ductr 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06d8a8a110ec6897312c7cd97aadcde63b5a5c303d7597dfb3a110ba202e94d3
-  data.tar.gz: ef127e746f6b4c8cee9d024a6368ee6f729626290106b4cc85d9600138d317f6
+  metadata.gz: 8ba418763bfafcbc8f42d9f78c3277ab24d7feac82f77aad21ddef34d9967c4a
+  data.tar.gz: 3a05f4e233052037ca21c0b8e4dfe374a09f0745cc1b07128f6111b55107088f
 SHA512:
-  metadata.gz: f29dd8bb58731e45a94e4ad800f2df651f98f260851509765e4a821312c1f3d026ebc4b2a68f479b2b4bee96c5e68f9c1633a9d2cc762a9b382aa2715e5ad220
-  data.tar.gz: e4a0cbe15974d7ea681eec581fe890cbed6799727b78342d5c4a7f39be065e3b31ab46da8bf0e453c6e8fa921bc867cc2d688a95396d02e63b585d5b35657fce
+  metadata.gz: 14f51aca717bf8d342f384756ed496261ca8626e3a010eb9790292a34f05e1419ee7e200618eb7e3d3a6bae8dfeb547a04fe7a2a69fa28a5d435a6af2e9d20df
+  data.tar.gz: bf8e6b8e2ab6941b66db0bb17a3360c44dd797ad04dab0ebbf245335090fe61d5b90b78fa6fe6266d4efd6a05ab11bf13eac7488b11583c3bb13ea7066275c75

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ductr (0.1.1)
+    ductr (0.2.0)
       activejob (~> 7.0)
       annotable (~> 0.1)
       colorize (~> 0.8)
@@ -27,8 +27,8 @@ GEM
     commander (4.6.0)
       highline (~> 2.0.0)
     concurrent-ruby (1.2.2)
-    debug (1.6.3)
-      irb (>= 1.3.6)
+    debug (1.7.2)
+      irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
     et-orbi (1.2.7)
@@ -41,10 +41,10 @@ GEM
     highline (2.0.3)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    io-console (0.5.11)
-    irb (1.4.2)
+    io-console (0.6.0)
+    irb (1.6.3)
       reline (>= 0.3.0)
-    json (2.6.2)
+    json (2.6.3)
     kiba (4.0.0)
     minitest (5.18.0)
     parallel (1.22.1)
@@ -53,44 +53,44 @@ GEM
       parser
       rainbow (~> 3.0)
       sorbet-runtime (>= 0.5)
-    parser (3.1.2.1)
+    parser (3.2.2.0)
       ast (~> 2.4.1)
     raabro (1.4.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.6.0)
-    reline (0.3.1)
+    regexp_parser (2.7.0)
+    reline (0.3.3)
       io-console (~> 0.5)
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
+    rspec-core (3.12.1)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.38.0)
+    rubocop (1.48.1)
       json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.1.2.1)
+      parser (>= 3.2.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.23.0, < 2.0)
+      rubocop-ast (>= 1.26.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.23.0)
-      parser (>= 3.1.1.0)
-    ruby-progressbar (1.11.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.28.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
     rufus-scheduler (3.8.2)
       fugit (~> 1.1, >= 1.1.6)
-    sorbet-runtime (0.5.10712)
+    sorbet-runtime (0.5.10746)
     sord (4.0.0)
       commander (~> 4.5)
       parlour (~> 5.0)
@@ -99,7 +99,7 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.3.0)
+    unicode-display_width (2.4.2)
     webrick (1.7.0)
     yard (0.9.28)
       webrick (~> 1.7.0)
@@ -118,4 +118,4 @@ DEPENDENCIES
   yard (~> 0.9)
 
 BUNDLED WITH
-   2.3.7
+   2.4.8

data/lib/ductr/cli/new_project_generator.rb

@@ -42,6 +42,8 @@ module Ductr
         copy_file "gemfile.rb", "Gemfile"
         copy_file "rubocop.yml", ".rubocop.yml"
 
+        create_file "config/initializers/.gitkeep"
+        create_file "lib/.gitkeep"
         create_file "app/jobs/.gitkeep"
         create_file "app/pipelines/.gitkeep"
         create_file "app/schedulers/.gitkeep"

data/lib/ductr/cli/templates/project/config_app.rb

@@ -3,3 +3,5 @@
 require "ductr"
 
 require_relative "environment/#{Ductr.env}"
+
+Dir[File.join(__dir__, "initializers", "*.rb")].each { |file| require file }

data/lib/ductr/rufus_trigger.rb

@@ -53,8 +53,9 @@ module Ductr
     def add(method, options)
       rufus_type = options.keys.first
       rufus_value = options.values.first
+      handler = callable(method, **options)
 
-      do_schedule(rufus_type, rufus_value, method)
+      do_schedule(rufus_type, rufus_value, handler)
     end
 
     #
@@ -78,16 +79,15 @@ module Ductr
     end
 
     #
-    # Returns a callable object based on given scheduler, method_name and options.
+    # Returns a callable object based on given method and options.
     #
-    # @param [Scheduler] scheduler The scheduler instance
-    # @param [Symbol] method_name The scheduler's method name
+    # @param [Method] method The scheduler's method
     # @param [Hash] ** The option passed to the trigger annotation
     #
     # @return [#call] A callable object
     #
-    def callable(scheduler, method_name, **)
-      scheduler.method(method_name)
+    def callable(method, **)
+      method
     end
 
     #
@@ -95,12 +95,12 @@ module Ductr
     #
     # @param [Symbol] rufus_type The rufus-scheduler type (`:once`, `:every`, `:interval` or `:cron`)
     # @param [String] rufus_value The rufus-scheduler value (e.g. `"10min"`)
-    # @param [#call] method The callable object (the scheduler's method in this case)
+    # @param [#call] handler The callable object (the scheduler's method in this case)
     #
     # @return [void]
     #
-    def do_schedule(rufus_type, rufus_value, method)
-      rufus.send(:do_schedule, rufus_type, rufus_value, nil, {}, false, method)
+    def do_schedule(rufus_type, rufus_value, handler)
+      rufus.send(:do_schedule, rufus_type, rufus_value, nil, {}, false, handler)
     end
   end
 end

data/lib/ductr/sequel/adapter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # The base class of all sequel-based adapters.
+    #
+    class Adapter < Ductr::Adapter
+      # @return [Sequel::Database, nil] The database connection instance
+      attr_reader :db
+    end
+  end
+end

data/lib/ductr/sequel/basic_destination.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A destination control that write rows one by one.
+    #
+    class BasicDestination < Ductr::ETL::Destination
+      #
+      # Opens the database if needed and call the job's method to insert one row at time.
+      #
+      # @param [Hash<Symbol, Object>] row The row to insert, preferably a Hash
+      #
+      # @return [void]
+      #
+      def write(row)
+        call_method(adapter.db, row)
+      end
+    end
+  end
+end

data/lib/ductr/sequel/basic_lookup.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A lookup control that execute one query per row.
+    #
+    class BasicLookup < Ductr::ETL::Transform
+      #
+      # Calls the job's method to merge its result with the current row.
+      #
+      # @param [Hash<Symbol, Object>] row The current row, preferably a Hash
+      #
+      # @return [Hash<Symbol, Object>] The row merged with looked up row or the untouched row if nothing was found
+      #
+      def process(row)
+        matching_row = call_method(adapter.db, row).first
+        return row unless matching_row
+
+        row.merge matching_row
+      end
+    end
+  end
+end

data/lib/ductr/sequel/basic_source.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A source control that yields rows one by one.
+    #
+    class BasicSource < Ductr::ETL::Source
+      #
+      # Opens the database, calls the job's method and iterate over the query results.
+      #
+      # @yield The each block
+      #
+      # @return [void]
+      #
+      def each(&)
+        call_method(adapter.db).each(&)
+      end
+    end
+  end
+end

data/lib/ductr/sequel/buffered_destination.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A destination control that accumulates rows in a buffer to write them by batch.
+    #
+    class BufferedDestination < Ductr::ETL::BufferedDestination
+      #
+      # Open the database if needed and call the job's method to run the query.
+      #
+      # @return [void]
+      #
+      def on_flush
+        call_method(adapter.db, buffer)
+      end
+    end
+  end
+end

data/lib/ductr/sequel/buffered_lookup.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A lookup control that execute the query for a bunch of rows.
+    #
+    class BufferedLookup < Ductr::ETL::BufferedTransform
+      #
+      # Opens the database if needed, calls the job's method and pass the each block to it.
+      #
+      # @yield The each block
+      #
+      # @return [void]
+      #
+      def on_flush(&)
+        call_method(adapter.db, buffer, &)
+      end
+    end
+  end
+end

data/lib/ductr/sequel/buffered_upsert_destination.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A destination control that accumulates rows in a buffer to upsert them by batch.
+    #
+    class BufferedUpsertDestination < Ductr::ETL::BufferedDestination
+      #
+      # Open the database if needed and call the job's method to run the query.
+      #
+      # @return [void]
+      #
+      def on_flush
+        call_method(adapter.db, excluded, buffer)
+      end
+
+      private
+
+      #
+      # Generate the excluded keys hash e.g.
+      #
+      # ```ruby
+      # {a: Sequel[:excluded][:a]}
+      # ```
+      #
+      # @return [Hash<Symbol, Sequel::SQL::QualifiedIdentifier>] The excluded keys hash
+      #
+      def excluded
+        keys = buffer.first.keys
+
+        excluded_keys = keys.map do |key|
+          Sequel[:excluded][key]
+        end
+
+        keys.zip(excluded_keys).to_h
+      end
+    end
+  end
+end

data/lib/ductr/sequel/match_lookup.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Ductr
+  module SQLite
+    #
+    # A lookup control that execute the query for a bunch of rows and merge them with the buffer's rows.
+    #
+    class MatchLookup < Ductr::ETL::BufferedTransform
+      #
+      # The looked up row key to match.
+      #
+      # @return [Symbol] The column name
+      #
+      def from_key
+        @options[:merge].first
+      end
+
+      #
+      # The buffer row key to match.
+      #
+      # @return [Symbol] The column name
+      #
+      def to_key
+        @options[:merge].last
+      end
+
+      #
+      # Opens the database if needed, calls the job's method and merges
+      # the looked up rows with corresponding buffer rows.
+      #
+      # @yield [row] The each block
+      # @yieldparam [Hash<Symbol, Object>] row The merged row
+      #
+      # @return [void]
+      #
+      def on_flush(&)
+        call_method(adapter.db, buffer_keys).each do |row|
+          match = buffer_find(row)
+          next yield(row) unless match
+
+          yield(row.merge match)
+        end
+      end
+
+      private
+
+      #
+      # Find the corresponding row into the buffer.
+      #
+      # @param [Hash<Symbol, Object>] row The looked up row
+      #
+      # @return [Hash<Symbol, Object>, nil] the matching row if exists
+      #
+      def buffer_find(row)
+        buffer.find { |r| r[from_key] == row[to_key] }
+      end
+
+      #
+      # Maps the buffer keys into an array.
+      #
+      # @return [Array<Integer, String>] The keys array
+      #
+      def buffer_keys
+        buffer.map { |row| row[from_key] }
+      end
+    end
+  end
+end

data/lib/ductr/sequel/paginated_source.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A source control that allows to select a big number of rows by relying on pagination.
+    #
+    class PaginatedSource < Ductr::ETL::PaginatedSource
+      #
+      # Calls the job's method and iterate on the query result.
+      # Returns true if the page is full, false otherwise.
+      #
+      # @yield The each block
+      #
+      # @raise [InconsistentPaginationError] When the query return more rows than the page size
+      # @return [Boolean] True if the page is full, false otherwise.
+      #
+      def each_page(&)
+        rows_count = 0
+
+        call_method(adapter.db, @offset, page_size).each do |row|
+          yield(row)
+          rows_count += 1
+        end
+
+        if rows_count > page_size
+          raise InconsistentPaginationError,
+                "The query returned #{rows_count} rows but the page size is #{page_size} rows"
+        end
+
+        rows_count == page_size
+      end
+    end
+  end
+end

data/lib/ductr/sequel/polling_handler.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # The rufus-scheduler handler class.
+    # @see https://github.com/jmettraux/rufus-scheduler#scheduling-handler-instances
+    #   For further information
+    #
+    class PollingHandler
+      #
+      # Creates the handler based on the given scheduler, its method name and the trigger's adapter instance.
+      #
+      # @param [Method] method The scheduler's method
+      # @param [Ductr::Adapter] adapter The trigger's adapter
+      #
+      def initialize(method, adapter)
+        @method = method
+        @adapter = adapter
+        @last_triggering_key = nil
+      end
+
+      #
+      # The callable method used by the trigger, actually calls the scheduler's method.
+      #
+      # @return [void]
+      #
+      def call
+        @adapter.open do |db|
+          @method.call(db) do |triggering_key|
+            return false if triggering_key == @last_triggering_key
+
+            @last_triggering_key = triggering_key
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ductr/sequel/polling_trigger.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Sequel
+    #
+    # A trigger based on the RufusTrigger, runs the PollingHandler at the given timing.
+    #
+    class PollingTrigger < Ductr::RufusTrigger
+      #
+      # Closes the connection if the scheduler is stopped.
+      #
+      # @return [void]
+      #
+      def stop
+        super
+        adapter.close!
+      end
+
+      private
+
+      #
+      # Returns a callable object, allowing rufus-scheduler to call it.
+      #
+      # @param [Ductr::Scheduler] scheduler The scheduler instance
+      # @param [Method] method The scheduler's method
+      # @param [Hash] ** The option passed to the trigger annotation
+      #
+      # @return [#call] A callable object
+      #
+      def callable(method, **)
+        PollingHandler.new(method, adapter)
+      end
+    end
+  end
+end

data/lib/ductr/version.rb

@@ -2,5 +2,5 @@
 
 module Ductr
   # @return [String] The ductr's version number
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/ductr.rb

@@ -133,10 +133,10 @@ end
 if File.directory?("#{pwd = Dir.pwd}/app")
   Zeitwerk::Loader.new.tap do |loader|
     loader.push_dir "#{pwd}/app"
+    loader.push_dir "#{pwd}/lib"
 
-    loader.collapse "#{pwd}/app/jobs"
-    loader.collapse "#{pwd}/app/pipelines"
-    loader.collapse "#{pwd}/app/schedulers"
+    app_paths = Dir.glob("#{pwd}/app/**/**").select { |f| File.directory? f }
+    loader.collapse(app_paths)
 
     loader.setup
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ductr
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Mathieu Morel
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-03-18 00:00:00.000000000 Z
+date: 2023-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: debug
@@ -252,6 +252,17 @@ files:
 - lib/ductr/registry.rb
 - lib/ductr/rufus_trigger.rb
 - lib/ductr/scheduler.rb
+- lib/ductr/sequel/adapter.rb
+- lib/ductr/sequel/basic_destination.rb
+- lib/ductr/sequel/basic_lookup.rb
+- lib/ductr/sequel/basic_source.rb
+- lib/ductr/sequel/buffered_destination.rb
+- lib/ductr/sequel/buffered_lookup.rb
+- lib/ductr/sequel/buffered_upsert_destination.rb
+- lib/ductr/sequel/match_lookup.rb
+- lib/ductr/sequel/paginated_source.rb
+- lib/ductr/sequel/polling_handler.rb
+- lib/ductr/sequel/polling_trigger.rb
 - lib/ductr/store.rb
 - lib/ductr/store/job_serializer.rb
 - lib/ductr/store/job_store.rb