ductr 0.1.0

data/lib/ductr/etl/fiber_control.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Glues ETL controls and the associated fibers together.
+    #
+    class FiberControl
+      extend Forwardable
+
+      #
+      # @!method resume
+      #   Resumes the control's fiber.
+      #   @param [Object] row The row to pass to right fiber controls
+      #   @return [void]
+      def_delegators :fiber, :resume
+
+      # @return [Array<FiberControl>] The next fiber controls
+      attr_accessor :right
+      # @return [Control] The ETL control instance
+      attr_reader :control
+
+      #
+      # Creates a new fiber control with the given control and control type.
+      #
+      # @param [Control] control The ETL control to work with in the fiber
+      # @param [Symbol] type The ETL control type, one of [:source, :transform, :destination]
+      #
+      def initialize(control, type:)
+        @control = control
+        @type = type
+
+        @right = []
+      end
+
+      #
+      # Memoizes the fiber to be associated with the ETL control based on its type.
+      #
+      # @return [Fiber] The fiber in charge of executing the control's logic
+      #
+      def fiber
+        @fiber ||= send(@type)
+      end
+
+      private
+
+      #
+      # Creates the fiber to run ETL sources.
+      #
+      # @return [Fiber]
+      #
+      def source
+        Fiber.new do
+          control.each do |row|
+            resume_right_fibers(row)
+          end
+
+          resume_right_fibers(:end)
+        end
+      end
+
+      #
+      # Creates the fiber to run ETL transforms.
+      #
+      # @return [Fiber]
+      #
+      def transform
+        resume_control(Fiber.new do
+          loop do
+            row_in = Fiber.yield
+            next close_transform if row_in == :end
+
+            row_out = control.process(row_in) do |r|
+              resume_right_fibers(r)
+            end
+
+            resume_right_fibers(row_out) if row_out
+          end
+        end)
+      end
+
+      #
+      # Creates the fiber to run ETL Destinations.
+      #
+      # @return [Fiber]
+      #
+      def destination
+        resume_control(Fiber.new do
+          loop do
+            row = Fiber.yield
+            next control.close if row == :end
+
+            control.write(row)
+          end
+        end)
+      end
+
+      #
+      # Call #close on control, resume resulting rows then ends following fibers.
+      #
+      # @return [void]
+      #
+      def close_transform
+        control.close do |row|
+          resume_right_fibers(row)
+        end
+        resume_right_fibers(:end)
+      end
+
+      #
+      # Resumes all fibers at the right of the current one.
+      #
+      # @param [Object] row The row to pass to the next fibers
+      #
+      # @return [void]
+      #
+      def resume_right_fibers(row)
+        right.each do |fiber|
+          fiber.resume(row)
+        end
+      end
+
+      #
+      # Resumes the given fiber and returns it.
+      #
+      # @param [Fiber] fiber The fiber to resume
+      #
+      # @return [Fiber] The resumed fiber
+      #
+      def resume_control(fiber)
+        fiber.resume
+        fiber
+      end
+    end
+  end
+end

data/lib/ductr/etl/fiber_runner.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # A runner built with fibers. Compared to KibaRunner,
+    # this one allows to define how control are related to each other.
+    # These definitions can be found in Runner#pipes method.
+    #
+    class FiberRunner < Runner
+      #
+      # Initializes fibers and waits for them to finish.
+      #
+      # @return [void]
+      #
+      def run
+        create_fibers!
+        @source_fibers.each_value(&:resume)
+      end
+
+      private
+
+      #
+      # Initializes control fibers and pipes them together.
+      #
+      # @return [void]
+      #
+      def create_fibers!
+        @source_fibers = create_control_fibers(sources) { |s| FiberControl.new(s, type: :source) }
+        @transform_fibers = create_control_fibers(transforms) { |t| FiberControl.new(t, type: :transform) }
+        @destination_fibers = create_control_fibers(destinations) { |d| FiberControl.new(d, type: :destination) }
+
+        apply_fibers_plumbing!
+      end
+
+      #
+      # Pipes fiber controls together based on the control plumbing hash.
+      #
+      # @return [void]
+      #
+      def apply_fibers_plumbing!
+        pipes.map do |from_to|
+          from = from_to.keys.first
+          to = from_to[from]
+
+          input = { **@source_fibers, **@transform_fibers }[from]
+          outputs = to.map { |out| { **@transform_fibers, **@destination_fibers }[out] }
+
+          input.right = outputs
+        end
+      end
+
+      #
+      # Maps controls into a hash with job's method name as keys and control fibers as values.
+      #
+      # @param [Array<Control>] controls The controls to map on the hash
+      # @yield [control] The block in which the control fiber has to be initialized
+      #
+      # @return [Hash{Symbol => FiberControl}] The mapped hash
+      #
+      def create_control_fibers(controls, &)
+        controls.to_h do |control|
+          [control.job_method.name, yield(control)]
+        end
+      end
+    end
+  end
+end

data/lib/ductr/etl/kiba_runner.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "kiba"
+
+module Ductr
+  module ETL
+    #
+    # A runner based on kiba's streaming runner
+    # @see Kiba's streaming runner source code to get details about its forwarded methods
+    #
+    class KibaRunner < Runner
+      extend Forwardable
+      def_delegators Kiba::StreamingRunner, :source_stream, :transform_stream, :process_rows, :close_destinations
+
+      #
+      # Calls kiba's streaming runner #process_rows and #close_destinations like Kiba::StreamingRunner#run
+      #
+      # @return [void]
+      #
+      def run
+        process_rows(sources, transforms, destinations)
+        close_destinations(destinations)
+      end
+    end
+  end
+end

data/lib/ductr/etl/parser.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Contains anything to "parse" ETL jobs annotations.
+    # #parse_annotations handles ETL controls and send_to directives.
+    #
+    module Parser
+      #
+      # Handles sources, transforms and destinations controls.
+      # Handles send_to directives, used to do the plumbing between controls.
+      # Used for both kiba and fiber runners initialization.
+      #
+      # @return [Array<Source, Transform, Destination, Hash{Symbol => Array<Symbol>}>] The job's controls
+      #
+      def parse_annotations
+        sources = init_adapter_controls(:source)
+        transforms = init_transform_controls(:transform, :lookup)
+        destinations = init_adapter_controls(:destination)
+        pipes = find_method(:send_to) do |method|
+          { method.name => method.find_annotation(:send_to).params }
+        end
+
+        [sources, transforms, destinations, pipes]
+      end
+
+      #
+      # Currently used adapters set.
+      #
+      # @return [Set] The current adapters
+      #
+      def adapters
+        @adapters ||= Set.new
+      end
+
+      private
+
+      #
+      # Finds the method(s) associated to the given annotation names in the job class.
+      #
+      # @param [Array<Symbol>] *annotation_names The annotation names of the searched methods
+      # @yield [method] The block to execute on each founded methods
+      # @yieldparam [method] A job's method
+      #
+      # @return [Array] Returns mapped array containing the block's returned value
+      #
+      def find_method(*annotation_names, &)
+        self.class.annotated_methods(*annotation_names).map(&)
+      end
+
+      #
+      # Initializes adapter controls for the given type.
+      #
+      # @param [Symbol] control_type The adapter control type, one of :source or :destination
+      #
+      # @return [Array<Source, Destination>] The initialized adapter controls
+      #
+      def init_adapter_controls(control_type)
+        find_method(control_type) do |method|
+          adapter_control(method)
+        end
+      end
+
+      #
+      # Initializes transform controls for the given types.
+      #
+      # @param [Array<Symbol>] *control_types The transform control types, :transform and/or :lookup
+      #
+      # @return [Array<Transform>] The initialized transform controls
+      #
+      def init_transform_controls(*control_types)
+        find_method(*control_types) do |method|
+          next adapter_control(method) if method.annotation_exist?(:lookup)
+
+          transform_control(method)
+        end
+      end
+
+      #
+      # Initializes an adapter control (source, lookup or destination) based on the given annotated method.
+      #
+      # @param [Annotable::Method] annotated_method The control's method
+      #
+      # @return [Control] The adapter control instance
+      #
+      def adapter_control(annotated_method)
+        annotation = annotated_method.find_annotation(:source, :destination, :lookup)
+        adapter_name, control_type = annotation.params
+
+        adapter = Ductr.config.adapter(adapter_name)
+        control_class = adapter.class.send("#{annotation.name}_registry").find(control_type)
+        job_method = method(annotated_method.name)
+
+        adapters.add(adapter)
+        control_class.new(job_method, adapter, **annotation.options)
+      end
+
+      #
+      # Initializes a transform control.
+      #
+      # @param [Annotable::Method] annotated_method The transform's method
+      #
+      # @return [Transform] The transform control instance
+      #
+      def transform_control(annotated_method)
+        annotation = annotated_method.find_annotation(:transform)
+        transform_class = annotation.params.first || Transform
+        job_method = method(annotated_method.name)
+
+        transform_class.new(job_method, **annotation.options)
+      end
+    end
+  end
+end

data/lib/ductr/etl/runner.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # The base class for all runners
+    #
+    class Runner
+      # @return [Array<Source>] The runner source controls
+      attr_accessor :sources
+
+      # @return [Array<Transform>] The runner transform controls
+      attr_accessor :transforms
+
+      # @return [Array<Destination>] The runner destination controls
+      attr_accessor :destinations
+
+      # @return [Array<Hash{Symbol => Array<Symbol>}>] The controls plumbing hashes
+      attr_accessor :pipes
+
+      #
+      # Creates the runner instance.
+      #
+      # @param [Array<Source>] sources The job's source controls
+      # @param [Array<Transform>] transforms The job's transform controls
+      # @param [Array<Destination>] destinations The job's destination controls
+      # @param [Array<Hash{Symbol => Array<Symbol>}>] pipes The controls plumbing hashes
+      #
+      def initialize(sources, transforms, destinations, pipes = [])
+        @sources = sources
+        @transforms = transforms
+        @destinations = destinations
+        @pipes = pipes
+      end
+    end
+  end
+end

data/lib/ductr/etl_job.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # Base class for ETL job using the experimental fiber runner.
+  # Usage example:
+  #
+  #   class MyETLJob < Ductr::ETLJob
+  #     source :first_db, :basic
+  #     send_to :the_transform, :the_other_transform
+  #     def the_source(db)
+  #       # ...
+  #     end
+  #
+  #     transform
+  #     send_to :the_destination
+  #     def the_transform(row)
+  #       # ...
+  #     end
+  #
+  #     destination :first_db, :basic
+  #     def the_destination(row, db)
+  #       # ...
+  #     end
+  #
+  #     transform
+  #     send_to :the_other_destination
+  #     def the_other_transform(row)
+  #       # ...
+  #     end
+  #
+  #     destination :second_db, :basic
+  #     def the_other_destination(row, db)
+  #       # ...
+  #     end
+  #   end
+  #
+  class ETLJob < Job
+    # @return [Class] The ETL runner class used by the job
+    ETL_RUNNER_CLASS = ETL::FiberRunner
+    include JobETLRunner
+
+    include ETL::Parser
+
+    #
+    # @!method self.source(adapter_name, source_type, **source_options)
+    #   Annotation to define a source method
+    #   @param adapter_name [Symbol] The adapter the source is running on
+    #   @param source_type [Symbol] The type of source to run
+    #   @param **source_options [Hash<Symbol: Object>] The options to pass to the source
+    #
+    #   @example Source with Sequel SQLite adapter
+    #     source :my_adapter, :paginated, page_size: 42
+    #     def my_source(db, offset, limit)
+    #       db[:items].offset(offset).limit(limit)
+    #     end
+    #
+    #   @see The chosen adapter documentation for further information on sources usage.
+    #
+    #   @return [void]
+    #
+    annotable :source
+
+    #
+    # @!method self.transform(transform_class, **transform_options)
+    #   Annotation to define a transform method
+    #   @param transform_class [Class, nil] The class the transform is running on
+    #   @param **transform_options [Hash<Symbol: Object>] The options to pass to the transform
+    #
+    #   @example Transform without params
+    #     transform
+    #     def rename_keys(row)
+    #       row[:new_name] = row.delete[:old_name]
+    #       row[:new_email] = row.delete[:old_email]
+    #     end
+    #
+    #   @example Transform with params
+    #     class RenameTransform < Ductr::ETL::Transform
+    #       def process(row)
+    #         call_method.each do |actual_name, new_name|
+    #           new_key = "#{options[:prefix]}#{new_name}".to_sym
+    #
+    #           row[new_key] = row.delete(actual_name)
+    #         end
+    #       end
+    #     end
+    #
+    #     transform RenameTransform, prefix: "some_"
+    #     def rename
+    #       { old_name: :new_name, old_email: :new_email }
+    #     end
+    #
+    #   @return [void]
+    #
+    annotable :transform
+
+    #
+    # @!method self.lookup(adapter_name, lookup_type, **lookup_options)
+    #   Annotation to define a lookup method
+    #   @param adapter_name [Symbol] The adapter the lookup is running on
+    #   @param lookup_type [Symbol] The type of lookup to run
+    #   @param **lookup_options [Hash<Symbol: Object>] The options to pass to the lookup
+    #
+    #   @example Lookup with Sequel SQLite adapter
+    #     lookup :my_other_adapter, :match, merge: [:id, :item], buffer_size: 4
+    #     def joining_different_adapters(db, ids)
+    #       db[:items_bis].select(:id, :item, :name).where(item: ids)
+    #     end
+    #
+    #   @see The chosen adapter documentation for further information on lookups usage.
+    #
+    #   @return [void]
+    #
+    annotable :lookup
+
+    #
+    # @!method self.destination(adapter_name, destination_type, **destination_options)
+    #   Annotation to define a destination method
+    #   @param adapter_name [Symbol] The adapter the destination is running on
+    #   @param destination_type [Symbol] The type of destination to run
+    #   @param **destination_options [Hash<Symbol: Object>] The options to pass to the destination
+    #
+    #   @example Destination with Sequel SQLite adapter
+    #     destination :my_other_adapter, :basic
+    #     def my_destination(row, db)
+    #       db[:new_items].insert(name: row[:name], new_name: row[:new_name])
+    #     end
+    #
+    #   @see The chosen adapter documentation for further information on destinations usage.
+    #
+    #   @return [void]
+    #
+    annotable :destination
+
+    #
+    # @!method self.send_to(*methods)
+    #   Annotation to define which methods will follow the current one
+    #   @param *methods [Array<Symbol>] The names of the following methods
+    #
+    #   @example Source with Sequel SQLite adapter sending rows to two transforms
+    #     source :my_adapter, :paginated, page_size: 42
+    #     send_to :my_first_transform, :my_second_transform
+    #     def my_source(db, offset, limit)
+    #       db[:items].offset(offset).limit(limit)
+    #     end
+    #
+    #     transform
+    #     def my_first_transform(row)
+    #       # ...
+    #     end
+    #
+    #     transform
+    #     def my_second_transform(row)
+    #       # ...
+    #     end
+    #
+    #   @return [void]
+    #
+    annotable :send_to
+  end
+end

data/lib/ductr/job.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # The base class for any job, you can use it directly if you don't need an ETL job.
+  #
+  class Job < ActiveJob::Base
+    extend Annotable
+    extend Forwardable
+
+    include JobStatus
+
+    # @return [Exception] The occurred error if any
+    attr_reader :error
+    # @return [Symbol] The job's status, one of `:queued`, `:working`, `:completed` and `:failed`
+    attr_reader :status
+
+    queue_as :ductr_jobs
+
+    #
+    # The active job's perform method. DO NOT override it, implement the #run method instead.
+    #
+    # @return [void]
+    #
+    def perform(*_)
+      run
+    end
+
+    #
+    # The configured adapter instances.
+    #
+    # @param [Symbol] name The adapter name
+    #
+    # @return [Adapter] The adapter corresponding to the given name
+    #
+    def adapter(name)
+      Ductr.config.adapter(name)
+    end
+
+    #
+    # The job's logger instance.
+    #
+    # @return [Ductr::Log::Logger] The logger instance
+    #
+    def logger
+      @logger ||= Ductr.config.logging.new(self.class)
+    end
+
+    #
+    # The entry point of jobs.
+    #
+    # @return [void]
+    #
+    def run
+      raise NotImplementedError, "A job must implement the `#run` method"
+    end
+  end
+end

data/lib/ductr/job_etl_runner.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # Allowing a job to execute ETL runners.
+  # You need to declare the ETL_RUNNER_CLASS constant in the including class:
+  #
+  #   class CustomJobClass < Job
+  #     ETL_RUNNER_CLASS = ETL::KibaRunner
+  #     include JobETLRunner
+  #   end
+  #
+  # The job must have the #parse_annotations method defined, which can be added by including ETL::Parser.
+  #
+  module JobETLRunner
+    #
+    # Parse job's annotations and create the runner instance.
+    #
+    def initialize(...)
+      super(...)
+
+      @runner = self.class::ETL_RUNNER_CLASS.new(*parse_annotations)
+    end
+
+    #
+    # Opens adapters, executes the runner and then closes back adapters.
+    #
+    # @return [void]
+    #
+    def run
+      adapters.each(&:open!)
+      @runner.run
+    ensure
+      adapters.each(&:close!)
+    end
+  end
+end

data/lib/ductr/job_status.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "annotable"
+
+module Ductr
+  #
+  # This module contains the job's status tracking logic.
+  # It relies on Active Job's callbacks to write status into the store.
+  #
+  module JobStatus
+    class << self
+      #
+      # Registers the ActiveJob's `before_enqueue`, `before_perform` and `after_perform` callbacks
+      # to write status in the Ductr's store.
+      # Intercepts and re-raises job's exceptions to write the `:failed` status.
+      #
+      # @param [Class<Job>] job_class The job's class
+      #
+      # @return [void]
+      #
+      def included(job_class)
+        job_class.before_enqueue { |job| job.status = :queued }
+        job_class.before_perform { |job| job.status = :working }
+        job_class.after_perform { |job| job.status = :completed }
+
+        job_class.rescue_from(Exception) do |e|
+          @error = e
+          self.status = :failed
+
+          raise e
+        end
+      end
+    end
+
+    #
+    # Writes the job's status into the Ductr's store.
+    #
+    # @param [Symbol] status The status of the job
+    #
+    # @return [void]
+    #
+    def status=(status)
+      @status = status
+      Store.write_job(self)
+    end
+
+    #
+    # Determines whether the job has a `completed` or `failed` status.
+    #
+    # @return [Boolean] True when the status is `completed` or `failed`
+    #
+    def stopped?
+      %i[completed failed].include? status
+    end
+  end
+end