ductr 0.1.0

data/lib/ductr/pipeline_step.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # Representation of a pipeline's step.
+  # Hold a fiber to execute steps concurrently.
+  #
+  class PipelineStep
+    extend Forwardable
+
+    #
+    # @!method resume
+    #   Resumes the step's fiber.
+    #   @return [void]
+    # @!method alive?
+    #   Check if the step's fiber is running.
+    #   @return [Boolean] True if step's fiber is running
+    #
+    def_delegators :fiber, :resume, :alive?
+
+    # @return [Pipeline] The step's pipeline
+    attr_reader :pipeline
+    # @return [Symbol] The step's name
+    attr_reader :name
+    # @return [Array<Job>] The step's queued jobs
+    attr_reader :jobs
+
+    # @return [PipelineStep] The previous step
+    attr_accessor :left
+
+    #
+    # Creates a step for the given pipeline.
+    #
+    # @param [Pipeline] pipeline The pipeline containing step's method
+    # @param [Symbol] The name of the pipeline's step method
+    #
+    def initialize(pipeline, name)
+      @pipeline = pipeline
+      @name = name
+
+      @jobs = []
+      @left = []
+    end
+
+    #
+    # Track, registers and enqueues the given job.
+    #
+    # @param [Job] job The job to enqueue
+    #
+    # @return [void]
+    #
+    def enqueue_job(job)
+      jobs.push(job)
+      Store.register_job(job)
+      job.enqueue
+    end
+
+    #
+    # Check if the step is done.
+    #
+    # @return [Boolean] True if the step is done
+    #
+    def done?
+      !fiber.alive?
+    end
+
+    #
+    # Waits until all step's jobs are stopped.
+    #
+    # @return [void]
+    #
+    def flush_jobs
+      return if jobs.empty?
+
+      Fiber.yield until Store.read_jobs(*jobs).all?(&:stopped?)
+    end
+
+    #
+    # The step's fiber instance, invokes the pipeline's step method.
+    #
+    # @return [Fiber] The step's fiber
+    #
+    def fiber
+      @fiber ||= Fiber.new do
+        Fiber.yield until left.all?(&:done?)
+
+        pipeline.send(name)
+        flush_jobs
+      end
+    end
+  end
+end

data/lib/ductr/registry.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Ductr
+  class NotFoundInRegistryError < StandardError; end
+
+  #
+  # The registry pattern to store adapters, controls and triggers.
+  #
+  class Registry
+    extend Forwardable
+
+    #
+    # @!method values
+    #   Get all registered adapters, controls or triggers.
+    #   @return [Array<Class<Adapter, ETL::Control, Trigger>>] The registered adapters, controls or triggers
+    #
+    def_delegators :@items, :values
+
+    #
+    # Initialize the registry as an empty hash with the given name.
+    #
+    # @param name [Symbol] The registry name, used in error message
+    #
+    def initialize(name)
+      @name = name
+      @items = {}
+    end
+
+    #
+    # Allow to add one class into the registry.
+    #
+    # @param item [Class<Adapter, ETL::Control, Trigger>] The class to add in the registry
+    # @param as: [Symbol] The adapter, control or trigger type
+    #
+    # @return [void]
+    #
+    def add(item, as:)
+      @items[as.to_sym] = item
+    end
+
+    #
+    # Find an adapter, control or trigger based on its type.
+    #
+    # @param type [Symbol] The adapter, control or trigger type
+    #
+    # @raise [NotFoundInRegistryError] If nothing matches the given type
+    # @return [Class<Adapter, ETL::Control, Trigger>] The found adapter
+    #
+    def find(type)
+      @items.fetch(type.to_sym) do
+        raise NotFoundInRegistryError, "The #{@name} of type \"#{type}\" does not exist"
+      end
+    end
+  end
+end

data/lib/ductr/rufus_trigger.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "rufus-scheduler"
+
+module Ductr
+  #
+  # A time related trigger, used to trigger jobs or pipelines based on temporal events.
+  # The trigger is registered as `:schedule`:
+  #
+  #   trigger :schedule, every: "1min"
+  #   def every_minute
+  #     MyPipeline.perform_later
+  #   end
+  #
+  # This trigger is based on the `rufus-scheduler` gem.
+  # Under the hood, the internal method `Rufus::Scheduler#do_schedule` is used.
+  #
+  # There are 4 types of options available:
+  #
+  # - `:once`, used to schedule at a given time
+  #
+  #   trigger :schedule, once: "10d" # Will run in ten days
+  #   trigger :schedule, once: "2042/12/12 23:59:00" # Will run at the given date
+  #
+  # - `:every`, triggers following the given frequency
+  #
+  #   trigger :schedule, every: "1min" # will run every minute.
+  #
+  # - `:interval`, run the trigger then waits the given interval before running again
+  #
+  #   # Will run every 4 + 1 seconds
+  #   trigger :schedule, interval: "4s"
+  #   def every_interval
+  #     sleep(1)
+  #     MyPipeline.perform_later
+  #   end
+  #
+  # - `:cron`, run the trigger following the given crontab pattern
+  #
+  #   trigger :schedule, cron: "00 01 * * *" # Will run every day at 1am.
+  #
+  class RufusTrigger < Trigger
+    Ductr.trigger_registry.add self, as: :schedule
+
+    #
+    # Adds a new trigger into rufus-scheduler.
+    #
+    # @param [Symbol] method The scheduler method to be called by rufus-scheduler
+    # @param [Hash<Symbol: String>] options The options to configure rufus-scheduler
+    #
+    # @return [void]
+    #
+    def add(method, options)
+      rufus_type = options.keys.first
+      rufus_value = options.values.first
+
+      do_schedule(rufus_type, rufus_value, method)
+    end
+
+    #
+    # Shutdown rufus-scheduler
+    #
+    # @return [void]
+    #
+    def stop
+      rufus.shutdown
+    end
+
+    private
+
+    #
+    # Shortcut to get the rufus-scheduler singleton
+    #
+    # @return [Rufus::Scheduler] The rufus-scheduler instance
+    #
+    def rufus
+      Rufus::Scheduler.singleton
+    end
+
+    #
+    # Returns a callable object based on given scheduler, method_name and options.
+    #
+    # @param [Scheduler] scheduler The scheduler instance
+    # @param [Symbol] method_name The scheduler's method name
+    # @param [Hash] ** The option passed to the trigger annotation
+    #
+    # @return [#call] A callable object
+    #
+    def callable(scheduler, method_name, **)
+      scheduler.method(method_name)
+    end
+
+    #
+    # Calls the Rufus::Scheduler#do_schedule private method with rufus-scheduler type, value and the callable object.
+    #
+    # @param [Symbol] rufus_type The rufus-scheduler type (`:once`, `:every`, `:interval` or `:cron`)
+    # @param [String] rufus_value The rufus-scheduler value (e.g. `"10min"`)
+    # @param [#call] method The callable object (the scheduler's method in this case)
+    #
+    # @return [void]
+    #
+    def do_schedule(rufus_type, rufus_value, method)
+      rufus.send(:do_schedule, rufus_type, rufus_value, nil, {}, false, method)
+    end
+  end
+end

data/lib/ductr/scheduler.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # Base class to declare event driven scheduling.
+  # Example using the schedule trigger:
+  #
+  #   class MyScheduler < Ductr::Scheduler
+  #     trigger :schedule, every: "10min"
+  #     def every_ten_minutes
+  #       MyJob.perform_later
+  #     end
+  #   end
+  #
+  class Scheduler
+    extend Annotable
+
+    #
+    # @!method self.trigger(trigger_type, adapter_name = nil, **trigger_options)
+    #   Annotation to define a trigger method
+    #   @param trigger_type [Symbol] The trigger's type
+    #   @param adapter_name [Symbol] The trigger's adapter (if any)
+    #   @param **trigger_options [Hash<Symbol: Object>] The options to pass to the trigger
+    #
+    #   @example A schedule trigger
+    #     trigger :schedule, every: "10min"
+    #     def every_ten_minutes
+    #       MyJob.perform_later
+    #     end
+    #
+    #   @see The chosen trigger documentation for further information.
+    #
+    #   @return [void]
+    #
+    annotable :trigger
+
+    class << self
+      #
+      # All trigger instances are stored in a singleton hash, avoiding to create the same trigger multiple times.
+      #
+      # @return [Hash<Symbol: Trigger>] The singleton hash containing all trigger instances
+      #
+      def triggers
+        @triggers ||= {}
+      end
+
+      #
+      # Calls #start on all created triggers.
+      #
+      # @return [void]
+      #
+      def start
+        triggers.values.each(&:start)
+      end
+
+      #
+      # Calls #stop on all created triggers.
+      #
+      # @return [void]
+      #
+      def stop
+        triggers.values.each(&:stop)
+      end
+    end
+
+    #
+    # Parses trigger annotations, creates triggers if needed and calls #add on trigger instances.
+    #
+    def initialize
+      self.class.annotated_methods.each do |method|
+        annotation = method.find_annotation(:trigger)
+        trigger = find_trigger(*annotation.params.reverse)
+        callable = self.method(method.name)
+
+        trigger.add(callable, annotation.options)
+      end
+    end
+
+    private
+
+    #
+    # Finds a trigger in the according trigger registry based on its type and adapter name if given.
+    #
+    # @param [Symbol] trigger_type The trigger type, e.g. `:schedule`
+    # @param [Symbol] adapter_name The adapter name, e.g. `:my_adapter`
+    #
+    # @return [Trigger] The found trigger
+    #
+    def find_trigger(trigger_type, adapter_name = nil)
+      return find_or_create_trigger(Ductr.trigger_registry, trigger_type) unless adapter_name
+
+      adapter = Ductr.config.adapter(adapter_name)
+      trigger_registry = adapter.class.trigger_registry
+      registry_key = "#{adapter_name}_#{trigger_type}".to_sym
+
+      find_or_create_trigger(trigger_registry, registry_key, trigger_type, adapter)
+    end
+
+    #
+    # Find a trigger in the singleton hash. if none found, find the trigger class in the given registry,
+    # initializes it and store the instance in the singleton hash.
+    #
+    # @param [Registry] trigger_registry The registry containing the required trigger class
+    # @param [Symbol] registry_key The key in which to store the trigger instance inside the singleton hash
+    # @param [Symbol] trigger_type The registry's key in which to find the trigger class
+    # @param [Adapter, Nil] adapter The adapter to pass to the trigger
+    #
+    # @return [Trigger] The found or created trigger
+    #
+    def find_or_create_trigger(trigger_registry, registry_key, trigger_type = registry_key, adapter = nil)
+      trigger = Scheduler.triggers[registry_key]
+      return trigger if trigger
+
+      Scheduler.triggers[registry_key] = trigger_registry.find(trigger_type).new(adapter)
+    end
+  end
+end

data/lib/ductr/store/job_serializer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Store
+    #
+    # Convert jobs into active job serializable structs.
+    #
+    module JobSerializer
+      #
+      # @!parse
+      #   #
+      #   # The job representation as a struct.
+      #   #
+      #   # @!attribute [r] job_id
+      #   #   @return [String] The active job's job id
+      #   #
+      #   # @!attribute [r] status
+      #   #   @return [Symbol] The job's status
+      #   #
+      #   # @!attribute [r] error
+      #   #   @return [Exception, nil] The job's error if any
+      #   #
+      #   class SerializedJob < Struct
+      #     #
+      #     # @param [String] job_id Active job's job id
+      #     # @param [Symbol] status Job's status
+      #     # @param [Exception, nil] error Job's error
+      #     #
+      #     def initialize(job_id, status, error)
+      #       @job_id = job_id
+      #       @status = status
+      #       @error = error
+      #     end
+      #   end
+      #
+      SerializedJob = Struct.new(:job_id, :status, :error) do
+        #
+        # Determines whether the job has a `completed` or `failed` status.
+        #
+        # @return [Boolean] True when the status is `completed` or `failed`
+        #
+        def stopped?
+          %i[completed failed].include? status
+        end
+      end
+
+      #
+      # Convert the given job into a `SerializedJob` struct.
+      #
+      # @param [Job] job The job to serialize
+      #
+      # @return [SerializedJob] The job converted into struct
+      #
+      def serialize_job(job)
+        SerializedJob.new(job.job_id, job.status, job.error)
+      end
+    end
+  end
+end

data/lib/ductr/store/job_store.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Store
+    #
+    # Job's level store interactions.
+    #
+    module JobStore
+      include JobSerializer
+
+      # @return [String] The job key prefix
+      JOB_KEY_PREFIX = "ductr:job"
+      # @return [String] The job registry key
+      JOB_REGISTRY_KEY = "ductr:job_registry"
+
+      #
+      # Get all known job instances.
+      #
+      # @return [Array<Job>] The job instances
+      #
+      def all_jobs
+        all(JOB_REGISTRY_KEY, JOB_KEY_PREFIX)
+      end
+
+      #
+      # Read all given jobs.
+      #
+      # @param [Array<Job>] *jobs The jobs to read
+      #
+      # @return [Array<Job>] The read jobs
+      #
+      def read_jobs(*jobs)
+        read(JOB_KEY_PREFIX, *jobs)
+      end
+
+      #
+      # Update the given job.
+      #
+      # @param [Job] job The job to update in the store
+      #
+      # @return [void]
+      #
+      def write_job(job)
+        write(JOB_KEY_PREFIX, serialize_job(job))
+      end
+
+      #
+      # Add the given job to the store's job registry. This method is NOT thread-safe.
+      #
+      # @param [Job] job The job to register
+      #
+      # @return [void]
+      #
+      def register_job(job)
+        register(JOB_REGISTRY_KEY, serialize_job(job))
+      end
+    end
+  end
+end

data/lib/ductr/store/pipeline_serializer.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Store
+    #
+    # Convert pipelines and steps into active job serializable structs.
+    #
+    module PipelineSerializer
+      include JobSerializer
+
+      #
+      # @!parse
+      #   #
+      #   # The pipeline representation as a struct.
+      #   #
+      #   # @!attribute [r] job_id
+      #   #   @return [String] The active job's job id
+      #   #
+      #   # @!attribute [r] status
+      #   #   @return [Symbol] The pipeline job status
+      #   #
+      #   # @!attribute [r] error
+      #   #   @return [Exception, nil] The pipeline job error if any
+      #   #
+      #   # @!attribute [r] steps
+      #   #   @return [Array<SerializedPipelineStep>] The pipeline steps as struct
+      #   #
+      #   class SerializedPipeline < Struct
+      #     #
+      #     # @param [String] job_id Pipeline job id
+      #     # @param [Symbol] status Pipeline status
+      #     # @param [Exception, nil] error Pipeline error
+      #     # @param [Array<SerializedPipelineStep>] steps Pipeline steps as struct
+      #     #
+      #     def initialize(job_id, status, error, steps)
+      #       @job_id = job_id
+      #       @status = status
+      #       @error = error
+      #       @steps = steps
+      #     end
+      #   end
+      #
+      SerializedPipeline = Struct.new(:job_id, :status, :error, :steps) do
+        #
+        # Determines whether the pipeline has a `completed` or `failed` status.
+        #
+        # @return [Boolean] True when the status is `completed` or `failed`
+        #
+        def stopped?
+          %i[completed failed].include? status
+        end
+      end
+
+      #
+      # @!parse
+      #   #
+      #   # The pipeline step representation as a struct.
+      #   #
+      #   # @!attribute [r] jobs
+      #   #   @return [Array<Job>] The step's jobs
+      #   #
+      #   # @!attribute [r] done
+      #   #   @return [Boolean] The step's fiber state
+      #   #
+      #   class SerializedPipelineStep < Struct
+      #     #
+      #     # @param [Array<Job>] jobs The step's jobs
+      #     # @param [Boolean] done The step's fiber state
+      #     #
+      #     def initialize(jobs, done)
+      #       @jobs = jobs
+      #       @done = done
+      #     end
+      #   end
+      #
+      SerializedPipelineStep = Struct.new(:jobs, :done) do
+        #
+        # Check if the step is done.
+        #
+        # @return [Boolean] True if the step is done
+        #
+        def done?
+          done
+        end
+      end
+
+      #
+      # Convert the given pipeline and its steps into
+      # `SerializedPipeline` and `SerializedPipelineStep` structs.
+      #
+      # @param [Pipeline] pipeline The pipeline to serialize
+      #
+      # @return [SerializedPipeline] The pipeline converted into struct
+      #
+      def serialize_pipeline(pipeline)
+        serialized_steps = pipeline.runner.steps.map do |step|
+          jobs = step.jobs.map { |j| serialize_job(j) }
+
+          SerializedPipelineStep.new(jobs, step.done?)
+        end
+
+        SerializedPipeline.new(pipeline.job_id, pipeline.status, pipeline.error, serialized_steps)
+      end
+    end
+  end
+end

data/lib/ductr/store/pipeline_store.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Ductr
+  module Store
+    #
+    # Pipeline's level store interactions.
+    #
+    module PipelineStore
+      include PipelineSerializer
+
+      # @return [String] The pipeline key prefix
+      PIPELINE_KEY_PREFIX = "ductr:pipeline"
+      # @return [String] The pipeline registry key
+      PIPELINE_REGISTRY_KEY = "ductr:pipeline_registry"
+
+      #
+      # Get all known pipeline instances.
+      #
+      # @return [Array<SerializedPipeline>] The pipeline instances
+      #
+      def all_pipelines
+        all(PIPELINE_REGISTRY_KEY, PIPELINE_KEY_PREFIX)
+      end
+
+      #
+      # Update the given pipeline.
+      #
+      # @param [Pipeline] pipeline The pipeline to update in the store
+      #
+      # @return [void]
+      #
+      def write_pipeline(pipeline)
+        write(PIPELINE_KEY_PREFIX, serialize_pipeline(pipeline))
+      end
+
+      #
+      # Add the given pipeline to the store's pipeline registry. This method is NOT thread-safe.
+      #
+      # @param [Pipeline] pipeline The job to register
+      #
+      # @return [void]
+      #
+      def register_pipeline(pipeline)
+        register(PIPELINE_REGISTRY_KEY, pipeline)
+      end
+    end
+  end
+end