ductr 0.1.0

data/lib/ductr/store.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Ductr
+  #
+  # Store interaction helpers for internal usage.
+  #
+  module Store
+    extend JobStore
+    extend PipelineStore
+
+    class << self
+      # @return [Integer] The cache expiration of job's status, default to one day
+      EXPIRATION_INTERVAL = 86_400
+
+      #
+      # Get all known job instances for the given registry_key and job's key_prefix.
+      #
+      # @param [String] registry_key The registry key in which job keys will be read
+      # @param [String] key_prefix The cache key prefix for the registry's job keys
+      #
+      # @return [Array<Job>] The job instances
+      #
+      def all(registry_key, key_prefix)
+        job_ids = Ductr.store.read(registry_key)
+        return [] unless job_ids
+
+        keys = job_ids.map { |job_id| "#{key_prefix}:#{job_id}" }
+        Ductr.store.read_multi(*keys).values
+      end
+
+      #
+      # Read all given jobs in the given key_prefix.
+      #
+      # @param [String] key_prefix The cache key prefix for the job_id
+      # @param [Array<Job>] *jobs The jobs to read
+      #
+      # @return [Array<Job>] The read jobs
+      #
+      def read(key_prefix, *jobs)
+        keys = jobs.map { |job| "#{key_prefix}:#{job.job_id}" }
+        Ductr.store.read_multi(*keys).values
+      end
+
+      #
+      # Update the given job in the given key_prefix.
+      #
+      # @param [Job] job The job to update in the store
+      #
+      # @return [void]
+      #
+      def write(key_prefix, job)
+        Ductr.store.write("#{key_prefix}:#{job.job_id}", job, expires_in: EXPIRATION_INTERVAL)
+      end
+
+      #
+      # Add the given job to the store's job registry. This method is NOT thread-safe.
+      #
+      # @param [Job] job The job to register
+      #
+      # @return [void]
+      #
+      def register(registry_key, job)
+        job_ids = Ductr.store.read(registry_key) || Set.new
+
+        job_ids.add(job.job_id)
+        Ductr.store.write(registry_key, job_ids, expires_in: EXPIRATION_INTERVAL)
+      end
+
+      #
+      # Determines whether all tracked jobs have either a completed or failed status.
+      #
+      # @return [Boolean] `true` when all jobs are done
+      #
+      def all_done?
+        [*all_jobs, *all_pipelines].all?(&:stopped?)
+      end
+    end
+  end
+end

data/lib/ductr/trigger.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Ductr
+  #
+  # The base class for any trigger, can be initialized by passing it its adapter name if any.
+  # A trigger must implement the #add method which is called for each trigger declaration.
+  # Depending on what your trigger do, you may have to implement the #start and #stop methods.
+  # #start is called when the scheduler relying on the trigger is started. #stop does the opposite:
+  # it is called when the scheduler relying on the trigger is stopped.
+  #
+  class Trigger
+    attr_reader :adapter
+
+    #
+    # Creates a new trigger instance, called by the scheduler.
+    #
+    # @param [Adapter, Nil] adapter The trigger's adapter, if any
+    #
+    def initialize(adapter = nil)
+      @adapter = adapter
+    end
+
+    #
+    # Adds a new trigger, called by a scheduler when a trigger is declared.
+    #
+    # @param [Method] _method The scheduler method to be called by the trigger
+    # @param [Hash<Symbol: Object>] _options options The options of the trigger declaration
+    #
+    # @return [void]
+    #
+    def add(_method, _options)
+      raise NotImplementedError, "A trigger must implement the #add method"
+    end
+
+    #
+    # Called when the scheduler relying on the trigger is started.
+    #
+    # @return [void]
+    #
+    def start; end
+
+    #
+    # Called when the scheduler relying on the trigger is stopped.
+    #
+    # @return [void]
+    #
+    def stop; end
+  end
+end

data/lib/ductr/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Ductr
+  # @return [String] The ductr's version number
+  VERSION = "0.1.0"
+end

data/lib/ductr.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "active_job"
+require "annotable"
+require "forwardable"
+require "zeitwerk"
+
+#
+# The main Ductr module.
+#
+module Ductr
+  class AdapterNotFoundError < StandardError; end
+  class ControlNotFoundError < StandardError; end
+  class InconsistentPaginationError < StandardError; end
+
+  class << self
+    #
+    # Contains all the Ductr configuration.
+    #
+    # @return [Configuration] The configuration instance
+    attr_reader :config
+
+    #
+    # The adapter classes registry, all declared adapters are in the registry.
+    #
+    # @return [Registry] The registry instance
+    #
+    def adapter_registry
+      @adapter_registry ||= Registry.new(:adapter)
+    end
+
+    #
+    # The trigger classes registry, all declared triggers are in the registry.
+    #
+    # @return [Registry] The registry instance
+    #
+    def trigger_registry
+      @trigger_registry ||= Registry.new(:trigger)
+    end
+
+    #
+    # The Ductr current environment, "development" by default.
+    # You can change it by setting the `DUCTR_ENV` environment variable.
+    #
+    # @return [String] The Ductr environment
+    #
+    def env
+      @env ||= ENV.fetch("DUCTR_ENV", "development").downcase
+    end
+
+    #
+    # Determines if Ductr is in development mode.
+    #
+    # @return [Boolean] True if DUCTR_ENV is set to "development" or nil
+    #
+    def development?
+      env == "development"
+    end
+
+    #
+    # Determines if Ductr is in production mode.
+    #
+    # @return [Boolean] True if DUCTR_ENV is set to "production"
+    #
+    def production?
+      env == "production"
+    end
+
+    #
+    # The configure block allows to configure Ductr internals.
+    # You must calls this method one and only one time to use the framework.
+    #
+    # @raise [ScriptError] Raises when called more than one time
+    # @return [void]
+    # @yield [config] Configure the framework
+    # @yieldparam [Configuration] config The configuration instance
+    #
+    def configure
+      raise ScriptError, "Ductr::configure must be called only once" if @config
+
+      @config = Configuration.new(env)
+      yield(@config)
+      @config.apply_active_job_config
+    end
+
+    #
+    # The Ductr main logger instance.
+    #
+    # @return [Log::Logger] The logger instance
+    #
+    def logger
+      @logger ||= config.logging.new
+    end
+
+    #
+    # The Ductr store, used to share information across different instances.
+    #
+    # @return [ActiveSupport::Cache::Store] The store instance
+    #
+    def store
+      @store ||= \
+        if config.store_adapter.is_a? Class
+          config.store_adapter.new(*config.store_parameters)
+        else
+          ActiveSupport::Cache.lookup_store(config.store_adapter, *config.store_parameters)
+        end
+    end
+  end
+end
+
+#
+# Framework auto loading
+#
+Zeitwerk::Loader.for_gem.tap do |loader|
+  loader.inflector.inflect "cli" => "CLI"
+  loader.inflector.inflect "etl" => "ETL"
+  loader.inflector.inflect "etl_job" => "ETLJob"
+  loader.inflector.inflect "job_etl_runner" => "JobETLRunner"
+
+  loader.collapse "#{__dir__}/ductr/etl/controls"
+  loader.collapse "#{__dir__}/ductr/log/outputs"
+  loader.collapse "#{__dir__}/ductr/log/formatters"
+
+  loader.ignore "#{__dir__}/ductr/cli/templates"
+
+  loader.setup
+  loader.eager_load_namespace(Ductr::RufusTrigger)
+end
+
+#
+# Application auto loading
+#
+if File.directory?("#{pwd = Dir.pwd}/app")
+  Zeitwerk::Loader.new.tap do |loader|
+    loader.push_dir "#{pwd}/app"
+
+    loader.collapse "#{pwd}/app/jobs"
+    loader.collapse "#{pwd}/app/pipelines"
+    loader.collapse "#{pwd}/app/schedulers"
+
+    loader.setup
+  end
+end