ductr 0.1.0

data/lib/ductr/cli/new_project_generator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "thor"
+require "thor/group"
+
+module Ductr
+  module CLI
+    #
+    # Thor generator to create a new project
+    #
+    class NewProjectGenerator < Thor::Group
+      include Thor::Actions
+      desc "Generate a new project"
+      argument :name, type: :string, optional: true, default: ""
+
+      #
+      # The templates source used to create a new project
+      #
+      # @return [String] the templates source absolute path
+      #
+      def self.source_root
+        "#{__dir__}/templates/project"
+      end
+
+      #
+      # Doing some setup before generating file,
+      # creates the project directory and sets it as destination for the generator
+      #
+      # @return [void]
+      #
+      def init
+        empty_directory name
+        self.destination_root = "#{destination_root}/#{name}"
+      end
+
+      #
+      # Creates files in the project's root
+      #
+      # @return [void]
+      #
+      def gen_root
+        copy_file "gemfile.rb", "Gemfile"
+        copy_file "rubocop.yml", ".rubocop.yml"
+        copy_file "tool-versions", ".tool-versions"
+
+        create_file "app/jobs/.gitkeep"
+        create_file "app/pipelines/.gitkeep"
+        create_file "app/schedulers/.gitkeep"
+      end
+
+      #
+      # Creates the bin file for the project
+      #
+      # @return [void]
+      #
+      def gen_bin
+        copy_file "bin_ductr.rb", "bin/ductr"
+      end
+
+      #
+      # Creates files in the `config` folder
+      #
+      # @return [void]
+      #
+      def gen_config
+        copy_file "config_app.rb", "config/app.rb"
+        copy_file "config_development.yml", "config/development.yml"
+        copy_file "config_environment_development.rb", "config/environment/development.rb"
+      end
+    end
+  end
+end

data/lib/ductr/cli/templates/project/bin_ductr.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "ductr"
+require_relative "../config/app"
+
+Ductr::CLI::Main.start

data/lib/ductr/cli/templates/project/config_app.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "ductr"
+
+require_relative "environment/#{Ductr.env}"

data/lib/ductr/cli/templates/project/config_development.yml

@@ -0,0 +1,8 @@
+# Add your vars here, you can make erb bindings e.g.:
+# example: <%= ENV.fetch("SOME_VAR", "some default value") %>
+
+# declare your adapters here
+adapters:
+  # my_adapter:
+  #   adapter: sqlite
+  #   database: some.db

data/lib/ductr/cli/templates/project/config_environment_development.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+require "active_support/cache/file_store"
+
+Ductr.configure do |config|
+  # Store configuration.
+  #
+  # You can pass an `ActiveSupport::Cache::Store` class or a symbol
+  config.store(ActiveSupport::Cache::FileStore, "tmp/store")
+
+  # Logging configuration.
+  #
+  # The following logging levels are available:
+  # :debug, :info, :warn, :error, :fatal
+  config.logging.level = :debug
+
+  # Append logs to the stdout by default, you can add/replace outputs at will.
+  config.logging.add_output(Ductr::Log::StandardOutput, Ductr::Log::ColorFormatter)
+end

data/lib/ductr/cli/templates/project/gemfile.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "annotable", git: "git@gitlab.com:la-manufacture/rocket/annotable.git", branch: "master"
+gem "ductr", git: "git@gitlab.com:la-manufacture/rocket/ductr.git", branch: "main"

data/lib/ductr/cli/templates/project/rubocop.yml

@@ -0,0 +1,14 @@
+AllCops:
+  TargetRubyVersion: 3.1
+  NewCops: enable
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/lib/ductr/cli/templates/project/tool-versions

@@ -0,0 +1 @@
+ruby 3.1.0

data/lib/ductr/configuration.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "erb"
+
+module Ductr
+  #
+  # Contains the framework's configuration, including:
+  # - `DUCTR_ENV` environment variable
+  # - `Ductr.configure` block
+  # - Project root path
+  # - YML file configuration
+  #
+  class Configuration
+    # @return [Struct] The active job configuration, available options are
+    #                  `queue_adapter`, `default_queue_name`, `queue_name_prefix` & `queue_name_delimiter`
+    attr_reader :active_job
+
+    # @return [Class<Ductr::Log::Logger>] The logger constant
+    attr_reader :logging
+
+    # @return [String] The project root
+    attr_reader :root
+
+    # @return [Class<ActiveSupport::Cache::Store>, Symbol] The store adapter to use
+    #   @see https://edgeapi.rubyonrails.org/classes/ActiveSupport/Cache.html#method-c-lookup_store
+    attr_reader :store_adapter
+
+    # @return [Array] The store adapter config
+    attr_reader :store_parameters
+
+    # @return [Hash] The parsed YML configuration
+    attr_reader :yml
+
+    #
+    # Initializing environment to "development" by default, setting project root, parsing YML with the config gem
+    # and aliasing semantic logger gem constant to make it usable through the `Ductr.configure` block
+    #
+    def initialize(env)
+      @root = Dir.pwd
+      @yml = load_yaml("#{root}/config/#{env}.yml")
+
+      @logging = Log::Logger
+      logging.level = :debug
+
+      @active_job = Struct.new(:queue_adapter, :default_queue_name, :queue_name_prefix, :queue_name_delimiter).new
+      @store_adapter = ActiveSupport::Cache::FileStore
+      @store_parameters = ["tmp/store"]
+    end
+
+    #
+    # Configures the store instance.
+    #
+    # @param [Class<ActiveSupport::Cache::Store>, Symbol] adapter The store adapter class
+    # @param [Array] *parameters The store adapter configuration
+    #
+    # @return [void]
+    #
+    def store(adapter, *parameters)
+      @store_adapter = adapter
+      @store_parameters = parameters
+    end
+
+    #
+    # Memoize configured adapters based on the YAML configuration.
+    #
+    # @return [Array<Adapter>] The configured Adapter instances
+    #
+    def adapters
+      @adapters ||= yml.adapters.to_h.map do |name, entry|
+        adapter_class = Ductr.adapter_registry.find(entry.adapter)
+        config = entry.to_h.except(:adapter)
+
+        adapter_class.new(name, **config)
+      end
+    end
+
+    #
+    # Find an adapter based on its name.
+    #
+    # @param [Symbol] name The name of the adapter to find
+    #
+    # @raise [AdapterNotFoundError] If no adapter match the given name
+    # @return [Adapter] The adapter found
+    #
+    def adapter(name)
+      not_found_error = -> { raise AdapterNotFoundError, "The adapter named \"#{name}\" does not exist" }
+
+      adapters.find(not_found_error) do |adapter|
+        adapter.name == name
+      end
+    end
+
+    #
+    # Configures active job with the given options.
+    #
+    # @return [void]
+    #
+    def apply_active_job_config
+      ActiveJob::Base.logger = logging.new("ActiveJob")
+
+      active_job.each_pair do |opt, value|
+        next unless value
+
+        ActiveJob::Base.send("#{opt}=", value)
+      end
+    end
+
+    private
+
+    #
+    # Load YAML configuration localized at given path.
+    # Parse the file with ERB before parsing YAML, so we can use env vars in config files.
+    #
+    # @param [String] path The path of the YAML file to load
+    #
+    # @return [Struct] The parsed YAML configuration
+    #
+    def load_yaml(path)
+      return {} unless path && File.exist?(path)
+
+      erb = ERB.new File.read(path)
+      yaml = YAML.load(erb.result, symbolize_names: true)
+
+      hash_to_struct(yaml)
+    end
+
+    #
+    # Recursively convert Hash into Struct.
+    #
+    # @param [Hash] hash The hash to convert
+    #
+    # @return [Struct] The converted hash
+    #
+    def hash_to_struct(hash)
+      values = hash.values.map do |value|
+        next hash_to_struct(value) if value.is_a?(Hash)
+
+        value
+      end
+
+      Struct.new(*hash.keys).new(*values)
+    end
+  end
+end

data/lib/ductr/etl/controls/buffered_destination.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class to implement buffered destinations.
+    #
+    class BufferedDestination < Destination
+      # @return [Array] The row buffer
+      attr_reader :buffer
+
+      #
+      # The buffer size option, default to 10_000.
+      #
+      # @return [Integer] The buffer size
+      #
+      def buffer_size
+        @options[:buffer_size] || 10_000
+      end
+
+      #
+      # Pushes the row inside the buffer or flushes it when full.
+      #
+      # @param [Object] row The row to write
+      #
+      # @return [void]
+      #
+      def write(row)
+        @buffer ||= []
+
+        @buffer.push row
+        flush_buffer if @buffer.size == buffer_size
+      end
+
+      #
+      # Flushes the buffer, called when the last row is reached.
+      #
+      # @return [void]
+      #
+      def close
+        flush_buffer unless @buffer.empty?
+        super
+      end
+
+      #
+      # Calls #on_flush and reset the buffer.
+      #
+      # @return [void]
+      #
+      def flush_buffer
+        on_flush
+        @buffer = []
+      end
+
+      #
+      # Called each time the buffer have to be emptied.
+      #
+      # @return [void]
+      #
+      def on_flush
+        raise NotImplementedError, "A buffered destination must implement the `#on_flush` method"
+      end
+    end
+  end
+end

data/lib/ductr/etl/controls/buffered_transform.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class to implement buffered transforms.
+    #
+    class BufferedTransform < Transform
+      # @return [Array] The row buffer
+      attr_reader :buffer
+
+      #
+      # The buffer size option, default to 10_000.
+      #
+      # @return [Integer] The buffer size
+      #
+      def buffer_size
+        @options[:buffer_size] || 10_000
+      end
+
+      #
+      # Pushes the row inside the buffer or flushes it when full.
+      #
+      # @param [Object] row The row to process
+      # @yield [row] The row yielder
+      #
+      # @return [nil] Returning nil to complies with kiba
+      #
+      def process(row, &)
+        @buffer ||= []
+
+        @buffer.push row
+        flush_buffer(&) if @buffer.size == buffer_size
+
+        # avoid returning a row, see
+        # https://github.com/thbar/kiba/wiki/Implementing-ETL-transforms#generating-more-than-one-output-row-per-input-row-aka-yielding-transforms
+        nil
+      end
+
+      #
+      # Called when the last row is reached.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def close(&)
+        flush_buffer(&) unless @buffer.empty?
+        super
+      end
+
+      #
+      # Calls #on_flush and reset the buffer.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def flush_buffer(&)
+        on_flush(&)
+        @buffer = []
+      end
+
+      #
+      # Called each time the buffer have to be emptied.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def on_flush(&)
+        raise NotImplementedError, "A buffered transform must implement the `#on_flush` method"
+      end
+    end
+  end
+end

data/lib/ductr/etl/controls/control.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class for all types of ETL control.
+    #
+    class Control
+      extend Forwardable
+
+      class << self
+        # @return [Symbol] The control type, written when registering the control into its adapter
+        attr_accessor :type
+      end
+
+      #
+      # @!method call_method
+      #   Invokes the job's method linked to the control.
+      #   @return [Object] Something returned by the method, e.g. a query, a file, a row, ...
+      #
+      def_delegator :@job_method, :call, :call_method
+
+      # @return [Symbol] The method to be called by the control
+      attr_reader :job_method
+
+      # @return [Hash] The configuration hash of the control's adapter
+      attr_reader :options
+
+      # @return [Adapter] The control's adapter
+      attr_reader :adapter
+
+      #
+      # Creates a new control based on the job instance and the configured adapter.
+      #
+      # @param [Method] job_method The job's method to be called by the control
+      # @param [Adapter] adapter The configured adapter
+      # @param [Hash] **options The configuration hash of the control's adapter
+      #
+      def initialize(job_method, adapter = nil, **options)
+        @job_method = job_method
+        @adapter = adapter
+        @options = options
+      end
+    end
+  end
+end

data/lib/ductr/etl/controls/destination.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class for implementing destinations.
+    #
+    class Destination < Control
+      #
+      # Writes the row into the destination.
+      #
+      # @param [Object] row The row to write
+      #
+      # @return [void]
+      #
+      def write(row)
+        call_method(row)
+      end
+
+      #
+      # Called when the last row is reached, closes the adapter.
+      #
+      # @return [void]
+      #
+      def close; end
+    end
+  end
+end

data/lib/ductr/etl/controls/paginated_source.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class to implement paginated source.
+    #
+    class PaginatedSource < Source
+      #
+      # The page size option, default to 10_000.
+      #
+      # @return [Integer] The page size
+      #
+      def page_size
+        @options[:page_size] || 10_000
+      end
+
+      #
+      # Iterates over pages and calls #each_page.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def each(&)
+        @offset ||= 0
+
+        loop do
+          break unless each_page(&)
+
+          @offset += page_size
+        end
+      end
+
+      #
+      # Called once per pages.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def each_page(&)
+        raise NotImplementedError, "A paginated source must implement the `#each_page` method"
+      end
+    end
+  end
+end

data/lib/ductr/etl/controls/source.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # The base class for implementing sources.
+    #
+    class Source < Control
+      #
+      # Iterates over rows.
+      #
+      # @yield [row] The row yielder
+      #
+      # @return [void]
+      #
+      def each(&)
+        call_method.each(&)
+      end
+    end
+  end
+end

data/lib/ductr/etl/controls/transform.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Ductr
+  module ETL
+    #
+    # Base class for implementing transforms.
+    #
+    class Transform < Control
+      #
+      # Calls the control method and passes the row.
+      #
+      # @param [Object] row The row to process
+      #
+      # @return [void]
+      #
+      def process(row)
+        call_method(row)
+      end
+
+      #
+      # Called when the last row is reached.
+      #
+      # @return [void]
+      #
+      def close; end
+    end
+  end
+end