beetle_etl 0.0.1

data/lib/beetle_etl/steps/load.rb

@@ -0,0 +1,108 @@
+module BeetleETL
+  class Load < Step
+
+    IMPORTER_COLUMNS = %i[
+      import_run_id
+      external_source
+      transition
+    ]
+
+    def run
+      %w(create update delete undelete).each do |transition|
+        public_send(:"load_#{transition}")
+      end
+    end
+
+    def load_create
+      just_now = now
+      database[table_name].import(
+        data_columns + [:external_source, :created_at, :updated_at],
+        database[:"#{stage_schema}__#{table_name}"]
+          .select(*data_columns)
+          .where(
+            import_run_id: run_id,
+            transition: 'CREATE'
+          )
+          .select_more(external_source, just_now, just_now)
+        )
+    end
+
+    def load_update
+      updates = updatable_columns.reduce({updated_at: now}) do |acc, column|
+        acc[column] = :"stage__#{column}"
+        acc
+      end
+
+      database.from(
+        :"#{table_name}___public",
+        :"#{stage_schema}__#{table_name}___stage"
+      )
+        .where(
+          stage__id: :public__id,
+          stage__transition: 'UPDATE',
+          stage__import_run_id: run_id,
+        )
+        .update(updates)
+    end
+
+    def load_delete
+      just_now = now
+      database.from(
+          :"#{table_name}___public",
+          :"#{stage_schema}__#{table_name}___stage"
+        )
+        .where(
+          stage__id: :public__id,
+          stage__transition: 'DELETE',
+          stage__import_run_id: run_id,
+        )
+        .update(
+          updated_at: just_now,
+          deleted_at: just_now,
+        )
+    end
+
+    def load_undelete
+      updates = updatable_columns.reduce({updated_at: now, deleted_at: nil}) do |acc, column|
+        acc[column] = :"stage__#{column}"
+        acc
+      end
+
+      database.from(
+        :"#{table_name}___public",
+        :"#{stage_schema}__#{table_name}___stage"
+      )
+        .where(
+          stage__id: :public__id,
+          stage__transition: 'UNDELETE',
+          stage__import_run_id: run_id,
+        )
+        .update(updates)
+    end
+
+    private
+
+    def data_columns
+      table_columns - ignored_columns
+    end
+
+    def table_columns
+      @table_columns ||= database[:"#{stage_schema}__#{table_name}"].columns
+    end
+
+    def ignored_columns
+      IMPORTER_COLUMNS + table_columns.select do |column_name|
+        column_name.to_s.index(/^external_.+_id$/)
+      end
+    end
+
+    def updatable_columns
+      data_columns - [:id, :external_source, :external_id]
+    end
+
+    def now
+      Time.now
+    end
+
+  end
+end

data/lib/beetle_etl/steps/map_relations.rb

@@ -0,0 +1,31 @@
+module BeetleETL
+  class MapRelations < Step
+
+    attr_reader :relations
+
+    def initialize(table_name, relations)
+      super(table_name)
+      @relations = relations
+    end
+
+    def dependencies
+      relations.values.map { |d| AssignIds.step_name(d) }.to_set << Transform.step_name(table_name)
+    end
+
+    def run
+      relations.each do |foreign_key_column, foreign_table_name|
+        database.from(
+          :"#{stage_schema}__#{table_name}___ST",
+          :"#{stage_schema}__#{foreign_table_name}___FT"
+        ).where(
+          ST__import_run_id: run_id,
+          FT__import_run_id: run_id,
+          FT__external_id: :"ST__external_#{foreign_key_column}",
+        ).update(
+          :"#{foreign_key_column}" => :"FT__id"
+        )
+      end
+    end
+
+  end
+end

data/lib/beetle_etl/steps/step.rb

@@ -0,0 +1,42 @@
+module BeetleETL
+
+  DependenciesNotDefinedError = Class.new(StandardError)
+
+  class Step
+
+    attr_reader :table_name
+
+    def initialize(table_name)
+      @table_name = table_name
+    end
+
+    def self.step_name(table_name)
+      "#{table_name}: #{name.split('::').last}"
+    end
+
+    def name
+      self.class.step_name(table_name)
+    end
+
+    def dependencies
+      raise DependenciesNotDefinedError
+    end
+
+    def run_id
+      BeetleETL.state.run_id
+    end
+
+    def stage_schema
+      BeetleETL.config.stage_schema
+    end
+
+    def external_source
+      BeetleETL.config.external_source
+    end
+
+    def database
+      BeetleETL.database
+    end
+
+  end
+end

data/lib/beetle_etl/steps/table_diff.rb

@@ -0,0 +1,155 @@
+module BeetleETL
+  class TableDiff < Step
+
+    IMPORTER_COLUMNS = %i[
+      import_run_id
+      external_id
+      transition
+    ]
+
+    def dependencies
+      [MapRelations.step_name(table_name)].to_set
+    end
+
+    def run
+      %w(create keep update delete undelete).each do |transition|
+        public_send(:"transition_#{transition}")
+      end
+    end
+
+    def transition_create
+      stage_table.where(
+        stage__import_run_id: run_id,
+      )
+      .where(Sequel.~(public_table.where(
+          public__external_id: :stage__external_id,
+          public__external_source: external_source,
+        )
+        .exists))
+      .update(transition: 'CREATE')
+    end
+
+    def transition_keep
+      stage_table.where(
+        stage__import_run_id: run_id,
+      )
+      .where(
+        public_table.where(
+          public__external_id: :stage__external_id,
+          public__external_source: external_source,
+          public__deleted_at: nil,
+        )
+        .where(
+          ':public_columns IS NOT DISTINCT FROM :stage_columns',
+          public_columns: public_record_columns,
+          stage_columns: stage_record_columns,
+        )
+        .exists)
+      .update(transition: 'KEEP')
+    end
+
+    def transition_update
+      stage_table.where(
+        stage__import_run_id: run_id,
+      )
+      .where(
+        public_table.where(
+          public__external_id: :stage__external_id,
+          public__external_source: external_source,
+          public__deleted_at: nil,
+        )
+        .where(
+          ':public_columns IS DISTINCT FROM :stage_columns',
+          public_columns: public_record_columns,
+          stage_columns: stage_record_columns,
+        )
+        .exists)
+      .update(transition: 'UPDATE')
+    end
+
+    def transition_delete
+      deleted_dataset = database.from(
+        :"#{stage_schema}__#{table_name}___stage",
+      ).right_join(
+        :"#{table_name}___public",
+        public__external_id: :stage__external_id,
+        public__external_source: external_source,
+      ).where(
+        stage__external_id: nil,
+        public__deleted_at: nil
+      )
+
+      database[:"#{stage_schema}__#{table_name}"]
+        .import(
+          [
+            :import_run_id,
+            :external_id,
+            :transition
+          ],
+          deleted_dataset
+            .select(
+              run_id,
+              :public__external_id,
+              'DELETE'
+            )
+        )
+    end
+
+    def transition_undelete
+      stage_table.where(
+        stage__import_run_id: run_id,
+      )
+      .where(
+        public_table.where(
+          public__external_id: :stage__external_id,
+          public__external_source: external_source,
+        )
+        .exclude(
+          public__deleted_at: nil
+        )
+        .exists)
+      .update(transition: 'UNDELETE')
+    end
+
+    private
+
+    def stage_table
+      @stage_table ||= database[:"#{stage_schema}__#{table_name}___stage"]
+    end
+
+    def public_table
+      @public_table ||= database[:"#{table_name}___public"]
+    end
+
+    def public_record_columns
+      prefixed_columns(data_columns, 'public')
+    end
+
+    def stage_record_columns
+      prefixed_columns(data_columns, 'stage')
+    end
+
+    def data_columns
+      table_columns - ignored_columns
+    end
+
+    def table_columns
+      @table_columns ||= database[:"#{stage_schema}__#{table_name}"].columns
+    end
+
+    def ignored_columns
+      importer_columns + [:id] + table_columns.select do |column_name|
+        column_name.to_s.index(/^external_.+_id$/)
+      end
+    end
+
+    def importer_columns
+      IMPORTER_COLUMNS
+    end
+
+    def prefixed_columns(columns, prefix)
+      columns.map { |column| "#{prefix}__#{column}".to_sym }
+    end
+
+  end
+end

data/lib/beetle_etl/steps/transform.rb

@@ -0,0 +1,22 @@
+module BeetleETL
+  class Transform < Step
+
+    attr_reader :query
+
+    def initialize(table_name, dependencies, query)
+      super(table_name)
+      @dependencies = dependencies
+      @query = query
+    end
+
+    def dependencies
+      Set.new(@dependencies.map { |d| self.class.step_name(d) })
+    end
+
+    def run
+      database.run(query)
+    end
+
+  end
+end
+

data/lib/beetle_etl/task_runner/dependency_resolver.rb

@@ -0,0 +1,39 @@
+module BeetleETL
+
+  UnsatisfiableDependenciesError = Class.new(StandardError)
+
+  class DependencyResolver
+
+    def initialize(items)
+      @items = items
+      check
+    end
+
+    def resolvables(resolved)
+      @items.select do |item|
+        (item.dependencies.subset?(resolved.to_set) || item.dependencies.empty?) && !resolved.include?(item.name)
+      end
+    end
+
+    private
+
+    def check
+      items = @items.dup
+      resolved = []
+
+      while not items.empty?
+        resolved_names = resolved.flatten.map(&:name).to_set
+
+        resolvable = items.select do |item|
+          item.dependencies.subset?(resolved_names) || item.dependencies.empty?
+        end
+
+        raise UnsatisfiableDependenciesError if resolvable.empty?
+
+        resolvable.each { |r| items.delete r }
+        resolved << resolvable
+      end
+    end
+
+  end
+end

data/lib/beetle_etl/task_runner/task_runner.rb

@@ -0,0 +1,64 @@
+require 'celluloid/autostart'
+
+module BeetleETL
+  class TaskRunner
+
+    include Celluloid
+
+    def initialize(runnables)
+      @runnables = runnables
+      @completed = Set.new
+      @running = Set.new
+      @dependency_resolver = DependencyResolver.new(runnables)
+
+      run_next
+    end
+
+    def completed(runnable_name)
+      @running.delete(runnable_name)
+      @completed << runnable_name
+
+      run_next
+    end
+
+    def run_next
+      if all_run?
+        terminate
+      else
+        resolvables.each do |runnable|
+          unless @running.include?(runnable.name)
+            Task.new(Actor.current, runnable).async.run_task
+            @running << runnable.name
+          end
+        end
+      end
+    end
+
+    private
+
+    def resolvables
+      @dependency_resolver.resolvables(@completed)
+    end
+
+    def all_run?
+      @completed == @runnables.map(&:name).to_set
+    end
+
+    class Task
+
+      include Celluloid
+
+      def initialize(runner, task)
+        @runner = runner
+        @task = task
+      end
+
+      def run_task
+        @task.run
+        @runner.async.completed(@task.name)
+        terminate
+      end
+    end
+
+  end
+end