beetle_etl 0.0.2 → 0.0.7

data/lib/beetle_etl/steps/map_relations.rb

@@ -1,29 +1,23 @@
 module BeetleETL
   class MapRelations < Step
 
-    attr_reader :relations
-
     def initialize(table_name, relations)
       super(table_name)
       @relations = relations
     end
 
     def dependencies
-      relations.values.map { |d| AssignIds.step_name(d) }.to_set << Transform.step_name(table_name)
+      @relations.values.map { |d| AssignIds.step_name(d) }.to_set << Transform.step_name(table_name)
     end
 
     def run
-      relations.each do |foreign_key_column, foreign_table_name|
-        database.from(
-          :"#{stage_schema}__#{table_name}___ST",
-          :"#{stage_schema}__#{foreign_table_name}___FT"
-        ).where(
-          ST__import_run_id: run_id,
-          FT__import_run_id: run_id,
-          FT__external_id: :"ST__external_#{foreign_key_column}",
-        ).update(
-          :"#{foreign_key_column}" => :"FT__id"
-        )
+      @relations.map do |foreign_key_column, foreign_table_name|
+        database.execute <<-SQL
+          UPDATE #{stage_table_name_sql} current_table
+          SET #{foreign_key_column} = foreign_table.id
+          FROM #{stage_table_name_sql(foreign_table_name)} foreign_table
+          WHERE current_table.external_#{foreign_key_column} = foreign_table.external_id
+        SQL
       end
     end
 

data/lib/beetle_etl/steps/step.rb

@@ -4,6 +4,7 @@ module BeetleETL
 
   class Step
 
+    include BeetleETL::Naming
     attr_reader :table_name
 
     def initialize(table_name)
@@ -22,14 +23,6 @@ module BeetleETL
       raise DependenciesNotDefinedError
     end
 
-    def run_id
-      BeetleETL.state.run_id
-    end
-
-    def stage_schema
-      BeetleETL.config.stage_schema
-    end
-
     def external_source
       BeetleETL.config.external_source
     end

data/lib/beetle_etl/steps/table_diff.rb

@@ -2,7 +2,6 @@ module BeetleETL
   class TableDiff < Step
 
     IMPORTER_COLUMNS = %i[
-      import_run_id
       external_id
       transition
     ]
@@ -12,115 +11,95 @@ module BeetleETL
     end
 
     def run
-      %w(create keep update delete undelete).map do |transition|
-        Thread.new { public_send(:"transition_#{transition}") }
-      end.each(&:join)
+      %w(create keep update delete undelete).each do |transition|
+        public_send(:"transition_#{transition}")
+      end
     end
 
     def transition_create
-      stage_table.where(
-        stage__import_run_id: run_id,
-      )
-      .where(Sequel.~(public_table.where(
-          public__external_id: :stage__external_id,
-          public__external_source: external_source,
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql} stage
+        SET transition = 'CREATE'
+        WHERE NOT EXISTS (
+          SELECT 1
+          FROM #{public_table_name} public
+          WHERE public.external_id = stage.external_id
+          AND public.external_source = '#{external_source}'
         )
-        .exists))
-      .update(transition: 'CREATE')
+      SQL
     end
 
     def transition_keep
-      stage_table.where(
-        stage__import_run_id: run_id,
-      )
-      .where(
-        public_table.where(
-          public__external_id: :stage__external_id,
-          public__external_source: external_source,
-          public__deleted_at: nil,
-        )
-        .where(
-          ':public_columns IS NOT DISTINCT FROM :stage_columns',
-          public_columns: public_record_columns,
-          stage_columns: stage_record_columns,
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql} stage
+        SET transition = 'KEEP'
+        WHERE EXISTS (
+          SELECT 1
+          FROM #{public_table_name} public
+          WHERE public.external_id = stage.external_id
+          AND public.external_source = '#{external_source}'
+          AND public.deleted_at IS NULL
+          AND
+            (#{public_record_columns.join(', ')})
+            IS NOT DISTINCT FROM
+            (#{stage_record_columns.join(', ')})
         )
-        .exists)
-      .update(transition: 'KEEP')
+      SQL
     end
 
     def transition_update
-      stage_table.where(
-        stage__import_run_id: run_id,
-      )
-      .where(
-        public_table.where(
-          public__external_id: :stage__external_id,
-          public__external_source: external_source,
-          public__deleted_at: nil,
-        )
-        .where(
-          ':public_columns IS DISTINCT FROM :stage_columns',
-          public_columns: public_record_columns,
-          stage_columns: stage_record_columns,
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql} stage
+        SET transition = 'UPDATE'
+        WHERE EXISTS (
+          SELECT 1
+          FROM #{public_table_name} public
+          WHERE public.external_id = stage.external_id
+          AND public.external_source = '#{external_source}'
+          AND public.deleted_at IS NULL
+          AND
+            (#{public_record_columns.join(', ')})
+            IS DISTINCT FROM
+            (#{stage_record_columns.join(', ')})
         )
-        .exists)
-      .update(transition: 'UPDATE')
+      SQL
     end
 
     def transition_delete
-      deleted_dataset = database.from(
-        :"#{stage_schema}__#{table_name}___stage",
-      ).right_join(
-        :"#{table_name}___public",
-        public__external_id: :stage__external_id,
-        public__external_source: external_source,
-      ).where(
-        stage__external_id: nil,
-        public__deleted_at: nil
-      )
-
-      database[:"#{stage_schema}__#{table_name}"]
-        .import(
-          [
-            :import_run_id,
-            :external_id,
-            :transition
-          ],
-          deleted_dataset
-            .select(
-              run_id,
-              :public__external_id,
-              'DELETE'
-            )
-        )
+      database.execute <<-SQL
+        INSERT INTO #{stage_table_name_sql}
+          (external_id, transition)
+        SELECT
+          public.external_id,
+          'DELETE'
+        FROM #{public_table_name_sql} public
+        LEFT OUTER JOIN (
+          SELECT *
+          FROM #{stage_table_name_sql}
+          ) stage
+          ON (stage.external_id = public.external_id)
+        WHERE stage.external_id IS NULL
+        AND public.external_source = '#{external_source}'
+        AND public.deleted_at IS NULL
+      SQL
     end
 
     def transition_undelete
-      stage_table.where(
-        stage__import_run_id: run_id,
-      )
-      .where(
-        public_table.where(
-          public__external_id: :stage__external_id,
-          public__external_source: external_source,
-        )
-        .exclude(
-          public__deleted_at: nil
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql} stage
+        SET transition = 'UNDELETE'
+        WHERE EXISTS (
+          SELECT 1
+          FROM #{public_table_name_sql} public
+          WHERE public.external_id = stage.external_id
+          AND public.external_source = '#{external_source}'
+          AND public.deleted_at IS NOT NULL
         )
-        .exists)
-      .update(transition: 'UNDELETE')
+      SQL
     end
 
     private
 
-    def stage_table
-      @stage_table ||= database[:"#{stage_schema}__#{table_name}___stage"]
-    end
-
-    def public_table
-      @public_table ||= database[:"#{table_name}___public"]
-    end
-
     def public_record_columns
       prefixed_columns(data_columns, 'public')
     end
@@ -134,7 +113,7 @@ module BeetleETL
     end
 
     def table_columns
-      @table_columns ||= database[:"#{stage_schema}__#{table_name}"].columns
+      @table_columns ||= database[stage_table_name.to_sym].columns
     end
 
     def ignored_columns
@@ -148,7 +127,7 @@ module BeetleETL
     end
 
     def prefixed_columns(columns, prefix)
-      columns.map { |column| "#{prefix}__#{column}".to_sym }
+      columns.map { |column| %Q("#{prefix}"."#{column}") }
     end
 
   end

data/lib/beetle_etl/steps/transform.rb

@@ -1,8 +1,6 @@
 module BeetleETL
   class Transform < Step
 
-    attr_reader :query
-
     def initialize(table_name, dependencies, query)
       super(table_name)
       @dependencies = dependencies
@@ -10,11 +8,11 @@ module BeetleETL
     end
 
     def dependencies
-      Set.new(@dependencies.map { |d| self.class.step_name(d) })
+      Set.new(@dependencies.map { |d| self.class.step_name(d) }) << CreateStage.step_name(table_name)
     end
 
     def run
-      database.run(query)
+      database.run(@query)
     end
 
   end

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "0.0.2"
+  VERSION = "0.0.7"
 end

data/spec/beetle_etl_spec.rb

@@ -1,34 +1,12 @@
 require 'spec_helper'
 
 describe BeetleETL do
-  describe '#import' do
 
+  describe '#import' do
     it 'runs the import' do
-      allow(BeetleETL).to receive(:state) { double(:state).as_null_object }
-      expect(BeetleETL::Import).to receive(:run)
+      expect(BeetleETL::Import).to receive_message_chain(:new, :run)
       BeetleETL.import
     end
-
-    context 'handling state' do
-      it 'starts the import and marks it as finished if no errors are thrown' do
-        allow(BeetleETL::Import).to receive(:run)
-
-        expect(BeetleETL.state).to receive(:start_import).ordered
-        expect(BeetleETL.state).to receive(:mark_as_succeeded).ordered
-
-        BeetleETL.import
-      end
-
-      it 'starts the import and marks it as failed if Import.run throws an error' do
-        exception = Exception.new
-        allow(BeetleETL::Import).to receive(:run).and_raise(exception)
-
-        expect(BeetleETL.state).to receive(:start_import).ordered
-        expect(BeetleETL.state).to receive(:mark_as_failed).ordered
-
-        expect { BeetleETL.import }.to raise_exception(exception)
-      end
-    end
   end
 
   describe '#config' do
@@ -65,6 +43,6 @@ describe BeetleETL do
       expect(BeetleETL.database).to eql(database)
       expect(BeetleETL.database).to eql(database)
     end
-
   end
+
 end

data/spec/dsl/dsl_spec.rb

@@ -6,15 +6,16 @@ module BeetleETL
     subject { DSL.new(:foo_table) }
 
     describe '#stage_table' do
-      it 'returns the stage table name including the schema defined in the config' do
-        BeetleETL.configure { |config| config.stage_schema = 'bar' }
-        expect(subject.stage_table).to eql('"bar"."foo_table"')
+      it 'returns the current stage table name' do
+        expect(subject.stage_table).to eql(
+          BeetleETL::Naming.stage_table_name_sql(:foo_table)
+        )
       end
-    end
 
-    describe '#external_source' do
-      it 'returns the external source‘s identifier' do
-        expect(subject.external_source).to eql('source')
+      it 'returns the stage table name for the given table' do
+        expect(subject.stage_table(:bar_table)).to eql(
+          BeetleETL::Naming.stage_table_name_sql(:bar_table)
+        )
       end
     end
 
@@ -32,13 +33,5 @@ module BeetleETL
       end
     end
 
-    describe '#import_run_id' do
-      it 'returns the import run id defined in the config' do
-        id = double(:id)
-        allow(BeetleETL.state).to receive(:run_id) { id }
-        expect(subject.import_run_id).to eql(id)
-      end
-    end
-
   end
 end

data/spec/dsl/transformation_loader_spec.rb

@@ -4,6 +4,8 @@ require 'tempfile'
 module BeetleETL
   describe TransformationLoader do
 
+    subject { TransformationLoader.new }
+
     before :example do
       data_file = tempfile_with_contents <<-FILE
         import :foo do
@@ -13,6 +15,10 @@ module BeetleETL
         import :bar do
           'bar'
         end
+
+        helpers do
+          "baz"
+        end
       FILE
 
       BeetleETL.configure do |config|
@@ -21,16 +27,17 @@ module BeetleETL
     end
 
     describe '#load' do
-      it 'loads runlist entries from the data file' do
-        expect(Transformation).to receive(:new) do |table_name, config|
+      it 'loads transformations from the data file' do
+        expect(Transformation).to receive(:new) do |table_name, config, helpers|
           expect(table_name.to_s).to eql(config.call)
+          expect(helpers.call).to eql("baz")
         end.exactly(2).times
 
         subject.load
       end
 
-      it 'adds every runlist entry to the entries array' do
-        allow(Transformation).to receive(:new) do |table_name, config|
+      it 'returns the list of transformations' do
+        allow(Transformation).to receive(:new) do |table_name, config, helpers|
           table_name
         end
 

data/spec/dsl/transformation_spec.rb

@@ -10,15 +10,36 @@ module BeetleETL
       end
     end
 
+    describe 'columns' do
+      it 'returns a list of payload column name symbols' do
+        setup = Proc.new do
+          columns :payload_1, 'payload_2'
+        end
+        transformation = Transformation.new(:table, setup)
+
+        expect(transformation.column_names).to match_array([
+          :payload_1, :payload_2
+        ])
+      end
+
+      it 'defaults to an empty array if no columns are defined' do
+        transformation = Transformation.new(:table, Proc.new {})
+
+        expect(transformation.column_names).to match_array([])
+      end
+    end
+
     describe '#relations' do
       it 'returns the list of foreign tables and their foreign key column' do
         setup = Proc.new do
           references :foreign_table, on: :foreign_table_id
+          references :another_foreign_table, on: :another_foreign_table_id
         end
         transformation = Transformation.new(:table, setup)
 
         expect(transformation.relations).to eql({
-          foreign_table_id: :foreign_table
+          foreign_table_id: :foreign_table,
+          another_foreign_table_id: :another_foreign_table
         })
       end
     end
@@ -36,18 +57,33 @@ module BeetleETL
     end
 
     describe '#query' do
-      it 'returns the query interpolating methods in scope' do
+      it 'returns the query interpolating methods defined as helpers' do
+        helpers = Proc.new do
+          def foo; "foo_string"; end
+        end
 
         setup = Proc.new do
-          def foo; "foo_string"; end
           query "SELECT '#{foo}' FROM some_table"
         end
-        transformation = Transformation.new(:table, setup)
+
+        transformation = Transformation.new(:table, setup, helpers)
 
         expect(transformation.query).to eql(
           "SELECT 'foo_string' FROM some_table"
         )
       end
+
+      it 'concatenates multiple queries' do
+        setup = Proc.new do
+          query "SOME QUERY"
+          query "ANOTHER QUERY"
+        end
+        transformation = Transformation.new(:table, setup)
+
+        expect(transformation.query).to eql(
+          "SOME QUERY;ANOTHER QUERY"
+        )
+      end
     end
 
   end