beetle_etl 2.0.1 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cd21d7530444cef54966138ce38a5ec6ca0f08b
-  data.tar.gz: 277b54c1b957edeeb78fab7434852215f571d486
+  metadata.gz: 114b9229ba18051ff3bb5d606af695df0b58649c
+  data.tar.gz: 869afc2743aa1c5a4028bc4d48b1fa7416c4bb43
 SHA512:
-  metadata.gz: 659b6e6cde8208578833b7fd4a870f15fd706300389aef4c2c8c93786a17954bddca324398f1ca2e32bba12a96ed89f2e26b237df78efe99f33e1cbb0d9cd476
-  data.tar.gz: cb8d7842912f24a4e82d0198f4508757f32c5051183face61e5359bfbe36a5ce353127e085992a33069d82e6afd6bc1d44a68c79d832d38fc079eca819d8726a
+  metadata.gz: 1c70a1b62aa9689174177d4021eb1499ad6afdbf688338affc682d99c21633f81a4f61a64b5db3f487b7cdeddef9511cf430cef743f3eb73eedcc1d8b11562f1
+  data.tar.gz: d61a1b35342ec7a4cf4d78ffee9efaed6b4ce7f3c44954a0e77b1e03c81e10f4d7edc9f26acaf5635a128f1e008302a59ab81121273003ee556899d1d42bf904

data/lib/beetle_etl.rb

@@ -19,11 +19,10 @@ module BeetleETL
   require 'beetle_etl/steps/transform'
   require 'beetle_etl/steps/map_relations'
   require 'beetle_etl/steps/table_diff'
-  require 'beetle_etl/steps/assign_ids'
   require 'beetle_etl/steps/load'
   require 'beetle_etl/steps/drop_stage'
 
-  require 'beetle_etl/step_runner/dependency_resolver'
+  require 'beetle_etl/step_runner/sequential_step_runner'
   require 'beetle_etl/step_runner/async_step_runner'
 
   require 'beetle_etl/import'

data/lib/beetle_etl/import.rb

@@ -5,57 +5,61 @@ module BeetleETL
 
     def initialize(config)
       @config = config
-    end
-
-    def run
-      setup
-      import
-    ensure
-      cleanup
-    end
+      @report = {}
 
-    def setup
-      transformations.each do |t|
-        CreateStage.new(@config, t.table_name, t.relations, t.column_names).run
-      end
+      @transformations ||= TransformationLoader.new(@config).load
     end
 
-    def import
-      data_report = AsyncStepRunner.new(@config, data_steps).run
-      load_report = @config.database.transaction do
-        AsyncStepRunner.new(@config, load_steps).run
+    def run
+      begin
+        run_setup
+        run_transform
+        run_load
+      ensure
+        run_cleanup
       end
 
-      data_report.deep_merge load_report
+      @report
     end
 
-    def cleanup
-      transformations.each do |t|
-        DropStage.new(@config, t.table_name).run
-      end
-    end
+    def run_setup
+      steps = @transformations.map { |t|
+        CreateStage.new(@config, t.table_name, t.relations, t.column_names)
+      }
 
-    private
+      @report.deep_merge SequentialStepRunner.new(@config, steps).run
+    end
 
-    def data_steps
-      transformations.flat_map do |t|
+    def run_transform
+      steps = @transformations.flat_map { |t|
         [
           Transform.new(@config, t.table_name, t.dependencies, t.query),
           MapRelations.new(@config, t.table_name, t.relations),
-          TableDiff.new(@config, t.table_name),
-          AssignIds.new(@config, t.table_name),
+          TableDiff.new(@config, t.table_name)
         ]
-      end
+      }
+
+      @report.deep_merge AsyncStepRunner.new(@config, steps).run
     end
 
-    def load_steps
-      transformations.map do |t|
+    def run_load
+      steps = @transformations.map { |t|
         Load.new(@config, t.table_name, t.relations)
+      }
+
+      result = @config.database.transaction do
+        SequentialStepRunner.new(@config, steps).run
       end
+
+      @report.deep_merge result
     end
 
-    def transformations
-      @transformations ||= TransformationLoader.new(@config).load
+    def run_cleanup
+      steps = @transformations.map { |t|
+        DropStage.new(@config, t.table_name)
+      }
+
+      @report.deep_merge SequentialStepRunner.new(@config, steps).run
     end
 
   end

data/lib/beetle_etl/step_runner/abstract_step_runner.rb

@@ -0,0 +1,47 @@
+require 'active_support/core_ext/hash/slice'
+
+module BeetleETL
+  class AbstractStepRunner
+
+    def initialize(config, steps)
+      @config = config
+      @steps = steps
+    end
+
+    def run
+      raise NotImplementedError
+    end
+
+    private
+
+    def run_step(step)
+      @config.logger.info("started step #{step.name}")
+
+      started_at = Time.now
+      step.run
+      finished_at = Time.now
+
+      duration = Time.at(finished_at - started_at).utc.strftime("%H:%M:%S")
+      @config.logger.info("finished #{step.name} in #{duration}")
+
+      {
+        step_name: step.name,
+        table_name: step.table_name,
+        started_at: started_at,
+        finished_at: finished_at
+      }
+    rescue => e
+      @config.logger.fatal(e.message)
+      raise e
+    end
+
+    def add_result!(results, step_data)
+      table_name = step_data[:table_name]
+      step_name = step_data[:step_name]
+
+      results[table_name] ||= {}
+      results[table_name][step_name] = step_data.slice(:started_at, :finished_at)
+    end
+
+  end
+end

data/lib/beetle_etl/step_runner/async_step_runner.rb

@@ -1,15 +1,17 @@
+require_relative './abstract_step_runner'
+require_relative './dependency_resolver'
+
 module BeetleETL
-  class AsyncStepRunner
+  class AsyncStepRunner < AbstractStepRunner
 
     def initialize(config, steps)
-      @config = config
+      super(config, steps)
 
       @dependency_resolver = DependencyResolver.new(steps)
-      @steps = steps
 
       @queue = Queue.new
       @completed = Set.new
-      @running = Set.new
+      @started = Set.new
     end
 
     def run
@@ -18,17 +20,13 @@ module BeetleETL
       until all_steps_complete?
         runnables.each do |step|
           run_step_async(step)
-          mark_step_running(step.name)
+          @started.add(step.name)
         end
 
-        table_name, step_name, step_data = @queue.pop
-
-        unless results.has_key?(table_name)
-          results[table_name] = {}
-        end
+        step_data = @queue.pop
+        add_result!(results, step_data)
 
-        results[table_name][step_name] = step_data
-        mark_step_completed(step_name)
+        @completed.add(step_data[:step_name])
       end
 
       results
@@ -36,49 +34,19 @@ module BeetleETL
 
     private
 
-    attr_reader :running, :completed
-
     def run_step_async(step)
       Thread.new do
-        begin
-          @config.logger.info("started step #{step.name}")
-
-          started_at = Time.now
-          step.run
-          finished_at = Time.now
-
-          duration = Time.at(finished_at - started_at).utc.strftime("%H:%M:%S")
-          @config.logger.info("finished #{step.name} in #{duration}")
-
-          @queue.push [
-            step.table_name,
-            step.name,
-            { started_at: started_at, finished_at: finished_at }
-          ]
-
-        rescue => e
-          @config.logger.fatal(e.message)
-          raise e
-        end
+        @queue.push run_step(step)
       end.abort_on_exception = true
     end
 
-    def mark_step_running(step_name)
-      running.add(step_name)
-    end
-
-    def mark_step_completed(step_name)
-      runnables.delete(step_name)
-      completed.add(step_name)
-    end
-
     def runnables
-      resolvables = @dependency_resolver.resolvables(completed)
-      resolvables.reject { |r| running.include? r.name }
+      resolvables = @dependency_resolver.resolvables(@completed)
+      resolvables.reject { |r| @started.include? r.name }
     end
 
     def all_steps_complete?
-      @steps.map(&:name).to_set == completed.to_set
+      @steps.map(&:name).to_set == @completed.to_set
     end
 
   end

data/lib/beetle_etl/step_runner/sequential_step_runner.rb

@@ -0,0 +1,13 @@
+require_relative './abstract_step_runner'
+
+module BeetleETL
+  class SequentialStepRunner < AbstractStepRunner
+
+    def run
+      @steps.reduce({}) do |results, step|
+        add_result!(results, run_step(step))
+      end
+    end
+
+  end
+end

data/lib/beetle_etl/steps/map_relations.rb

@@ -8,7 +8,7 @@ module BeetleETL
 
     def dependencies
       result = Set.new([Transform.step_name(table_name)])
-      result.merge @relations.values.map { |d| AssignIds.step_name(d) }
+      result.merge @relations.values.map { |d| TableDiff.step_name(d) }
     end
 
     def run

data/lib/beetle_etl/steps/table_diff.rb

@@ -11,7 +11,7 @@ module BeetleETL
     end
 
     def run
-      %w(create update delete reinstate).each do |transition|
+      %w(create update delete reinstate keep).each do |transition|
         public_send(:"transition_#{transition}")
       end
     end
@@ -19,7 +19,9 @@ module BeetleETL
     def transition_create
       database.execute <<-SQL
         UPDATE "#{target_schema}"."#{stage_table_name}" stage
-        SET transition = 'CREATE'
+        SET
+          transition = 'CREATE',
+          id = NEXTVAL('#{target_schema}.#{table_name}_id_seq')
         WHERE NOT EXISTS (
           SELECT 1
           FROM "#{target_schema}"."#{table_name}" target
@@ -31,12 +33,13 @@ module BeetleETL
 
     def transition_update
       database.execute <<-SQL
-        UPDATE "#{target_schema}"."#{stage_table_name}" stage
-        SET transition = 'UPDATE'
-        WHERE EXISTS (
-          SELECT 1
-          FROM "#{target_schema}"."#{table_name}" target
-          WHERE target.external_id = stage.external_id
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
+        SET
+          transition = 'UPDATE',
+          id = target.id
+        FROM "#{target_schema}"."#{stage_table_name}" stage
+        JOIN "#{target_schema}"."#{table_name}" target ON (
+          target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
           AND target.deleted_at IS NULL
           AND
@@ -44,16 +47,17 @@ module BeetleETL
             IS DISTINCT FROM
             (#{stage_record_columns.join(', ')})
         )
+        WHERE stage_update.external_id = stage.external_id
       SQL
     end
 
     def transition_delete
       database.execute <<-SQL
         INSERT INTO "#{target_schema}"."#{stage_table_name}"
-          (external_id, transition)
+          (transition, id)
         SELECT
-          target.external_id,
-          'DELETE'
+          'DELETE',
+          target.id
         FROM "#{target_schema}"."#{table_name}" target
         LEFT OUTER JOIN "#{target_schema}"."#{stage_table_name}" stage
           ON (stage.external_id = target.external_id)
@@ -65,15 +69,37 @@ module BeetleETL
 
     def transition_reinstate
       database.execute <<-SQL
-        UPDATE "#{target_schema}"."#{stage_table_name}" stage
-        SET transition = 'REINSTATE'
-        WHERE EXISTS (
-          SELECT 1
-          FROM "#{target_schema}"."#{table_name}" target
-          WHERE target.external_id = stage.external_id
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
+        SET
+          transition = 'REINSTATE',
+          id = target.id
+        FROM "#{target_schema}"."#{stage_table_name}" stage
+        JOIN "#{target_schema}"."#{table_name}" target ON (
+          target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
           AND target.deleted_at IS NOT NULL
         )
+        WHERE stage_update.external_id = stage.external_id
+      SQL
+    end
+
+    def transition_keep
+      database.execute <<-SQL
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
+        SET
+          transition = 'KEEP',
+          id = target.id
+        FROM "#{target_schema}"."#{stage_table_name}" stage
+        JOIN "#{target_schema}"."#{table_name}" target ON (
+          target.external_id = stage.external_id
+          AND target.external_source = '#{external_source}'
+          AND target.deleted_at IS NULL
+          AND
+            (#{target_record_columns.join(', ')})
+            IS NOT DISTINCT FROM
+            (#{stage_record_columns.join(', ')})
+        )
+        WHERE stage_update.external_id = stage.external_id
       SQL
     end
 

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "2.0.1"
+  VERSION = "2.0.3"
 end

data/spec/steps/map_relations_spec.rb

@@ -52,11 +52,11 @@ module BeetleETL
     end
 
     describe '#depenencies' do
-      it 'depends on Transform of the same table and AssignIds of its dependees' do
+      it 'depends on Transform of the same table and TableDiff of its dependees' do
         expect(subject.dependencies).to eql(
           [
-            'dependee_a: AssignIds',
-            'dependee_b: AssignIds',
+            'dependee_a: TableDiff',
+            'dependee_b: TableDiff',
             'depender: Transform',
           ].to_set
         )

data/spec/steps/table_diff_spec.rb

@@ -18,6 +18,7 @@ module BeetleETL
 
     before do
       test_database.create_table(subject.stage_table_name.to_sym) do
+        Integer :id
         String :external_id, size: 255
         String :transition, size: 20
 
@@ -28,7 +29,7 @@ module BeetleETL
       end
 
       test_database.create_table(:example_table) do
-        Integer :id
+        primary_key :id
         String :external_id, size: 255
         String :external_source, size: 255
         DateTime :deleted_at
@@ -47,7 +48,7 @@ module BeetleETL
 
     describe '#run' do
       it 'runs all transitions' do
-        %w(create update delete reinstate).each do |transition|
+        %w(create update delete reinstate keep).each do |transition|
           expect(subject).to receive(:"transition_#{transition}")
         end
 
@@ -59,11 +60,13 @@ module BeetleETL
       it 'assigns CREATE to new records' do
 
         insert_into(:example_table).values(
-          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
         )
 
+        test_database.run "SELECT setval('public.example_table_id_seq', 99)"
+
         insert_into(subject.stage_table_name.to_sym).values(
           [ :external_id ] ,
           [ 'created'    ] ,
@@ -72,10 +75,10 @@ module BeetleETL
 
         subject.transition_create
 
-        insert_into(subject.stage_table_name.to_sym).values(
-          [ :external_id , :transition ] ,
-          [ 'created'    , 'CREATE'    ] ,
-          [ 'existing'   , nil         ] ,
+        expect(subject.stage_table_name.to_sym).to have_values(
+          [ :external_id , :id , :transition ] ,
+          [ 'created'    , 100 , 'CREATE'    ] ,
+          [ 'existing'   , nil , nil         ] ,
         )
       end
     end
@@ -85,10 +88,10 @@ module BeetleETL
         except externald_*_id columns and columns not contained in the stage table' do
 
         insert_into(:example_table).values(
-          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 'existing_1' , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 'existing_2' , external_source  , 'existing content' , 'ignored content'  , 2       , nil         ] ,
-          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 3       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 1   , 'existing_1' , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 2   , 'existing_2' , external_source  , 'existing content' , 'ignored content'  , 2       , nil         ] ,
+          [ 3   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 3       , 1.day.ago   ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(
@@ -101,10 +104,10 @@ module BeetleETL
         subject.transition_update
 
         expect(subject.stage_table_name.to_sym).to have_values(
-          [ :external_id , :transition ] ,
-          [ 'existing_1' , 'UPDATE'    ] ,
-          [ 'existing_2' , 'UPDATE'    ] ,
-          [ 'deleted'    , nil         ] ,
+          [ :external_id , :id , :transition ] ,
+          [ 'existing_1' , 1   , 'UPDATE'    ] ,
+          [ 'existing_2' , 2   , 'UPDATE'    ] ,
+          [ 'deleted'    , nil , nil         ] ,
         )
       end
     end
@@ -112,16 +115,16 @@ module BeetleETL
     describe 'transition_delete' do
       it 'creates records with DELETE that no loger exist in the stage table for the given run' do
         insert_into(:example_table).values(
-          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
         )
 
         subject.transition_delete
 
         expect(subject.stage_table_name.to_sym).to have_values(
-          [ :external_id , :transition ] ,
-          [ 'existing'   , 'DELETE'    ] ,
+          [ :id , :transition ] ,
+          [ 1   , 'DELETE'    ] ,
         )
       end
     end
@@ -129,9 +132,9 @@ module BeetleETL
     describe 'transition_reinstate' do
       it 'assigns REINSTATE to previously deleted records' do
         insert_into(:example_table).values(
-          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(
@@ -143,9 +146,34 @@ module BeetleETL
         subject.transition_reinstate
 
         expect(subject.stage_table_name.to_sym).to have_values(
-          [ :external_id , :transition  ] ,
-          [ 'existing'   , nil          ] ,
-          [ 'deleted'    , 'REINSTATE'  ] ,
+          [ :external_id , :id , :transition  ] ,
+          [ 'existing'   , nil , nil          ] ,
+          [ 'deleted'    , 2   , 'REINSTATE'  ] ,
+        )
+      end
+    end
+
+    describe '#transition_keep' do
+      it 'assigns KEEP to unchanged records' do
+
+        insert_into(:example_table).values(
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+        )
+
+        insert_into(subject.stage_table_name.to_sym).values(
+          [ :external_id , :payload           , :foo_id ] ,
+          [ 'created'    , nil                , nil     ] ,
+          [ 'existing'   , 'existing content' , 1       ] ,
+        )
+
+        subject.transition_keep
+
+        expect(subject.stage_table_name.to_sym).to have_values(
+          [ :external_id , :id , :transition ] ,
+          [ 'created'    , nil , nil         ] ,
+          [ 'existing'   , 1   , 'KEEP'      ] ,
         )
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.3
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-24 00:00:00.000000000 Z
+date: 2017-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -132,9 +132,10 @@ files:
 - lib/beetle_etl/import.rb
 - lib/beetle_etl/naming.rb
 - lib/beetle_etl/reporter.rb
+- lib/beetle_etl/step_runner/abstract_step_runner.rb
 - lib/beetle_etl/step_runner/async_step_runner.rb
 - lib/beetle_etl/step_runner/dependency_resolver.rb
-- lib/beetle_etl/steps/assign_ids.rb
+- lib/beetle_etl/step_runner/sequential_step_runner.rb
 - lib/beetle_etl/steps/create_stage.rb
 - lib/beetle_etl/steps/drop_stage.rb
 - lib/beetle_etl/steps/load.rb
@@ -158,7 +159,6 @@ files:
 - spec/feature/feature_spec.rb
 - spec/reporter_spec.rb
 - spec/spec_helper.rb
-- spec/steps/assign_ids_spec.rb
 - spec/steps/create_stage_spec.rb
 - spec/steps/load_spec.rb
 - spec/steps/map_relations_spec.rb
@@ -208,7 +208,6 @@ test_files:
 - spec/feature/feature_spec.rb
 - spec/reporter_spec.rb
 - spec/spec_helper.rb
-- spec/steps/assign_ids_spec.rb
 - spec/steps/create_stage_spec.rb
 - spec/steps/load_spec.rb
 - spec/steps/map_relations_spec.rb

data/lib/beetle_etl/steps/assign_ids.rb

@@ -1,23 +0,0 @@
-module BeetleETL
-  class AssignIds < Step
-
-    def dependencies
-      [TableDiff.step_name(table_name)].to_set
-    end
-
-    def run
-      database.execute <<-SQL
-        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
-        SET id = COALESCE(target.id, NEXTVAL('#{target_schema}.#{table_name}_id_seq'))
-        FROM "#{target_schema}"."#{stage_table_name}" stage
-        LEFT OUTER JOIN "#{target_schema}"."#{table_name}" target
-          on (
-            stage.external_id = target.external_id
-            AND target.external_source = '#{external_source}'
-          )
-        WHERE stage_update.external_id = stage.external_id
-      SQL
-    end
-
-  end
-end

data/spec/steps/assign_ids_spec.rb

@@ -1,74 +0,0 @@
-require 'spec_helper'
-
-module BeetleETL
-  describe AssignIds do
-
-    let(:external_source) { 'my_source' }
-    let(:another_source) { 'another_source' }
-
-    let(:config) do
-      OpenStruct.new({
-        stage_schema: 'stage',
-        target_schema: 'public',
-        external_source: external_source,
-        database: test_database,
-      })
-    end
-
-    subject { AssignIds.new(config, :example_table) }
-
-    describe '#dependencies' do
-      it 'depends on TableDiff of the same table' do
-        expect(subject.dependencies).to eql(['example_table: TableDiff'].to_set)
-      end
-    end
-
-    describe '#run' do
-      before do
-        test_database.create_table(subject.stage_table_name.to_sym) do
-          Integer :id
-          String :external_id, size: 255
-          String :transition, size: 255
-        end
-
-        test_database.create_table(:example_table) do
-          primary_key :id
-          String :external_id, size: 255
-          String :external_source, size: 255
-        end
-      end
-
-      it 'assigns ids for' do
-        # - generated ones for new records
-        # - mapped ones by external_id for existing records
-
-        insert_into(:example_table).values(
-          [ :external_id , :external_source ] ,
-          [ 'a'          , external_source  ] ,
-          [ 'b'          , external_source  ] ,
-          [ 'c'          , external_source  ] ,
-          [ 'd'          , another_source   ] ,
-        )
-
-        insert_into(subject.stage_table_name.to_sym).values(
-          [ :external_id ] ,
-          [ 'new value'  ] ,
-          [ 'a'          ] ,
-          [ 'b'          ] ,
-          [ 'c'          ] ,
-        )
-
-        subject.run
-
-        expect(subject.stage_table_name.to_sym).to have_values(
-          [ :id , :external_id ] ,
-          [ 1   , 'a'          ] ,
-          [ 2   , 'b'          ] ,
-          [ 3   , 'c'          ] ,
-          [ 5   , 'new value'  ] ,
-        )
-      end
-    end
-
-  end
-end