beetle_etl 0.0.13 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7fb51d510b64c866a5b928cda8cfac73f8970805
-  data.tar.gz: 114785c8b95adc867d5ec3cae6839acf338df7b8
+  metadata.gz: 07d1e92398037e916f19762ae9a373de8cd3edfa
+  data.tar.gz: c064dfbfd3dfd4103864c51d8742f537963abf8c
 SHA512:
-  metadata.gz: 983c54e043e1671a20d452438d9e15058806fba4a4567a935b7a1c275a7221b0bee2d685d3f4bccc2ac1b3f5aae9430280101f6d9e198644e6b439af1c9ec33e
-  data.tar.gz: cc6c0bf17ac7c8969e8daf69acaed3b91a918d971e2bbb26a605365972889245c2bb956916ca129953840ec12823e8349982d7a3ebbd1cec9dfb6b4e70e0e2a0
+  metadata.gz: b984f80cdba06d018d49fa9414cf5443b8098b0edf34f1eb149df32f5c718f1af08676ca05743e064bf7e19475ad32a31459feedb68e0a45b091378c8c869665
+  data.tar.gz: 3b80f7d9ca88ed49130a6ef6719e9d65fb28d0dca101a2ee23e191bcb413dcf93ae213fefe194e736169cca2b2ab3e0a67444f975a741a7bf323944c4328fb1d

data/lib/beetle_etl/steps/assign_ids.rb

@@ -6,25 +6,16 @@ module BeetleETL
     end
 
     def run
-      assign_new_ids
-      map_existing_ids
-    end
-
-    def assign_new_ids
-      database.execute <<-SQL
-        UPDATE #{stage_table_name_sql}
-        SET id = nextval('#{table_name}_id_seq')
-        WHERE transition = 'CREATE'
-      SQL
-    end
-
-    def map_existing_ids
       database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage
-        SET id = public.id
-        FROM #{public_table_name_sql} public
-        WHERE stage.transition IN ('KEEP', 'UPDATE', 'DELETE', 'UNDELETE')
-        AND stage.external_id = public.external_id
+        UPDATE #{stage_table_name_sql} stage_update
+        SET id = COALESCE(public.id, nextval('#{table_name}_id_seq'))
+        FROM #{stage_table_name_sql} stage
+        LEFT OUTER JOIN #{public_table_name_sql} public
+          on (
+            stage.external_id = public.external_id
+            AND public.external_source = '#{external_source}'
+          )
+        WHERE stage_update.external_id = stage.external_id
       SQL
     end
 

data/lib/beetle_etl/steps/load.rb

@@ -12,7 +12,7 @@ module BeetleETL
     end
 
     def run
-      %w(create update delete undelete).each do |transition|
+      %w(create update delete reinstate).each do |transition|
         public_send(:"load_#{transition}")
       end
     end
@@ -63,7 +63,7 @@ module BeetleETL
       SQL
     end
 
-    def load_undelete
+    def load_reinstate
       database.execute <<-SQL
         UPDATE #{public_table_name_sql} public
         SET
@@ -72,7 +72,7 @@ module BeetleETL
           deleted_at = NULL
         FROM #{stage_table_name_sql} stage
         WHERE stage.id = public.id
-          AND stage.transition = 'UNDELETE'
+          AND stage.transition = 'REINSTATE'
       SQL
     end
 

data/lib/beetle_etl/steps/table_diff.rb

@@ -11,7 +11,7 @@ module BeetleETL
     end
 
     def run
-      %w(create keep update delete undelete).each do |transition|
+      %w(create update delete reinstate).each do |transition|
         public_send(:"transition_#{transition}")
       end
     end
@@ -29,24 +29,6 @@ module BeetleETL
       SQL
     end
 
-    def transition_keep
-      database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage
-        SET transition = 'KEEP'
-        WHERE EXISTS (
-          SELECT 1
-          FROM #{public_table_name} public
-          WHERE public.external_id = stage.external_id
-          AND public.external_source = '#{external_source}'
-          AND public.deleted_at IS NULL
-          AND
-            (#{public_record_columns.join(', ')})
-            IS NOT DISTINCT FROM
-            (#{stage_record_columns.join(', ')})
-        )
-      SQL
-    end
-
     def transition_update
       database.execute <<-SQL
         UPDATE #{stage_table_name_sql} stage
@@ -73,10 +55,7 @@ module BeetleETL
           public.external_id,
           'DELETE'
         FROM #{public_table_name_sql} public
-        LEFT OUTER JOIN (
-          SELECT *
-          FROM #{stage_table_name_sql}
-          ) stage
+        LEFT OUTER JOIN #{stage_table_name_sql} stage
           ON (stage.external_id = public.external_id)
         WHERE stage.external_id IS NULL
         AND public.external_source = '#{external_source}'
@@ -84,10 +63,10 @@ module BeetleETL
       SQL
     end
 
-    def transition_undelete
+    def transition_reinstate
       database.execute <<-SQL
         UPDATE #{stage_table_name_sql} stage
-        SET transition = 'UNDELETE'
+        SET transition = 'REINSTATE'
         WHERE EXISTS (
           SELECT 1
           FROM #{public_table_name_sql} public

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "0.0.13"
+  VERSION = "0.0.16"
 end

data/spec/feature/feature_spec.rb

@@ -36,7 +36,7 @@ describe BeetleETL do
   end
 
   it 'performs all possible transitions', :feature do
-    # create, keep, update, delete, undelete
+    # create, keep, update, delete, reinstate
 
     import1
     import2
@@ -108,7 +108,7 @@ describe BeetleETL do
   end
 
   def import3
-    # undelete with update
+    # reinstate with update
     insert_into(:source__Organisation).values(
       [ :pkOrgId , :Name    , :Adresse            , :Abteilung ] ,
       [ 1        , 'Apple'  , 'Apple Street'      , 'iPhone'   ] ,

data/spec/steps/assign_ids_spec.rb

@@ -4,6 +4,8 @@ module BeetleETL
   describe AssignIds do
 
     let(:external_source) { 'my_source' }
+    let(:another_source) { 'another_source' }
+
     subject { AssignIds.new(:example_table) }
 
     before do
@@ -12,19 +14,6 @@ module BeetleETL
         config.external_source = external_source
         config.database = test_database
       end
-
-      test_database.create_table(subject.stage_table_name.to_sym) do
-        Integer :id
-        String :external_id, size: 255
-        String :transition, size: 255
-      end
-
-      test_database.create_table(:example_table) do
-        primary_key :id
-        String :external_id, size: 255
-        String :external_source, size: 255
-      end
-
     end
 
     describe '#dependencies' do
@@ -34,66 +23,48 @@ module BeetleETL
     end
 
     describe '#run' do
-      it 'runs all transitions' do
-        %i(assign_new_ids map_existing_ids).each do |method|
-          expect(subject).to receive(method)
+      before do
+        test_database.create_table(subject.stage_table_name.to_sym) do
+          Integer :id
+          String :external_id, size: 255
+          String :transition, size: 255
         end
 
-        subject.run
+        test_database.create_table(:example_table) do
+          primary_key :id
+          String :external_id, size: 255
+          String :external_source, size: 255
+        end
       end
-    end
 
-    describe '#assign_new_ids' do
-      it 'generates new ids for newly created records' do
-        insert_into(:example_table).values(
-          [ :external_id , :external_source ] ,
-          [ 'keep_id'    , external_source  ] ,
-        )
+      it 'assigns ids for' do
+        # - generated ones for new records
+        # - mapped ones by external_id for existing records
 
-        insert_into(subject.stage_table_name.to_sym).values(
-          [ :external_id , :transition ] ,
-          [ 'create_id'  , 'CREATE'    ] ,
-          [ 'keep_id'    , 'KEEP'      ] ,
-        )
-
-        subject.assign_new_ids
-
-        expect(subject.stage_table_name.to_sym).to have_values(
-          [ :id , :external_id , :transition ] ,
-          [ 2   , 'create_id'  , 'CREATE'    ] ,
-          [ nil , 'keep_id'    , 'KEEP'      ] ,
-        )
-      end
-    end
-
-    describe '#map_existing_ids' do
-      it 'assigns ids for existing records by their external id' do
         insert_into(:example_table).values(
-          [ :external_id  , :external_source ] ,
-          [ 'keep_id'     , external_source  ] ,
-          [ 'update_id'   , external_source  ] ,
-          [ 'delete_id'   , external_source  ] ,
-          [ 'undelete_id' , external_source  ] ,
+          [ :external_id , :external_source ] ,
+          [ 'a'          , external_source  ] ,
+          [ 'b'          , external_source  ] ,
+          [ 'c'          , external_source  ] ,
+          [ 'd'          , another_source   ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(
-          [ :external_id  , :transition ] ,
-          [ 'create_id'   , 'CREATE'    ] ,
-          [ 'keep_id'     , 'KEEP'      ] ,
-          [ 'update_id'   , 'UPDATE'    ] ,
-          [ 'delete_id'   , 'DELETE'    ] ,
-          [ 'undelete_id' , 'UNDELETE'  ] ,
+          [ :external_id ] ,
+          [ 'new value'  ] ,
+          [ 'a'          ] ,
+          [ 'b'          ] ,
+          [ 'c'          ] ,
         )
 
-        subject.map_existing_ids
+        subject.run
 
         expect(subject.stage_table_name.to_sym).to have_values(
-          [ :id , :external_id  , :transition ] ,
-          [ nil , 'create_id'   , 'CREATE'    ] ,
-          [ 1   , 'keep_id'     , 'KEEP'      ] ,
-          [ 2   , 'update_id'   , 'UPDATE'    ] ,
-          [ 3   , 'delete_id'   , 'DELETE'    ] ,
-          [ 4   , 'undelete_id' , 'UNDELETE'  ] ,
+          [ :id , :external_id ] ,
+          [ 1   , 'a'          ] ,
+          [ 2   , 'b'          ] ,
+          [ 3   , 'c'          ] ,
+          [ 5   , 'new value'  ] ,
         )
       end
     end

data/spec/steps/load_spec.rb

@@ -65,7 +65,7 @@ module BeetleETL
 
     describe '#run' do
       it 'runs all load steps' do
-        %w(create update delete undelete).each do |transition|
+        %w(create update delete reinstate).each do |transition|
           expect(subject).to receive(:"load_#{transition}")
         end
 
@@ -131,7 +131,7 @@ module BeetleETL
       end
     end
 
-    describe '#load_undelete' do
+    describe '#load_reinstate' do
       it 'restores deleted records' do
         insert_into(:example_table).values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
@@ -140,10 +140,10 @@ module BeetleETL
 
         insert_into(subject.stage_table_name.to_sym).values(
           [ :id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
-          [ 1   , 'external_id' , 'UNDELETE'  , 'foo_id'         , 33      , 'updated content' ] ,
+          [ 1   , 'external_id' , 'REINSTATE' , 'foo_id'         , 33      , 'updated content' ] ,
         )
 
-        subject.load_undelete
+        subject.load_reinstate
 
         expect(:example_table).to have_values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,

data/spec/steps/table_diff_spec.rb

@@ -46,7 +46,7 @@ module BeetleETL
 
     describe '#run' do
       it 'runs all transitions' do
-        %w(create keep update delete undelete).each do |transition|
+        %w(create update delete reinstate).each do |transition|
           expect(subject).to receive(:"transition_#{transition}")
         end
 
@@ -79,32 +79,6 @@ module BeetleETL
       end
     end
 
-    describe 'transition_keep' do
-      it 'assigns KEEP if the record already exists and is not deleted, comparing all columns
-        except externald_*_id columns and columns not contained in the stage table' do
-
-        insert_into(:example_table).values(
-          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
-        )
-
-        insert_into(subject.stage_table_name.to_sym).values(
-          [ :external_id , :payload           , :foo_id , :external_foo_id ] ,
-          [ 'existing'   , 'existing content' , 1       , 'ignored column' ] ,
-          [ 'deleted'    , 'deleted content'  , 2       , 'ignored column' ] ,
-        )
-
-        subject.transition_keep
-
-        expect(subject.stage_table_name.to_sym).to have_values(
-          [ :external_id , :transition ] ,
-          [ 'existing'   , 'KEEP'      ] ,
-          [ 'deleted'    , nil         ] ,
-        )
-      end
-    end
-
     describe '#transition_update' do
       it 'assigns UPDATE to non-deleted records with changed values comparing all columns
         except externald_*_id columns and columns not contained in the stage table' do
@@ -151,8 +125,8 @@ module BeetleETL
       end
     end
 
-    describe 'transition_undelete' do
-      it 'assigns UNDELETE to previously deleted records' do
+    describe 'transition_reinstate' do
+      it 'assigns REINSTATE to previously deleted records' do
         insert_into(:example_table).values(
           [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
           [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
@@ -165,12 +139,12 @@ module BeetleETL
           [ 'deleted'    , 'updated content' , 2       , 'ignored_column' ] ,
         )
 
-        subject.transition_undelete
+        subject.transition_reinstate
 
         expect(subject.stage_table_name.to_sym).to have_values(
-          [ :external_id , :transition ] ,
-          [ 'existing'   , nil         ] ,
-          [ 'deleted'    , 'UNDELETE'  ] ,
+          [ :external_id , :transition  ] ,
+          [ 'existing'   , nil          ] ,
+          [ 'deleted'    , 'REINSTATE'  ] ,
         )
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.16
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-12 00:00:00.000000000 Z
+date: 2015-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel