beetle_etl 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13376260ac59a025f892af7ab4c47346ef7bd399
-  data.tar.gz: e5e0bcad031552006cc3e650725b245e121acc25
+  metadata.gz: 1edda8a1e337f1aa6f17031c95d6eb8b1672c2a1
+  data.tar.gz: bd24f5983ecafd0e08b0573e50e690fdd90ca989
 SHA512:
-  metadata.gz: e06aa20a649cd4caf061dc9f847df5446c5d09b1f60b86ecc9c450cb1de3670deff56564e9636e86865748e6b2fcf1697832d6464fd6f5d07d16ad6af987d225
-  data.tar.gz: 68e3004d46baf5d6f1d9eaf9e88f79ff6617e00ce013bb0b7faad89e9b175bd7827f02f6cd82a45f64acf9a1e8e08bcb3f40bd8fe3c5811c5d52256a3df5fa5e
+  metadata.gz: 7318b99966f73b114196f5d0f371220557caf54870c74426e1468178ae63afc4f318a5292922d86977e9d8b028fd12e15d3cf6409c5695e7c7775878cc3e4894
+  data.tar.gz: 5db98f072ccf66e5a373f443d2ce4e9a5134592abfd196b75173413e3c84cc08babf106af5da6c97806481cd9d54217d7cf6b4a80943491bfd23312a1a75ce79

data/.travis.yml

@@ -1,14 +1,13 @@
 language: ruby
 rvm:
-  - 2.0.0
-  - 2.2.0
-  - 2.3.0
-  - 2.4.0
-addons:
-  postgresql: "9.3"
-before_install:
-  - gem update --system
-  - gem update bundler
+  - 2.0
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.6
+
+services:
+  - postgresql
 
 before_script:
   - psql -c 'create database travis_ci_test;' -U postgres

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Luciano Maiwald
+Copyright (c) 2014 - 2017 Luciano Maiwald
 
 MIT License
 

data/beetle_etl.gemspec

@@ -18,10 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'activesupport', '>= 4.0.0'
-
   spec.add_development_dependency 'sequel', '>= 4.0.0'
-  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'bundler', '>= 1.11'
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'timecop', '>= 0.7.0'
   spec.add_development_dependency 'pg', '>= 0.18.0'

data/lib/beetle_etl/import.rb

@@ -1,5 +1,3 @@
-require 'active_support/core_ext/hash/deep_merge'
-
 module BeetleETL
   class Import
 
@@ -27,7 +25,7 @@ module BeetleETL
         CreateStage.new(@config, t.table_name, t.relations, t.column_names)
       }
 
-      @report.deep_merge StepRunner.new(@config, steps).run
+      merge_report! StepRunner.new(@config, steps).run
     end
 
     def run_transform
@@ -39,7 +37,7 @@ module BeetleETL
         ]
       }
 
-      @report.deep_merge AsyncStepRunner.new(@config, steps).run
+      merge_report! AsyncStepRunner.new(@config, steps).run
     end
 
     def run_load
@@ -51,7 +49,7 @@ module BeetleETL
         StepRunner.new(@config, steps).run
       end
 
-      @report.deep_merge result
+      merge_report! result
     end
 
     def run_cleanup
@@ -59,7 +57,11 @@ module BeetleETL
         DropStage.new(@config, t.table_name)
       }
 
-      @report.deep_merge StepRunner.new(@config, steps).run
+      merge_report! StepRunner.new(@config, steps).run
+    end
+
+    def merge_report!(new_report)
+      @report.merge!(new_report) { |key, oldval, newval| oldval.merge(newval) }
     end
 
   end

data/lib/beetle_etl/step_runner/step_runner.rb

@@ -55,7 +55,10 @@ module BeetleETL
       step_name = step_data[:step_name]
 
       results[table_name] ||= {}
-      results[table_name][step_name] = step_data.slice(:started_at, :finished_at)
+      results[table_name][step_name] = {
+        started_at: step_data[:started_at],
+        finished_at: step_data[:finished_at]
+      }
     end
 
     def all_steps_complete?

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "2.0.4"
+  VERSION = "2.0.5"
 end

data/spec/feature/feature_spec.rb

@@ -4,18 +4,15 @@ require 'timecop'
 require_relative 'example_schema'
 require 'yaml'
 
-require 'active_support/core_ext/date/calculations'
-require 'active_support/core_ext/numeric/time'
-
 Thread::abort_on_exception = true
 
 describe BeetleETL do
 
   include ExampleSchema
 
-  let!(:time1) { Time.new(2014 ,  7 , 17 , 16 , 12).beginning_of_day }
-  let!(:time2) { Time.new(2015 ,  2 ,  8 , 22 , 18).beginning_of_day }
-  let!(:time3) { Time.new(2015 , 11 ,  3 , 12 , 17).beginning_of_day }
+  let!(:time1) { Time.new(2014 ,  7 , 17 , 16 , 12) }
+  let!(:time2) { Time.new(2015 ,  2 ,  8 , 22 , 18) }
+  let!(:time3) { Time.new(2015 , 11 ,  3 , 12 , 17) }
 
   before :each do
     create_tables

data/spec/steps/load_spec.rb

@@ -1,15 +1,12 @@
 require 'spec_helper'
 
-require 'active_support/core_ext/date/calculations'
-require 'active_support/core_ext/numeric/time'
-
 module BeetleETL
   describe Load do
 
     let(:external_source) { 'my_source' }
 
-    let(:now) { Time.now.beginning_of_day }
-    let(:yesterday) { 1.day.ago.beginning_of_day }
+    let(:today) { Time.new(2020, 3, 24) }
+    let(:yesterday) { Time.new(2020, 3, 23) }
 
     let(:config) do
       Configuration.new.tap do |c|
@@ -21,7 +18,7 @@ module BeetleETL
     subject { Load.new(config, :example_table, []) }
 
     before do
-      allow(subject).to receive(:now) { now }
+      allow(subject).to receive(:now) { today }
 
       test_database.create_schema(:stage)
       test_database.create_table(subject.stage_table_name.to_sym) do
@@ -86,7 +83,7 @@ module BeetleETL
 
         expect(:example_table).to have_values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
-          [ 3   , 'external_id' , external_source  , 22      , now         , now         , nil         , 'content' ] ,
+          [ 3   , 'external_id' , external_source  , 22      , today       , today       , nil         , 'content' ] ,
         )
       end
     end
@@ -107,7 +104,7 @@ module BeetleETL
 
         expect(:example_table).to have_values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
-          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
+          [ 1   , 'external_id' , external_source  , 33      , yesterday   , today       , nil         , 'updated content' ] ,
         )
       end
 
@@ -126,7 +123,7 @@ module BeetleETL
 
         expect(:example_table).to have_values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
-          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
+          [ 1   , 'external_id' , external_source  , 33      , yesterday   , today       , nil         , 'updated content' ] ,
         )
       end
     end
@@ -147,7 +144,7 @@ module BeetleETL
 
         expect(:example_table).to have_values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
-          [ 1   , 'external_id' , external_source  , 22      , yesterday   , now         , now         , 'content' ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , today       , today       , 'content' ] ,
         )
       end
     end

data/spec/steps/table_diff_spec.rb

@@ -1,8 +1,5 @@
 require 'spec_helper'
 
-require 'active_support/core_ext/date/calculations'
-require 'active_support/core_ext/numeric/time'
-
 module BeetleETL
   describe TableDiff do
 
@@ -14,6 +11,8 @@ module BeetleETL
       end
     end
 
+    let(:time_in_past) { Time.new(2020, 3, 23) }
+
     subject { TableDiff.new(config, :example_table) }
 
     before do
@@ -60,9 +59,9 @@ module BeetleETL
       it 'assigns CREATE to new records' do
 
         insert_into(:example_table).values(
-          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at  ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil          ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , time_in_past ] ,
         )
 
         test_database.run "SELECT setval('public.example_table_id_seq', 99)"
@@ -88,10 +87,10 @@ module BeetleETL
         except externald_*_id columns and columns not contained in the stage table' do
 
         insert_into(:example_table).values(
-          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 1   , 'existing_1' , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 2   , 'existing_2' , external_source  , 'existing content' , 'ignored content'  , 2       , nil         ] ,
-          [ 3   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 3       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at  ] ,
+          [ 1   , 'existing_1' , external_source  , 'existing content' , 'ignored content'  , 1       , nil          ] ,
+          [ 2   , 'existing_2' , external_source  , 'existing content' , 'ignored content'  , 2       , nil          ] ,
+          [ 3   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 3       , time_in_past ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(
@@ -115,9 +114,9 @@ module BeetleETL
     describe 'transition_delete' do
       it 'creates records with DELETE that no loger exist in the stage table for the given run' do
         insert_into(:example_table).values(
-          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at  ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil          ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , time_in_past ] ,
         )
 
         subject.transition_delete
@@ -132,9 +131,9 @@ module BeetleETL
     describe 'transition_reinstate' do
       it 'assigns REINSTATE to previously deleted records' do
         insert_into(:example_table).values(
-          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
-          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at  ] ,
+          [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil          ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , time_in_past ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(
@@ -159,7 +158,7 @@ module BeetleETL
         insert_into(:example_table).values(
           [ :id , :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
           [ 1   , 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
-          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+          [ 2   , 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , time_in_past   ] ,
         )
 
         insert_into(subject.stage_table_name.to_sym).values(

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-14 00:00:00.000000000 Z
+date: 2020-03-24 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -42,14 +28,14 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.11'
 - !ruby/object:Gem::Dependency