beetle_etl 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 114b9229ba18051ff3bb5d606af695df0b58649c
-  data.tar.gz: 869afc2743aa1c5a4028bc4d48b1fa7416c4bb43
+  metadata.gz: 13376260ac59a025f892af7ab4c47346ef7bd399
+  data.tar.gz: e5e0bcad031552006cc3e650725b245e121acc25
 SHA512:
-  metadata.gz: 1c70a1b62aa9689174177d4021eb1499ad6afdbf688338affc682d99c21633f81a4f61a64b5db3f487b7cdeddef9511cf430cef743f3eb73eedcc1d8b11562f1
-  data.tar.gz: d61a1b35342ec7a4cf4d78ffee9efaed6b4ce7f3c44954a0e77b1e03c81e10f4d7edc9f26acaf5635a128f1e008302a59ab81121273003ee556899d1d42bf904
+  metadata.gz: e06aa20a649cd4caf061dc9f847df5446c5d09b1f60b86ecc9c450cb1de3670deff56564e9636e86865748e6b2fcf1697832d6464fd6f5d07d16ad6af987d225
+  data.tar.gz: 68e3004d46baf5d6f1d9eaf9e88f79ff6617e00ce013bb0b7faad89e9b175bd7827f02f6cd82a45f64acf9a1e8e08bcb3f40bd8fe3c5811c5d52256a3df5fa5e

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 rvm:
   - 2.0.0
-  - 2.1.0
   - 2.2.0
   - 2.3.0
+  - 2.4.0
 addons:
   postgresql: "9.3"
 before_install:

data/lib/beetle_etl.rb

@@ -22,7 +22,7 @@ module BeetleETL
   require 'beetle_etl/steps/load'
   require 'beetle_etl/steps/drop_stage'
 
-  require 'beetle_etl/step_runner/sequential_step_runner'
+  require 'beetle_etl/step_runner/step_runner'
   require 'beetle_etl/step_runner/async_step_runner'
 
   require 'beetle_etl/import'

data/lib/beetle_etl/import.rb

@@ -27,7 +27,7 @@ module BeetleETL
         CreateStage.new(@config, t.table_name, t.relations, t.column_names)
       }
 
-      @report.deep_merge SequentialStepRunner.new(@config, steps).run
+      @report.deep_merge StepRunner.new(@config, steps).run
     end
 
     def run_transform
@@ -48,7 +48,7 @@ module BeetleETL
       }
 
       result = @config.database.transaction do
-        SequentialStepRunner.new(@config, steps).run
+        StepRunner.new(@config, steps).run
       end
 
       @report.deep_merge result
@@ -59,7 +59,7 @@ module BeetleETL
         DropStage.new(@config, t.table_name)
       }
 
-      @report.deep_merge SequentialStepRunner.new(@config, steps).run
+      @report.deep_merge StepRunner.new(@config, steps).run
     end
 
   end

data/lib/beetle_etl/step_runner/async_step_runner.rb

@@ -1,16 +1,10 @@
-require_relative './abstract_step_runner'
-require_relative './dependency_resolver'
-
 module BeetleETL
-  class AsyncStepRunner < AbstractStepRunner
+  class AsyncStepRunner < StepRunner
 
     def initialize(config, steps)
       super(config, steps)
 
-      @dependency_resolver = DependencyResolver.new(steps)
-
       @queue = Queue.new
-      @completed = Set.new
       @started = Set.new
     end
 
@@ -45,9 +39,5 @@ module BeetleETL
       resolvables.reject { |r| @started.include? r.name }
     end
 
-    def all_steps_complete?
-      @steps.map(&:name).to_set == @completed.to_set
-    end
-
   end
 end

data/lib/beetle_etl/step_runner/{abstract_step_runner.rb → step_runner.rb}

@@ -1,15 +1,26 @@
-require 'active_support/core_ext/hash/slice'
+require_relative './dependency_resolver'
 
 module BeetleETL
-  class AbstractStepRunner
+  class StepRunner
 
     def initialize(config, steps)
       @config = config
       @steps = steps
+      @dependency_resolver = DependencyResolver.new(steps)
+      @completed = Set.new
     end
 
     def run
-      raise NotImplementedError
+      results = {}
+
+      until all_steps_complete?
+        runnables.each do |step|
+          add_result!(results, run_step(step))
+          @completed.add(step.name)
+        end
+      end
+
+      results
     end
 
     private
@@ -35,6 +46,10 @@ module BeetleETL
       raise e
     end
 
+    def runnables
+      @dependency_resolver.resolvables(@completed)
+    end
+
     def add_result!(results, step_data)
       table_name = step_data[:table_name]
       step_name = step_data[:step_name]
@@ -43,5 +58,9 @@ module BeetleETL
       results[table_name][step_name] = step_data.slice(:started_at, :finished_at)
     end
 
+    def all_steps_complete?
+      @steps.map(&:name).to_set == @completed.to_set
+    end
+
   end
 end

data/lib/beetle_etl/steps/table_diff.rb

@@ -18,16 +18,17 @@ module BeetleETL
 
     def transition_create
       database.execute <<-SQL
-        UPDATE "#{target_schema}"."#{stage_table_name}" stage
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
         SET
           transition = 'CREATE',
           id = NEXTVAL('#{target_schema}.#{table_name}_id_seq')
-        WHERE NOT EXISTS (
-          SELECT 1
-          FROM "#{target_schema}"."#{table_name}" target
-          WHERE target.external_id = stage.external_id
+        FROM "#{target_schema}"."#{stage_table_name}" stage
+        LEFT JOIN "#{target_schema}"."#{table_name}" target ON (
+          target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
         )
+        WHERE stage_update.external_id = stage.external_id
+          AND target.external_id IS NULL
       SQL
     end
 

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "2.0.3"
+  VERSION = "2.0.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-11 00:00:00.000000000 Z
+date: 2017-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -132,10 +132,9 @@ files:
 - lib/beetle_etl/import.rb
 - lib/beetle_etl/naming.rb
 - lib/beetle_etl/reporter.rb
-- lib/beetle_etl/step_runner/abstract_step_runner.rb
 - lib/beetle_etl/step_runner/async_step_runner.rb
 - lib/beetle_etl/step_runner/dependency_resolver.rb
-- lib/beetle_etl/step_runner/sequential_step_runner.rb
+- lib/beetle_etl/step_runner/step_runner.rb
 - lib/beetle_etl/steps/create_stage.rb
 - lib/beetle_etl/steps/drop_stage.rb
 - lib/beetle_etl/steps/load.rb
@@ -191,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: BeetleETL helps you with your recurring ETL imports.

data/lib/beetle_etl/step_runner/sequential_step_runner.rb

@@ -1,13 +0,0 @@
-require_relative './abstract_step_runner'
-
-module BeetleETL
-  class SequentialStepRunner < AbstractStepRunner
-
-    def run
-      @steps.reduce({}) do |results, step|
-        add_result!(results, run_step(step))
-      end
-    end
-
-  end
-end