beetle_etl 0.0.7 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d792ed373d43d6c0ab7ec27241eaf18fb59736b
-  data.tar.gz: 7b08688ce87fbff7eea72b362773444ec4700b2c
+  metadata.gz: 788ce4239b271ab02fe67da3642f8f17e6fd275e
+  data.tar.gz: e89e37f2bd8ec970599249b73e97f9d66be60555
 SHA512:
-  metadata.gz: 7022ab2cc6a60f57f061d3b1acaa8a125da6db7e9cec76421189ae2fa2c4729b4134cb8246a8c14283aeb7769e1ef92a75f480e2d6be3a3e41ba2e8017b7ee2c
-  data.tar.gz: fa6e2f2f74cf1c4a6a03e53c2d76c962154e8d64f656d32690101bdd4dfa636dda9036213b29dfc675d07e9ed381a6527e94abb462f4f7eabfa3c3f11f279ee1
+  metadata.gz: 3df6bfbcadd41a1e98f330aa35c9bcc0701c3450d9425c7c156520cae467cdccbb4adc5075a7923261d972c1341f50c49f3a09e35536270f2cdd165414b14ea7
+  data.tar.gz: dd92ef629e21523001a4d8371b0a8b2432845ecb371b0bde278d752b06f989e19c1b61321dc28686220f14a0b428f33b749bab52c119ec1d56284c6a71ae0e44

data/beetle_etl.gemspec

@@ -19,10 +19,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'sequel', '>= 4.0.0'
+  spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'timecop', '>= 0.7.0'
   spec.add_development_dependency 'pg', '>= 0.18.0'
-  spec.add_development_dependency 'activesupport', '>= 4.2.0'
+  spec.add_development_dependency 'unindent', '~> 1.0'
 end

data/lib/beetle_etl/import.rb

@@ -1,15 +1,34 @@
+require 'active_support/core_ext/hash/deep_merge'
+
 module BeetleETL
   class Import
 
     def run
-      TaskRunner.new(data_steps).run
-      BeetleETL.database.transaction do
-        TaskRunner.new(load_steps).run
-      end
-    rescue => e
-      raise e
+      setup
+      import
     ensure
-      TaskRunner.new(cleanup_steps).run
+      cleanup
+    end
+
+    def setup
+      transformations.each do |t|
+        CreateStage.new(t.table_name, t.relations, t.column_names).run
+      end
+    end
+
+    def import
+      data_report = AsyncStepRunner.new(data_steps).run
+      load_report = BeetleETL.database.transaction do
+        AsyncStepRunner.new(load_steps).run
+      end
+
+      data_report.deep_merge load_report
+    end
+
+    def cleanup
+      transformations.each do |t|
+        DropStage.new(t.table_name).run
+      end
     end
 
     private
@@ -17,7 +36,6 @@ module BeetleETL
     def data_steps
       transformations.flat_map do |t|
         [
-          CreateStage.new(t.table_name, t.relations, t.column_names),
           Transform.new(t.table_name, t.dependencies, t.query),
           MapRelations.new(t.table_name, t.relations),
           TableDiff.new(t.table_name),
@@ -32,10 +50,6 @@ module BeetleETL
       end
     end
 
-    def cleanup_steps
-      transformations.map { |t| DropStage.new(t.table_name) }
-    end
-
     def transformations
       @transformations ||= TransformationLoader.new.load
     end

data/lib/beetle_etl/reporter.rb

@@ -0,0 +1,65 @@
+module BeetleETL
+  class Reporter
+
+    def initialize(report)
+      @report = report
+    end
+
+    def log_summary
+      BeetleETL.logger.info(summary)
+    end
+
+    private
+
+    def summary
+      "\n\n" +
+      @report.map do |(table_name, steps)|
+        total_duration = format_duration(sum_durations(steps))
+        [
+          table_name,
+          seperator("="),
+          step_rows(steps).join("\n"),
+          seperator("-"),
+          total_duration.rjust(line_width)
+        ].join("\n")
+      end.join("\n\n") + "\n"
+    end
+
+    def step_rows(steps)
+      steps.map do |step_name, data|
+        label = step_name.split(": ")[1] + ":"
+        duration = format_duration(data[:finished_at] - data[:started_at])
+        line = duration.rjust(line_width)
+        line[2, label.length] = label
+        line
+      end
+    end
+
+    def format_duration(duration)
+      Time.at(duration).utc.strftime("%H:%M:%S")
+    end
+
+    def sum_durations(steps)
+      steps.inject(0) do |acc, (_step_name, data)|
+        acc + (data[:finished_at] - data[:started_at])
+      end
+    end
+
+    def line_width
+      #   2 spaces
+      # + 1 colon
+      # + 1 space
+      # + 8 duration
+      12 + longest_step_name_length
+    end
+
+    def longest_step_name_length
+      @report.keys.max_by(&:length).length - 1
+    end
+
+    def seperator(character)
+      character * line_width
+    end
+
+  end
+end

data/lib/beetle_etl/step_runner/async_step_runner.rb

@@ -0,0 +1,83 @@
+module BeetleETL
+  class AsyncStepRunner
+
+    def initialize(steps)
+      @dependency_resolver = DependencyResolver.new(steps)
+      @steps = steps
+
+      @queue = Queue.new
+      @completed = Set.new
+      @running = Set.new
+    end
+
+    def run
+      results = {}
+
+      until all_steps_complete?
+        runnables.each do |step|
+          run_step_async(step)
+          mark_step_running(step.name)
+        end
+
+        table_name, step_name, step_data = @queue.pop
+
+        unless results.has_key?(table_name)
+          results[table_name] = {}
+        end
+
+        results[table_name][step_name] = step_data
+        mark_step_completed(step_name)
+      end
+
+      results
+    end
+
+    private
+
+    attr_reader :running, :completed
+
+    def run_step_async(step)
+      Thread.new do
+        begin
+          BeetleETL.logger.info("started step #{step.name}")
+
+          started_at = Time.now
+          step.run
+          finished_at = Time.now
+
+          duration = Time.at(finished_at - started_at).utc.strftime("%H:%M:%S")
+          BeetleETL.logger.info("finished #{step.name} in #{duration}")
+
+          @queue.push [
+            step.table_name,
+            step.name,
+            { started_at: started_at, finished_at: finished_at }
+          ]
+
+        rescue => e
+          BeetleETL.logger.fatal(e.message)
+          raise e
+        end
+      end
+    end
+
+    def mark_step_running(step_name)
+      running.add(step_name)
+    end
+
+    def mark_step_completed(step_name)
+      runnables.delete(step_name)
+      completed.add(step_name)
+    end
+
+    def runnables
+      resolvables = @dependency_resolver.resolvables(completed)
+      resolvables.reject { |r| running.include? r.name }
+    end
+
+    def all_steps_complete?
+      @steps.map(&:name).to_set == completed.to_set
+    end
+
+  end
+end

data/lib/beetle_etl/steps/create_stage.rb

@@ -1,4 +1,8 @@
 module BeetleETL
+
+  ColumnDefinitionNotFoundError = Class.new(StandardError)
+  NoColumnsDefinedError = Class.new(StandardError)
+
   class CreateStage < Step
 
     def initialize(table_name, relations, column_names)
@@ -7,27 +11,38 @@ module BeetleETL
       @column_names = column_names
     end
 
-    def dependencies
-      Set.new
-    end
-
     def run
       database.execute <<-SQL
-        CREATE TABLE #{stage_table_name_sql} (
+        CREATE UNLOGGED TABLE #{stage_table_name_sql} (
           id integer,
           external_id character varying(255),
           transition character varying(255),
 
-          #{[
-            payload_column_definitions,
-            relation_column_definitions
-          ].compact.join(',')}
-        )
+          #{column_definitions}
+        );
+
+        #{index_definitions}
       SQL
     end
 
     private
 
+    def column_definitions
+      definitions = [
+        payload_column_definitions,
+        relation_column_definitions
+      ].compact
+
+      if definitions.empty?
+        raise NoColumnsDefinedError.new <<-MSG
+          Transformation for #{table_name} has no column definitions.
+          Either add an array of columns or references to other tables.
+        MSG
+      end
+
+      definitions.join(',')
+    end
+
     def payload_column_definitions
       definitions = (@column_names - @relations.keys).map do |column_name|
         "#{column_name} #{column_type(column_name)}"
@@ -45,6 +60,13 @@ module BeetleETL
       definitions.join(',') if definitions.any?
     end
 
+    def index_definitions
+      index_columns = [:external_id] + @relations.keys.map { |c| "external_#{c}" }
+      index_columns.map do |column_name|
+        "CREATE INDEX ON #{stage_table_name_sql} (#{column_name})"
+      end.join(";")
+    end
+
     def column_type(column_name)
       @column_types ||= Hash[database.schema(public_table_name.to_sym)]
         .reduce({}) do |acc, (name, schema)|
@@ -52,6 +74,12 @@ module BeetleETL
           acc
         end
 
+      unless @column_types.has_key?(column_name)
+        raise ColumnDefinitionNotFoundError.new <<-MSG
+          Table "#{table_name}" has no column "#{column_name}".
+        MSG
+      end
+
       @column_types[column_name]
     end
 

data/lib/beetle_etl/steps/drop_stage.rb

@@ -1,10 +1,6 @@
 module BeetleETL
   class DropStage < Step
 
-    def dependencies
-      Set.new
-    end
-
     def run
       database.execute <<-SQL
         DROP TABLE IF EXISTS #{stage_table_name_sql}

data/lib/beetle_etl/steps/step.rb

@@ -1,7 +1,5 @@
 module BeetleETL
 
-  DependenciesNotDefinedError = Class.new(StandardError)
-
   class Step
 
     include BeetleETL::Naming
@@ -20,7 +18,7 @@ module BeetleETL
     end
 
     def dependencies
-      raise DependenciesNotDefinedError
+      Set.new
     end
 
     def external_source

data/lib/beetle_etl/steps/transform.rb

@@ -8,7 +8,7 @@ module BeetleETL
     end
 
     def dependencies
-      Set.new(@dependencies.map { |d| self.class.step_name(d) }) << CreateStage.step_name(table_name)
+      Set.new(@dependencies.map { |d| self.class.step_name(d) })
     end
 
     def run

data/lib/beetle_etl/testing.rb

@@ -0,0 +1,46 @@
+module BeetleETL
+  module Testing
+
+    TargetTableNotFoundError = Class.new(StandardError)
+    NoTransformationFoundError = Class.new(StandardError)
+
+    def with_stage_tables_for(*table_names, &block)
+      table_names.each do |table_name|
+        unless BeetleETL.database.table_exists?(table_name)
+          raise TargetTableNotFoundError.new <<-MSG
+            Missing target table "#{table_name}".
+            In order to create stage tables, BeetleETL requires the target tables to exist because they provide the column definitions.
+          MSG
+        end
+      end
+
+      import = Import.new
+      begin
+        import.setup
+        block.call
+      ensure
+        import.cleanup
+      end
+    end
+
+    def run_transformation(table_name)
+      transformations = TransformationLoader.new.load
+
+      unless transformations.map(&:table_name).include?(table_name)
+        raise NoTransformationFoundError.new <<-MSG
+          No transformation definition found for table "#{table_name}".
+        MSG
+      end
+
+      transformation = transformations.find { |t| t.table_name == table_name }
+      transform = Transform.new(transformation.table_name, transformation.dependencies, transformation.query)
+      transform.run
+    end
+
+
+    def stage_table_name(table_name)
+      BeetleETL::Naming.stage_table_name(table_name)
+    end
+
+  end
+end

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "0.0.7"
+  VERSION = "0.0.11"
 end

data/lib/beetle_etl.rb

@@ -1,6 +1,7 @@
 require 'beetle_etl/version'
 
 require 'sequel'
+require 'logger'
 
 module BeetleETL
 
@@ -21,10 +22,11 @@ module BeetleETL
   require 'beetle_etl/steps/load'
   require 'beetle_etl/steps/drop_stage'
 
-  require 'beetle_etl/task_runner/dependency_resolver'
-  require 'beetle_etl/task_runner/task_runner'
+  require 'beetle_etl/step_runner/dependency_resolver'
+  require 'beetle_etl/step_runner/async_step_runner'
 
   require 'beetle_etl/import'
+  require 'beetle_etl/reporter'
 
   class Configuration
     attr_accessor \
@@ -33,10 +35,12 @@ module BeetleETL
       :transformation_file,
       :stage_schema,
       :public_schema,
-      :external_source
+      :external_source,
+      :logger
 
     def initialize
       @public_schema = 'public'
+      @logger = ::Logger.new(STDOUT)
     end
   end
 
@@ -44,9 +48,9 @@ module BeetleETL
 
     def import
       begin
-        Import.new.run
-      rescue Exception => e
-        raise e
+        report = Import.new.run
+        Reporter.new(report).log_summary
+        report
       ensure
         @database.disconnect if @database
       end
@@ -60,6 +64,10 @@ module BeetleETL
       @config ||= Configuration.new
     end
 
+    def logger
+      config.logger
+    end
+
     def database
       if config.database
         config.database

data/spec/beetle_etl_spec.rb

@@ -3,9 +3,13 @@ require 'spec_helper'
 describe BeetleETL do
 
   describe '#import' do
-    it 'runs the import' do
-      expect(BeetleETL::Import).to receive_message_chain(:new, :run)
-      BeetleETL.import
+    it 'runs the import with reporting' do
+      report = double(:report)
+      reporter = double(:reporter, log_summary: nil)
+
+      expect(BeetleETL::Import).to receive_message_chain(:new, :run).and_return report
+      expect(BeetleETL::Reporter).to receive(:new).with(report).and_return reporter
+      expect(BeetleETL.import).to eql(report)
     end
   end
 

data/spec/dsl/transformation_loader_spec.rb

@@ -1,12 +1,11 @@
 require 'spec_helper'
-require 'tempfile'
 
 module BeetleETL
   describe TransformationLoader do
 
     subject { TransformationLoader.new }
 
-    before :example do
+    before do
       data_file = tempfile_with_contents <<-FILE
         import :foo do
           'foo'
@@ -47,12 +46,5 @@ module BeetleETL
       end
     end
 
-    def tempfile_with_contents(contents)
-      Tempfile.new('transform').tap do |file|
-        file.write(contents)
-        file.close
-      end
-    end
-
   end
 end

data/spec/feature/feature_spec.rb

@@ -13,9 +13,9 @@ describe BeetleETL do
 
   include ExampleSchema
 
-  let!(:time1) { Time.new(2014 , 7 , 17 , 16 , 12).beginning_of_day }
-  let!(:time2) { Time.new(2015 , 2 ,  8 , 22 , 18).beginning_of_day }
-  let!(:time3) { Time.new(2015 , 11 , 3 , 12 , 17).beginning_of_day }
+  let!(:time1) { Time.new(2014 ,  7 , 17 , 16 , 12).beginning_of_day }
+  let!(:time2) { Time.new(2015 ,  2 ,  8 , 22 , 18).beginning_of_day }
+  let!(:time3) { Time.new(2015 , 11 ,  3 , 12 , 17).beginning_of_day }
 
   before :each do
     create_tables
@@ -27,6 +27,7 @@ describe BeetleETL do
       config.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
       config.database_config = database_config
       config.external_source = 'source_name'
+      config.logger = Logger.new(Tempfile.new("log"))
     end
   end
 

data/spec/reporter_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+require 'unindent'
+
+module BeetleETL
+  describe Reporter do
+
+    let(:report) do
+      {
+        organisations: {
+          "organisations: Transform" => {
+            :started_at  => Time.new(2015, 03, 14, 16, 0),
+            :finished_at => Time.new(2015, 03, 14, 16, 10)
+          },
+          "organisations: MapRelations" => {
+            :started_at  => Time.new(2015, 03, 14, 17, 0),
+            :finished_at => Time.new(2015, 03, 14, 17, 10)
+          },
+          "organisations: Load" => {
+            :started_at  => Time.new(2015, 03, 14, 18, 0),
+            :finished_at => Time.new(2015, 03, 14, 18, 10)
+          },
+        },
+        departments: {
+          "departments: Transform" => {
+            :started_at  => Time.new(2015, 03, 14, 16, 0),
+            :finished_at => Time.new(2015, 03, 14, 16, 12)
+          },
+          "departments: MapRelations" => {
+            :started_at  => Time.new(2015, 03, 14, 17, 2),
+            :finished_at => Time.new(2015, 03, 14, 17, 10)
+          },
+          "departments: Load" => {
+            :started_at  => Time.new(2015, 03, 14, 18, 10),
+            :finished_at => Time.new(2015, 03, 14, 19, 21, 39)
+          },
+        }
+      }
+    end
+
+    it "loggs a summary of all step times by table name" do
+      expect(BeetleETL.logger).to receive(:info).with <<-LOG.unindent
+
+
+        organisations
+        ========================
+          Transform:    00:10:00
+          MapRelations: 00:10:00
+          Load:         00:10:00
+        ------------------------
+                        00:30:00
+
+        departments
+        ========================
+          Transform:    00:12:00
+          MapRelations: 00:08:00
+          Load:         01:11:39
+        ------------------------
+                        01:31:39
+      LOG
+
+      Reporter.new(report).log_summary
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -3,14 +3,18 @@ CodeClimate::TestReporter.start
 
 require_relative '../lib/beetle_etl.rb'
 require_relative 'support/database_helpers.rb'
+require_relative 'support/file_helpers.rb'
 
 RSpec.configure do |config|
 
   config.include SpecSupport::DatabaseHelpers
+  config.include SpecSupport::FileHelpers
+
   config.backtrace_exclusion_patterns = [/rspec-core/]
 
   config.around(:each) do |example|
     BeetleETL.reset
+
     if example.metadata[:feature]
       example.run
     else

data/spec/steps/create_stage_spec.rb

@@ -47,42 +47,54 @@ module BeetleETL
       it 'creates a stage table table with all payload columns' do
         subject.run
 
-        columns = Hash[test_database.schema(subject.stage_table_name.to_sym)]
+        schema = Hash[test_database.schema(subject.stage_table_name.to_sym)]
 
         expected_columns = %i(id external_id some_string some_integer some_float)
-        expect(columns.keys).to include(*expected_columns)
+        expect(schema.keys).to include(*expected_columns)
 
-        expect(columns[:id][:db_type]).to eq('integer')
-        expect(columns[:external_id][:db_type]).to eq('character varying(255)')
-        expect(columns[:transition][:db_type]).to eq('character varying(255)')
+        expect(schema[:id][:db_type]).to eq('integer')
+        expect(schema[:external_id][:db_type]).to eq('character varying(255)')
+        expect(schema[:transition][:db_type]).to eq('character varying(255)')
 
-        expect(columns[:some_string][:db_type]).to eq('character varying(200)')
-        expect(columns[:some_integer][:db_type]).to eq('integer')
-        expect(columns[:some_float][:db_type]).to eq('double precision')
+        expect(schema[:some_string][:db_type]).to eq('character varying(200)')
+        expect(schema[:some_integer][:db_type]).to eq('integer')
+        expect(schema[:some_float][:db_type]).to eq('double precision')
       end
 
       it 'adds columns for dependent foreign key associations' do
         subject.run
 
-        columns = Hash[test_database.schema(subject.stage_table_name)]
+        schema = Hash[test_database.schema(subject.stage_table_name)]
 
         expected_columns = %i(
           dependee_a_id external_dependee_a_id
           dependee_b_id external_dependee_b_id
         )
-        expect(columns.keys).to include(*expected_columns)
+        expect(schema.keys).to include(*expected_columns)
 
-        expect(columns[:dependee_a_id][:db_type]).to eq('integer')
-        expect(columns[:external_dependee_a_id][:db_type]).to eq('character varying(255)')
+        expect(schema[:dependee_a_id][:db_type]).to eq('integer')
+        expect(schema[:external_dependee_a_id][:db_type]).to eq('character varying(255)')
 
-        expect(columns[:dependee_b_id][:db_type]).to eq('integer')
-        expect(columns[:external_dependee_b_id][:db_type]).to eq('character varying(255)')
+        expect(schema[:dependee_b_id][:db_type]).to eq('integer')
+        expect(schema[:external_dependee_b_id][:db_type]).to eq('character varying(255)')
       end
 
       it 'does not add foreign key columns twice if defined as payload column' do
         columns = [:some_string, :dependee_a_id]
         CreateStage.new(:example_table, @relations, columns).run
       end
+
+      it 'raises an error if no columns and no relations are defined' do
+        expect do
+          CreateStage.new(:example_table, {}, []).run
+        end.to raise_error(BeetleETL::NoColumnsDefinedError)
+      end
+
+      it 'raises an error when given columns with no definition' do
+        expect do
+          CreateStage.new(:example_table, @relations, [:undefined_column]).run
+        end.to raise_error(BeetleETL::ColumnDefinitionNotFoundError)
+      end
     end
 
   end

data/spec/steps/step_spec.rb

@@ -27,8 +27,8 @@ module BeetleETL
     end
 
     describe '#dependencies' do
-      it 'raises an exception' do
-        expect { subject.dependencies }.to raise_error(DependenciesNotDefinedError)
+      it 'returns an empty set' do
+        expect(subject.dependencies).to eql(Set.new)
       end
     end
 

data/spec/steps/transform_spec.rb

@@ -15,7 +15,6 @@ module BeetleETL
           [
             'some_table: Transform',
             'some_other_table: Transform',
-            'example_table: CreateStage',
           ].to_set
         )
       end

data/spec/support/database_helpers.rb

@@ -37,7 +37,7 @@ end
 
 RSpec::Matchers.define :have_values do |*rows|
   match do |table_description|
-    dataset = test_database[table_description]
+    dataset = test_database[table_description.to_sym]
 
     columns = rows[0].map(&:to_sym)
     values = rows[1..-1]

data/spec/support/file_helpers.rb

@@ -0,0 +1,14 @@
+require "tempfile"
+
+module SpecSupport
+  module FileHelpers
+
+    def tempfile_with_contents(contents)
+      ::Tempfile.new('transform').tap do |file|
+        file.write(contents)
+        file.close
+      end
+    end
+
+  end
+end

data/spec/testing_spec.rb

@@ -0,0 +1,76 @@
+require 'byebug'
+require "spec_helper"
+require "beetle_etl/testing"
+
+describe "BeetleETL:Testing" do
+
+  include BeetleETL::Testing
+
+  before do
+    data_file = tempfile_with_contents <<-'FILE'
+      import :some_table do
+        columns :some_attribute
+      end
+
+      import :organisations do
+        references :some_table, on: :some_table_id
+        columns :name, :address
+
+        query <<-SQL
+          INSERT INTO #{stage_table} (external_id, address, name)
+          VALUES ('external_id', 'address', 'name')
+        SQL
+      end
+    FILE
+
+    BeetleETL.configure do |config|
+      config.database = test_database
+      config.transformation_file = data_file.path
+    end
+  end
+
+  context "with properly defined target tables" do
+    before do
+      test_database.create_table :some_table do
+        primary_key :id
+        String :external_id, size: 255
+        String :some_attribute, size: 255
+      end
+
+      test_database.create_table :organisations do
+        primary_key :id
+        String :external_id, size: 255
+        String :name, size: 255
+        String :address, size: 255
+      end
+    end
+
+    it "makes stage tables available in the block" do
+      with_stage_tables_for(:organisations, :some_table) do
+        expect(test_database.table_exists?(stage_table_name(:organisations))).to be_truthy
+        expect(test_database.table_exists?(stage_table_name(:some_table))).to be_truthy
+      end
+
+      expect(test_database.table_exists?(stage_table_name(:organisations))).to be_falsey
+      expect(test_database.table_exists?(stage_table_name(:some_table))).to be_falsey
+    end
+
+    it "allows the transformation to be run insiede the block" do
+      with_stage_tables_for(:organisations, :some_table) do
+        run_transformation(:organisations)
+
+        expect(stage_table_name(:organisations)).to have_values(
+          [ :external_id  , :address  , :name  ] ,
+          [ "external_id" , "address" , "name" ]
+        )
+      end
+    end
+
+  end
+
+  it "raises an error if the target table cannot be found" do
+    expect do
+      with_stage_tables_for(:organisations)
+    end.to raise_error(BeetleETL::Testing::TargetTableNotFoundError)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.11
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-28 00:00:00.000000000 Z
+date: 2015-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -81,19 +95,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.18.0
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: unindent
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '1.0'
 description: Taking care of synchronizing external data with referential data in your
   application.
 email:
@@ -115,6 +129,9 @@ files:
 - lib/beetle_etl/dsl/transformation_loader.rb
 - lib/beetle_etl/import.rb
 - lib/beetle_etl/naming.rb
+- lib/beetle_etl/reporter.rb
+- lib/beetle_etl/step_runner/async_step_runner.rb
+- lib/beetle_etl/step_runner/dependency_resolver.rb
 - lib/beetle_etl/steps/assign_ids.rb
 - lib/beetle_etl/steps/create_stage.rb
 - lib/beetle_etl/steps/drop_stage.rb
@@ -123,8 +140,7 @@ files:
 - lib/beetle_etl/steps/step.rb
 - lib/beetle_etl/steps/table_diff.rb
 - lib/beetle_etl/steps/transform.rb
-- lib/beetle_etl/task_runner/dependency_resolver.rb
-- lib/beetle_etl/task_runner/task_runner.rb
+- lib/beetle_etl/testing.rb
 - lib/beetle_etl/version.rb
 - script/postgres
 - spec/beetle_etl_spec.rb
@@ -134,6 +150,7 @@ files:
 - spec/feature/example_schema.rb
 - spec/feature/example_transform.rb
 - spec/feature/feature_spec.rb
+- spec/reporter_spec.rb
 - spec/spec_helper.rb
 - spec/steps/assign_ids_spec.rb
 - spec/steps/create_stage_spec.rb
@@ -145,7 +162,9 @@ files:
 - spec/support/database.yml.example
 - spec/support/database.yml.travis
 - spec/support/database_helpers.rb
+- spec/support/file_helpers.rb
 - spec/task_runner/dependency_resolver_spec.rb
+- spec/testing_spec.rb
 homepage: https://github.com/maiwald/beetle_etl
 licenses:
 - MIT
@@ -178,6 +197,7 @@ test_files:
 - spec/feature/example_schema.rb
 - spec/feature/example_transform.rb
 - spec/feature/feature_spec.rb
+- spec/reporter_spec.rb
 - spec/spec_helper.rb
 - spec/steps/assign_ids_spec.rb
 - spec/steps/create_stage_spec.rb
@@ -189,4 +209,6 @@ test_files:
 - spec/support/database.yml.example
 - spec/support/database.yml.travis
 - spec/support/database_helpers.rb
+- spec/support/file_helpers.rb
 - spec/task_runner/dependency_resolver_spec.rb
+- spec/testing_spec.rb

data/lib/beetle_etl/task_runner/task_runner.rb

@@ -1,71 +0,0 @@
-module BeetleETL
-  class TaskRunner
-
-    def initialize(tasks)
-      @dependency_resolver = DependencyResolver.new(tasks)
-      @tasks = tasks
-
-      @queue = Queue.new
-      @completed = Set.new
-      @running = Set.new
-    end
-
-    def run
-      results = {}
-
-      until all_tasks_complete?
-        runnables.each do |task|
-          run_task_async(task)
-          mark_task_running(task.name)
-        end
-
-        task_name, task_data = @queue.pop
-        results[task_name] = task_data
-        mark_task_completed(task_name)
-      end
-
-      results
-    end
-
-    private
-
-    attr_reader :running, :completed
-
-    def run_task_async(task)
-      Thread.new do
-        started_at = now
-        result = task.run
-        finished_at = now
-
-        @queue.push [task.name, {
-          started_at: started_at,
-          finished_at: finished_at,
-          result: result,
-        }]
-      end
-    end
-
-    def mark_task_running(task_name)
-      running.add(task_name)
-    end
-
-    def mark_task_completed(task_name)
-      runnables.delete(task_name)
-      completed.add(task_name)
-    end
-
-    def runnables
-      resolvables = @dependency_resolver.resolvables(completed)
-      resolvables.reject { |r| running.include? r.name }
-    end
-
-    def all_tasks_complete?
-      @tasks.map(&:name).to_set == completed.to_set
-    end
-
-    def now
-      Time.now
-    end
-
-  end
-end