beetle_etl 1.0.1 → 2.0.0

data/spec/reporter_spec.rb

@@ -38,7 +38,10 @@ module BeetleETL
     end
 
     it "loggs a summary of all step times by table name" do
-      expect(BeetleETL.logger).to receive(:info).with <<-LOG.unindent
+      config = Configuration.new.tap do |c|
+        c.logger = double(:logger)
+      end
+      expect(config.logger).to receive(:info).with <<-LOG.unindent
 
 
         organisations
@@ -58,7 +61,7 @@ module BeetleETL
                         01:31:39
       LOG
 
-      Reporter.new(report).log_summary
+      Reporter.new(config, report).log_summary
     end
 
   end

data/spec/spec_helper.rb

@@ -1,9 +1,10 @@
+require "byebug"
 require "codeclimate-test-reporter"
 CodeClimate::TestReporter.start
 
-require_relative '../lib/beetle_etl.rb'
-require_relative 'support/database_helpers.rb'
-require_relative 'support/file_helpers.rb'
+require_relative "../lib/beetle_etl.rb"
+require_relative "support/database_helpers.rb"
+require_relative "support/file_helpers.rb"
 
 RSpec.configure do |config|
 
@@ -13,8 +14,6 @@ RSpec.configure do |config|
   config.backtrace_exclusion_patterns = [/rspec-core/]
 
   config.around(:each) do |example|
-    BeetleETL.reset
-
     if example.metadata[:feature]
       example.run
     else

data/spec/steps/assign_ids_spec.rb

@@ -6,16 +6,16 @@ module BeetleETL
     let(:external_source) { 'my_source' }
     let(:another_source) { 'another_source' }
 
-    subject { AssignIds.new(:example_table) }
-
-    before do
-      BeetleETL.configure do |config|
-        config.stage_schema = 'stage'
-        config.external_source = external_source
-        config.database = test_database
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.stage_schema = 'stage'
+        c.external_source = external_source
+        c.database = test_database
       end
     end
 
+    subject { AssignIds.new(config, :example_table) }
+
     describe '#dependencies' do
       it 'depends on TableDiff of the same table' do
         expect(subject.dependencies).to eql(['example_table: TableDiff'].to_set)

data/spec/steps/create_stage_spec.rb

@@ -3,20 +3,22 @@ require 'spec_helper'
 module BeetleETL
   describe CreateStage do
 
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.database = test_database
+        c.external_source = "source"
+      end
+    end
+
     describe '#dependencies' do
       it 'has no dependencies' do
-        subject = CreateStage.new(:example_table, double(:dependencies), double(:columns))
+        subject = CreateStage.new(config, :example_table, double(:dependencies), double(:columns))
         expect(subject.dependencies).to eql(Set.new)
       end
     end
 
     describe '#run' do
       before do
-        BeetleETL.configure do |config|
-          config.database = test_database
-          config.external_source = "source"
-        end
-
         test_database.execute <<-SQL
           CREATE TABLE example_table (
             id INTEGER,
@@ -42,7 +44,7 @@ module BeetleETL
       end
 
       let(:subject) do
-        CreateStage.new(:example_table, @relations, @columns)
+        CreateStage.new(config, :example_table, @relations, @columns)
       end
 
       it 'creates a stage table table with all payload columns' do
@@ -82,30 +84,30 @@ module BeetleETL
 
       it 'does not add foreign key columns twice if defined as payload column' do
         columns = [:some_string, :dependee_a_id]
-        CreateStage.new(:example_table, @relations, columns).run
+        CreateStage.new(config, :example_table, @relations, columns).run
       end
 
       it 'raises an error if no columns and no relations are defined' do
         expect do
-          CreateStage.new(:example_table, {}, []).run
+          CreateStage.new(config, :example_table, {}, []).run
         end.to raise_error(BeetleETL::NoColumnsDefinedError)
       end
 
       it 'raises an error when given columns with no definition' do
         expect do
-          CreateStage.new(:example_table, @relations, [:undefined_column]).run
+          CreateStage.new(config, :example_table, @relations, [:undefined_column]).run
         end.to raise_error(BeetleETL::ColumnDefinitionNotFoundError)
       end
 
       it "truncates the stage table if it already exists" do
-        CreateStage.new(:example_table, {}, @columns).run
+        CreateStage.new(config, :example_table, {}, @columns).run
 
         insert_into(subject.stage_table_name.to_sym).values(
           [ :some_string , :some_integer , :some_float ] ,
           [ "hello"     , 123           , 123.456      ]
         )
 
-        CreateStage.new(:example_table, {}, @columns).run
+        CreateStage.new(config, :example_table, {}, @columns).run
 
         expect(subject.stage_table_name).to have_values(
           [:some_string, :some_integer, :some_float]

data/spec/steps/load_spec.rb

@@ -11,14 +11,16 @@ module BeetleETL
     let(:now) { Time.now.beginning_of_day }
     let(:yesterday) { 1.day.ago.beginning_of_day }
 
-    subject { Load.new(:example_table, []) }
-
-    before do
-      BeetleETL.configure do |config|
-        config.external_source = external_source
-        config.database = test_database
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.external_source = external_source
+        c.database = test_database
       end
+    end
 
+    subject { Load.new(config, :example_table, []) }
+
+    before do
       allow(subject).to receive(:now) { now }
 
       test_database.create_schema(:stage)
@@ -54,7 +56,7 @@ module BeetleETL
           dependee_b_id: :dependee_b,
         }
 
-        expect(Load.new(:depender, relations).dependencies).to eql(
+        expect(Load.new(config, :depender, relations).dependencies).to eql(
           [
             'dependee_a: Load',
             'dependee_b: Load',

data/spec/steps/map_relations_spec.rb

@@ -3,8 +3,19 @@ require 'spec_helper'
 module BeetleETL
   describe MapRelations do
 
-    let(:dependee_a) { BeetleETL::Naming.stage_table_name(:dependee_a).to_sym }
-    let(:dependee_b) { BeetleETL::Naming.stage_table_name(:dependee_b).to_sym }
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.external_source = 'my_source'
+        c.database = test_database
+      end
+    end
+
+    let(:dependee_a) do
+      BeetleETL::Naming.stage_table_name('my_source', :dependee_a).to_sym
+    end
+    let(:dependee_b) do
+      BeetleETL::Naming.stage_table_name('my_source', :dependee_b).to_sym
+    end
 
     let(:relations) do
       {
@@ -14,15 +25,10 @@ module BeetleETL
     end
 
     subject do
-      MapRelations.new(:depender, relations)
+      MapRelations.new(config, :depender, relations)
     end
 
     before do
-      BeetleETL.configure do |config|
-        config.external_source = 'my_source'
-        config.database = test_database
-      end
-
       test_database.create_table(dependee_a) do
         Integer :id
         String :external_id, size: 255

data/spec/steps/step_spec.rb

@@ -3,7 +3,9 @@ require 'spec_helper'
 module BeetleETL
   describe Step do
 
-    subject { Step.new(:example_table) }
+    let(:config) { Configuration.new }
+
+    subject { Step.new(config, :example_table) }
     FooStep = Class.new(Step)
 
     describe '.step_name' do
@@ -18,11 +20,11 @@ module BeetleETL
 
     describe '#name' do
       it 'returns the steps name' do
-        expect(Step.new(:example_table).name).to eql('example_table: Step')
+        expect(Step.new(config, :example_table).name).to eql('example_table: Step')
       end
 
       it 'returns the step name of inheriting steps' do
-        expect(FooStep.new(:foo_table).name).to eql('foo_table: FooStep')
+        expect(FooStep.new(config, :foo_table).name).to eql('foo_table: FooStep')
       end
     end
 

data/spec/steps/table_diff_spec.rb

@@ -7,15 +7,16 @@ module BeetleETL
   describe TableDiff do
 
     let(:external_source) { 'my_source' }
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.external_source = external_source
+        c.database = test_database
+      end
+    end
 
-    subject { TableDiff.new(:example_table) }
+    subject { TableDiff.new(config, :example_table) }
 
     before do
-      BeetleETL.configure do |config|
-        config.external_source = external_source
-        config.database = test_database
-      end
-
       test_database.create_table(subject.stage_table_name.to_sym) do
         String :external_id, size: 255
         String :transition, size: 20

data/spec/steps/transform_spec.rb

@@ -3,10 +3,17 @@ require 'spec_helper'
 module BeetleETL
   describe Transform do
 
+    let(:database) { double(:database) }
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.database = database
+      end
+    end
     let(:query) { double(:query) }
+
     subject do
       deps = [:some_table, :some_other_table].to_set
-      Transform.new(:example_table, deps, query)
+      Transform.new(config, :example_table, deps, query)
     end
 
     describe '#dependencies' do
@@ -22,9 +29,6 @@ module BeetleETL
 
     describe '#run' do
       it 'runs a query in the database' do
-        database = double(:database)
-        BeetleETL.configure { |config| config.database = database }
-
         expect(database).to receive(:run).with(query)
 
         subject.run

data/spec/testing_spec.rb

@@ -22,7 +22,7 @@ describe "BeetleETL:Testing" do
       end
     FILE
 
-    BeetleETL.configure do |config|
+    BeetleETL::Testing.configure do |config|
       config.database = test_database
       config.transformation_file = data_file.path
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-16 00:00:00.000000000 Z
+date: 2016-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -116,6 +116,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".byebug_history"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
@@ -124,6 +125,7 @@ files:
 - Rakefile
 - beetle_etl.gemspec
 - lib/beetle_etl.rb
+- lib/beetle_etl/configuration.rb
 - lib/beetle_etl/dsl/dsl.rb
 - lib/beetle_etl/dsl/transformation.rb
 - lib/beetle_etl/dsl/transformation_loader.rb
@@ -145,6 +147,7 @@ files:
 - lib/beetle_etl/version.rb
 - script/postgres
 - spec/beetle_etl_spec.rb
+- spec/configuration_spec.rb
 - spec/dsl/dsl_spec.rb
 - spec/dsl/transformation_loader_spec.rb
 - spec/dsl/transformation_spec.rb
@@ -186,12 +189,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: BeetleETL helps you with your recurring ETL imports.
 test_files:
 - spec/beetle_etl_spec.rb
+- spec/configuration_spec.rb
 - spec/dsl/dsl_spec.rb
 - spec/dsl/transformation_loader_spec.rb
 - spec/dsl/transformation_spec.rb