beetle_etl 1.0.1 → 2.0.0

data/lib/beetle_etl/steps/create_stage.rb

@@ -5,8 +5,8 @@ module BeetleETL
 
   class CreateStage < Step
 
-    def initialize(table_name, relations, column_names)
-      super(table_name)
+    def initialize(config, table_name, relations, column_names)
+      super(config, table_name)
       @relations = relations
       @column_names = column_names
     end

data/lib/beetle_etl/steps/load.rb

@@ -6,8 +6,8 @@ module BeetleETL
       transition
     ]
 
-    def initialize(table_name, relations)
-      super(table_name)
+    def initialize(config, table_name, relations)
+      super(config, table_name)
       @relations = relations
     end
 

data/lib/beetle_etl/steps/map_relations.rb

@@ -1,8 +1,8 @@
 module BeetleETL
   class MapRelations < Step
 
-    def initialize(table_name, relations)
-      super(table_name)
+    def initialize(config, table_name, relations)
+      super(config, table_name)
       @relations = relations
     end
 

data/lib/beetle_etl/steps/step.rb

@@ -2,10 +2,10 @@ module BeetleETL
 
   class Step
 
-    include BeetleETL::Naming
     attr_reader :table_name
 
-    def initialize(table_name)
+    def initialize(config, table_name)
+      @config = config
       @table_name = table_name
     end
 
@@ -22,11 +22,30 @@ module BeetleETL
     end
 
     def external_source
-      BeetleETL.config.external_source
+      @config.external_source
     end
 
     def database
-      BeetleETL.database
+      @config.database
+    end
+
+    # naming
+
+    def stage_table_name
+      BeetleETL::Naming.stage_table_name(@config.external_source, @table_name)
+    end
+
+    def stage_table_name_sql(table_name = nil)
+      table_name ||= @table_name
+      BeetleETL::Naming.stage_table_name_sql(@config.external_source, table_name)
+    end
+
+    def target_table_name
+      BeetleETL::Naming.target_table_name(@config.target_schema, @table_name)
+    end
+
+    def target_table_name_sql
+      BeetleETL::Naming.target_table_name_sql(@config.target_schema, @table_name)
     end
 
   end

data/lib/beetle_etl/steps/transform.rb

@@ -1,8 +1,8 @@
 module BeetleETL
   class Transform < Step
 
-    def initialize(table_name, dependencies, query)
-      super(table_name)
+    def initialize(config, table_name, dependencies, query)
+      super(config, table_name)
       @dependencies = dependencies
       @query = query
     end

data/lib/beetle_etl/testing.rb

@@ -6,9 +6,14 @@ module BeetleETL
     TargetTableNotFoundError = Class.new(StandardError)
     NoTransformationFoundError = Class.new(StandardError)
 
+    def self.configure
+      @@config = Configuration.new
+      yield(@@config)
+    end
+
     def with_stage_tables_for(*table_names, &block)
       table_names.each do |table_name|
-        unless BeetleETL.database.table_exists?(table_name)
+        unless @@config.database.table_exists?(table_name)
           raise TargetTableNotFoundError.new <<-MSG
             Missing target table "#{table_name}".
             In order to create stage tables, BeetleETL requires the target tables to exist because they provide the column definitions.
@@ -16,12 +21,12 @@ module BeetleETL
         end
       end
 
-      test_wrapper = TestWrapper.new(table_names)
+      test_wrapper = TestWrapper.new(@@config, table_names)
       test_wrapper.run block
     end
 
     def run_transformation(table_name)
-      transformations = TransformationLoader.new.load
+      transformations = TransformationLoader.new(@@config).load
 
       unless transformations.map(&:table_name).include?(table_name)
         raise NoTransformationFoundError.new <<-MSG
@@ -30,13 +35,13 @@ module BeetleETL
       end
 
       transformation = transformations.find { |t| t.table_name == table_name }
-      transform = Transform.new(transformation.table_name, transformation.dependencies, transformation.query)
+      transform = Transform.new(@@config, transformation.table_name, transformation.dependencies, transformation.query)
       transform.run
     end
 
 
     def stage_table_name(table_name)
-      BeetleETL::Naming.stage_table_name(table_name)
+      BeetleETL::Naming.stage_table_name(@@config.external_source, table_name)
     end
 
   end

data/lib/beetle_etl/testing/test_wrapper.rb

@@ -1,6 +1,6 @@
 module BeetleETL
   module Testing
-    class TestWrapper < Struct.new(:table_names)
+    class TestWrapper < Struct.new(:config, :table_names)
 
       def run(block)
         begin
@@ -15,18 +15,18 @@ module BeetleETL
 
       def create_stages
         transformations.each do |t|
-          CreateStage.new(t.table_name, t.relations, t.column_names).run
+          CreateStage.new(config, t.table_name, t.relations, t.column_names).run
         end
       end
 
       def drop_stages
         transformations.each do |t|
-          DropStage.new(t.table_name).run
+          DropStage.new(config, t.table_name).run
         end
       end
 
       def transformations
-        @transformations ||= TransformationLoader.new.load.find_all do |transformation|
+        @transformations ||= TransformationLoader.new(config).load.find_all do |transformation|
           table_names.include? transformation.table_name
         end
       end

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "1.0.1"
+  VERSION = "2.0.0"
 end

data/spec/beetle_etl_spec.rb

@@ -4,48 +4,16 @@ describe BeetleETL do
 
   describe '#import' do
     it 'runs the import with reporting' do
+      config = double(:config, disconnect_database: nil)
+      runner = double(:runner)
       report = double(:report)
       reporter = double(:reporter, log_summary: nil)
 
-      expect(BeetleETL::Import).to receive_message_chain(:new, :run).and_return report
-      expect(BeetleETL::Reporter).to receive(:new).with(report).and_return reporter
-      expect(BeetleETL.import).to eql(report)
-    end
-  end
-
-  describe '#config' do
-    it 'returns a configuration object' do
-      expect(BeetleETL.config).to be_a(BeetleETL::Configuration)
-    end
-  end
-
-  describe '#configure' do
-    it 'allows the configuration to be changed' do
-      expect(BeetleETL.config.external_source).to be_nil
-
-      BeetleETL.configure { |config| config.external_source = 'foo' }
-
-      expect(BeetleETL.config.external_source).to eql('foo')
-    end
-  end
-
-  describe '#database' do
-    let(:database) { double(:database) }
-
-    it 'returns the Sequel Database object stored in the config' do
-      BeetleETL.configure { |config| config.database = database }
-
-      expect(BeetleETL.database).to eql(database)
-    end
-
-    it 'builds and caches a Sequel Database from config when no database is passed' do
-      database_config = double(:database_config)
-      BeetleETL.configure { |config| config.database_config = database_config }
-
-      expect(Sequel).to receive(:connect).with(database_config).once { database }
+      expect(BeetleETL::Import).to receive(:new).with(config).and_return runner
+      expect(runner).to receive(:run).and_return report
+      expect(BeetleETL::Reporter).to receive(:new).with(config, report).and_return reporter
 
-      expect(BeetleETL.database).to eql(database)
-      expect(BeetleETL.database).to eql(database)
+      expect(BeetleETL.import(config)).to eql(report)
     end
   end
 

data/spec/configuration_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe Configuration do
+
+    subject { Configuration.new }
+
+    describe "#database" do
+      let(:database) { double(:database) }
+
+      it "returns the object if present" do
+        subject.database = database
+
+        expect(subject.database).to eql(database)
+      end
+
+      it "builds a Sequel Database from config when no database is passed" do
+        database_config = double(:database_config)
+        subject.database_config = database_config
+
+        expect(Sequel).to receive(:connect).with(database_config).once { database }
+
+        expect(subject.database).to eql(database)
+        expect(subject.database).to eql(database)
+      end
+
+      it "raises an error if no database or database_config is passed" do
+        expect { subject.database }
+          .to raise_error(BeetleETL::InvalidConfigurationError)
+      end
+    end
+
+    describe "#disconnect_database" do
+      let(:database) { double(:database) }
+
+      it "disconnects from database if database_config was passed" do
+        database_config = double(:database_config)
+
+        expect(Sequel).to receive(:connect).with(database_config) { database }
+        expect(database).to receive(:disconnect)
+
+        subject.database_config = database_config
+        subject.disconnect_database
+      end
+
+      it "does not disconnect from database if database object was passed" do
+        expect(database).not_to receive(:disconnect)
+
+        subject.database = database
+        subject.disconnect_database
+      end
+    end
+
+    describe "#target_schema" do
+      it "returns nil if target_schema is 'public'" do
+        subject.target_schema = "public"
+        expect(subject.target_schema).to be_nil
+      end
+
+      it "returns target_schema if target_schema is not 'public'" do
+        subject.target_schema = "foo"
+        expect(subject.target_schema).to eql("foo")
+      end
+    end
+  end
+end

data/spec/dsl/dsl_spec.rb

@@ -3,18 +3,24 @@ require 'spec_helper'
 module BeetleETL
   describe DSL do
 
-    subject { DSL.new(:foo_table) }
+    let(:config) do
+      Configuration.new.tap do |c|
+        c.external_source = "bar"
+      end
+    end
+
+    subject { DSL.new(config, :foo_table) }
 
     describe '#stage_table' do
       it 'returns the current stage table name' do
         expect(subject.stage_table).to eql(
-          BeetleETL::Naming.stage_table_name_sql(:foo_table)
+          BeetleETL::Naming.stage_table_name_sql("bar", :foo_table)
         )
       end
 
       it 'returns the stage table name for the given table' do
         expect(subject.stage_table(:bar_table)).to eql(
-          BeetleETL::Naming.stage_table_name_sql(:bar_table)
+          BeetleETL::Naming.stage_table_name_sql("bar", :bar_table)
         )
       end
     end

data/spec/dsl/transformation_loader_spec.rb

@@ -3,9 +3,7 @@ require 'spec_helper'
 module BeetleETL
   describe TransformationLoader do
 
-    subject { TransformationLoader.new }
-
-    before do
+    let(:config) do
       data_file = tempfile_with_contents <<-FILE
         import :foo do
           'foo'
@@ -20,15 +18,18 @@ module BeetleETL
         end
       FILE
 
-      BeetleETL.configure do |config|
-        config.transformation_file = data_file.path
+      Configuration.new.tap do |c|
+        c.transformation_file = data_file.path
       end
     end
 
+    subject { TransformationLoader.new(config) }
+
     describe '#load' do
       it 'loads transformations from the data file' do
-        expect(Transformation).to receive(:new) do |table_name, config, helpers|
-          expect(table_name.to_s).to eql(config.call)
+        expect(Transformation).to receive(:new) do |configuration, table_name, setup, helpers|
+          expect(configuration).to eql(config)
+          expect(table_name.to_s).to eql(setup.call)
           expect(helpers.call).to eql("baz")
         end.exactly(2).times
 
@@ -36,7 +37,7 @@ module BeetleETL
       end
 
       it 'returns the list of transformations' do
-        allow(Transformation).to receive(:new) do |table_name, config, helpers|
+        allow(Transformation).to receive(:new) do |configuration, table_name, setup, helpers|
           table_name
         end
 

data/spec/dsl/transformation_spec.rb

@@ -3,9 +3,11 @@ require 'spec_helper'
 module BeetleETL
   describe Transformation do
 
+    let(:config) { Configuration.new }
+
     describe '#table_name' do
       it 'returns the given table name' do
-        transformation = Transformation.new(:table, Proc.new {})
+        transformation = Transformation.new(config, :table, Proc.new {})
         expect(transformation.table_name).to eql(:table)
       end
     end
@@ -15,7 +17,7 @@ module BeetleETL
         setup = Proc.new do
           columns :payload_1, 'payload_2'
         end
-        transformation = Transformation.new(:table, setup)
+        transformation = Transformation.new(config, :table, setup)
 
         expect(transformation.column_names).to match_array([
           :payload_1, :payload_2
@@ -23,7 +25,7 @@ module BeetleETL
       end
 
       it 'defaults to an empty array if no columns are defined' do
-        transformation = Transformation.new(:table, Proc.new {})
+        transformation = Transformation.new(config, :table, Proc.new {})
 
         expect(transformation.column_names).to match_array([])
       end
@@ -35,7 +37,7 @@ module BeetleETL
           references :foreign_table, on: :foreign_table_id
           references :another_foreign_table, on: :another_foreign_table_id
         end
-        transformation = Transformation.new(:table, setup)
+        transformation = Transformation.new(config, :table, setup)
 
         expect(transformation.relations).to eql({
           foreign_table_id: :foreign_table,
@@ -50,7 +52,7 @@ module BeetleETL
           references :foreign_table, on: :foreign_table_id
           references :another_foreign_table, on: :another_foreign_table_id
         end
-        transformation = Transformation.new(:table, setup)
+        transformation = Transformation.new(config, :table, setup)
 
         expect(transformation.dependencies).to eql(Set.new([:foreign_table, :another_foreign_table]))
       end
@@ -66,7 +68,7 @@ module BeetleETL
           query "SELECT '#{foo}' FROM some_table"
         end
 
-        transformation = Transformation.new(:table, setup, helpers)
+        transformation = Transformation.new(config, :table, setup, helpers)
 
         expect(transformation.query).to eql(
           "SELECT 'foo_string' FROM some_table"
@@ -78,7 +80,7 @@ module BeetleETL
           query "SOME QUERY"
           query "ANOTHER QUERY"
         end
-        transformation = Transformation.new(:table, setup)
+        transformation = Transformation.new(config, :table, setup)
 
         expect(transformation.query).to eql(
           "SOME QUERY;ANOTHER QUERY"

data/spec/feature/feature_spec.rb

@@ -23,11 +23,11 @@ describe BeetleETL do
     database_config_path = File.expand_path('../support/database.yml', File.dirname(__FILE__))
     database_config = YAML.load(File.read(database_config_path))
 
-    BeetleETL.configure do |config|
-      config.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
-      config.database_config = database_config
-      config.external_source = 'source_name'
-      config.logger = Logger.new(Tempfile.new("log"))
+    @config = BeetleETL::Configuration.new.tap do |c|
+      c.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
+      c.database_config = database_config
+      c.external_source = 'source_name'
+      c.logger = Logger.new(Tempfile.new("log"))
     end
   end
 
@@ -54,7 +54,7 @@ describe BeetleETL do
     )
 
     Timecop.freeze(time1) do
-      BeetleETL.import
+      BeetleETL.import(@config)
     end
 
     expect(:organisations).to have_values(
@@ -86,7 +86,7 @@ describe BeetleETL do
     )
 
     Timecop.freeze(time2) do
-      BeetleETL.import
+      BeetleETL.import(@config)
     end
 
     expect(:organisations).to have_values(
@@ -118,7 +118,7 @@ describe BeetleETL do
     )
 
     Timecop.freeze(time3) do
-      BeetleETL.import
+      BeetleETL.import(@config)
     end
 
     expect(:organisations).to have_values(