beetle_etl 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a726a9734d6866687319a6742cc8db41ef68b64
-  data.tar.gz: 82be15d660033bd3d957879ec351c0479b26623f
+  metadata.gz: 2cd21d7530444cef54966138ce38a5ec6ca0f08b
+  data.tar.gz: 277b54c1b957edeeb78fab7434852215f571d486
 SHA512:
-  metadata.gz: 01c1408c035afb9d0dcadb9382a3db442318f48c74517f1376c3a6ef615675ef891df8559c172be6e6b1f7f47d2f29c4a6965493a613d93a24d439c40f224246
-  data.tar.gz: 111f73fd2692ed88f4ce91c6f4da838efe13d1d12fc00bfb4f63cb00383509ffa1d171bf265362ded8f01ddde7f0a0e0659d552f7be8b31188c94bb4bb984e59
+  metadata.gz: 659b6e6cde8208578833b7fd4a870f15fd706300389aef4c2c8c93786a17954bddca324398f1ca2e32bba12a96ed89f2e26b237df78efe99f33e1cbb0d9cd476
+  data.tar.gz: cb8d7842912f24a4e82d0198f4508757f32c5051183face61e5359bfbe36a5ce353127e085992a33069d82e6afd6bc1d44a68c79d832d38fc079eca819d8726a

data/.gitignore

@@ -1,5 +1,6 @@
 *.gem
 *.rbc
+.byebug_history
 .bundle
 .config
 .yardoc

data/.travis.yml

@@ -6,8 +6,6 @@ rvm:
   - 2.3.0
 addons:
   postgresql: "9.3"
-  code_climate:
-    repo_token: fcd6d8c28da900609a2cf903716d858621b8ce68152edbcebe6908a9a3f5d3d5
 before_install:
   - gem update --system
   - gem update bundler

data/Gemfile

@@ -5,6 +5,5 @@ gemspec
 
 group :test do
   gem 'rake'
-  gem 'codeclimate-test-reporter'
   gem 'byebug'
 end

data/README.md

@@ -1,12 +1,11 @@
 # BeetleETL
 [![Build Status](https://travis-ci.org/maiwald/beetle_etl.svg?branch=master)](https://travis-ci.org/maiwald/beetle_etl)
-[![Code Climate](https://codeclimate.com/github/maiwald/beetle_etl.png)](https://codeclimate.com/github/maiwald/beetle_etl)
 
 BeetleETL helps you with synchronising relational databases and recurring imports of reference data. It is actually quite nice.
 
 Consider you have a set of database tables representing third party data (i.e. the ```source```) and you want to synchronize a set of tables in your application (i.e. the ```target```) with that third party data. Further consider that you want to apply transformations to that ```source``` data before you import it.
 
-You define your transformations and BeetleETL will to the rest. Even when your ```source``` data changes, when you run BeetleETL again, it can keep track of what changes need to be applied to what records in your application’s tables.
+You define your transformations and BeetleETL will do the rest. Even when your ```source``` data changes, when you run BeetleETL again, it can keep track of what changes need to be applied to what records in your application’s tables.
 
 It currently only works with PostgreSQL databases.
 
@@ -34,61 +33,64 @@ Make sure the tables you want to import contain columns named ```external_id```
 
 Create a configuration object
 
-    configuration = BeetleETL::Configuration.new do |config|
-      # path to your transformation file
-      config.transformation_file = "../my_fancy_transformations"
-
-      # sequel database config
-      config.database_config = {
-        adapter: 'postgres'
-        encoding: utf8
-        host: my_host
-        database: my_database
-        username: 'foo'
-        password: 'bar'
-        pool: 5
-        pool_timeout: 360
-        connect_timeout: 360
-      }
-      # or config.database = # sequel database instance
-
-      # name of your soruce
-      config.external_source = "important_data"
-
-      # target schema in case you use postgres schemas
-      config.target_schema = "public" # default
-
-      # logger
-      config.logger = Logger.new(STDOUT) # default
-    end
+```ruby
+configuration = BeetleETL::Configuration.new do |config|
+  # path to your transformation file
+  config.transformation_file = "../my_fancy_transformations"
+
+  # sequel database config
+  config.database_config = {
+    adapter: 'postgres'
+    encoding: utf8
+    host: my_host
+    database: my_database
+    username: 'foo'
+    password: 'bar'
+    pool: 5
+    pool_timeout: 360
+    connect_timeout: 360
+  }
+  # or config.database = # sequel database instance
+
+  # name of your soruce
+  config.external_source = "important_data"
+
+  # target schema in case you use postgres schemas
+  config.target_schema = "public" # default
+
+  # logger
+  config.logger = Logger.new(STDOUT) # default
+end
+```
 
 ### Defining Imports
 
 Fill a ```transformation``` file with import directives like this:
 
-    import :departments do
-      columns :name
-
-      references :organisations, on: :organisation_id
-
-      query <<-SQL
-        INSERT INTO #{stage_table} (
-          external_id,
-          name,
-          external_organisation_id
-        )
-
-        SELECT
-          o.id,
-          o.”dep_name”,
-          data.”address”
-
-        FROM ”Organisation” o
-        JOIN additional_data data
-          ON data.org_id = o.id
-      SQL
-    end
-
+```ruby
+import :departments do
+  columns :name
+
+  references :organisations, on: :organisation_id
+
+  query <<-SQL
+    INSERT INTO #{stage_table} (
+      external_id,
+      name,
+      external_organisation_id
+    )
+
+    SELECT
+      o.id,
+      o.”dep_name”,
+      data.”address”
+
+    FROM ”Organisation” o
+    JOIN additional_data data
+      ON data.org_id = o.id
+  SQL
+end
+```
 
 ```import``` takes the name of the table you want to fill and the configuration as arguments.
 With ```columns``` you define what columns BeetleETL is supposed to fill in your application’s table.

data/beetle_etl.gemspec

@@ -18,9 +18,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'sequel', '>= 4.0.0'
   spec.add_runtime_dependency 'activesupport', '>= 4.0.0'
 
+  spec.add_development_dependency 'sequel', '>= 4.0.0'
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'timecop', '>= 0.7.0'

data/lib/beetle_etl.rb

@@ -5,6 +5,7 @@ require 'logger'
 
 module BeetleETL
 
+  require 'beetle_etl/adapters/sequel_adapter'
   require 'beetle_etl/configuration'
 
   require 'beetle_etl/dsl/dsl'

data/lib/beetle_etl/adapters/sequel_adapter.rb

@@ -0,0 +1,35 @@
+module BeetleETL
+  class SequelAdapter
+    attr_reader :database
+    def initialize(database)
+      @database = database
+    end
+
+    def execute(query)
+      @database.run(query)
+    end
+
+    def column_names(schema_name, table_name)
+      @database[Sequel.qualify(schema_name, table_name)].columns
+    end
+
+    def column_types(schema_name, table_name)
+      Hash[@database.schema(Sequel.qualify(schema_name, table_name))].reduce({}) do |acc, (name, column_config)|
+        acc[name.to_sym] = column_config.fetch(:db_type)
+        acc
+      end
+    end
+
+    def table_exists?(schema_name, table_name)
+      @database.table_exists?(Sequel.qualify(schema_name, table_name))
+    end
+
+    def transaction(&block)
+      @database.transaction(&block)
+    end
+
+    def disconnect
+      @database.disconnect
+    end
+  end
+end

data/lib/beetle_etl/configuration.rb

@@ -1,38 +1,54 @@
 module BeetleETL
+
   InvalidConfigurationError = Class.new(StandardError)
 
   class Configuration
     attr_accessor \
       :transformation_file,
+      :target_schema,
       :stage_schema,
       :external_source,
       :logger
 
     attr_writer \
       :database,
-      :database_config,
-      :target_schema
+      :database_config
 
     def initialize
       @target_schema = 'public'
       @logger = ::Logger.new(STDOUT)
     end
 
+    def database=(database)
+      @database_config = nil
+      @adapter ||= case
+        when sequel?(database) then SequelAdapter.new(database)
+        else nil
+      end
+    end
+
+    def database_config=(database_config)
+      @database_config = database_config
+      @adapter = SequelAdapter.new(Sequel.connect(@database_config))
+    end
+
     def database
-      if [@database, @database_config].none?
-        msg = "Either Sequel connection database_config or a Sequel Database object required"
+      if @adapter.nil?
+        msg = "Either Sequel connection database_config, Sequel::Database object or ActiveRecord::Base.connection required!"
         raise InvalidConfigurationError.new(msg)
       end
 
-      @database ||= Sequel.connect(@database_config)
+      @adapter
     end
 
     def disconnect_database
-      database.disconnect if @database_config
+      @adapter.disconnect if @database_config
     end
 
-    def target_schema
-      @target_schema != 'public' ? @target_schema : nil
+    private
+
+    def sequel?(database)
+      defined?(::Sequel::Database) && database.is_a?(::Sequel::Database)
     end
 
   end

data/lib/beetle_etl/dsl/dsl.rb

@@ -26,10 +26,12 @@ module BeetleETL
     # query helper methods
 
     def stage_table(table_name = nil)
-      BeetleETL::Naming.stage_table_name_sql(
+      stage_table_name = BeetleETL::Naming.stage_table_name(
         @config.external_source,
         table_name || @table_name
       )
+
+      %Q("#{@config.target_schema}"."#{stage_table_name}")
     end
 
     def combined_key(*args)

data/lib/beetle_etl/naming.rb

@@ -10,18 +10,5 @@ module BeetleETL
       "#{external_source.to_s}-#{table_name.to_s}-#{digest}"[0, 63]
     end
 
-    def stage_table_name_sql(external_source, table_name)
-      %Q("#{stage_table_name(external_source, table_name)}")
-    end
-
-    def target_table_name(target_schema, table_name)
-      schema = target_schema ? target_schema.to_s : nil
-      [schema, table_name.to_s].compact.join('.')
-    end
-
-    def target_table_name_sql(target_schema, table_name)
-      %Q("#{target_table_name(target_schema, table_name)}")
-    end
-
   end
 end

data/lib/beetle_etl/steps/assign_ids.rb

@@ -7,10 +7,10 @@ module BeetleETL
 
     def run
       database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage_update
-        SET id = COALESCE(target.id, nextval('#{table_name}_id_seq'))
-        FROM #{stage_table_name_sql} stage
-        LEFT OUTER JOIN #{target_table_name_sql} target
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
+        SET id = COALESCE(target.id, NEXTVAL('#{target_schema}.#{table_name}_id_seq'))
+        FROM "#{target_schema}"."#{stage_table_name}" stage
+        LEFT OUTER JOIN "#{target_schema}"."#{table_name}" target
           on (
             stage.external_id = target.external_id
             AND target.external_source = '#{external_source}'

data/lib/beetle_etl/steps/create_stage.rb

@@ -13,7 +13,7 @@ module BeetleETL
 
     def run
       database.execute <<-SQL
-        CREATE UNLOGGED TABLE IF NOT EXISTS #{stage_table_name_sql} (
+        CREATE UNLOGGED TABLE IF NOT EXISTS "#{target_schema}"."#{stage_table_name}" (
           id integer,
           external_id character varying(255),
           transition character varying(255),
@@ -23,13 +23,13 @@ module BeetleETL
 
         #{index_definitions};
 
-        ALTER TABLE #{stage_table_name_sql}
+        ALTER TABLE "#{target_schema}"."#{stage_table_name}"
         SET (
           autovacuum_enabled = false,
           toast.autovacuum_enabled = false
         );
 
-        TRUNCATE TABLE #{stage_table_name_sql} RESTART IDENTITY CASCADE;
+        TRUNCATE TABLE "#{target_schema}"."#{stage_table_name}" RESTART IDENTITY CASCADE;
       SQL
     end
 
@@ -39,7 +39,7 @@ module BeetleETL
       definitions = [
         payload_column_definitions,
         relation_column_definitions
-      ].compact
+      ].flatten
 
       if definitions.empty?
         raise NoColumnsDefinedError.new <<-MSG
@@ -52,35 +52,29 @@ module BeetleETL
     end
 
     def payload_column_definitions
-      definitions = (@column_names - @relations.keys).map do |column_name|
+      (@column_names - @relations.keys).map do |column_name|
         "#{column_name} #{column_type(column_name)}"
       end
-      definitions.join(',') if definitions.any?
     end
 
     def relation_column_definitions
-      definitions = @relations.map do |foreign_key_name, table|
+      @relations.map do |foreign_key_name, table|
         <<-SQL
           #{foreign_key_name} integer,
           external_#{foreign_key_name} character varying(255)
         SQL
       end
-      definitions.join(',') if definitions.any?
     end
 
     def index_definitions
       index_columns = [:external_id] + @relations.keys.map { |c| "external_#{c}" }
       index_columns.map do |column_name|
-        "CREATE INDEX ON #{stage_table_name_sql} (#{column_name})"
+        %Q[CREATE INDEX ON "#{target_schema}"."#{stage_table_name}" (#{column_name});]
       end.join(";")
     end
 
     def column_type(column_name)
-      @column_types ||= Hash[database.schema(target_table_name.to_sym)]
-        .reduce({}) do |acc, (name, schema)|
-          acc[name.to_sym] = schema.fetch(:db_type)
-          acc
-        end
+      @column_types ||= database.column_types(target_schema, table_name)
 
       unless @column_types.has_key?(column_name)
         raise ColumnDefinitionNotFoundError.new <<-MSG

data/lib/beetle_etl/steps/drop_stage.rb

@@ -3,7 +3,7 @@ module BeetleETL
 
     def run
       database.execute <<-SQL
-        DROP TABLE IF EXISTS #{stage_table_name_sql}
+        DROP TABLE IF EXISTS "#{target_schema}"."#{stage_table_name}";
       SQL
     end
 

data/lib/beetle_etl/steps/load.rb

@@ -25,26 +25,26 @@ module BeetleETL
       just_now = now
 
       database.execute <<-SQL
-        INSERT INTO #{target_table_name_sql}
+        INSERT INTO "#{target_schema}"."#{table_name}"
           (#{data_columns.join(', ')}, external_source, created_at, updated_at)
         SELECT
           #{data_columns.join(', ')},
           '#{external_source}',
           '#{just_now}',
           '#{just_now}'
-        FROM #{stage_table_name_sql}
+        FROM "#{target_schema}"."#{stage_table_name}"
         WHERE transition = 'CREATE'
       SQL
     end
 
     def load_update
       database.execute <<-SQL
-        UPDATE #{target_table_name_sql} target
+        UPDATE "#{target_schema}"."#{table_name}" target
         SET
           #{updatable_columns.map { |c| %Q("#{c}" = stage."#{c}") }.join(',')},
           "updated_at" = '#{now}',
           deleted_at = NULL
-        FROM #{stage_table_name_sql} stage
+        FROM "#{target_schema}"."#{stage_table_name}" stage
         WHERE stage.id = target.id
           AND stage.transition IN ('UPDATE', 'REINSTATE')
       SQL
@@ -54,11 +54,11 @@ module BeetleETL
       just_now = now
 
       database.execute <<-SQL
-        UPDATE #{target_table_name_sql} target
+        UPDATE "#{target_schema}"."#{table_name}" target
         SET
           updated_at = '#{just_now}',
           deleted_at = '#{just_now}'
-        FROM #{stage_table_name_sql} stage
+        FROM "#{target_schema}"."#{stage_table_name}" stage
         WHERE stage.id = target.id
           AND stage.transition = 'DELETE'
       SQL
@@ -71,7 +71,7 @@ module BeetleETL
     end
 
     def table_columns
-      @table_columns ||= database[stage_table_name.to_sym].columns
+      @table_columns ||= database.column_names(target_schema, stage_table_name)
     end
 
     def ignored_columns

data/lib/beetle_etl/steps/map_relations.rb

@@ -14,9 +14,9 @@ module BeetleETL
     def run
       @relations.map do |foreign_key_column, foreign_table_name|
         database.execute <<-SQL
-          UPDATE #{stage_table_name_sql} current_table
+          UPDATE "#{target_schema}"."#{stage_table_name}" current_table
           SET #{foreign_key_column} = foreign_table.id
-          FROM #{stage_table_name_sql(foreign_table_name)} foreign_table
+          FROM "#{target_schema}"."#{stage_table_name(foreign_table_name)}" foreign_table
           WHERE current_table.external_#{foreign_key_column} = foreign_table.external_id
         SQL
       end

data/lib/beetle_etl/steps/step.rb

@@ -29,23 +29,12 @@ module BeetleETL
       @config.database
     end
 
-    # naming
-
-    def stage_table_name
-      BeetleETL::Naming.stage_table_name(@config.external_source, @table_name)
-    end
-
-    def stage_table_name_sql(table_name = nil)
-      table_name ||= @table_name
-      BeetleETL::Naming.stage_table_name_sql(@config.external_source, table_name)
-    end
-
-    def target_table_name
-      BeetleETL::Naming.target_table_name(@config.target_schema, @table_name)
+    def target_schema
+      @config.target_schema
     end
 
-    def target_table_name_sql
-      BeetleETL::Naming.target_table_name_sql(@config.target_schema, @table_name)
+    def stage_table_name(table_name = nil)
+      BeetleETL::Naming.stage_table_name(external_source, table_name || @table_name)
     end
 
   end

data/lib/beetle_etl/steps/table_diff.rb

@@ -18,11 +18,11 @@ module BeetleETL
 
     def transition_create
       database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage
         SET transition = 'CREATE'
         WHERE NOT EXISTS (
           SELECT 1
-          FROM #{target_table_name} target
+          FROM "#{target_schema}"."#{table_name}" target
           WHERE target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
         )
@@ -31,11 +31,11 @@ module BeetleETL
 
     def transition_update
       database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage
         SET transition = 'UPDATE'
         WHERE EXISTS (
           SELECT 1
-          FROM #{target_table_name} target
+          FROM "#{target_schema}"."#{table_name}" target
           WHERE target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
           AND target.deleted_at IS NULL
@@ -49,13 +49,13 @@ module BeetleETL
 
     def transition_delete
       database.execute <<-SQL
-        INSERT INTO #{stage_table_name_sql}
+        INSERT INTO "#{target_schema}"."#{stage_table_name}"
           (external_id, transition)
         SELECT
           target.external_id,
           'DELETE'
-        FROM #{target_table_name_sql} target
-        LEFT OUTER JOIN #{stage_table_name_sql} stage
+        FROM "#{target_schema}"."#{table_name}" target
+        LEFT OUTER JOIN "#{target_schema}"."#{stage_table_name}" stage
           ON (stage.external_id = target.external_id)
         WHERE stage.external_id IS NULL
         AND target.external_source = '#{external_source}'
@@ -65,11 +65,11 @@ module BeetleETL
 
     def transition_reinstate
       database.execute <<-SQL
-        UPDATE #{stage_table_name_sql} stage
+        UPDATE "#{target_schema}"."#{stage_table_name}" stage
         SET transition = 'REINSTATE'
         WHERE EXISTS (
           SELECT 1
-          FROM #{target_table_name_sql} target
+          FROM "#{target_schema}"."#{table_name}" target
           WHERE target.external_id = stage.external_id
           AND target.external_source = '#{external_source}'
           AND target.deleted_at IS NOT NULL
@@ -92,7 +92,7 @@ module BeetleETL
     end
 
     def table_columns
-      @table_columns ||= database[stage_table_name.to_sym].columns
+      @table_columns ||= database.column_names(target_schema, stage_table_name)
     end
 
     def ignored_columns

data/lib/beetle_etl/steps/transform.rb

@@ -12,7 +12,7 @@ module BeetleETL
     end
 
     def run
-      database.run(@query)
+      database.execute(@query)
     end
 
   end

data/lib/beetle_etl/testing.rb

@@ -13,7 +13,7 @@ module BeetleETL
 
     def with_stage_tables_for(*table_names, &block)
       table_names.each do |table_name|
-        unless @@config.database.table_exists?(table_name)
+        unless @@config.database.table_exists?(@@config.target_schema, table_name)
           raise TargetTableNotFoundError.new <<-MSG
             Missing target table "#{table_name}".
             In order to create stage tables, BeetleETL requires the target tables to exist because they provide the column definitions.

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/spec/adapters/sequel_adapter_spec.rb

@@ -0,0 +1,10 @@
+require "spec_helper"
+require_relative "shared_examples"
+
+module BeetleETL
+  describe SequelAdapter do
+    it_behaves_like "database adapter" do
+      let(:adapter) { SequelAdapter.new(test_database) }
+    end
+  end
+end

data/spec/adapters/shared_examples.rb

@@ -0,0 +1,58 @@
+require "spec_helper"
+
+shared_examples "database adapter" do
+  before do
+    test_database.run <<-SQL
+      CREATE SCHEMA foo;
+      CREATE TABLE foo.persons (
+        id int,
+        first_name varchar(255),
+        last_name varchar(255)
+      );
+    SQL
+  end
+
+  after do
+    test_database.run <<-SQL
+      DROP SCHEMA foo CASCADE;
+    SQL
+  end
+
+  describe "#execute" do
+    it "executes SQL" do
+      adapter.execute <<-SQL
+          INSERT INTO foo.persons VALUES (1, 'hugo', 'warzenkopp');
+      SQL
+
+      expect(Sequel.qualify("foo", "persons")).to have_values(
+        [ :id , :first_name , :last_name   ],
+        [ 1   , "hugo"      , "warzenkopp" ]
+      )
+    end
+  end
+
+  describe "#column_names" do
+    it "returns a tables column names" do
+      expect(adapter.column_names("foo", "persons")).to match_array([
+        :id, :first_name, :last_name
+      ])
+    end
+  end
+
+  describe "#column_types" do
+    it "returns a tables column names" do
+      expect(adapter.column_types("foo", "persons")).to match({
+        id: 'integer',
+        first_name: 'character varying(255)',
+        last_name: 'character varying(255)'
+      })
+    end
+  end
+
+  describe "#table_exists?" do
+    it "returns whether a table exists" do
+      expect(adapter.table_exists?("foo", "persons")).to eql(true)
+      expect(adapter.table_exists?("foo", "persons200")).to eql(false)
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -1,27 +1,27 @@
 require 'spec_helper'
+require 'yaml'
 
 module BeetleETL
   describe Configuration do
 
     subject { Configuration.new }
 
-    describe "#database" do
-      let(:database) { double(:database) }
+    let(:database_config) do
+      config_path = File.expand_path('../support/database.yml', __FILE__)
+      YAML.load(File.read(config_path))
+    end
 
-      it "returns the object if present" do
-        subject.database = database
+    describe "#database" do
+     it "builds a SequelAdapter when passed a Sequel::Database" do
+        subject.database = test_database
 
-        expect(subject.database).to eql(database)
+        expect { subject.database.execute("SELECT 1") }.not_to raise_error
       end
 
-      it "builds a Sequel Database from config when no database is passed" do
-        database_config = double(:database_config)
+      it "builds a SequelAdapter from config when no database is passed" do
         subject.database_config = database_config
 
-        expect(Sequel).to receive(:connect).with(database_config).once { database }
-
-        expect(subject.database).to eql(database)
-        expect(subject.database).to eql(database)
+        expect { subject.database.execute("SELECT 1") }.not_to raise_error
       end
 
       it "raises an error if no database or database_config is passed" do
@@ -31,30 +31,26 @@ module BeetleETL
     end
 
     describe "#disconnect_database" do
-      let(:database) { double(:database) }
-
       it "disconnects from database if database_config was passed" do
-        database_config = double(:database_config)
+        subject.database_config = database_config
 
-        expect(Sequel).to receive(:connect).with(database_config) { database }
-        expect(database).to receive(:disconnect)
+        expect(subject.database).to receive(:disconnect)
 
-        subject.database_config = database_config
         subject.disconnect_database
       end
 
       it "does not disconnect from database if database object was passed" do
-        expect(database).not_to receive(:disconnect)
+        subject.database = test_database
+
+        expect(subject.database).not_to receive(:disconnect)
 
-        subject.database = database
         subject.disconnect_database
       end
     end
 
     describe "#target_schema" do
-      it "returns nil if target_schema is 'public'" do
-        subject.target_schema = "public"
-        expect(subject.target_schema).to be_nil
+      it "returns 'public' by default" do
+        expect(subject.target_schema).to eql("public")
       end
 
       it "returns target_schema if target_schema is not 'public'" do

data/spec/dsl/dsl_spec.rb

@@ -6,6 +6,7 @@ module BeetleETL
     let(:config) do
       Configuration.new.tap do |c|
         c.external_source = "bar"
+        c.target_schema = "baz_schema"
       end
     end
 
@@ -14,13 +15,13 @@ module BeetleETL
     describe '#stage_table' do
       it 'returns the current stage table name' do
         expect(subject.stage_table).to eql(
-          BeetleETL::Naming.stage_table_name_sql("bar", :foo_table)
+          %Q["baz_schema"."#{BeetleETL::Naming.stage_table_name("bar", :foo_table)}"]
         )
       end
 
       it 'returns the stage table name for the given table' do
         expect(subject.stage_table(:bar_table)).to eql(
-          BeetleETL::Naming.stage_table_name_sql("bar", :bar_table)
+          %Q["baz_schema"."#{BeetleETL::Naming.stage_table_name("bar", :bar_table)}"]
         )
       end
     end

data/spec/feature/example_schema.rb

@@ -13,7 +13,7 @@ module ExampleSchema
   def create_source_tables
     test_database.create_schema :source
 
-    test_database.create_table :source__Organisation do
+    test_database.create_table Sequel.qualify("source", "Organisation") do
       Integer :pkOrgId
       String :Name, size: 255
       String :Adresse, size: 255
@@ -26,7 +26,9 @@ module ExampleSchema
   end
 
   def create_target_tables
-    test_database.create_table :organisations do
+    test_database.create_schema :my_target
+
+    test_database.create_table Sequel.qualify("my_target", "organisations") do
       primary_key :id
       String :external_id, size: 255
       String :external_source, size: 255
@@ -37,12 +39,12 @@ module ExampleSchema
       DateTime :deleted_at
     end
 
-    test_database.create_table :departments do
+    test_database.create_table Sequel.qualify("my_target", "departments") do
       primary_key :id
       String :external_id, size: 255
       String :external_source, size: 255
       String :name, size: 255
-      foreign_key :organisation_id, :organisations
+      foreign_key :organisation_id, Sequel.qualify("my_target", "organisations")
       DateTime :created_at
       DateTime :updated_at
       DateTime :deleted_at
@@ -50,8 +52,7 @@ module ExampleSchema
   end
 
   def drop_target_tables
-    test_database.drop_table :departments
-    test_database.drop_table :organisations
+    test_database.drop_schema :my_target, cascade: true
   end
 
 end

data/spec/feature/feature_spec.rb

@@ -27,6 +27,7 @@ describe BeetleETL do
       c.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
       c.database_config = database_config
       c.external_source = 'source_name'
+      c.target_schema = 'my_target'
       c.logger = Logger.new(Tempfile.new("log"))
     end
   end
@@ -45,7 +46,7 @@ describe BeetleETL do
 
   def import1
     # create
-    insert_into(:source__Organisation).values(
+    insert_into(Sequel.qualify("source", "Organisation")).values(
       [ :pkOrgId , :Name    , :Adresse        , :Abteilung ] ,
       [ 1        , 'Apple'  , 'Apple Street'  , 'iPhone'   ] ,
       [ 2        , 'Apple'  , 'Apple Street'  , 'MacBook'  ] ,
@@ -57,14 +58,14 @@ describe BeetleETL do
       BeetleETL.import(@config)
     end
 
-    expect(:organisations).to have_values(
+    expect(Sequel.qualify("my_target", "organisations")).to have_values(
       [ :id                       , :external_id , :external_source , :name    , :address        , :created_at , :updated_at , :deleted_at ] ,
       [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'  , time1       , time1       , nil         ] ,
       [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'Google Street' , time1       , time1       , nil         ] ,
       [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'Audi Street'   , time1       , time1       , nil         ]
     )
 
-    expect(:departments).to have_values(
+    expect(Sequel.qualify("my_target", "departments")).to have_values(
       [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
       [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
       [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
@@ -72,12 +73,12 @@ describe BeetleETL do
       [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time1       , nil         ] ,
     )
 
-    test_database[:source__Organisation].truncate
+    test_database[Sequel.qualify("source", "Organisation")].truncate
   end
 
   def import2
     # keep, update, delete
-    insert_into(:source__Organisation).values(
+    insert_into(Sequel.qualify("source", "Organisation")).values(
       [ :pkOrgId , :Name    , :Adresse            , :Abteilung ] ,
       [ 1        , 'Apple'  , 'Apple Street'      , 'iPhone'   ] ,
       [ 2        , 'Apple'  , 'Apple Street'      , 'MacBook'  ] ,
@@ -89,14 +90,14 @@ describe BeetleETL do
       BeetleETL.import(@config)
     end
 
-    expect(:organisations).to have_values(
+    expect(Sequel.qualify("my_target", "organisations")).to have_values(
       [ :id                       , :external_id , :external_source , :name    , :address            , :created_at , :updated_at , :deleted_at ] ,
       [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'      , time1       , time1       , nil         ] ,
       [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'NEW Google Street' , time1       , time2       , nil         ] ,
       [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'Audi Street'       , time1       , time2       , time2       ]
     )
 
-    expect(:departments).to have_values(
+    expect(Sequel.qualify("my_target", "departments")).to have_values(
       [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
       [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
       [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
@@ -104,12 +105,12 @@ describe BeetleETL do
       [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time2       , time2       ] ,
     )
 
-    test_database[:source__Organisation].truncate
+    test_database[Sequel.qualify("source", "Organisation")].truncate
   end
 
   def import3
     # reinstate with update
-    insert_into(:source__Organisation).values(
+    insert_into(Sequel.qualify("source", "Organisation")).values(
       [ :pkOrgId , :Name    , :Adresse            , :Abteilung ] ,
       [ 1        , 'Apple'  , 'Apple Street'      , 'iPhone'   ] ,
       [ 2        , 'Apple'  , 'Apple Street'      , 'MacBook'  ] ,
@@ -121,14 +122,14 @@ describe BeetleETL do
       BeetleETL.import(@config)
     end
 
-    expect(:organisations).to have_values(
+    expect(Sequel.qualify("my_target", "organisations")).to have_values(
       [ :id                       , :external_id , :external_source , :name    , :address            , :created_at , :updated_at , :deleted_at ] ,
       [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'      , time1       , time1       , nil         ] ,
       [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'NEW Google Street' , time1       , time2       , nil         ] ,
       [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'NEW Audi Street'   , time1       , time3       , nil         ]
     )
 
-    expect(:departments).to have_values(
+    expect(Sequel.qualify("my_target", "departments")).to have_values(
       [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
       [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
       [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
@@ -136,15 +137,15 @@ describe BeetleETL do
       [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time3       , nil         ] ,
     )
 
-    test_database[:source__Organisation].truncate
+    test_database[Sequel.qualify("source", "Organisation")].truncate
   end
 
   def organisation_id(external_id)
-    test_database[:organisations].first(external_id: external_id)[:id]
+    test_database[Sequel.qualify("my_target", "organisations")].first(external_id: external_id)[:id]
   end
 
   def department_id(external_id)
-    test_database[:departments].first(external_id: external_id)[:id]
+    test_database[Sequel.qualify("my_target", "departments")].first(external_id: external_id)[:id]
   end
 
 end

data/spec/spec_helper.rb

@@ -1,6 +1,5 @@
 require "byebug"
-require "codeclimate-test-reporter"
-CodeClimate::TestReporter.start
+require "ostruct"
 
 require_relative "../lib/beetle_etl.rb"
 require_relative "support/database_helpers.rb"
@@ -19,10 +18,9 @@ RSpec.configure do |config|
     else
       test_database.transaction do
         example.run
-        raise Sequel::Error::Rollback
+        raise Sequel::Rollback
       end
     end
   end
 
 end
-

data/spec/steps/assign_ids_spec.rb

@@ -7,11 +7,12 @@ module BeetleETL
     let(:another_source) { 'another_source' }
 
     let(:config) do
-      Configuration.new.tap do |c|
-        c.stage_schema = 'stage'
-        c.external_source = external_source
-        c.database = test_database
-      end
+      OpenStruct.new({
+        stage_schema: 'stage',
+        target_schema: 'public',
+        external_source: external_source,
+        database: test_database,
+      })
     end
 
     subject { AssignIds.new(config, :example_table) }

data/spec/steps/create_stage_spec.rb

@@ -102,14 +102,14 @@ module BeetleETL
       it "truncates the stage table if it already exists" do
         CreateStage.new(config, :example_table, {}, @columns).run
 
-        insert_into(subject.stage_table_name.to_sym).values(
+        insert_into(Sequel.qualify("public", subject.stage_table_name)).values(
           [ :some_string , :some_integer , :some_float ] ,
           [ "hello"     , 123           , 123.456      ]
         )
 
         CreateStage.new(config, :example_table, {}, @columns).run
 
-        expect(subject.stage_table_name).to have_values(
+        expect(Sequel.qualify("public", subject.stage_table_name)).to have_values(
           [:some_string, :some_integer, :some_float]
         )
       end

data/spec/steps/map_relations_spec.rb

@@ -4,10 +4,11 @@ module BeetleETL
   describe MapRelations do
 
     let(:config) do
-      Configuration.new.tap do |c|
-        c.external_source = 'my_source'
-        c.database = test_database
-      end
+      OpenStruct.new({
+        external_source: 'my_source',
+        target_schema: 'public',
+        database: test_database
+      })
     end
 
     let(:dependee_a) do
@@ -75,15 +76,14 @@ module BeetleETL
           [ 26  , 'b_id'       ] ,
         )
 
-        insert_into(subject.stage_table_name.to_sym).values(
+        insert_into(Sequel.qualify("public", subject.stage_table_name)).values(
           [ :external_dependee_a_id , :external_dependee_b_id ] ,
           [ 'a_id'                  , 'b_id'                  ] ,
         )
 
-
         subject.run
 
-        expect(subject.stage_table_name.to_sym).to have_values(
+        expect(Sequel.qualify("public", subject.stage_table_name)).to have_values(
           [ :dependee_a_id , :dependee_b_id ] ,
           [ 1              , 26             ] ,
         )

data/spec/steps/step_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 module BeetleETL
   describe Step do
 
-    let(:config) { Configuration.new }
+    let(:config) { OpenStruct.new }
 
     subject { Step.new(config, :example_table) }
     FooStep = Class.new(Step)

data/spec/steps/transform_spec.rb

@@ -5,9 +5,9 @@ module BeetleETL
 
     let(:database) { double(:database) }
     let(:config) do
-      Configuration.new.tap do |c|
-        c.database = database
-      end
+      OpenStruct.new({
+        database: database
+      })
     end
     let(:query) { double(:query) }
 
@@ -29,7 +29,7 @@ module BeetleETL
 
     describe '#run' do
       it 'runs a query in the database' do
-        expect(database).to receive(:run).with(query)
+        expect(database).to receive(:execute).with(query)
 
         subject.run
       end

data/spec/support/database_helpers.rb

@@ -37,7 +37,7 @@ end
 
 RSpec::Matchers.define :have_values do |*rows|
   match do |table_description|
-    dataset = test_database[table_description.to_sym]
+    dataset = test_database[table_description]
 
     columns = rows[0].map(&:to_sym)
     values = rows[1..-1]

data/spec/testing_spec.rb

@@ -75,7 +75,7 @@ describe "BeetleETL:Testing" do
       with_stage_tables_for(:organisations, :some_table) do
         run_transformation(:organisations)
 
-        expect(stage_table_name(:organisations)).to have_values(
+        expect(Sequel.qualify("public", stage_table_name(:organisations))).to have_values(
           [ :external_id  , :address  , :name  ] ,
           [ "external_id" , "address" , "name" ]
         )

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-19 00:00:00.000000000 Z
+date: 2017-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,13 +25,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -116,7 +116,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".byebug_history"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
@@ -125,6 +124,7 @@ files:
 - Rakefile
 - beetle_etl.gemspec
 - lib/beetle_etl.rb
+- lib/beetle_etl/adapters/sequel_adapter.rb
 - lib/beetle_etl/configuration.rb
 - lib/beetle_etl/dsl/dsl.rb
 - lib/beetle_etl/dsl/transformation.rb
@@ -146,6 +146,8 @@ files:
 - lib/beetle_etl/testing/test_wrapper.rb
 - lib/beetle_etl/version.rb
 - script/postgres
+- spec/adapters/sequel_adapter_spec.rb
+- spec/adapters/shared_examples.rb
 - spec/beetle_etl_spec.rb
 - spec/configuration_spec.rb
 - spec/dsl/dsl_spec.rb
@@ -189,11 +191,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: BeetleETL helps you with your recurring ETL imports.
 test_files:
+- spec/adapters/sequel_adapter_spec.rb
+- spec/adapters/shared_examples.rb
 - spec/beetle_etl_spec.rb
 - spec/configuration_spec.rb
 - spec/dsl/dsl_spec.rb

data/.byebug_history

@@ -1,8 +0,0 @@
-continue
-backtrace
-stack
-trace
-c
-continue
-c
-target_table_name