beetle_etl 0.0.1

data/spec/feature/example_transform.rb

@@ -0,0 +1,37 @@
+import :organisations do
+  query <<-SQL
+    INSERT INTO #{stage_table} (
+      external_id,
+      import_run_id,
+      name
+    )
+
+    SELECT DISTINCT
+      o."Name",
+      #{import_run_id},
+      o."Name"
+
+    FROM source."Organisation" o
+  SQL
+end
+
+import :departments do
+  references :organisations, on: :organisation_id
+
+  query <<-SQL
+    INSERT INTO #{stage_table} (
+      external_id,
+      import_run_id,
+      name,
+      external_organisation_id
+    )
+
+    SELECT
+      #{combined_key('o."Name"', 'o."pkOrgId"')},
+      #{import_run_id},
+      o."Abteilung",
+      o."Name"
+
+    FROM source."Organisation" o
+  SQL
+end

data/spec/feature/feature_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require_relative 'example_schema'
+require 'yaml'
+
+require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/numeric/time'
+
+describe BeetleETL do
+
+  include ExampleSchema
+
+  let!(:now) { Time.new(2014, 07, 17, 16, 12).beginning_of_day }
+  before { allow(Time).to receive(:now) { now } }
+
+  before { create_tables }
+  after { drop_tables }
+
+  it 'is a working', :feature do
+    insert_into(:source__Organisation).values(
+      [ :pkOrgId , :Name   , :Abteilung ] ,
+      [ 1        , 'Apple' , 'iPhone'   ] ,
+      [ 2        , 'Apple' , 'MacBook'  ] ,
+    )
+
+    BeetleETL.configure do |config|
+      config.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
+      config.database = test_database
+      config.external_source = 'source_name'
+      config.stage_schema = 'stage'
+    end
+
+
+    BeetleETL.import
+
+
+    expect(:organisations).to have_values(
+      [ :id , :external_id , :external_source , :name   , :created_at , :updated_at , :deleted_at ] ,
+      [ 1   , 'Apple'      , 'source_name'    , 'Apple' , now         , now         , nil         ]
+    )
+
+    expect(:departments).to have_values(
+      [ :id , :external_id , :external_source , :name     , :organisation_id , :created_at , :updated_at , :deleted_at ] ,
+      [ 1   , '[Apple,1]'  , 'source_name'    , 'iPhone'  , 1                , now         , now         , nil         ] ,
+      [ 2   , '[Apple,2]'  , 'source_name'    , 'MacBook' , 1                , now         , now         , nil         ] ,
+    )
+  end
+
+end

data/spec/import_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe Import do
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+
+require_relative '../lib/beetle_etl.rb'
+require_relative 'support/database_helpers.rb'
+
+RSpec.configure do |config|
+
+  config.include SpecSupport::DatabaseHelpers
+  config.backtrace_exclusion_patterns = [/rspec-core/]
+
+  config.around(:each) do |example|
+    BeetleETL.reset
+    if example.metadata[:feature]
+      example.run
+    else
+      test_database.transaction do
+        example.run
+        raise Sequel::Error::Rollback
+      end
+    end
+  end
+
+end
+

data/spec/state_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+
+require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/numeric/time'
+
+module BeetleETL
+  describe State do
+    subject { State.new }
+
+    before do
+      BeetleETL.configure do |config|
+        config.stage_schema = 'stage'
+        config.database = test_database
+      end
+
+      test_database.create_schema 'stage'
+      test_database.create_table :stage__import_runs do
+        primary_key :id
+        String :state, size: 10, null: false
+        DateTime :started_at, null: false
+        DateTime :finished_at
+      end
+    end
+
+    describe '#start_import' do
+      let(:now) { 1.minute.ago.beginning_of_day }
+
+      it 'registers a new import in the import_runs table' do
+        allow(subject).to receive(:now) { now }
+
+        subject.start_import
+
+        expect(:stage__import_runs).to have_values(
+          [ :id , :state    , :started_at , :finished_at ] ,
+          [ 1   , 'RUNNING' , now         , nil          ]
+        )
+      end
+
+      it 'raises an exception if there is alreay an import marked as running' do
+        insert_into(:stage__import_runs).values(
+          [ :id , :state    , :started_at , :finished_at ] ,
+          [ 1   , 'RUNNING' , now         , nil          ]
+        )
+
+        expect { subject.start_import }.to raise_exception(BeetleETL::ImportAleadyRunning)
+      end
+    end
+
+    context 'run ids' do
+      before do
+        insert_into(:stage__import_runs).values(
+          [ :state      , :started_at , :finished_at ] ,
+          [ 'FAILED'    , 8.days.ago  , 7.days.ago   ] ,
+          [ 'SUCCEEDED' , 6.days.ago  , 5.day.ago    ] ,
+          [ 'SUCCEEDED' , 4.days.ago  , 3.days.ago   ] ,
+          [ 'FAILED'    , 2.days.ago  , 1.day.ago    ] ,
+        )
+      end
+
+      describe '#run_id' do
+        it 'returns the import‘s id after it has been started' do
+          subject.start_import
+          expect(subject.run_id).to eql(5)
+        end
+
+        it 'raises an exception when the import has not been started' do
+          expect { subject.run_id }.to raise_exception(BeetleETL::ImportNotRunning)
+        end
+      end
+
+      describe '#last_run_id' do
+        it 'returns nil if there is no last successful import' do
+          test_database[:stage__import_runs].update(state: 'FAILED')
+
+          subject.start_import
+          expect(subject.last_run_id).to be_nil
+        end
+
+        it 'returns the id of the last successul import' do
+          subject.start_import
+          expect(subject.last_run_id).to eql(3)
+        end
+      end
+    end
+
+    context 'marking imports' do
+      let(:now) { 1.minute.ago.beginning_of_day }
+      let(:one_day_ago) { 1.day.ago.beginning_of_day }
+
+      before do
+        insert_into(:stage__import_runs).values(
+          [ :state      , :started_at , :finished_at ] ,
+          [ 'SUCCEEDED' , 2.days.ago  , one_day_ago  ] ,
+        )
+        allow(subject).to receive(:now) { now }
+        subject.start_import
+      end
+
+      describe '#mark_as_failed' do
+        it 'marks the current import as FAILED' do
+          subject.mark_as_failed
+
+          expect(:stage__import_runs).to have_values(
+            [ :id , :state      , :finished_at ] ,
+            [ 1   , 'SUCCEEDED' , one_day_ago  ] ,
+            [ 2   , 'FAILED'    , now          ] ,
+          )
+        end
+      end
+
+      describe '#mark_as_succeeded' do
+        it 'marks the current import as SUCCEEDED' do
+          subject.mark_as_succeeded
+
+          expect(:stage__import_runs).to have_values(
+            [ :id , :state      , :finished_at ] ,
+            [ 1   , 'SUCCEEDED' , one_day_ago  ] ,
+            [ 2   , 'SUCCEEDED' , now          ] ,
+          )
+        end
+      end
+    end
+  end
+end

data/spec/steps/assign_ids_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe AssignIds do
+
+    let(:run_id) { 1 }
+    let(:external_source) { 'my_source' }
+    subject { AssignIds.new(:example_table) }
+
+    before do
+      BeetleETL.configure do |config|
+        config.stage_schema = 'stage'
+        config.external_source = external_source
+        config.database = test_database
+      end
+
+      allow(BeetleETL).to receive(:state) { double(:state, run_id: run_id) }
+
+      test_database.create_schema(:stage)
+      test_database.create_table(:stage__example_table) do
+        Integer :id
+        Integer :import_run_id
+        String :external_id, size: 255
+        String :transition, size: 255
+      end
+
+      test_database.create_table(:example_table) do
+        primary_key :id
+        String :external_id, size: 255
+        String :external_source, size: 255
+      end
+
+    end
+
+    describe '#dependencies' do
+      it 'depends on TableDiff of the same table' do
+        expect(subject.dependencies).to eql(['example_table: TableDiff'].to_set)
+      end
+    end
+
+    describe '#run' do
+      it 'runs all transitions' do
+        %i(assign_new_ids map_existing_ids).each do |method|
+          expect(subject).to receive(method)
+        end
+
+        subject.run
+      end
+    end
+
+    describe '#assign_new_ids' do
+      it 'generates new ids for newly created records' do
+        insert_into(:example_table).values(
+          [ :external_id , :external_source ] ,
+          [ 'keep_id'    , external_source  ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'create_id'  , 'CREATE'    ] ,
+          [ run_id         , 'keep_id'    , 'KEEP'      ] ,
+        )
+
+        subject.assign_new_ids
+
+        expect(:stage__example_table).to have_values(
+          [ :id , :import_run_id , :external_id , :transition ] ,
+          [ 2   , run_id         , 'create_id'  , 'CREATE'    ] ,
+          [ nil , run_id         , 'keep_id'    , 'KEEP'      ] ,
+        )
+      end
+    end
+
+    describe '#map_existing_ids' do
+      it 'assigns ids for existing records by their external id' do
+        insert_into(:example_table).values(
+          [ :external_id  , :external_source ] ,
+          [ 'keep_id'     , external_source  ] ,
+          [ 'update_id'   , external_source  ] ,
+          [ 'delete_id'   , external_source  ] ,
+          [ 'undelete_id' , external_source  ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id  , :transition ] ,
+          [ run_id         , 'create_id'   , 'CREATE'    ] ,
+          [ run_id         , 'keep_id'     , 'KEEP'      ] ,
+          [ run_id         , 'update_id'   , 'UPDATE'    ] ,
+          [ run_id         , 'delete_id'   , 'DELETE'    ] ,
+          [ run_id         , 'undelete_id' , 'UNDELETE'  ] ,
+        )
+
+        subject.map_existing_ids
+
+        expect(:stage__example_table).to have_values(
+          [ :id , :import_run_id , :external_id  , :transition ] ,
+          [ nil , run_id         , 'create_id'   , 'CREATE'    ] ,
+          [ 1   , run_id         , 'keep_id'     , 'KEEP'      ] ,
+          [ 2   , run_id         , 'update_id'   , 'UPDATE'    ] ,
+          [ 3   , run_id         , 'delete_id'   , 'DELETE'    ] ,
+          [ 4   , run_id         , 'undelete_id' , 'UNDELETE'  ] ,
+        )
+      end
+    end
+
+  end
+end

data/spec/steps/load_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper'
+
+require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/numeric/time'
+
+module BeetleETL
+  describe Load do
+
+    let(:run_id) { 1 }
+    let(:old_run_id) { 5000 }
+    let(:external_source) { 'my_source' }
+
+    let(:now) { Time.now.beginning_of_day }
+    let(:yesterday) { 1.day.ago.beginning_of_day }
+
+    subject { Load.new(:example_table) }
+
+    before do
+      BeetleETL.configure do |config|
+        config.stage_schema = 'stage'
+        config.external_source = external_source
+        config.database = test_database
+      end
+
+      allow(BeetleETL).to receive(:state) { double(:state, run_id: run_id) }
+      allow(subject).to receive(:now) { now }
+
+      test_database.create_schema(:stage)
+      test_database.create_table(:stage__example_table) do
+        Integer :import_run_id
+        Integer :id
+        String :external_id, size: 255
+        String :transition, size: 20
+
+        String :external_foo_id, size: 255
+        Integer :foo_id
+
+        String :payload, size: 255
+      end
+
+      test_database.create_table(:example_table) do
+        primary_key :id
+        String :external_id, size: 255
+        String :external_source, size: 255
+        DateTime :created_at
+        DateTime :updated_at
+        DateTime :deleted_at
+
+        String :payload, size: 255
+        String :ignored_attribute, size: 255
+        Integer :foo_id
+      end
+    end
+
+    describe '#run' do
+      it 'runs all load steps' do
+        %w(create update delete undelete).each do |transition|
+          expect(subject).to receive(:"load_#{transition}")
+        end
+
+        subject.run
+      end
+    end
+
+    describe '#load_create' do
+      it 'loads records into the public table' do
+        insert_into(:stage__example_table).values(
+          [ :id , :import_run_id , :external_id  , :transition , :external_foo_id , :foo_id , :payload       ] ,
+          [ 3   , old_run_id     , 'external_id' , 'CREATE'    , 'foo_id'         , 999     , 'some content' ] ,
+          [ 3   , run_id         , 'external_id' , 'CREATE'    , 'foo_id'         , 22      , 'content'      ] ,
+        )
+
+        subject.load_create
+
+        expect(:example_table).to have_values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 3   , 'external_id' , external_source  , 22      , now         , now         , nil         , 'content' ] ,
+        )
+      end
+    end
+
+    describe '#load_update' do
+      it 'updates existing records' do
+        insert_into(:example_table).values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , yesterday   , nil         , 'content' ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :id , :import_run_id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
+          [ 1   , old_run_id     , 'external_id' , 'UPDATE'    , 'foo_id'         , 999     , 'some content'    ] ,
+          [ 1   , run_id         , 'external_id' , 'UPDATE'    , 'foo_id'         , 33      , 'updated content' ] ,
+        )
+
+        subject.load_update
+
+        expect(:example_table).to have_values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
+          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
+        )
+      end
+    end
+
+    describe '#load_delete' do
+      it 'marks existing records as deleted' do
+        insert_into(:example_table).values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , yesterday   , nil         , 'content' ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :id , :import_run_id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
+          [ 1   , old_run_id     , 'external_id' , 'UPDATE'    , 'foo_id'         , 999     , 'some content'    ] ,
+          [ 1   , run_id         , 'external_id' , 'DELETE'    , 'foo_id'         , 33      , 'updated content' ] ,
+        )
+
+        subject.load_delete
+
+        expect(:example_table).to have_values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , now         , now         , 'content' ] ,
+        )
+      end
+    end
+
+    describe '#load_undelete' do
+      it 'reinstates deleted records' do
+        insert_into(:example_table).values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , yesterday   , nil         , 'content' ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :id , :import_run_id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
+          [ 1   , old_run_id     , 'external_id' , 'UPDATE'    , 'foo_id'         , 999     , 'some content'    ] ,
+          [ 1   , run_id         , 'external_id' , 'UNDELETE'  , 'foo_id'         , 33      , 'updated content' ] ,
+        )
+
+        subject.load_undelete
+
+        expect(:example_table).to have_values(
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
+          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
+        )
+      end
+    end
+  end
+end