beetle_etl 0.0.2 → 0.0.7

data/spec/feature/example_schema.rb

@@ -2,13 +2,11 @@ module ExampleSchema
 
   def create_tables
     create_source_tables
-    create_stage_tables
     create_target_tables
   end
 
   def drop_tables
     drop_source_tables
-    drop_stage_tables
     drop_target_tables
   end
 
@@ -18,118 +16,22 @@ module ExampleSchema
     test_database.create_table :source__Organisation do
       Integer :pkOrgId
       String :Name, size: 255
+      String :Adresse, size: 255
       String :Abteilung, size: 255
     end
-
-    test_database.create_table :source__Person do
-      Integer :pkPersID
-      String :Vorname, size: 255
-      String :Nachname, size: 255
-      Integer :fkFirma
-      Integer :fkAdresse
-      Integer :fkTyp
-    end
-
-    test_database.create_table :source__Veranstaltung do
-      Integer :pkVeranstaltungId
-      Integer :fkOrganisation
-    end
-
-    test_database.create_table :source__Veranstaltungsbesuch do
-      Integer :fkVeranstaltung
-      Integer :fkBesucher
-    end
   end
 
   def drop_source_tables
     test_database.drop_schema :source, cascade: true
   end
 
-  def create_stage_tables
-    test_database.create_schema :stage
-
-    test_database.create_table :stage__import_runs do
-      primary_key :id
-      DateTime :started_at
-      DateTime :finished_at
-      String :state, size: 255
-    end
-
-    test_database.create_table :stage__organisations do
-      Integer :id
-      String :external_id, size: 255
-      foreign_key :import_run_id, :stage__import_runs
-      index [:external_id, :import_run_id]
-      String :transition, size: 255
-
-      String :name, size: 255
-    end
-
-    test_database.create_table :stage__departments do
-      Integer :id
-      String :external_id, size: 255
-      foreign_key :import_run_id, :stage__import_runs
-      index [:external_id, :import_run_id]
-      String :transition, size: 255
-
-      String :name, size: 255
-
-      String :external_organisation_id, size: 255
-      Integer :organisation_id
-
-    end
-
-    test_database.create_table :stage__attendees do
-      Integer :id
-      String :external_id, size: 255
-      foreign_key :import_run_id, :stage__import_runs
-      index [:external_id, :import_run_id]
-      String :transition, size: 255
-
-      String :first_name, size: 255
-      String :last_name, size: 255
-    end
-
-    test_database.create_table :stage__events do
-      Integer :id
-      String :external_id, size: 255
-      foreign_key :import_run_id, :stage__import_runs
-      index [:external_id, :import_run_id]
-      String :transition, size: 255
-
-      String :name, size: 255
-      DateTime :starts_at
-      DateTime :ends_at
-
-      String :external_organisations_id, size: 255
-      Integer :organisation_id
-    end
-
-    test_database.create_table :stage__attendees_events do
-      Integer :id
-      String :external_id, size: 255
-      foreign_key :import_run_id, :stage__import_runs
-      index [:external_id, :import_run_id]
-      String :transition, size: 255
-
-      String :external_attendee_id, size: 255
-      Integer :attendee_id
-
-      String :external_event_id, size: 255
-      Integer :event_id
-    end
-  end
-
-  def drop_stage_tables
-    test_database.drop_schema :stage, cascade: true
-  end
-
   def create_target_tables
     test_database.create_table :organisations do
       primary_key :id
       String :external_id, size: 255
       String :external_source, size: 255
       String :name, size: 255
+      String :address, size: 255
       DateTime :created_at
       DateTime :updated_at
       DateTime :deleted_at
@@ -145,46 +47,9 @@ module ExampleSchema
       DateTime :updated_at
       DateTime :deleted_at
     end
-
-    test_database.create_table :attendees do
-      primary_key :id
-      String :external_id, size: 255
-      String :external_source, size: 255
-      String :first_name, size: 255
-      String :last_name, size: 255
-      DateTime :created_at
-      DateTime :updated_at
-      DateTime :deleted_at
-    end
-
-    test_database.create_table :events do
-      primary_key :id
-      String :external_id, size: 255
-      String :external_source, size: 255
-      String :name, size: 255
-      DateTime :starts_at
-      DateTime :ends_at
-      foreign_key :organisation, :organisations
-      DateTime :created_at
-      DateTime :updated_at
-      DateTime :deleted_at
-    end
-
-    test_database.create_table :attendees_events do
-      foreign_key :attendee_id, :attendees, null: false
-      foreign_key :event_id, :events, null: false
-      primary_key [:attendee_id, :event_id]
-      index [:attendee_id, :event_id]
-      DateTime :created_at
-      DateTime :updated_at
-      DateTime :deleted_at
-    end
   end
 
   def drop_target_tables
-    test_database.drop_table :attendees_events
-    test_database.drop_table :events
-    test_database.drop_table :attendees
     test_database.drop_table :departments
     test_database.drop_table :organisations
   end

data/spec/feature/example_transform.rb

@@ -1,37 +1,44 @@
+helpers do
+  def source_schema_helper
+    'source'
+  end
+end
+
 import :organisations do
+  columns :name, :address
+
   query <<-SQL
     INSERT INTO #{stage_table} (
       external_id,
-      import_run_id,
+      address,
       name
     )
 
     SELECT DISTINCT
       o."Name",
-      #{import_run_id},
+      o."Adresse",
       o."Name"
 
-    FROM source."Organisation" o
+    FROM #{source_schema_helper}."Organisation" o
   SQL
 end
 
 import :departments do
+  columns :name
   references :organisations, on: :organisation_id
 
   query <<-SQL
     INSERT INTO #{stage_table} (
       external_id,
-      import_run_id,
       name,
       external_organisation_id
     )
 
     SELECT
       #{combined_key('o."Name"', 'o."pkOrgId"')},
-      #{import_run_id},
       o."Abteilung",
       o."Name"
 
-    FROM source."Organisation" o
+    FROM #{source_schema_helper}."Organisation" o
   SQL
 end

data/spec/feature/feature_spec.rb

@@ -1,48 +1,149 @@
 require 'spec_helper'
+require 'timecop'
+
 require_relative 'example_schema'
 require 'yaml'
 
 require 'active_support/core_ext/date/calculations'
 require 'active_support/core_ext/numeric/time'
 
+Thread::abort_on_exception = true
+
 describe BeetleETL do
 
   include ExampleSchema
 
-  let!(:now) { Time.new(2014, 07, 17, 16, 12).beginning_of_day }
-  before { allow(Time).to receive(:now) { now } }
+  let!(:time1) { Time.new(2014 , 7 , 17 , 16 , 12).beginning_of_day }
+  let!(:time2) { Time.new(2015 , 2 ,  8 , 22 , 18).beginning_of_day }
+  let!(:time3) { Time.new(2015 , 11 , 3 , 12 , 17).beginning_of_day }
 
-  before { create_tables }
-  after { drop_tables }
+  before :each do
+    create_tables
 
-  it 'is a working', :feature do
-    insert_into(:source__Organisation).values(
-      [ :pkOrgId , :Name   , :Abteilung ] ,
-      [ 1        , 'Apple' , 'iPhone'   ] ,
-      [ 2        , 'Apple' , 'MacBook'  ] ,
-    )
+    database_config_path = File.expand_path('../support/database.yml', File.dirname(__FILE__))
+    database_config = YAML.load(File.read(database_config_path))
 
     BeetleETL.configure do |config|
       config.transformation_file = File.expand_path('../example_transform.rb', __FILE__)
-      config.database = test_database
+      config.database_config = database_config
       config.external_source = 'source_name'
-      config.stage_schema = 'stage'
     end
+  end
 
+  after do
+    drop_tables
+  end
 
-    BeetleETL.import
+  it 'performs all possible transitions', :feature do
+    # create, keep, update, delete, undelete
+
+    import1
+    import2
+    import3
+  end
+
+  def import1
+    # create
+    insert_into(:source__Organisation).values(
+      [ :pkOrgId , :Name    , :Adresse        , :Abteilung ] ,
+      [ 1        , 'Apple'  , 'Apple Street'  , 'iPhone'   ] ,
+      [ 2        , 'Apple'  , 'Apple Street'  , 'MacBook'  ] ,
+      [ 3        , 'Google' , 'Google Street' , 'Gmail'    ] ,
+      [ 4        , 'Audi'   , 'Audi Street'   , 'A4'       ] ,
+    )
 
+    Timecop.freeze(time1) do
+      BeetleETL.import
+    end
 
     expect(:organisations).to have_values(
-      [ :id , :external_id , :external_source , :name   , :created_at , :updated_at , :deleted_at ] ,
-      [ 1   , 'Apple'      , 'source_name'    , 'Apple' , now         , now         , nil         ]
+      [ :id                       , :external_id , :external_source , :name    , :address        , :created_at , :updated_at , :deleted_at ] ,
+      [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'  , time1       , time1       , nil         ] ,
+      [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'Google Street' , time1       , time1       , nil         ] ,
+      [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'Audi Street'   , time1       , time1       , nil         ]
     )
 
     expect(:departments).to have_values(
-      [ :id , :external_id , :external_source , :name     , :organisation_id , :created_at , :updated_at , :deleted_at ] ,
-      [ 1   , '[Apple,1]'  , 'source_name'    , 'iPhone'  , 1                , now         , now         , nil         ] ,
-      [ 2   , '[Apple,2]'  , 'source_name'    , 'MacBook' , 1                , now         , now         , nil         ] ,
+      [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
+      [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
+      [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
+      [ department_id('[Google,3]') , '[Google,3]' , organisation_id('Google') , 'source_name'    , 'Gmail'   , time1       , time1       , nil         ] ,
+      [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time1       , nil         ] ,
+    )
+
+    test_database[:source__Organisation].truncate
+  end
+
+  def import2
+    # keep, update, delete
+    insert_into(:source__Organisation).values(
+      [ :pkOrgId , :Name    , :Adresse            , :Abteilung ] ,
+      [ 1        , 'Apple'  , 'Apple Street'      , 'iPhone'   ] ,
+      [ 2        , 'Apple'  , 'Apple Street'      , 'MacBook'  ] ,
+      [ 3        , 'Google' , 'NEW Google Street' , 'Google+'  ] ,
+    # [ 4        , 'Audi'   , 'Audi Street'       , 'A4'       ] ,
+    )
+
+    Timecop.freeze(time2) do
+      BeetleETL.import
+    end
+
+    expect(:organisations).to have_values(
+      [ :id                       , :external_id , :external_source , :name    , :address            , :created_at , :updated_at , :deleted_at ] ,
+      [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'      , time1       , time1       , nil         ] ,
+      [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'NEW Google Street' , time1       , time2       , nil         ] ,
+      [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'Audi Street'       , time1       , time2       , time2       ]
     )
+
+    expect(:departments).to have_values(
+      [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
+      [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
+      [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
+      [ department_id('[Google,3]') , '[Google,3]' , organisation_id('Google') , 'source_name'    , 'Google+' , time1       , time2       , nil         ] ,
+      [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time2       , time2       ] ,
+    )
+
+    test_database[:source__Organisation].truncate
+  end
+
+  def import3
+    # undelete with update
+    insert_into(:source__Organisation).values(
+      [ :pkOrgId , :Name    , :Adresse            , :Abteilung ] ,
+      [ 1        , 'Apple'  , 'Apple Street'      , 'iPhone'   ] ,
+      [ 2        , 'Apple'  , 'Apple Street'      , 'MacBook'  ] ,
+      [ 3        , 'Google' , 'NEW Google Street' , 'Google+'  ] ,
+      [ 4        , 'Audi'   , 'NEW Audi Street'   , 'A4'       ] ,
+    )
+
+    Timecop.freeze(time3) do
+      BeetleETL.import
+    end
+
+    expect(:organisations).to have_values(
+      [ :id                       , :external_id , :external_source , :name    , :address            , :created_at , :updated_at , :deleted_at ] ,
+      [ organisation_id('Apple')  , 'Apple'      , 'source_name'    , 'Apple'  , 'Apple Street'      , time1       , time1       , nil         ] ,
+      [ organisation_id('Google') , 'Google'     , 'source_name'    , 'Google' , 'NEW Google Street' , time1       , time2       , nil         ] ,
+      [ organisation_id('Audi')   , 'Audi'       , 'source_name'    , 'Audi'   , 'NEW Audi Street'   , time1       , time3       , nil         ]
+    )
+
+    expect(:departments).to have_values(
+      [ :id                         , :external_id , :organisation_id          , :external_source , :name     , :created_at , :updated_at , :deleted_at ] ,
+      [ department_id('[Apple,1]')  , '[Apple,1]'  , organisation_id('Apple')  , 'source_name'    , 'iPhone'  , time1       , time1       , nil         ] ,
+      [ department_id('[Apple,2]')  , '[Apple,2]'  , organisation_id('Apple')  , 'source_name'    , 'MacBook' , time1       , time1       , nil         ] ,
+      [ department_id('[Google,3]') , '[Google,3]' , organisation_id('Google') , 'source_name'    , 'Google+' , time1       , time2       , nil         ] ,
+      [ department_id('[Audi,4]')   , '[Audi,4]'   , organisation_id('Audi')   , 'source_name'    , 'A4'      , time1       , time3       , nil         ] ,
+    )
+
+    test_database[:source__Organisation].truncate
+  end
+
+  def organisation_id(external_id)
+    test_database[:organisations].first(external_id: external_id)[:id]
+  end
+
+  def department_id(external_id)
+    test_database[:departments].first(external_id: external_id)[:id]
   end
 
 end

data/spec/steps/assign_ids_spec.rb

@@ -3,7 +3,6 @@ require 'spec_helper'
 module BeetleETL
   describe AssignIds do
 
-    let(:run_id) { 1 }
     let(:external_source) { 'my_source' }
     subject { AssignIds.new(:example_table) }
 
@@ -14,12 +13,8 @@ module BeetleETL
         config.database = test_database
       end
 
-      allow(BeetleETL).to receive(:state) { double(:state, run_id: run_id) }
-
-      test_database.create_schema(:stage)
-      test_database.create_table(:stage__example_table) do
+      test_database.create_table(subject.stage_table_name.to_sym) do
         Integer :id
-        Integer :import_run_id
         String :external_id, size: 255
         String :transition, size: 255
       end
@@ -55,18 +50,18 @@ module BeetleETL
           [ 'keep_id'    , external_source  ] ,
         )
 
-        insert_into(:stage__example_table).values(
-          [ :import_run_id , :external_id , :transition ] ,
-          [ run_id         , 'create_id'  , 'CREATE'    ] ,
-          [ run_id         , 'keep_id'    , 'KEEP'      ] ,
+        insert_into(subject.stage_table_name.to_sym).values(
+          [ :external_id , :transition ] ,
+          [ 'create_id'  , 'CREATE'    ] ,
+          [ 'keep_id'    , 'KEEP'      ] ,
         )
 
         subject.assign_new_ids
 
-        expect(:stage__example_table).to have_values(
-          [ :id , :import_run_id , :external_id , :transition ] ,
-          [ 2   , run_id         , 'create_id'  , 'CREATE'    ] ,
-          [ nil , run_id         , 'keep_id'    , 'KEEP'      ] ,
+        expect(subject.stage_table_name.to_sym).to have_values(
+          [ :id , :external_id , :transition ] ,
+          [ 2   , 'create_id'  , 'CREATE'    ] ,
+          [ nil , 'keep_id'    , 'KEEP'      ] ,
         )
       end
     end
@@ -81,24 +76,24 @@ module BeetleETL
           [ 'undelete_id' , external_source  ] ,
         )
 
-        insert_into(:stage__example_table).values(
-          [ :import_run_id , :external_id  , :transition ] ,
-          [ run_id         , 'create_id'   , 'CREATE'    ] ,
-          [ run_id         , 'keep_id'     , 'KEEP'      ] ,
-          [ run_id         , 'update_id'   , 'UPDATE'    ] ,
-          [ run_id         , 'delete_id'   , 'DELETE'    ] ,
-          [ run_id         , 'undelete_id' , 'UNDELETE'  ] ,
+        insert_into(subject.stage_table_name.to_sym).values(
+          [ :external_id  , :transition ] ,
+          [ 'create_id'   , 'CREATE'    ] ,
+          [ 'keep_id'     , 'KEEP'      ] ,
+          [ 'update_id'   , 'UPDATE'    ] ,
+          [ 'delete_id'   , 'DELETE'    ] ,
+          [ 'undelete_id' , 'UNDELETE'  ] ,
         )
 
         subject.map_existing_ids
 
-        expect(:stage__example_table).to have_values(
-          [ :id , :import_run_id , :external_id  , :transition ] ,
-          [ nil , run_id         , 'create_id'   , 'CREATE'    ] ,
-          [ 1   , run_id         , 'keep_id'     , 'KEEP'      ] ,
-          [ 2   , run_id         , 'update_id'   , 'UPDATE'    ] ,
-          [ 3   , run_id         , 'delete_id'   , 'DELETE'    ] ,
-          [ 4   , run_id         , 'undelete_id' , 'UNDELETE'  ] ,
+        expect(subject.stage_table_name.to_sym).to have_values(
+          [ :id , :external_id  , :transition ] ,
+          [ nil , 'create_id'   , 'CREATE'    ] ,
+          [ 1   , 'keep_id'     , 'KEEP'      ] ,
+          [ 2   , 'update_id'   , 'UPDATE'    ] ,
+          [ 3   , 'delete_id'   , 'DELETE'    ] ,
+          [ 4   , 'undelete_id' , 'UNDELETE'  ] ,
         )
       end
     end