beetle_etl 0.0.1

data/spec/steps/map_relations_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe MapRelations do
+
+    let(:run_id) { 1 }
+    let(:previous_run_id) { 5000 }
+
+    before do
+      BeetleETL.configure do |config|
+        config.stage_schema = 'stage'
+        config.external_source = 'my_source'
+        config.database = test_database
+      end
+
+      allow(BeetleETL).to receive(:state) { double(:state, run_id: run_id) }
+
+      test_database.create_schema(:stage)
+      test_database.create_table(:stage__dependee_a) do
+        Integer :import_run_id
+        Integer :id
+        String :external_id, size: 255
+      end
+
+      test_database.create_table(:stage__dependee_b) do
+        Integer :import_run_id
+        Integer :id
+        String :external_id, size: 255
+      end
+
+      test_database.create_table(:stage__depender) do
+        Integer :import_run_id
+        String :external_id, size: 255
+
+        String :external_dependee_a_id
+        Integer :dependee_a_id
+
+        String :external_dependee_b_id
+        Integer :dependee_b_id
+      end
+    end
+
+    describe '#depenencies' do
+      it 'depends on Transform of the same table and AssignIds of its dependees' do
+        relations = {
+          dependee_a_id: :dependee_a,
+          dependee_b_id: :dependee_b,
+        }
+
+        expect(MapRelations.new(:depender, relations).dependencies).to eql(
+          [
+            'dependee_a: AssignIds',
+            'dependee_b: AssignIds',
+            'depender: Transform',
+          ].to_set
+        )
+      end
+    end
+
+    describe '#run' do
+      it 'maps external foreign key references to id references ' do
+        insert_into(:stage__dependee_a).values(
+          [ :import_run_id  , :id , :external_id ] ,
+          [ run_id          , 1   , 'a_id'       ] ,
+          [ previous_run_id , 2   , 'a_id'       ] ,
+        )
+
+        insert_into(:stage__dependee_b).values(
+          [ :import_run_id , :id , :external_id ] ,
+          [ run_id         , 26  , 'b_id'       ] ,
+        )
+
+        insert_into(:stage__depender).values(
+          [ :import_run_id , :external_dependee_a_id , :external_dependee_b_id ] ,
+          [ run_id         , 'a_id'                  , 'b_id'                  ] ,
+        )
+
+        relations = {
+          dependee_a_id: :dependee_a,
+          dependee_b_id: :dependee_b,
+        }
+
+        MapRelations.new(:depender, relations).run
+
+        expect(:stage__depender).to have_values(
+          [ :import_run_id , :dependee_a_id , :dependee_b_id ] ,
+          [ run_id         , 1              , 26             ] ,
+        )
+      end
+    end
+  end
+end

data/spec/steps/step_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe Step do
+
+    subject { Step.new(:example_table) }
+    FooStep = Class.new(Step)
+
+    describe '.step_name' do
+      it 'returns the steps name' do
+        expect(Step.step_name(:example_table)).to eql('example_table: Step')
+      end
+
+      it 'returns the step name of inheriting steps' do
+        expect(FooStep.step_name(:foo_table)).to eql('foo_table: FooStep')
+      end
+    end
+
+    describe '#name' do
+      it 'returns the steps name' do
+        expect(Step.new(:example_table).name).to eql('example_table: Step')
+      end
+
+      it 'returns the step name of inheriting steps' do
+        expect(FooStep.new(:foo_table).name).to eql('foo_table: FooStep')
+      end
+    end
+
+    describe '#dependencies' do
+      it 'raises an exception' do
+        expect { subject.dependencies }.to raise_error(DependenciesNotDefinedError)
+      end
+
+    end
+
+  end
+end

data/spec/steps/table_diff_spec.rb

@@ -0,0 +1,183 @@
+require 'spec_helper'
+
+require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/numeric/time'
+
+module BeetleETL
+  describe TableDiff do
+
+    let(:run_id) { 1 }
+    let(:external_source) { 'my_source' }
+    subject { TableDiff.new(:example_table) }
+
+    before do
+      BeetleETL.configure do |config|
+        config.stage_schema = 'stage'
+        config.external_source = external_source
+        config.database = test_database
+      end
+
+      allow(BeetleETL).to receive(:state) { double(:state, run_id: run_id) }
+
+      test_database.create_schema(:stage)
+      test_database.create_table(:stage__example_table) do
+        Integer :import_run_id
+        String :external_id, size: 255
+        String :transition, size: 20
+
+        String :external_foo_id, size: 255
+        Integer :foo_id
+
+        String :payload, size: 255
+      end
+
+      test_database.create_table(:example_table) do
+        Integer :id
+        String :external_id, size: 255
+        String :external_source, size: 255
+        DateTime :deleted_at
+
+        String :payload, size: 255
+        String :ignored_attribute, size: 255
+        Integer :foo_id
+      end
+    end
+
+    describe '#depenencies' do
+      it 'depends on MapRelations of the same table' do
+        expect(subject.dependencies).to eql(['example_table: MapRelations'].to_set)
+      end
+    end
+
+    describe '#run' do
+      it 'runs all transitions' do
+        %w(create keep update delete undelete).each do |transition|
+          expect(subject).to receive(:"transition_#{transition}")
+        end
+
+        subject.run
+      end
+    end
+
+    describe '#transition_create' do
+      it 'assigns CREATE to new records' do
+
+        insert_into(:example_table).values(
+          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id ] ,
+          [ run_id         , 'created'    ] ,
+          [ run_id         , 'existing'   ] ,
+        )
+
+        subject.transition_create
+
+        expect(:stage__example_table).to have_values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'created'    , 'CREATE'    ] ,
+          [ run_id         , 'existing'   , nil         ] ,
+        )
+      end
+    end
+
+    describe 'transition_keep' do
+      it 'assigns KEEP if the record already exists and is not deleted, comparing all columns
+        except externald_*_id columns and columns not contained in the stage table' do
+
+        insert_into(:example_table).values(
+          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id , :payload           , :foo_id , :external_foo_id ] ,
+          [ run_id         , 'existing'   , 'existing content' , 1       , 'ignored column' ] ,
+          [ run_id         , 'deleted'    , 'deleted content'  , 2       , 'ignored column' ] ,
+        )
+
+        subject.transition_keep
+
+        expect(:stage__example_table).to have_values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'existing'   , 'KEEP'      ] ,
+          [ run_id         , 'deleted'    , nil         ] ,
+        )
+      end
+    end
+
+    describe '#transition_update' do
+      it 'assigns UPDATE to non-deleted records with changed values comparing all columns
+        except externald_*_id columns and columns not contained in the stage table' do
+
+        insert_into(:example_table).values(
+          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 'existing_1' , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 'existing_2' , external_source  , 'existing content' , 'ignored content'  , 2       , nil         ] ,
+          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 3       , 1.day.ago   ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id , :payload           , :foo_id , :external_foo_id ] ,
+          [ run_id         , 'existing_1' , 'updated content'  , 1       , 'ignored_column' ] ,
+          [ run_id         , 'existing_2' , 'existing content' , 4       , 'ignored_column' ] ,
+          [ run_id         , 'deleted'    , 'updated content'  , 3       , 'ignored_column' ] ,
+        )
+
+        subject.transition_update
+
+        expect(:stage__example_table).to have_values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'existing_1' , 'UPDATE'    ] ,
+          [ run_id         , 'existing_2' , 'UPDATE'    ] ,
+          [ run_id         , 'deleted'    , nil         ] ,
+        )
+      end
+    end
+
+    describe 'transition_delete' do
+      it 'creates records with DELETE that no loger exist in the stage table' do
+        insert_into(:example_table).values(
+          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+        )
+
+        subject.transition_delete
+
+        expect(:stage__example_table).to have_values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'existing'   , 'DELETE'    ] ,
+        )
+      end
+    end
+
+    describe 'transition_undelete' do
+      it 'assigns UNDELETE to previously deleted records' do
+        insert_into(:example_table).values(
+          [ :external_id , :external_source , :payload           , :ignored_attribute , :foo_id , :deleted_at ] ,
+          [ 'existing'   , external_source  , 'existing content' , 'ignored content'  , 1       , nil         ] ,
+          [ 'deleted'    , external_source  , 'deleted content'  , 'ignored content'  , 2       , 1.day.ago   ] ,
+        )
+
+        insert_into(:stage__example_table).values(
+          [ :import_run_id , :external_id , :payload          , :foo_id , :external_foo_id ] ,
+          [ run_id         , 'existing'   , 'updated content' , 1       , 'ignored_column' ] ,
+          [ run_id         , 'deleted'    , 'updated content' , 2       , 'ignored_column' ] ,
+        )
+
+        subject.transition_undelete
+
+        expect(:stage__example_table).to have_values(
+          [ :import_run_id , :external_id , :transition ] ,
+          [ run_id         , 'existing'   , nil         ] ,
+          [ run_id         , 'deleted'    , 'UNDELETE'  ] ,
+        )
+      end
+    end
+  end
+end

data/spec/steps/transform_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe Transform do
+
+    let(:query) { double(:query) }
+    subject do
+      deps = [:some_table, :some_other_table].to_set
+      Transform.new(:example_table, deps, query)
+    end
+
+    describe '#dependencies' do
+      it 'depends on Transform of all dependencies' do
+        expect(subject.dependencies).to eql(
+          [
+            'some_table: Transform',
+            'some_other_table: Transform',
+          ].to_set
+        )
+      end
+    end
+
+    describe '#run' do
+      it 'runs a query in the database' do
+        database = double(:database)
+        BeetleETL.configure { |config| config.database = database }
+
+        expect(database).to receive(:run).with(query)
+
+        subject.run
+      end
+    end
+  end
+end

data/spec/support/database.yml.example

@@ -0,0 +1,9 @@
+adapter: 'postgres'
+encoding: utf8
+host: localhost
+database: beetle_etl_test
+username: ''
+password: ''
+pool: 5
+pool_timeout: 360
+connect_timeout: 360

data/spec/support/database.yml.travis

@@ -0,0 +1,4 @@
+adapter: 'postgres'
+encoding: utf8
+database: travis_ci_test
+username: postgres

data/spec/support/database_helpers.rb

@@ -0,0 +1,58 @@
+require 'sequel'
+require 'yaml'
+require 'rspec/expectations'
+
+module SpecSupport
+  module DatabaseHelpers
+
+    module Test
+      class Table
+        def initialize dataset
+          @dataset = dataset
+        end
+
+        def values(*data)
+          columns = data[0]
+          values = data[1..-1]
+          @dataset.import(columns, values)
+        end
+      end
+    end
+
+    def test_database
+      @database ||= begin
+        config_path = File.expand_path('./database.yml', File.dirname(__FILE__))
+        config = File.read(config_path)
+        Sequel.connect(YAML.load(config))
+      end
+    end
+
+    def insert_into(table_description)
+      dataset = test_database[table_description]
+      Test::Table.new(dataset)
+    end
+
+  end
+end
+
+RSpec::Matchers.define :have_values do |*rows|
+  match do |table_description|
+    dataset = test_database[table_description]
+
+    columns = rows[0].map(&:to_sym)
+    values = rows[1..-1]
+
+    begin
+      expect{dataset.select(columns).all}.not_to raise_error
+      expect(dataset.map(columns)).to match_array(values)
+      true
+    rescue RSpec::Expectations::ExpectationNotMetError => error
+      @expectation_not_met_error = error
+      false
+    end
+  end
+
+  failure_message do |dataset|
+    @expectation_not_met_error.message
+  end
+end

metadata

@@ -0,0 +1,220 @@
+--- !ruby/object:Gem::Specification
+name: beetle_etl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Luciano Maiwald
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.13.0
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.15.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.15.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Taking care of synchronizing external data with referential data in your
+  application.
+email:
+- luciano.maiwald@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- beetle_etl.gemspec
+- lib/beetle_etl.rb
+- lib/beetle_etl/dsl/dsl.rb
+- lib/beetle_etl/dsl/transformation.rb
+- lib/beetle_etl/dsl/transformation_loader.rb
+- lib/beetle_etl/import.rb
+- lib/beetle_etl/state.rb
+- lib/beetle_etl/steps/assign_ids.rb
+- lib/beetle_etl/steps/load.rb
+- lib/beetle_etl/steps/map_relations.rb
+- lib/beetle_etl/steps/step.rb
+- lib/beetle_etl/steps/table_diff.rb
+- lib/beetle_etl/steps/transform.rb
+- lib/beetle_etl/task_runner/dependency_resolver.rb
+- lib/beetle_etl/task_runner/task_runner.rb
+- lib/beetle_etl/version.rb
+- script/postgres
+- spec/beetle_etl_spec.rb
+- spec/dependency_resolver_spec.rb
+- spec/dsl/dsl_spec.rb
+- spec/dsl/transformation_loader_spec.rb
+- spec/dsl/transformation_spec.rb
+- spec/feature/example_schema.rb
+- spec/feature/example_transform.rb
+- spec/feature/feature_spec.rb
+- spec/import_spec.rb
+- spec/spec_helper.rb
+- spec/state_spec.rb
+- spec/steps/assign_ids_spec.rb
+- spec/steps/load_spec.rb
+- spec/steps/map_relations_spec.rb
+- spec/steps/step_spec.rb
+- spec/steps/table_diff_spec.rb
+- spec/steps/transform_spec.rb
+- spec/support/database.yml.example
+- spec/support/database.yml.travis
+- spec/support/database_helpers.rb
+homepage: https://github.com/maiwald/beetle_etl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: BeetleETL helps you with your recurring ETL imports.
+test_files:
+- spec/beetle_etl_spec.rb
+- spec/dependency_resolver_spec.rb
+- spec/dsl/dsl_spec.rb
+- spec/dsl/transformation_loader_spec.rb
+- spec/dsl/transformation_spec.rb
+- spec/feature/example_schema.rb
+- spec/feature/example_transform.rb
+- spec/feature/feature_spec.rb
+- spec/import_spec.rb
+- spec/spec_helper.rb
+- spec/state_spec.rb
+- spec/steps/assign_ids_spec.rb
+- spec/steps/load_spec.rb
+- spec/steps/map_relations_spec.rb
+- spec/steps/step_spec.rb
+- spec/steps/table_diff_spec.rb
+- spec/steps/transform_spec.rb
+- spec/support/database.yml.example
+- spec/support/database.yml.travis
+- spec/support/database_helpers.rb