beetle_etl 0.0.1

data/lib/beetle_etl/version.rb

@@ -0,0 +1,3 @@
+module BeetleETL
+  VERSION = "0.0.1"
+end

data/script/postgres

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+config_file="/usr/local/var/postgres/postgresql.conf"
+
+case "$1" in
+  "run" )
+    postgres -D /usr/local/var/postgres -c config_file=$config_file
+    ;;
+  * )
+    echo "USAGE: $0 {brew|…} run"
+    ;;
+esac

data/spec/beetle_etl_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe BeetleETL do
+  describe '#import' do
+
+    it 'runs the import' do
+      allow(BeetleETL).to receive(:state) { double(:state).as_null_object }
+      expect(BeetleETL::Import).to receive(:run)
+      BeetleETL.import
+    end
+
+    context 'handling state' do
+      it 'starts the import and marks it as finished if no errors are thrown' do
+        allow(BeetleETL::Import).to receive(:run)
+
+        expect(BeetleETL.state).to receive(:start_import).ordered
+        expect(BeetleETL.state).to receive(:mark_as_succeeded).ordered
+
+        BeetleETL.import
+      end
+
+      it 'starts the import and marks it as failed if Import.run throws an error' do
+        exception = Exception.new
+        allow(BeetleETL::Import).to receive(:run).and_raise(exception)
+
+        expect(BeetleETL.state).to receive(:start_import).ordered
+        expect(BeetleETL.state).to receive(:mark_as_failed).ordered
+
+        expect { BeetleETL.import }.to raise_exception(exception)
+      end
+    end
+  end
+
+  describe '#config' do
+    it 'returns a configuration object' do
+      expect(BeetleETL.config).to be_a(BeetleETL::Configuration)
+    end
+  end
+
+  describe '#configure' do
+    it 'allows the configuration to be changed' do
+      expect(BeetleETL.config.external_source).to be_nil
+
+      BeetleETL.configure { |config| config.external_source = 'foo' }
+
+      expect(BeetleETL.config.external_source).to eql('foo')
+    end
+  end
+
+  describe '#database' do
+    let(:database) { double(:database) }
+
+    it 'returns the Sequel Database object stored in the config' do
+      BeetleETL.configure { |config| config.database = database }
+
+      expect(BeetleETL.database).to eql(database)
+    end
+
+    it 'builds and caches a Sequel Database from config when no database is passed' do
+      database_config = double(:database_config)
+      BeetleETL.configure { |config| config.database_config = database_config }
+
+      expect(Sequel).to receive(:connect).with(database_config).once { database }
+
+      expect(BeetleETL.database).to eql(database)
+      expect(BeetleETL.database).to eql(database)
+    end
+
+  end
+end

data/spec/dependency_resolver_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe DependencyResolver do
+
+    # test dependencies
+    #
+    #     A
+    #   / | \
+    #  B  |  C
+    #  |  \ /|
+    #  |   D |
+    #   \ / \|
+    #    E   F
+
+    Item = Struct.new(:name, :dependencies)
+
+    let(:a) { Item.new(:a, Set.new) }
+    let(:b) { Item.new(:b, Set.new([:a])) }
+    let(:c) { Item.new(:c, Set.new([:a])) }
+    let(:d) { Item.new(:d, Set.new([:a, :c])) }
+    let(:e) { Item.new(:e, Set.new([:c, :d])) }
+    let(:f) { Item.new(:f, Set.new([:c, :d])) }
+
+    def items
+      [a, b, c, d, e, f].shuffle
+    end
+
+    describe '#resolvables' do
+      let(:resolver) { DependencyResolver.new(items) }
+
+      it 'returns all items without dependencies when given an empty array' do
+        expect(resolver.resolvables([])).to match_array([a])
+      end
+
+      it 'returns all items with met dependencies' do
+        expect(resolver.resolvables([:a, :b, :c])).to match_array([d])
+        expect(resolver.resolvables([:a, :b, :c, :d])).to match_array([e, f])
+      end
+    end
+
+    context 'with cyclic or missing dependencies' do
+      let(:cyclic) { Item.new(:a, Set.new([:b])) }
+
+      it 'detects cyclic dependencies' do
+        expect { DependencyResolver.new([cyclic, b]) }.to \
+          raise_error(BeetleETL::UnsatisfiableDependenciesError)
+      end
+
+      it 'detects unsatisfiable dependencies' do
+        expect { DependencyResolver.new([b]) }.to \
+          raise_error(BeetleETL::UnsatisfiableDependenciesError)
+      end
+    end
+
+  end
+end

data/spec/dsl/dsl_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe DSL do
+
+    subject { DSL.new(:foo_table) }
+
+    describe '#stage_table' do
+      it 'returns the stage table name including the schema defined in the config' do
+        BeetleETL.configure { |config| config.stage_schema = 'bar' }
+        expect(subject.stage_table).to eql('"bar"."foo_table"')
+      end
+    end
+
+    describe '#external_source' do
+      it 'returns the external source‘s identifier' do
+        expect(subject.external_source).to eql('source')
+      end
+    end
+
+    describe '#combined_key' do
+      it 'returns an SQL string for combined external ids' do
+        expect(subject.combined_key('foo', 'bar')).to eql(
+          %q('[' || foo || ',' || bar || ']')
+        )
+      end
+
+      it 'works with multiple arguments' do
+        expect(subject.combined_key('foo', 'bar', 'baz')).to eql(
+          %q('[' || foo || ',' || bar || ',' || baz || ']')
+        )
+      end
+    end
+
+    describe '#import_run_id' do
+      it 'returns the import run id defined in the config' do
+        id = double(:id)
+        allow(BeetleETL.state).to receive(:run_id) { id }
+        expect(subject.import_run_id).to eql(id)
+      end
+    end
+
+  end
+end

data/spec/dsl/transformation_loader_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'tempfile'
+
+module BeetleETL
+  describe TransformationLoader do
+
+    before :example do
+      data_file = tempfile_with_contents <<-FILE
+        import :foo do
+          'foo'
+        end
+
+        import :bar do
+          'bar'
+        end
+      FILE
+
+      BeetleETL.configure do |config|
+        config.transformation_file = data_file.path
+      end
+    end
+
+    describe '#load' do
+      it 'loads runlist entries from the data file' do
+        expect(Transformation).to receive(:new) do |table_name, config|
+          expect(table_name.to_s).to eql(config.call)
+        end.exactly(2).times
+
+        subject.load
+      end
+
+      it 'adds every runlist entry to the entries array' do
+        allow(Transformation).to receive(:new) do |table_name, config|
+          table_name
+        end
+
+        transformations = subject.load
+
+        expect(transformations).to eql(%i[foo bar])
+      end
+    end
+
+    def tempfile_with_contents(contents)
+      Tempfile.new('transform').tap do |file|
+        file.write(contents)
+        file.close
+      end
+    end
+
+  end
+end

data/spec/dsl/transformation_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+module BeetleETL
+  describe Transformation do
+
+    describe '#table_name' do
+      it 'returns the given table name' do
+        transformation = Transformation.new(:table, Proc.new {})
+        expect(transformation.table_name).to eql(:table)
+      end
+    end
+
+    describe '#relations' do
+      it 'returns the list of foreign tables and their foreign key column' do
+        setup = Proc.new do
+          references :foreign_table, on: :foreign_table_id
+        end
+        transformation = Transformation.new(:table, setup)
+
+        expect(transformation.relations).to eql({
+          foreign_table_id: :foreign_table
+        })
+      end
+    end
+
+    describe '#dependencies' do
+      it 'returns the depending tables' do
+        setup = Proc.new do
+          references :foreign_table, on: :foreign_table_id
+          references :another_foreign_table, on: :another_foreign_table_id
+        end
+        transformation = Transformation.new(:table, setup)
+
+        expect(transformation.dependencies).to eql(Set.new([:foreign_table, :another_foreign_table]))
+      end
+    end
+
+    describe '#query' do
+      it 'returns the query interpolating methods in scope' do
+
+        setup = Proc.new do
+          def foo; "foo_string"; end
+          query "SELECT '#{foo}' FROM some_table"
+        end
+        transformation = Transformation.new(:table, setup)
+
+        expect(transformation.query).to eql(
+          "SELECT 'foo_string' FROM some_table"
+        )
+      end
+    end
+
+  end
+end

data/spec/feature/example_schema.rb

@@ -0,0 +1,192 @@
+module ExampleSchema
+
+  def create_tables
+    create_source_tables
+    create_stage_tables
+    create_target_tables
+  end
+
+  def drop_tables
+    drop_source_tables
+    drop_stage_tables
+    drop_target_tables
+  end
+
+  def create_source_tables
+    test_database.create_schema :source
+
+    test_database.create_table :source__Organisation do
+      Integer :pkOrgId
+      String :Name, size: 255
+      String :Abteilung, size: 255
+    end
+
+    test_database.create_table :source__Person do
+      Integer :pkPersID
+      String :Vorname, size: 255
+      String :Nachname, size: 255
+      Integer :fkFirma
+      Integer :fkAdresse
+      Integer :fkTyp
+    end
+
+    test_database.create_table :source__Veranstaltung do
+      Integer :pkVeranstaltungId
+      Integer :fkOrganisation
+    end
+
+    test_database.create_table :source__Veranstaltungsbesuch do
+      Integer :fkVeranstaltung
+      Integer :fkBesucher
+    end
+  end
+
+  def drop_source_tables
+    test_database.drop_schema :source, cascade: true
+  end
+
+  def create_stage_tables
+    test_database.create_schema :stage
+
+    test_database.create_table :stage__import_runs do
+      primary_key :id
+      DateTime :started_at
+      DateTime :finished_at
+      String :state, size: 255
+    end
+
+    test_database.create_table :stage__organisations do
+      Integer :id
+      String :external_id, size: 255
+      foreign_key :import_run_id, :stage__import_runs
+      index [:external_id, :import_run_id]
+      String :transition, size: 255
+
+      String :name, size: 255
+    end
+
+    test_database.create_table :stage__departments do
+      Integer :id
+      String :external_id, size: 255
+      foreign_key :import_run_id, :stage__import_runs
+      index [:external_id, :import_run_id]
+      String :transition, size: 255
+
+      String :name, size: 255
+
+      String :external_organisation_id, size: 255
+      Integer :organisation_id
+
+    end
+
+    test_database.create_table :stage__attendees do
+      Integer :id
+      String :external_id, size: 255
+      foreign_key :import_run_id, :stage__import_runs
+      index [:external_id, :import_run_id]
+      String :transition, size: 255
+
+      String :first_name, size: 255
+      String :last_name, size: 255
+    end
+
+    test_database.create_table :stage__events do
+      Integer :id
+      String :external_id, size: 255
+      foreign_key :import_run_id, :stage__import_runs
+      index [:external_id, :import_run_id]
+      String :transition, size: 255
+
+      String :name, size: 255
+      DateTime :starts_at
+      DateTime :ends_at
+
+      String :external_organisations_id, size: 255
+      Integer :organisation_id
+    end
+
+    test_database.create_table :stage__attendees_events do
+      Integer :id
+      String :external_id, size: 255
+      foreign_key :import_run_id, :stage__import_runs
+      index [:external_id, :import_run_id]
+      String :transition, size: 255
+
+      String :external_attendee_id, size: 255
+      Integer :attendee_id
+
+      String :external_event_id, size: 255
+      Integer :event_id
+    end
+  end
+
+  def drop_stage_tables
+    test_database.drop_schema :stage, cascade: true
+  end
+
+  def create_target_tables
+    test_database.create_table :organisations do
+      primary_key :id
+      String :external_id, size: 255
+      String :external_source, size: 255
+      String :name, size: 255
+      DateTime :created_at
+      DateTime :updated_at
+      DateTime :deleted_at
+    end
+
+    test_database.create_table :departments do
+      primary_key :id
+      String :external_id, size: 255
+      String :external_source, size: 255
+      String :name, size: 255
+      foreign_key :organisation_id, :organisations
+      DateTime :created_at
+      DateTime :updated_at
+      DateTime :deleted_at
+    end
+
+    test_database.create_table :attendees do
+      primary_key :id
+      String :external_id, size: 255
+      String :external_source, size: 255
+      String :first_name, size: 255
+      String :last_name, size: 255
+      DateTime :created_at
+      DateTime :updated_at
+      DateTime :deleted_at
+    end
+
+    test_database.create_table :events do
+      primary_key :id
+      String :external_id, size: 255
+      String :external_source, size: 255
+      String :name, size: 255
+      DateTime :starts_at
+      DateTime :ends_at
+      foreign_key :organisation, :organisations
+      DateTime :created_at
+      DateTime :updated_at
+      DateTime :deleted_at
+    end
+
+    test_database.create_table :attendees_events do
+      foreign_key :attendee_id, :attendees, null: false
+      foreign_key :event_id, :events, null: false
+      primary_key [:attendee_id, :event_id]
+      index [:attendee_id, :event_id]
+      DateTime :created_at
+      DateTime :updated_at
+      DateTime :deleted_at
+    end
+  end
+
+  def drop_target_tables
+    test_database.drop_table :attendees_events
+    test_database.drop_table :events
+    test_database.drop_table :attendees
+    test_database.drop_table :departments
+    test_database.drop_table :organisations
+  end
+
+end