beetle_etl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ebb61022b0e58e217f1b215486a993c7e43799a2
+  data.tar.gz: e00ad7086cf5be7c4cadf520cbd21c1ebd47202d
+SHA512:
+  metadata.gz: 8e3c2be8adf3cb65807fddb95d1b09d1de9064f586f3e0aefa6026ff3e7b572b8f9653c31ca759114228e6cb6470495f21b74f7470ad6fb7745136210b87f31e
+  data.tar.gz: 3d91601c914486564b8db8e3afdef62c90b04b54fe3633321547de3ec437ee1f12ead442265e84561c4578d2dae482be2cebc85dfd610c76e3db877239d4a2d5

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/support/database.yml
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.2
+addons:
+  postgresql: "9.3"
+  code_climate:
+    repo_token: dd18697b0acb6be343db62982b753b72676e8342701cc0442121de2d12ee6549
+
+before_script:
+  - psql -c 'create database travis_ci_test;' -U postgres
+  - cp spec/support/database.yml.travis spec/support/database.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in beetle_etl.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Luciano Maiwald
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# BeetleETL
+[![Build Status](https://travis-ci.org/maiwald/beetle_etl.svg?branch=master)](https://travis-ci.org/maiwald/beetle_etl)
+[![Code Climate](https://codeclimate.com/github/maiwald/beetle_etl.png)](https://codeclimate.com/github/maiwald/beetle_etl)
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'beetle_etl'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install beetle_etl
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/maiwald/beetle_etl/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+task :default => :spec
+RSpec::Core::RakeTask.new

data/beetle_etl.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'beetle_etl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'beetle_etl'
+  spec.version       = BeetleETL::VERSION
+  spec.authors       = ['Luciano Maiwald']
+  spec.email         = ['luciano.maiwald@gmail.com']
+  spec.summary       = %q{BeetleETL helps you with your recurring ETL imports.}
+  spec.description   = %q{Taking care of synchronizing external data with referential data in your application.}
+  spec.homepage      = 'https://github.com/maiwald/beetle_etl'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'sequel', '>= 4.13.0'
+  spec.add_runtime_dependency 'celluloid', '>= 0.15.2'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '~> 3.0.0'
+  spec.add_development_dependency 'pg'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'activesupport'
+end

data/lib/beetle_etl.rb

@@ -0,0 +1,85 @@
+require 'beetle_etl/version'
+
+require 'sequel'
+
+module BeetleETL
+
+  InvalidConfigurationError = Class.new(StandardError)
+
+  require 'beetle_etl/dsl/dsl'
+  require 'beetle_etl/dsl/transformation'
+  require 'beetle_etl/dsl/transformation_loader'
+
+  require 'beetle_etl/steps/step'
+  require 'beetle_etl/steps/transform'
+  require 'beetle_etl/steps/map_relations'
+  require 'beetle_etl/steps/table_diff'
+  require 'beetle_etl/steps/assign_ids'
+  require 'beetle_etl/steps/load'
+
+  require 'beetle_etl/task_runner/dependency_resolver'
+  require 'beetle_etl/task_runner/task_runner'
+
+  require 'beetle_etl/state'
+  require 'beetle_etl/import'
+
+  class Configuration
+    attr_accessor \
+      :database_config,
+      :database,
+      :transformation_file,
+      :stage_schema,
+      :external_source
+
+    def initialize
+      @stage_schema = 'stage'
+    end
+  end
+
+  class << self
+
+    def import
+      state.start_import
+
+      begin
+        Import.run
+        state.mark_as_succeeded
+      rescue Exception => e
+        state.mark_as_failed
+        raise e
+      ensure
+        @database.disconnect if @database
+      end
+    end
+
+    def configure
+      yield(config)
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def database
+      if config.database
+        config.database
+      elsif config.database_config
+        @database ||= Sequel.connect(config.database_config)
+      else
+        msg = "Either Sequel connection database_config or a Sequel Database object required"
+        raise InvalidConfigurationError.new(msg)
+      end
+    end
+
+    def state
+      @state ||= State.new
+    end
+
+    def reset
+      @config = nil
+      @state = nil
+      @database = nil
+    end
+
+  end
+end

data/lib/beetle_etl/dsl/dsl.rb

@@ -0,0 +1,37 @@
+module BeetleETL
+  class DSL
+
+    attr_reader :relations, :query_string
+
+    def initialize(table_name)
+      @table_name = table_name
+      @relations = {}
+    end
+
+    def references(foreign_table, on: foreign_key)
+      @relations[on] = foreign_table
+    end
+
+    def query(query)
+      @query_string = query
+    end
+
+
+    def stage_table
+      %Q("#{BeetleETL.config.stage_schema}"."#{@table_name}")
+    end
+
+    def external_source
+      'source'
+    end
+
+    def combined_key(*args)
+      %Q('[' || #{args.join(%q[ || ',' || ])} || ']')
+    end
+
+    def import_run_id
+      BeetleETL.state.run_id
+    end
+
+  end
+end

data/lib/beetle_etl/dsl/transformation.rb

@@ -0,0 +1,26 @@
+require 'set'
+
+module BeetleETL
+  class Transformation
+
+    attr_reader :table_name
+
+    def initialize(table_name, setup)
+      @table_name = table_name
+      (@parsed = DSL.new(table_name)).instance_eval(&setup)
+    end
+
+    def relations
+      @parsed.relations
+    end
+
+    def dependencies
+      relations.values.to_set
+    end
+
+    def query
+      @parsed.query_string
+    end
+
+  end
+end

data/lib/beetle_etl/dsl/transformation_loader.rb

@@ -0,0 +1,22 @@
+module BeetleETL
+  module TransformationLoader
+    extend self
+
+    def load
+      @transformations = []
+
+      File.open(BeetleETL.config.transformation_file, 'r') do |file|
+        instance_eval file.read
+      end
+
+      @transformations
+    end
+
+    private
+
+    def import(table_name, &setup)
+      @transformations << Transformation.new(table_name, setup)
+    end
+
+  end
+end

data/lib/beetle_etl/import.rb

@@ -0,0 +1,37 @@
+module BeetleETL
+  module Import
+
+    extend self
+
+    def run
+      TaskRunner.run(data_steps)
+      BeetleETL.database.transaction do
+        load_steps.each(&:run)
+      end
+    end
+
+    private
+
+    def data_steps
+      transformations.map do |t|
+        [
+          Transform.new(t.table_name, t.dependencies, t.query),
+          MapRelations.new(t.table_name, t.relations),
+          TableDiff.new(t.table_name),
+          AssignIds.new(t.table_name),
+        ]
+      end.flatten
+    end
+
+    def load_steps
+      transformations.map do |t|
+        Load.new(t.table_name)
+      end
+    end
+
+    def transformations
+      @transformations ||= TransformationLoader.load
+    end
+
+  end
+end

data/lib/beetle_etl/state.rb

@@ -0,0 +1,67 @@
+module BeetleETL
+
+  ImportAleadyRunning = Class.new(StandardError)
+  ImportSchemaNotFound = Class.new(StandardError)
+  ImportNotRunning = Class.new(StandardError)
+
+  class State
+
+    def start_import
+      raise ImportAleadyRunning if import_already_running?
+
+      @run_id = import_runs_dataset.insert(
+        state: 'RUNNING',
+        started_at: now
+      )
+    end
+
+    def mark_as_succeeded
+      mark_as('SUCCEEDED')
+    end
+
+    def mark_as_failed
+      mark_as('FAILED')
+    end
+
+    def run_id
+      raise ImportNotRunning if @run_id.nil?
+      @run_id
+    end
+
+    def last_run_id
+      last_import = import_runs_dataset.
+        select(:id).
+        where(state: 'SUCCEEDED').
+        order(Sequel.desc(:id)).
+        first
+
+      last_import.nil? ? nil : last_import[:id]
+    end
+
+  private
+
+    def import_runs_table
+      "#{BeetleETL.config.stage_schema}__import_runs".to_sym
+    end
+
+    def import_already_running?
+      import_runs_dataset.where(state: 'RUNNING').count > 0
+    end
+
+    def now
+      Time.now
+    end
+
+    def mark_as(state)
+      import_runs_dataset.filter(id: run_id).update(
+        state: state,
+        finished_at: now
+      )
+    end
+
+    def import_runs_dataset
+      BeetleETL.database[import_runs_table]
+    end
+
+  end
+end

data/lib/beetle_etl/steps/assign_ids.rb

@@ -0,0 +1,54 @@
+module BeetleETL
+  class AssignIds < Step
+
+    def dependencies
+      [TableDiff.step_name(table_name)].to_set
+    end
+
+    def run
+      assign_new_ids
+      map_existing_ids
+    end
+
+    def assign_new_ids
+      stage_table
+        .where(
+          import_run_id: run_id,
+          transition: 'CREATE'
+        )
+        .update(
+          id: Sequel.function(:NEXTVAL, "public.#{table_name}_id_seq")
+        )
+    end
+
+    def map_existing_ids
+      stage_table
+        .from(stage_table_identifier, public_table_identifier)
+        .where(
+          stage__import_run_id: run_id,
+          stage__transition: %w(KEEP UPDATE DELETE UNDELETE),
+          stage__external_id: :public__external_id
+        )
+        .update(id: :public__id)
+    end
+
+    private
+
+    def stage_table_identifier
+      :"#{stage_schema}__#{table_name}___stage"
+    end
+
+    def stage_table
+      database[stage_table_identifier]
+    end
+
+    def public_table_identifier
+      :"#{table_name}___public"
+    end
+
+    def public_table
+      database[public_table_identifier]
+    end
+
+  end
+end