beetle_etl 0.0.2 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b08e51fe6819079c6b8317636030d5d678b102cc
-  data.tar.gz: 6afa19db3171f105fa275f99c82c07306aa9c568
+  metadata.gz: 9d792ed373d43d6c0ab7ec27241eaf18fb59736b
+  data.tar.gz: 7b08688ce87fbff7eea72b362773444ec4700b2c
 SHA512:
-  metadata.gz: a915efc65e4450aa4ba7cec71a75b89082d6b65be4cd2e6db88ee8e540f9bc0d1db96afd18c787ac504309460447bfdbb20ab462e012ac503e8fe8cdecb65880
-  data.tar.gz: afc74720f9875d7a447030bb14fbc4da90c1004b5c725fb55b0630bbc073d3d15560e6c5e2b7002951e1989eb4b63b956508411811c146986820aee2679a6462
+  metadata.gz: 7022ab2cc6a60f57f061d3b1acaa8a125da6db7e9cec76421189ae2fa2c4729b4134cb8246a8c14283aeb7769e1ef92a75f480e2d6be3a3e41ba2e8017b7ee2c
+  data.tar.gz: fa6e2f2f74cf1c4a6a03e53c2d76c962154e8d64f656d32690101bdd4dfa636dda9036213b29dfc675d07e9ed381a6527e94abb462f4f7eabfa3c3f11f279ee1

data/Gemfile

@@ -2,3 +2,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in beetle_etl.gemspec
 gemspec
+
+group :test do
+  gem 'rake'
+  gem 'codeclimate-test-reporter'
+  gem 'byebug'
+end

data/beetle_etl.gemspec

@@ -18,12 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'sequel', '>= 4.13.0'
+  spec.add_runtime_dependency 'sequel', '>= 4.0.0'
 
   spec.add_development_dependency 'bundler', '~> 1.6'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec', '~> 3.1.0'
-  spec.add_development_dependency 'pg'
-  spec.add_development_dependency 'codeclimate-test-reporter'
-  spec.add_development_dependency 'activesupport'
+  spec.add_development_dependency 'rspec', '>= 3.0.0'
+  spec.add_development_dependency 'timecop', '>= 0.7.0'
+  spec.add_development_dependency 'pg', '>= 0.18.0'
+  spec.add_development_dependency 'activesupport', '>= 4.2.0'
 end

data/lib/beetle_etl.rb

@@ -10,17 +10,20 @@ module BeetleETL
   require 'beetle_etl/dsl/transformation'
   require 'beetle_etl/dsl/transformation_loader'
 
+  require 'beetle_etl/naming'
+
   require 'beetle_etl/steps/step'
+  require 'beetle_etl/steps/create_stage'
   require 'beetle_etl/steps/transform'
   require 'beetle_etl/steps/map_relations'
   require 'beetle_etl/steps/table_diff'
   require 'beetle_etl/steps/assign_ids'
   require 'beetle_etl/steps/load'
+  require 'beetle_etl/steps/drop_stage'
 
   require 'beetle_etl/task_runner/dependency_resolver'
   require 'beetle_etl/task_runner/task_runner'
 
-  require 'beetle_etl/state'
   require 'beetle_etl/import'
 
   class Configuration
@@ -29,23 +32,20 @@ module BeetleETL
       :database,
       :transformation_file,
       :stage_schema,
+      :public_schema,
       :external_source
 
     def initialize
-      @stage_schema = 'stage'
+      @public_schema = 'public'
     end
   end
 
   class << self
 
     def import
-      state.start_import
-
       begin
-        Import.run
-        state.mark_as_succeeded
+        Import.new.run
       rescue Exception => e
-        state.mark_as_failed
         raise e
       ensure
         @database.disconnect if @database
@@ -71,13 +71,8 @@ module BeetleETL
       end
     end
 
-    def state
-      @state ||= State.new
-    end
-
     def reset
       @config = nil
-      @state = nil
       @database = nil
     end
 

data/lib/beetle_etl/dsl/dsl.rb

@@ -1,11 +1,17 @@
 module BeetleETL
   class DSL
 
-    attr_reader :relations, :query_string
+    attr_reader :column_names, :relations, :query_strings
 
     def initialize(table_name)
       @table_name = table_name
+      @column_names = []
       @relations = {}
+      @query_strings = []
+    end
+
+    def columns(*column_names)
+      @column_names = column_names
     end
 
     def references(foreign_table, on: foreign_key)
@@ -13,25 +19,18 @@ module BeetleETL
     end
 
     def query(query)
-      @query_string = query
+      @query_strings << query
     end
 
+    # query helper methods
 
-    def stage_table
-      %Q("#{BeetleETL.config.stage_schema}"."#{@table_name}")
-    end
-
-    def external_source
-      'source'
+    def stage_table(table_name = nil)
+      BeetleETL::Naming.stage_table_name_sql(table_name || @table_name)
     end
 
     def combined_key(*args)
       %Q('[' || #{args.join(%q[ || ',' || ])} || ']')
     end
 
-    def import_run_id
-      BeetleETL.state.run_id
-    end
-
   end
 end

data/lib/beetle_etl/dsl/transformation.rb

@@ -5,9 +5,16 @@ module BeetleETL
 
     attr_reader :table_name
 
-    def initialize(table_name, setup)
+    def initialize(table_name, setup, helpers = nil)
       @table_name = table_name
-      (@parsed = DSL.new(table_name)).instance_eval(&setup)
+      @parsed = DSL.new(table_name).tap do |dsl|
+        dsl.instance_eval(&helpers) if helpers
+        dsl.instance_eval(&setup)
+      end
+    end
+
+    def column_names
+      @parsed.column_names.map(&:to_sym)
     end
 
     def relations
@@ -19,7 +26,7 @@ module BeetleETL
     end
 
     def query
-      @parsed.query_string
+      @parsed.query_strings.join(';')
     end
 
   end

data/lib/beetle_etl/dsl/transformation_loader.rb

@@ -1,21 +1,29 @@
 module BeetleETL
-  module TransformationLoader
-    extend self
+  class TransformationLoader
 
-    def load
+    def initialize
       @transformations = []
+      @helper_definitions = nil
+    end
 
+    def load
       File.open(BeetleETL.config.transformation_file, 'r') do |file|
         instance_eval file.read
       end
 
-      @transformations
+      @transformations.map do |(table_name, setup)|
+        Transformation.new(table_name, setup, @helper_definitions)
+      end
     end
 
     private
 
     def import(table_name, &setup)
-      @transformations << Transformation.new(table_name, setup)
+      @transformations << [table_name, setup]
+    end
+
+    def helpers(&helper_definitions)
+      @helper_definitions = helper_definitions
     end
 
   end

data/lib/beetle_etl/import.rb

@@ -1,13 +1,15 @@
 module BeetleETL
-  module Import
-
-    extend self
+  class Import
 
     def run
       TaskRunner.new(data_steps).run
       BeetleETL.database.transaction do
         TaskRunner.new(load_steps).run
       end
+    rescue => e
+      raise e
+    ensure
+      TaskRunner.new(cleanup_steps).run
     end
 
     private
@@ -15,6 +17,7 @@ module BeetleETL
     def data_steps
       transformations.flat_map do |t|
         [
+          CreateStage.new(t.table_name, t.relations, t.column_names),
           Transform.new(t.table_name, t.dependencies, t.query),
           MapRelations.new(t.table_name, t.relations),
           TableDiff.new(t.table_name),
@@ -29,8 +32,12 @@ module BeetleETL
       end
     end
 
+    def cleanup_steps
+      transformations.map { |t| DropStage.new(t.table_name) }
+    end
+
     def transformations
-      @transformations ||= TransformationLoader.load
+      @transformations ||= TransformationLoader.new.load
     end
 
   end

data/lib/beetle_etl/naming.rb

@@ -0,0 +1,37 @@
+require 'digest'
+
+module BeetleETL
+  module Naming
+
+    extend self
+
+    def stage_table_name(table_name = nil)
+      name = (table_name || @table_name).to_s
+      digest = Digest::MD5.hexdigest(name)
+      "#{name}-#{digest}"[0, 63]
+    end
+
+    def stage_table_name_sql(table_name = nil)
+      %Q("#{stage_table_name(table_name)}")
+    end
+
+    def public_table_name(table_name = nil)
+      name = (table_name || @table_name).to_s
+      [public_schema, name].compact.join('.')
+    end
+
+    def public_table_name_sql(table_name = nil)
+      name = (table_name || @table_name).to_s
+      public_table_name= [public_schema, name].compact.join('"."')
+      %Q("#{public_table_name}")
+    end
+
+    private
+
+    def public_schema
+      public_schema = BeetleETL.config.public_schema
+      public_schema != 'public' ? public_schema : nil
+    end
+
+  end
+end

data/lib/beetle_etl/steps/assign_ids.rb

@@ -6,50 +6,26 @@ module BeetleETL
     end
 
     def run
-      [
-        Thread.new { assign_new_ids },
-        Thread.new { map_existing_ids }
-      ].each(&:join)
+      assign_new_ids
+      map_existing_ids
     end
 
     def assign_new_ids
-      stage_table
-        .where(
-          import_run_id: run_id,
-          transition: 'CREATE'
-        )
-        .update(
-          id: Sequel.function(:NEXTVAL, "public.#{table_name}_id_seq")
-        )
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql}
+        SET id = nextval('#{table_name}_id_seq')
+        WHERE transition = 'CREATE'
+      SQL
     end
 
     def map_existing_ids
-      stage_table
-        .from(stage_table_identifier, public_table_identifier)
-        .where(
-          stage__import_run_id: run_id,
-          stage__transition: %w(KEEP UPDATE DELETE UNDELETE),
-          stage__external_id: :public__external_id
-        )
-        .update(id: :public__id)
-    end
-
-    private
-
-    def stage_table_identifier
-      :"#{stage_schema}__#{table_name}___stage"
-    end
-
-    def stage_table
-      database[stage_table_identifier]
-    end
-
-    def public_table_identifier
-      :"#{table_name}___public"
-    end
-
-    def public_table
-      database[public_table_identifier]
+      database.execute <<-SQL
+        UPDATE #{stage_table_name_sql} stage
+        SET id = public.id
+        FROM #{public_table_name_sql} public
+        WHERE stage.transition IN ('KEEP', 'UPDATE', 'DELETE', 'UNDELETE')
+        AND stage.external_id = public.external_id
+      SQL
     end
 
   end

data/lib/beetle_etl/steps/create_stage.rb

@@ -0,0 +1,59 @@
+module BeetleETL
+  class CreateStage < Step
+
+    def initialize(table_name, relations, column_names)
+      super(table_name)
+      @relations = relations
+      @column_names = column_names
+    end
+
+    def dependencies
+      Set.new
+    end
+
+    def run
+      database.execute <<-SQL
+        CREATE TABLE #{stage_table_name_sql} (
+          id integer,
+          external_id character varying(255),
+          transition character varying(255),
+
+          #{[
+            payload_column_definitions,
+            relation_column_definitions
+          ].compact.join(',')}
+        )
+      SQL
+    end
+
+    private
+
+    def payload_column_definitions
+      definitions = (@column_names - @relations.keys).map do |column_name|
+        "#{column_name} #{column_type(column_name)}"
+      end
+      definitions.join(',') if definitions.any?
+    end
+
+    def relation_column_definitions
+      definitions = @relations.map do |foreign_key_name, table|
+        <<-SQL
+          #{foreign_key_name} integer,
+          external_#{foreign_key_name} character varying(255)
+        SQL
+      end
+      definitions.join(',') if definitions.any?
+    end
+
+    def column_type(column_name)
+      @column_types ||= Hash[database.schema(public_table_name.to_sym)]
+        .reduce({}) do |acc, (name, schema)|
+          acc[name.to_sym] = schema.fetch(:db_type)
+          acc
+        end
+
+      @column_types[column_name]
+    end
+
+  end
+end

data/lib/beetle_etl/steps/drop_stage.rb

@@ -0,0 +1,15 @@
+module BeetleETL
+  class DropStage < Step
+
+    def dependencies
+      Set.new
+    end
+
+    def run
+      database.execute <<-SQL
+        DROP TABLE IF EXISTS #{stage_table_name_sql}
+      SQL
+    end
+
+  end
+end

data/lib/beetle_etl/steps/load.rb

@@ -2,93 +2,78 @@ module BeetleETL
   class Load < Step
 
     IMPORTER_COLUMNS = %i[
-      import_run_id
       external_source
       transition
     ]
 
-    attr_reader :relations
-
     def initialize(table_name, relations)
       super(table_name)
       @relations = relations
     end
 
     def run
-      %w(create update delete undelete).map do |transition|
-        Thread.new { public_send(:"load_#{transition}") }
-      end.each(&:join)
+      %w(create update delete undelete).each do |transition|
+        public_send(:"load_#{transition}")
+      end
     end
 
     def dependencies
-      relations.values.map { |d| Load.step_name(d) }.to_set
+      @relations.values.map { |d| Load.step_name(d) }.to_set
     end
 
     def load_create
       just_now = now
-      database[table_name].import(
-        data_columns + [:external_source, :created_at, :updated_at],
-        database[:"#{stage_schema}__#{table_name}"]
-          .select(*data_columns)
-          .where(
-            import_run_id: run_id,
-            transition: 'CREATE'
-          )
-          .select_more(external_source, just_now, just_now)
-        )
+
+      database.execute <<-SQL
+        INSERT INTO #{public_table_name_sql}
+          (#{data_columns.join(', ')}, external_source, created_at, updated_at)
+        SELECT
+          #{data_columns.join(', ')},
+          '#{external_source}',
+          '#{just_now}',
+          '#{just_now}'
+        FROM #{stage_table_name_sql}
+        WHERE transition = 'CREATE'
+      SQL
     end
 
     def load_update
-      updates = updatable_columns.reduce({updated_at: now}) do |acc, column|
-        acc[column] = :"stage__#{column}"
-        acc
-      end
-
-      database.from(
-        :"#{table_name}___public",
-        :"#{stage_schema}__#{table_name}___stage"
-      )
-        .where(
-          stage__id: :public__id,
-          stage__transition: 'UPDATE',
-          stage__import_run_id: run_id,
-        )
-        .update(updates)
+      database.execute <<-SQL
+        UPDATE #{public_table_name_sql} public
+        SET
+          #{updatable_columns.map { |c| %Q("#{c}" = stage."#{c}") }.join(',')},
+          "updated_at" = '#{now}'
+        FROM #{stage_table_name_sql} stage
+        WHERE stage.id = public.id
+          AND stage.transition = 'UPDATE'
+      SQL
     end
 
     def load_delete
       just_now = now
-      database.from(
-          :"#{table_name}___public",
-          :"#{stage_schema}__#{table_name}___stage"
-        )
-        .where(
-          stage__id: :public__id,
-          stage__transition: 'DELETE',
-          stage__import_run_id: run_id,
-        )
-        .update(
-          updated_at: just_now,
-          deleted_at: just_now,
-        )
+
+      database.execute <<-SQL
+        UPDATE #{public_table_name_sql} public
+        SET
+          updated_at = '#{just_now}',
+          deleted_at = '#{just_now}'
+        FROM #{stage_table_name_sql} stage
+        WHERE stage.id = public.id
+          AND stage.transition = 'DELETE'
+      SQL
     end
 
     def load_undelete
-      updates = updatable_columns.reduce({updated_at: now, deleted_at: nil}) do |acc, column|
-        acc[column] = :"stage__#{column}"
-        acc
-      end
-
-      database.from(
-        :"#{table_name}___public",
-        :"#{stage_schema}__#{table_name}___stage"
-      )
-        .where(
-          stage__id: :public__id,
-          stage__transition: 'UNDELETE',
-          stage__import_run_id: run_id,
-        )
-        .update(updates)
+      database.execute <<-SQL
+        UPDATE #{public_table_name_sql} public
+        SET
+          #{updatable_columns.map { |c| %Q("#{c}" = stage."#{c}") }.join(',')},
+          updated_at = '#{now}',
+          deleted_at = NULL
+        FROM #{stage_table_name_sql} stage
+        WHERE stage.id = public.id
+          AND stage.transition = 'UNDELETE'
+      SQL
     end
 
     private
@@ -98,7 +83,7 @@ module BeetleETL
     end
 
     def table_columns
-      @table_columns ||= database[:"#{stage_schema}__#{table_name}"].columns
+      @table_columns ||= database[stage_table_name.to_sym].columns
     end
 
     def ignored_columns