beetle_etl 0.0.16 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07d1e92398037e916f19762ae9a373de8cd3edfa
-  data.tar.gz: c064dfbfd3dfd4103864c51d8742f537963abf8c
+  metadata.gz: dacfd8801cf603b9cec442ee9ea025bc14026234
+  data.tar.gz: 1a183e975200bb9bdc6e631130903b6ead9739be
 SHA512:
-  metadata.gz: b984f80cdba06d018d49fa9414cf5443b8098b0edf34f1eb149df32f5c718f1af08676ca05743e064bf7e19475ad32a31459feedb68e0a45b091378c8c869665
-  data.tar.gz: 3b80f7d9ca88ed49130a6ef6719e9d65fb28d0dca101a2ee23e191bcb413dcf93ae213fefe194e736169cca2b2ab3e0a67444f975a741a7bf323944c4328fb1d
+  metadata.gz: bacbe3f6292e52b1ad23d38f5e128b1cf39ec63ded808310a8353956f2b11d349b4ea546012beb22e03edbd3de9bdd787ca0bbde329e1755ebacc23c9561a8ca
+  data.tar.gz: cbc51b9be8c35b70c21829af0c38f6db0898f4b5c38a2fc6138968bc63591e91f51d7558b2fff62196ba43789b3255f2347c9982d78de33a4d025e3da167084d

data/README.md

@@ -2,7 +2,9 @@
 [![Build Status](https://travis-ci.org/maiwald/beetle_etl.svg?branch=master)](https://travis-ci.org/maiwald/beetle_etl)
 [![Code Climate](https://codeclimate.com/github/maiwald/beetle_etl.png)](https://codeclimate.com/github/maiwald/beetle_etl)
 
-TODO: Write a gem description
+BeetleETL helps you with synchronising relational databases and recurring imports of data. It is actually quite nice.
+
+It currently only works with PostgreSQL databases.
 
 ## Installation
 
@@ -20,7 +22,53 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### Configuration
+
+    BeetleETL.configure do |config|
+      config.transformation_file = # path to your imports
+      config.database_config = # sequel database config
+    # or config.database = # sequel database instance
+      config.external_source = ‘source_name’
+      config.logger = Logger.new(STDOUT)
+    end
+
+### Defining Imports
+
+Fill a file with all the tables you wish to import and write queries to select the data you want.
+
+    import :departments do
+      columns :name
+
+      references :organisations, on: :organisation_id
+
+      query <<-SQL
+        INSERT INTO #{stage_table} (
+          external_id,
+          name,
+          external_organisation_id
+        )
+
+        SELECT
+          o.id,
+          o.”dep_name”,
+          data.”address”
+
+        FROM ”Organisation” o
+        JOIN additional_data data
+          ON data.org_id = o.id
+      SQL
+    end
+
+
+### Running BeetleETL
+
+    BeetleETL.import
+
+## Development
+
+To run the specs call
+
+    $ bundle exec rspec
 
 ## Contributing
 

data/lib/beetle_etl.rb

@@ -34,12 +34,12 @@ module BeetleETL
       :database,
       :transformation_file,
       :stage_schema,
-      :public_schema,
+      :target_schema,
       :external_source,
       :logger
 
     def initialize
-      @public_schema = 'public'
+      @target_schema = 'public'
       @logger = ::Logger.new(STDOUT)
     end
   end

data/lib/beetle_etl/dsl/transformation.rb

@@ -8,8 +8,8 @@ module BeetleETL
     def initialize(table_name, setup, helpers = nil)
       @table_name = table_name
       @parsed = DSL.new(table_name).tap do |dsl|
-        dsl.instance_eval(&helpers) if helpers
-        dsl.instance_eval(&setup)
+        dsl.instance_exec(&helpers) if helpers
+        dsl.instance_exec(&setup)
       end
     end
 

data/lib/beetle_etl/naming.rb

@@ -15,22 +15,22 @@ module BeetleETL
       %Q("#{stage_table_name(table_name)}")
     end
 
-    def public_table_name(table_name = nil)
+    def target_table_name(table_name = nil)
       name = (table_name || @table_name).to_s
-      [public_schema, name].compact.join('.')
+      [target_schema, name].compact.join('.')
     end
 
-    def public_table_name_sql(table_name = nil)
+    def target_table_name_sql(table_name = nil)
       name = (table_name || @table_name).to_s
-      public_table_name= [public_schema, name].compact.join('"."')
-      %Q("#{public_table_name}")
+      target_table_name= [target_schema, name].compact.join('"."')
+      %Q("#{target_table_name}")
     end
 
     private
 
-    def public_schema
-      public_schema = BeetleETL.config.public_schema
-      public_schema != 'public' ? public_schema : nil
+    def target_schema
+      target_schema = BeetleETL.config.target_schema
+      target_schema != 'public' ? target_schema : nil
     end
 
   end

data/lib/beetle_etl/step_runner/dependency_resolver.rb

@@ -11,7 +11,7 @@ module BeetleETL
 
     def resolvables(resolved)
       @items.select do |item|
-        (item.dependencies.subset?(resolved.to_set) || item.dependencies.empty?) && !resolved.include?(item.name)
+        !resolved.include?(item.name) && all_dependencies_met?(item, resolved)
       end
     end
 
@@ -22,18 +22,15 @@ module BeetleETL
       resolved = []
 
       until items.empty?
-        resolved_names = resolved.flatten.map(&:name).to_set
-
-        resolvable = items.select do |item|
-          item.dependencies.subset?(resolved_names) || item.dependencies.empty?
-        end
-
-        raise UnsatisfiableDependenciesError if resolvable.empty?
-
-        resolvable.each { |r| items.delete r }
-        resolved << resolvable
+        resolvables = items.select { |item| all_dependencies_met?(item, resolved.map(&:name)) }
+        raise UnsatisfiableDependenciesError if resolvables.empty?
+        resolvables.each { |r| resolved << items.delete(r) }
       end
     end
 
+    def all_dependencies_met?(item, resolved)
+      item.dependencies.empty? || item.dependencies.subset?(resolved.to_set)
+    end
+
   end
 end

data/lib/beetle_etl/steps/assign_ids.rb

@@ -8,12 +8,12 @@ module BeetleETL
     def run
       database.execute <<-SQL
         UPDATE #{stage_table_name_sql} stage_update
-        SET id = COALESCE(public.id, nextval('#{table_name}_id_seq'))
+        SET id = COALESCE(target.id, nextval('#{table_name}_id_seq'))
         FROM #{stage_table_name_sql} stage
-        LEFT OUTER JOIN #{public_table_name_sql} public
+        LEFT OUTER JOIN #{target_table_name_sql} target
           on (
-            stage.external_id = public.external_id
-            AND public.external_source = '#{external_source}'
+            stage.external_id = target.external_id
+            AND target.external_source = '#{external_source}'
           )
         WHERE stage_update.external_id = stage.external_id
       SQL

data/lib/beetle_etl/steps/create_stage.rb

@@ -23,6 +23,12 @@ module BeetleETL
 
         #{index_definitions};
 
+        ALTER TABLE #{stage_table_name_sql}
+        SET (
+          autovacuum_enabled = false,
+          toast.autovacuum_enabled = false
+        );
+
         TRUNCATE TABLE #{stage_table_name_sql} RESTART IDENTITY CASCADE;
       SQL
     end
@@ -70,7 +76,7 @@ module BeetleETL
     end
 
     def column_type(column_name)
-      @column_types ||= Hash[database.schema(public_table_name.to_sym)]
+      @column_types ||= Hash[database.schema(target_table_name.to_sym)]
         .reduce({}) do |acc, (name, schema)|
           acc[name.to_sym] = schema.fetch(:db_type)
           acc

data/lib/beetle_etl/steps/load.rb

@@ -12,7 +12,7 @@ module BeetleETL
     end
 
     def run
-      %w(create update delete reinstate).each do |transition|
+      %w(create update delete).each do |transition|
         public_send(:"load_#{transition}")
       end
     end
@@ -25,7 +25,7 @@ module BeetleETL
       just_now = now
 
       database.execute <<-SQL
-        INSERT INTO #{public_table_name_sql}
+        INSERT INTO #{target_table_name_sql}
           (#{data_columns.join(', ')}, external_source, created_at, updated_at)
         SELECT
           #{data_columns.join(', ')},
@@ -39,13 +39,14 @@ module BeetleETL
 
     def load_update
       database.execute <<-SQL
-        UPDATE #{public_table_name_sql} public
+        UPDATE #{target_table_name_sql} target
         SET
           #{updatable_columns.map { |c| %Q("#{c}" = stage."#{c}") }.join(',')},
-          "updated_at" = '#{now}'
+          "updated_at" = '#{now}',
+          deleted_at = NULL
         FROM #{stage_table_name_sql} stage
-        WHERE stage.id = public.id
-          AND stage.transition = 'UPDATE'
+        WHERE stage.id = target.id
+          AND stage.transition IN ('UPDATE', 'REINSTATE')
       SQL
     end
 
@@ -53,29 +54,16 @@ module BeetleETL
       just_now = now
 
       database.execute <<-SQL
-        UPDATE #{public_table_name_sql} public
+        UPDATE #{target_table_name_sql} target
         SET
           updated_at = '#{just_now}',
           deleted_at = '#{just_now}'
         FROM #{stage_table_name_sql} stage
-        WHERE stage.id = public.id
+        WHERE stage.id = target.id
           AND stage.transition = 'DELETE'
       SQL
     end
 
-    def load_reinstate
-      database.execute <<-SQL
-        UPDATE #{public_table_name_sql} public
-        SET
-          #{updatable_columns.map { |c| %Q("#{c}" = stage."#{c}") }.join(',')},
-          updated_at = '#{now}',
-          deleted_at = NULL
-        FROM #{stage_table_name_sql} stage
-        WHERE stage.id = public.id
-          AND stage.transition = 'REINSTATE'
-      SQL
-    end
-
     private
 
     def data_columns

data/lib/beetle_etl/steps/map_relations.rb

@@ -7,7 +7,8 @@ module BeetleETL
     end
 
     def dependencies
-      @relations.values.map { |d| AssignIds.step_name(d) }.to_set << Transform.step_name(table_name)
+      result = Set.new([Transform.step_name(table_name)])
+      result.merge @relations.values.map { |d| AssignIds.step_name(d) }
     end
 
     def run

data/lib/beetle_etl/steps/table_diff.rb

@@ -22,9 +22,9 @@ module BeetleETL
         SET transition = 'CREATE'
         WHERE NOT EXISTS (
           SELECT 1
-          FROM #{public_table_name} public
-          WHERE public.external_id = stage.external_id
-          AND public.external_source = '#{external_source}'
+          FROM #{target_table_name} target
+          WHERE target.external_id = stage.external_id
+          AND target.external_source = '#{external_source}'
         )
       SQL
     end
@@ -35,12 +35,12 @@ module BeetleETL
         SET transition = 'UPDATE'
         WHERE EXISTS (
           SELECT 1
-          FROM #{public_table_name} public
-          WHERE public.external_id = stage.external_id
-          AND public.external_source = '#{external_source}'
-          AND public.deleted_at IS NULL
+          FROM #{target_table_name} target
+          WHERE target.external_id = stage.external_id
+          AND target.external_source = '#{external_source}'
+          AND target.deleted_at IS NULL
           AND
-            (#{public_record_columns.join(', ')})
+            (#{target_record_columns.join(', ')})
             IS DISTINCT FROM
             (#{stage_record_columns.join(', ')})
         )
@@ -52,14 +52,14 @@ module BeetleETL
         INSERT INTO #{stage_table_name_sql}
           (external_id, transition)
         SELECT
-          public.external_id,
+          target.external_id,
           'DELETE'
-        FROM #{public_table_name_sql} public
+        FROM #{target_table_name_sql} target
         LEFT OUTER JOIN #{stage_table_name_sql} stage
-          ON (stage.external_id = public.external_id)
+          ON (stage.external_id = target.external_id)
         WHERE stage.external_id IS NULL
-        AND public.external_source = '#{external_source}'
-        AND public.deleted_at IS NULL
+        AND target.external_source = '#{external_source}'
+        AND target.deleted_at IS NULL
       SQL
     end
 
@@ -69,18 +69,18 @@ module BeetleETL
         SET transition = 'REINSTATE'
         WHERE EXISTS (
           SELECT 1
-          FROM #{public_table_name_sql} public
-          WHERE public.external_id = stage.external_id
-          AND public.external_source = '#{external_source}'
-          AND public.deleted_at IS NOT NULL
+          FROM #{target_table_name_sql} target
+          WHERE target.external_id = stage.external_id
+          AND target.external_source = '#{external_source}'
+          AND target.deleted_at IS NOT NULL
         )
       SQL
     end
 
     private
 
-    def public_record_columns
-      prefixed_columns(data_columns, 'public')
+    def target_record_columns
+      prefixed_columns(data_columns, 'target')
     end
 
     def stage_record_columns

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "0.0.16"
+  VERSION = "0.0.19"
 end

data/spec/steps/load_spec.rb

@@ -65,7 +65,7 @@ module BeetleETL
 
     describe '#run' do
       it 'runs all load steps' do
-        %w(create update delete reinstate).each do |transition|
+        %w(create update delete).each do |transition|
           expect(subject).to receive(:"load_#{transition}")
         end
 
@@ -74,7 +74,7 @@ module BeetleETL
     end
 
     describe '#load_create' do
-      it 'loads records into the public table' do
+      it 'loads records into the target table' do
         insert_into(subject.stage_table_name.to_sym).values(
           [ :id , :external_id  , :transition , :external_foo_id , :foo_id , :payload       ] ,
           [ 3   , 'external_id' , 'CREATE'    , 'foo_id'         , 22      , 'content'      ] ,
@@ -108,10 +108,8 @@ module BeetleETL
           [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
         )
       end
-    end
 
-    describe '#load_delete' do
-      it 'marks existing records as deleted' do
+      it 'restores deleted records' do
         insert_into(:example_table).values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
           [ 1   , 'external_id' , external_source  , 22      , yesterday   , yesterday   , nil         , 'content' ] ,
@@ -119,20 +117,20 @@ module BeetleETL
 
         insert_into(subject.stage_table_name.to_sym).values(
           [ :id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
-          [ 1   , 'external_id' , 'DELETE'    , 'foo_id'         , 33      , 'updated content' ] ,
+          [ 1   , 'external_id' , 'REINSTATE' , 'foo_id'         , 33      , 'updated content' ] ,
         )
 
-        subject.load_delete
+        subject.load_update
 
         expect(:example_table).to have_values(
-          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
-          [ 1   , 'external_id' , external_source  , 22      , yesterday   , now         , now         , 'content' ] ,
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
+          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
         )
       end
     end
 
-    describe '#load_reinstate' do
-      it 'restores deleted records' do
+    describe '#load_delete' do
+      it 'marks existing records as deleted' do
         insert_into(:example_table).values(
           [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
           [ 1   , 'external_id' , external_source  , 22      , yesterday   , yesterday   , nil         , 'content' ] ,
@@ -140,14 +138,14 @@ module BeetleETL
 
         insert_into(subject.stage_table_name.to_sym).values(
           [ :id , :external_id  , :transition , :external_foo_id , :foo_id , :payload          ] ,
-          [ 1   , 'external_id' , 'REINSTATE' , 'foo_id'         , 33      , 'updated content' ] ,
+          [ 1   , 'external_id' , 'DELETE'    , 'foo_id'         , 33      , 'updated content' ] ,
         )
 
-        subject.load_reinstate
+        subject.load_delete
 
         expect(:example_table).to have_values(
-          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload          ] ,
-          [ 1   , 'external_id' , external_source  , 33      , yesterday   , now         , nil         , 'updated content' ] ,
+          [ :id , :external_id  , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload  ] ,
+          [ 1   , 'external_id' , external_source  , 22      , yesterday   , now         , now         , 'content' ] ,
         )
       end
     end

data/spec/task_runner/dependency_resolver_spec.rb

@@ -34,6 +34,7 @@ module BeetleETL
       end
 
       it 'returns all items with met dependencies' do
+        expect(resolver.resolvables([:a])).to match_array([b, c])
         expect(resolver.resolvables([:a, :b, :c])).to match_array([d])
         expect(resolver.resolvables([:a, :b, :c, :d])).to match_array([e, f])
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.19
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel