beetle_etl 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c5a5a3f4d0b4170ccb1581a50a2fccaf32be2cb
-  data.tar.gz: c8beb0f668010410cadefe1ae7ab7c7708032acc
+  metadata.gz: 6a726a9734d6866687319a6742cc8db41ef68b64
+  data.tar.gz: 82be15d660033bd3d957879ec351c0479b26623f
 SHA512:
-  metadata.gz: 3b88ee3cd93eb9344b95bc0b31e01da0a1419d90115fe7a435541b8a336cc3eef8dbad68df819f9993dab3f326be7a91f7b89f8f1e3133b81584834c8f973d65
-  data.tar.gz: e696f99dee13095e6ac7ee32a4fc7e0364744bafbc28caca41b8ab5ce87af3c66e832493b116070545401e9e43bbc34d7582f9d004991beac6b994d3a2d2a9f4
+  metadata.gz: 01c1408c035afb9d0dcadb9382a3db442318f48c74517f1376c3a6ef615675ef891df8559c172be6e6b1f7f47d2f29c4a6965493a613d93a24d439c40f224246
+  data.tar.gz: 111f73fd2692ed88f4ce91c6f4da838efe13d1d12fc00bfb4f63cb00383509ffa1d171bf265362ded8f01ddde7f0a0e0659d552f7be8b31188c94bb4bb984e59

data/.byebug_history

@@ -0,0 +1,8 @@
+continue
+backtrace
+stack
+trace
+c
+continue
+c
+target_table_name

data/.travis.yml

@@ -1,11 +1,16 @@
 language: ruby
 rvm:
   - 2.0.0
-  - 2.1.2
+  - 2.1.0
+  - 2.2.0
+  - 2.3.0
 addons:
   postgresql: "9.3"
   code_climate:
     repo_token: fcd6d8c28da900609a2cf903716d858621b8ce68152edbcebe6908a9a3f5d3d5
+before_install:
+  - gem update --system
+  - gem update bundler
 
 before_script:
   - psql -c 'create database travis_ci_test;' -U postgres

data/README.md

@@ -32,12 +32,34 @@ Make sure the tables you want to import contain columns named ```external_id```
 
 ### Configuration
 
-    BeetleETL.configure do |config|
-      config.transformation_file = # path to your transformation file
-      config.database_config = # sequel database config
-    # or config.database = # sequel database instance
-      config.external_source = ‘name_of_your_source’
-      config.logger = Logger.new(STDOUT)
+Create a configuration object
+
+    configuration = BeetleETL::Configuration.new do |config|
+      # path to your transformation file
+      config.transformation_file = "../my_fancy_transformations"
+
+      # sequel database config
+      config.database_config = {
+        adapter: 'postgres'
+        encoding: utf8
+        host: my_host
+        database: my_database
+        username: 'foo'
+        password: 'bar'
+        pool: 5
+        pool_timeout: 360
+        connect_timeout: 360
+      }
+      # or config.database = # sequel database instance
+
+      # name of your soruce
+      config.external_source = "important_data"
+
+      # target schema in case you use postgres schemas
+      config.target_schema = "public" # default
+
+      # logger
+      config.logger = Logger.new(STDOUT) # default
     end
 
 ### Defining Imports
@@ -66,8 +88,8 @@ Fill a ```transformation``` file with import directives like this:
           ON data.org_id = o.id
       SQL
     end
-    
-    
+
+
 ```import``` takes the name of the table you want to fill and the configuration as arguments.
 With ```columns``` you define what columns BeetleETL is supposed to fill in your application’s table.
 The ```query``` transforms the data. Make sure that you insert into ```#{stage_table}``` as the name of the actual table, that this inserts into will be filled in by BeetleETL during runtime.
@@ -76,7 +98,7 @@ Define any foreign references your table has to other tables using the ```refrec
 
 ### Running BeetleETL
 
-    BeetleETL.import
+    BeetleETL.import(configuration)
 
 ## Development
 

data/beetle_etl.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'sequel', '>= 4.0.0'
   spec.add_runtime_dependency 'activesupport', '>= 4.0.0'
 
-  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'timecop', '>= 0.7.0'
   spec.add_development_dependency 'pg', '>= 0.18.0'

data/lib/beetle_etl.rb

@@ -5,7 +5,7 @@ require 'logger'
 
 module BeetleETL
 
-  InvalidConfigurationError = Class.new(StandardError)
+  require 'beetle_etl/configuration'
 
   require 'beetle_etl/dsl/dsl'
   require 'beetle_etl/dsl/transformation'
@@ -28,61 +28,19 @@ module BeetleETL
   require 'beetle_etl/import'
   require 'beetle_etl/reporter'
 
-  class Configuration
-    attr_accessor \
-      :database_config,
-      :database,
-      :transformation_file,
-      :stage_schema,
-      :target_schema,
-      :external_source,
-      :logger
-
-    def initialize
-      @target_schema = 'public'
-      @logger = ::Logger.new(STDOUT)
-    end
-  end
-
   class << self
 
-    def import
+    def import(config = Configuration.new)
+      yield config if block_given?
+
       begin
-        report = Import.new.run
-        Reporter.new(report).log_summary
+        report = Import.new(config).run
+        Reporter.new(config, report).log_summary
         report
       ensure
-        @database.disconnect if @database
-      end
-    end
-
-    def configure
-      yield(config)
-    end
-
-    def config
-      @config ||= Configuration.new
-    end
-
-    def logger
-      config.logger
-    end
-
-    def database
-      if config.database
-        config.database
-      elsif config.database_config
-        @database ||= Sequel.connect(config.database_config)
-      else
-        msg = "Either Sequel connection database_config or a Sequel Database object required"
-        raise InvalidConfigurationError.new(msg)
+        config.disconnect_database
       end
     end
 
-    def reset
-      @config = nil
-      @database = nil
-    end
-
   end
 end

data/lib/beetle_etl/configuration.rb

@@ -0,0 +1,39 @@
+module BeetleETL
+  InvalidConfigurationError = Class.new(StandardError)
+
+  class Configuration
+    attr_accessor \
+      :transformation_file,
+      :stage_schema,
+      :external_source,
+      :logger
+
+    attr_writer \
+      :database,
+      :database_config,
+      :target_schema
+
+    def initialize
+      @target_schema = 'public'
+      @logger = ::Logger.new(STDOUT)
+    end
+
+    def database
+      if [@database, @database_config].none?
+        msg = "Either Sequel connection database_config or a Sequel Database object required"
+        raise InvalidConfigurationError.new(msg)
+      end
+
+      @database ||= Sequel.connect(@database_config)
+    end
+
+    def disconnect_database
+      database.disconnect if @database_config
+    end
+
+    def target_schema
+      @target_schema != 'public' ? @target_schema : nil
+    end
+
+  end
+end

data/lib/beetle_etl/dsl/dsl.rb

@@ -3,7 +3,8 @@ module BeetleETL
 
     attr_reader :column_names, :relations, :query_strings
 
-    def initialize(table_name)
+    def initialize(config, table_name)
+      @config = config
       @table_name = table_name
       @column_names = []
       @relations = {}
@@ -25,7 +26,10 @@ module BeetleETL
     # query helper methods
 
     def stage_table(table_name = nil)
-      BeetleETL::Naming.stage_table_name_sql(table_name || @table_name)
+      BeetleETL::Naming.stage_table_name_sql(
+        @config.external_source,
+        table_name || @table_name
+      )
     end
 
     def combined_key(*args)

data/lib/beetle_etl/dsl/transformation.rb

@@ -5,9 +5,9 @@ module BeetleETL
 
     attr_reader :table_name
 
-    def initialize(table_name, setup, helpers = nil)
+    def initialize(config, table_name, setup, helpers = nil)
       @table_name = table_name
-      @parsed = DSL.new(table_name).tap do |dsl|
+      @parsed = DSL.new(config, table_name).tap do |dsl|
         dsl.instance_exec(&helpers) if helpers
         dsl.instance_exec(&setup)
       end

data/lib/beetle_etl/dsl/transformation_loader.rb

@@ -1,18 +1,19 @@
 module BeetleETL
   class TransformationLoader
 
-    def initialize
+    def initialize(config)
+      @config = config
       @transformations = []
       @helper_definitions = nil
     end
 
     def load
-      File.open(BeetleETL.config.transformation_file, 'r') do |file|
+      File.open(@config.transformation_file, 'r') do |file|
         instance_eval file.read
       end
 
       @transformations.map do |(table_name, setup)|
-        Transformation.new(table_name, setup, @helper_definitions)
+        Transformation.new(@config, table_name, setup, @helper_definitions)
       end
     end
 

data/lib/beetle_etl/import.rb

@@ -3,6 +3,10 @@ require 'active_support/core_ext/hash/deep_merge'
 module BeetleETL
   class Import
 
+    def initialize(config)
+      @config = config
+    end
+
     def run
       setup
       import
@@ -12,14 +16,14 @@ module BeetleETL
 
     def setup
       transformations.each do |t|
-        CreateStage.new(t.table_name, t.relations, t.column_names).run
+        CreateStage.new(@config, t.table_name, t.relations, t.column_names).run
       end
     end
 
     def import
-      data_report = AsyncStepRunner.new(data_steps).run
-      load_report = BeetleETL.database.transaction do
-        AsyncStepRunner.new(load_steps).run
+      data_report = AsyncStepRunner.new(@config, data_steps).run
+      load_report = @config.database.transaction do
+        AsyncStepRunner.new(@config, load_steps).run
       end
 
       data_report.deep_merge load_report
@@ -27,7 +31,7 @@ module BeetleETL
 
     def cleanup
       transformations.each do |t|
-        DropStage.new(t.table_name).run
+        DropStage.new(@config, t.table_name).run
       end
     end
 
@@ -36,22 +40,22 @@ module BeetleETL
     def data_steps
       transformations.flat_map do |t|
         [
-          Transform.new(t.table_name, t.dependencies, t.query),
-          MapRelations.new(t.table_name, t.relations),
-          TableDiff.new(t.table_name),
-          AssignIds.new(t.table_name),
+          Transform.new(@config, t.table_name, t.dependencies, t.query),
+          MapRelations.new(@config, t.table_name, t.relations),
+          TableDiff.new(@config, t.table_name),
+          AssignIds.new(@config, t.table_name),
         ]
       end
     end
 
     def load_steps
       transformations.map do |t|
-        Load.new(t.table_name, t.relations)
+        Load.new(@config, t.table_name, t.relations)
       end
     end
 
     def transformations
-      @transformations ||= TransformationLoader.new.load
+      @transformations ||= TransformationLoader.new(@config).load
     end
 
   end

data/lib/beetle_etl/naming.rb

@@ -5,32 +5,22 @@ module BeetleETL
 
     extend self
 
-    def stage_table_name(table_name = nil)
-      name = (table_name || @table_name).to_s
-      digest = Digest::MD5.hexdigest(name)
-      "#{BeetleETL.config.external_source}-#{name}-#{digest}"[0, 63]
+    def stage_table_name(external_source, table_name)
+      digest = Digest::MD5.hexdigest(table_name.to_s)
+      "#{external_source.to_s}-#{table_name.to_s}-#{digest}"[0, 63]
     end
 
-    def stage_table_name_sql(table_name = nil)
-      %Q("#{stage_table_name(table_name)}")
+    def stage_table_name_sql(external_source, table_name)
+      %Q("#{stage_table_name(external_source, table_name)}")
     end
 
-    def target_table_name(table_name = nil)
-      name = (table_name || @table_name).to_s
-      [target_schema, name].compact.join('.')
+    def target_table_name(target_schema, table_name)
+      schema = target_schema ? target_schema.to_s : nil
+      [schema, table_name.to_s].compact.join('.')
     end
 
-    def target_table_name_sql(table_name = nil)
-      name = (table_name || @table_name).to_s
-      target_table_name= [target_schema, name].compact.join('"."')
-      %Q("#{target_table_name}")
-    end
-
-    private
-
-    def target_schema
-      target_schema = BeetleETL.config.target_schema
-      target_schema != 'public' ? target_schema : nil
+    def target_table_name_sql(target_schema, table_name)
+      %Q("#{target_table_name(target_schema, table_name)}")
     end
 
   end

data/lib/beetle_etl/reporter.rb

@@ -1,12 +1,13 @@
 module BeetleETL
   class Reporter
 
-    def initialize(report)
+    def initialize(config, report)
+      @config = config
       @report = report
     end
 
     def log_summary
-      BeetleETL.logger.info(summary)
+      @config.logger.info(summary)
     end
 
     private

data/lib/beetle_etl/step_runner/async_step_runner.rb

@@ -1,7 +1,9 @@
 module BeetleETL
   class AsyncStepRunner
 
-    def initialize(steps)
+    def initialize(config, steps)
+      @config = config
+
       @dependency_resolver = DependencyResolver.new(steps)
       @steps = steps
 
@@ -39,14 +41,14 @@ module BeetleETL
     def run_step_async(step)
       Thread.new do
         begin
-          BeetleETL.logger.info("started step #{step.name}")
+          @config.logger.info("started step #{step.name}")
 
           started_at = Time.now
           step.run
           finished_at = Time.now
 
           duration = Time.at(finished_at - started_at).utc.strftime("%H:%M:%S")
-          BeetleETL.logger.info("finished #{step.name} in #{duration}")
+          @config.logger.info("finished #{step.name} in #{duration}")
 
           @queue.push [
             step.table_name,
@@ -55,7 +57,7 @@ module BeetleETL
           ]
 
         rescue => e
-          BeetleETL.logger.fatal(e.message)
+          @config.logger.fatal(e.message)
           raise e
         end
       end.abort_on_exception = true