chicago-etl 0.0.13 → 0.1.0

data/Gemfile

@@ -1,6 +1,10 @@
 source "http://rubygems.org"
 
 gem "chicagowarehouse", "~> 0.4"
+gem "fastercsv", :platform => :ruby_18
+gem "sequel"
+gem "sequel_load_data_infile", ">= 0.0.2", :require => "sequel/load_data_infile"
+gem "sequel_fast_columns", :require => "sequel/fast_columns"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
@@ -9,8 +13,9 @@ group :development do
   gem "timecop"
   gem "yard"
   gem "flog"
-  gem "jeweler"
-  gem "rcov", :platforms => :mri_18
-  gem "simplecov", :platforms => :mri_19
+  gem "simplecov", :platforms => :mri_19, :require => false
   gem "ZenTest"
+  gem "mysql", "2.8.1"  
+  gem "bundler", "~> 1"
+  gem "jeweler"
 end

data/README.rdoc

@@ -14,8 +14,11 @@ An ETL pipeline for use with Chicago Warehouse.
 * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
+== Authors
+
+Roland Swingler (@knaveofdiamonds)
+
 == Copyright
 
 Copyright (c) 2012 notonthehighstreet.com. See LICENSE.txt for
 further details.
-

data/VERSION

@@ -1 +1 @@
-0.0.13
+0.1.0

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.0.13"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-04-16"
+  s.date = "2013-09-05"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -25,52 +25,74 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "chicago-etl.gemspec",
+    "chicago-flow.gemspec",
     "lib/chicago-etl.rb",
     "lib/chicago/etl.rb",
     "lib/chicago/etl/batch.rb",
-    "lib/chicago/etl/buffering_insert_writer.rb",
+    "lib/chicago/etl/core_extensions.rb",
     "lib/chicago/etl/counter.rb",
+    "lib/chicago/etl/dataset_batch_stage.rb",
     "lib/chicago/etl/key_builder.rb",
     "lib/chicago/etl/load_dataset_builder.rb",
-    "lib/chicago/etl/mysql_dumpfile.rb",
-    "lib/chicago/etl/mysql_load_file_value_transformer.rb",
+    "lib/chicago/etl/load_pipeline_stage_builder.rb",
+    "lib/chicago/etl/pipeline.rb",
+    "lib/chicago/etl/schema_table_sink_factory.rb",
     "lib/chicago/etl/screens/column_screen.rb",
-    "lib/chicago/etl/screens/composite_screen.rb",
     "lib/chicago/etl/screens/invalid_element.rb",
     "lib/chicago/etl/screens/missing_value.rb",
     "lib/chicago/etl/screens/out_of_bounds.rb",
     "lib/chicago/etl/sequel/dependant_tables.rb",
     "lib/chicago/etl/sequel/filter_to_etl_batch.rb",
-    "lib/chicago/etl/sequel/load_data_infile.rb",
-    "lib/chicago/etl/sink.rb",
     "lib/chicago/etl/table_builder.rb",
     "lib/chicago/etl/task_invocation.rb",
     "lib/chicago/etl/tasks.rb",
-    "lib/chicago/etl/transformations/add_insert_timestamp.rb",
+    "lib/chicago/etl/transformations.rb",
     "lib/chicago/etl/transformations/uk_post_code.rb",
     "lib/chicago/etl/transformations/uk_post_code_field.rb",
+    "lib/chicago/flow/array_sink.rb",
+    "lib/chicago/flow/array_source.rb",
+    "lib/chicago/flow/dataset_source.rb",
+    "lib/chicago/flow/errors.rb",
+    "lib/chicago/flow/filter.rb",
+    "lib/chicago/flow/mysql.rb",
+    "lib/chicago/flow/mysql_file_serializer.rb",
+    "lib/chicago/flow/mysql_file_sink.rb",
+    "lib/chicago/flow/null_sink.rb",
+    "lib/chicago/flow/pipeline_endpoint.rb",
+    "lib/chicago/flow/pipeline_stage.rb",
+    "lib/chicago/flow/sink.rb",
+    "lib/chicago/flow/transformation.rb",
+    "lib/chicago/flow/transformation_chain.rb",
     "spec/db_connections.yml.dist",
     "spec/etl/batch_spec.rb",
+    "spec/etl/core_extensions_spec.rb",
     "spec/etl/counter_spec.rb",
+    "spec/etl/dataset_batch_stage_spec.rb",
     "spec/etl/etl_batch_id_dataset_filter.rb",
     "spec/etl/key_builder_spec.rb",
     "spec/etl/load_dataset_builder_spec.rb",
-    "spec/etl/mysql_dumpfile_spec.rb",
-    "spec/etl/mysql_load_file_value_transformer_spec.rb",
-    "spec/etl/screens/composite_screen_spec.rb",
+    "spec/etl/pipeline_stage_builder_spec.rb",
+    "spec/etl/schema_table_sink_factory_spec.rb",
     "spec/etl/screens/invalid_element_spec.rb",
     "spec/etl/screens/missing_value_spec.rb",
     "spec/etl/screens/out_of_bounds_spec.rb",
     "spec/etl/sequel/dependant_tables_spec.rb",
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
-    "spec/etl/sequel/load_data_infile_expression_spec.rb",
-    "spec/etl/sequel/load_data_infile_spec.rb",
-    "spec/etl/sink_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",
-    "spec/etl/transformations/add_insert_timestamp_spec.rb",
     "spec/etl/transformations/uk_post_code_field_spec.rb",
     "spec/etl/transformations/uk_post_code_spec.rb",
+    "spec/etl/transformations_spec.rb",
+    "spec/flow/array_sink_spec.rb",
+    "spec/flow/array_source_spec.rb",
+    "spec/flow/dataset_source_spec.rb",
+    "spec/flow/filter_spec.rb",
+    "spec/flow/mysql_file_serializer_spec.rb",
+    "spec/flow/mysql_file_sink_spec.rb",
+    "spec/flow/mysql_integration_spec.rb",
+    "spec/flow/pipeline_stage_spec.rb",
+    "spec/flow/transformation_chain_spec.rb",
+    "spec/flow/transformation_spec.rb",
     "spec/spec_helper.rb"
   ]
   s.homepage = "http://github.com/notonthehighstreet/chicago-etl"
@@ -84,35 +106,50 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+      s.add_runtime_dependency(%q<sequel>, [">= 0"])
+      s.add_runtime_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+      s.add_runtime_dependency(%q<sequel_fast_columns>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2"])
       s.add_development_dependency(%q<timecop>, [">= 0"])
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<flog>, [">= 0"])
-      s.add_development_dependency(%q<jeweler>, [">= 0"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
       s.add_development_dependency(%q<ZenTest>, [">= 0"])
+      s.add_development_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_development_dependency(%q<bundler>, ["~> 1"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
       s.add_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+      s.add_dependency(%q<sequel>, [">= 0"])
+      s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+      s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2"])
       s.add_dependency(%q<timecop>, [">= 0"])
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<flog>, [">= 0"])
-      s.add_dependency(%q<jeweler>, [">= 0"])
-      s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<simplecov>, [">= 0"])
       s.add_dependency(%q<ZenTest>, [">= 0"])
+      s.add_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_dependency(%q<bundler>, ["~> 1"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
     end
   else
     s.add_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+    s.add_dependency(%q<sequel>, [">= 0"])
+    s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+    s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2"])
     s.add_dependency(%q<timecop>, [">= 0"])
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<flog>, [">= 0"])
-    s.add_dependency(%q<jeweler>, [">= 0"])
-    s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<simplecov>, [">= 0"])
     s.add_dependency(%q<ZenTest>, [">= 0"])
+    s.add_dependency(%q<mysql>, ["= 2.8.1"])
+    s.add_dependency(%q<bundler>, ["~> 1"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
   end
 end
 

data/chicago-flow.gemspec

@@ -0,0 +1,92 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "chicago-flow"
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Roland Swingler"]
+  s.date = "2013-06-05"
+  s.description = "Dataflow-style processing for hash-like rows"
+  s.email = "roland.swingler@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/chicago/flow.rb",
+    "lib/chicago/flow/array_sink.rb",
+    "lib/chicago/flow/array_source.rb",
+    "lib/chicago/flow/dataset_source.rb",
+    "lib/chicago/flow/filter.rb",
+    "lib/chicago/flow/mysql.rb",
+    "lib/chicago/flow/mysql_file_serializer.rb",
+    "lib/chicago/flow/mysql_file_sink.rb",
+    "lib/chicago/flow/pipeline_endpoint.rb",
+    "lib/chicago/flow/pipeline_stage.rb",
+    "lib/chicago/flow/sink.rb",
+    "lib/chicago/flow/transformation.rb",
+    "lib/chicago/flow/transformation_chain.rb",
+    "spec/array_sink_spec.rb",
+    "spec/array_source_spec.rb",
+    "spec/database.yml.dist",
+    "spec/dataset_source_spec.rb",
+    "spec/filter_spec.rb",
+    "spec/mysql_file_serializer_spec.rb",
+    "spec/mysql_file_sink_spec.rb",
+    "spec/mysql_integration_spec.rb",
+    "spec/pipeline_stage_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/transformation_chain_spec.rb",
+    "spec/transformation_spec.rb"
+  ]
+  s.homepage = "http://github.com/notonthehighstreet/chicago-flow"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.24"
+  s.summary = "Dataflow-style processing for hash-like rows"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+      s.add_runtime_dependency(%q<sequel>, [">= 0"])
+      s.add_runtime_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+      s.add_runtime_dependency(%q<sequel_fast_columns>, [">= 0"])
+      s.add_development_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_development_dependency(%q<rspec>, ["~> 2"])
+      s.add_development_dependency(%q<bundler>, ["~> 1"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
+    else
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+      s.add_dependency(%q<sequel>, [">= 0"])
+      s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+      s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
+      s.add_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_dependency(%q<rspec>, ["~> 2"])
+      s.add_dependency(%q<bundler>, ["~> 1"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+    end
+  else
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+    s.add_dependency(%q<sequel>, [">= 0"])
+    s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
+    s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
+    s.add_dependency(%q<mysql>, ["= 2.8.1"])
+    s.add_dependency(%q<rspec>, ["~> 2"])
+    s.add_dependency(%q<bundler>, ["~> 1"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+  end
+end
+

data/lib/chicago/etl/batch.rb

@@ -32,21 +32,27 @@ module Chicago
       end
 
       # Deprecated.
+      #
+      # @deprecated Use perform_task instead
       def load(task_name, &block)
         perform_task(:load, task_name, &block)
       end
 
       # Deprecated.
+      #
+      # @deprecated Use perform_task instead
       def transform(task_name, &block)
         perform_task(:extract, task_name, &block)
       end
 
       # Deprecated.
+      #
+      # @deprecated Use perform_task instead
       def extract(task_name, &block)
         perform_task(:extract, task_name, &block)
       end
 
-      # Perform a named task if it hasn't already run successfully in
+      # Performs a named task if it hasn't already run successfully in
       # this batch.
       def perform_task(stage, task_name, &block)
         task = find_or_create_task_invocation(stage, task_name)
@@ -95,7 +101,8 @@ module Chicago
         @log ||= Logger.new(File.join(dir, "log"))
       end
 
-      def after_create # :nodoc:
+      # @api private
+      def after_create
         FileUtils.mkdir_p(dir, :mode => 0777)
       end
 

data/lib/chicago/etl/core_extensions.rb

@@ -0,0 +1,12 @@
+class Hash
+  def put(key, value)
+    store(key, value)
+    self
+  end
+
+  def modify_existing(key)
+    value = self[key]
+    self[key] = yield value unless value.nil?
+    self
+  end
+end

data/lib/chicago/etl/counter.rb

@@ -12,7 +12,14 @@ module Chicago
       # Returns the current number this counter is on.
       attr_reader :current
 
-      # Creates a new counter, optionally with a starting count.
+      # Creates a new counter.
+      #
+      # May optionally be created with a starting count, either as a
+      # number or as a block which generates a number.
+      #
+      #     Counter.new(41).next # returns 42
+      #     Counter.new { 2 + 2 }.next # returns 5
+      #     
       def initialize(current_number=0, &block)
         @mutex = Mutex.new
         if block

data/lib/chicago/etl/dataset_batch_stage.rb

@@ -0,0 +1,52 @@
+module Chicago
+  module ETL
+    # Links a PipelineStage to a Dataset.
+    #
+    # Allows deferring constructing a DatasetSource until extract
+    # time, so that it can be filtered to an ETL batch appropriately.
+    class DatasetBatchStage
+      attr_reader :name
+
+      def initialize(name, dataset, pipeline_stage, options={})
+        @name = name
+        @dataset = dataset
+        @pipeline_stage = pipeline_stage
+        @filter_strategy = options[:filter_strategy] || lambda {|dataset, etl_batch|
+          dataset.filter_to_etl_batch(etl_batch)
+        }
+        @truncate_pre_load = !!options[:truncate_pre_load]
+      end
+
+      # Executes this ETL stage.
+      #
+      # Configures the dataset and flows rows into the pipeline.
+      def execute(etl_batch, reextract=false)
+        if @truncate_pre_load
+          pipeline_stage.sinks.each {|sink| sink.truncate }
+        elsif reextract && pipeline_stage.sink(:error)
+          pipeline_stage.sink(:error).truncate
+        end
+
+        pipeline_stage.execute(source(etl_batch, reextract))
+      end
+
+      # Returns the pipeline for this stage.
+      def pipeline_stage
+        @pipeline_stage.sink(:default).
+          set_constant_values(:_inserted_at => Time.now)
+        @pipeline_stage
+      end
+
+      # Returns a DatasetSource for the provided dataset filtered to
+      # the ETL batch as appropriate.
+      def source(etl_batch, reextract=false)
+        if reextract
+          filtered_dataset = @dataset
+        else
+          filtered_dataset = @filter_strategy.call(@dataset, etl_batch)
+        end
+        Chicago::Flow::DatasetSource.new(filtered_dataset)
+      end
+    end
+  end
+end

data/lib/chicago/etl/key_builder.rb

@@ -1,5 +1,4 @@
 require 'digest/md5'
-require 'chicago/etl/buffering_insert_writer'
 
 module Chicago
   module ETL
@@ -12,6 +11,8 @@ module Chicago
     #
     # @api public
     class KeyBuilder
+      # Creates the appropriate KeyBuilder for a star schema table.
+      #
       # @api private
       class Factory
         attr_reader :table, :staging_db
@@ -20,19 +21,17 @@ module Chicago
           @table = table
           @staging_db = staging_db
         end
-
+        
         def make
           if dimension?
             key_table = staging_db[table.key_table_name]
-            key_sink = BufferingInsertWriter.new(key_table,
-                                                 [:original_id, :dimension_id])
             
             if table.identifiable?
-              IdentifiableDimensionKeyBuilder.new(key_table, key_sink)
+              IdentifiableDimensionKeyBuilder.new(key_table)
             elsif existing_hash_column?(table)
-              ExistingHashColumnKeyBuilder.new(key_table, key_sink)
+              ExistingHashColumnKeyBuilder.new(key_table)
             else
-              HashingKeyBuilder.new(key_table, key_sink, columns_to_hash)
+              HashingKeyBuilder.new(key_table, columns_to_hash)
             end
           elsif fact?
             FactKeyBuilder.new(staging_db[table.table_name])
@@ -68,9 +67,8 @@ module Chicago
         Factory.new(table, staging_db).make
       end
 
-      def initialize(key_table, key_sink)
+      def initialize(key_table)
         @key_table = key_table
-        @new_keys = key_sink
         @counter = Counter.new { key_table.max(:dimension_id) }
       end
 
@@ -84,14 +82,15 @@ module Chicago
         new_key = @key_mapping[row_id]
         
         if new_key
-          new_key
+          [new_key, nil]
         else
           new_key = @counter.next
-          @new_keys << {
-            :original_id => key_for_insert(row_id), 
-            :dimension_id => new_key
-          }
           @key_mapping[row_id] = new_key
+
+          [new_key, {
+             :original_id => row_id, 
+             :dimension_id => new_key
+           }]
         end
       end
 
@@ -100,11 +99,6 @@ module Chicago
       # Overridden by subclasses.
       def original_key(row)
       end
-
-      # Flushes any newly created keys to the key table.
-      def flush
-        @new_keys.flush
-      end
       
       protected
 
@@ -132,10 +126,6 @@ module Chicago
         row[:original_id]
       end
 
-      def key_for_insert(original_id)
-        original_id
-      end
-
       def original_key_select_fragment
         :original_id
       end
@@ -150,10 +140,6 @@ module Chicago
         row[:hash].upcase
       end
 
-      def key_for_insert(original_id)
-        ("0x" + original_id).lit
-      end
-
       def original_key_select_fragment
         :hex.sql_function(:original_id).as(:original_id)
       end
@@ -170,8 +156,8 @@ module Chicago
       attr_reader   :columns
       attr_accessor :hash_preparation
 
-      def initialize(key_table, key_sink, columns)
-        super(key_table, key_sink)
+      def initialize(key_table, columns)
+        super(key_table)
         @columns = columns
         @hash_preparation = lambda {|column| column.to_s.upcase }
       end
@@ -181,10 +167,6 @@ module Chicago
         Digest::MD5.hexdigest(str).upcase
       end
 
-      def key_for_insert(original_id)
-        ("0x" + original_id).lit
-      end
-
       def original_key_select_fragment
         :hex.sql_function(:original_id).as(:original_id)
       end
@@ -203,7 +185,7 @@ module Chicago
     #
     # In addition, the same row passed twice will get a different id. 
     class FactKeyBuilder
-      def initialize(db_table, key_sink=nil)
+      def initialize(db_table)
         @db_table = db_table
         @counter = Counter.new { @db_table.max(:id) }
       end
@@ -211,11 +193,7 @@ module Chicago
       # Returns an id given a row - the row actually has no bearing on
       # the id returned.
       def key(row)
-        @counter.next
-      end
-
-      # No-op, provided for interface compatability.
-      def flush
+        [@counter.next, nil]
       end
     end
   end

data/lib/chicago/etl/load_dataset_builder.rb

@@ -2,6 +2,9 @@ require 'set'
 
 module Chicago
   module ETL
+    # Currently unused - work in progress.
+    # 
+    # @api private
     class LoadDatasetBuilder
       def initialize(&block)
         @constructed_columns = {}
@@ -69,7 +72,6 @@ module Chicago
           raise "Column #{name} was either ambiguous or non-existant"
         end
       end
-        
     end
   end
 end