chicago-etl 0.0.9

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+
+gem "chicagowarehouse", "~> 0.4"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2"
+  gem "timecop"
+  gem "yard"
+  gem "flog"
+  gem "jeweler"
+  gem "rcov", :platforms => :mri_18
+  gem "simplecov", :platforms => :mri_19
+  gem "ZenTest"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 notonthehighstreet.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,21 @@
+= chicago-etl
+
+HIGHLY EXPERIMENTAL. If you use this, you'll find that things will vanish without warning and you'll be terrified.
+
+An ETL pipeline for use with Chicago Warehouse.
+
+== Contributing to chicago-etl
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2012 notonthehighstreet.com. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "chicago-etl"
+  gem.homepage = "http://github.com/notonthehighstreet/chicago-etl"
+  gem.license = "MIT"
+  gem.summary = "Chicago ETL"
+  gem.description = "ETL tools for Chicago"
+  gem.email = "roland.swingler@gmail.com"
+  gem.authors = ["Roland Swingler"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/VERSION

@@ -0,0 +1 @@
+0.0.9

data/chicago-etl.gemspec

@@ -0,0 +1,117 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "chicago-etl"
+  s.version = "0.0.9"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Roland Swingler"]
+  s.date = "2013-02-19"
+  s.description = "ETL tools for Chicago"
+  s.email = "roland.swingler@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "chicago-etl.gemspec",
+    "lib/chicago-etl.rb",
+    "lib/chicago/etl.rb",
+    "lib/chicago/etl/batch.rb",
+    "lib/chicago/etl/buffering_insert_writer.rb",
+    "lib/chicago/etl/counter.rb",
+    "lib/chicago/etl/key_builder.rb",
+    "lib/chicago/etl/load_dataset_builder.rb",
+    "lib/chicago/etl/mysql_dumpfile.rb",
+    "lib/chicago/etl/mysql_load_file_value_transformer.rb",
+    "lib/chicago/etl/screens/column_screen.rb",
+    "lib/chicago/etl/screens/composite_screen.rb",
+    "lib/chicago/etl/screens/invalid_element.rb",
+    "lib/chicago/etl/screens/missing_value.rb",
+    "lib/chicago/etl/screens/out_of_bounds.rb",
+    "lib/chicago/etl/sequel/dependant_tables.rb",
+    "lib/chicago/etl/sequel/filter_to_etl_batch.rb",
+    "lib/chicago/etl/sequel/load_data_infile.rb",
+    "lib/chicago/etl/sink.rb",
+    "lib/chicago/etl/table_builder.rb",
+    "lib/chicago/etl/task_invocation.rb",
+    "lib/chicago/etl/tasks.rb",
+    "lib/chicago/etl/transformations/add_insert_timestamp.rb",
+    "lib/chicago/etl/transformations/uk_post_code.rb",
+    "lib/chicago/etl/transformations/uk_post_code_field.rb",
+    "spec/db_connections.yml.dist",
+    "spec/etl/batch_spec.rb",
+    "spec/etl/counter_spec.rb",
+    "spec/etl/etl_batch_id_dataset_filter.rb",
+    "spec/etl/key_builder_spec.rb",
+    "spec/etl/load_dataset_builder_spec.rb",
+    "spec/etl/mysql_dumpfile_spec.rb",
+    "spec/etl/mysql_load_file_value_transformer_spec.rb",
+    "spec/etl/screens/composite_screen_spec.rb",
+    "spec/etl/screens/invalid_element_spec.rb",
+    "spec/etl/screens/missing_value_spec.rb",
+    "spec/etl/screens/out_of_bounds_spec.rb",
+    "spec/etl/sequel/dependant_tables_spec.rb",
+    "spec/etl/sequel/filter_to_etl_batch_spec.rb",
+    "spec/etl/sequel/load_data_infile_spec.rb",
+    "spec/etl/sink_spec.rb",
+    "spec/etl/table_builder_spec.rb",
+    "spec/etl/task_spec.rb",
+    "spec/etl/transformations/add_insert_timestamp_spec.rb",
+    "spec/etl/transformations/uk_post_code_field_spec.rb",
+    "spec/etl/transformations/uk_post_code_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/notonthehighstreet/chicago-etl"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.25"
+  s.summary = "Chicago ETL"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+      s.add_development_dependency(%q<rspec>, ["~> 2"])
+      s.add_development_dependency(%q<timecop>, [">= 0"])
+      s.add_development_dependency(%q<yard>, [">= 0"])
+      s.add_development_dependency(%q<flog>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<ZenTest>, [">= 0"])
+    else
+      s.add_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+      s.add_dependency(%q<rspec>, ["~> 2"])
+      s.add_dependency(%q<timecop>, [">= 0"])
+      s.add_dependency(%q<yard>, [">= 0"])
+      s.add_dependency(%q<flog>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<ZenTest>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<chicagowarehouse>, ["~> 0.4"])
+    s.add_dependency(%q<rspec>, ["~> 2"])
+    s.add_dependency(%q<timecop>, [">= 0"])
+    s.add_dependency(%q<yard>, [">= 0"])
+    s.add_dependency(%q<flog>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<ZenTest>, [">= 0"])
+  end
+end
+

data/lib/chicago/etl/batch.rb

@@ -0,0 +1,110 @@
+require 'fileutils'
+require 'logger'
+
+module Chicago
+  module ETL
+    # A particular "run" of the ETL process.
+    #
+    # All ETL tasks should be executed in the context of a Batch.
+    #
+    # A batch creates a temporary directory under tmp/batches/:id
+    # where it stores various logs and extract files.
+    class Batch < Sequel::Model
+      set_dataset :etl_batches
+
+      one_to_many :task_invocations
+
+      class << self
+        # Returns the Batch that should be used for the ETL process.
+        #
+        # A new batch is returned, unless the previous batch did not
+        # finish successfully.
+        #
+        # This should be used in preference to new or create.
+        def instance
+          (last_batch.nil? || last_batch.finished?) ? new : last_batch
+        end
+      
+        # Returns the last batch run, or nil if this is the first batch.
+        def last_batch
+          order(:started_at).last
+        end
+      end
+
+      # Deprecated.
+      def load(task_name, &block)
+        perform_task(:load, task_name, &block)
+      end
+
+      # Deprecated.
+      def transform(task_name, &block)
+        perform_task(:extract, task_name, &block)
+      end
+
+      # Deprecated.
+      def extract(task_name, &block)
+        perform_task(:extract, task_name, &block)
+      end
+
+      # Perform a named task if it hasn't already run successfully in
+      # this batch.
+      def perform_task(stage, task_name, &block)
+        task = find_or_create_task_invocation(stage, task_name)
+        task.perform(&block) unless task.finished?
+      end
+
+      # Returns the directory files & batch logs will be written to.
+      def dir
+        @dir ||= File.join(Chicago.project_root, "tmp", "batches", id.to_s)
+      end
+
+      # Starts this batch.
+      def start(extract_to=nil)
+        self.extracted_to = extract_to || Date.today
+        save
+        if state == "Started"
+          log.info "Started ETL batch #{id}."
+        else
+          log.info "Resumed ETL batch #{id}."
+        end
+        self
+      end
+
+      # Finishes this batch, and sets the finished_at timestamp.
+      def finish
+        update(:state => "Finished", :finished_at => Time.now)
+      end
+
+      # Sets this batch to the Error state.
+      def error
+        update(:state => "Error")
+      end
+
+      # Returns true if this batch is finished.
+      def finished?
+        state == "Finished"
+      end
+
+      # Returns true if in the error state
+      def in_error?
+        state == "Error"
+      end
+
+      # Returns the logger for this batch
+      def log
+        @log ||= Logger.new(File.join(dir, "log"))
+      end
+
+      def after_create # :nodoc:
+        FileUtils.mkdir_p(dir, :mode => 0777)
+      end
+
+      private
+
+      def find_or_create_task_invocation(stage, name)
+        attrs = {:stage => stage.to_s.downcase, :name => name.to_s}
+        task_invocations_dataset.filter(attrs).first || add_task_invocation(attrs)
+      end
+    end
+  end
+end

data/lib/chicago/etl/buffering_insert_writer.rb

@@ -0,0 +1,36 @@
+require 'chicago/etl/sink'
+
+module Chicago
+  module ETL
+    # Wrapper around a dataset to allowed buffered inserts.
+    #
+    # @api public
+    class BufferingInsertWriter < Sink
+      # The number of rows written before inserting to the DB.
+      BUFFER_SIZE = 10_000
+      
+      def initialize(dataset, column_names, key=nil)
+        super([], column_names, key)
+        @dataset = dataset
+      end
+      
+      def flush
+        @dataset.insert_replace.import(column_names, output)
+        output.clear
+      end
+
+      protected
+
+      def write(row)
+        output << @column_names.map {|name| row[name] }
+        flush if reached_buffer_limit?
+      end
+
+      private
+
+      def reached_buffer_limit?
+        output.size >= BUFFER_SIZE
+      end
+    end
+  end
+end

data/lib/chicago/etl/counter.rb

@@ -0,0 +1,36 @@
+require 'thread'
+
+module Chicago
+  module ETL
+    # Provides a thread-safe wrapper around an incrementing number.
+    #
+    # Intended to be used for key builders, rather than using the
+    # database's AUTO INCREMENT functionality.
+    #
+    # @api private
+    class Counter
+      # Returns the current number this counter is on.
+      attr_reader :current
+
+      # Creates a new counter, optionally with a starting count.
+      def initialize(current_number=0, &block)
+        @mutex = Mutex.new
+        if block
+          @block = block
+        else
+          @current = current_number || 0
+        end
+      end
+
+      # Returns the next number.
+      #
+      # Modifies the current state of the counter.
+      def next
+        @current = (@block.call || 0) if @current.nil?
+        @mutex.synchronize do
+          @current += 1
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/key_builder.rb

@@ -0,0 +1,198 @@
+require 'digest/md5'
+require 'chicago/etl/buffering_insert_writer'
+
+module Chicago
+  module ETL
+    # Builds a surrogate key for a dimension record, without relying
+    # on the database's AUTO_INCREMENT functionality.
+    #
+    # We avoid AUTO_INCREMENT because we need to be able to get the
+    # key mappings without having anything to do with the database -
+    # this allows us to use bulk load.
+    #
+    # @api public
+    class KeyBuilder
+      # @api private
+      class Factory
+        attr_reader :table, :staging_db
+
+        def initialize(table, staging_db)
+          @table = table
+          @staging_db = staging_db
+        end
+
+        def make
+          if dimension?
+            key_table = staging_db[table.key_table_name]
+            key_sink = BufferingInsertWriter.new(key_table,
+                                                 [:original_id, :dimension_id])
+            
+            if table.identifiable?
+              IdentifiableDimensionKeyBuilder.new(key_table, key_sink)
+            else
+              HashingKeyBuilder.new(key_table, key_sink, columns_to_hash)
+            end
+          elsif fact?
+            FactKeyBuilder.new(staging_db[table.table_name])
+          end
+        end
+
+        private
+
+        def dimension?
+          table.kind_of?(Chicago::Schema::Dimension)
+        end
+
+        def fact?
+          table.kind_of?(Chicago::Schema::Fact)
+        end
+
+        def columns_to_hash
+          if table.natural_key.nil?
+            table.columns.map(&:name)
+          else
+            table.natural_key
+          end
+        end
+      end
+
+      # Returns an appropriate key builder for a schema table, using
+      # the staging database for key management where necessary.
+      def self.for_table(table, staging_db)
+        Factory.new(table, staging_db).make
+      end
+
+      def initialize(key_table, key_sink)
+        @key_table = key_table
+        @new_keys = key_sink
+        @counter = Counter.new { key_table.max(:dimension_id) }
+      end
+
+      # Returns a surrogate key, given a record row.
+      #
+      # @raises Chicago::ETL::KeyError if the surrogate key cannot be
+      #   determined from the row data.
+      def key(row)
+        fetch_cache unless @key_mapping
+        row_id = original_key(row)
+        new_key = @key_mapping[row_id]
+        
+        if new_key
+          new_key
+        else
+          new_key = @counter.next
+          @new_keys << {
+            :original_id => key_for_insert(row_id), 
+            :dimension_id => new_key
+          }
+          @key_mapping[row_id] = new_key
+        end
+      end
+
+      # Returns the original key for the row.
+      #
+      # Overridden by subclasses.
+      def original_key(row)
+      end
+
+      # Flushes any newly created keys to the key table.
+      def flush
+        @new_keys.flush
+      end
+      
+      protected
+
+      attr_reader :key_table
+
+      def fetch_cache
+        @key_mapping = key_table.
+          select_hash(original_key_select_fragment, :dimension_id)
+      end
+    end
+
+    # Key builder for identifiable dimensions.
+    #
+    # This should not be instantiated directly, use
+    # KeyBuilder.for_dimension.
+    #
+    # @api private
+    class IdentifiableDimensionKeyBuilder < KeyBuilder
+      def key(row)
+        raise KeyError.new("Row does not have an original_id field") unless row.has_key?(:original_id)
+        super
+      end
+
+      def original_key(row)
+        row[:original_id]
+      end
+
+      def key_for_insert(original_id)
+        original_id
+      end
+
+      def original_key_select_fragment
+        :original_id
+      end
+    end
+
+    # Key builder for dimensions with natuaral keys, but no simple
+    # key.
+    #
+    # This should not be instantiated directly, use
+    # KeyBuilder.for_dimension.
+    #
+    # @api private
+    class HashingKeyBuilder < KeyBuilder
+      attr_reader   :columns
+      attr_accessor :hash_preparation
+
+      def initialize(key_table, key_sink, columns)
+        super(key_table, key_sink)
+        @columns = columns
+        @hash_preparation = lambda {|column| column.to_s.upcase }
+      end
+
+      def original_key(row)
+        str = columns.map {|column| hash_preparation.call(row[column]) }.join
+        Digest::MD5.hexdigest(str).upcase
+      end
+
+      def key_for_insert(original_id)
+        ("0x" + original_id).lit
+      end
+
+      def original_key_select_fragment
+        :hex.sql_function(:original_id).as(:original_id)
+      end
+    end
+
+    # Returns ids for Fact tables.
+    #
+    # Fact table surrogate ids are transient - there is no expectation
+    # that the same fact row will have the same id between
+    # invocations. This is ok, because all facts should have a natural
+    # key defined - the id generated by this is purely for convenience
+    # and linking to error rows.
+    #
+    # As a result fact keys aren't stored in a key table - they are
+    # never referenced by any other tables in the system.
+    #
+    # In addition, the same row passed twice will get a different id. 
+    class FactKeyBuilder
+      def initialize(db_table, key_sink=nil)
+        @db_table = db_table
+        @counter = Counter.new { @db_table.max(:id) }
+      end
+
+      # Returns an id given a row - the row actually has no bearing on
+      # the id returned.
+      def key(row)
+        @counter.next
+      end
+
+      # No-op, provided for interface compatability.
+      def flush
+      end
+    end
+  end
+end