chicago-etl 0.0.13 → 0.1.0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 13
-  version: 0.0.13
+  version: 0.1.0
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-04-16 00:00:00 Z
+date: 2013-09-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -36,16 +36,16 @@ dependencies:
   version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 3
         segments: 
-        - 2
-        version: "2"
+        - 0
+        version: "0"
   requirement: *id002
-  type: :development
+  type: :runtime
   prerelease: false
-  name: rspec
+  name: fastercsv
 - !ruby/object:Gem::Dependency 
   version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -57,23 +57,25 @@ dependencies:
         - 0
         version: "0"
   requirement: *id003
-  type: :development
+  type: :runtime
   prerelease: false
-  name: timecop
+  name: sequel
 - !ruby/object:Gem::Dependency 
   version_requirements: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 27
         segments: 
         - 0
-        version: "0"
+        - 0
+        - 2
+        version: 0.0.2
   requirement: *id004
-  type: :development
+  type: :runtime
   prerelease: false
-  name: yard
+  name: sequel_load_data_infile
 - !ruby/object:Gem::Dependency 
   version_requirements: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -85,23 +87,23 @@ dependencies:
         - 0
         version: "0"
   requirement: *id005
-  type: :development
+  type: :runtime
   prerelease: false
-  name: flog
+  name: sequel_fast_columns
 - !ruby/object:Gem::Dependency 
   version_requirements: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 7
         segments: 
-        - 0
-        version: "0"
+        - 2
+        version: "2"
   requirement: *id006
   type: :development
   prerelease: false
-  name: jeweler
+  name: rspec
 - !ruby/object:Gem::Dependency 
   version_requirements: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -115,7 +117,7 @@ dependencies:
   requirement: *id007
   type: :development
   prerelease: false
-  name: rcov
+  name: timecop
 - !ruby/object:Gem::Dependency 
   version_requirements: &id008 !ruby/object:Gem::Requirement 
     none: false
@@ -129,7 +131,7 @@ dependencies:
   requirement: *id008
   type: :development
   prerelease: false
-  name: simplecov
+  name: yard
 - !ruby/object:Gem::Dependency 
   version_requirements: &id009 !ruby/object:Gem::Requirement 
     none: false
@@ -143,7 +145,79 @@ dependencies:
   requirement: *id009
   type: :development
   prerelease: false
+  name: flog
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id010
+  type: :development
+  prerelease: false
+  name: simplecov
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id011
+  type: :development
+  prerelease: false
   name: ZenTest
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id012 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 45
+        segments: 
+        - 2
+        - 8
+        - 1
+        version: 2.8.1
+  requirement: *id012
+  type: :development
+  prerelease: false
+  name: mysql
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id013 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        version: "1"
+  requirement: *id013
+  type: :development
+  prerelease: false
+  name: bundler
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id014 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id014
+  type: :development
+  prerelease: false
+  name: jeweler
 description: ETL tools for Chicago
 email: roland.swingler@gmail.com
 executables: []
@@ -162,52 +236,74 @@ files:
 - Rakefile
 - VERSION
 - chicago-etl.gemspec
+- chicago-flow.gemspec
 - lib/chicago-etl.rb
 - lib/chicago/etl.rb
 - lib/chicago/etl/batch.rb
-- lib/chicago/etl/buffering_insert_writer.rb
+- lib/chicago/etl/core_extensions.rb
 - lib/chicago/etl/counter.rb
+- lib/chicago/etl/dataset_batch_stage.rb
 - lib/chicago/etl/key_builder.rb
 - lib/chicago/etl/load_dataset_builder.rb
-- lib/chicago/etl/mysql_dumpfile.rb
-- lib/chicago/etl/mysql_load_file_value_transformer.rb
+- lib/chicago/etl/load_pipeline_stage_builder.rb
+- lib/chicago/etl/pipeline.rb
+- lib/chicago/etl/schema_table_sink_factory.rb
 - lib/chicago/etl/screens/column_screen.rb
-- lib/chicago/etl/screens/composite_screen.rb
 - lib/chicago/etl/screens/invalid_element.rb
 - lib/chicago/etl/screens/missing_value.rb
 - lib/chicago/etl/screens/out_of_bounds.rb
 - lib/chicago/etl/sequel/dependant_tables.rb
 - lib/chicago/etl/sequel/filter_to_etl_batch.rb
-- lib/chicago/etl/sequel/load_data_infile.rb
-- lib/chicago/etl/sink.rb
 - lib/chicago/etl/table_builder.rb
 - lib/chicago/etl/task_invocation.rb
 - lib/chicago/etl/tasks.rb
-- lib/chicago/etl/transformations/add_insert_timestamp.rb
+- lib/chicago/etl/transformations.rb
 - lib/chicago/etl/transformations/uk_post_code.rb
 - lib/chicago/etl/transformations/uk_post_code_field.rb
+- lib/chicago/flow/array_sink.rb
+- lib/chicago/flow/array_source.rb
+- lib/chicago/flow/dataset_source.rb
+- lib/chicago/flow/errors.rb
+- lib/chicago/flow/filter.rb
+- lib/chicago/flow/mysql.rb
+- lib/chicago/flow/mysql_file_serializer.rb
+- lib/chicago/flow/mysql_file_sink.rb
+- lib/chicago/flow/null_sink.rb
+- lib/chicago/flow/pipeline_endpoint.rb
+- lib/chicago/flow/pipeline_stage.rb
+- lib/chicago/flow/sink.rb
+- lib/chicago/flow/transformation.rb
+- lib/chicago/flow/transformation_chain.rb
 - spec/db_connections.yml.dist
 - spec/etl/batch_spec.rb
+- spec/etl/core_extensions_spec.rb
 - spec/etl/counter_spec.rb
+- spec/etl/dataset_batch_stage_spec.rb
 - spec/etl/etl_batch_id_dataset_filter.rb
 - spec/etl/key_builder_spec.rb
 - spec/etl/load_dataset_builder_spec.rb
-- spec/etl/mysql_dumpfile_spec.rb
-- spec/etl/mysql_load_file_value_transformer_spec.rb
-- spec/etl/screens/composite_screen_spec.rb
+- spec/etl/pipeline_stage_builder_spec.rb
+- spec/etl/schema_table_sink_factory_spec.rb
 - spec/etl/screens/invalid_element_spec.rb
 - spec/etl/screens/missing_value_spec.rb
 - spec/etl/screens/out_of_bounds_spec.rb
 - spec/etl/sequel/dependant_tables_spec.rb
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
-- spec/etl/sequel/load_data_infile_expression_spec.rb
-- spec/etl/sequel/load_data_infile_spec.rb
-- spec/etl/sink_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
-- spec/etl/transformations/add_insert_timestamp_spec.rb
 - spec/etl/transformations/uk_post_code_field_spec.rb
 - spec/etl/transformations/uk_post_code_spec.rb
+- spec/etl/transformations_spec.rb
+- spec/flow/array_sink_spec.rb
+- spec/flow/array_source_spec.rb
+- spec/flow/dataset_source_spec.rb
+- spec/flow/filter_spec.rb
+- spec/flow/mysql_file_serializer_spec.rb
+- spec/flow/mysql_file_sink_spec.rb
+- spec/flow/mysql_integration_spec.rb
+- spec/flow/pipeline_stage_spec.rb
+- spec/flow/transformation_chain_spec.rb
+- spec/flow/transformation_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/notonthehighstreet/chicago-etl
 licenses: 

data/lib/chicago/etl/buffering_insert_writer.rb

@@ -1,36 +0,0 @@
-require 'chicago/etl/sink'
-
-module Chicago
-  module ETL
-    # Wrapper around a dataset to allowed buffered inserts.
-    #
-    # @api public
-    class BufferingInsertWriter < Sink
-      # The number of rows written before inserting to the DB.
-      BUFFER_SIZE = 10_000
-      
-      def initialize(dataset, column_names, key=nil)
-        super([], column_names, key)
-        @dataset = dataset
-      end
-      
-      def flush
-        @dataset.insert_replace.import(column_names, output)
-        output.clear
-      end
-
-      protected
-
-      def write(row)
-        output << @column_names.map {|name| row[name] }
-        flush if reached_buffer_limit?
-      end
-
-      private
-
-      def reached_buffer_limit?
-        output.size >= BUFFER_SIZE
-      end
-    end
-  end
-end

data/lib/chicago/etl/mysql_dumpfile.rb

@@ -1,32 +0,0 @@
-require 'chicago/etl/sink'
-
-module Chicago
-  module ETL
-    # Wrapper around FasterCSV's output object, to convert values to a
-    # format required by MySQL's LOAD DATA INFILE command.
-    #
-    # @api public
-    class MysqlDumpfile < Sink
-      # Creates a new writer.
-      #
-      # @param csv a FasterCSV output object
-      # @param [Symbol] column_names columns to be output
-      # @param key an optional key to ensure rows are written only once.
-      def initialize(csv, column_names, key=nil)
-        super(csv, column_names, key)
-        @transformer = MysqlLoadFileValueTransformer.new
-      end
-
-      protected
-
-      # Writes a row to the output.
-      #
-      # @param Hash row Only keys in column_names will be output.
-      def write(row)
-        output << @column_names.map {|name| 
-          @transformer.transform(row[name]) 
-        }
-      end
-    end
-  end
-end

data/lib/chicago/etl/screens/composite_screen.rb

@@ -1,17 +0,0 @@
-module Chicago
-  module ETL
-    module Screens
-      class CompositeScreen
-        def initialize(*screens)
-          @screens = screens.flatten
-        end
-
-        def call(row, errors=[])
-          @screens.inject([row,errors]) do |(row, errors), screen| 
-            screen.call(row, errors) 
-          end
-        end
-      end
-    end
-  end
-end

data/lib/chicago/etl/sequel/load_data_infile.rb

@@ -1,141 +0,0 @@
-module Chicago
-  module ETL
-    module SequelExtensions
-      # @api private
-      class LoadDataInfileExpression
-        attr_reader :path, :table, :columns, :ignore, :character_set
-
-        def initialize(path, table, columns, opts={})
-          @path    = path
-          @table   = table
-          @columns = columns
-          @ignore  = opts[:ignore]
-          @update  = opts[:update]
-          @set     = opts[:set] || {}
-          @character_set = opts[:character_set] || "utf8"
-          if opts[:format] == :csv
-            @field_terminator = ","
-            @enclosed_by = '"'
-            @escaped_by = '"'
-          end
-        end
-
-        def replace?
-          @update == :replace
-        end
-
-        def ignore?
-          @update == :ignore
-        end
-
-        def to_sql(db)
-          @db = db
-          [load_fragment,
-           replace_fragment,
-           table_fragment,
-           character_set_fragment,
-           field_terminator_fragment,
-           field_enclosure_fragment,
-           escape_fragment,
-           ignore_fragment, 
-           column_fragment,
-           set_fragment].compact.join(" ")
-        end
-
-        private
-
-        def load_fragment
-          "LOAD DATA INFILE '#{path}'"
-        end
-
-        def replace_fragment
-          @update.to_s.upcase if replace? || ignore?
-        end
-
-        def table_fragment
-          "INTO TABLE `#{table}`"
-        end
-
-        def character_set_fragment
-          "CHARACTER SET '#{character_set}'"
-        end
-
-        def field_terminator_fragment
-          "FIELDS TERMINATED BY '#{@field_terminator}'" if @field_terminator
-        end
-
-        def field_enclosure_fragment
-          "OPTIONALLY ENCLOSED BY '#{@enclosed_by}'" if @enclosed_by
-        end
-
-        def escape_fragment
-          "ESCAPED BY '#{@escaped_by}'" if @escaped_by
-        end
-
-        def ignore_fragment
-          "IGNORE #{ignore} LINES" if ignore
-        end
-
-        def column_fragment
-          "(" + columns.map {|c| format_column(c) }.join(",") + ")"
-        end
-
-        def set_fragment
-          unless @set.empty?
-            "SET " + @set.map do |k, v|
-              "#{@db.literal(k)} = #{@db.literal(v)}"
-            end.join(", ")
-          end
-        end
-
-        def format_column(column)
-          column.to_s[0..0] == "@" ? column : "`#{column}`"
-        end
-      end
-
-      module LoadDataInfile
-        # Load data in file specified at path.
-        #
-        # Columns is a list of columns to load - column names starting
-        # with an @ symbol will be treated as variables.
-        #
-        # By default, this will generate a REPLACE INTO TABLE
-        # statement.
-        #
-        # Options:
-        # :ignore - the number of lines to ignore in the source file
-        # :update - nil, :ignore or :replace
-        # :set - a hash specifying autopopulation of columns
-        # :character_set - the character set of the file, UTF8 default
-        # :format - either nil or :csv
-        def load_infile(path, columns, options={})
-          execute_dui(load_infile_sql(path, columns, options))
-        end
-
-        def load_infile_sql(path, columns, options={})
-          replacement = opts[:insert_ignore] ? :ignore : :replace
-          options = {:update => replacement}.merge(options)
-          LoadDataInfileExpression.new(path, 
-                                       opts[:from].first, 
-                                       columns, 
-                                       options).
-            to_sql(db)
-        end
-
-        # Loads the CSV data columns in path into this dataset's
-        # table.
-        #
-        # See load_infile for more options.
-        def load_csv_infile(path, columns, options={})
-          execute_dui(load_csv_infile_sql(path, columns, options))
-        end
-        
-        def load_csv_infile_sql(path, columns, options={})
-          load_infile_sql(path, columns, options.merge(:format => :csv))
-        end
-      end
-    end
-  end
-end
-
-Sequel::Dataset.send :include, Chicago::ETL::SequelExtensions::LoadDataInfile

data/lib/chicago/etl/sink.rb

@@ -1,61 +0,0 @@
-require 'set'
-
-module Chicago
-  module ETL
-    # An end point to write rows.
-    #
-    # @abstract
-    # @api public
-    class Sink
-      # Returns the column names expected to be written to this sink.
-      # @api public
-      attr_reader :column_names
-
-      # @abstract
-      def initialize(output, column_names, unique_row_key=nil)
-        @output = output
-        @column_names = column_names
-        @written_rows = Set.new
-        @unique_row_key = unique_row_key
-      end
-
-      # Writes a row to the output.
-      #
-      # Row will not be written to the output if it has already been
-      # written, as identified by the unique row key.
-      #
-      # Should not be overridden by subclasses - overwrite write instead.
-      def <<(row)
-        unless written?(row)
-          write row
-          @written_rows << row[@unique_row_key]
-        end
-      end
-      
-      # Flushes any remaining writes to the output.
-      #
-      # By default does nothing, subclasses should override where
-      # necessary.
-      def flush
-      end
-
-      # Returns true if this row has previously been written to the
-      # output.
-      #
-      # Always returns false if no key to determine row uniqueness has
-      # been provided.
-      def written?(row)
-        return false if @unique_row_key.nil?
-        @written_rows.include?(row[@unique_row_key])
-      end
-
-      protected
-
-      attr_reader :output
-
-      # @abstract
-      def write(row)
-      end
-    end
-  end
-end

data/lib/chicago/etl/transformations/add_insert_timestamp.rb

@@ -1,16 +0,0 @@
-module Chicago
-  module ETL
-    module Transformations
-      class AddInsertTimestamp
-        def initialize(timestamp=Time.now)
-          @insert_timestamp = timestamp.utc
-        end
-
-        def call(row, errors=[])
-          row[:_inserted_at] = @insert_timestamp
-          [row, errors]
-        end
-      end
-    end
-  end
-end

data/spec/etl/mysql_dumpfile_spec.rb

@@ -1,42 +0,0 @@
-require 'spec_helper'
-
-describe Chicago::ETL::MysqlDumpfile do
-  before :each do
-    @csv = mock(:csv)
-  end
-  
-  it "outputs specified column values in order" do
-    dumpfile = described_class.new(@csv, [:foo, :bar])
-    @csv.should_receive(:<<).with(["1", "2"])
-
-    dumpfile << {:foo => "1", :bar => "2", :baz => "not output"}
-  end
-
-  it "transforms values with a MysqlLoadFileValueTransformer" do
-    transformer = mock(:transformer)
-    Chicago::ETL::MysqlLoadFileValueTransformer.stub(:new).and_return(transformer)
-
-    transformer.should_receive(:transform).with("bar").and_return("baz")
-    @csv.should_receive(:<<).with(["baz"])
-
-    dumpfile = described_class.new(@csv, [:foo])
-    dumpfile << {:foo => "bar"}
-  end
-
-  it "will write a row only once with the same key" do
-    dumpfile = described_class.new(@csv, [:foo], :id)
-    @csv.should_receive(:<<).with(["bar"])
-    
-    dumpfile << {:id => 1, :foo => "bar"}
-    dumpfile << {:id => 1, :foo => "baz"}
-  end
-
-  it "will write a row multiple times if no key is specified" do
-    dumpfile = described_class.new(@csv, [:foo])
-    @csv.should_receive(:<<).with(["bar"])
-    @csv.should_receive(:<<).with(["baz"])
-    
-    dumpfile << {:id => 1, :foo => "bar"}
-    dumpfile << {:id => 1, :foo => "baz"}
-  end
-end

data/spec/etl/mysql_load_file_value_transformer_spec.rb

@@ -1,27 +0,0 @@
-require 'spec_helper'
-
-describe Chicago::ETL::MysqlLoadFileValueTransformer do
-  it "transforms nil into \\N" do
-    subject.transform(nil).should == "\\N"
-  end
-
-  it "transforms true into '1'" do
-    subject.transform(true).should == "1"
-  end
-
-  it "transforms false into '0'" do
-    subject.transform(false).should == "0"
-  end
-
-  it "transforms times into mysql time format" do
-    subject.transform(Time.local(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
-  end
-
-  it "transforms datetimes into mysql time format" do
-    subject.transform(DateTime.new(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
-  end
-
-  it "transforms dates into mysql date format" do
-    subject.transform(Date.new(2011,01,02)).should == "2011-01-02"
-  end
-end

data/spec/etl/screens/composite_screen_spec.rb

@@ -1,25 +0,0 @@
-require 'spec_helper'
-
-describe Chicago::ETL::Screens::CompositeScreen do
-  let(:screen) do
-    i = 0
-
-    lambda {|row, errors| 
-      i += 1 
-      errors << i 
-      [row, errors]
-    }
-  end
-
-  it "calls all child screens" do
-    row, errors = described_class.new([screen, screen]).call({:a => 1}, [])
-    row.should == {:a => 1}
-    errors.should == [1,2]
-  end
-
-  it "supports variable arguments in the constructor" do
-    row, errors = described_class.new(screen, screen).call({:a => 1}, [])
-    row.should == {:a => 1}
-    errors.should == [1,2]
-  end
-end