chicago-etl 0.1.4 → 0.2.0

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.2.0

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.1.4"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-11-13"
+  s.date = "2013-11-18"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -28,14 +28,24 @@ Gem::Specification.new do |s|
     "chicago-flow.gemspec",
     "lib/chicago-etl.rb",
     "lib/chicago/etl.rb",
+    "lib/chicago/etl/array_sink.rb",
+    "lib/chicago/etl/array_source.rb",
     "lib/chicago/etl/batch.rb",
     "lib/chicago/etl/core_extensions.rb",
     "lib/chicago/etl/counter.rb",
     "lib/chicago/etl/dataset_batch_stage.rb",
     "lib/chicago/etl/dataset_builder.rb",
+    "lib/chicago/etl/dataset_source.rb",
+    "lib/chicago/etl/errors.rb",
+    "lib/chicago/etl/filter.rb",
     "lib/chicago/etl/key_builder.rb",
     "lib/chicago/etl/load_dataset_builder.rb",
+    "lib/chicago/etl/mysql.rb",
+    "lib/chicago/etl/mysql_file_serializer.rb",
+    "lib/chicago/etl/mysql_file_sink.rb",
+    "lib/chicago/etl/null_sink.rb",
     "lib/chicago/etl/pipeline.rb",
+    "lib/chicago/etl/pipeline_endpoint.rb",
     "lib/chicago/etl/schema_sinks_and_transformations_builder.rb",
     "lib/chicago/etl/schema_table_sink_factory.rb",
     "lib/chicago/etl/screens/column_screen.rb",
@@ -44,37 +54,34 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/screens/out_of_bounds.rb",
     "lib/chicago/etl/sequel/dependant_tables.rb",
     "lib/chicago/etl/sequel/filter_to_etl_batch.rb",
+    "lib/chicago/etl/sink.rb",
     "lib/chicago/etl/stage.rb",
     "lib/chicago/etl/stage_builder.rb",
     "lib/chicago/etl/table_builder.rb",
     "lib/chicago/etl/task_invocation.rb",
     "lib/chicago/etl/tasks.rb",
+    "lib/chicago/etl/transformation.rb",
+    "lib/chicago/etl/transformation_chain.rb",
     "lib/chicago/etl/transformations.rb",
     "lib/chicago/etl/transformations/deduplicate_rows.rb",
     "lib/chicago/etl/transformations/uk_post_code.rb",
     "lib/chicago/etl/transformations/uk_post_code_field.rb",
-    "lib/chicago/flow/array_sink.rb",
-    "lib/chicago/flow/array_source.rb",
-    "lib/chicago/flow/dataset_source.rb",
-    "lib/chicago/flow/errors.rb",
-    "lib/chicago/flow/filter.rb",
-    "lib/chicago/flow/mysql.rb",
-    "lib/chicago/flow/mysql_file_serializer.rb",
-    "lib/chicago/flow/mysql_file_sink.rb",
-    "lib/chicago/flow/null_sink.rb",
-    "lib/chicago/flow/pipeline_endpoint.rb",
-    "lib/chicago/flow/sink.rb",
-    "lib/chicago/flow/transformation.rb",
-    "lib/chicago/flow/transformation_chain.rb",
     "spec/db_connections.yml.dist",
+    "spec/etl/array_sink_spec.rb",
+    "spec/etl/array_source_spec.rb",
     "spec/etl/batch_spec.rb",
     "spec/etl/core_extensions_spec.rb",
     "spec/etl/counter_spec.rb",
+    "spec/etl/dataset_source_spec.rb",
     "spec/etl/define_dimension_stage_spec.rb",
     "spec/etl/define_stage_spec.rb",
     "spec/etl/etl_batch_id_dataset_filter.rb",
+    "spec/etl/filter_spec.rb",
     "spec/etl/key_builder_spec.rb",
     "spec/etl/load_dataset_builder_spec.rb",
+    "spec/etl/mysql_file_serializer_spec.rb",
+    "spec/etl/mysql_file_sink_spec.rb",
+    "spec/etl/mysql_integration_spec.rb",
     "spec/etl/pipeline_stage_builder_spec.rb",
     "spec/etl/schema_table_sink_factory_spec.rb",
     "spec/etl/screens/invalid_element_spec.rb",
@@ -85,19 +92,12 @@ Gem::Specification.new do |s|
     "spec/etl/stage_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",
+    "spec/etl/transformation_chain_spec.rb",
+    "spec/etl/transformation_spec.rb",
     "spec/etl/transformations/deduplicate_rows_spec.rb",
     "spec/etl/transformations/uk_post_code_field_spec.rb",
     "spec/etl/transformations/uk_post_code_spec.rb",
     "spec/etl/transformations_spec.rb",
-    "spec/flow/array_sink_spec.rb",
-    "spec/flow/array_source_spec.rb",
-    "spec/flow/dataset_source_spec.rb",
-    "spec/flow/filter_spec.rb",
-    "spec/flow/mysql_file_serializer_spec.rb",
-    "spec/flow/mysql_file_sink_spec.rb",
-    "spec/flow/mysql_integration_spec.rb",
-    "spec/flow/transformation_chain_spec.rb",
-    "spec/flow/transformation_spec.rb",
     "spec/spec_helper.rb"
   ]
   s.homepage = "http://github.com/notonthehighstreet/chicago-etl"

data/lib/chicago/{flow → etl}/array_sink.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # An endpoint that stores rows in an Array.
     #
     # @api public

data/lib/chicago/{flow → etl}/array_source.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # @api public
     class ArraySource < PipelineEndpoint
       def initialize(array, fields=[])

data/lib/chicago/etl/dataset_batch_stage.rb

@@ -11,7 +11,6 @@ module Chicago
         super
         @filter_strategy = options[:filter_strategy] ||
           lambda { |dataset, etl_batch| @source.filter_to_etl_batch(etl_batch)}
-        @truncate_pre_load = !!options[:truncate_pre_load]
      end
 
       # Executes this ETL stage.

data/lib/chicago/{flow → etl}/dataset_source.rb

@@ -2,7 +2,7 @@ require 'sequel'
 require 'sequel/fast_columns'
 
 module Chicago
-  module Flow
+  module ETL
     # @api public
     class DatasetSource < PipelineEndpoint
       attr_reader :dataset

data/lib/chicago/{flow → etl}/errors.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # @api public
     class Error < RuntimeError
     end

data/lib/chicago/{flow → etl}/filter.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # @api public
     class Filter < Transformation
       def initialize(stream=:default, &block)

data/lib/chicago/etl/mysql.rb

@@ -0,0 +1,4 @@
+require 'sequel'
+require 'sequel/load_data_infile'
+require 'chicago/etl/mysql_file_serializer'
+require 'chicago/etl/mysql_file_sink'

data/lib/chicago/{flow → etl}/mysql_file_serializer.rb

@@ -1,7 +1,7 @@
 require 'date'
 
 module Chicago
-  module Flow
+  module ETL
     # @api private
     class MysqlFileSerializer
       # Transforms a value to be suitable for use in file in a LOAD

data/lib/chicago/{flow → etl}/mysql_file_sink.rb

@@ -5,7 +5,7 @@ require 'tmpdir'
 Sequel.extension :core_extensions
 
 module Chicago
-  module Flow
+  module ETL
     # @api public
     class MysqlFileSink < Sink
       attr_reader :filepath

data/lib/chicago/{flow → etl}/null_sink.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # Supports the Sink interface, but discards all rows written to
     # it.
     class NullSink < Sink

data/lib/chicago/{flow → etl}/pipeline_endpoint.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # A Source or a Sink.
     #
     # @api public

data/lib/chicago/etl/schema_sinks_and_transformations_builder.rb

@@ -88,7 +88,7 @@ module Chicago
                      @sink_factory.key_sink
                    else
                      # Facts have no key table to write to.
-                     Flow::NullSink.new
+                     NullSink.new
                    end
 
         {

data/lib/chicago/etl/schema_table_sink_factory.rb

@@ -13,10 +13,10 @@ module Chicago
       # Pass an :exclude option if you don't want all columns of the
       # schema table to be loaded via this sink.
       def sink(options={})
-        Flow::MysqlFileSink.new(@db,
-                                @schema_table.table_name,
-                                load_columns(options[:exclude]),
-                                mysql_options(options))
+        MysqlFileSink.new(@db,
+                          @schema_table.table_name,
+                          load_columns(options[:exclude]),
+                          mysql_options(options))
       end
       
       # Returns a sink to load data into the MySQL table backing the
@@ -26,20 +26,20 @@ module Chicago
       #   schema table's key table name will be used otherwise.
       def key_sink(options={})
         table = options.delete(:table) || @schema_table.key_table_name
-        sink = Flow::MysqlFileSink.new(@db,
-                                       table,
-                                       [:original_id, :dimension_id],
-                                       mysql_options(options))
+        sink = MysqlFileSink.new(@db,
+                                 table,
+                                 [:original_id, :dimension_id],
+                                 mysql_options(options))
         sink.truncation_strategy = lambda do
           # No Op - we want to maintain keys to avoid having to sort
           # out fact tables.
         end
         sink
       end
-
+      
       # Returns a sink to load errors generated in the ETL process.
       def error_sink(options={})
-        sink = Flow::MysqlFileSink.
+        sink = MysqlFileSink.
           new(@db, :etl_error_log, 
               [:column, :row_id, :error, :severity, :error_detail], mysql_options(options)).
           set_constant_values(:table => @schema_table.table_name.to_s,
@@ -53,7 +53,7 @@ module Chicago
         end
         sink
       end
-
+      
       private
       
       def load_columns(exclude=nil)

data/lib/chicago/etl/screens/column_screen.rb

@@ -2,7 +2,7 @@ module Chicago
   module ETL
     module Screens
       # @abstract
-      class ColumnScreen < Flow::Transformation
+      class ColumnScreen < Transformation
         def self.for_columns(columns)
           columns.map {|column|
             new(:default, :column => column) 

data/lib/chicago/{flow → etl}/sink.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # The destination for rows passing through a pipeline stage.
     #
     # @api public

data/lib/chicago/etl/stage.rb

@@ -39,7 +39,7 @@ module Chicago
         filtered_dataset = reextract ? source : 
           @filter_strategy.call(source, etl_batch)
 
-        Chicago::Flow::DatasetSource.new(filtered_dataset)
+        DatasetSource.new(filtered_dataset)
       end
 
       private
@@ -58,8 +58,7 @@ module Chicago
       end
 
       def transformation_chain
-        @transformation_chain ||= Chicago::Flow::TransformationChain.
-          new(*@transformations)
+        @transformation_chain ||= TransformationChain.new(*@transformations)
       end
 
       def process_row(row)

data/lib/chicago/{flow → etl}/transformation.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # The key used to store the stream in the row.
     #
     # @api private

data/lib/chicago/{flow → etl}/transformation_chain.rb

@@ -1,5 +1,5 @@
 module Chicago
-  module Flow
+  module ETL
     # @api private
     class TransformationChain
       def initialize(*transforms)

data/lib/chicago/etl/transformations/deduplicate_rows.rb

@@ -1,6 +1,6 @@
 module Chicago
   module ETL
-    class DeduplicateRows < Chicago::Flow::Transformation
+    class DeduplicateRows < Transformation
       def process_row(row)
         if @working_row.nil?
           @working_row = new_row(row)

data/lib/chicago/etl/transformations.rb

@@ -2,7 +2,7 @@ module Chicago
   module ETL
     module Transformations
       # Filters rows so they only get output once, based on a :key.
-      class WrittenRowFilter < Flow::Transformation
+      class WrittenRowFilter < Transformation
         requires_options :key
 
         def initialize(*args)
@@ -30,7 +30,7 @@ module Chicago
       # :_errors field.
       #
       # Pass the :key_builder option to set the KeyBuilder.
-      class AddKey < Flow::Transformation
+      class AddKey < Transformation
         requires_options :key_builder
         adds_fields :id
 
@@ -57,7 +57,7 @@ module Chicago
       end
 
       # Removes embedded :_errors and puts them on the error stream.
-      class DemultiplexErrors < Flow::Transformation
+      class DemultiplexErrors < Transformation
         def output_streams
           [:default, :error]
         end
@@ -73,7 +73,7 @@ module Chicago
 
       # Removes a field from the row, and creates a row on a
       # designated key stream
-      class DimensionKeyMapping < Flow::Transformation
+      class DimensionKeyMapping < Transformation
         requires_options :original_key, :key_table
 
         def removed_fields
@@ -103,7 +103,7 @@ module Chicago
       end
 
       # Adds a hash of the specified columns as a field in the row.
-      class HashColumns < Flow::Transformation
+      class HashColumns < Transformation
         requires_options :columns
 
         def process_row(row)

data/lib/chicago/etl.rb

@@ -6,17 +6,17 @@ else
 end
 
 require 'sequel'
-require 'chicago/flow/errors'
-require 'chicago/flow/transformation'
-require 'chicago/flow/filter'
-require 'chicago/flow/transformation_chain'
-require 'chicago/flow/pipeline_endpoint'
-require 'chicago/flow/array_source'
-require 'chicago/flow/dataset_source'
-require 'chicago/flow/sink'
-require 'chicago/flow/array_sink'
-require 'chicago/flow/null_sink'
-require 'chicago/flow/mysql'
+require 'chicago/etl/errors'
+require 'chicago/etl/transformation'
+require 'chicago/etl/filter'
+require 'chicago/etl/transformation_chain'
+require 'chicago/etl/pipeline_endpoint'
+require 'chicago/etl/array_source'
+require 'chicago/etl/dataset_source'
+require 'chicago/etl/sink'
+require 'chicago/etl/array_sink'
+require 'chicago/etl/null_sink'
+require 'chicago/etl/mysql'
 
 require 'chicago/etl/core_extensions'
 require 'chicago/etl/counter'
@@ -66,4 +66,7 @@ module Chicago
       end
     end
   end
+
+  # Deprecated, allows clients to transition when they like.
+  Flow = ETL
 end

data/spec/{flow → etl}/array_sink_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::ArraySink do
+describe Chicago::ETL::ArraySink do
   let(:sink) { described_class.new(:foo) }
 
   it "has a name" do

data/spec/{flow → etl}/array_source_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::ArraySource do
+describe Chicago::ETL::ArraySource do
   it "has an each method that yields rows" do
     described_class.new([{:a => 1}]).each do |row|
       row.should == {:a => 1}

data/spec/{flow → etl}/dataset_source_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::DatasetSource do
+describe Chicago::ETL::DatasetSource do
   let(:dataset) { stub(:dataset) }
 
   it "should delegtate each to the dataset" do

data/spec/etl/define_stage_spec.rb

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-class TestTransformation < Chicago::Flow::Transformation
+class TestTransformation < Chicago::ETL::Transformation
   def output_streams
     [:another_stream]
   end
@@ -23,8 +23,8 @@ describe "defining and executing a stage" do
       end
 
       sinks do
-        add Chicago::Flow::ArraySink.new(:test)
-        add Chicago::Flow::ArraySink.new(:test), :stream => :another_stream
+        add Chicago::ETL::ArraySink.new(:test)
+        add Chicago::ETL::ArraySink.new(:test), :stream => :another_stream
       end
     end
 
@@ -50,8 +50,8 @@ describe "defining and executing a stage" do
       end
 
       sinks do
-        add Chicago::Flow::ArraySink.new(:test)
-        add Chicago::Flow::ArraySink.new(:test), :stream => :another_stream
+        add Chicago::ETL::ArraySink.new(:test)
+        add Chicago::ETL::ArraySink.new(:test), :stream => :another_stream
       end
     end
 
@@ -78,7 +78,7 @@ describe "defining and executing a stage" do
       end
 
       sinks do
-        add Chicago::Flow::ArraySink.new(:test)
+        add Chicago::ETL::ArraySink.new(:test)
       end
 
       filter_strategy do |source, etl_batch|

data/spec/{flow → etl}/filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::Filter do
+describe Chicago::ETL::Filter do
   it "filters all rows by default" do
     subject.process({:a => 1}).should be_nil
   end

data/spec/{flow → etl}/mysql_file_serializer_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::MysqlFileSerializer do
+describe Chicago::ETL::MysqlFileSerializer do
   it "serializes nil into NULL" do
     subject.serialize(nil).should == "NULL"
   end

data/spec/{flow → etl}/mysql_file_sink_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 require 'sequel'
 
-describe Chicago::Flow::MysqlFileSink do
+describe Chicago::ETL::MysqlFileSink do
   let(:dataset) { mock(:dataset).as_null_object }
   let(:db) { mock(:db, :[] => dataset, :schema => []) }
   let(:csv) { mock(:csv) }
@@ -29,7 +29,7 @@ describe Chicago::Flow::MysqlFileSink do
   end
 
   it "serializes values before writing to the file" do
-    Chicago::Flow::MysqlFileSerializer.any_instance.
+    Chicago::ETL::MysqlFileSerializer.any_instance.
       should_receive(:serialize).with(1).and_return(1)
     sink << {:foo => 1}
   end

data/spec/{flow → etl}/mysql_integration_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe "Mysql -> Mysql through transformation chain" do
   let(:dup_row) {
-    Class.new(Chicago::Flow::Transformation) {
+    Class.new(Chicago::ETL::Transformation) {
       def output_streams
         [:default, @options[:onto]].flatten
       end
@@ -46,15 +46,15 @@ describe "Mysql -> Mysql through transformation chain" do
     TEST_DB[:source].multi_insert([{:foo => nil, :bin => :unhex.sql_function("1F")},
                                    {:foo => "Hello", :bin => :unhex.sql_function("1F")}])
     
-    source = Chicago::Flow::DatasetSource.
+    source = Chicago::ETL::DatasetSource.
       new(TEST_DB[:source].
           select(:id, :foo, :hex.sql_function(:bin).as(:bin)))
 
     transformations = [dup_row.new(:onto => :other)]
 
-    sink_1 = Chicago::Flow::MysqlFileSink.
+    sink_1 = Chicago::ETL::MysqlFileSink.
       new(TEST_DB, :destination, [:id, :foo, :bin])
-    sink_2 = Chicago::Flow::ArraySink.new([:id, :foo, :bin])
+    sink_2 = Chicago::ETL::ArraySink.new([:id, :foo, :bin])
 
     stage = Chicago::ETL::Stage.new(:test, 
                                     :source => source, 

data/spec/etl/schema_table_sink_factory_spec.rb

@@ -12,7 +12,7 @@ describe Chicago::ETL::SchemaTableSinkFactory do
     end
   }
 
-  let(:sink_class) { Chicago::Flow::MysqlFileSink }
+  let(:sink_class) { Chicago::ETL::MysqlFileSink }
 
   it "builds a MysqlFileSink" do
     sink_class.should_receive(:new).

data/spec/{flow → etl}/transformation_chain_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 
-describe Chicago::Flow::TransformationChain do
+describe Chicago::ETL::TransformationChain do
   let(:add_1_to_a) {
-    Class.new(Chicago::Flow::Transformation) {
+    Class.new(Chicago::ETL::Transformation) {
       def process_row(row)
         row[:a] += 1
         row
@@ -11,7 +11,7 @@ describe Chicago::Flow::TransformationChain do
   }
 
   let(:dup_row) {
-    Class.new(Chicago::Flow::Transformation) {
+    Class.new(Chicago::ETL::Transformation) {
       def output_streams
         [:default, @options[:onto]].flatten
       end
@@ -24,7 +24,7 @@ describe Chicago::Flow::TransformationChain do
   }
 
   let(:store_until_flush) {
-    Class.new(Chicago::Flow::Transformation) {
+    Class.new(Chicago::ETL::Transformation) {
       def process_row(row)
         @cache ||= []
         @cache << row
@@ -48,7 +48,7 @@ describe Chicago::Flow::TransformationChain do
   end
 
   it "can cope with a filter returning nil" do
-    described_class.new(Chicago::Flow::Filter.new, 
+    described_class.new(Chicago::ETL::Filter.new, 
                         dup_row.new, add_1_to_a.new).process({:a => 1}).
       should == []
   end
@@ -56,7 +56,7 @@ describe Chicago::Flow::TransformationChain do
   it "can write to different streams" do
     described_class.new(dup_row.new(:onto => :other),
                         add_1_to_a.new).process({:a => 1}).
-      should == [{:a => 2}, {:a => 1, Chicago::Flow::STREAM => :other}]
+      should == [{:a => 2}, {:a => 1, Chicago::ETL::STREAM => :other}]
   end
 
   it "knows what streams it writes to as a chain" do

data/spec/{flow → etl}/transformation_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::Flow::Transformation do
+describe Chicago::ETL::Transformation do
   let(:add_1_to_a) {
     Class.new(described_class) {
       def process_row(row)
@@ -45,8 +45,8 @@ describe Chicago::Flow::Transformation do
 
   it "can apply to all streams using :all" do
     add_1_to_a.new(:all).process({:a => 1}).should == {:a => 2}
-    add_1_to_a.new(:all).process({:a => 1, Chicago::Flow::STREAM => :other}).
-      should == {:a => 2, Chicago::Flow::STREAM => :other}
+    add_1_to_a.new(:all).process({:a => 1, Chicago::ETL::STREAM => :other}).
+      should == {:a => 2, Chicago::ETL::STREAM => :other}
   end
 
   it "can be flushed" do

data/spec/etl/transformations_spec.rb

@@ -16,7 +16,7 @@ describe Chicago::ETL::Transformations::DemultiplexErrors do
   it "adds the errors onto the error stream" do
     subject.process(:_errors => [{:error => 1}]).last.should == {
       :error => 1,
-      Chicago::Flow::STREAM => :error
+      Chicago::ETL::STREAM => :error
     }
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
-  - 4
-  version: 0.1.4
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-11-13 00:00:00 Z
+date: 2013-11-18 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -239,14 +239,24 @@ files:
 - chicago-flow.gemspec
 - lib/chicago-etl.rb
 - lib/chicago/etl.rb
+- lib/chicago/etl/array_sink.rb
+- lib/chicago/etl/array_source.rb
 - lib/chicago/etl/batch.rb
 - lib/chicago/etl/core_extensions.rb
 - lib/chicago/etl/counter.rb
 - lib/chicago/etl/dataset_batch_stage.rb
 - lib/chicago/etl/dataset_builder.rb
+- lib/chicago/etl/dataset_source.rb
+- lib/chicago/etl/errors.rb
+- lib/chicago/etl/filter.rb
 - lib/chicago/etl/key_builder.rb
 - lib/chicago/etl/load_dataset_builder.rb
+- lib/chicago/etl/mysql.rb
+- lib/chicago/etl/mysql_file_serializer.rb
+- lib/chicago/etl/mysql_file_sink.rb
+- lib/chicago/etl/null_sink.rb
 - lib/chicago/etl/pipeline.rb
+- lib/chicago/etl/pipeline_endpoint.rb
 - lib/chicago/etl/schema_sinks_and_transformations_builder.rb
 - lib/chicago/etl/schema_table_sink_factory.rb
 - lib/chicago/etl/screens/column_screen.rb
@@ -255,37 +265,34 @@ files:
 - lib/chicago/etl/screens/out_of_bounds.rb
 - lib/chicago/etl/sequel/dependant_tables.rb
 - lib/chicago/etl/sequel/filter_to_etl_batch.rb
+- lib/chicago/etl/sink.rb
 - lib/chicago/etl/stage.rb
 - lib/chicago/etl/stage_builder.rb
 - lib/chicago/etl/table_builder.rb
 - lib/chicago/etl/task_invocation.rb
 - lib/chicago/etl/tasks.rb
+- lib/chicago/etl/transformation.rb
+- lib/chicago/etl/transformation_chain.rb
 - lib/chicago/etl/transformations.rb
 - lib/chicago/etl/transformations/deduplicate_rows.rb
 - lib/chicago/etl/transformations/uk_post_code.rb
 - lib/chicago/etl/transformations/uk_post_code_field.rb
-- lib/chicago/flow/array_sink.rb
-- lib/chicago/flow/array_source.rb
-- lib/chicago/flow/dataset_source.rb
-- lib/chicago/flow/errors.rb
-- lib/chicago/flow/filter.rb
-- lib/chicago/flow/mysql.rb
-- lib/chicago/flow/mysql_file_serializer.rb
-- lib/chicago/flow/mysql_file_sink.rb
-- lib/chicago/flow/null_sink.rb
-- lib/chicago/flow/pipeline_endpoint.rb
-- lib/chicago/flow/sink.rb
-- lib/chicago/flow/transformation.rb
-- lib/chicago/flow/transformation_chain.rb
 - spec/db_connections.yml.dist
+- spec/etl/array_sink_spec.rb
+- spec/etl/array_source_spec.rb
 - spec/etl/batch_spec.rb
 - spec/etl/core_extensions_spec.rb
 - spec/etl/counter_spec.rb
+- spec/etl/dataset_source_spec.rb
 - spec/etl/define_dimension_stage_spec.rb
 - spec/etl/define_stage_spec.rb
 - spec/etl/etl_batch_id_dataset_filter.rb
+- spec/etl/filter_spec.rb
 - spec/etl/key_builder_spec.rb
 - spec/etl/load_dataset_builder_spec.rb
+- spec/etl/mysql_file_serializer_spec.rb
+- spec/etl/mysql_file_sink_spec.rb
+- spec/etl/mysql_integration_spec.rb
 - spec/etl/pipeline_stage_builder_spec.rb
 - spec/etl/schema_table_sink_factory_spec.rb
 - spec/etl/screens/invalid_element_spec.rb
@@ -296,19 +303,12 @@ files:
 - spec/etl/stage_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
+- spec/etl/transformation_chain_spec.rb
+- spec/etl/transformation_spec.rb
 - spec/etl/transformations/deduplicate_rows_spec.rb
 - spec/etl/transformations/uk_post_code_field_spec.rb
 - spec/etl/transformations/uk_post_code_spec.rb
 - spec/etl/transformations_spec.rb
-- spec/flow/array_sink_spec.rb
-- spec/flow/array_source_spec.rb
-- spec/flow/dataset_source_spec.rb
-- spec/flow/filter_spec.rb
-- spec/flow/mysql_file_serializer_spec.rb
-- spec/flow/mysql_file_sink_spec.rb
-- spec/flow/mysql_integration_spec.rb
-- spec/flow/transformation_chain_spec.rb
-- spec/flow/transformation_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/notonthehighstreet/chicago-etl
 licenses: 

data/lib/chicago/flow/mysql.rb

@@ -1,4 +0,0 @@
-require 'sequel'
-require 'sequel/load_data_infile'
-require 'chicago/flow/mysql_file_serializer'
-require 'chicago/flow/mysql_file_sink'