chicago-etl 0.0.13 → 0.1.0

data/spec/flow/array_source_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Chicago::Flow::ArraySource do
+  it "has an each method that yields rows" do
+    described_class.new([{:a => 1}]).each do |row|
+      row.should == {:a => 1}
+    end
+  end
+
+  it "doesn't know about any fields rows have by default" do
+    described_class.new([]).fields.should == []
+    described_class.new([]).should_not have_defined_fields
+  end
+  
+  it "can optionally define which fields will be in rows" do
+    described_class.new([], [:a, :b]).fields.should == [:a, :b]
+    described_class.new([], :a).fields.should == [:a]
+    described_class.new([], :a).should have_defined_fields
+  end
+end

data/spec/flow/dataset_source_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Chicago::Flow::DatasetSource do
+  let(:dataset) { stub(:dataset) }
+
+  it "should delegtate each to the dataset" do
+    dataset.should_receive(:each)
+    described_class.new(dataset).each {|row| }
+  end
+
+  it "gets columns from the dataset" do
+    dataset.should_receive(:columns)
+    described_class.new(dataset).fields
+  end
+end

data/spec/flow/filter_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Chicago::Flow::Filter do
+  it "filters all rows by default" do
+    subject.process({:a => 1}).should be_nil
+  end
+
+  it "filters rows given a block" do
+    filter = described_class.new {|row| row.has_key?(:a) }
+    filter.process(:a => 1).should == {:a => 1}
+    filter.process(:b => 1).should be_nil
+  end
+end

data/spec/flow/mysql_file_serializer_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Chicago::Flow::MysqlFileSerializer do
+  it "serializes nil into NULL" do
+    subject.serialize(nil).should == "NULL"
+  end
+
+  it "serializes true into '1'" do
+    subject.serialize(true).should == "1"
+  end
+
+  it "serializes false into '0'" do
+    subject.serialize(false).should == "0"
+  end
+
+  it "serializes times into mysql time format" do
+    subject.serialize(Time.local(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
+  end
+
+  it "serializes datetimes into mysql time format" do
+    subject.serialize(DateTime.new(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
+  end
+
+  it "serializes dates into mysql date format" do
+    subject.serialize(Date.new(2011,01,02)).should == "2011-01-02"
+  end
+end

data/spec/flow/mysql_file_sink_spec.rb

@@ -0,0 +1,94 @@
+require 'spec_helper'
+require 'sequel'
+
+describe Chicago::Flow::MysqlFileSink do
+  let(:dataset) { mock(:dataset).as_null_object }
+  let(:db) { mock(:db, :[] => dataset, :schema => []) }
+  let(:csv) { mock(:csv) }
+
+  let(:sink) {
+    described_class.new(db, :table, [:foo], :filepath => "test_file")
+  }
+  
+  before :each do
+    CSV.stub(:open).and_return(csv)
+    csv.stub(:<<)
+    csv.stub(:close).and_return(csv)
+    csv.stub(:flush)
+
+    File.stub(:size?).and_return(true)
+  end
+  
+  it "has the same name as the table it is loading into" do
+    sink.name.should == :table
+  end
+
+  it "writes specified columns to rows in a file" do
+    csv.should_receive(:<<).with([1])
+    sink << {:foo => 1, :bar => 2}
+  end
+
+  it "serializes values before writing to the file" do
+    Chicago::Flow::MysqlFileSerializer.any_instance.
+      should_receive(:serialize).with(1).and_return(1)
+    sink << {:foo => 1}
+  end
+
+  it "has defined fields" do
+    sink.should have_defined_fields
+    sink.fields.should == [:foo]
+  end
+
+  it "loads the csv file into the database when closed" do
+    dataset.should_receive(:load_csv_infile).
+      with("test_file", [:foo], :set => {})
+    sink.close
+  end
+
+  it "uses the :set hash to load constant values" do
+    sink.set_constant_values(:bar => 1).should == sink
+    dataset.should_receive(:load_csv_infile).
+      with("test_file", [:foo], :set => {:bar => 1})
+    sink.close
+  end
+
+  it "does not IGNORE rows by default" do
+    dataset.should_not_receive(:insert_ignore)
+    sink.close
+  end
+
+  it "can specify that INSERT IGNORE should be used" do
+    dataset.should_receive(:insert_ignore)
+    described_class.new(db, :table, [:foo], 
+                        :filepath => "test_file", :ignore => true).close
+  end
+
+  it "writes csv to a tempfile if no explicit filepath is given" do
+    described_class.new(db, :table, [:foo]).filepath.should match(/table\.\d+\.csv/)
+  end
+
+  it "doesn't attempt to load data if the file is empty or does not exist" do
+    File.stub(:size?).and_return(false)
+    dataset.should_not_receive(:load_csv_infile)
+    sink.close
+  end
+
+  it "removes the temporary file when closed" do
+    File.stub(:exists?).and_return(true)
+    File.should_receive(:unlink).with("test_file")
+
+    sink.close
+  end
+
+  it "truncates the table by default" do
+    dataset.should_receive(:truncate)
+    sink.truncate
+  end
+
+  it "can have a truncation strategy set" do
+    x = nil    
+    sink.truncation_strategy = lambda { x = "deleted table" }
+    sink.truncate
+    x.should == "deleted table"
+  end
+end

data/spec/flow/mysql_integration_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe "Mysql -> Mysql through transformation chain" do
+  let(:dup_row) {
+    Class.new(Chicago::Flow::Transformation) {
+      def output_streams
+        [:default, @options[:onto]].flatten
+      end
+
+      def process_row(row)
+        new_row = assign_stream(row.dup, @options[:onto])
+        [row, new_row]
+      end
+    }
+  }
+
+  before :all do
+    unless TEST_DB.table_exists?(:source)
+      TEST_DB.create_table(:source) do
+        primary_key :id
+        varchar :foo
+        binary  :bin, :size => 1
+      end
+    end
+
+    unless TEST_DB.table_exists?(:destination)
+      TEST_DB.create_table(:destination) do
+        primary_key :id
+        varchar :foo
+        binary  :bin, :size => 1
+      end
+    end
+  end
+
+  before :each do
+    TEST_DB[:source].truncate
+    TEST_DB[:destination].truncate
+  end
+
+  after :each do
+    TEST_DB[:source].truncate
+    TEST_DB[:destination].truncate
+  end
+
+  it "copies data from source to destination" do
+    TEST_DB[:source].multi_insert([{:foo => nil, :bin => :unhex.sql_function("1F")},
+                                  {:foo => "Hello", :bin => :unhex.sql_function("1F")}])
+
+    source = Chicago::Flow::DatasetSource.
+      new(TEST_DB[:source].
+          select(:id, :foo, :hex.sql_function(:bin).as(:bin)))
+    sink_1 = Chicago::Flow::MysqlFileSink.
+      new(TEST_DB, :destination, [:id, :foo, :bin])
+    sink_2 = Chicago::Flow::ArraySink.new([:id, :foo, :bin])
+
+    stage = Chicago::Flow::PipelineStage.
+      new(:transformations => [dup_row.new(:onto => :other)])
+
+    expect { stage.execute(source) }.to raise_error
+
+    stage.register_sink(:default, sink_1)
+    stage.register_sink(:other, sink_2)
+
+    stage.execute(source)
+
+    expected = [{:id => 1, :foo => nil, :bin => "1F"},
+                {:id => 2, :foo => "Hello", :bin => "1F"}]
+
+    sink_2.data.should == expected
+    TEST_DB[:destination].select(:id, :foo, :hex.sql_function(:bin).as(:bin)).all.should == expected
+  end
+end

data/spec/flow/pipeline_stage_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe Chicago::Flow::PipelineStage do
+  let(:transform) {
+    Class.new(Chicago::Flow::Transformation) {
+      def process_row(row)
+        row[:a] += 1
+        row
+      end
+    }
+  }
+  
+  let(:add_error) {
+    Class.new(Chicago::Flow::Transformation) {
+      # add_output_stream :error
+      def output_streams
+        [:default, :error]
+      end
+      
+      def process_row(row)
+        [row, {Chicago::Flow::STREAM => :error, :message => "error"}]
+      end
+    }
+  }
+
+  let(:sink) { Chicago::Flow::ArraySink.new(:test) }
+  let(:source) { Chicago::Flow::ArraySource.new([{:a => 1}]) }
+
+  it "returns all sinks" do
+    stage = described_class.new.register_sink(:default, sink)
+    stage.sinks.should == [sink]
+  end
+
+  it "returns a sink by name" do
+    stage = described_class.new.register_sink(:default, sink)
+    stage.sink(:default).should == sink
+  end
+
+  it "reads from source to sink" do
+    pipeline = described_class.new.register_sink(:default, sink)
+    pipeline.execute(source)
+    sink.data.should == [{:a => 1}]
+  end
+
+  it "passes rows through transforms" do
+    pipeline = described_class.new(:transformations => [transform.new]).
+      register_sink(:default, sink)
+                                   
+    pipeline.execute(source)
+    sink.data.should == [{:a => 2}]
+  end
+
+  it "writes rows to the appropriate sink for their stream, and strips the stream tag" do
+    error_sink = Chicago::Flow::ArraySink.new(:test)
+
+    pipeline = described_class.new(:transformations => [add_error.new]).
+      register_sink(:default, sink).
+      register_sink(:error, error_sink)
+
+    pipeline.execute(source)
+    sink.data.should == [{:a => 1}]
+    error_sink.data.should == [{:message => "error"}]
+  end
+
+  it "calls an error handler if sinks are not registered" do
+    error_handler = mock(:error_handler)
+    error_handler.should_receive(:unregistered_sinks).
+      with([:default, :error])
+
+    pipeline = described_class.new(:transformations => [add_error.new],
+                                   :error_handler => error_handler)
+
+    pipeline.validate_pipeline
+  end
+
+  it "by default raises an exception if the pipeline is not valid when executed" do
+    pipeline = described_class.new(:transformations => [add_error.new])
+    expect { pipeline.execute(source) }.to raise_error(Chicago::Flow::Error)
+  end
+
+  it "opens sinks before writing and closes them afterwards" do
+    sink = mock(:sink)
+    pipeline = described_class.new.register_sink(:default, sink)
+    sink.should_receive(:open)
+    sink.stub(:<<)
+    sink.should_receive(:close)
+    pipeline.execute(source)
+  end
+end

data/spec/flow/transformation_chain_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Chicago::Flow::TransformationChain do
+  let(:add_1_to_a) {
+    Class.new(Chicago::Flow::Transformation) {
+      def process_row(row)
+        row[:a] += 1
+        row
+      end
+    }
+  }
+
+  let(:dup_row) {
+    Class.new(Chicago::Flow::Transformation) {
+      def output_streams
+        [:default, @options[:onto]].flatten
+      end
+
+      def process_row(row)
+        new_row = assign_stream(row.dup, @options[:onto])
+        [row, new_row]
+      end
+    }
+  }
+
+  let(:store_until_flush) {
+    Class.new(Chicago::Flow::Transformation) {
+      def process_row(row)
+        @cache ||= []
+        @cache << row
+        nil
+      end
+
+      def flush
+        @cache
+      end
+    }
+  }
+
+  it "chains transformations" do
+    described_class.new(add_1_to_a.new, add_1_to_a.new).process({:a => 1}).
+      should == [{:a => 3}]
+  end
+
+  it "can cope with multiple return rows from transformations" do
+    described_class.new(add_1_to_a.new, dup_row.new, add_1_to_a.new).process({:a => 1}).
+      should == [{:a => 3}, {:a => 3}]
+  end
+
+  it "can cope with a filter returning nil" do
+    described_class.new(Chicago::Flow::Filter.new, 
+                        dup_row.new, add_1_to_a.new).process({:a => 1}).
+      should == []
+  end
+
+  it "can write to different streams" do
+    described_class.new(dup_row.new(:onto => :other),
+                        add_1_to_a.new).process({:a => 1}).
+      should == [{:a => 2}, {:a => 1, Chicago::Flow::STREAM => :other}]
+  end
+
+  it "knows what streams it writes to as a chain" do
+    described_class.new(dup_row.new(:onto => :other),
+                        add_1_to_a.new).output_streams.should == [:default, :other]
+  end
+
+  it "can flush rows held back by transforms" do
+    chain = described_class.new(store_until_flush.new,
+                                add_1_to_a.new,
+                                store_until_flush.new,
+                                add_1_to_a.new)
+    chain.process({:a => 1}).should == []
+    chain.process({:a => 2}).should == []
+    chain.flush.should == [{:a => 3}, {:a => 4}]
+  end
+end

data/spec/flow/transformation_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe Chicago::Flow::Transformation do
+  let(:add_1_to_a) {
+    Class.new(described_class) {
+      def process_row(row)
+        row[:a] += 1
+        row
+      end
+    }
+  }
+
+  let(:add_and_remove) {
+    Class.new(described_class) {
+      adds_fields :b, :c
+      removes_fields :a
+      
+      def process_row(row)
+        row.delete(:a)
+        row[:b] = 1
+        row[:c] = 2
+        row
+      end
+    }
+  }
+  
+  it "writes to the :default stream by default" do
+    subject.output_streams.should == [:default]
+  end
+
+  it "may apply to a particular stream" do
+    subject.applies_to_stream?(:default).should be_true
+    subject.applies_to_stream?(nil).should be_true
+    described_class.new(:other).applies_to_stream?(:default).should be_false
+    described_class.new(:other).applies_to_stream?(:other).should be_true
+  end
+
+  it "processes a row via #process_row" do
+    add_1_to_a.new.process({:a => 1}).should == {:a => 2}
+  end
+
+  it "passes through rows not on its stream" do
+    add_1_to_a.new(:other).process({:a => 1}).should == {:a => 1}
+  end
+
+  it "can apply to all streams using :all" do
+    add_1_to_a.new(:all).process({:a => 1}).should == {:a => 2}
+    add_1_to_a.new(:all).process({:a => 1, Chicago::Flow::STREAM => :other}).
+      should == {:a => 2, Chicago::Flow::STREAM => :other}
+  end
+
+  it "can be flushed" do
+    subject.flush.should == []
+  end
+
+  it "can specify which fields are added" do
+    add_and_remove.new.added_fields.should == [:b, :c]
+  end
+
+  it "can specify which fields are removed" do
+    add_and_remove.new.removed_fields.should == [:a]
+  end
+
+  it "can calculate downstream fields" do
+    Set.new(add_and_remove.new.downstream_fields([:a, :b, :d])).
+      should == Set.new([:b, :c, :d])
+  end
+
+  it "can calculate upstream fields" do
+    Set.new(add_and_remove.new.upstream_fields([:b, :c, :d])).
+      should == Set.new([:a, :d])
+  end
+
+  it "has an empty array of added fields by default" do
+    subject.added_fields.should == []
+  end
+
+  it "has an empty array of removed fields by default" do
+    subject.removed_fields.should == []
+  end
+
+  it "has an empty array of required options by default" do
+    subject.required_options.should == []
+  end
+
+  it "can enforce options" do
+    klass = Class.new(described_class) { requires_options :foo }
+    expect { klass.new }.to raise_error(ArgumentError)
+    expect { klass.new(:foo => :bar) }.to_not raise_error(ArgumentError)
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,8 @@
+if RUBY_VERSION.to_f >= 1.9
+  require 'simplecov'
+  SimpleCov.start { add_filter 'spec' }
+end
+
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'rspec'