chicago-etl 0.0.9

data/spec/etl/sink_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Sink do
+  it "writes rows once if given a key" do
+    
+  end
+end

data/spec/etl/table_builder_spec.rb

@@ -0,0 +1,22 @@
+require "spec_helper"
+
+describe Chicago::ETL::TableBuilder do
+  before :each do
+    TEST_DB.drop_table(*(TEST_DB.tables))
+  end
+
+  it "should create an etl_batches table" do
+    ETL::TableBuilder.build(TEST_DB)
+    TEST_DB.tables.should include(:etl_batches)
+  end
+
+  it "should create an etl_task_invocations table" do
+    ETL::TableBuilder.build(TEST_DB)
+    TEST_DB.tables.should include(:etl_task_invocations)
+  end
+
+  it "should do nothing and not raise an error if run more times than necessary" do
+    ETL::TableBuilder.build(TEST_DB)
+    lambda { ETL::TableBuilder.build(TEST_DB) }.should_not raise_error(Sequel::DatabaseError)
+  end
+end

data/spec/etl/task_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+require 'timeout'
+
+describe Chicago::ETL::TaskInvocation do
+  before :all do
+    TEST_DB.drop_table(*(TEST_DB.tables))
+    ETL::TableBuilder.build(TEST_DB)
+    ETL::TaskInvocation.db = TEST_DB
+  end
+
+  before :each do
+    TEST_DB[:etl_task_invocations].truncate
+    TEST_DB[:etl_batches].truncate
+  end
+  
+  after :each do
+    tmpdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "tmp"))
+    FileUtils.rm_r(tmpdir) if File.exists?(tmpdir)
+  end
+  
+  it "should be associated with an ETL Batch" do
+    batch = create_batch
+    batch.should_not be_nil
+    t = ETL::TaskInvocation.create(:batch => batch)
+    t.reload.batch.should == batch
+  end
+
+  it "should be unqiue by name across an ETL Batch and stage" do
+    attrs = {:batch_id => 1, :stage => "Extract", :name => "foo"}
+    ETL::TaskInvocation.create(attrs)
+    lambda { ETL::TaskInvocation.create(attrs) }.should raise_error(Sequel::DatabaseError)
+  end
+
+  it "should be in the 'Created' state by default" do
+    ETL::TaskInvocation.create.state.should == "Created"
+  end
+
+  it "should perform an action in a block, and set it's state to Error if an exception is raised" do
+    task = ETL::TaskInvocation.create
+    lambda { task.perform { raise "Boom" } }.should raise_error(RuntimeError, "Boom")
+    task.state.should == "Error"
+  end
+
+  it "should perform an action in a block, and set it's state to Error if an timeout error is raised" do
+    task = ETL::TaskInvocation.create
+    lambda { task.perform { raise Timeout::Error.new("Timeout error") } }.should raise_error(Timeout::Error)
+    task.state.should == "Error"
+  end
+
+  it "should be in the 'Started' state while perform is called" do
+    task = ETL::TaskInvocation.create
+    task.perform { task.state.should == "Started" }
+  end
+
+  it "should be in the 'Finished' state after #perform has finished" do
+    task = ETL::TaskInvocation.create
+    task.perform {}
+    task.state.should == "Finished"
+    task.finished_at.should be_kind_of(Time)
+  end
+
+  it "should raise a RuntimeError if perform is attempted after the task has finished" do
+    task = ETL::TaskInvocation.create
+    task.perform {}
+    lambda { task.perform {} }.should raise_error(RuntimeError)
+  end
+
+  it "should increment the number of attempts each time the task is performed" do
+    task = ETL::TaskInvocation.create
+    task.perform { raise "Boom" } rescue nil
+    task.attempts.should == 1
+    task.perform { raise "Boom" } rescue nil
+    task.attempts.should == 2
+  end
+
+  it "should set the associated batch into an error state" do
+    batch = create_batch
+    task = ETL::TaskInvocation.create(:batch => batch)
+    task.perform { raise "Boom" } rescue nil
+    task.batch.should be_in_error
+  end
+
+  def create_batch
+    Chicago.project_root = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+    ETL::Batch.create
+  end
+end

data/spec/etl/transformations/add_insert_timestamp_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Transformations::AddInsertTimestamp do
+  it "adds a timestamp in UTC in the _inserted_at field" do
+    time = subject.call({}).first[:_inserted_at]
+    time.should be_kind_of(Time)
+    time.zone.should == "UTC"
+  end
+end

data/spec/etl/transformations/uk_post_code_field_spec.rb

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Chicago::ETL::Transformations::UkPostCodeField do
+  it "reformats a 6 character postcode" do
+    data = described_class.new.normalize(" SW 30 TA")
+    data[:post_code].should == "SW3 0TA"
+  end
+
+  it "reformats a 7 character postcode" do
+    data = described_class.new.normalize(" G L515 FD")
+    data[:post_code].should == "GL51 5FD"
+  end
+
+  it "reformats a BFPO postcode" do
+    data = described_class.new.normalize("bfpo123")
+    data[:post_code].should == "BFPO 123"
+  end
+
+  it "returns the outcode" do
+    data = described_class.new.normalize(" G L515 FD")
+    data[:outcode].should == "GL51"
+  end
+
+  it "returns the incode" do
+    data = described_class.new.normalize(" G L515 FD")
+    data[:outcode].should == "GL51"
+  end
+
+  it "fixes people using 0 instead of O for OX and OL post codes" do
+    data = described_class.new.normalize(" 0X1 4HG")
+    data[:post_code].should == "OX1 4HG"
+
+    data = described_class.new.normalize(" 0L1 4HG")
+    data[:post_code].should == "OL1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for OX post codes" do
+    data = described_class.new.normalize(" 0X1 4HG")
+    data[:post_code].should == "OX1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for OL post codes" do
+    data = described_class.new.normalize(" 0L1 0LG")
+    data[:post_code].should == "OL1 0LG"
+  end
+
+  it "fixes people using 0 instead of O for PO post codes" do
+    data = described_class.new.normalize(" P01 4HG")
+    data[:post_code].should == "PO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for SO post codes" do
+    data = described_class.new.normalize(" S01 4HG")
+    data[:post_code].should == "SO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for CO post codes" do
+    data = described_class.new.normalize(" C01 4HG")
+    data[:post_code].should == "CO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for YO post codes" do
+    data = described_class.new.normalize(" Y01 4HG")
+    data[:post_code].should == "YO1 4HG"
+  end
+
+  it "fixes shift key slips" do
+    data = described_class.new.normalize(")X!£ $HG")
+    data[:post_code].should == "OX13 4HG"
+  end
+
+  it "uppercases post code letters" do
+    data = described_class.new.normalize("sw3 0ta")
+    data[:post_code].should == "SW3 0TA"
+  end
+
+  it "returns the postcode with an invalid flag if invalid" do
+    data = described_class.new.normalize("zzz zzz")
+    data.should == {:post_code => "zzz zzz", :invalid => true}
+  end
+
+  it "has no outcode if not valid" do
+    data = described_class.new.normalize("zzz zzz")
+    data[:outcode].should == nil
+  end
+
+  it "has the outcode for partial postcodes" do
+    data = described_class.new.normalize("W1a")
+    data[:post_code].should == "W1A"
+    data[:outcode].should == "W1A"
+    data[:incode].should be_nil
+    data[:invalid].should_not be_true
+  end
+end

data/spec/etl/transformations/uk_post_code_spec.rb

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Chicago::ETL::Transformations::UkPostCode do
+  it "reformats a 6 character postcode" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " SW 30 TA")
+
+    row[:post_code].should == "SW3 0TA"
+  end
+
+  it "reformats a 7 character postcode" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " G L515 FD")
+
+    row[:post_code].should == "GL51 5FD"
+  end
+
+  it "doesn't reformat a BFPO post code" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => "BFPO 23")
+
+    row[:post_code].should == "BFPO 23"
+  end
+
+  it "fixes people using 0 instead of O for OX and OL post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " 0X1 4HG")
+
+    row[:post_code].should == "OX1 4HG"
+
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " 0L1 4HG")
+
+    row[:post_code].should == "OL1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for OX post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " 0X1 4HG")
+
+    row[:post_code].should == "OX1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for OL post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " 0L1 0LG")
+
+    row[:post_code].should == "OL1 0LG"
+  end
+
+  it "fixes people using 0 instead of O for PO post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " P01 4HG")
+
+    row[:post_code].should == "PO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for SO post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " S01 4HG")
+
+    row[:post_code].should == "SO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for CO post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " C01 4HG")
+
+    row[:post_code].should == "CO1 4HG"
+  end
+
+  it "fixes people using 0 instead of O for YO post codes" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => " Y01 4HG")
+
+    row[:post_code].should == "YO1 4HG"
+  end
+
+  it "fixes shift key slips" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => ")X!£ $HG")
+
+    row[:post_code].should == "OX13 4HG"
+  end
+
+  it "uppercases post code letters" do
+    row, _ = described_class.new(:post_code).
+      call(:post_code => "sw3 0ta")
+
+    row[:post_code].should == "SW3 0TA"
+  end
+
+  it "can be configured to only run given a block with success" do
+    transform = described_class.new(:post_code) {|row| row[:country] == "GB" }
+
+    transform.call(:post_code => "sw1 7yh").first[:post_code].
+      should == "sw1 7yh"
+    transform.call(:country => "GB", :post_code => "sw1 7yh").
+      first[:post_code].should == "SW1 7YH"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'chicago'
+require 'chicago/etl'
+require 'yaml'
+require 'timecop'
+
+include Chicago
+
+unless defined? TEST_DB
+  TEST_DB = Sequel.connect(YAML.load(File.read(File.dirname(__FILE__) + "/db_connections.yml")))
+end
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+end

metadata

@@ -0,0 +1,245 @@
+--- !ruby/object:Gem::Specification 
+name: chicago-etl
+version: !ruby/object:Gem::Version 
+  hash: 13
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 9
+  version: 0.0.9
+platform: ruby
+authors: 
+- Roland Swingler
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-02-19 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        - 4
+        version: "0.4"
+  requirement: *id001
+  type: :runtime
+  prerelease: false
+  name: chicagowarehouse
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        version: "2"
+  requirement: *id002
+  type: :development
+  prerelease: false
+  name: rspec
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id003
+  type: :development
+  prerelease: false
+  name: timecop
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+  type: :development
+  prerelease: false
+  name: yard
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id005
+  type: :development
+  prerelease: false
+  name: flog
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id006
+  type: :development
+  prerelease: false
+  name: jeweler
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id007
+  type: :development
+  prerelease: false
+  name: rcov
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id008
+  type: :development
+  prerelease: false
+  name: simplecov
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id009
+  type: :development
+  prerelease: false
+  name: ZenTest
+description: ETL tools for Chicago
+email: roland.swingler@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- chicago-etl.gemspec
+- lib/chicago-etl.rb
+- lib/chicago/etl.rb
+- lib/chicago/etl/batch.rb
+- lib/chicago/etl/buffering_insert_writer.rb
+- lib/chicago/etl/counter.rb
+- lib/chicago/etl/key_builder.rb
+- lib/chicago/etl/load_dataset_builder.rb
+- lib/chicago/etl/mysql_dumpfile.rb
+- lib/chicago/etl/mysql_load_file_value_transformer.rb
+- lib/chicago/etl/screens/column_screen.rb
+- lib/chicago/etl/screens/composite_screen.rb
+- lib/chicago/etl/screens/invalid_element.rb
+- lib/chicago/etl/screens/missing_value.rb
+- lib/chicago/etl/screens/out_of_bounds.rb
+- lib/chicago/etl/sequel/dependant_tables.rb
+- lib/chicago/etl/sequel/filter_to_etl_batch.rb
+- lib/chicago/etl/sequel/load_data_infile.rb
+- lib/chicago/etl/sink.rb
+- lib/chicago/etl/table_builder.rb
+- lib/chicago/etl/task_invocation.rb
+- lib/chicago/etl/tasks.rb
+- lib/chicago/etl/transformations/add_insert_timestamp.rb
+- lib/chicago/etl/transformations/uk_post_code.rb
+- lib/chicago/etl/transformations/uk_post_code_field.rb
+- spec/db_connections.yml.dist
+- spec/etl/batch_spec.rb
+- spec/etl/counter_spec.rb
+- spec/etl/etl_batch_id_dataset_filter.rb
+- spec/etl/key_builder_spec.rb
+- spec/etl/load_dataset_builder_spec.rb
+- spec/etl/mysql_dumpfile_spec.rb
+- spec/etl/mysql_load_file_value_transformer_spec.rb
+- spec/etl/screens/composite_screen_spec.rb
+- spec/etl/screens/invalid_element_spec.rb
+- spec/etl/screens/missing_value_spec.rb
+- spec/etl/screens/out_of_bounds_spec.rb
+- spec/etl/sequel/dependant_tables_spec.rb
+- spec/etl/sequel/filter_to_etl_batch_spec.rb
+- spec/etl/sequel/load_data_infile_spec.rb
+- spec/etl/sink_spec.rb
+- spec/etl/table_builder_spec.rb
+- spec/etl/task_spec.rb
+- spec/etl/transformations/add_insert_timestamp_spec.rb
+- spec/etl/transformations/uk_post_code_field_spec.rb
+- spec/etl/transformations/uk_post_code_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/notonthehighstreet/chicago-etl
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Chicago ETL
+test_files: []
+