etl 0.9.5.rc1

data/spec/fixtures/nokogiri_all.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_all.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/spec/fixtures/nokogiri_select.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_select.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person[@type="client"]',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/spec/fixtures/output/.ignore

@@ -0,0 +1 @@
+*.txt

data/spec/fixtures/output/delimited.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,23cc5914d48b146f0fbb73c4,24,Male,test!,2011-09-15 10:42:00
+2,Jim,Foxworthy,596e3534978b8c2b47851e37,51,Male,test!,2011-09-15 10:42:00
+3,Brian,Collingsworth,535776974fbf28b1dfbab5bd,10,Male,test!,2011-09-15 10:42:00

data/spec/fixtures/output/encode_destination_utf-8.txt

@@ -0,0 +1,2 @@
+éphémère has accents.
+let's encode them.

data/spec/fixtures/output/fixed_width.txt

@@ -0,0 +1,3 @@
+Bob,Smith,2b7a4a0b79c7d22dd7703a65,23
+Jane,Doe,d23b42528de0c6c5615cb438,45
+Abcdefghi,Jklmnopqrstu,f7c3bc1d808e04732adf6799,20

data/spec/fixtures/output/inline_parser.txt

@@ -0,0 +1,3 @@
+foo
+bar
+baz

data/spec/fixtures/output/multiple_source_delimited.txt

@@ -0,0 +1,6 @@
+Chris,Smith,23cc5914d48b146f0fbb73c4,24,,12
+Jim,Foxworthy,596e3534978b8c2b47851e37,51,,14
+Brian,Collingsworth,535776974fbf28b1dfbab5bd,10,,83
+Chris,Smith,23cc5914d48b146f0fbb73c4,24,dude,
+Jim,Foxworthy,596e3534978b8c2b47851e37,51,broseph,
+Brian,Collingsworth,535776974fbf28b1dfbab5bd,10,maaan,

data/spec/fixtures/output/test_file_destination.2.txt

@@ -0,0 +1,2 @@
+123 SW 1st Street|Melbourne|Florida|United States|US[EOL]
+Allen's Way|Some City|Some State|Mexico|MX[EOL]

data/spec/fixtures/output/test_file_destination.txt

@@ -0,0 +1,2 @@
+123 SW 1st Street,Melbourne,Florida,United States,US
+Allen's Way,Some City,Some State,Mexico,MX

data/spec/fixtures/output/test_multiple_unique.txt

@@ -0,0 +1 @@
+Bob,Smith,111234444

data/spec/fixtures/output/test_unique.txt

@@ -0,0 +1,2 @@
+Bob,Smith,111234444
+John,Smith,000112222

data/spec/fixtures/sax.ctl

@@ -0,0 +1,26 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/sax.xml',
+  :parser => :sax
+}, 
+{
+  :write_trigger => 'people/person',
+  :fields => {
+    :first_name => 'people/person/first_name',
+    :last_name => 'people/person/last_name',
+    :ssn => 'people/person/social_security_number',
+    :age => 'people/person[age]'
+  }
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+transform :age, :type, {:type => :number}
+
+destination :out, {
+  :file => 'output/sax.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/spec/fixtures/scd/1.txt

@@ -0,0 +1 @@
+Bob,Smith,200 South Drive,Boston,MA,32123

data/spec/fixtures/scd/2.txt

@@ -0,0 +1 @@
+Bob,Smith,1010 SW 23rd St,Los Angeles,CA,90392

data/spec/fixtures/scd/3.txt

@@ -0,0 +1 @@
+Bob,Smith,280 Pine Street,Los Angeles,CA,90392

data/spec/fixtures/scd_test_type_1.ctl

@@ -0,0 +1,43 @@
+source :in, {
+  :file => "scd/#{ENV['run_number']}.txt",
+  :parser => :csv
+},
+[
+  :first_name,
+  :last_name,
+  :address,
+  :city,
+  :state,
+  :zip_code
+]
+
+# NOTE: These are not usually required for a type 1 SCD dimension, but since
+# we're sharing this table with the type 2 tests, they're necessary.
+transform :effective_date, :default, :default_value => Time.now.to_s(:db)
+transform :end_date, :default, :default_value => '9999-12-31 00:00:00'
+transform :latest_version, :default, :default_value => true
+
+destination :out, {
+  :file => 'output/scd_test_type_1.txt',
+  :natural_key => [:first_name, :last_name],
+  :scd => {
+    :type => 1,
+    :dimension_target => :data_warehouse,
+    :dimension_table => 'person_dimension'
+  },
+  :scd_fields => [:address, :city, :state, :zip_code]
+}, 
+{
+  :order => [
+    :id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
+  ],
+  :virtual => {
+    :id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
+  }
+}
+
+post_process :bulk_import, {
+  :file => 'output/scd_test_type_1.txt',
+  :target => :data_warehouse,
+  :table => 'person_dimension'
+}

data/spec/fixtures/scd_test_type_2.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => "scd/#{ENV['run_number']}.txt",
+  :parser => :csv
+},
+[
+  :first_name,
+  :last_name,
+  :address,
+  :city,
+  :state,
+  :zip_code
+]
+
+destination :out, {
+  :type => :database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'person_dimension',
+  :natural_key => [:first_name, :last_name],
+  :scd => {
+    :type => 2,
+    :dimension_target => :data_warehouse,
+    :dimension_table => 'person_dimension'
+  },
+  :scd_fields => ENV['type_2_scd_fields'] ? Marshal.load(ENV['type_2_scd_fields']) : [:address, :city, :state, :zip_code]
+}, 
+{
+  :order => [
+    :id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
+  ],
+  :virtual => {
+    :id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
+  }
+}

data/spec/fixtures/screen_test_error.ctl

@@ -0,0 +1,3 @@
+screen(:error){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/spec/fixtures/screen_test_fatal.ctl

@@ -0,0 +1,3 @@
+screen(:fatal){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/spec/fixtures/xml.ctl

@@ -0,0 +1,31 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/xml.xml',
+  :parser => :xml
+}, 
+{
+  :collection => 'people/person',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => 'social_security_number'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/spec/quality_spec.rb

@@ -0,0 +1,11 @@
+require "spec_helper"
+
+IGNORE = /\.(gitmodules|txt$|png$|tar$|gz$|rbc$|gem$|pdf$)/
+
+describe "The application itself" do
+  it "has no malformed whitespace" do
+    files = `git ls-files`.split("\n").select {|fn| fn !~ IGNORE}
+
+    files.should be_well_formed
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'bundler/setup'
+
+Bundler.require :default, :development, :test
+
+Dir[ Bundler.root.join("spec/support/**/*.rb") ].each{|f| require f}
+
+RSpec.configure do |c|
+  c.include CustomMatchers
+  c.include CustomFixtures
+end

data/spec/support/custom_fixtures.rb

@@ -0,0 +1,54 @@
+require 'bundler'
+
+module CustomFixtures
+  attr_reader :fixtures
+
+  # @param [#to_sym]
+  # @return [String]
+  def fixture_for(name)
+    fixtures[name.to_sym][:fixture]
+  end
+
+  # @param [#to_sym]
+  # @return [String]
+  def path_for(name)
+    fixtures[name.to_sym][:path]
+  end
+
+  # @see ConfigFixtures#create_fixture_hash
+  def fixtures
+    @fixtures ||= begin
+      hash = Hash.new {|k,v| k[v] = {}}
+      hash.merge!(create_fixture_hash)
+    end
+  end
+
+  # @return [Hash{Symbol => String}]
+  def create_fixture_hash
+    Hash[ find_fixtures.map{|fpath| map_fixture(fpath) } ]
+  end
+
+  # @param [String]
+  # @return [Array<Symbol, String>]
+  def map_fixture(fpath)
+    [symbolize_filename(fpath), {:path => fpath, :fixture => read_file(fpath)}]
+  end
+
+  # @return [Array<String>]
+  def find_fixtures
+    Dir.glob Bundler.root.join('spec/fixtures/**/*')
+  end
+
+  # @param [String]
+  # @return [Symbol]
+  def symbolize_filename(fpath)
+    fname = File.basename(fpath)
+    fname.split(/\W/).shift.to_sym
+  end
+
+  # @param [String]
+  # @return [String]
+  def read_file(fpath)
+    File.read(fpath)
+  end
+end

data/spec/support/custom_matchers.rb

@@ -0,0 +1,54 @@
+module CustomMatchers
+  class BeWellFormed
+    attr_writer :errors
+
+    def errors
+      @errors ||= []
+    end
+
+    def matches?(files)
+      self.errors = files.dup
+
+      errors.map! do |filename|
+        file = File.read(filename).encode!("UTF-8")
+
+        next unless file.valid_encoding?
+
+        [ check_for_tabs(filename, file), excessive_spacing(filename, file), newline_precedes_eof(filename, file) ]
+      end
+
+      errors.flatten!
+      errors.compact!
+
+      errors.empty?
+    end
+
+    def failure_message_for_should
+      errors.join("\n")
+    end
+
+    def check_for_tabs(filename, file)
+      bad_lines = file.lines.each_with_index.map do |line, line_no|
+                    line_no + 1 if line["\t"] and line !~ /^\s+#.*\s+\n$/
+                  end.flatten.compact
+
+      "#{filename} has tab characters on lines #{bad_lines.join(', ')}" if bad_lines.any?
+    end
+
+    def excessive_spacing(filename, file)
+      bad_lines = file.lines.each_with_index.map do |line, line_no|
+                    line_no + 1 if line =~ /\s+\n$/ and line !~ /^\s+#.*\s+\n$/
+                  end.flatten.compact
+
+      "#{filename} has spaces on the EOL on lines #{bad_lines.join(', ')}" if bad_lines.any?
+    end
+
+    def newline_precedes_eof(filename, file)
+      "#{filename} does not have a newline (\\n) before EOF" if file !~ /\n$/
+    end
+  end
+
+  def be_well_formed
+    BeWellFormed.new
+  end
+end

data/test-matrix.yml

@@ -0,0 +1,10 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+#  - jruby-1.6.2
+rails:
+  - 2.3.x
+  - 3.0.x
+database:
+  - mysql
+  - postgresql

data/test/.gitignore

@@ -0,0 +1 @@
+database*.yml

data/test/.ignore

@@ -0,0 +1,2 @@
+database.yml
+*.txt

data/test/batch_test.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BatchTest < Test::Unit::TestCase
+  attr_reader :file, :db_yaml, :engine
+  def setup
+    @file =  File.dirname(__FILE__) + '/all.ebf'
+    @db_yaml = File.dirname(__FILE__) + '/database.yml'
+    @engine = ETL::Engine.new
+  end
+  def teardown
+    
+  end
+  def test_etl_batch_file
+    #`etl #{file} -c #{db_yaml}`
+  end
+  def test_batch
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(file, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_file
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(File.new(file), engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_batch_object
+    assert_nothing_raised do
+      batch_instance = ETL::Batch::Batch.new(File.new(file))
+      batch_instance.engine = engine
+      batch = ETL::Batch::Batch.resolve(batch_instance, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_object_should_fail
+    assert_raise(RuntimeError) do
+      batch = ETL::Batch::Batch.resolve(0, engine)
+    end
+  end
+end