etl 0.9.5.rc1

data/spec/fixtures/data/people.txt

@@ -0,0 +1,3 @@
+Bob,Smith
+Jane,Doe
+Chris,Cornell

data/spec/fixtures/data/sax.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+
+<people>
+  <person age="24">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+  </person>
+  <person age="31">
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+  </person>
+</people>

data/spec/fixtures/data/xml.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<people>
+  <person>
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+    <age>24</age>
+  </person>
+  <person>
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+    <age>31</age>
+  </person>
+</people>

data/spec/fixtures/delimited.ctl

@@ -0,0 +1,30 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :file => 'output/delimited.txt'
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/spec/fixtures/delimited_absolute.ctl

@@ -0,0 +1,31 @@
+source :in, {
+  :file => '/tmp/delimited_abs.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  },
+  :sex
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :file => 'data/delimited_abs.txt'
+},
+{
+  :order => [:first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :test => "test!",
+    :calc_test => Time.now
+  }
+}

data/spec/fixtures/delimited_destination_db.ctl

@@ -0,0 +1,23 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => :csv
+}, 
+[ 
+  :id,
+  :first_name,
+  :last_name,
+  :ssn
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+
+destination :out, {
+  :type => :database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people',
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn]
+}

data/spec/fixtures/delimited_excel.ctl

@@ -0,0 +1,31 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :excel,
+  :file => 'output/delimited_excel.xls'
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/spec/fixtures/delimited_insert_update.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :insert_update_database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people'
+},
+{
+  :primarykey => [:id],
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/spec/fixtures/delimited_update.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :update_database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people'
+},
+{
+  :conditions => [{:field => "\#{conn.quote_column_name(:id)}", :value => "\#{conn.quote(row[:id])}", :comp => "="}],
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/spec/fixtures/delimited_with_bulk_load.ctl

@@ -0,0 +1,34 @@
+infile = 'data/people.txt'
+outfile = 'output/people.txt'
+
+source :in, {
+  :file => infile,
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+]
+
+before_write :surrogate_key, :target => :data_warehouse, :table => 'person_dimension', :column => 'id'
+before_write :check_exist, {
+  :target => :data_warehouse, 
+  :table => 'person_dimension', 
+  :columns => [:first_name, :last_name]
+}
+
+destination :out, {
+  :file => outfile
+},
+{
+  :order => [:id, :first_name, :last_name]
+}
+
+post_process :bulk_import, {
+  :file => outfile,
+  :target => :data_warehouse,
+  :table => 'person_dimension',
+  :order => [:id, :first_name, :last_name]
+}

data/spec/fixtures/errors.ctl

@@ -0,0 +1,24 @@
+class ErrorProcessor < ETL::Processor::RowProcessor
+  def initialize(control, configuration)
+    super
+  end
+  def process(row)
+    raise RuntimeError, "Generated error"
+  end
+end
+
+set_error_threshold 1
+
+source :in, {
+  :type => :enumerable,
+  :enumerable => [
+    {:first_name => 'Bob',:last_name => 'Smith'},
+    {:first_name => 'Joe', :last_name => 'Thompson'}
+  ]
+},
+[
+  :first_name,
+  :last_name
+]
+
+after_read ErrorProcessor

data/spec/fixtures/excel.ctl

@@ -0,0 +1,24 @@
+source :in, {
+  :file => 'data/excel.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => false,
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/spec/fixtures/excel2.ctl

@@ -0,0 +1,25 @@
+source :in, {
+  :file => 'data/excel2.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => true,
+  :worksheets => [ 1 ],
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel2.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/spec/fixtures/fixed_width.ctl

@@ -0,0 +1,35 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/fixed_width.txt',
+  :parser => :fixed_width
+}, 
+{
+  :first_name => {
+    :start => 1,
+    :length => 9
+  },
+  :last_name => {
+    :start => 10,
+    :length => 12
+  },
+  :ssn => {
+    :start => 22,
+    :length => 9
+  },
+  :age => {
+    :start => 31,
+    :length => 2,
+    :type => :integer
+  }
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+destination :out, {
+  :file => 'output/fixed_width.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/spec/fixtures/inline_parser.ctl

@@ -0,0 +1,17 @@
+class MyParser < ETL::Parser::Parser
+  def each
+    [{:name => 'foo'},{:name => 'bar'},{:name => 'baz'}].each do |row|
+      yield row
+    end
+  end
+end
+
+source :in, {
+  :file => '',
+  :parser => MyParser
+}, 
+[ 
+  :name
+]
+
+destination :out, {:file => 'output/inline_parser.txt'},{:order => [:name]}

data/spec/fixtures/model_source.ctl

@@ -0,0 +1,14 @@
+source :in, {
+  :type => :model
+}, 
+[ 
+  :first_name,
+  :last_name
+]
+
+destination :out, {
+  :file => 'data/model_out.txt'
+},
+{
+  :order => [:first_name, :last_name],
+}

data/spec/fixtures/multiple_delimited.ctl

@@ -0,0 +1,22 @@
+# puts "executing delimited.ctl"
+
+source :in, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+destination :out, {
+  :file => 'output/multiple_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/spec/fixtures/multiple_source_delimited.ctl

@@ -0,0 +1,39 @@
+# puts "executing delimited.ctl"
+
+source :source1, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+source :source2, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+
+destination :out, {
+  :file => 'output/multiple_source_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}