activewarehouse-etl-sgonyea 0.9.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +9 -0
- data/0.9-UPGRADE +6 -0
- data/CHANGELOG +236 -0
- data/Gemfile +4 -0
- data/HOW_TO_RELEASE +13 -0
- data/LICENSE +7 -0
- data/README.textile +111 -0
- data/Rakefile +103 -0
- data/TODO +28 -0
- data/active_support_logger.patch +78 -0
- data/activewarehouse-etl.gemspec +36 -0
- data/bin/etl +28 -0
- data/bin/etl.cmd +8 -0
- data/examples/database.example.yml +16 -0
- data/lib/etl.rb +97 -0
- data/lib/etl/batch.rb +2 -0
- data/lib/etl/batch/batch.rb +111 -0
- data/lib/etl/batch/directives.rb +65 -0
- data/lib/etl/builder.rb +2 -0
- data/lib/etl/builder/date_dimension_builder.rb +96 -0
- data/lib/etl/builder/time_dimension_builder.rb +31 -0
- data/lib/etl/commands/etl.rb +89 -0
- data/lib/etl/control.rb +3 -0
- data/lib/etl/control/control.rb +405 -0
- data/lib/etl/control/destination.rb +438 -0
- data/lib/etl/control/destination/csv_destination.rb +113 -0
- data/lib/etl/control/destination/database_destination.rb +97 -0
- data/lib/etl/control/destination/excel_destination.rb +91 -0
- data/lib/etl/control/destination/file_destination.rb +126 -0
- data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
- data/lib/etl/control/destination/update_database_destination.rb +109 -0
- data/lib/etl/control/destination/yaml_destination.rb +74 -0
- data/lib/etl/control/source.rb +132 -0
- data/lib/etl/control/source/database_source.rb +224 -0
- data/lib/etl/control/source/enumerable_source.rb +11 -0
- data/lib/etl/control/source/file_source.rb +90 -0
- data/lib/etl/control/source/model_source.rb +39 -0
- data/lib/etl/core_ext.rb +1 -0
- data/lib/etl/core_ext/time.rb +5 -0
- data/lib/etl/core_ext/time/calculations.rb +42 -0
- data/lib/etl/engine.rb +582 -0
- data/lib/etl/execution.rb +19 -0
- data/lib/etl/execution/base.rb +8 -0
- data/lib/etl/execution/batch.rb +10 -0
- data/lib/etl/execution/job.rb +8 -0
- data/lib/etl/execution/migration.rb +90 -0
- data/lib/etl/generator.rb +2 -0
- data/lib/etl/generator/generator.rb +20 -0
- data/lib/etl/generator/surrogate_key_generator.rb +39 -0
- data/lib/etl/http_tools.rb +139 -0
- data/lib/etl/parser.rb +11 -0
- data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
- data/lib/etl/parser/csv_parser.rb +93 -0
- data/lib/etl/parser/excel_parser.rb +112 -0
- data/lib/etl/parser/fixed_width_parser.rb +65 -0
- data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
- data/lib/etl/parser/parser.rb +41 -0
- data/lib/etl/parser/sax_parser.rb +218 -0
- data/lib/etl/parser/xml_parser.rb +65 -0
- data/lib/etl/processor.rb +11 -0
- data/lib/etl/processor/block_processor.rb +14 -0
- data/lib/etl/processor/bulk_import_processor.rb +94 -0
- data/lib/etl/processor/check_exist_processor.rb +80 -0
- data/lib/etl/processor/check_unique_processor.rb +39 -0
- data/lib/etl/processor/copy_field_processor.rb +26 -0
- data/lib/etl/processor/database_join_processor.rb +82 -0
- data/lib/etl/processor/encode_processor.rb +55 -0
- data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
- data/lib/etl/processor/escape_csv_processor.rb +77 -0
- data/lib/etl/processor/filter_row_processor.rb +51 -0
- data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
- data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
- data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
- data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
- data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
- data/lib/etl/processor/print_row_processor.rb +12 -0
- data/lib/etl/processor/processor.rb +25 -0
- data/lib/etl/processor/rename_processor.rb +24 -0
- data/lib/etl/processor/require_non_blank_processor.rb +26 -0
- data/lib/etl/processor/row_processor.rb +27 -0
- data/lib/etl/processor/sequence_processor.rb +23 -0
- data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
- data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
- data/lib/etl/processor/surrogate_key_processor.rb +53 -0
- data/lib/etl/processor/truncate_processor.rb +40 -0
- data/lib/etl/processor/zip_file_processor.rb +27 -0
- data/lib/etl/row.rb +20 -0
- data/lib/etl/screen.rb +14 -0
- data/lib/etl/screen/row_count_screen.rb +20 -0
- data/lib/etl/transform.rb +2 -0
- data/lib/etl/transform/block_transform.rb +13 -0
- data/lib/etl/transform/calculation_transform.rb +71 -0
- data/lib/etl/transform/date_to_string_transform.rb +20 -0
- data/lib/etl/transform/decode_transform.rb +51 -0
- data/lib/etl/transform/default_transform.rb +20 -0
- data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
- data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
- data/lib/etl/transform/md5_transform.rb +13 -0
- data/lib/etl/transform/ordinalize_transform.rb +14 -0
- data/lib/etl/transform/sha1_transform.rb +13 -0
- data/lib/etl/transform/split_fields_transform.rb +27 -0
- data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
- data/lib/etl/transform/string_to_date_transform.rb +16 -0
- data/lib/etl/transform/string_to_time_transform.rb +11 -0
- data/lib/etl/transform/transform.rb +61 -0
- data/lib/etl/transform/trim_transform.rb +26 -0
- data/lib/etl/transform/type_transform.rb +35 -0
- data/lib/etl/util.rb +59 -0
- data/lib/etl/version.rb +3 -0
- data/test-matrix.yml +10 -0
- data/test/.gitignore +1 -0
- data/test/.ignore +2 -0
- data/test/all.ebf +6 -0
- data/test/apache_combined_log.ctl +11 -0
- data/test/batch_test.rb +41 -0
- data/test/batch_with_error.ebf +6 -0
- data/test/batched1.ctl +0 -0
- data/test/batched2.ctl +0 -0
- data/test/block_processor.ctl +6 -0
- data/test/block_processor_error.ctl +1 -0
- data/test/block_processor_pre_post_process.ctl +4 -0
- data/test/block_processor_remove_rows.ctl +5 -0
- data/test/block_processor_test.rb +38 -0
- data/test/check_exist_processor_test.rb +92 -0
- data/test/check_unique_processor_test.rb +40 -0
- data/test/config/Gemfile.rails-2.3.x +3 -0
- data/test/config/Gemfile.rails-2.3.x.lock +53 -0
- data/test/config/Gemfile.rails-3.0.x +3 -0
- data/test/config/Gemfile.rails-3.0.x.lock +61 -0
- data/test/config/common.rb +29 -0
- data/test/connection/mysql/connection.rb +9 -0
- data/test/connection/mysql/schema.sql +37 -0
- data/test/connection/postgresql/connection.rb +13 -0
- data/test/connection/postgresql/schema.sql +40 -0
- data/test/control_test.rb +43 -0
- data/test/data/apache_combined_log.txt +3 -0
- data/test/data/bulk_import.txt +3 -0
- data/test/data/bulk_import_with_empties.txt +3 -0
- data/test/data/decode.txt +3 -0
- data/test/data/delimited.txt +3 -0
- data/test/data/encode_source_latin1.txt +2 -0
- data/test/data/excel.xls +0 -0
- data/test/data/excel2.xls +0 -0
- data/test/data/fixed_width.txt +3 -0
- data/test/data/multiple_delimited_1.txt +3 -0
- data/test/data/multiple_delimited_2.txt +3 -0
- data/test/data/nokogiri.xml +38 -0
- data/test/data/people.txt +3 -0
- data/test/data/sax.xml +14 -0
- data/test/data/xml.xml +16 -0
- data/test/database_join_processor_test.rb +43 -0
- data/test/date_dimension_builder_test.rb +96 -0
- data/test/delimited.ctl +30 -0
- data/test/delimited_absolute.ctl +31 -0
- data/test/delimited_destination_db.ctl +23 -0
- data/test/delimited_excel.ctl +31 -0
- data/test/delimited_insert_update.ctl +34 -0
- data/test/delimited_update.ctl +34 -0
- data/test/delimited_with_bulk_load.ctl +34 -0
- data/test/destination_test.rb +275 -0
- data/test/directive_test.rb +23 -0
- data/test/encode_processor_test.rb +32 -0
- data/test/engine_test.rb +78 -0
- data/test/ensure_fields_presence_processor_test.rb +28 -0
- data/test/errors.ctl +24 -0
- data/test/etl_test.rb +42 -0
- data/test/excel.ctl +24 -0
- data/test/excel2.ctl +25 -0
- data/test/fixed_width.ctl +35 -0
- data/test/foreign_key_lookup_transform_test.rb +50 -0
- data/test/generator_test.rb +14 -0
- data/test/inline_parser.ctl +17 -0
- data/test/mocks/mock_destination.rb +26 -0
- data/test/mocks/mock_source.rb +25 -0
- data/test/model_source.ctl +14 -0
- data/test/multiple_delimited.ctl +22 -0
- data/test/multiple_source_delimited.ctl +39 -0
- data/test/nokogiri_all.ctl +35 -0
- data/test/nokogiri_select.ctl +35 -0
- data/test/nokogiri_test.rb +35 -0
- data/test/parser_test.rb +224 -0
- data/test/performance/delimited.ctl +30 -0
- data/test/processor_test.rb +44 -0
- data/test/row_processor_test.rb +17 -0
- data/test/sax.ctl +26 -0
- data/test/scd/1.txt +1 -0
- data/test/scd/2.txt +1 -0
- data/test/scd/3.txt +1 -0
- data/test/scd_test.rb +257 -0
- data/test/scd_test_type_1.ctl +43 -0
- data/test/scd_test_type_2.ctl +34 -0
- data/test/screen_test.rb +9 -0
- data/test/screen_test_error.ctl +3 -0
- data/test/screen_test_fatal.ctl +3 -0
- data/test/source_test.rb +154 -0
- data/test/test_helper.rb +37 -0
- data/test/transform_test.rb +101 -0
- data/test/truncate_processor_test.rb +37 -0
- data/test/xml.ctl +31 -0
- metadata +370 -0
@@ -0,0 +1,43 @@
|
|
1
|
+
source :in, {
|
2
|
+
:file => "scd/#{ENV['run_number']}.txt",
|
3
|
+
:parser => :csv
|
4
|
+
},
|
5
|
+
[
|
6
|
+
:first_name,
|
7
|
+
:last_name,
|
8
|
+
:address,
|
9
|
+
:city,
|
10
|
+
:state,
|
11
|
+
:zip_code
|
12
|
+
]
|
13
|
+
|
14
|
+
# NOTE: These are not usually required for a type 1 SCD dimension, but since
|
15
|
+
# we're sharing this table with the type 2 tests, they're necessary.
|
16
|
+
transform :effective_date, :default, :default_value => Time.now.to_s(:db)
|
17
|
+
transform :end_date, :default, :default_value => '9999-12-31 00:00:00'
|
18
|
+
transform :latest_version, :default, :default_value => true
|
19
|
+
|
20
|
+
destination :out, {
|
21
|
+
:file => 'output/scd_test_type_1.txt',
|
22
|
+
:natural_key => [:first_name, :last_name],
|
23
|
+
:scd => {
|
24
|
+
:type => 1,
|
25
|
+
:dimension_target => :data_warehouse,
|
26
|
+
:dimension_table => 'person_dimension'
|
27
|
+
},
|
28
|
+
:scd_fields => [:address, :city, :state, :zip_code]
|
29
|
+
},
|
30
|
+
{
|
31
|
+
:order => [
|
32
|
+
:id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
|
33
|
+
],
|
34
|
+
:virtual => {
|
35
|
+
:id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
|
36
|
+
}
|
37
|
+
}
|
38
|
+
|
39
|
+
post_process :bulk_import, {
|
40
|
+
:file => 'output/scd_test_type_1.txt',
|
41
|
+
:target => :data_warehouse,
|
42
|
+
:table => 'person_dimension'
|
43
|
+
}
|
@@ -0,0 +1,34 @@
|
|
1
|
+
source :in, {
|
2
|
+
:file => "scd/#{ENV['run_number']}.txt",
|
3
|
+
:parser => :csv
|
4
|
+
},
|
5
|
+
[
|
6
|
+
:first_name,
|
7
|
+
:last_name,
|
8
|
+
:address,
|
9
|
+
:city,
|
10
|
+
:state,
|
11
|
+
:zip_code
|
12
|
+
]
|
13
|
+
|
14
|
+
destination :out, {
|
15
|
+
:type => :database,
|
16
|
+
:target => :data_warehouse,
|
17
|
+
:database => 'etl_unittest',
|
18
|
+
:table => 'person_dimension',
|
19
|
+
:natural_key => [:first_name, :last_name],
|
20
|
+
:scd => {
|
21
|
+
:type => 2,
|
22
|
+
:dimension_target => :data_warehouse,
|
23
|
+
:dimension_table => 'person_dimension'
|
24
|
+
},
|
25
|
+
:scd_fields => ENV['type_2_scd_fields'] ? Marshal.load(ENV['type_2_scd_fields']) : [:address, :city, :state, :zip_code]
|
26
|
+
},
|
27
|
+
{
|
28
|
+
:order => [
|
29
|
+
:id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
|
30
|
+
],
|
31
|
+
:virtual => {
|
32
|
+
:id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
|
33
|
+
}
|
34
|
+
}
|
data/test/screen_test.rb
ADDED
data/test/source_test.rb
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class Person < ActiveRecord::Base
|
4
|
+
end
|
5
|
+
class SourceTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
context "source" do
|
8
|
+
should "set store_locally to true by default" do
|
9
|
+
assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
|
10
|
+
end
|
11
|
+
|
12
|
+
should "let the user set store_locally to true" do
|
13
|
+
assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
|
14
|
+
end
|
15
|
+
|
16
|
+
should "let the user set store_locally to false" do
|
17
|
+
assert_equal false, Source.new(nil, { :store_locally => false }, nil).store_locally
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "a file source" do
|
22
|
+
context "with delimited data" do
|
23
|
+
setup do
|
24
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
25
|
+
configuration = {
|
26
|
+
:file => 'data/delimited.txt',
|
27
|
+
:parser => :csv
|
28
|
+
}
|
29
|
+
definition = self.definition + [:sex]
|
30
|
+
|
31
|
+
source = ETL::Control::FileSource.new(control, configuration, definition)
|
32
|
+
@rows = source.collect { |row| row }
|
33
|
+
end
|
34
|
+
should "find 3 rows in the delimited file" do
|
35
|
+
assert_equal 3, @rows.length
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context "a file source with a glob" do
|
41
|
+
setup do
|
42
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/multiple_delimited.ctl')
|
43
|
+
configuration = {
|
44
|
+
:file => 'data/multiple_delimited_*.txt',
|
45
|
+
:parser => :csv
|
46
|
+
}
|
47
|
+
|
48
|
+
source = ETL::Control::FileSource.new(control, configuration, definition)
|
49
|
+
@rows = source.collect { |row| row }
|
50
|
+
end
|
51
|
+
should "find 6 rows in total" do
|
52
|
+
assert_equal 6, @rows.length
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context "a file source with an absolute path" do
|
57
|
+
setup do
|
58
|
+
FileUtils.cp(File.dirname(__FILE__) + '/data/delimited.txt', '/tmp/delimited_abs.txt')
|
59
|
+
|
60
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
61
|
+
'/delimited_absolute.ctl')
|
62
|
+
configuration = {
|
63
|
+
:file => '/tmp/delimited_abs.txt',
|
64
|
+
:parser => :csv
|
65
|
+
}
|
66
|
+
definition = self.definition + [:sex]
|
67
|
+
|
68
|
+
source = ETL::Control::FileSource.new(control, configuration, definition)
|
69
|
+
@rows = source.collect { |row| row }
|
70
|
+
end
|
71
|
+
should "find 3 rows" do
|
72
|
+
assert_equal 3, @rows.length
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
context "multiple sources" do
|
77
|
+
setup do
|
78
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
79
|
+
'/multiple_source_delimited.ctl')
|
80
|
+
@rows = control.sources.collect { |source| source.collect { |row| row }}.flatten!
|
81
|
+
end
|
82
|
+
should "find 12 rows" do
|
83
|
+
assert_equal 12, @rows.length
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
context "a database source" do
|
88
|
+
setup do
|
89
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
90
|
+
configuration = {
|
91
|
+
:database => 'etl_unittest',
|
92
|
+
:target => :operational_database,
|
93
|
+
:table => 'people',
|
94
|
+
}
|
95
|
+
definition = [
|
96
|
+
:first_name,
|
97
|
+
:last_name,
|
98
|
+
:ssn,
|
99
|
+
]
|
100
|
+
@source = ETL::Control::DatabaseSource.new(control, configuration, definition)
|
101
|
+
end
|
102
|
+
should "set the local file for extraction storage" do
|
103
|
+
assert_match %r{source_data/localhost/etl_unittest/people/\d+.csv}, @source.local_file.to_s
|
104
|
+
end
|
105
|
+
should "find 1 row" do
|
106
|
+
Person.delete_all
|
107
|
+
assert_equal 0, Person.count
|
108
|
+
Person.create!(:first_name => 'Bob', :last_name => 'Smith', :ssn => '123456789')
|
109
|
+
assert_equal 1, Person.count
|
110
|
+
rows = @source.collect { |row| row }
|
111
|
+
assert_equal 1, rows.length
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
context "a file source with an xml parser" do
|
116
|
+
setup do
|
117
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
118
|
+
'/xml.ctl')
|
119
|
+
@rows = control.sources.collect{ |source| source.collect { |row| row }}.flatten!
|
120
|
+
end
|
121
|
+
should "find 2 rows" do
|
122
|
+
assert_equal 2, @rows.length
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context "a model source" do
|
127
|
+
setup do
|
128
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/model_source.ctl')
|
129
|
+
configuration = {
|
130
|
+
|
131
|
+
}
|
132
|
+
definition = [
|
133
|
+
:first_name,
|
134
|
+
:last_name,
|
135
|
+
:ssn
|
136
|
+
]
|
137
|
+
end
|
138
|
+
should_eventually "find n rows" do
|
139
|
+
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def definition
|
144
|
+
[
|
145
|
+
:first_name,
|
146
|
+
:last_name,
|
147
|
+
:ssn,
|
148
|
+
{
|
149
|
+
:name => :age,
|
150
|
+
:type => :integer
|
151
|
+
}
|
152
|
+
]
|
153
|
+
end
|
154
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
|
4
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
5
|
+
$:.unshift(File.dirname(__FILE__))
|
6
|
+
|
7
|
+
require 'test/unit'
|
8
|
+
require 'pp'
|
9
|
+
require 'etl'
|
10
|
+
require 'shoulda'
|
11
|
+
require 'flexmock/test_unit'
|
12
|
+
|
13
|
+
database_yml = File.dirname(__FILE__) + '/config/database.yml'
|
14
|
+
ETL::Engine.init(:config => database_yml)
|
15
|
+
ETL::Engine.logger = Logger.new(STDOUT)
|
16
|
+
# ETL::Engine.logger.level = Logger::DEBUG
|
17
|
+
ETL::Engine.logger.level = Logger::FATAL
|
18
|
+
|
19
|
+
db = YAML::load(IO.read(database_yml))['operational_database']['adapter']
|
20
|
+
# allow both mysql2 and mysql adapters
|
21
|
+
db = db.gsub('mysql2', 'mysql')
|
22
|
+
raise "Unsupported test db '#{db}'" unless ['mysql', 'postgresql'].include?(db)
|
23
|
+
|
24
|
+
require "connection/#{db}/connection"
|
25
|
+
ActiveRecord::Base.establish_connection :operational_database
|
26
|
+
ETL::Execution::Job.delete_all
|
27
|
+
|
28
|
+
require 'mocks/mock_source'
|
29
|
+
require 'mocks/mock_destination'
|
30
|
+
|
31
|
+
# shortcut to launch a ctl file
|
32
|
+
def process(file)
|
33
|
+
Engine.process(File.join(File.dirname(__FILE__), file))
|
34
|
+
end
|
35
|
+
|
36
|
+
puts "ActiveRecord::VERSION = #{ActiveRecord::VERSION::STRING}"
|
37
|
+
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class MyResolver
|
4
|
+
def resolve(value)
|
5
|
+
4
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
class TransformTest < Test::Unit::TestCase
|
10
|
+
def test_sha1_transform
|
11
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
12
|
+
digest_value = 'a9993e364706816aba3e25717850c26c9cd0d89d'
|
13
|
+
assert_equal digest_value, ETL::Transform::Sha1Transform.new(
|
14
|
+
control, nil
|
15
|
+
).transform('test', 'abc', [])
|
16
|
+
end
|
17
|
+
def test_block_transform
|
18
|
+
#transforms = [Proc.new(){|name, value, row| value[0,2]}]
|
19
|
+
#assert_equal '11', ETL::Transform::Transform.transform(:ssn, '1111223333', [], transforms)
|
20
|
+
end
|
21
|
+
def test_decode_transform
|
22
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
23
|
+
configuration = {:decode_table_path => 'data/decode.txt'}
|
24
|
+
|
25
|
+
t = ETL::Transform::DecodeTransform.new(control, nil, configuration)
|
26
|
+
|
27
|
+
assert_equal 'Male', t.transform(nil, 'M', [])
|
28
|
+
assert_equal 'Female', t.transform(nil, 'F', [])
|
29
|
+
assert_equal 'Unknown', t.transform(nil, '', [])
|
30
|
+
assert_equal 'Unknown', t.transform(nil, 'blah', [])
|
31
|
+
end
|
32
|
+
def test_string_to_date_transform
|
33
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
34
|
+
t = ETL::Transform::StringToDateTransform.new(control, nil)
|
35
|
+
|
36
|
+
assert_equal Date.parse('2005-01-01'), t.transform(nil, '2005-01-01', [])
|
37
|
+
assert_equal Date.parse('2004-10-20 20:30:00'), t.transform(nil, '2004-10-20', [])
|
38
|
+
end
|
39
|
+
def test_date_to_string_transform
|
40
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
41
|
+
t = ETL::Transform::DateToStringTransform.new(control, nil)
|
42
|
+
|
43
|
+
d1 = Date.parse('2005-01-01')
|
44
|
+
t1 = Time.parse('2004-10-20 23:03:23')
|
45
|
+
assert_equal '2005-01-01', t.transform(nil, d1, [])
|
46
|
+
assert_equal '2004-10-20', t.transform(nil, t1, [])
|
47
|
+
|
48
|
+
t = ETL::Transform::DateToStringTransform.new(control, nil, {:format => '%m/%d/%Y'})
|
49
|
+
|
50
|
+
assert_equal '01/01/2005', t.transform(nil, d1, [])
|
51
|
+
assert_equal '10/20/2004', t.transform(nil, t1, [])
|
52
|
+
end
|
53
|
+
def test_string_to_datetime_transform
|
54
|
+
v = '1/1/1900 04:34:30'
|
55
|
+
t = ETL::Transform::StringToDateTimeTransform.new(flexmock(:control), nil)
|
56
|
+
assert_equal DateTime.parse(v), t.transform(nil, v, nil)
|
57
|
+
end
|
58
|
+
def test_foreign_key_lookup_transform
|
59
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
60
|
+
configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}}
|
61
|
+
t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
|
62
|
+
|
63
|
+
assert_equal 1, t.transform(nil, 'foo', nil)
|
64
|
+
assert_equal 2, t.transform(nil, 'bar', nil)
|
65
|
+
assert_equal 3, t.transform(nil, 'baz', nil)
|
66
|
+
assert_raises(ETL::ResolverError, 'Foreign key for bing not found and no resolver specified') do
|
67
|
+
assert_equal 4, t.transform(nil, 'bing', nil)
|
68
|
+
end
|
69
|
+
|
70
|
+
configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}, :resolver => MyResolver}
|
71
|
+
t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
|
72
|
+
assert_equal 1, t.transform(nil, 'foo', nil)
|
73
|
+
assert_equal 2, t.transform(nil, 'bar', nil)
|
74
|
+
assert_equal 3, t.transform(nil, 'baz', nil)
|
75
|
+
assert_equal 4, t.transform(nil, 'bing', nil)
|
76
|
+
end
|
77
|
+
def test_type_transform
|
78
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
79
|
+
assert_equal 10, ETL::Transform::TypeTransform.new(control, nil, {:type => :number}).transform(nil, '10', nil)
|
80
|
+
|
81
|
+
assert_equal BigDecimal::ROUND_HALF_UP, BigDecimal.mode(BigDecimal::ROUND_MODE)
|
82
|
+
decimal_transformed = ETL::Transform::TypeTransform.new(
|
83
|
+
control, nil, {:type => :decimal, :scale => 4}
|
84
|
+
).transform(nil, '10.0000000000000000000000000000000001', nil)
|
85
|
+
assert_equal '10.0000000000000000000000000000000001', decimal_transformed.to_s('F')
|
86
|
+
end
|
87
|
+
def test_non_existent_transformer
|
88
|
+
|
89
|
+
end
|
90
|
+
def test_default_transform
|
91
|
+
t = ETL::Transform::DefaultTransform.new(flexmock('control'), nil, {:default_value => 'foo'})
|
92
|
+
assert_equal 'foo', t.transform(nil, '', nil)
|
93
|
+
assert_equal 'foo', t.transform(nil, nil, nil)
|
94
|
+
assert_equal 'bar', t.transform(nil, 'bar', nil)
|
95
|
+
end
|
96
|
+
def test_ordinalize_transform
|
97
|
+
t = ETL::Transform::OrdinalizeTransform.new(flexmock('control'), nil, {})
|
98
|
+
assert_equal '1st', t.transform(nil, 1, nil)
|
99
|
+
assert_equal '10th', t.transform(nil, 10, nil)
|
100
|
+
end
|
101
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
include ETL::Processor
|
4
|
+
|
5
|
+
class TruncateTest < ActiveRecord::Base
|
6
|
+
set_table_name 'truncate_test'
|
7
|
+
end
|
8
|
+
|
9
|
+
class TruncateProcessorTest < Test::Unit::TestCase
|
10
|
+
|
11
|
+
def create_item!
|
12
|
+
TruncateTest.create!(:x => 'ABC')
|
13
|
+
end
|
14
|
+
|
15
|
+
def truncate!(options=nil)
|
16
|
+
TruncateProcessor.new(nil,
|
17
|
+
:target => :data_warehouse,
|
18
|
+
:table => TruncateTest.table_name,
|
19
|
+
:options => options
|
20
|
+
).process
|
21
|
+
end
|
22
|
+
|
23
|
+
should 'reset ids by default' do
|
24
|
+
create_item!
|
25
|
+
truncate!
|
26
|
+
assert_equal 1, create_item!.id
|
27
|
+
end
|
28
|
+
|
29
|
+
if ETL::Engine.connection(:data_warehouse).class.name =~ /postgres/i
|
30
|
+
should 'allow disabling id reset for postgres' do
|
31
|
+
truncate!
|
32
|
+
create_item!
|
33
|
+
truncate!('CONTINUE IDENTITY')
|
34
|
+
assert_equal 2, create_item!.id
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/test/xml.ctl
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# puts "executing fixed_width.ctl"
|
2
|
+
|
3
|
+
source :in, {
|
4
|
+
:file => 'data/xml.xml',
|
5
|
+
:parser => :xml
|
6
|
+
},
|
7
|
+
{
|
8
|
+
:collection => 'people/person',
|
9
|
+
:fields => [
|
10
|
+
:first_name,
|
11
|
+
:last_name,
|
12
|
+
{
|
13
|
+
:name => :ssn,
|
14
|
+
:xpath => 'social_security_number'
|
15
|
+
},
|
16
|
+
{
|
17
|
+
:name => :age,
|
18
|
+
:type => :integer
|
19
|
+
}
|
20
|
+
]
|
21
|
+
}
|
22
|
+
|
23
|
+
destination :out, {
|
24
|
+
:file => 'output/xml.txt'
|
25
|
+
},
|
26
|
+
{
|
27
|
+
:order => [:first_name, :last_name, :ssn]
|
28
|
+
}
|
29
|
+
|
30
|
+
transform :ssn, :sha1
|
31
|
+
transform(:ssn){ |v| v[0,24] }
|