balinterdi-activewarehouse-etl 0.9.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/0.9-UPGRADE +6 -0
- data/CHANGELOG +198 -0
- data/HOW_TO_RELEASE +4 -0
- data/LICENSE +7 -0
- data/README +85 -0
- data/Rakefile +169 -0
- data/TODO +28 -0
- data/VERSION +1 -0
- data/active_support_logger.patch +78 -0
- data/balinterdi-activewarehouse-etl.gemspec +221 -0
- data/bin/etl +28 -0
- data/bin/etl.cmd +8 -0
- data/examples/database.example.yml +16 -0
- data/lib/etl.rb +78 -0
- data/lib/etl/batch.rb +2 -0
- data/lib/etl/batch/batch.rb +111 -0
- data/lib/etl/batch/directives.rb +55 -0
- data/lib/etl/builder.rb +2 -0
- data/lib/etl/builder/date_dimension_builder.rb +96 -0
- data/lib/etl/builder/time_dimension_builder.rb +31 -0
- data/lib/etl/commands/etl.rb +89 -0
- data/lib/etl/control.rb +3 -0
- data/lib/etl/control/control.rb +405 -0
- data/lib/etl/control/destination.rb +420 -0
- data/lib/etl/control/destination/database_destination.rb +95 -0
- data/lib/etl/control/destination/file_destination.rb +124 -0
- data/lib/etl/control/source.rb +109 -0
- data/lib/etl/control/source/database_source.rb +220 -0
- data/lib/etl/control/source/enumerable_source.rb +11 -0
- data/lib/etl/control/source/file_source.rb +90 -0
- data/lib/etl/control/source/model_source.rb +39 -0
- data/lib/etl/core_ext.rb +1 -0
- data/lib/etl/core_ext/time.rb +5 -0
- data/lib/etl/core_ext/time/calculations.rb +42 -0
- data/lib/etl/engine.rb +556 -0
- data/lib/etl/execution.rb +20 -0
- data/lib/etl/execution/base.rb +9 -0
- data/lib/etl/execution/batch.rb +8 -0
- data/lib/etl/execution/job.rb +8 -0
- data/lib/etl/execution/migration.rb +85 -0
- data/lib/etl/generator.rb +2 -0
- data/lib/etl/generator/generator.rb +20 -0
- data/lib/etl/generator/surrogate_key_generator.rb +39 -0
- data/lib/etl/http_tools.rb +139 -0
- data/lib/etl/parser.rb +11 -0
- data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
- data/lib/etl/parser/delimited_parser.rb +75 -0
- data/lib/etl/parser/fixed_width_parser.rb +65 -0
- data/lib/etl/parser/parser.rb +41 -0
- data/lib/etl/parser/sax_parser.rb +218 -0
- data/lib/etl/parser/xml_parser.rb +65 -0
- data/lib/etl/processor.rb +11 -0
- data/lib/etl/processor/block_processor.rb +14 -0
- data/lib/etl/processor/bulk_import_processor.rb +81 -0
- data/lib/etl/processor/check_exist_processor.rb +80 -0
- data/lib/etl/processor/check_unique_processor.rb +35 -0
- data/lib/etl/processor/copy_field_processor.rb +26 -0
- data/lib/etl/processor/encode_processor.rb +55 -0
- data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
- data/lib/etl/processor/print_row_processor.rb +12 -0
- data/lib/etl/processor/processor.rb +25 -0
- data/lib/etl/processor/rename_processor.rb +24 -0
- data/lib/etl/processor/require_non_blank_processor.rb +26 -0
- data/lib/etl/processor/row_processor.rb +17 -0
- data/lib/etl/processor/sequence_processor.rb +23 -0
- data/lib/etl/processor/surrogate_key_processor.rb +53 -0
- data/lib/etl/processor/truncate_processor.rb +35 -0
- data/lib/etl/row.rb +20 -0
- data/lib/etl/screen.rb +14 -0
- data/lib/etl/screen/row_count_screen.rb +20 -0
- data/lib/etl/transform.rb +2 -0
- data/lib/etl/transform/block_transform.rb +13 -0
- data/lib/etl/transform/date_to_string_transform.rb +20 -0
- data/lib/etl/transform/decode_transform.rb +51 -0
- data/lib/etl/transform/default_transform.rb +20 -0
- data/lib/etl/transform/foreign_key_lookup_transform.rb +151 -0
- data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
- data/lib/etl/transform/ordinalize_transform.rb +12 -0
- data/lib/etl/transform/sha1_transform.rb +13 -0
- data/lib/etl/transform/string_to_date_transform.rb +16 -0
- data/lib/etl/transform/string_to_datetime_transform.rb +14 -0
- data/lib/etl/transform/string_to_time_transform.rb +11 -0
- data/lib/etl/transform/transform.rb +61 -0
- data/lib/etl/transform/trim_transform.rb +26 -0
- data/lib/etl/transform/type_transform.rb +35 -0
- data/lib/etl/util.rb +59 -0
- data/lib/etl/version.rb +9 -0
- data/test/.ignore +2 -0
- data/test/all.ebf +6 -0
- data/test/apache_combined_log.ctl +11 -0
- data/test/batch_test.rb +41 -0
- data/test/batch_with_error.ebf +6 -0
- data/test/batched1.ctl +0 -0
- data/test/batched2.ctl +0 -0
- data/test/block_processor.ctl +6 -0
- data/test/block_processor_error.ctl +1 -0
- data/test/block_processor_pre_post_process.ctl +4 -0
- data/test/block_processor_remove_rows.ctl +5 -0
- data/test/block_processor_test.rb +38 -0
- data/test/connection/native_mysql/connection.rb +9 -0
- data/test/connection/native_mysql/schema.sql +36 -0
- data/test/connection/postgresql/connection.rb +13 -0
- data/test/connection/postgresql/schema.sql +43 -0
- data/test/control_test.rb +43 -0
- data/test/data/apache_combined_log.txt +3 -0
- data/test/data/bulk_import.txt +3 -0
- data/test/data/bulk_import_with_empties.txt +3 -0
- data/test/data/decode.txt +3 -0
- data/test/data/delimited.txt +3 -0
- data/test/data/encode_source_latin1.txt +2 -0
- data/test/data/fixed_width.txt +3 -0
- data/test/data/multiple_delimited_1.txt +3 -0
- data/test/data/multiple_delimited_2.txt +3 -0
- data/test/data/people.txt +3 -0
- data/test/data/sax.xml +14 -0
- data/test/data/xml.xml +16 -0
- data/test/database.example.yml +18 -0
- data/test/database.mysql.yml +18 -0
- data/test/database.postgres.yml +18 -0
- data/test/database.yml +18 -0
- data/test/date_dimension_builder_test.rb +96 -0
- data/test/delimited.ctl +30 -0
- data/test/delimited_absolute.ctl +33 -0
- data/test/delimited_destination_db.ctl +25 -0
- data/test/delimited_with_bulk_load.ctl +34 -0
- data/test/destination_test.rb +171 -0
- data/test/directive_test.rb +23 -0
- data/test/encode_processor_test.rb +31 -0
- data/test/engine_test.rb +32 -0
- data/test/errors.ctl +24 -0
- data/test/etl_test.rb +42 -0
- data/test/fixed_width.ctl +35 -0
- data/test/generator_test.rb +14 -0
- data/test/inline_parser.ctl +17 -0
- data/test/mocks/mock_destination.rb +26 -0
- data/test/mocks/mock_source.rb +25 -0
- data/test/model_source.ctl +14 -0
- data/test/multiple_delimited.ctl +22 -0
- data/test/multiple_source_delimited.ctl +39 -0
- data/test/parser_test.rb +200 -0
- data/test/performance/delimited.ctl +30 -0
- data/test/processor_test.rb +38 -0
- data/test/row_processor_test.rb +17 -0
- data/test/sax.ctl +26 -0
- data/test/scd/1.txt +1 -0
- data/test/scd/2.txt +1 -0
- data/test/scd/3.txt +1 -0
- data/test/scd_test.rb +263 -0
- data/test/scd_test_type_1.ctl +43 -0
- data/test/scd_test_type_2.ctl +42 -0
- data/test/screen_test.rb +9 -0
- data/test/screen_test_error.ctl +3 -0
- data/test/screen_test_fatal.ctl +3 -0
- data/test/source_test.rb +139 -0
- data/test/test_helper.rb +33 -0
- data/test/transform_test.rb +101 -0
- data/test/xml.ctl +31 -0
- metadata +237 -0
@@ -0,0 +1,43 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class ControlTest < Test::Unit::TestCase
|
4
|
+
# Test the ability to parse control files.
|
5
|
+
def test_parse
|
6
|
+
assert_nothing_raised do
|
7
|
+
Dir.glob('*.ctl') do |f|
|
8
|
+
ETL::Control::Control.parse(File.join(File.dirname(__FILE__), f))
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_bad_control_raises_error
|
14
|
+
assert_raise ETL::ControlError do
|
15
|
+
ETL::Control::Control.resolve(0)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_resolve_control_object
|
20
|
+
assert_nothing_raised do
|
21
|
+
ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_set_error_threshold
|
26
|
+
assert_nothing_raised do
|
27
|
+
ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_bad_processor_name
|
32
|
+
assert_raise ETL::ControlError do
|
33
|
+
s = "before_write :chunky_monkey"
|
34
|
+
ETL::Control::Control.parse_text(s)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_dependencies
|
39
|
+
s = "depends_on 'foo', 'bar'"
|
40
|
+
control = ETL::Control::Control.parse_text(s)
|
41
|
+
assert_equal control.dependencies, ['foo','bar']
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,3 @@
|
|
1
|
+
127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
2
|
+
127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
3
|
+
127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
data/test/data/sax.xml
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
|
3
|
+
<people>
|
4
|
+
<person age="24">
|
5
|
+
<first_name>Bob</first_name>
|
6
|
+
<last_name>Smith</last_name>
|
7
|
+
<social_security_number>123456789</social_security_number>
|
8
|
+
</person>
|
9
|
+
<person age="31">
|
10
|
+
<first_name>John</first_name>
|
11
|
+
<last_name>Doe</last_name>
|
12
|
+
<social_security_number>222114545</social_security_number>
|
13
|
+
</person>
|
14
|
+
</people>
|
data/test/data/xml.xml
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
|
3
|
+
<people>
|
4
|
+
<person>
|
5
|
+
<first_name>Bob</first_name>
|
6
|
+
<last_name>Smith</last_name>
|
7
|
+
<social_security_number>123456789</social_security_number>
|
8
|
+
<age>24</age>
|
9
|
+
</person>
|
10
|
+
<person>
|
11
|
+
<first_name>John</first_name>
|
12
|
+
<last_name>Doe</last_name>
|
13
|
+
<social_security_number>222114545</social_security_number>
|
14
|
+
<age>31</age>
|
15
|
+
</person>
|
16
|
+
</people>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
operational_database:
|
2
|
+
adapter: mysql
|
3
|
+
username: root
|
4
|
+
host: localhost
|
5
|
+
database: etl_unittest
|
6
|
+
encoding: utf8
|
7
|
+
data_warehouse:
|
8
|
+
adapter: mysql
|
9
|
+
username: root
|
10
|
+
host: localhost
|
11
|
+
database: etl_unittest
|
12
|
+
encoding: utf8
|
13
|
+
etl_execution:
|
14
|
+
adapter: mysql
|
15
|
+
username: root
|
16
|
+
host: localhost
|
17
|
+
database: etl_unittest_execution
|
18
|
+
encoding: utf8
|
@@ -0,0 +1,18 @@
|
|
1
|
+
operational_database:
|
2
|
+
adapter: mysql
|
3
|
+
username: root
|
4
|
+
host: localhost
|
5
|
+
database: etl_unittest
|
6
|
+
encoding: utf8
|
7
|
+
data_warehouse:
|
8
|
+
adapter: mysql
|
9
|
+
username: root
|
10
|
+
host: localhost
|
11
|
+
database: etl_unittest
|
12
|
+
encoding: utf8
|
13
|
+
etl_execution:
|
14
|
+
adapter: mysql
|
15
|
+
username: root
|
16
|
+
host: localhost
|
17
|
+
database: etl_unittest_execution
|
18
|
+
encoding: utf8
|
@@ -0,0 +1,18 @@
|
|
1
|
+
operational_database:
|
2
|
+
adapter: postgresql
|
3
|
+
username: postgres
|
4
|
+
host: localhost
|
5
|
+
database: etl_unittest
|
6
|
+
encoding: utf8
|
7
|
+
data_warehouse:
|
8
|
+
adapter: postgresql
|
9
|
+
username: postgres
|
10
|
+
host: localhost
|
11
|
+
database: etl_unittest
|
12
|
+
encoding: utf8
|
13
|
+
etl_execution:
|
14
|
+
adapter: postgresql
|
15
|
+
username: postgres
|
16
|
+
host: localhost
|
17
|
+
database: etl_unittest_execution
|
18
|
+
encoding: utf8
|
data/test/database.yml
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
operational_database:
|
2
|
+
adapter: postgresql
|
3
|
+
username: postgres
|
4
|
+
host: localhost
|
5
|
+
database: etl_unittest
|
6
|
+
encoding: utf8
|
7
|
+
data_warehouse:
|
8
|
+
adapter: postgresql
|
9
|
+
username: postgres
|
10
|
+
host: localhost
|
11
|
+
database: etl_unittest
|
12
|
+
encoding: utf8
|
13
|
+
etl_execution:
|
14
|
+
adapter: postgresql
|
15
|
+
username: postgres
|
16
|
+
host: localhost
|
17
|
+
database: etl_unittest_execution
|
18
|
+
encoding: utf8
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class DateDimensionBuilderTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
context "the DateDimensionBuilder" do
|
6
|
+
context "when initialized with defaults" do
|
7
|
+
setup do
|
8
|
+
@builder = ETL::Builder::DateDimensionBuilder.new
|
9
|
+
end
|
10
|
+
should "have a start date of 5 years ago" do
|
11
|
+
assert_equal Time.now.years_ago(5).to_date, @builder.start_date.to_date
|
12
|
+
end
|
13
|
+
should "have an end date of now" do
|
14
|
+
assert_equal Time.now.to_date, @builder.end_date.to_date
|
15
|
+
end
|
16
|
+
should "have an empty of array of holiday indicators" do
|
17
|
+
assert_equal [], @builder.holiday_indicators
|
18
|
+
end
|
19
|
+
end
|
20
|
+
context "when initialized with arguments" do
|
21
|
+
setup do
|
22
|
+
@start_date = Time.now.years_ago(2)
|
23
|
+
@end_date = Time.now.years_ago(1)
|
24
|
+
@builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
|
25
|
+
end
|
26
|
+
should "respect a custom start date" do
|
27
|
+
assert_equal @start_date.to_date, @builder.start_date.to_date
|
28
|
+
end
|
29
|
+
should "respect a custom end date" do
|
30
|
+
assert_equal @end_date.to_date, @builder.end_date.to_date
|
31
|
+
end
|
32
|
+
end
|
33
|
+
context "when building a date dimension using the default settings" do
|
34
|
+
setup do
|
35
|
+
# specific dates required when testing, because leap years affect
|
36
|
+
# how many records are built
|
37
|
+
@start_date = Date.parse('2002-05-19').to_time
|
38
|
+
@end_date = Date.parse('2007-05-19').to_time
|
39
|
+
@builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
|
40
|
+
@records = @builder.build
|
41
|
+
end
|
42
|
+
should "build a dimension with the correct number of records" do
|
43
|
+
assert_equal 1827, @records.length
|
44
|
+
end
|
45
|
+
should "have the correct first date" do
|
46
|
+
assert_date_dimension_record_equal(@builder.start_date, @records.first)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
context "when building a date dimension with a fiscal year offset month" do
|
50
|
+
should_eventually "respect the fiscal year offset month" do
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def assert_date_dimension_record_equal(date, record)
|
57
|
+
real_date = date
|
58
|
+
date = date.to_time
|
59
|
+
assert_equal date.strftime("%m/%d/%Y"), record[:date]
|
60
|
+
assert_equal date.strftime("%B %d,%Y"), record[:full_date_description]
|
61
|
+
assert_equal date.strftime("%A"), record[:day_of_week]
|
62
|
+
assert_equal date.day, record[:day_number_in_calendar_month]
|
63
|
+
assert_equal date.yday, record[:day_number_in_calendar_year]
|
64
|
+
assert_equal date.day, record[:day_number_in_fiscal_month]
|
65
|
+
assert_equal date.fiscal_year_yday, record[:day_number_in_fiscal_year]
|
66
|
+
assert_equal "Week #{date.week}", record[:calendar_week]
|
67
|
+
assert_equal date.week, record[:calendar_week_number_in_year]
|
68
|
+
assert_equal date.strftime("%B"), record[:calendar_month_name]
|
69
|
+
assert_equal date.month, record[:calendar_month_number_in_year]
|
70
|
+
assert_equal date.strftime("%Y-%m"), record[:calendar_year_month]
|
71
|
+
assert_equal "Q#{date.quarter}", record[:calendar_quarter]
|
72
|
+
assert_equal date.quarter, record[:calendar_quarter_number_in_year]
|
73
|
+
assert_equal "#{date.strftime('%Y')}-#{record[:calendar_quarter]}", record[:calendar_year_quarter]
|
74
|
+
assert_equal "#{date.year}", record[:calendar_year]
|
75
|
+
assert_equal "FY Week #{date.fiscal_year_week}", record[:fiscal_week]
|
76
|
+
assert_equal date.fiscal_year_week, record[:fiscal_week_number_in_year]
|
77
|
+
assert_equal date.fiscal_year_month, record[:fiscal_month]
|
78
|
+
assert_equal date.fiscal_year_month, record[:fiscal_month_number_in_year]
|
79
|
+
assert_equal "FY#{date.fiscal_year}-" + date.fiscal_year_month.to_s.rjust(2, '0'), record[:fiscal_year_month]
|
80
|
+
assert_equal "FY Q#{date.fiscal_year_quarter}", record[:fiscal_quarter]
|
81
|
+
assert_equal "FY#{date.fiscal_year}-Q#{date.fiscal_year_quarter}", record[:fiscal_year_quarter]
|
82
|
+
assert_equal date.fiscal_year_quarter, record[:fiscal_year_quarter_number]
|
83
|
+
assert_equal "FY#{date.fiscal_year}", record[:fiscal_year]
|
84
|
+
assert_equal date.fiscal_year, record[:fiscal_year_number]
|
85
|
+
assert_equal 'Nonholiday', record[:holiday_indicator]
|
86
|
+
assert_equal weekday_indicators[date.wday], record[:weekday_indicator]
|
87
|
+
assert_equal 'None', record[:selling_season]
|
88
|
+
assert_equal 'None', record[:major_event]
|
89
|
+
assert_equal record[:sql_date_stamp], real_date
|
90
|
+
end
|
91
|
+
|
92
|
+
private
|
93
|
+
def weekday_indicators
|
94
|
+
['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
|
95
|
+
end
|
96
|
+
end
|
data/test/delimited.ctl
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
source :in, {
|
2
|
+
:file => 'data/delimited.txt',
|
3
|
+
:parser => {
|
4
|
+
:name => :delimited
|
5
|
+
}
|
6
|
+
},
|
7
|
+
[
|
8
|
+
:first_name,
|
9
|
+
:last_name,
|
10
|
+
:ssn,
|
11
|
+
:age,
|
12
|
+
:sex
|
13
|
+
]
|
14
|
+
|
15
|
+
#transform :age, :type, :type => :number
|
16
|
+
transform :ssn, :sha1
|
17
|
+
transform(:ssn){ |n, v, row| v[0,24] }
|
18
|
+
transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
|
19
|
+
|
20
|
+
destination :out, {
|
21
|
+
:file => 'output/delimited.txt'
|
22
|
+
},
|
23
|
+
{
|
24
|
+
:order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
|
25
|
+
:virtual => {
|
26
|
+
:id => :surrogate_key,
|
27
|
+
:test => "test!",
|
28
|
+
:calc_test => Time.now
|
29
|
+
},
|
30
|
+
}
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# puts "executing delimited.ctl"
|
2
|
+
|
3
|
+
source :in, {
|
4
|
+
:file => '/tmp/delimited_abs.txt',
|
5
|
+
:parser => {
|
6
|
+
:name => :delimited
|
7
|
+
}
|
8
|
+
},
|
9
|
+
[
|
10
|
+
:first_name,
|
11
|
+
:last_name,
|
12
|
+
:ssn,
|
13
|
+
{
|
14
|
+
:name => :age,
|
15
|
+
:type => :integer
|
16
|
+
},
|
17
|
+
:sex
|
18
|
+
]
|
19
|
+
|
20
|
+
transform :ssn, :sha1
|
21
|
+
transform(:ssn){ |n, v, row| v[0,24] }
|
22
|
+
transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
|
23
|
+
|
24
|
+
destination :out, {
|
25
|
+
:file => 'data/delimited_abs.txt'
|
26
|
+
},
|
27
|
+
{
|
28
|
+
:order => [:first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
|
29
|
+
:virtual => {
|
30
|
+
:test => "test!",
|
31
|
+
:calc_test => Time.now
|
32
|
+
}
|
33
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# puts "executing delimited.ctl"
|
2
|
+
|
3
|
+
source :in, {
|
4
|
+
:file => 'data/delimited.txt',
|
5
|
+
:parser => :delimited
|
6
|
+
},
|
7
|
+
[
|
8
|
+
:id,
|
9
|
+
:first_name,
|
10
|
+
:last_name,
|
11
|
+
:ssn
|
12
|
+
]
|
13
|
+
|
14
|
+
transform :ssn, :sha1
|
15
|
+
transform(:ssn){ |v| v[0,24] }
|
16
|
+
|
17
|
+
destination :out, {
|
18
|
+
:type => :database,
|
19
|
+
:target => :data_warehouse,
|
20
|
+
:database => 'etl_unittest',
|
21
|
+
:table => 'people',
|
22
|
+
},
|
23
|
+
{
|
24
|
+
:order => [:id, :first_name, :last_name, :ssn]
|
25
|
+
}
|
@@ -0,0 +1,34 @@
|
|
1
|
+
infile = 'data/people.txt'
|
2
|
+
outfile = 'output/people.txt'
|
3
|
+
|
4
|
+
source :in, {
|
5
|
+
:file => infile,
|
6
|
+
:parser => {
|
7
|
+
:name => :delimited
|
8
|
+
}
|
9
|
+
},
|
10
|
+
[
|
11
|
+
:first_name,
|
12
|
+
:last_name,
|
13
|
+
]
|
14
|
+
|
15
|
+
before_write :surrogate_key, :target => :data_warehouse, :table => 'person_dimension', :column => 'id'
|
16
|
+
before_write :check_exists, {
|
17
|
+
:target => :data_warehouse,
|
18
|
+
:table => 'person_dimension',
|
19
|
+
:columns => [:first_name, :last_name]
|
20
|
+
}
|
21
|
+
|
22
|
+
destination :out, {
|
23
|
+
:file => outfile
|
24
|
+
},
|
25
|
+
{
|
26
|
+
:order => [:id, :first_name, :last_name]
|
27
|
+
}
|
28
|
+
|
29
|
+
post_process :bulk_import, {
|
30
|
+
:file => outfile,
|
31
|
+
:target => :data_warehouse,
|
32
|
+
:table => 'person_dimension',
|
33
|
+
:order => [:id, :first_name, :last_name]
|
34
|
+
}
|
@@ -0,0 +1,171 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class Person < ActiveRecord::Base
|
4
|
+
end
|
5
|
+
|
6
|
+
class BadDestination < ETL::Control::Destination
|
7
|
+
def initialize(control, configuration, mapping)
|
8
|
+
super
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
# Test the functionality of destinations
|
13
|
+
class DestinationTest < Test::Unit::TestCase
|
14
|
+
# Test a file destination
|
15
|
+
def test_file_destination
|
16
|
+
outfile = 'output/test_file_destination.txt'
|
17
|
+
outfile2 = 'output/test_file_destination.2.txt'
|
18
|
+
row = ETL::Row[ :address => '123 SW 1st Street', :city => 'Melbourne',
|
19
|
+
:state => 'Florida', :country => 'United States' ]
|
20
|
+
row_needs_escape = ETL::Row[ :address => "Allen's Way",
|
21
|
+
:city => 'Some City', :state => 'Some State', :country => 'Mexico' ]
|
22
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
23
|
+
'/delimited.ctl')
|
24
|
+
|
25
|
+
# First define a basic configuration to check defaults
|
26
|
+
configuration = {
|
27
|
+
:file => outfile,
|
28
|
+
:buffer_size => 0,
|
29
|
+
}
|
30
|
+
mapping = {
|
31
|
+
:order => [:address, :city, :state, :country, :country_code],
|
32
|
+
:virtual => {
|
33
|
+
:country_code => Proc.new do |r|
|
34
|
+
{
|
35
|
+
'United States' => 'US',
|
36
|
+
'Mexico' => 'MX'
|
37
|
+
}[r[:country]]
|
38
|
+
end
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
dest = ETL::Control::FileDestination.new(control, configuration, mapping)
|
43
|
+
dest.write(row)
|
44
|
+
dest.write(row_needs_escape)
|
45
|
+
dest.close
|
46
|
+
|
47
|
+
configuration[:file] = outfile2
|
48
|
+
configuration[:separator] = '|'
|
49
|
+
configuration[:eol] = "[EOL]\n"
|
50
|
+
|
51
|
+
dest = ETL::Control::FileDestination.new(control, configuration, mapping)
|
52
|
+
dest.write(row)
|
53
|
+
dest.write(row_needs_escape)
|
54
|
+
dest.close
|
55
|
+
|
56
|
+
# Read back the resulting
|
57
|
+
lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
|
58
|
+
assert_equal "123 SW 1st Street,Melbourne,Florida,United States,US\n", lines[0]
|
59
|
+
assert_equal "Allen's Way,Some City,Some State,Mexico,MX\n", lines[1]
|
60
|
+
|
61
|
+
lines = open(File.join(File.dirname(__FILE__), outfile2), 'r').readlines
|
62
|
+
assert_equal "123 SW 1st Street|Melbourne|Florida|United States|US[EOL]\n", lines[0]
|
63
|
+
assert_equal "Allen's Way|Some City|Some State|Mexico|MX[EOL]\n", lines[1]
|
64
|
+
end
|
65
|
+
|
66
|
+
# Test a database destination
|
67
|
+
def test_database_destination
|
68
|
+
row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
69
|
+
row_needs_escape = ETL::Row[:id => 2, :first_name => "Foo's", :last_name => "Bar", :ssn => '000000000' ]
|
70
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
71
|
+
'/delimited.ctl')
|
72
|
+
|
73
|
+
Person.delete_all
|
74
|
+
assert_equal 0, Person.count
|
75
|
+
|
76
|
+
# First define a basic configuration to check defaults
|
77
|
+
configuration = {
|
78
|
+
:target => :data_warehouse,
|
79
|
+
:database => 'etl_unittest',
|
80
|
+
:table => 'people',
|
81
|
+
:buffer_size => 0
|
82
|
+
}
|
83
|
+
mapping = { :order => [:id, :first_name, :last_name, :ssn] }
|
84
|
+
dest = ETL::Control::DatabaseDestination.new(control, configuration, mapping)
|
85
|
+
dest.write(row)
|
86
|
+
dest.close
|
87
|
+
|
88
|
+
assert_equal 1, Person.find(:all).length
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_database_destination_with_control
|
92
|
+
row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
93
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) +
|
94
|
+
'/delimited_destination_db.ctl')
|
95
|
+
Person.delete_all
|
96
|
+
assert_equal 0, Person.count
|
97
|
+
d = control.destinations.first
|
98
|
+
dest = ETL::Control::DatabaseDestination.new(control, d.configuration, d.mapping)
|
99
|
+
dest.write(row)
|
100
|
+
dest.close
|
101
|
+
assert_equal 1, Person.count
|
102
|
+
end
|
103
|
+
|
104
|
+
def test_unique
|
105
|
+
row1 = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
106
|
+
row2 = ETL::Row[:id => 2, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
107
|
+
row3 = ETL::Row[:id => 3, :first_name => 'John', :last_name => 'Smith', :ssn => '000112222']
|
108
|
+
|
109
|
+
outfile = 'output/test_unique.txt'
|
110
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
111
|
+
|
112
|
+
# First define a basic configuration to check defaults
|
113
|
+
configuration = { :file => outfile, :buffer_size => 0, :unique => [:ssn]}
|
114
|
+
mapping = {
|
115
|
+
:order => [:first_name, :last_name, :ssn]
|
116
|
+
}
|
117
|
+
dest = ETL::Control::FileDestination.new(control, configuration, mapping)
|
118
|
+
dest.write(row1)
|
119
|
+
dest.write(row2)
|
120
|
+
dest.write(row3)
|
121
|
+
|
122
|
+
# Close (and flush) the destination
|
123
|
+
dest.close
|
124
|
+
|
125
|
+
# Read back the resulting
|
126
|
+
lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
|
127
|
+
assert_equal "Bob,Smith,111234444\n", lines[0]
|
128
|
+
assert_equal "John,Smith,000112222\n", lines[1]
|
129
|
+
end
|
130
|
+
|
131
|
+
def test_multiple_unique
|
132
|
+
row1 = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
133
|
+
row2 = ETL::Row[:id => 2, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
|
134
|
+
row3 = ETL::Row[:id => 3, :first_name => 'Bob', :last_name => 'Smith', :ssn => '000112222']
|
135
|
+
|
136
|
+
outfile = 'output/test_multiple_unique.txt'
|
137
|
+
control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
|
138
|
+
|
139
|
+
# First define a basic configuration to check defaults
|
140
|
+
configuration = { :file => outfile, :buffer_size => 0, :unique => [:last_name,:first_name]}
|
141
|
+
mapping = {
|
142
|
+
:order => [:first_name, :last_name, :ssn]
|
143
|
+
}
|
144
|
+
dest = ETL::Control::FileDestination.new(control, configuration, mapping)
|
145
|
+
dest.write(row1)
|
146
|
+
dest.write(row2)
|
147
|
+
dest.write(row3)
|
148
|
+
|
149
|
+
# Close (and flush) the destination
|
150
|
+
dest.close
|
151
|
+
|
152
|
+
# Read back the resulting
|
153
|
+
lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
|
154
|
+
assert_equal "Bob,Smith,111234444\n", lines[0]
|
155
|
+
assert_equal 1, lines.length
|
156
|
+
end
|
157
|
+
|
158
|
+
def test_bad_destination
|
159
|
+
control = ETL::Control::Control.parse_text('')
|
160
|
+
configuration = {}
|
161
|
+
mapping = {}
|
162
|
+
dest = BadDestination.new(control, configuration, mapping)
|
163
|
+
dest.write(nil)
|
164
|
+
assert_raise NotImplementedError do
|
165
|
+
dest.flush
|
166
|
+
end
|
167
|
+
assert_raise NotImplementedError do
|
168
|
+
dest.close
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|