activewarehouse-etl-sgonyea 0.9.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +9 -0
- data/0.9-UPGRADE +6 -0
- data/CHANGELOG +236 -0
- data/Gemfile +4 -0
- data/HOW_TO_RELEASE +13 -0
- data/LICENSE +7 -0
- data/README.textile +111 -0
- data/Rakefile +103 -0
- data/TODO +28 -0
- data/active_support_logger.patch +78 -0
- data/activewarehouse-etl.gemspec +36 -0
- data/bin/etl +28 -0
- data/bin/etl.cmd +8 -0
- data/examples/database.example.yml +16 -0
- data/lib/etl.rb +97 -0
- data/lib/etl/batch.rb +2 -0
- data/lib/etl/batch/batch.rb +111 -0
- data/lib/etl/batch/directives.rb +65 -0
- data/lib/etl/builder.rb +2 -0
- data/lib/etl/builder/date_dimension_builder.rb +96 -0
- data/lib/etl/builder/time_dimension_builder.rb +31 -0
- data/lib/etl/commands/etl.rb +89 -0
- data/lib/etl/control.rb +3 -0
- data/lib/etl/control/control.rb +405 -0
- data/lib/etl/control/destination.rb +438 -0
- data/lib/etl/control/destination/csv_destination.rb +113 -0
- data/lib/etl/control/destination/database_destination.rb +97 -0
- data/lib/etl/control/destination/excel_destination.rb +91 -0
- data/lib/etl/control/destination/file_destination.rb +126 -0
- data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
- data/lib/etl/control/destination/update_database_destination.rb +109 -0
- data/lib/etl/control/destination/yaml_destination.rb +74 -0
- data/lib/etl/control/source.rb +132 -0
- data/lib/etl/control/source/database_source.rb +224 -0
- data/lib/etl/control/source/enumerable_source.rb +11 -0
- data/lib/etl/control/source/file_source.rb +90 -0
- data/lib/etl/control/source/model_source.rb +39 -0
- data/lib/etl/core_ext.rb +1 -0
- data/lib/etl/core_ext/time.rb +5 -0
- data/lib/etl/core_ext/time/calculations.rb +42 -0
- data/lib/etl/engine.rb +582 -0
- data/lib/etl/execution.rb +19 -0
- data/lib/etl/execution/base.rb +8 -0
- data/lib/etl/execution/batch.rb +10 -0
- data/lib/etl/execution/job.rb +8 -0
- data/lib/etl/execution/migration.rb +90 -0
- data/lib/etl/generator.rb +2 -0
- data/lib/etl/generator/generator.rb +20 -0
- data/lib/etl/generator/surrogate_key_generator.rb +39 -0
- data/lib/etl/http_tools.rb +139 -0
- data/lib/etl/parser.rb +11 -0
- data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
- data/lib/etl/parser/csv_parser.rb +93 -0
- data/lib/etl/parser/excel_parser.rb +112 -0
- data/lib/etl/parser/fixed_width_parser.rb +65 -0
- data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
- data/lib/etl/parser/parser.rb +41 -0
- data/lib/etl/parser/sax_parser.rb +218 -0
- data/lib/etl/parser/xml_parser.rb +65 -0
- data/lib/etl/processor.rb +11 -0
- data/lib/etl/processor/block_processor.rb +14 -0
- data/lib/etl/processor/bulk_import_processor.rb +94 -0
- data/lib/etl/processor/check_exist_processor.rb +80 -0
- data/lib/etl/processor/check_unique_processor.rb +39 -0
- data/lib/etl/processor/copy_field_processor.rb +26 -0
- data/lib/etl/processor/database_join_processor.rb +82 -0
- data/lib/etl/processor/encode_processor.rb +55 -0
- data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
- data/lib/etl/processor/escape_csv_processor.rb +77 -0
- data/lib/etl/processor/filter_row_processor.rb +51 -0
- data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
- data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
- data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
- data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
- data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
- data/lib/etl/processor/print_row_processor.rb +12 -0
- data/lib/etl/processor/processor.rb +25 -0
- data/lib/etl/processor/rename_processor.rb +24 -0
- data/lib/etl/processor/require_non_blank_processor.rb +26 -0
- data/lib/etl/processor/row_processor.rb +27 -0
- data/lib/etl/processor/sequence_processor.rb +23 -0
- data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
- data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
- data/lib/etl/processor/surrogate_key_processor.rb +53 -0
- data/lib/etl/processor/truncate_processor.rb +40 -0
- data/lib/etl/processor/zip_file_processor.rb +27 -0
- data/lib/etl/row.rb +20 -0
- data/lib/etl/screen.rb +14 -0
- data/lib/etl/screen/row_count_screen.rb +20 -0
- data/lib/etl/transform.rb +2 -0
- data/lib/etl/transform/block_transform.rb +13 -0
- data/lib/etl/transform/calculation_transform.rb +71 -0
- data/lib/etl/transform/date_to_string_transform.rb +20 -0
- data/lib/etl/transform/decode_transform.rb +51 -0
- data/lib/etl/transform/default_transform.rb +20 -0
- data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
- data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
- data/lib/etl/transform/md5_transform.rb +13 -0
- data/lib/etl/transform/ordinalize_transform.rb +14 -0
- data/lib/etl/transform/sha1_transform.rb +13 -0
- data/lib/etl/transform/split_fields_transform.rb +27 -0
- data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
- data/lib/etl/transform/string_to_date_transform.rb +16 -0
- data/lib/etl/transform/string_to_time_transform.rb +11 -0
- data/lib/etl/transform/transform.rb +61 -0
- data/lib/etl/transform/trim_transform.rb +26 -0
- data/lib/etl/transform/type_transform.rb +35 -0
- data/lib/etl/util.rb +59 -0
- data/lib/etl/version.rb +3 -0
- data/test-matrix.yml +10 -0
- data/test/.gitignore +1 -0
- data/test/.ignore +2 -0
- data/test/all.ebf +6 -0
- data/test/apache_combined_log.ctl +11 -0
- data/test/batch_test.rb +41 -0
- data/test/batch_with_error.ebf +6 -0
- data/test/batched1.ctl +0 -0
- data/test/batched2.ctl +0 -0
- data/test/block_processor.ctl +6 -0
- data/test/block_processor_error.ctl +1 -0
- data/test/block_processor_pre_post_process.ctl +4 -0
- data/test/block_processor_remove_rows.ctl +5 -0
- data/test/block_processor_test.rb +38 -0
- data/test/check_exist_processor_test.rb +92 -0
- data/test/check_unique_processor_test.rb +40 -0
- data/test/config/Gemfile.rails-2.3.x +3 -0
- data/test/config/Gemfile.rails-2.3.x.lock +53 -0
- data/test/config/Gemfile.rails-3.0.x +3 -0
- data/test/config/Gemfile.rails-3.0.x.lock +61 -0
- data/test/config/common.rb +29 -0
- data/test/connection/mysql/connection.rb +9 -0
- data/test/connection/mysql/schema.sql +37 -0
- data/test/connection/postgresql/connection.rb +13 -0
- data/test/connection/postgresql/schema.sql +40 -0
- data/test/control_test.rb +43 -0
- data/test/data/apache_combined_log.txt +3 -0
- data/test/data/bulk_import.txt +3 -0
- data/test/data/bulk_import_with_empties.txt +3 -0
- data/test/data/decode.txt +3 -0
- data/test/data/delimited.txt +3 -0
- data/test/data/encode_source_latin1.txt +2 -0
- data/test/data/excel.xls +0 -0
- data/test/data/excel2.xls +0 -0
- data/test/data/fixed_width.txt +3 -0
- data/test/data/multiple_delimited_1.txt +3 -0
- data/test/data/multiple_delimited_2.txt +3 -0
- data/test/data/nokogiri.xml +38 -0
- data/test/data/people.txt +3 -0
- data/test/data/sax.xml +14 -0
- data/test/data/xml.xml +16 -0
- data/test/database_join_processor_test.rb +43 -0
- data/test/date_dimension_builder_test.rb +96 -0
- data/test/delimited.ctl +30 -0
- data/test/delimited_absolute.ctl +31 -0
- data/test/delimited_destination_db.ctl +23 -0
- data/test/delimited_excel.ctl +31 -0
- data/test/delimited_insert_update.ctl +34 -0
- data/test/delimited_update.ctl +34 -0
- data/test/delimited_with_bulk_load.ctl +34 -0
- data/test/destination_test.rb +275 -0
- data/test/directive_test.rb +23 -0
- data/test/encode_processor_test.rb +32 -0
- data/test/engine_test.rb +78 -0
- data/test/ensure_fields_presence_processor_test.rb +28 -0
- data/test/errors.ctl +24 -0
- data/test/etl_test.rb +42 -0
- data/test/excel.ctl +24 -0
- data/test/excel2.ctl +25 -0
- data/test/fixed_width.ctl +35 -0
- data/test/foreign_key_lookup_transform_test.rb +50 -0
- data/test/generator_test.rb +14 -0
- data/test/inline_parser.ctl +17 -0
- data/test/mocks/mock_destination.rb +26 -0
- data/test/mocks/mock_source.rb +25 -0
- data/test/model_source.ctl +14 -0
- data/test/multiple_delimited.ctl +22 -0
- data/test/multiple_source_delimited.ctl +39 -0
- data/test/nokogiri_all.ctl +35 -0
- data/test/nokogiri_select.ctl +35 -0
- data/test/nokogiri_test.rb +35 -0
- data/test/parser_test.rb +224 -0
- data/test/performance/delimited.ctl +30 -0
- data/test/processor_test.rb +44 -0
- data/test/row_processor_test.rb +17 -0
- data/test/sax.ctl +26 -0
- data/test/scd/1.txt +1 -0
- data/test/scd/2.txt +1 -0
- data/test/scd/3.txt +1 -0
- data/test/scd_test.rb +257 -0
- data/test/scd_test_type_1.ctl +43 -0
- data/test/scd_test_type_2.ctl +34 -0
- data/test/screen_test.rb +9 -0
- data/test/screen_test_error.ctl +3 -0
- data/test/screen_test_fatal.ctl +3 -0
- data/test/source_test.rb +154 -0
- data/test/test_helper.rb +37 -0
- data/test/transform_test.rb +101 -0
- data/test/truncate_processor_test.rb +37 -0
- data/test/xml.ctl +31 -0
- metadata +370 -0
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class Person < ActiveRecord::Base
|
4
|
+
end
|
5
|
+
|
6
|
+
class CheckExistProcessorTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
context 'CheckExistProcessor' do
|
9
|
+
|
10
|
+
setup do
|
11
|
+
@config = {
|
12
|
+
:target => :data_warehouse,
|
13
|
+
:table => 'people',
|
14
|
+
:columns => [:first_name, :last_name]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
should_eventually "compare based on all columns if no columns are provided" do
|
19
|
+
# TBI
|
20
|
+
end
|
21
|
+
|
22
|
+
should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
|
23
|
+
# TBI
|
24
|
+
end
|
25
|
+
|
26
|
+
should "raise an error if no table is provided" do
|
27
|
+
error = assert_raises(ETL::ControlError) do
|
28
|
+
ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
|
29
|
+
end
|
30
|
+
# bug #2413 on assert_raises won't let me check error message above
|
31
|
+
assert_equal 'table must be specified', error.message
|
32
|
+
end
|
33
|
+
|
34
|
+
should "raise an error if no target is provided" do
|
35
|
+
error = assert_raises(ETL::ControlError) do
|
36
|
+
ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
|
37
|
+
end
|
38
|
+
|
39
|
+
assert_equal 'target must be specified', error.message
|
40
|
+
end
|
41
|
+
|
42
|
+
should "bypass checking if the table has no rows" do
|
43
|
+
Person.delete_all
|
44
|
+
|
45
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
46
|
+
assert_equal false, processor.should_check?
|
47
|
+
end
|
48
|
+
|
49
|
+
should "raise an error if one of the keys used for checking existence is not available in a row" do
|
50
|
+
Person.delete_all
|
51
|
+
# we need at least one record to avoid automatic skipping
|
52
|
+
# this should be mocked instead, probably
|
53
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
54
|
+
|
55
|
+
error = assert_raise(ETL::ControlError) do
|
56
|
+
row = ETL::Row[:first_name => 'John']
|
57
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
58
|
+
|
59
|
+
# guard against bypassing
|
60
|
+
assert_equal true, processor.should_check?
|
61
|
+
|
62
|
+
processor.process(row)
|
63
|
+
end
|
64
|
+
|
65
|
+
assert_equal "Row missing required field :last_name for existence check", error.message
|
66
|
+
end
|
67
|
+
|
68
|
+
should "return nil if the same row is found in database" do
|
69
|
+
Person.delete_all
|
70
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
71
|
+
|
72
|
+
row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
|
73
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
74
|
+
assert_equal true, processor.should_check? # guard against bypassing
|
75
|
+
|
76
|
+
assert_equal nil, processor.process(row)
|
77
|
+
end
|
78
|
+
|
79
|
+
should "return the row if no same row is found in database" do
|
80
|
+
Person.delete_all
|
81
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
82
|
+
|
83
|
+
row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
|
84
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
85
|
+
assert_equal true, processor.should_check? # guard against bypassing
|
86
|
+
|
87
|
+
assert_equal row, processor.process(row)
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class CheckUniqueProcessorTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
context 'CheckUniqueProcessor' do
|
6
|
+
attr_reader :processor
|
7
|
+
|
8
|
+
setup do
|
9
|
+
@processor = ETL::Processor::CheckUniqueProcessor.new(nil,
|
10
|
+
:keys => [:first, :second])
|
11
|
+
end
|
12
|
+
|
13
|
+
should "keep a row whose keys didn't already appear in the pipeline" do
|
14
|
+
row = ETL::Row[:first => 'A', :second => 'B']
|
15
|
+
|
16
|
+
assert_equal row, processor.process(row)
|
17
|
+
|
18
|
+
assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
|
19
|
+
end
|
20
|
+
|
21
|
+
should "remove a row whose keys already appeared in the pipeline" do
|
22
|
+
row = ETL::Row[:first => 'A', :second => 'B']
|
23
|
+
|
24
|
+
assert_equal row, processor.process(row)
|
25
|
+
assert_equal nil, processor.process(row)
|
26
|
+
end
|
27
|
+
|
28
|
+
should "raise an error if a row lacks one of the keys specified" do
|
29
|
+
row = ETL::Row[:first => 'A']
|
30
|
+
|
31
|
+
error = assert_raises(ETL::ControlError) do
|
32
|
+
processor.process(row)
|
33
|
+
end
|
34
|
+
|
35
|
+
assert_equal "Row missing required field :second for unicity check", error.message
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git@github.com:activewarehouse/adapter_extensions.git
|
3
|
+
revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
|
4
|
+
specs:
|
5
|
+
adapter_extensions (0.9.5.rc1)
|
6
|
+
activerecord (>= 2.1.0)
|
7
|
+
activesupport (>= 2.1.0)
|
8
|
+
rake (>= 0.8.3)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activerecord (2.3.11)
|
14
|
+
activesupport (= 2.3.11)
|
15
|
+
activesupport (2.3.11)
|
16
|
+
fastercsv (1.5.4)
|
17
|
+
flexmock (0.9.0)
|
18
|
+
mysql (2.8.1)
|
19
|
+
net-sftp (2.0.5)
|
20
|
+
net-ssh (>= 2.0.9)
|
21
|
+
net-ssh (2.1.4)
|
22
|
+
nokogiri (1.4.4)
|
23
|
+
nokogiri (1.4.4-java)
|
24
|
+
weakling (>= 0.0.3)
|
25
|
+
pg (0.11.0)
|
26
|
+
rdoc (3.6.1)
|
27
|
+
ruby-ole (1.2.11.1)
|
28
|
+
shoulda (2.11.3)
|
29
|
+
spreadsheet (0.6.5.4)
|
30
|
+
ruby-ole (>= 1.0)
|
31
|
+
tmail (1.2.7.1)
|
32
|
+
weakling (0.0.4-java)
|
33
|
+
zip (2.0.2)
|
34
|
+
|
35
|
+
PLATFORMS
|
36
|
+
java
|
37
|
+
ruby
|
38
|
+
|
39
|
+
DEPENDENCIES
|
40
|
+
activerecord (= 2.3.11)
|
41
|
+
activesupport (= 2.3.11)
|
42
|
+
adapter_extensions!
|
43
|
+
fastercsv (= 1.5.4)
|
44
|
+
flexmock (= 0.9.0)
|
45
|
+
mysql (= 2.8.1)
|
46
|
+
net-sftp (= 2.0.5)
|
47
|
+
nokogiri (= 1.4.4)
|
48
|
+
pg (= 0.11.0)
|
49
|
+
rdoc
|
50
|
+
shoulda (= 2.11.3)
|
51
|
+
spreadsheet (= 0.6.5.4)
|
52
|
+
tmail (= 1.2.7.1)
|
53
|
+
zip (= 2.0.2)
|
@@ -0,0 +1,61 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git@github.com:activewarehouse/adapter_extensions.git
|
3
|
+
revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
|
4
|
+
specs:
|
5
|
+
adapter_extensions (0.9.5.rc1)
|
6
|
+
activerecord (>= 2.1.0)
|
7
|
+
activesupport (>= 2.1.0)
|
8
|
+
rake (>= 0.8.3)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activemodel (3.0.7)
|
14
|
+
activesupport (= 3.0.7)
|
15
|
+
builder (~> 2.1.2)
|
16
|
+
i18n (~> 0.5.0)
|
17
|
+
activerecord (3.0.7)
|
18
|
+
activemodel (= 3.0.7)
|
19
|
+
activesupport (= 3.0.7)
|
20
|
+
arel (~> 2.0.2)
|
21
|
+
tzinfo (~> 0.3.23)
|
22
|
+
activesupport (3.0.7)
|
23
|
+
arel (2.0.10)
|
24
|
+
builder (2.1.2)
|
25
|
+
fastercsv (1.5.4)
|
26
|
+
flexmock (0.9.0)
|
27
|
+
i18n (0.5.0)
|
28
|
+
mysql (2.8.1)
|
29
|
+
net-sftp (2.0.5)
|
30
|
+
net-ssh (>= 2.0.9)
|
31
|
+
net-ssh (2.1.4)
|
32
|
+
nokogiri (1.4.4)
|
33
|
+
pg (0.11.0)
|
34
|
+
rdoc (3.6.1)
|
35
|
+
ruby-ole (1.2.11.1)
|
36
|
+
shoulda (2.11.3)
|
37
|
+
spreadsheet (0.6.5.4)
|
38
|
+
ruby-ole (>= 1.0)
|
39
|
+
tmail (1.2.7.1)
|
40
|
+
tzinfo (0.3.27)
|
41
|
+
zip (2.0.2)
|
42
|
+
|
43
|
+
PLATFORMS
|
44
|
+
java
|
45
|
+
ruby
|
46
|
+
|
47
|
+
DEPENDENCIES
|
48
|
+
activerecord (= 3.0.7)
|
49
|
+
activesupport (= 3.0.7)
|
50
|
+
adapter_extensions!
|
51
|
+
fastercsv (= 1.5.4)
|
52
|
+
flexmock (= 0.9.0)
|
53
|
+
mysql (= 2.8.1)
|
54
|
+
net-sftp (= 2.0.5)
|
55
|
+
nokogiri (= 1.4.4)
|
56
|
+
pg (= 0.11.0)
|
57
|
+
rdoc
|
58
|
+
shoulda (= 2.11.3)
|
59
|
+
spreadsheet (= 0.6.5.4)
|
60
|
+
tmail (= 1.2.7.1)
|
61
|
+
zip (= 2.0.2)
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rbconfig'
|
2
|
+
|
3
|
+
def common_gemfile(rails_version)
|
4
|
+
source :rubygems
|
5
|
+
|
6
|
+
# using explicit versions for the gems to avoid any weirdness later on
|
7
|
+
gem "activesupport", rails_version
|
8
|
+
gem "activerecord", rails_version
|
9
|
+
|
10
|
+
gem "fastercsv", "1.5.4"
|
11
|
+
gem "spreadsheet", "0.6.5.4"
|
12
|
+
gem "tmail", "1.2.7.1"
|
13
|
+
gem "net-sftp", "2.0.5"
|
14
|
+
gem "zip", "2.0.2"
|
15
|
+
|
16
|
+
gem "shoulda", "2.11.3"
|
17
|
+
gem "flexmock", "0.9.0"
|
18
|
+
|
19
|
+
gem "mysql", "2.8.1"
|
20
|
+
gem "pg", "0.11.0"
|
21
|
+
|
22
|
+
gem "nokogiri", "1.4.4"
|
23
|
+
|
24
|
+
gem "rdoc"
|
25
|
+
|
26
|
+
gem "adapter_extensions", :git => 'git@github.com:activewarehouse/adapter_extensions.git'
|
27
|
+
|
28
|
+
gem "jruby-openssl" if RUBY_PLATFORM == "java"
|
29
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
print "Using native MySQL\n"
|
2
|
+
|
3
|
+
puts "Resetting database"
|
4
|
+
conn = ETL::Engine.connection(:data_warehouse)
|
5
|
+
conn.recreate_database(conn.current_database)
|
6
|
+
conn.reconnect!
|
7
|
+
lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
|
8
|
+
lines.join.split(';').each { |line| conn.execute(line) }
|
9
|
+
conn.disconnect!
|
@@ -0,0 +1,37 @@
|
|
1
|
+
drop table if exists people;
|
2
|
+
create table people (
|
3
|
+
id int not null primary key,
|
4
|
+
first_name char(255) not null,
|
5
|
+
last_name char(255) not null,
|
6
|
+
ssn char(64) not null
|
7
|
+
);
|
8
|
+
drop table if exists places;
|
9
|
+
create table places (
|
10
|
+
address text,
|
11
|
+
city char(255),
|
12
|
+
state char(255),
|
13
|
+
country char(2)
|
14
|
+
);
|
15
|
+
|
16
|
+
drop table if exists person_dimension;
|
17
|
+
create table person_dimension (
|
18
|
+
id int not null primary key,
|
19
|
+
first_name char(50),
|
20
|
+
last_name char(50),
|
21
|
+
address char(100),
|
22
|
+
city char(50),
|
23
|
+
state char(50),
|
24
|
+
zip_code char(20),
|
25
|
+
effective_date datetime,
|
26
|
+
end_date datetime,
|
27
|
+
latest_version boolean not null
|
28
|
+
);
|
29
|
+
|
30
|
+
drop table if exists truncate_test;
|
31
|
+
create table truncate_test (
|
32
|
+
id int not null primary key auto_increment,
|
33
|
+
x char(4)
|
34
|
+
);
|
35
|
+
insert into truncate_test (x) values ('a');
|
36
|
+
insert into truncate_test (x) values ('b');
|
37
|
+
insert into truncate_test (x) values ('c');
|
@@ -0,0 +1,13 @@
|
|
1
|
+
print "Using PostgreSQL\n"
|
2
|
+
|
3
|
+
puts "Resetting database"
|
4
|
+
conn = ETL::Engine.connection(:data_warehouse)
|
5
|
+
|
6
|
+
lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
|
7
|
+
lines.join.split(';').each_with_index do |line, index|
|
8
|
+
begin
|
9
|
+
conn.execute(line)
|
10
|
+
rescue => e
|
11
|
+
puts "failed to load line #{index}: #{e}"
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
drop table people;
|
2
|
+
create table people (
|
3
|
+
id SERIAL PRIMARY KEY,
|
4
|
+
first_name character varying(255) not null,
|
5
|
+
/* null below allowed for bulk_import_with_empties.txt test */
|
6
|
+
last_name character varying(255) null,
|
7
|
+
ssn character varying(64) not null
|
8
|
+
);
|
9
|
+
|
10
|
+
drop table places;
|
11
|
+
create table places (
|
12
|
+
id SERIAL PRIMARY KEY,
|
13
|
+
address text,
|
14
|
+
city character varying(255),
|
15
|
+
state character varying(255),
|
16
|
+
country character varying(2)
|
17
|
+
);
|
18
|
+
|
19
|
+
drop table person_dimension;
|
20
|
+
create table person_dimension (
|
21
|
+
id SERIAL PRIMARY KEY,
|
22
|
+
first_name character varying(50),
|
23
|
+
last_name character varying(50),
|
24
|
+
address character varying(100),
|
25
|
+
city character varying(50),
|
26
|
+
state character varying(50),
|
27
|
+
zip_code character varying(20),
|
28
|
+
effective_date timestamp without time zone,
|
29
|
+
end_date timestamp without time zone,
|
30
|
+
latest_version boolean not null
|
31
|
+
);
|
32
|
+
|
33
|
+
drop table truncate_test;
|
34
|
+
create table truncate_test (
|
35
|
+
id SERIAL PRIMARY KEY,
|
36
|
+
x character varying(4)
|
37
|
+
);
|
38
|
+
insert into truncate_test (x) values ('a');
|
39
|
+
insert into truncate_test (x) values ('b');
|
40
|
+
insert into truncate_test (x) values ('c');
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class ControlTest < Test::Unit::TestCase
|
4
|
+
# Test the ability to parse control files.
|
5
|
+
def test_parse
|
6
|
+
assert_nothing_raised do
|
7
|
+
Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
|
8
|
+
ETL::Control::Control.parse(f)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_bad_control_raises_error
|
14
|
+
assert_raise ETL::ControlError do
|
15
|
+
ETL::Control::Control.resolve(0)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_resolve_control_object
|
20
|
+
assert_nothing_raised do
|
21
|
+
ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_set_error_threshold
|
26
|
+
assert_nothing_raised do
|
27
|
+
ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_bad_processor_name
|
32
|
+
assert_raise ETL::ControlError do
|
33
|
+
s = "before_write :chunky_monkey"
|
34
|
+
ETL::Control::Control.parse_text(s)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_dependencies
|
39
|
+
s = "depends_on 'foo', 'bar'"
|
40
|
+
control = ETL::Control::Control.parse_text(s)
|
41
|
+
assert_equal control.dependencies, ['foo','bar']
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,3 @@
|
|
1
|
+
127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
2
|
+
127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
3
|
+
127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|