activewarehouse-etl-sgonyea 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +9 -0
- data/0.9-UPGRADE +6 -0
- data/CHANGELOG +236 -0
- data/Gemfile +4 -0
- data/HOW_TO_RELEASE +13 -0
- data/LICENSE +7 -0
- data/README.textile +111 -0
- data/Rakefile +103 -0
- data/TODO +28 -0
- data/active_support_logger.patch +78 -0
- data/activewarehouse-etl.gemspec +36 -0
- data/bin/etl +28 -0
- data/bin/etl.cmd +8 -0
- data/examples/database.example.yml +16 -0
- data/lib/etl.rb +97 -0
- data/lib/etl/batch.rb +2 -0
- data/lib/etl/batch/batch.rb +111 -0
- data/lib/etl/batch/directives.rb +65 -0
- data/lib/etl/builder.rb +2 -0
- data/lib/etl/builder/date_dimension_builder.rb +96 -0
- data/lib/etl/builder/time_dimension_builder.rb +31 -0
- data/lib/etl/commands/etl.rb +89 -0
- data/lib/etl/control.rb +3 -0
- data/lib/etl/control/control.rb +405 -0
- data/lib/etl/control/destination.rb +438 -0
- data/lib/etl/control/destination/csv_destination.rb +113 -0
- data/lib/etl/control/destination/database_destination.rb +97 -0
- data/lib/etl/control/destination/excel_destination.rb +91 -0
- data/lib/etl/control/destination/file_destination.rb +126 -0
- data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
- data/lib/etl/control/destination/update_database_destination.rb +109 -0
- data/lib/etl/control/destination/yaml_destination.rb +74 -0
- data/lib/etl/control/source.rb +132 -0
- data/lib/etl/control/source/database_source.rb +224 -0
- data/lib/etl/control/source/enumerable_source.rb +11 -0
- data/lib/etl/control/source/file_source.rb +90 -0
- data/lib/etl/control/source/model_source.rb +39 -0
- data/lib/etl/core_ext.rb +1 -0
- data/lib/etl/core_ext/time.rb +5 -0
- data/lib/etl/core_ext/time/calculations.rb +42 -0
- data/lib/etl/engine.rb +582 -0
- data/lib/etl/execution.rb +19 -0
- data/lib/etl/execution/base.rb +8 -0
- data/lib/etl/execution/batch.rb +10 -0
- data/lib/etl/execution/job.rb +8 -0
- data/lib/etl/execution/migration.rb +90 -0
- data/lib/etl/generator.rb +2 -0
- data/lib/etl/generator/generator.rb +20 -0
- data/lib/etl/generator/surrogate_key_generator.rb +39 -0
- data/lib/etl/http_tools.rb +139 -0
- data/lib/etl/parser.rb +11 -0
- data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
- data/lib/etl/parser/csv_parser.rb +93 -0
- data/lib/etl/parser/excel_parser.rb +112 -0
- data/lib/etl/parser/fixed_width_parser.rb +65 -0
- data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
- data/lib/etl/parser/parser.rb +41 -0
- data/lib/etl/parser/sax_parser.rb +218 -0
- data/lib/etl/parser/xml_parser.rb +65 -0
- data/lib/etl/processor.rb +11 -0
- data/lib/etl/processor/block_processor.rb +14 -0
- data/lib/etl/processor/bulk_import_processor.rb +94 -0
- data/lib/etl/processor/check_exist_processor.rb +80 -0
- data/lib/etl/processor/check_unique_processor.rb +39 -0
- data/lib/etl/processor/copy_field_processor.rb +26 -0
- data/lib/etl/processor/database_join_processor.rb +82 -0
- data/lib/etl/processor/encode_processor.rb +55 -0
- data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
- data/lib/etl/processor/escape_csv_processor.rb +77 -0
- data/lib/etl/processor/filter_row_processor.rb +51 -0
- data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
- data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
- data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
- data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
- data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
- data/lib/etl/processor/print_row_processor.rb +12 -0
- data/lib/etl/processor/processor.rb +25 -0
- data/lib/etl/processor/rename_processor.rb +24 -0
- data/lib/etl/processor/require_non_blank_processor.rb +26 -0
- data/lib/etl/processor/row_processor.rb +27 -0
- data/lib/etl/processor/sequence_processor.rb +23 -0
- data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
- data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
- data/lib/etl/processor/surrogate_key_processor.rb +53 -0
- data/lib/etl/processor/truncate_processor.rb +40 -0
- data/lib/etl/processor/zip_file_processor.rb +27 -0
- data/lib/etl/row.rb +20 -0
- data/lib/etl/screen.rb +14 -0
- data/lib/etl/screen/row_count_screen.rb +20 -0
- data/lib/etl/transform.rb +2 -0
- data/lib/etl/transform/block_transform.rb +13 -0
- data/lib/etl/transform/calculation_transform.rb +71 -0
- data/lib/etl/transform/date_to_string_transform.rb +20 -0
- data/lib/etl/transform/decode_transform.rb +51 -0
- data/lib/etl/transform/default_transform.rb +20 -0
- data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
- data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
- data/lib/etl/transform/md5_transform.rb +13 -0
- data/lib/etl/transform/ordinalize_transform.rb +14 -0
- data/lib/etl/transform/sha1_transform.rb +13 -0
- data/lib/etl/transform/split_fields_transform.rb +27 -0
- data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
- data/lib/etl/transform/string_to_date_transform.rb +16 -0
- data/lib/etl/transform/string_to_time_transform.rb +11 -0
- data/lib/etl/transform/transform.rb +61 -0
- data/lib/etl/transform/trim_transform.rb +26 -0
- data/lib/etl/transform/type_transform.rb +35 -0
- data/lib/etl/util.rb +59 -0
- data/lib/etl/version.rb +3 -0
- data/test-matrix.yml +10 -0
- data/test/.gitignore +1 -0
- data/test/.ignore +2 -0
- data/test/all.ebf +6 -0
- data/test/apache_combined_log.ctl +11 -0
- data/test/batch_test.rb +41 -0
- data/test/batch_with_error.ebf +6 -0
- data/test/batched1.ctl +0 -0
- data/test/batched2.ctl +0 -0
- data/test/block_processor.ctl +6 -0
- data/test/block_processor_error.ctl +1 -0
- data/test/block_processor_pre_post_process.ctl +4 -0
- data/test/block_processor_remove_rows.ctl +5 -0
- data/test/block_processor_test.rb +38 -0
- data/test/check_exist_processor_test.rb +92 -0
- data/test/check_unique_processor_test.rb +40 -0
- data/test/config/Gemfile.rails-2.3.x +3 -0
- data/test/config/Gemfile.rails-2.3.x.lock +53 -0
- data/test/config/Gemfile.rails-3.0.x +3 -0
- data/test/config/Gemfile.rails-3.0.x.lock +61 -0
- data/test/config/common.rb +29 -0
- data/test/connection/mysql/connection.rb +9 -0
- data/test/connection/mysql/schema.sql +37 -0
- data/test/connection/postgresql/connection.rb +13 -0
- data/test/connection/postgresql/schema.sql +40 -0
- data/test/control_test.rb +43 -0
- data/test/data/apache_combined_log.txt +3 -0
- data/test/data/bulk_import.txt +3 -0
- data/test/data/bulk_import_with_empties.txt +3 -0
- data/test/data/decode.txt +3 -0
- data/test/data/delimited.txt +3 -0
- data/test/data/encode_source_latin1.txt +2 -0
- data/test/data/excel.xls +0 -0
- data/test/data/excel2.xls +0 -0
- data/test/data/fixed_width.txt +3 -0
- data/test/data/multiple_delimited_1.txt +3 -0
- data/test/data/multiple_delimited_2.txt +3 -0
- data/test/data/nokogiri.xml +38 -0
- data/test/data/people.txt +3 -0
- data/test/data/sax.xml +14 -0
- data/test/data/xml.xml +16 -0
- data/test/database_join_processor_test.rb +43 -0
- data/test/date_dimension_builder_test.rb +96 -0
- data/test/delimited.ctl +30 -0
- data/test/delimited_absolute.ctl +31 -0
- data/test/delimited_destination_db.ctl +23 -0
- data/test/delimited_excel.ctl +31 -0
- data/test/delimited_insert_update.ctl +34 -0
- data/test/delimited_update.ctl +34 -0
- data/test/delimited_with_bulk_load.ctl +34 -0
- data/test/destination_test.rb +275 -0
- data/test/directive_test.rb +23 -0
- data/test/encode_processor_test.rb +32 -0
- data/test/engine_test.rb +78 -0
- data/test/ensure_fields_presence_processor_test.rb +28 -0
- data/test/errors.ctl +24 -0
- data/test/etl_test.rb +42 -0
- data/test/excel.ctl +24 -0
- data/test/excel2.ctl +25 -0
- data/test/fixed_width.ctl +35 -0
- data/test/foreign_key_lookup_transform_test.rb +50 -0
- data/test/generator_test.rb +14 -0
- data/test/inline_parser.ctl +17 -0
- data/test/mocks/mock_destination.rb +26 -0
- data/test/mocks/mock_source.rb +25 -0
- data/test/model_source.ctl +14 -0
- data/test/multiple_delimited.ctl +22 -0
- data/test/multiple_source_delimited.ctl +39 -0
- data/test/nokogiri_all.ctl +35 -0
- data/test/nokogiri_select.ctl +35 -0
- data/test/nokogiri_test.rb +35 -0
- data/test/parser_test.rb +224 -0
- data/test/performance/delimited.ctl +30 -0
- data/test/processor_test.rb +44 -0
- data/test/row_processor_test.rb +17 -0
- data/test/sax.ctl +26 -0
- data/test/scd/1.txt +1 -0
- data/test/scd/2.txt +1 -0
- data/test/scd/3.txt +1 -0
- data/test/scd_test.rb +257 -0
- data/test/scd_test_type_1.ctl +43 -0
- data/test/scd_test_type_2.ctl +34 -0
- data/test/screen_test.rb +9 -0
- data/test/screen_test_error.ctl +3 -0
- data/test/screen_test_fatal.ctl +3 -0
- data/test/source_test.rb +154 -0
- data/test/test_helper.rb +37 -0
- data/test/transform_test.rb +101 -0
- data/test/truncate_processor_test.rb +37 -0
- data/test/xml.ctl +31 -0
- metadata +370 -0
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class Person < ActiveRecord::Base
|
4
|
+
end
|
5
|
+
|
6
|
+
class CheckExistProcessorTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
context 'CheckExistProcessor' do
|
9
|
+
|
10
|
+
setup do
|
11
|
+
@config = {
|
12
|
+
:target => :data_warehouse,
|
13
|
+
:table => 'people',
|
14
|
+
:columns => [:first_name, :last_name]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
should_eventually "compare based on all columns if no columns are provided" do
|
19
|
+
# TBI
|
20
|
+
end
|
21
|
+
|
22
|
+
should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
|
23
|
+
# TBI
|
24
|
+
end
|
25
|
+
|
26
|
+
should "raise an error if no table is provided" do
|
27
|
+
error = assert_raises(ETL::ControlError) do
|
28
|
+
ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
|
29
|
+
end
|
30
|
+
# bug #2413 on assert_raises won't let me check error message above
|
31
|
+
assert_equal 'table must be specified', error.message
|
32
|
+
end
|
33
|
+
|
34
|
+
should "raise an error if no target is provided" do
|
35
|
+
error = assert_raises(ETL::ControlError) do
|
36
|
+
ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
|
37
|
+
end
|
38
|
+
|
39
|
+
assert_equal 'target must be specified', error.message
|
40
|
+
end
|
41
|
+
|
42
|
+
should "bypass checking if the table has no rows" do
|
43
|
+
Person.delete_all
|
44
|
+
|
45
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
46
|
+
assert_equal false, processor.should_check?
|
47
|
+
end
|
48
|
+
|
49
|
+
should "raise an error if one of the keys used for checking existence is not available in a row" do
|
50
|
+
Person.delete_all
|
51
|
+
# we need at least one record to avoid automatic skipping
|
52
|
+
# this should be mocked instead, probably
|
53
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
54
|
+
|
55
|
+
error = assert_raise(ETL::ControlError) do
|
56
|
+
row = ETL::Row[:first_name => 'John']
|
57
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
58
|
+
|
59
|
+
# guard against bypassing
|
60
|
+
assert_equal true, processor.should_check?
|
61
|
+
|
62
|
+
processor.process(row)
|
63
|
+
end
|
64
|
+
|
65
|
+
assert_equal "Row missing required field :last_name for existence check", error.message
|
66
|
+
end
|
67
|
+
|
68
|
+
should "return nil if the same row is found in database" do
|
69
|
+
Person.delete_all
|
70
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
71
|
+
|
72
|
+
row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
|
73
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
74
|
+
assert_equal true, processor.should_check? # guard against bypassing
|
75
|
+
|
76
|
+
assert_equal nil, processor.process(row)
|
77
|
+
end
|
78
|
+
|
79
|
+
should "return the row if no same row is found in database" do
|
80
|
+
Person.delete_all
|
81
|
+
Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
|
82
|
+
|
83
|
+
row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
|
84
|
+
processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
|
85
|
+
assert_equal true, processor.should_check? # guard against bypassing
|
86
|
+
|
87
|
+
assert_equal row, processor.process(row)
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class CheckUniqueProcessorTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
context 'CheckUniqueProcessor' do
|
6
|
+
attr_reader :processor
|
7
|
+
|
8
|
+
setup do
|
9
|
+
@processor = ETL::Processor::CheckUniqueProcessor.new(nil,
|
10
|
+
:keys => [:first, :second])
|
11
|
+
end
|
12
|
+
|
13
|
+
should "keep a row whose keys didn't already appear in the pipeline" do
|
14
|
+
row = ETL::Row[:first => 'A', :second => 'B']
|
15
|
+
|
16
|
+
assert_equal row, processor.process(row)
|
17
|
+
|
18
|
+
assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
|
19
|
+
end
|
20
|
+
|
21
|
+
should "remove a row whose keys already appeared in the pipeline" do
|
22
|
+
row = ETL::Row[:first => 'A', :second => 'B']
|
23
|
+
|
24
|
+
assert_equal row, processor.process(row)
|
25
|
+
assert_equal nil, processor.process(row)
|
26
|
+
end
|
27
|
+
|
28
|
+
should "raise an error if a row lacks one of the keys specified" do
|
29
|
+
row = ETL::Row[:first => 'A']
|
30
|
+
|
31
|
+
error = assert_raises(ETL::ControlError) do
|
32
|
+
processor.process(row)
|
33
|
+
end
|
34
|
+
|
35
|
+
assert_equal "Row missing required field :second for unicity check", error.message
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git@github.com:activewarehouse/adapter_extensions.git
|
3
|
+
revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
|
4
|
+
specs:
|
5
|
+
adapter_extensions (0.9.5.rc1)
|
6
|
+
activerecord (>= 2.1.0)
|
7
|
+
activesupport (>= 2.1.0)
|
8
|
+
rake (>= 0.8.3)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activerecord (2.3.11)
|
14
|
+
activesupport (= 2.3.11)
|
15
|
+
activesupport (2.3.11)
|
16
|
+
fastercsv (1.5.4)
|
17
|
+
flexmock (0.9.0)
|
18
|
+
mysql (2.8.1)
|
19
|
+
net-sftp (2.0.5)
|
20
|
+
net-ssh (>= 2.0.9)
|
21
|
+
net-ssh (2.1.4)
|
22
|
+
nokogiri (1.4.4)
|
23
|
+
nokogiri (1.4.4-java)
|
24
|
+
weakling (>= 0.0.3)
|
25
|
+
pg (0.11.0)
|
26
|
+
rdoc (3.6.1)
|
27
|
+
ruby-ole (1.2.11.1)
|
28
|
+
shoulda (2.11.3)
|
29
|
+
spreadsheet (0.6.5.4)
|
30
|
+
ruby-ole (>= 1.0)
|
31
|
+
tmail (1.2.7.1)
|
32
|
+
weakling (0.0.4-java)
|
33
|
+
zip (2.0.2)
|
34
|
+
|
35
|
+
PLATFORMS
|
36
|
+
java
|
37
|
+
ruby
|
38
|
+
|
39
|
+
DEPENDENCIES
|
40
|
+
activerecord (= 2.3.11)
|
41
|
+
activesupport (= 2.3.11)
|
42
|
+
adapter_extensions!
|
43
|
+
fastercsv (= 1.5.4)
|
44
|
+
flexmock (= 0.9.0)
|
45
|
+
mysql (= 2.8.1)
|
46
|
+
net-sftp (= 2.0.5)
|
47
|
+
nokogiri (= 1.4.4)
|
48
|
+
pg (= 0.11.0)
|
49
|
+
rdoc
|
50
|
+
shoulda (= 2.11.3)
|
51
|
+
spreadsheet (= 0.6.5.4)
|
52
|
+
tmail (= 1.2.7.1)
|
53
|
+
zip (= 2.0.2)
|
@@ -0,0 +1,61 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git@github.com:activewarehouse/adapter_extensions.git
|
3
|
+
revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
|
4
|
+
specs:
|
5
|
+
adapter_extensions (0.9.5.rc1)
|
6
|
+
activerecord (>= 2.1.0)
|
7
|
+
activesupport (>= 2.1.0)
|
8
|
+
rake (>= 0.8.3)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activemodel (3.0.7)
|
14
|
+
activesupport (= 3.0.7)
|
15
|
+
builder (~> 2.1.2)
|
16
|
+
i18n (~> 0.5.0)
|
17
|
+
activerecord (3.0.7)
|
18
|
+
activemodel (= 3.0.7)
|
19
|
+
activesupport (= 3.0.7)
|
20
|
+
arel (~> 2.0.2)
|
21
|
+
tzinfo (~> 0.3.23)
|
22
|
+
activesupport (3.0.7)
|
23
|
+
arel (2.0.10)
|
24
|
+
builder (2.1.2)
|
25
|
+
fastercsv (1.5.4)
|
26
|
+
flexmock (0.9.0)
|
27
|
+
i18n (0.5.0)
|
28
|
+
mysql (2.8.1)
|
29
|
+
net-sftp (2.0.5)
|
30
|
+
net-ssh (>= 2.0.9)
|
31
|
+
net-ssh (2.1.4)
|
32
|
+
nokogiri (1.4.4)
|
33
|
+
pg (0.11.0)
|
34
|
+
rdoc (3.6.1)
|
35
|
+
ruby-ole (1.2.11.1)
|
36
|
+
shoulda (2.11.3)
|
37
|
+
spreadsheet (0.6.5.4)
|
38
|
+
ruby-ole (>= 1.0)
|
39
|
+
tmail (1.2.7.1)
|
40
|
+
tzinfo (0.3.27)
|
41
|
+
zip (2.0.2)
|
42
|
+
|
43
|
+
PLATFORMS
|
44
|
+
java
|
45
|
+
ruby
|
46
|
+
|
47
|
+
DEPENDENCIES
|
48
|
+
activerecord (= 3.0.7)
|
49
|
+
activesupport (= 3.0.7)
|
50
|
+
adapter_extensions!
|
51
|
+
fastercsv (= 1.5.4)
|
52
|
+
flexmock (= 0.9.0)
|
53
|
+
mysql (= 2.8.1)
|
54
|
+
net-sftp (= 2.0.5)
|
55
|
+
nokogiri (= 1.4.4)
|
56
|
+
pg (= 0.11.0)
|
57
|
+
rdoc
|
58
|
+
shoulda (= 2.11.3)
|
59
|
+
spreadsheet (= 0.6.5.4)
|
60
|
+
tmail (= 1.2.7.1)
|
61
|
+
zip (= 2.0.2)
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rbconfig'
|
2
|
+
|
3
|
+
def common_gemfile(rails_version)
|
4
|
+
source :rubygems
|
5
|
+
|
6
|
+
# using explicit versions for the gems to avoid any weirdness later on
|
7
|
+
gem "activesupport", rails_version
|
8
|
+
gem "activerecord", rails_version
|
9
|
+
|
10
|
+
gem "fastercsv", "1.5.4"
|
11
|
+
gem "spreadsheet", "0.6.5.4"
|
12
|
+
gem "tmail", "1.2.7.1"
|
13
|
+
gem "net-sftp", "2.0.5"
|
14
|
+
gem "zip", "2.0.2"
|
15
|
+
|
16
|
+
gem "shoulda", "2.11.3"
|
17
|
+
gem "flexmock", "0.9.0"
|
18
|
+
|
19
|
+
gem "mysql", "2.8.1"
|
20
|
+
gem "pg", "0.11.0"
|
21
|
+
|
22
|
+
gem "nokogiri", "1.4.4"
|
23
|
+
|
24
|
+
gem "rdoc"
|
25
|
+
|
26
|
+
gem "adapter_extensions", :git => 'git@github.com:activewarehouse/adapter_extensions.git'
|
27
|
+
|
28
|
+
gem "jruby-openssl" if RUBY_PLATFORM == "java"
|
29
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
print "Using native MySQL\n"
|
2
|
+
|
3
|
+
puts "Resetting database"
|
4
|
+
conn = ETL::Engine.connection(:data_warehouse)
|
5
|
+
conn.recreate_database(conn.current_database)
|
6
|
+
conn.reconnect!
|
7
|
+
lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
|
8
|
+
lines.join.split(';').each { |line| conn.execute(line) }
|
9
|
+
conn.disconnect!
|
@@ -0,0 +1,37 @@
|
|
1
|
+
drop table if exists people;
|
2
|
+
create table people (
|
3
|
+
id int not null primary key,
|
4
|
+
first_name char(255) not null,
|
5
|
+
last_name char(255) not null,
|
6
|
+
ssn char(64) not null
|
7
|
+
);
|
8
|
+
drop table if exists places;
|
9
|
+
create table places (
|
10
|
+
address text,
|
11
|
+
city char(255),
|
12
|
+
state char(255),
|
13
|
+
country char(2)
|
14
|
+
);
|
15
|
+
|
16
|
+
drop table if exists person_dimension;
|
17
|
+
create table person_dimension (
|
18
|
+
id int not null primary key,
|
19
|
+
first_name char(50),
|
20
|
+
last_name char(50),
|
21
|
+
address char(100),
|
22
|
+
city char(50),
|
23
|
+
state char(50),
|
24
|
+
zip_code char(20),
|
25
|
+
effective_date datetime,
|
26
|
+
end_date datetime,
|
27
|
+
latest_version boolean not null
|
28
|
+
);
|
29
|
+
|
30
|
+
drop table if exists truncate_test;
|
31
|
+
create table truncate_test (
|
32
|
+
id int not null primary key auto_increment,
|
33
|
+
x char(4)
|
34
|
+
);
|
35
|
+
insert into truncate_test (x) values ('a');
|
36
|
+
insert into truncate_test (x) values ('b');
|
37
|
+
insert into truncate_test (x) values ('c');
|
@@ -0,0 +1,13 @@
|
|
1
|
+
print "Using PostgreSQL\n"
|
2
|
+
|
3
|
+
puts "Resetting database"
|
4
|
+
conn = ETL::Engine.connection(:data_warehouse)
|
5
|
+
|
6
|
+
lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
|
7
|
+
lines.join.split(';').each_with_index do |line, index|
|
8
|
+
begin
|
9
|
+
conn.execute(line)
|
10
|
+
rescue => e
|
11
|
+
puts "failed to load line #{index}: #{e}"
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
drop table people;
|
2
|
+
create table people (
|
3
|
+
id SERIAL PRIMARY KEY,
|
4
|
+
first_name character varying(255) not null,
|
5
|
+
/* null below allowed for bulk_import_with_empties.txt test */
|
6
|
+
last_name character varying(255) null,
|
7
|
+
ssn character varying(64) not null
|
8
|
+
);
|
9
|
+
|
10
|
+
drop table places;
|
11
|
+
create table places (
|
12
|
+
id SERIAL PRIMARY KEY,
|
13
|
+
address text,
|
14
|
+
city character varying(255),
|
15
|
+
state character varying(255),
|
16
|
+
country character varying(2)
|
17
|
+
);
|
18
|
+
|
19
|
+
drop table person_dimension;
|
20
|
+
create table person_dimension (
|
21
|
+
id SERIAL PRIMARY KEY,
|
22
|
+
first_name character varying(50),
|
23
|
+
last_name character varying(50),
|
24
|
+
address character varying(100),
|
25
|
+
city character varying(50),
|
26
|
+
state character varying(50),
|
27
|
+
zip_code character varying(20),
|
28
|
+
effective_date timestamp without time zone,
|
29
|
+
end_date timestamp without time zone,
|
30
|
+
latest_version boolean not null
|
31
|
+
);
|
32
|
+
|
33
|
+
drop table truncate_test;
|
34
|
+
create table truncate_test (
|
35
|
+
id SERIAL PRIMARY KEY,
|
36
|
+
x character varying(4)
|
37
|
+
);
|
38
|
+
insert into truncate_test (x) values ('a');
|
39
|
+
insert into truncate_test (x) values ('b');
|
40
|
+
insert into truncate_test (x) values ('c');
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class ControlTest < Test::Unit::TestCase
|
4
|
+
# Test the ability to parse control files.
|
5
|
+
def test_parse
|
6
|
+
assert_nothing_raised do
|
7
|
+
Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
|
8
|
+
ETL::Control::Control.parse(f)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_bad_control_raises_error
|
14
|
+
assert_raise ETL::ControlError do
|
15
|
+
ETL::Control::Control.resolve(0)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_resolve_control_object
|
20
|
+
assert_nothing_raised do
|
21
|
+
ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_set_error_threshold
|
26
|
+
assert_nothing_raised do
|
27
|
+
ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_bad_processor_name
|
32
|
+
assert_raise ETL::ControlError do
|
33
|
+
s = "before_write :chunky_monkey"
|
34
|
+
ETL::Control::Control.parse_text(s)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_dependencies
|
39
|
+
s = "depends_on 'foo', 'bar'"
|
40
|
+
control = ETL::Control::Control.parse_text(s)
|
41
|
+
assert_equal control.dependencies, ['foo','bar']
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,3 @@
|
|
1
|
+
127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
2
|
+
127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|
3
|
+
127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"
|