activewarehouse-etl-sgonyea 0.9.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (200) hide show
  1. data/.gitignore +9 -0
  2. data/0.9-UPGRADE +6 -0
  3. data/CHANGELOG +236 -0
  4. data/Gemfile +4 -0
  5. data/HOW_TO_RELEASE +13 -0
  6. data/LICENSE +7 -0
  7. data/README.textile +111 -0
  8. data/Rakefile +103 -0
  9. data/TODO +28 -0
  10. data/active_support_logger.patch +78 -0
  11. data/activewarehouse-etl.gemspec +36 -0
  12. data/bin/etl +28 -0
  13. data/bin/etl.cmd +8 -0
  14. data/examples/database.example.yml +16 -0
  15. data/lib/etl.rb +97 -0
  16. data/lib/etl/batch.rb +2 -0
  17. data/lib/etl/batch/batch.rb +111 -0
  18. data/lib/etl/batch/directives.rb +65 -0
  19. data/lib/etl/builder.rb +2 -0
  20. data/lib/etl/builder/date_dimension_builder.rb +96 -0
  21. data/lib/etl/builder/time_dimension_builder.rb +31 -0
  22. data/lib/etl/commands/etl.rb +89 -0
  23. data/lib/etl/control.rb +3 -0
  24. data/lib/etl/control/control.rb +405 -0
  25. data/lib/etl/control/destination.rb +438 -0
  26. data/lib/etl/control/destination/csv_destination.rb +113 -0
  27. data/lib/etl/control/destination/database_destination.rb +97 -0
  28. data/lib/etl/control/destination/excel_destination.rb +91 -0
  29. data/lib/etl/control/destination/file_destination.rb +126 -0
  30. data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
  31. data/lib/etl/control/destination/update_database_destination.rb +109 -0
  32. data/lib/etl/control/destination/yaml_destination.rb +74 -0
  33. data/lib/etl/control/source.rb +132 -0
  34. data/lib/etl/control/source/database_source.rb +224 -0
  35. data/lib/etl/control/source/enumerable_source.rb +11 -0
  36. data/lib/etl/control/source/file_source.rb +90 -0
  37. data/lib/etl/control/source/model_source.rb +39 -0
  38. data/lib/etl/core_ext.rb +1 -0
  39. data/lib/etl/core_ext/time.rb +5 -0
  40. data/lib/etl/core_ext/time/calculations.rb +42 -0
  41. data/lib/etl/engine.rb +582 -0
  42. data/lib/etl/execution.rb +19 -0
  43. data/lib/etl/execution/base.rb +8 -0
  44. data/lib/etl/execution/batch.rb +10 -0
  45. data/lib/etl/execution/job.rb +8 -0
  46. data/lib/etl/execution/migration.rb +90 -0
  47. data/lib/etl/generator.rb +2 -0
  48. data/lib/etl/generator/generator.rb +20 -0
  49. data/lib/etl/generator/surrogate_key_generator.rb +39 -0
  50. data/lib/etl/http_tools.rb +139 -0
  51. data/lib/etl/parser.rb +11 -0
  52. data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
  53. data/lib/etl/parser/csv_parser.rb +93 -0
  54. data/lib/etl/parser/excel_parser.rb +112 -0
  55. data/lib/etl/parser/fixed_width_parser.rb +65 -0
  56. data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
  57. data/lib/etl/parser/parser.rb +41 -0
  58. data/lib/etl/parser/sax_parser.rb +218 -0
  59. data/lib/etl/parser/xml_parser.rb +65 -0
  60. data/lib/etl/processor.rb +11 -0
  61. data/lib/etl/processor/block_processor.rb +14 -0
  62. data/lib/etl/processor/bulk_import_processor.rb +94 -0
  63. data/lib/etl/processor/check_exist_processor.rb +80 -0
  64. data/lib/etl/processor/check_unique_processor.rb +39 -0
  65. data/lib/etl/processor/copy_field_processor.rb +26 -0
  66. data/lib/etl/processor/database_join_processor.rb +82 -0
  67. data/lib/etl/processor/encode_processor.rb +55 -0
  68. data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
  69. data/lib/etl/processor/escape_csv_processor.rb +77 -0
  70. data/lib/etl/processor/filter_row_processor.rb +51 -0
  71. data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
  72. data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
  73. data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
  74. data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
  75. data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
  76. data/lib/etl/processor/print_row_processor.rb +12 -0
  77. data/lib/etl/processor/processor.rb +25 -0
  78. data/lib/etl/processor/rename_processor.rb +24 -0
  79. data/lib/etl/processor/require_non_blank_processor.rb +26 -0
  80. data/lib/etl/processor/row_processor.rb +27 -0
  81. data/lib/etl/processor/sequence_processor.rb +23 -0
  82. data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
  83. data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
  84. data/lib/etl/processor/surrogate_key_processor.rb +53 -0
  85. data/lib/etl/processor/truncate_processor.rb +40 -0
  86. data/lib/etl/processor/zip_file_processor.rb +27 -0
  87. data/lib/etl/row.rb +20 -0
  88. data/lib/etl/screen.rb +14 -0
  89. data/lib/etl/screen/row_count_screen.rb +20 -0
  90. data/lib/etl/transform.rb +2 -0
  91. data/lib/etl/transform/block_transform.rb +13 -0
  92. data/lib/etl/transform/calculation_transform.rb +71 -0
  93. data/lib/etl/transform/date_to_string_transform.rb +20 -0
  94. data/lib/etl/transform/decode_transform.rb +51 -0
  95. data/lib/etl/transform/default_transform.rb +20 -0
  96. data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
  97. data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
  98. data/lib/etl/transform/md5_transform.rb +13 -0
  99. data/lib/etl/transform/ordinalize_transform.rb +14 -0
  100. data/lib/etl/transform/sha1_transform.rb +13 -0
  101. data/lib/etl/transform/split_fields_transform.rb +27 -0
  102. data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
  103. data/lib/etl/transform/string_to_date_transform.rb +16 -0
  104. data/lib/etl/transform/string_to_time_transform.rb +11 -0
  105. data/lib/etl/transform/transform.rb +61 -0
  106. data/lib/etl/transform/trim_transform.rb +26 -0
  107. data/lib/etl/transform/type_transform.rb +35 -0
  108. data/lib/etl/util.rb +59 -0
  109. data/lib/etl/version.rb +3 -0
  110. data/test-matrix.yml +10 -0
  111. data/test/.gitignore +1 -0
  112. data/test/.ignore +2 -0
  113. data/test/all.ebf +6 -0
  114. data/test/apache_combined_log.ctl +11 -0
  115. data/test/batch_test.rb +41 -0
  116. data/test/batch_with_error.ebf +6 -0
  117. data/test/batched1.ctl +0 -0
  118. data/test/batched2.ctl +0 -0
  119. data/test/block_processor.ctl +6 -0
  120. data/test/block_processor_error.ctl +1 -0
  121. data/test/block_processor_pre_post_process.ctl +4 -0
  122. data/test/block_processor_remove_rows.ctl +5 -0
  123. data/test/block_processor_test.rb +38 -0
  124. data/test/check_exist_processor_test.rb +92 -0
  125. data/test/check_unique_processor_test.rb +40 -0
  126. data/test/config/Gemfile.rails-2.3.x +3 -0
  127. data/test/config/Gemfile.rails-2.3.x.lock +53 -0
  128. data/test/config/Gemfile.rails-3.0.x +3 -0
  129. data/test/config/Gemfile.rails-3.0.x.lock +61 -0
  130. data/test/config/common.rb +29 -0
  131. data/test/connection/mysql/connection.rb +9 -0
  132. data/test/connection/mysql/schema.sql +37 -0
  133. data/test/connection/postgresql/connection.rb +13 -0
  134. data/test/connection/postgresql/schema.sql +40 -0
  135. data/test/control_test.rb +43 -0
  136. data/test/data/apache_combined_log.txt +3 -0
  137. data/test/data/bulk_import.txt +3 -0
  138. data/test/data/bulk_import_with_empties.txt +3 -0
  139. data/test/data/decode.txt +3 -0
  140. data/test/data/delimited.txt +3 -0
  141. data/test/data/encode_source_latin1.txt +2 -0
  142. data/test/data/excel.xls +0 -0
  143. data/test/data/excel2.xls +0 -0
  144. data/test/data/fixed_width.txt +3 -0
  145. data/test/data/multiple_delimited_1.txt +3 -0
  146. data/test/data/multiple_delimited_2.txt +3 -0
  147. data/test/data/nokogiri.xml +38 -0
  148. data/test/data/people.txt +3 -0
  149. data/test/data/sax.xml +14 -0
  150. data/test/data/xml.xml +16 -0
  151. data/test/database_join_processor_test.rb +43 -0
  152. data/test/date_dimension_builder_test.rb +96 -0
  153. data/test/delimited.ctl +30 -0
  154. data/test/delimited_absolute.ctl +31 -0
  155. data/test/delimited_destination_db.ctl +23 -0
  156. data/test/delimited_excel.ctl +31 -0
  157. data/test/delimited_insert_update.ctl +34 -0
  158. data/test/delimited_update.ctl +34 -0
  159. data/test/delimited_with_bulk_load.ctl +34 -0
  160. data/test/destination_test.rb +275 -0
  161. data/test/directive_test.rb +23 -0
  162. data/test/encode_processor_test.rb +32 -0
  163. data/test/engine_test.rb +78 -0
  164. data/test/ensure_fields_presence_processor_test.rb +28 -0
  165. data/test/errors.ctl +24 -0
  166. data/test/etl_test.rb +42 -0
  167. data/test/excel.ctl +24 -0
  168. data/test/excel2.ctl +25 -0
  169. data/test/fixed_width.ctl +35 -0
  170. data/test/foreign_key_lookup_transform_test.rb +50 -0
  171. data/test/generator_test.rb +14 -0
  172. data/test/inline_parser.ctl +17 -0
  173. data/test/mocks/mock_destination.rb +26 -0
  174. data/test/mocks/mock_source.rb +25 -0
  175. data/test/model_source.ctl +14 -0
  176. data/test/multiple_delimited.ctl +22 -0
  177. data/test/multiple_source_delimited.ctl +39 -0
  178. data/test/nokogiri_all.ctl +35 -0
  179. data/test/nokogiri_select.ctl +35 -0
  180. data/test/nokogiri_test.rb +35 -0
  181. data/test/parser_test.rb +224 -0
  182. data/test/performance/delimited.ctl +30 -0
  183. data/test/processor_test.rb +44 -0
  184. data/test/row_processor_test.rb +17 -0
  185. data/test/sax.ctl +26 -0
  186. data/test/scd/1.txt +1 -0
  187. data/test/scd/2.txt +1 -0
  188. data/test/scd/3.txt +1 -0
  189. data/test/scd_test.rb +257 -0
  190. data/test/scd_test_type_1.ctl +43 -0
  191. data/test/scd_test_type_2.ctl +34 -0
  192. data/test/screen_test.rb +9 -0
  193. data/test/screen_test_error.ctl +3 -0
  194. data/test/screen_test_fatal.ctl +3 -0
  195. data/test/source_test.rb +154 -0
  196. data/test/test_helper.rb +37 -0
  197. data/test/transform_test.rb +101 -0
  198. data/test/truncate_processor_test.rb +37 -0
  199. data/test/xml.ctl +31 -0
  200. metadata +370 -0
@@ -0,0 +1,92 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class Person < ActiveRecord::Base
4
+ end
5
+
6
+ class CheckExistProcessorTest < Test::Unit::TestCase
7
+
8
+ context 'CheckExistProcessor' do
9
+
10
+ setup do
11
+ @config = {
12
+ :target => :data_warehouse,
13
+ :table => 'people',
14
+ :columns => [:first_name, :last_name]
15
+ }
16
+ end
17
+
18
+ should_eventually "compare based on all columns if no columns are provided" do
19
+ # TBI
20
+ end
21
+
22
+ should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
23
+ # TBI
24
+ end
25
+
26
+ should "raise an error if no table is provided" do
27
+ error = assert_raises(ETL::ControlError) do
28
+ ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
29
+ end
30
+ # bug #2413 on assert_raises won't let me check error message above
31
+ assert_equal 'table must be specified', error.message
32
+ end
33
+
34
+ should "raise an error if no target is provided" do
35
+ error = assert_raises(ETL::ControlError) do
36
+ ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
37
+ end
38
+
39
+ assert_equal 'target must be specified', error.message
40
+ end
41
+
42
+ should "bypass checking if the table has no rows" do
43
+ Person.delete_all
44
+
45
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
46
+ assert_equal false, processor.should_check?
47
+ end
48
+
49
+ should "raise an error if one of the keys used for checking existence is not available in a row" do
50
+ Person.delete_all
51
+ # we need at least one record to avoid automatic skipping
52
+ # this should be mocked instead, probably
53
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
54
+
55
+ error = assert_raise(ETL::ControlError) do
56
+ row = ETL::Row[:first_name => 'John']
57
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
58
+
59
+ # guard against bypassing
60
+ assert_equal true, processor.should_check?
61
+
62
+ processor.process(row)
63
+ end
64
+
65
+ assert_equal "Row missing required field :last_name for existence check", error.message
66
+ end
67
+
68
+ should "return nil if the same row is found in database" do
69
+ Person.delete_all
70
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
71
+
72
+ row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
73
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
74
+ assert_equal true, processor.should_check? # guard against bypassing
75
+
76
+ assert_equal nil, processor.process(row)
77
+ end
78
+
79
+ should "return the row if no same row is found in database" do
80
+ Person.delete_all
81
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
82
+
83
+ row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
84
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
85
+ assert_equal true, processor.should_check? # guard against bypassing
86
+
87
+ assert_equal row, processor.process(row)
88
+ end
89
+
90
+ end
91
+
92
+ end
@@ -0,0 +1,40 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class CheckUniqueProcessorTest < Test::Unit::TestCase
4
+
5
+ context 'CheckUniqueProcessor' do
6
+ attr_reader :processor
7
+
8
+ setup do
9
+ @processor = ETL::Processor::CheckUniqueProcessor.new(nil,
10
+ :keys => [:first, :second])
11
+ end
12
+
13
+ should "keep a row whose keys didn't already appear in the pipeline" do
14
+ row = ETL::Row[:first => 'A', :second => 'B']
15
+
16
+ assert_equal row, processor.process(row)
17
+
18
+ assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
19
+ end
20
+
21
+ should "remove a row whose keys already appeared in the pipeline" do
22
+ row = ETL::Row[:first => 'A', :second => 'B']
23
+
24
+ assert_equal row, processor.process(row)
25
+ assert_equal nil, processor.process(row)
26
+ end
27
+
28
+ should "raise an error if a row lacks one of the keys specified" do
29
+ row = ETL::Row[:first => 'A']
30
+
31
+ error = assert_raises(ETL::ControlError) do
32
+ processor.process(row)
33
+ end
34
+
35
+ assert_equal "Row missing required field :second for unicity check", error.message
36
+ end
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,3 @@
1
+ require File.dirname(__FILE__) + '/common'
2
+
3
+ common_gemfile('2.3.11')
@@ -0,0 +1,53 @@
1
+ GIT
2
+ remote: git@github.com:activewarehouse/adapter_extensions.git
3
+ revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
4
+ specs:
5
+ adapter_extensions (0.9.5.rc1)
6
+ activerecord (>= 2.1.0)
7
+ activesupport (>= 2.1.0)
8
+ rake (>= 0.8.3)
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ activerecord (2.3.11)
14
+ activesupport (= 2.3.11)
15
+ activesupport (2.3.11)
16
+ fastercsv (1.5.4)
17
+ flexmock (0.9.0)
18
+ mysql (2.8.1)
19
+ net-sftp (2.0.5)
20
+ net-ssh (>= 2.0.9)
21
+ net-ssh (2.1.4)
22
+ nokogiri (1.4.4)
23
+ nokogiri (1.4.4-java)
24
+ weakling (>= 0.0.3)
25
+ pg (0.11.0)
26
+ rdoc (3.6.1)
27
+ ruby-ole (1.2.11.1)
28
+ shoulda (2.11.3)
29
+ spreadsheet (0.6.5.4)
30
+ ruby-ole (>= 1.0)
31
+ tmail (1.2.7.1)
32
+ weakling (0.0.4-java)
33
+ zip (2.0.2)
34
+
35
+ PLATFORMS
36
+ java
37
+ ruby
38
+
39
+ DEPENDENCIES
40
+ activerecord (= 2.3.11)
41
+ activesupport (= 2.3.11)
42
+ adapter_extensions!
43
+ fastercsv (= 1.5.4)
44
+ flexmock (= 0.9.0)
45
+ mysql (= 2.8.1)
46
+ net-sftp (= 2.0.5)
47
+ nokogiri (= 1.4.4)
48
+ pg (= 0.11.0)
49
+ rdoc
50
+ shoulda (= 2.11.3)
51
+ spreadsheet (= 0.6.5.4)
52
+ tmail (= 1.2.7.1)
53
+ zip (= 2.0.2)
@@ -0,0 +1,3 @@
1
+ require File.dirname(__FILE__) + '/common'
2
+
3
+ common_gemfile('3.0.7')
@@ -0,0 +1,61 @@
1
+ GIT
2
+ remote: git@github.com:activewarehouse/adapter_extensions.git
3
+ revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
4
+ specs:
5
+ adapter_extensions (0.9.5.rc1)
6
+ activerecord (>= 2.1.0)
7
+ activesupport (>= 2.1.0)
8
+ rake (>= 0.8.3)
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ activemodel (3.0.7)
14
+ activesupport (= 3.0.7)
15
+ builder (~> 2.1.2)
16
+ i18n (~> 0.5.0)
17
+ activerecord (3.0.7)
18
+ activemodel (= 3.0.7)
19
+ activesupport (= 3.0.7)
20
+ arel (~> 2.0.2)
21
+ tzinfo (~> 0.3.23)
22
+ activesupport (3.0.7)
23
+ arel (2.0.10)
24
+ builder (2.1.2)
25
+ fastercsv (1.5.4)
26
+ flexmock (0.9.0)
27
+ i18n (0.5.0)
28
+ mysql (2.8.1)
29
+ net-sftp (2.0.5)
30
+ net-ssh (>= 2.0.9)
31
+ net-ssh (2.1.4)
32
+ nokogiri (1.4.4)
33
+ pg (0.11.0)
34
+ rdoc (3.6.1)
35
+ ruby-ole (1.2.11.1)
36
+ shoulda (2.11.3)
37
+ spreadsheet (0.6.5.4)
38
+ ruby-ole (>= 1.0)
39
+ tmail (1.2.7.1)
40
+ tzinfo (0.3.27)
41
+ zip (2.0.2)
42
+
43
+ PLATFORMS
44
+ java
45
+ ruby
46
+
47
+ DEPENDENCIES
48
+ activerecord (= 3.0.7)
49
+ activesupport (= 3.0.7)
50
+ adapter_extensions!
51
+ fastercsv (= 1.5.4)
52
+ flexmock (= 0.9.0)
53
+ mysql (= 2.8.1)
54
+ net-sftp (= 2.0.5)
55
+ nokogiri (= 1.4.4)
56
+ pg (= 0.11.0)
57
+ rdoc
58
+ shoulda (= 2.11.3)
59
+ spreadsheet (= 0.6.5.4)
60
+ tmail (= 1.2.7.1)
61
+ zip (= 2.0.2)
@@ -0,0 +1,29 @@
1
+ require 'rbconfig'
2
+
3
+ def common_gemfile(rails_version)
4
+ source :rubygems
5
+
6
+ # using explicit versions for the gems to avoid any weirdness later on
7
+ gem "activesupport", rails_version
8
+ gem "activerecord", rails_version
9
+
10
+ gem "fastercsv", "1.5.4"
11
+ gem "spreadsheet", "0.6.5.4"
12
+ gem "tmail", "1.2.7.1"
13
+ gem "net-sftp", "2.0.5"
14
+ gem "zip", "2.0.2"
15
+
16
+ gem "shoulda", "2.11.3"
17
+ gem "flexmock", "0.9.0"
18
+
19
+ gem "mysql", "2.8.1"
20
+ gem "pg", "0.11.0"
21
+
22
+ gem "nokogiri", "1.4.4"
23
+
24
+ gem "rdoc"
25
+
26
+ gem "adapter_extensions", :git => 'git@github.com:activewarehouse/adapter_extensions.git'
27
+
28
+ gem "jruby-openssl" if RUBY_PLATFORM == "java"
29
+ end
@@ -0,0 +1,9 @@
1
+ print "Using native MySQL\n"
2
+
3
+ puts "Resetting database"
4
+ conn = ETL::Engine.connection(:data_warehouse)
5
+ conn.recreate_database(conn.current_database)
6
+ conn.reconnect!
7
+ lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
8
+ lines.join.split(';').each { |line| conn.execute(line) }
9
+ conn.disconnect!
@@ -0,0 +1,37 @@
1
+ drop table if exists people;
2
+ create table people (
3
+ id int not null primary key,
4
+ first_name char(255) not null,
5
+ last_name char(255) not null,
6
+ ssn char(64) not null
7
+ );
8
+ drop table if exists places;
9
+ create table places (
10
+ address text,
11
+ city char(255),
12
+ state char(255),
13
+ country char(2)
14
+ );
15
+
16
+ drop table if exists person_dimension;
17
+ create table person_dimension (
18
+ id int not null primary key,
19
+ first_name char(50),
20
+ last_name char(50),
21
+ address char(100),
22
+ city char(50),
23
+ state char(50),
24
+ zip_code char(20),
25
+ effective_date datetime,
26
+ end_date datetime,
27
+ latest_version boolean not null
28
+ );
29
+
30
+ drop table if exists truncate_test;
31
+ create table truncate_test (
32
+ id int not null primary key auto_increment,
33
+ x char(4)
34
+ );
35
+ insert into truncate_test (x) values ('a');
36
+ insert into truncate_test (x) values ('b');
37
+ insert into truncate_test (x) values ('c');
@@ -0,0 +1,13 @@
1
+ print "Using PostgreSQL\n"
2
+
3
+ puts "Resetting database"
4
+ conn = ETL::Engine.connection(:data_warehouse)
5
+
6
+ lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
7
+ lines.join.split(';').each_with_index do |line, index|
8
+ begin
9
+ conn.execute(line)
10
+ rescue => e
11
+ puts "failed to load line #{index}: #{e}"
12
+ end
13
+ end
@@ -0,0 +1,40 @@
1
+ drop table people;
2
+ create table people (
3
+ id SERIAL PRIMARY KEY,
4
+ first_name character varying(255) not null,
5
+ /* null below allowed for bulk_import_with_empties.txt test */
6
+ last_name character varying(255) null,
7
+ ssn character varying(64) not null
8
+ );
9
+
10
+ drop table places;
11
+ create table places (
12
+ id SERIAL PRIMARY KEY,
13
+ address text,
14
+ city character varying(255),
15
+ state character varying(255),
16
+ country character varying(2)
17
+ );
18
+
19
+ drop table person_dimension;
20
+ create table person_dimension (
21
+ id SERIAL PRIMARY KEY,
22
+ first_name character varying(50),
23
+ last_name character varying(50),
24
+ address character varying(100),
25
+ city character varying(50),
26
+ state character varying(50),
27
+ zip_code character varying(20),
28
+ effective_date timestamp without time zone,
29
+ end_date timestamp without time zone,
30
+ latest_version boolean not null
31
+ );
32
+
33
+ drop table truncate_test;
34
+ create table truncate_test (
35
+ id SERIAL PRIMARY KEY,
36
+ x character varying(4)
37
+ );
38
+ insert into truncate_test (x) values ('a');
39
+ insert into truncate_test (x) values ('b');
40
+ insert into truncate_test (x) values ('c');
@@ -0,0 +1,43 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class ControlTest < Test::Unit::TestCase
4
+ # Test the ability to parse control files.
5
+ def test_parse
6
+ assert_nothing_raised do
7
+ Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
8
+ ETL::Control::Control.parse(f)
9
+ end
10
+ end
11
+ end
12
+
13
+ def test_bad_control_raises_error
14
+ assert_raise ETL::ControlError do
15
+ ETL::Control::Control.resolve(0)
16
+ end
17
+ end
18
+
19
+ def test_resolve_control_object
20
+ assert_nothing_raised do
21
+ ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
22
+ end
23
+ end
24
+
25
+ def test_set_error_threshold
26
+ assert_nothing_raised do
27
+ ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
28
+ end
29
+ end
30
+
31
+ def test_bad_processor_name
32
+ assert_raise ETL::ControlError do
33
+ s = "before_write :chunky_monkey"
34
+ ETL::Control::Control.parse_text(s)
35
+ end
36
+ end
37
+
38
+ def test_dependencies
39
+ s = "depends_on 'foo', 'bar'"
40
+ control = ETL::Control::Control.parse_text(s)
41
+ assert_equal control.dependencies, ['foo','bar']
42
+ end
43
+ end
@@ -0,0 +1,3 @@
1
+ 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
2
+ 127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
3
+ 127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"
@@ -0,0 +1,3 @@
1
+ 1,Chris,Smith,111223333
2
+ 2,Jim,Foxworthy,444332222
3
+ 3,Brian,Collingsworth,123443435