activewarehouse-etl-sgonyea 0.9.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. data/.gitignore +9 -0
  2. data/0.9-UPGRADE +6 -0
  3. data/CHANGELOG +236 -0
  4. data/Gemfile +4 -0
  5. data/HOW_TO_RELEASE +13 -0
  6. data/LICENSE +7 -0
  7. data/README.textile +111 -0
  8. data/Rakefile +103 -0
  9. data/TODO +28 -0
  10. data/active_support_logger.patch +78 -0
  11. data/activewarehouse-etl.gemspec +36 -0
  12. data/bin/etl +28 -0
  13. data/bin/etl.cmd +8 -0
  14. data/examples/database.example.yml +16 -0
  15. data/lib/etl.rb +97 -0
  16. data/lib/etl/batch.rb +2 -0
  17. data/lib/etl/batch/batch.rb +111 -0
  18. data/lib/etl/batch/directives.rb +65 -0
  19. data/lib/etl/builder.rb +2 -0
  20. data/lib/etl/builder/date_dimension_builder.rb +96 -0
  21. data/lib/etl/builder/time_dimension_builder.rb +31 -0
  22. data/lib/etl/commands/etl.rb +89 -0
  23. data/lib/etl/control.rb +3 -0
  24. data/lib/etl/control/control.rb +405 -0
  25. data/lib/etl/control/destination.rb +438 -0
  26. data/lib/etl/control/destination/csv_destination.rb +113 -0
  27. data/lib/etl/control/destination/database_destination.rb +97 -0
  28. data/lib/etl/control/destination/excel_destination.rb +91 -0
  29. data/lib/etl/control/destination/file_destination.rb +126 -0
  30. data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
  31. data/lib/etl/control/destination/update_database_destination.rb +109 -0
  32. data/lib/etl/control/destination/yaml_destination.rb +74 -0
  33. data/lib/etl/control/source.rb +132 -0
  34. data/lib/etl/control/source/database_source.rb +224 -0
  35. data/lib/etl/control/source/enumerable_source.rb +11 -0
  36. data/lib/etl/control/source/file_source.rb +90 -0
  37. data/lib/etl/control/source/model_source.rb +39 -0
  38. data/lib/etl/core_ext.rb +1 -0
  39. data/lib/etl/core_ext/time.rb +5 -0
  40. data/lib/etl/core_ext/time/calculations.rb +42 -0
  41. data/lib/etl/engine.rb +582 -0
  42. data/lib/etl/execution.rb +19 -0
  43. data/lib/etl/execution/base.rb +8 -0
  44. data/lib/etl/execution/batch.rb +10 -0
  45. data/lib/etl/execution/job.rb +8 -0
  46. data/lib/etl/execution/migration.rb +90 -0
  47. data/lib/etl/generator.rb +2 -0
  48. data/lib/etl/generator/generator.rb +20 -0
  49. data/lib/etl/generator/surrogate_key_generator.rb +39 -0
  50. data/lib/etl/http_tools.rb +139 -0
  51. data/lib/etl/parser.rb +11 -0
  52. data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
  53. data/lib/etl/parser/csv_parser.rb +93 -0
  54. data/lib/etl/parser/excel_parser.rb +112 -0
  55. data/lib/etl/parser/fixed_width_parser.rb +65 -0
  56. data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
  57. data/lib/etl/parser/parser.rb +41 -0
  58. data/lib/etl/parser/sax_parser.rb +218 -0
  59. data/lib/etl/parser/xml_parser.rb +65 -0
  60. data/lib/etl/processor.rb +11 -0
  61. data/lib/etl/processor/block_processor.rb +14 -0
  62. data/lib/etl/processor/bulk_import_processor.rb +94 -0
  63. data/lib/etl/processor/check_exist_processor.rb +80 -0
  64. data/lib/etl/processor/check_unique_processor.rb +39 -0
  65. data/lib/etl/processor/copy_field_processor.rb +26 -0
  66. data/lib/etl/processor/database_join_processor.rb +82 -0
  67. data/lib/etl/processor/encode_processor.rb +55 -0
  68. data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
  69. data/lib/etl/processor/escape_csv_processor.rb +77 -0
  70. data/lib/etl/processor/filter_row_processor.rb +51 -0
  71. data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
  72. data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
  73. data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
  74. data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
  75. data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
  76. data/lib/etl/processor/print_row_processor.rb +12 -0
  77. data/lib/etl/processor/processor.rb +25 -0
  78. data/lib/etl/processor/rename_processor.rb +24 -0
  79. data/lib/etl/processor/require_non_blank_processor.rb +26 -0
  80. data/lib/etl/processor/row_processor.rb +27 -0
  81. data/lib/etl/processor/sequence_processor.rb +23 -0
  82. data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
  83. data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
  84. data/lib/etl/processor/surrogate_key_processor.rb +53 -0
  85. data/lib/etl/processor/truncate_processor.rb +40 -0
  86. data/lib/etl/processor/zip_file_processor.rb +27 -0
  87. data/lib/etl/row.rb +20 -0
  88. data/lib/etl/screen.rb +14 -0
  89. data/lib/etl/screen/row_count_screen.rb +20 -0
  90. data/lib/etl/transform.rb +2 -0
  91. data/lib/etl/transform/block_transform.rb +13 -0
  92. data/lib/etl/transform/calculation_transform.rb +71 -0
  93. data/lib/etl/transform/date_to_string_transform.rb +20 -0
  94. data/lib/etl/transform/decode_transform.rb +51 -0
  95. data/lib/etl/transform/default_transform.rb +20 -0
  96. data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
  97. data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
  98. data/lib/etl/transform/md5_transform.rb +13 -0
  99. data/lib/etl/transform/ordinalize_transform.rb +14 -0
  100. data/lib/etl/transform/sha1_transform.rb +13 -0
  101. data/lib/etl/transform/split_fields_transform.rb +27 -0
  102. data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
  103. data/lib/etl/transform/string_to_date_transform.rb +16 -0
  104. data/lib/etl/transform/string_to_time_transform.rb +11 -0
  105. data/lib/etl/transform/transform.rb +61 -0
  106. data/lib/etl/transform/trim_transform.rb +26 -0
  107. data/lib/etl/transform/type_transform.rb +35 -0
  108. data/lib/etl/util.rb +59 -0
  109. data/lib/etl/version.rb +3 -0
  110. data/test-matrix.yml +10 -0
  111. data/test/.gitignore +1 -0
  112. data/test/.ignore +2 -0
  113. data/test/all.ebf +6 -0
  114. data/test/apache_combined_log.ctl +11 -0
  115. data/test/batch_test.rb +41 -0
  116. data/test/batch_with_error.ebf +6 -0
  117. data/test/batched1.ctl +0 -0
  118. data/test/batched2.ctl +0 -0
  119. data/test/block_processor.ctl +6 -0
  120. data/test/block_processor_error.ctl +1 -0
  121. data/test/block_processor_pre_post_process.ctl +4 -0
  122. data/test/block_processor_remove_rows.ctl +5 -0
  123. data/test/block_processor_test.rb +38 -0
  124. data/test/check_exist_processor_test.rb +92 -0
  125. data/test/check_unique_processor_test.rb +40 -0
  126. data/test/config/Gemfile.rails-2.3.x +3 -0
  127. data/test/config/Gemfile.rails-2.3.x.lock +53 -0
  128. data/test/config/Gemfile.rails-3.0.x +3 -0
  129. data/test/config/Gemfile.rails-3.0.x.lock +61 -0
  130. data/test/config/common.rb +29 -0
  131. data/test/connection/mysql/connection.rb +9 -0
  132. data/test/connection/mysql/schema.sql +37 -0
  133. data/test/connection/postgresql/connection.rb +13 -0
  134. data/test/connection/postgresql/schema.sql +40 -0
  135. data/test/control_test.rb +43 -0
  136. data/test/data/apache_combined_log.txt +3 -0
  137. data/test/data/bulk_import.txt +3 -0
  138. data/test/data/bulk_import_with_empties.txt +3 -0
  139. data/test/data/decode.txt +3 -0
  140. data/test/data/delimited.txt +3 -0
  141. data/test/data/encode_source_latin1.txt +2 -0
  142. data/test/data/excel.xls +0 -0
  143. data/test/data/excel2.xls +0 -0
  144. data/test/data/fixed_width.txt +3 -0
  145. data/test/data/multiple_delimited_1.txt +3 -0
  146. data/test/data/multiple_delimited_2.txt +3 -0
  147. data/test/data/nokogiri.xml +38 -0
  148. data/test/data/people.txt +3 -0
  149. data/test/data/sax.xml +14 -0
  150. data/test/data/xml.xml +16 -0
  151. data/test/database_join_processor_test.rb +43 -0
  152. data/test/date_dimension_builder_test.rb +96 -0
  153. data/test/delimited.ctl +30 -0
  154. data/test/delimited_absolute.ctl +31 -0
  155. data/test/delimited_destination_db.ctl +23 -0
  156. data/test/delimited_excel.ctl +31 -0
  157. data/test/delimited_insert_update.ctl +34 -0
  158. data/test/delimited_update.ctl +34 -0
  159. data/test/delimited_with_bulk_load.ctl +34 -0
  160. data/test/destination_test.rb +275 -0
  161. data/test/directive_test.rb +23 -0
  162. data/test/encode_processor_test.rb +32 -0
  163. data/test/engine_test.rb +78 -0
  164. data/test/ensure_fields_presence_processor_test.rb +28 -0
  165. data/test/errors.ctl +24 -0
  166. data/test/etl_test.rb +42 -0
  167. data/test/excel.ctl +24 -0
  168. data/test/excel2.ctl +25 -0
  169. data/test/fixed_width.ctl +35 -0
  170. data/test/foreign_key_lookup_transform_test.rb +50 -0
  171. data/test/generator_test.rb +14 -0
  172. data/test/inline_parser.ctl +17 -0
  173. data/test/mocks/mock_destination.rb +26 -0
  174. data/test/mocks/mock_source.rb +25 -0
  175. data/test/model_source.ctl +14 -0
  176. data/test/multiple_delimited.ctl +22 -0
  177. data/test/multiple_source_delimited.ctl +39 -0
  178. data/test/nokogiri_all.ctl +35 -0
  179. data/test/nokogiri_select.ctl +35 -0
  180. data/test/nokogiri_test.rb +35 -0
  181. data/test/parser_test.rb +224 -0
  182. data/test/performance/delimited.ctl +30 -0
  183. data/test/processor_test.rb +44 -0
  184. data/test/row_processor_test.rb +17 -0
  185. data/test/sax.ctl +26 -0
  186. data/test/scd/1.txt +1 -0
  187. data/test/scd/2.txt +1 -0
  188. data/test/scd/3.txt +1 -0
  189. data/test/scd_test.rb +257 -0
  190. data/test/scd_test_type_1.ctl +43 -0
  191. data/test/scd_test_type_2.ctl +34 -0
  192. data/test/screen_test.rb +9 -0
  193. data/test/screen_test_error.ctl +3 -0
  194. data/test/screen_test_fatal.ctl +3 -0
  195. data/test/source_test.rb +154 -0
  196. data/test/test_helper.rb +37 -0
  197. data/test/transform_test.rb +101 -0
  198. data/test/truncate_processor_test.rb +37 -0
  199. data/test/xml.ctl +31 -0
  200. metadata +370 -0
@@ -0,0 +1,92 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class Person < ActiveRecord::Base
4
+ end
5
+
6
+ class CheckExistProcessorTest < Test::Unit::TestCase
7
+
8
+ context 'CheckExistProcessor' do
9
+
10
+ setup do
11
+ @config = {
12
+ :target => :data_warehouse,
13
+ :table => 'people',
14
+ :columns => [:first_name, :last_name]
15
+ }
16
+ end
17
+
18
+ should_eventually "compare based on all columns if no columns are provided" do
19
+ # TBI
20
+ end
21
+
22
+ should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
23
+ # TBI
24
+ end
25
+
26
+ should "raise an error if no table is provided" do
27
+ error = assert_raises(ETL::ControlError) do
28
+ ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
29
+ end
30
+ # bug #2413 on assert_raises won't let me check error message above
31
+ assert_equal 'table must be specified', error.message
32
+ end
33
+
34
+ should "raise an error if no target is provided" do
35
+ error = assert_raises(ETL::ControlError) do
36
+ ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
37
+ end
38
+
39
+ assert_equal 'target must be specified', error.message
40
+ end
41
+
42
+ should "bypass checking if the table has no rows" do
43
+ Person.delete_all
44
+
45
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
46
+ assert_equal false, processor.should_check?
47
+ end
48
+
49
+ should "raise an error if one of the keys used for checking existence is not available in a row" do
50
+ Person.delete_all
51
+ # we need at least one record to avoid automatic skipping
52
+ # this should be mocked instead, probably
53
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
54
+
55
+ error = assert_raise(ETL::ControlError) do
56
+ row = ETL::Row[:first_name => 'John']
57
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
58
+
59
+ # guard against bypassing
60
+ assert_equal true, processor.should_check?
61
+
62
+ processor.process(row)
63
+ end
64
+
65
+ assert_equal "Row missing required field :last_name for existence check", error.message
66
+ end
67
+
68
+ should "return nil if the same row is found in database" do
69
+ Person.delete_all
70
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
71
+
72
+ row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
73
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
74
+ assert_equal true, processor.should_check? # guard against bypassing
75
+
76
+ assert_equal nil, processor.process(row)
77
+ end
78
+
79
+ should "return the row if no same row is found in database" do
80
+ Person.delete_all
81
+ Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
82
+
83
+ row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
84
+ processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
85
+ assert_equal true, processor.should_check? # guard against bypassing
86
+
87
+ assert_equal row, processor.process(row)
88
+ end
89
+
90
+ end
91
+
92
+ end
@@ -0,0 +1,40 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class CheckUniqueProcessorTest < Test::Unit::TestCase
4
+
5
+ context 'CheckUniqueProcessor' do
6
+ attr_reader :processor
7
+
8
+ setup do
9
+ @processor = ETL::Processor::CheckUniqueProcessor.new(nil,
10
+ :keys => [:first, :second])
11
+ end
12
+
13
+ should "keep a row whose keys didn't already appear in the pipeline" do
14
+ row = ETL::Row[:first => 'A', :second => 'B']
15
+
16
+ assert_equal row, processor.process(row)
17
+
18
+ assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
19
+ end
20
+
21
+ should "remove a row whose keys already appeared in the pipeline" do
22
+ row = ETL::Row[:first => 'A', :second => 'B']
23
+
24
+ assert_equal row, processor.process(row)
25
+ assert_equal nil, processor.process(row)
26
+ end
27
+
28
+ should "raise an error if a row lacks one of the keys specified" do
29
+ row = ETL::Row[:first => 'A']
30
+
31
+ error = assert_raises(ETL::ControlError) do
32
+ processor.process(row)
33
+ end
34
+
35
+ assert_equal "Row missing required field :second for unicity check", error.message
36
+ end
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,3 @@
1
+ require File.dirname(__FILE__) + '/common'
2
+
3
+ common_gemfile('2.3.11')
@@ -0,0 +1,53 @@
1
+ GIT
2
+ remote: git@github.com:activewarehouse/adapter_extensions.git
3
+ revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
4
+ specs:
5
+ adapter_extensions (0.9.5.rc1)
6
+ activerecord (>= 2.1.0)
7
+ activesupport (>= 2.1.0)
8
+ rake (>= 0.8.3)
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ activerecord (2.3.11)
14
+ activesupport (= 2.3.11)
15
+ activesupport (2.3.11)
16
+ fastercsv (1.5.4)
17
+ flexmock (0.9.0)
18
+ mysql (2.8.1)
19
+ net-sftp (2.0.5)
20
+ net-ssh (>= 2.0.9)
21
+ net-ssh (2.1.4)
22
+ nokogiri (1.4.4)
23
+ nokogiri (1.4.4-java)
24
+ weakling (>= 0.0.3)
25
+ pg (0.11.0)
26
+ rdoc (3.6.1)
27
+ ruby-ole (1.2.11.1)
28
+ shoulda (2.11.3)
29
+ spreadsheet (0.6.5.4)
30
+ ruby-ole (>= 1.0)
31
+ tmail (1.2.7.1)
32
+ weakling (0.0.4-java)
33
+ zip (2.0.2)
34
+
35
+ PLATFORMS
36
+ java
37
+ ruby
38
+
39
+ DEPENDENCIES
40
+ activerecord (= 2.3.11)
41
+ activesupport (= 2.3.11)
42
+ adapter_extensions!
43
+ fastercsv (= 1.5.4)
44
+ flexmock (= 0.9.0)
45
+ mysql (= 2.8.1)
46
+ net-sftp (= 2.0.5)
47
+ nokogiri (= 1.4.4)
48
+ pg (= 0.11.0)
49
+ rdoc
50
+ shoulda (= 2.11.3)
51
+ spreadsheet (= 0.6.5.4)
52
+ tmail (= 1.2.7.1)
53
+ zip (= 2.0.2)
@@ -0,0 +1,3 @@
1
+ require File.dirname(__FILE__) + '/common'
2
+
3
+ common_gemfile('3.0.7')
@@ -0,0 +1,61 @@
1
+ GIT
2
+ remote: git@github.com:activewarehouse/adapter_extensions.git
3
+ revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
4
+ specs:
5
+ adapter_extensions (0.9.5.rc1)
6
+ activerecord (>= 2.1.0)
7
+ activesupport (>= 2.1.0)
8
+ rake (>= 0.8.3)
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ activemodel (3.0.7)
14
+ activesupport (= 3.0.7)
15
+ builder (~> 2.1.2)
16
+ i18n (~> 0.5.0)
17
+ activerecord (3.0.7)
18
+ activemodel (= 3.0.7)
19
+ activesupport (= 3.0.7)
20
+ arel (~> 2.0.2)
21
+ tzinfo (~> 0.3.23)
22
+ activesupport (3.0.7)
23
+ arel (2.0.10)
24
+ builder (2.1.2)
25
+ fastercsv (1.5.4)
26
+ flexmock (0.9.0)
27
+ i18n (0.5.0)
28
+ mysql (2.8.1)
29
+ net-sftp (2.0.5)
30
+ net-ssh (>= 2.0.9)
31
+ net-ssh (2.1.4)
32
+ nokogiri (1.4.4)
33
+ pg (0.11.0)
34
+ rdoc (3.6.1)
35
+ ruby-ole (1.2.11.1)
36
+ shoulda (2.11.3)
37
+ spreadsheet (0.6.5.4)
38
+ ruby-ole (>= 1.0)
39
+ tmail (1.2.7.1)
40
+ tzinfo (0.3.27)
41
+ zip (2.0.2)
42
+
43
+ PLATFORMS
44
+ java
45
+ ruby
46
+
47
+ DEPENDENCIES
48
+ activerecord (= 3.0.7)
49
+ activesupport (= 3.0.7)
50
+ adapter_extensions!
51
+ fastercsv (= 1.5.4)
52
+ flexmock (= 0.9.0)
53
+ mysql (= 2.8.1)
54
+ net-sftp (= 2.0.5)
55
+ nokogiri (= 1.4.4)
56
+ pg (= 0.11.0)
57
+ rdoc
58
+ shoulda (= 2.11.3)
59
+ spreadsheet (= 0.6.5.4)
60
+ tmail (= 1.2.7.1)
61
+ zip (= 2.0.2)
@@ -0,0 +1,29 @@
1
+ require 'rbconfig'
2
+
3
+ def common_gemfile(rails_version)
4
+ source :rubygems
5
+
6
+ # using explicit versions for the gems to avoid any weirdness later on
7
+ gem "activesupport", rails_version
8
+ gem "activerecord", rails_version
9
+
10
+ gem "fastercsv", "1.5.4"
11
+ gem "spreadsheet", "0.6.5.4"
12
+ gem "tmail", "1.2.7.1"
13
+ gem "net-sftp", "2.0.5"
14
+ gem "zip", "2.0.2"
15
+
16
+ gem "shoulda", "2.11.3"
17
+ gem "flexmock", "0.9.0"
18
+
19
+ gem "mysql", "2.8.1"
20
+ gem "pg", "0.11.0"
21
+
22
+ gem "nokogiri", "1.4.4"
23
+
24
+ gem "rdoc"
25
+
26
+ gem "adapter_extensions", :git => 'git@github.com:activewarehouse/adapter_extensions.git'
27
+
28
+ gem "jruby-openssl" if RUBY_PLATFORM == "java"
29
+ end
@@ -0,0 +1,9 @@
1
+ print "Using native MySQL\n"
2
+
3
+ puts "Resetting database"
4
+ conn = ETL::Engine.connection(:data_warehouse)
5
+ conn.recreate_database(conn.current_database)
6
+ conn.reconnect!
7
+ lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
8
+ lines.join.split(';').each { |line| conn.execute(line) }
9
+ conn.disconnect!
@@ -0,0 +1,37 @@
1
+ drop table if exists people;
2
+ create table people (
3
+ id int not null primary key,
4
+ first_name char(255) not null,
5
+ last_name char(255) not null,
6
+ ssn char(64) not null
7
+ );
8
+ drop table if exists places;
9
+ create table places (
10
+ address text,
11
+ city char(255),
12
+ state char(255),
13
+ country char(2)
14
+ );
15
+
16
+ drop table if exists person_dimension;
17
+ create table person_dimension (
18
+ id int not null primary key,
19
+ first_name char(50),
20
+ last_name char(50),
21
+ address char(100),
22
+ city char(50),
23
+ state char(50),
24
+ zip_code char(20),
25
+ effective_date datetime,
26
+ end_date datetime,
27
+ latest_version boolean not null
28
+ );
29
+
30
+ drop table if exists truncate_test;
31
+ create table truncate_test (
32
+ id int not null primary key auto_increment,
33
+ x char(4)
34
+ );
35
+ insert into truncate_test (x) values ('a');
36
+ insert into truncate_test (x) values ('b');
37
+ insert into truncate_test (x) values ('c');
@@ -0,0 +1,13 @@
1
+ print "Using PostgreSQL\n"
2
+
3
+ puts "Resetting database"
4
+ conn = ETL::Engine.connection(:data_warehouse)
5
+
6
+ lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
7
+ lines.join.split(';').each_with_index do |line, index|
8
+ begin
9
+ conn.execute(line)
10
+ rescue => e
11
+ puts "failed to load line #{index}: #{e}"
12
+ end
13
+ end
@@ -0,0 +1,40 @@
1
+ drop table people;
2
+ create table people (
3
+ id SERIAL PRIMARY KEY,
4
+ first_name character varying(255) not null,
5
+ /* null below allowed for bulk_import_with_empties.txt test */
6
+ last_name character varying(255) null,
7
+ ssn character varying(64) not null
8
+ );
9
+
10
+ drop table places;
11
+ create table places (
12
+ id SERIAL PRIMARY KEY,
13
+ address text,
14
+ city character varying(255),
15
+ state character varying(255),
16
+ country character varying(2)
17
+ );
18
+
19
+ drop table person_dimension;
20
+ create table person_dimension (
21
+ id SERIAL PRIMARY KEY,
22
+ first_name character varying(50),
23
+ last_name character varying(50),
24
+ address character varying(100),
25
+ city character varying(50),
26
+ state character varying(50),
27
+ zip_code character varying(20),
28
+ effective_date timestamp without time zone,
29
+ end_date timestamp without time zone,
30
+ latest_version boolean not null
31
+ );
32
+
33
+ drop table truncate_test;
34
+ create table truncate_test (
35
+ id SERIAL PRIMARY KEY,
36
+ x character varying(4)
37
+ );
38
+ insert into truncate_test (x) values ('a');
39
+ insert into truncate_test (x) values ('b');
40
+ insert into truncate_test (x) values ('c');
@@ -0,0 +1,43 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class ControlTest < Test::Unit::TestCase
4
+ # Test the ability to parse control files.
5
+ def test_parse
6
+ assert_nothing_raised do
7
+ Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
8
+ ETL::Control::Control.parse(f)
9
+ end
10
+ end
11
+ end
12
+
13
+ def test_bad_control_raises_error
14
+ assert_raise ETL::ControlError do
15
+ ETL::Control::Control.resolve(0)
16
+ end
17
+ end
18
+
19
+ def test_resolve_control_object
20
+ assert_nothing_raised do
21
+ ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
22
+ end
23
+ end
24
+
25
+ def test_set_error_threshold
26
+ assert_nothing_raised do
27
+ ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
28
+ end
29
+ end
30
+
31
+ def test_bad_processor_name
32
+ assert_raise ETL::ControlError do
33
+ s = "before_write :chunky_monkey"
34
+ ETL::Control::Control.parse_text(s)
35
+ end
36
+ end
37
+
38
+ def test_dependencies
39
+ s = "depends_on 'foo', 'bar'"
40
+ control = ETL::Control::Control.parse_text(s)
41
+ assert_equal control.dependencies, ['foo','bar']
42
+ end
43
+ end
@@ -0,0 +1,3 @@
1
+ 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
2
+ 127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
3
+ 127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"
@@ -0,0 +1,3 @@
1
+ 1,Chris,Smith,111223333
2
+ 2,Jim,Foxworthy,444332222
3
+ 3,Brian,Collingsworth,123443435