activewarehouse-etl-sgonyea 0.9.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (200) hide show
  1. data/.gitignore +9 -0
  2. data/0.9-UPGRADE +6 -0
  3. data/CHANGELOG +236 -0
  4. data/Gemfile +4 -0
  5. data/HOW_TO_RELEASE +13 -0
  6. data/LICENSE +7 -0
  7. data/README.textile +111 -0
  8. data/Rakefile +103 -0
  9. data/TODO +28 -0
  10. data/active_support_logger.patch +78 -0
  11. data/activewarehouse-etl.gemspec +36 -0
  12. data/bin/etl +28 -0
  13. data/bin/etl.cmd +8 -0
  14. data/examples/database.example.yml +16 -0
  15. data/lib/etl.rb +97 -0
  16. data/lib/etl/batch.rb +2 -0
  17. data/lib/etl/batch/batch.rb +111 -0
  18. data/lib/etl/batch/directives.rb +65 -0
  19. data/lib/etl/builder.rb +2 -0
  20. data/lib/etl/builder/date_dimension_builder.rb +96 -0
  21. data/lib/etl/builder/time_dimension_builder.rb +31 -0
  22. data/lib/etl/commands/etl.rb +89 -0
  23. data/lib/etl/control.rb +3 -0
  24. data/lib/etl/control/control.rb +405 -0
  25. data/lib/etl/control/destination.rb +438 -0
  26. data/lib/etl/control/destination/csv_destination.rb +113 -0
  27. data/lib/etl/control/destination/database_destination.rb +97 -0
  28. data/lib/etl/control/destination/excel_destination.rb +91 -0
  29. data/lib/etl/control/destination/file_destination.rb +126 -0
  30. data/lib/etl/control/destination/insert_update_database_destination.rb +136 -0
  31. data/lib/etl/control/destination/update_database_destination.rb +109 -0
  32. data/lib/etl/control/destination/yaml_destination.rb +74 -0
  33. data/lib/etl/control/source.rb +132 -0
  34. data/lib/etl/control/source/database_source.rb +224 -0
  35. data/lib/etl/control/source/enumerable_source.rb +11 -0
  36. data/lib/etl/control/source/file_source.rb +90 -0
  37. data/lib/etl/control/source/model_source.rb +39 -0
  38. data/lib/etl/core_ext.rb +1 -0
  39. data/lib/etl/core_ext/time.rb +5 -0
  40. data/lib/etl/core_ext/time/calculations.rb +42 -0
  41. data/lib/etl/engine.rb +582 -0
  42. data/lib/etl/execution.rb +19 -0
  43. data/lib/etl/execution/base.rb +8 -0
  44. data/lib/etl/execution/batch.rb +10 -0
  45. data/lib/etl/execution/job.rb +8 -0
  46. data/lib/etl/execution/migration.rb +90 -0
  47. data/lib/etl/generator.rb +2 -0
  48. data/lib/etl/generator/generator.rb +20 -0
  49. data/lib/etl/generator/surrogate_key_generator.rb +39 -0
  50. data/lib/etl/http_tools.rb +139 -0
  51. data/lib/etl/parser.rb +11 -0
  52. data/lib/etl/parser/apache_combined_log_parser.rb +49 -0
  53. data/lib/etl/parser/csv_parser.rb +93 -0
  54. data/lib/etl/parser/excel_parser.rb +112 -0
  55. data/lib/etl/parser/fixed_width_parser.rb +65 -0
  56. data/lib/etl/parser/nokogiri_xml_parser.rb +83 -0
  57. data/lib/etl/parser/parser.rb +41 -0
  58. data/lib/etl/parser/sax_parser.rb +218 -0
  59. data/lib/etl/parser/xml_parser.rb +65 -0
  60. data/lib/etl/processor.rb +11 -0
  61. data/lib/etl/processor/block_processor.rb +14 -0
  62. data/lib/etl/processor/bulk_import_processor.rb +94 -0
  63. data/lib/etl/processor/check_exist_processor.rb +80 -0
  64. data/lib/etl/processor/check_unique_processor.rb +39 -0
  65. data/lib/etl/processor/copy_field_processor.rb +26 -0
  66. data/lib/etl/processor/database_join_processor.rb +82 -0
  67. data/lib/etl/processor/encode_processor.rb +55 -0
  68. data/lib/etl/processor/ensure_fields_presence_processor.rb +24 -0
  69. data/lib/etl/processor/escape_csv_processor.rb +77 -0
  70. data/lib/etl/processor/filter_row_processor.rb +51 -0
  71. data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
  72. data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
  73. data/lib/etl/processor/hierarchy_exploder_processor.rb +55 -0
  74. data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
  75. data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
  76. data/lib/etl/processor/print_row_processor.rb +12 -0
  77. data/lib/etl/processor/processor.rb +25 -0
  78. data/lib/etl/processor/rename_processor.rb +24 -0
  79. data/lib/etl/processor/require_non_blank_processor.rb +26 -0
  80. data/lib/etl/processor/row_processor.rb +27 -0
  81. data/lib/etl/processor/sequence_processor.rb +23 -0
  82. data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
  83. data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
  84. data/lib/etl/processor/surrogate_key_processor.rb +53 -0
  85. data/lib/etl/processor/truncate_processor.rb +40 -0
  86. data/lib/etl/processor/zip_file_processor.rb +27 -0
  87. data/lib/etl/row.rb +20 -0
  88. data/lib/etl/screen.rb +14 -0
  89. data/lib/etl/screen/row_count_screen.rb +20 -0
  90. data/lib/etl/transform.rb +2 -0
  91. data/lib/etl/transform/block_transform.rb +13 -0
  92. data/lib/etl/transform/calculation_transform.rb +71 -0
  93. data/lib/etl/transform/date_to_string_transform.rb +20 -0
  94. data/lib/etl/transform/decode_transform.rb +51 -0
  95. data/lib/etl/transform/default_transform.rb +20 -0
  96. data/lib/etl/transform/foreign_key_lookup_transform.rb +211 -0
  97. data/lib/etl/transform/hierarchy_lookup_transform.rb +49 -0
  98. data/lib/etl/transform/md5_transform.rb +13 -0
  99. data/lib/etl/transform/ordinalize_transform.rb +14 -0
  100. data/lib/etl/transform/sha1_transform.rb +13 -0
  101. data/lib/etl/transform/split_fields_transform.rb +27 -0
  102. data/lib/etl/transform/string_to_date_time_transform.rb +14 -0
  103. data/lib/etl/transform/string_to_date_transform.rb +16 -0
  104. data/lib/etl/transform/string_to_time_transform.rb +11 -0
  105. data/lib/etl/transform/transform.rb +61 -0
  106. data/lib/etl/transform/trim_transform.rb +26 -0
  107. data/lib/etl/transform/type_transform.rb +35 -0
  108. data/lib/etl/util.rb +59 -0
  109. data/lib/etl/version.rb +3 -0
  110. data/test-matrix.yml +10 -0
  111. data/test/.gitignore +1 -0
  112. data/test/.ignore +2 -0
  113. data/test/all.ebf +6 -0
  114. data/test/apache_combined_log.ctl +11 -0
  115. data/test/batch_test.rb +41 -0
  116. data/test/batch_with_error.ebf +6 -0
  117. data/test/batched1.ctl +0 -0
  118. data/test/batched2.ctl +0 -0
  119. data/test/block_processor.ctl +6 -0
  120. data/test/block_processor_error.ctl +1 -0
  121. data/test/block_processor_pre_post_process.ctl +4 -0
  122. data/test/block_processor_remove_rows.ctl +5 -0
  123. data/test/block_processor_test.rb +38 -0
  124. data/test/check_exist_processor_test.rb +92 -0
  125. data/test/check_unique_processor_test.rb +40 -0
  126. data/test/config/Gemfile.rails-2.3.x +3 -0
  127. data/test/config/Gemfile.rails-2.3.x.lock +53 -0
  128. data/test/config/Gemfile.rails-3.0.x +3 -0
  129. data/test/config/Gemfile.rails-3.0.x.lock +61 -0
  130. data/test/config/common.rb +29 -0
  131. data/test/connection/mysql/connection.rb +9 -0
  132. data/test/connection/mysql/schema.sql +37 -0
  133. data/test/connection/postgresql/connection.rb +13 -0
  134. data/test/connection/postgresql/schema.sql +40 -0
  135. data/test/control_test.rb +43 -0
  136. data/test/data/apache_combined_log.txt +3 -0
  137. data/test/data/bulk_import.txt +3 -0
  138. data/test/data/bulk_import_with_empties.txt +3 -0
  139. data/test/data/decode.txt +3 -0
  140. data/test/data/delimited.txt +3 -0
  141. data/test/data/encode_source_latin1.txt +2 -0
  142. data/test/data/excel.xls +0 -0
  143. data/test/data/excel2.xls +0 -0
  144. data/test/data/fixed_width.txt +3 -0
  145. data/test/data/multiple_delimited_1.txt +3 -0
  146. data/test/data/multiple_delimited_2.txt +3 -0
  147. data/test/data/nokogiri.xml +38 -0
  148. data/test/data/people.txt +3 -0
  149. data/test/data/sax.xml +14 -0
  150. data/test/data/xml.xml +16 -0
  151. data/test/database_join_processor_test.rb +43 -0
  152. data/test/date_dimension_builder_test.rb +96 -0
  153. data/test/delimited.ctl +30 -0
  154. data/test/delimited_absolute.ctl +31 -0
  155. data/test/delimited_destination_db.ctl +23 -0
  156. data/test/delimited_excel.ctl +31 -0
  157. data/test/delimited_insert_update.ctl +34 -0
  158. data/test/delimited_update.ctl +34 -0
  159. data/test/delimited_with_bulk_load.ctl +34 -0
  160. data/test/destination_test.rb +275 -0
  161. data/test/directive_test.rb +23 -0
  162. data/test/encode_processor_test.rb +32 -0
  163. data/test/engine_test.rb +78 -0
  164. data/test/ensure_fields_presence_processor_test.rb +28 -0
  165. data/test/errors.ctl +24 -0
  166. data/test/etl_test.rb +42 -0
  167. data/test/excel.ctl +24 -0
  168. data/test/excel2.ctl +25 -0
  169. data/test/fixed_width.ctl +35 -0
  170. data/test/foreign_key_lookup_transform_test.rb +50 -0
  171. data/test/generator_test.rb +14 -0
  172. data/test/inline_parser.ctl +17 -0
  173. data/test/mocks/mock_destination.rb +26 -0
  174. data/test/mocks/mock_source.rb +25 -0
  175. data/test/model_source.ctl +14 -0
  176. data/test/multiple_delimited.ctl +22 -0
  177. data/test/multiple_source_delimited.ctl +39 -0
  178. data/test/nokogiri_all.ctl +35 -0
  179. data/test/nokogiri_select.ctl +35 -0
  180. data/test/nokogiri_test.rb +35 -0
  181. data/test/parser_test.rb +224 -0
  182. data/test/performance/delimited.ctl +30 -0
  183. data/test/processor_test.rb +44 -0
  184. data/test/row_processor_test.rb +17 -0
  185. data/test/sax.ctl +26 -0
  186. data/test/scd/1.txt +1 -0
  187. data/test/scd/2.txt +1 -0
  188. data/test/scd/3.txt +1 -0
  189. data/test/scd_test.rb +257 -0
  190. data/test/scd_test_type_1.ctl +43 -0
  191. data/test/scd_test_type_2.ctl +34 -0
  192. data/test/screen_test.rb +9 -0
  193. data/test/screen_test_error.ctl +3 -0
  194. data/test/screen_test_fatal.ctl +3 -0
  195. data/test/source_test.rb +154 -0
  196. data/test/test_helper.rb +37 -0
  197. data/test/transform_test.rb +101 -0
  198. data/test/truncate_processor_test.rb +37 -0
  199. data/test/xml.ctl +31 -0
  200. metadata +370 -0
@@ -0,0 +1,2 @@
1
+ require 'etl/builder/date_dimension_builder'
2
+ require 'etl/builder/time_dimension_builder'
@@ -0,0 +1,96 @@
1
+ module ETL #:nodoc:
2
+ module Builder #:nodoc:
3
+ # A builder which will build a data structure which can be used to populate a date dimension using
4
+ # commonly used date dimension columns.
5
+ class DateDimensionBuilder
6
+ # Specify the start date for the first record
7
+ attr_accessor :start_date
8
+
9
+ # Specify the end date for the last record
10
+ attr_accessor :end_date
11
+
12
+ # Define any holiday indicators
13
+ attr_accessor :holiday_indicators
14
+
15
+ # Add offset month for fiscal year
16
+ attr_accessor :fiscal_year_offset_month
17
+
18
+ # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
19
+ cattr_accessor :weekday_indicators
20
+ @@weekday_indicators = ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
21
+
22
+ # Initialize the builder.
23
+ #
24
+ # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
25
+ # * <tt>end_date</tt>: The end date. Defaults to now.
26
+ def initialize(start_date=Time.now.years_ago(5), end_date=Time.now, fiscal_year_offset_month=10)
27
+ @start_date = start_date.to_date
28
+ @end_date = end_date.to_date
29
+ @fiscal_year_offset_month = fiscal_year_offset_month.to_i
30
+ @holiday_indicators = []
31
+ end
32
+
33
+ # Returns an array of hashes representing records in the dimension.
34
+ def build(options={})
35
+ (start_date..end_date).map { |date| record_from_date(date) }
36
+ end
37
+
38
+ private
39
+
40
+ # Returns a hash representing a record in the dimension. The values for each record are
41
+ # accessed by name.
42
+ def record_from_date(date)
43
+ time = date.to_time # need methods only available in Time
44
+ record = {}
45
+ record[:date] = time.strftime("%m/%d/%Y")
46
+ record[:full_date_description] = time.strftime("%B %d,%Y")
47
+ record[:day_of_week] = time.strftime("%A")
48
+ record[:day_in_week] = record[:day_of_week] # alias
49
+ #record[:day_number_in_epoch] = time.to_i / 24
50
+ #record[:week_number_in_epoch] = time.to_i / (24 * 7)
51
+ #record[:month_number_in_epoch] = time.to_i / (24 * 7 * 30)
52
+ record[:day_number_in_calendar_month] = time.day
53
+ record[:day_number_in_calendar_year] = time.yday
54
+ record[:day_number_in_fiscal_month] = time.day # should this be different from CY?
55
+ record[:day_number_in_fiscal_year] = time.fiscal_year_yday(fiscal_year_offset_month)
56
+ #record[:last_day_in_week_indicator] =
57
+ #record[:last_day_in_month_indicator] =
58
+ #record[:calendar_week_ending_date] =
59
+ record[:calendar_week] = "Week #{time.week}"
60
+ record[:calendar_week_number] = time.week
61
+ record[:calendar_week_number_in_year] = time.week # DEPRECATED
62
+ record[:calendar_month_name] = time.strftime("%B")
63
+ record[:calendar_month_number_in_year] = time.month # DEPRECATED
64
+ record[:calendar_month_number] = time.month
65
+ record[:calendar_year_month] = time.strftime("%Y-%m")
66
+ record[:calendar_quarter] = "Q#{time.quarter}"
67
+ record[:calendar_quarter_number] = time.quarter
68
+ record[:calendar_quarter_number_in_year] = time.quarter # DEPRECATED
69
+ record[:calendar_year_quarter] = "#{time.strftime('%Y')}-#{record[:calendar_quarter]}"
70
+ #record[:calendar_half_year] =
71
+ record[:calendar_year] = "#{time.year}"
72
+ record[:fiscal_week] = "FY Week #{time.fiscal_year_week(fiscal_year_offset_month)}"
73
+ record[:fiscal_week_number_in_year] = time.fiscal_year_week(fiscal_year_offset_month) # DEPRECATED
74
+ record[:fiscal_week_number] = time.fiscal_year_week(fiscal_year_offset_month)
75
+ record[:fiscal_month] = time.fiscal_year_month(fiscal_year_offset_month)
76
+ record[:fiscal_month_number] = time.fiscal_year_month(fiscal_year_offset_month)
77
+ record[:fiscal_month_number_in_year] = time.fiscal_year_month(fiscal_year_offset_month) # DEPRECATED
78
+ record[:fiscal_year_month] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-" + time.fiscal_year_month(fiscal_year_offset_month).to_s.rjust(2, '0')
79
+ record[:fiscal_quarter] = "FY Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
80
+ record[:fiscal_year_quarter] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
81
+ record[:fiscal_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month) # DEPRECATED
82
+ record[:fiscal_year_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month)
83
+ #record[:fiscal_half_year] =
84
+ record[:fiscal_year] = "FY#{time.fiscal_year(fiscal_year_offset_month)}"
85
+ record[:fiscal_year_number] = time.fiscal_year(fiscal_year_offset_month)
86
+ record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
87
+ record[:weekday_indicator] = weekday_indicators[time.wday]
88
+ record[:selling_season] = 'None'
89
+ record[:major_event] = 'None'
90
+ record[:sql_date_stamp] = date
91
+
92
+ record
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,31 @@
1
+ module ETL #:nodoc:
2
+ module Builder #:nodoc:
3
+ # Builder that creates a simple time dimension.
4
+ class TimeDimensionBuilder
5
+ def initialize
6
+ # Returns an array of hashes representing records in the dimension. The values for each record are
7
+ # accessed by name.
8
+ def build(options={})
9
+ records = []
10
+ 0.upto(23) do |t_hour|
11
+ 0.upto(59) do |t_minute|
12
+ 0.upto(59) do |t_second|
13
+ t_hour_string = t_hour.to_s.rjust(2, '0')
14
+ t_minute_string = t_minute.to_s.rjust(2, '0')
15
+ t_second_string = t_second.to_s.rjust(2, '0')
16
+ record = {}
17
+ record[:hour] = t_hour
18
+ record[:minute] = t_minute
19
+ record[:second] = t_second
20
+ record[:minute_description] = "#{t_hour_string}:#{t_minute_string}"
21
+ record[:full_description] = "#{t_hour_string}:#{t_minute_string}:#{t_second_string}"
22
+ records << record
23
+ end
24
+ end
25
+ end
26
+ records
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,89 @@
1
+ #--
2
+ # Copyright (c) 2006 Anthony Eden
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #++
23
+
24
+ require 'benchmark'
25
+ require 'getoptlong'
26
+
27
+ # Print a usage statement
28
+ def usage #:nodoc:
29
+ puts "Usage: etl file [file file ...]" # TODO: add the command line options
30
+ end
31
+
32
+ def execute
33
+ opts = GetoptLong.new(
34
+ [ '--version', '-v', GetoptLong::NO_ARGUMENT],
35
+ [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
36
+ [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
37
+ [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
38
+ [ '--offset', '-o', GetoptLong::REQUIRED_ARGUMENT],
39
+ [ '--newlog', '-n', GetoptLong::NO_ARGUMENT ],
40
+ [ '--skip-bulk-import', '-s', GetoptLong::NO_ARGUMENT ],
41
+ [ '--read-locally', GetoptLong::NO_ARGUMENT],
42
+ [ '--rails-root', GetoptLong::REQUIRED_ARGUMENT]
43
+ )
44
+
45
+ options = {}
46
+ opts.each do |opt, arg|
47
+ case opt
48
+ when '--version'
49
+ puts "ActiveWarehouse ETL version #{ETL::VERSION::STRING}"
50
+ return
51
+ when '--help'
52
+ usage
53
+ return
54
+ when '--config'
55
+ options[:config] = arg
56
+ when '--limit'
57
+ options[:limit] = arg.to_i
58
+ when '--offset'
59
+ options[:offset] = arg.to_i
60
+ when '--newlog'
61
+ options[:newlog] = true
62
+ when '--skip-bulk-import'
63
+ puts "skip bulk import enabled"
64
+ options[:skip_bulk_import] = true
65
+ when '--read-locally'
66
+ puts "read locally enabled"
67
+ options[:read_locally] = true
68
+ when '--rails-root'
69
+ options[:rails_root] = arg
70
+ puts "rails root set to #{options[:rails_root]}"
71
+ end
72
+ end
73
+
74
+ if ARGV.length < 1
75
+ usage
76
+ else
77
+ puts "Starting ETL process"
78
+
79
+ ETL::Engine.init(options)
80
+ ARGV.each do |f|
81
+ ETL::Engine.realtime_activity = true
82
+ ETL::Engine.process(f)
83
+ end
84
+
85
+ puts "ETL process complete\n\n"
86
+ end
87
+ end
88
+
89
+ execute
@@ -0,0 +1,3 @@
1
+ require 'etl/control/control'
2
+ require 'etl/control/source'
3
+ require 'etl/control/destination'
@@ -0,0 +1,405 @@
1
+ module ETL #:nodoc:
2
+ module Control #:nodoc:
3
+ # The Context is passed to eval.
4
+ class Context
5
+ require 'test/unit/assertions'
6
+ include Test::Unit::Assertions
7
+ attr_reader :control
8
+
9
+ class << self
10
+ # Create a Context instance
11
+ def create(control)
12
+ Context.new(control).get_binding
13
+ end
14
+ end
15
+
16
+ # Initialize the context
17
+ def initialize(control)
18
+ @control = control
19
+ end
20
+
21
+ # Get the control file
22
+ def file
23
+ control.file
24
+ end
25
+
26
+ # Set the allowed error threshold
27
+ def set_error_threshold(error_threshold)
28
+ control.error_threshold = error_threshold
29
+ end
30
+
31
+ # Define a list of control files that this file depends on. Those control
32
+ # files will be executed prior to this control file. The list may
33
+ # contain symbols that will be converted to file names by calling
34
+ # to_s + '.ctl', or they may be strings in which case they will be used
35
+ # as is
36
+ def depends_on(*args)
37
+ (dependencies << args).flatten!
38
+ end
39
+
40
+ # Get the defined dependencies
41
+ def dependencies
42
+ control.dependencies
43
+ end
44
+
45
+ # Define a source.
46
+ def source(name, configuration={}, definition={})
47
+ if configuration[:type]
48
+ case configuration[:type]
49
+ when Class
50
+ source_class = configuration[:type]
51
+ sources << source_class.new(self, configuration, definition)
52
+ when String, Symbol
53
+ source_class = ETL::Control::Source.class_for_name(configuration[:type])
54
+ sources << source_class.new(self, configuration, definition)
55
+ else
56
+ if configuration[:type].is_a?(ETL::Control::Source)
57
+ sources << configuration[:type]
58
+ else
59
+ raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Source"
60
+ end
61
+ end
62
+ else
63
+ source_types.each do |source_type|
64
+ if configuration[source_type]
65
+ source_class = ETL::Control::Source.class_for_name(source_type)
66
+ sources << source_class.new(self, configuration, definition)
67
+ break
68
+ end
69
+ end
70
+ raise ControlError, "A source was specified but no matching type was found" if sources.empty?
71
+ end
72
+ end
73
+
74
+ # Get the defined source
75
+ def sources
76
+ control.sources
77
+ end
78
+
79
+ # Define a destination
80
+ def destination(name, configuration={}, mapping={})
81
+ if configuration[:type]
82
+ case configuration[:type]
83
+ when Class
84
+ dest_class = configuration[:type]
85
+ destinations << dest_class.new(self, configuration, mapping)
86
+ when String, Symbol
87
+ dest_class = ETL::Control::Destination.class_for_name(configuration[:type])
88
+ destinations << dest_class.new(self, configuration, mapping)
89
+ else
90
+ if configuration[:type].is_a?(ETL::Control::Destination)
91
+ destinations << configuration[:type]
92
+ else
93
+ raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Destination"
94
+ end
95
+ end
96
+ else
97
+ destination_types.each do |dest_type|
98
+ if configuration[dest_type]
99
+ dest_class = ETL::Control::Destination.class_for_name(dest_type)
100
+ destinations << dest_class.new(self, configuration, mapping)
101
+ break
102
+ end
103
+ end
104
+ raise ControlError, "A destination was specified but no matching destination type was found" if destinations.empty?
105
+ end
106
+ end
107
+
108
+ # Get the defined destinations
109
+ def destinations
110
+ control.destinations
111
+ end
112
+
113
+ # Define a transform
114
+ def transform(name, transformer=nil, configuration={}, &block)
115
+ if transformer
116
+ case transformer
117
+ when String, Symbol
118
+ class_name = "#{transformer.to_s.camelize}Transform"
119
+ begin
120
+ transform_class = ETL::Transform.const_get(class_name)
121
+ transforms << transform_class.new(self, name, configuration)
122
+ rescue NameError => e
123
+ raise ControlError, "Unable to find transformer #{class_name}: #{e}"
124
+ end
125
+ when Class
126
+ transforms << transformer.new(self, transformer.name, configuration)
127
+ else
128
+ #transformer.class.inspect
129
+ if transformer.is_a?(ETL::Transform::Transform)
130
+ Engine.logger.debug "Adding transformer #{transformer.inspect} for field #{name}"
131
+ t = transformer.dup
132
+ t.name = name
133
+ transforms << t
134
+ else
135
+ raise ControlError, "Transformer must be a String, Symbol, Class or Transform instance"
136
+ end
137
+ end
138
+ elsif block_given?
139
+ transforms << ETL::Transform::BlockTransform.new(self, name, :block => block)
140
+ else
141
+ raise ControlError, "Either a transformer or a block must be specified"
142
+ end
143
+ end
144
+
145
+ # Get the defined transforms
146
+ def transforms
147
+ control.transforms
148
+ end
149
+
150
+ # Define a before post-process screen block. The type argument must be
151
+ # one of :fatal, :error or :warn
152
+ def screen(type, &block)
153
+ screens[type] << block
154
+ end
155
+
156
+ # Get the before post-process screen blocks
157
+ def screens
158
+ control.screens
159
+ end
160
+
161
+ # Define an after post-proces screen block. The type argument must be
162
+ # one of :fatal, :error or :warn
163
+ def after_post_process_screen(type, &block)
164
+ after_post_process_screens[type] << block
165
+ end
166
+
167
+ # Get the after post-process screen blocks
168
+ def after_post_process_screens
169
+ control.after_post_process_screens
170
+ end
171
+
172
+ # Rename the source field to the destination field
173
+ def rename(source, destination)
174
+ after_read :rename, :source => source, :dest => destination
175
+ end
176
+
177
+ # Copy the source field to the destination field
178
+ def copy(source, destination)
179
+ after_read :copy_field, :source => source, :dest => destination
180
+ end
181
+
182
+ protected
183
+ # This method is used to define a processor and insert into the specified processor
184
+ # collection.
185
+ def define_processor(name, processor_collection, configuration, proc)
186
+ case name
187
+ when String, Symbol, nil
188
+ name ||= 'block'
189
+ class_name = "#{name.to_s.camelize}Processor"
190
+ begin
191
+ processor_class = ETL::Processor.const_get(class_name)
192
+ if name == 'block'
193
+ raise ControlError, "A block must be passed for block processor" if proc.nil?
194
+ configuration[:block] = proc
195
+ end
196
+ processor_collection << processor_class.new(self, configuration)
197
+ rescue NameError => e
198
+ raise ControlError, "Unable to find processor #{class_name}: #{e}"
199
+ end
200
+ when Class
201
+ processor_collection << name.new(self, configuration)
202
+ else
203
+ raise ControlError, "The process declaration requires a String, Symbol, Class, or a Block to be passed"
204
+ end
205
+ end
206
+
207
+ public
208
+ # Define an "after read" processor. This must be a row-level processor.
209
+ def after_read(name='block', configuration={}, &block)
210
+ define_processor(name, after_read_processors, configuration, block)
211
+ end
212
+
213
+ # Get the defined "after read" processors
214
+ def after_read_processors
215
+ control.after_read_processors
216
+ end
217
+
218
+ # Define a "before write" processor. This must be a row-level processor.
219
+ def before_write(name='block', configuration={}, &block)
220
+ define_processor(name, before_write_processors, configuration, block)
221
+ end
222
+
223
+ # Get the defined "before write" processors
224
+ def before_write_processors
225
+ control.before_write_processors
226
+ end
227
+
228
+ # Define a pre-processor
229
+ def pre_process(name='block', configuration={}, &block)
230
+ define_processor(name, pre_processors, configuration, block)
231
+ end
232
+
233
+ # Get the defined pre-processors
234
+ def pre_processors
235
+ control.pre_processors
236
+ end
237
+
238
+ # Define a post-processor
239
+ def post_process(name='block', configuration={}, &block)
240
+ define_processor(name, post_processors, configuration, block)
241
+ end
242
+
243
+ # Get the defined post-processors
244
+ def post_processors
245
+ control.post_processors
246
+ end
247
+
248
+ # Get the binding object
249
+ def get_binding
250
+ binding
251
+ end
252
+
253
+ protected
254
+ # Get an array of supported source types
255
+ def source_types
256
+ control.source_types
257
+ end
258
+
259
+ # Get an array of supported destination types
260
+ def destination_types
261
+ control.destination_types
262
+ end
263
+
264
+ end
265
+
266
+ # Object representation of a control file
267
+ class Control
268
+ # The File object
269
+ attr_reader :file
270
+
271
+ # The error threshold
272
+ attr_accessor :error_threshold
273
+
274
+ class << self
275
+ # Parse a control file and return a Control instance
276
+ def parse(control_file)
277
+ control_file = control_file.path if control_file.instance_of?(File)
278
+ control = ETL::Control::Control.new(control_file)
279
+ # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
280
+ eval(IO.readlines(control_file).join("\n"), Context.create(control), control_file)
281
+ control.validate
282
+ control
283
+ end
284
+
285
+ def parse_text(text)
286
+ control = ETL::Control::Control.new('no-file')
287
+ eval(text, Context.create(control), 'inline')
288
+ control.validate
289
+ control
290
+ end
291
+
292
+ # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
293
+ # are:
294
+ # * The path to a control file as a String
295
+ # * A File object referencing the control file
296
+ # * The ETL::Control::Control object (which will just be returned)
297
+ #
298
+ # Raises a ControlError if any other type is given
299
+ def resolve(control)
300
+ case control
301
+ when String
302
+ ETL::Control::Control.parse(File.new(control))
303
+ when File
304
+ ETL::Control::Control.parse(control)
305
+ when ETL::Control::Control
306
+ control
307
+ else
308
+ raise ControlError, "Control must be a String, File or Control object"
309
+ end
310
+ end
311
+ end
312
+
313
+ # Initialize the instance with the given File object
314
+ def initialize(file)
315
+ @file = file
316
+ end
317
+
318
+ # Get a list of dependencies
319
+ def dependencies
320
+ @dependencies ||= []
321
+ end
322
+
323
+ # Get the defined source
324
+ def sources
325
+ @sources ||= []
326
+ end
327
+
328
+ # Get the defined destinations
329
+ def destinations
330
+ @destinations ||= []
331
+ end
332
+
333
+ # Get the transforms with the specified name
334
+ # def transform(name)
335
+ # transforms[name] ||= []
336
+ # end
337
+
338
+ def after_read_processors
339
+ @after_read_processors ||= []
340
+ end
341
+
342
+ # Get all of the "before write" processors
343
+ def before_write_processors
344
+ @before_write_processors ||= []
345
+ end
346
+
347
+ # Get an Array of preprocessors
348
+ def pre_processors
349
+ @pre_processors ||= []
350
+ end
351
+
352
+ # Get an Array of post processors
353
+ def post_processors
354
+ @post_processors ||= []
355
+ end
356
+
357
+ # Get an Array of all transforms for this control
358
+ def transforms
359
+ @transforms ||= []
360
+ end
361
+
362
+ # A hash of the screens executed before post-process
363
+ def screens
364
+ @screens ||= {
365
+ :fatal => [],
366
+ :error => [],
367
+ :warn => []
368
+ }
369
+ end
370
+
371
+ # A hash of the screens executed after post-process
372
+ def after_post_process_screens
373
+ @after_post_process_screens ||= {
374
+ :fatal => [],
375
+ :error => [],
376
+ :warn => []
377
+ }
378
+ end
379
+
380
+ # Get the error threshold. Defaults to 100.
381
+ def error_threshold
382
+ @error_threshold ||= 100
383
+ end
384
+
385
+ # Validate the control file
386
+ def validate
387
+ #unless sources.length > 0
388
+ # raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
389
+ #end
390
+ #unless destinations.length > 0
391
+ # raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
392
+ #end
393
+ end
394
+
395
+ def source_types
396
+ [:file, :database]
397
+ end
398
+
399
+ def destination_types
400
+ [:file, :database]
401
+ end
402
+
403
+ end
404
+ end
405
+ end