activewarehouse-etl 0.9.1 → 0.9.5.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (135) hide show
  1. data/.gitignore +7 -0
  2. data/0.9-UPGRADE +6 -0
  3. data/CHANGELOG +182 -150
  4. data/Gemfile +4 -0
  5. data/HOW_TO_RELEASE +9 -0
  6. data/README +18 -2
  7. data/Rakefile +35 -91
  8. data/active_support_logger.patch +78 -0
  9. data/activewarehouse-etl.gemspec +30 -0
  10. data/lib/etl.rb +10 -2
  11. data/lib/etl/batch/directives.rb +11 -1
  12. data/lib/etl/control/control.rb +2 -2
  13. data/lib/etl/control/destination.rb +27 -7
  14. data/lib/etl/control/destination/database_destination.rb +8 -6
  15. data/lib/etl/control/destination/excel_destination.rb +91 -0
  16. data/lib/etl/control/destination/file_destination.rb +6 -4
  17. data/lib/etl/control/destination/insert_update_database_destination.rb +133 -0
  18. data/lib/etl/control/destination/update_database_destination.rb +109 -0
  19. data/lib/etl/control/source.rb +3 -2
  20. data/lib/etl/control/source/database_source.rb +14 -10
  21. data/lib/etl/control/source/file_source.rb +2 -2
  22. data/lib/etl/engine.rb +17 -15
  23. data/lib/etl/execution.rb +0 -1
  24. data/lib/etl/execution/batch.rb +3 -1
  25. data/lib/etl/execution/migration.rb +5 -0
  26. data/lib/etl/parser/delimited_parser.rb +20 -1
  27. data/lib/etl/parser/excel_parser.rb +112 -0
  28. data/lib/etl/processor/bulk_import_processor.rb +4 -2
  29. data/lib/etl/processor/database_join_processor.rb +68 -0
  30. data/lib/etl/processor/escape_csv_processor.rb +77 -0
  31. data/lib/etl/processor/filter_row_processor.rb +51 -0
  32. data/lib/etl/processor/ftp_downloader_processor.rb +68 -0
  33. data/lib/etl/processor/ftp_uploader_processor.rb +65 -0
  34. data/lib/etl/processor/imapattachment_downloader_processor.rb +91 -0
  35. data/lib/etl/processor/pop3attachment_downloader_processor.rb +90 -0
  36. data/lib/etl/processor/sftp_downloader_processor.rb +63 -0
  37. data/lib/etl/processor/sftp_uploader_processor.rb +63 -0
  38. data/lib/etl/processor/zip_file_processor.rb +27 -0
  39. data/lib/etl/transform/calculation_transform.rb +71 -0
  40. data/lib/etl/transform/foreign_key_lookup_transform.rb +25 -7
  41. data/lib/etl/transform/ordinalize_transform.rb +3 -1
  42. data/lib/etl/transform/split_fields_transform.rb +27 -0
  43. data/lib/etl/version.rb +1 -7
  44. data/test-matrix.yml +10 -0
  45. data/test/.gitignore +1 -0
  46. data/test/.ignore +2 -0
  47. data/test/all.ebf +6 -0
  48. data/test/apache_combined_log.ctl +11 -0
  49. data/test/batch_test.rb +41 -0
  50. data/test/batch_with_error.ebf +6 -0
  51. data/test/batched1.ctl +0 -0
  52. data/test/batched2.ctl +0 -0
  53. data/test/block_processor.ctl +6 -0
  54. data/test/block_processor_error.ctl +1 -0
  55. data/test/block_processor_pre_post_process.ctl +4 -0
  56. data/test/block_processor_remove_rows.ctl +5 -0
  57. data/test/block_processor_test.rb +38 -0
  58. data/test/config/Gemfile.rails-2.3.x +3 -0
  59. data/test/config/Gemfile.rails-2.3.x.lock +38 -0
  60. data/test/config/Gemfile.rails-3.0.x +3 -0
  61. data/test/config/Gemfile.rails-3.0.x.lock +49 -0
  62. data/test/config/common.rb +21 -0
  63. data/test/connection/mysql/connection.rb +9 -0
  64. data/test/connection/mysql/schema.sql +36 -0
  65. data/test/connection/postgresql/connection.rb +13 -0
  66. data/test/connection/postgresql/schema.sql +39 -0
  67. data/test/control_test.rb +43 -0
  68. data/test/data/apache_combined_log.txt +3 -0
  69. data/test/data/bulk_import.txt +3 -0
  70. data/test/data/bulk_import_with_empties.txt +3 -0
  71. data/test/data/decode.txt +3 -0
  72. data/test/data/delimited.txt +3 -0
  73. data/test/data/encode_source_latin1.txt +2 -0
  74. data/test/data/excel.xls +0 -0
  75. data/test/data/excel2.xls +0 -0
  76. data/test/data/fixed_width.txt +3 -0
  77. data/test/data/multiple_delimited_1.txt +3 -0
  78. data/test/data/multiple_delimited_2.txt +3 -0
  79. data/test/data/people.txt +3 -0
  80. data/test/data/sax.xml +14 -0
  81. data/test/data/xml.xml +16 -0
  82. data/test/date_dimension_builder_test.rb +96 -0
  83. data/test/delimited.ctl +30 -0
  84. data/test/delimited_absolute.ctl +33 -0
  85. data/test/delimited_destination_db.ctl +25 -0
  86. data/test/delimited_excel.ctl +31 -0
  87. data/test/delimited_insert_update.ctl +34 -0
  88. data/test/delimited_update.ctl +34 -0
  89. data/test/delimited_with_bulk_load.ctl +34 -0
  90. data/test/destination_test.rb +275 -0
  91. data/test/directive_test.rb +23 -0
  92. data/test/encode_processor_test.rb +32 -0
  93. data/test/engine_test.rb +32 -0
  94. data/test/errors.ctl +24 -0
  95. data/test/etl_test.rb +42 -0
  96. data/test/excel.ctl +24 -0
  97. data/test/excel2.ctl +25 -0
  98. data/test/fixed_width.ctl +35 -0
  99. data/test/generator_test.rb +14 -0
  100. data/test/inline_parser.ctl +17 -0
  101. data/test/mocks/mock_destination.rb +26 -0
  102. data/test/mocks/mock_source.rb +25 -0
  103. data/test/model_source.ctl +14 -0
  104. data/test/multiple_delimited.ctl +22 -0
  105. data/test/multiple_source_delimited.ctl +39 -0
  106. data/test/parser_test.rb +224 -0
  107. data/test/performance/delimited.ctl +30 -0
  108. data/test/processor_test.rb +44 -0
  109. data/test/row_processor_test.rb +17 -0
  110. data/test/sax.ctl +26 -0
  111. data/test/scd/1.txt +1 -0
  112. data/test/scd/2.txt +1 -0
  113. data/test/scd/3.txt +1 -0
  114. data/test/scd_test.rb +257 -0
  115. data/test/scd_test_type_1.ctl +43 -0
  116. data/test/scd_test_type_2.ctl +34 -0
  117. data/test/screen_test.rb +9 -0
  118. data/test/screen_test_error.ctl +3 -0
  119. data/test/screen_test_fatal.ctl +3 -0
  120. data/test/source_test.rb +139 -0
  121. data/test/test_helper.rb +34 -0
  122. data/test/transform_test.rb +101 -0
  123. data/test/vendor/adapter_extensions-0.5.0/CHANGELOG +26 -0
  124. data/test/vendor/adapter_extensions-0.5.0/LICENSE +16 -0
  125. data/test/vendor/adapter_extensions-0.5.0/README +7 -0
  126. data/test/vendor/adapter_extensions-0.5.0/Rakefile +158 -0
  127. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions.rb +12 -0
  128. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/abstract_adapter.rb +44 -0
  129. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/mysql_adapter.rb +63 -0
  130. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/postgresql_adapter.rb +52 -0
  131. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/sqlserver_adapter.rb +44 -0
  132. data/test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/version.rb +10 -0
  133. data/test/xml.ctl +31 -0
  134. metadata +229 -70
  135. data/lib/etl/execution/record.rb +0 -18
data/lib/etl/execution/record.rb DELETED
@@ -1,18 +0,0 @@
1
- module ETL #:nodoc:
2
- module Execution #:nodoc:
3
- # Represents a single record
4
- class Record < ETL::Execution::Base
5
- belongs_to :table
6
- class << self
7
- attr_accessor :time_spent
8
- def time_spent
9
- @time_spent ||= 0
10
- end
11
- def average_time_spent
12
- return 0 if time_spent == 0
13
- ETL::Engine.rows_read / time_spent
14
- end
15
- end
16
- end
17
- end
18
- end