activewarehouse-etl 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +26 -14
- data/TODO +4 -2
- data/lib/etl.rb +10 -3
- data/lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb +2 -2
- data/lib/etl/commands/etl.rb +6 -7
- data/lib/etl/control/control.rb +120 -52
- data/lib/etl/control/destination.rb +46 -5
- data/lib/etl/control/destination/database_destination.rb +45 -7
- data/lib/etl/control/destination/file_destination.rb +28 -4
- data/lib/etl/control/source.rb +16 -1
- data/lib/etl/control/source/database_source.rb +35 -5
- data/lib/etl/control/source/file_source.rb +33 -3
- data/lib/etl/engine.rb +129 -9
- data/lib/etl/generator/generator.rb +11 -2
- data/lib/etl/generator/surrogate_key_generator.rb +3 -2
- data/lib/etl/parser/delimited_parser.rb +3 -4
- data/lib/etl/parser/fixed_width_parser.rb +3 -4
- data/lib/etl/parser/parser.rb +7 -1
- data/lib/etl/parser/sax_parser.rb +190 -0
- data/lib/etl/parser/xml_parser.rb +2 -2
- data/lib/etl/processor/bulk_import_processor.rb +4 -4
- data/lib/etl/processor/processor.rb +1 -1
- data/lib/etl/processor/truncate_processor.rb +4 -4
- data/lib/etl/transform/date_to_string_transform.rb +19 -0
- data/lib/etl/transform/decode_transform.rb +15 -1
- data/lib/etl/transform/foreign_key_lookup_transform.rb +53 -0
- data/lib/etl/transform/string_to_date_transform.rb +14 -0
- data/lib/etl/transform/transform.rb +28 -9
- data/lib/etl/transform/type_transform.rb +22 -0
- data/lib/etl/version.rb +2 -2
- metadata +8 -3
@@ -0,0 +1,14 @@
|
|
1
|
+
module ETL #:nodoc:
|
2
|
+
module Transform #:nodoc:
|
3
|
+
# Transform a String representation of a date to a Date instance
|
4
|
+
class StringToDateTransform < ETL::Transform::Transform
|
5
|
+
def initialize(control, configuration={})
|
6
|
+
super
|
7
|
+
end
|
8
|
+
# Transform the value using Time.parse
|
9
|
+
def transform(value)
|
10
|
+
t = Date.parse(value)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -1,5 +1,20 @@
|
|
1
|
-
module ETL
|
2
|
-
module Transform
|
1
|
+
module ETL#:nodoc:
|
2
|
+
module Transform#:nodoc:
|
3
|
+
# Base class for transforms.
|
4
|
+
#
|
5
|
+
# A transform converts one value to another value using some sort of algorithm.
|
6
|
+
#
|
7
|
+
# A simple transform has two arguments, the field to transform and the name of the transform:
|
8
|
+
#
|
9
|
+
# transform :ssn, :sha1
|
10
|
+
#
|
11
|
+
# Transforms can also be blocks:
|
12
|
+
#
|
13
|
+
# transform(:ssn){ |v| v[0,24] }
|
14
|
+
#
|
15
|
+
# Finally, a transform can include a configuration hash:
|
16
|
+
#
|
17
|
+
# transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
|
3
18
|
class Transform
|
4
19
|
class << self
|
5
20
|
# Transform the specified value using the given transforms. The transforms can either be
|
@@ -8,13 +23,17 @@ module ETL
|
|
8
23
|
def transform(name, value, transforms)
|
9
24
|
# logger.debug "Transforming field #{name}" if transforms.length > 0
|
10
25
|
transforms.each do |transform|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
26
|
+
begin
|
27
|
+
case transform
|
28
|
+
when Proc
|
29
|
+
value = transform.call(value)
|
30
|
+
when Transform
|
31
|
+
value = transform.transform(value)
|
32
|
+
else
|
33
|
+
raise ControlError, "Unsupported transform configuration type: #{transform}"
|
34
|
+
end
|
35
|
+
rescue
|
36
|
+
raise TransformError, "Error transforming #{value} with #{transform}"
|
18
37
|
end
|
19
38
|
end
|
20
39
|
value
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module ETL #:nodoc:
|
2
|
+
module Transform #:nodoc:
|
3
|
+
# Transform from one type to another
|
4
|
+
class TypeTransform < ETL::Transform::Transform
|
5
|
+
def initialize(control, configuration={})
|
6
|
+
super
|
7
|
+
@type = configuration[:type]
|
8
|
+
end
|
9
|
+
# Transform the value using Time.parse
|
10
|
+
def transform(value)
|
11
|
+
case @type
|
12
|
+
when :string
|
13
|
+
value.to_s
|
14
|
+
when :number
|
15
|
+
value.to_i
|
16
|
+
else
|
17
|
+
raise "Unsupported type: #{@type}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/etl/version.rb
CHANGED
metadata
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
|
-
rubygems_version: 0.9.0
|
2
|
+
rubygems_version: 0.9.0.10
|
3
3
|
specification_version: 1
|
4
4
|
name: activewarehouse-etl
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.
|
7
|
-
date: 2007-
|
6
|
+
version: 0.5.0
|
7
|
+
date: 2007-02-17 00:00:00 -05:00
|
8
8
|
summary: Pure Ruby ETL package.
|
9
9
|
require_paths:
|
10
10
|
- lib
|
@@ -68,13 +68,18 @@ files:
|
|
68
68
|
- lib/etl/parser/delimited_parser.rb
|
69
69
|
- lib/etl/parser/fixed_width_parser.rb
|
70
70
|
- lib/etl/parser/parser.rb
|
71
|
+
- lib/etl/parser/sax_parser.rb
|
71
72
|
- lib/etl/parser/xml_parser.rb
|
72
73
|
- lib/etl/processor/bulk_import_processor.rb
|
73
74
|
- lib/etl/processor/processor.rb
|
74
75
|
- lib/etl/processor/truncate_processor.rb
|
76
|
+
- lib/etl/transform/date_to_string_transform.rb
|
75
77
|
- lib/etl/transform/decode_transform.rb
|
78
|
+
- lib/etl/transform/foreign_key_lookup_transform.rb
|
76
79
|
- lib/etl/transform/sha1_transform.rb
|
80
|
+
- lib/etl/transform/string_to_date_transform.rb
|
77
81
|
- lib/etl/transform/transform.rb
|
82
|
+
- lib/etl/transform/type_transform.rb
|
78
83
|
test_files: []
|
79
84
|
|
80
85
|
rdoc_options:
|