activewarehouse-etl 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +26 -14
- data/TODO +4 -2
- data/lib/etl.rb +10 -3
- data/lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb +2 -2
- data/lib/etl/commands/etl.rb +6 -7
- data/lib/etl/control/control.rb +120 -52
- data/lib/etl/control/destination.rb +46 -5
- data/lib/etl/control/destination/database_destination.rb +45 -7
- data/lib/etl/control/destination/file_destination.rb +28 -4
- data/lib/etl/control/source.rb +16 -1
- data/lib/etl/control/source/database_source.rb +35 -5
- data/lib/etl/control/source/file_source.rb +33 -3
- data/lib/etl/engine.rb +129 -9
- data/lib/etl/generator/generator.rb +11 -2
- data/lib/etl/generator/surrogate_key_generator.rb +3 -2
- data/lib/etl/parser/delimited_parser.rb +3 -4
- data/lib/etl/parser/fixed_width_parser.rb +3 -4
- data/lib/etl/parser/parser.rb +7 -1
- data/lib/etl/parser/sax_parser.rb +190 -0
- data/lib/etl/parser/xml_parser.rb +2 -2
- data/lib/etl/processor/bulk_import_processor.rb +4 -4
- data/lib/etl/processor/processor.rb +1 -1
- data/lib/etl/processor/truncate_processor.rb +4 -4
- data/lib/etl/transform/date_to_string_transform.rb +19 -0
- data/lib/etl/transform/decode_transform.rb +15 -1
- data/lib/etl/transform/foreign_key_lookup_transform.rb +53 -0
- data/lib/etl/transform/string_to_date_transform.rb +14 -0
- data/lib/etl/transform/transform.rb +28 -9
- data/lib/etl/transform/type_transform.rb +22 -0
- data/lib/etl/version.rb +2 -2
- metadata +8 -3
@@ -0,0 +1,14 @@
|
|
1
|
+
module ETL #:nodoc:
|
2
|
+
module Transform #:nodoc:
|
3
|
+
# Transform a String representation of a date to a Date instance
|
4
|
+
class StringToDateTransform < ETL::Transform::Transform
|
5
|
+
def initialize(control, configuration={})
|
6
|
+
super
|
7
|
+
end
|
8
|
+
# Transform the value using Time.parse
|
9
|
+
def transform(value)
|
10
|
+
t = Date.parse(value)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -1,5 +1,20 @@
|
|
1
|
-
module ETL
|
2
|
-
module Transform
|
1
|
+
module ETL#:nodoc:
|
2
|
+
module Transform#:nodoc:
|
3
|
+
# Base class for transforms.
|
4
|
+
#
|
5
|
+
# A transform converts one value to another value using some sort of algorithm.
|
6
|
+
#
|
7
|
+
# A simple transform has two arguments, the field to transform and the name of the transform:
|
8
|
+
#
|
9
|
+
# transform :ssn, :sha1
|
10
|
+
#
|
11
|
+
# Transforms can also be blocks:
|
12
|
+
#
|
13
|
+
# transform(:ssn){ |v| v[0,24] }
|
14
|
+
#
|
15
|
+
# Finally, a transform can include a configuration hash:
|
16
|
+
#
|
17
|
+
# transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
|
3
18
|
class Transform
|
4
19
|
class << self
|
5
20
|
# Transform the specified value using the given transforms. The transforms can either be
|
@@ -8,13 +23,17 @@ module ETL
|
|
8
23
|
def transform(name, value, transforms)
|
9
24
|
# logger.debug "Transforming field #{name}" if transforms.length > 0
|
10
25
|
transforms.each do |transform|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
26
|
+
begin
|
27
|
+
case transform
|
28
|
+
when Proc
|
29
|
+
value = transform.call(value)
|
30
|
+
when Transform
|
31
|
+
value = transform.transform(value)
|
32
|
+
else
|
33
|
+
raise ControlError, "Unsupported transform configuration type: #{transform}"
|
34
|
+
end
|
35
|
+
rescue
|
36
|
+
raise TransformError, "Error transforming #{value} with #{transform}"
|
18
37
|
end
|
19
38
|
end
|
20
39
|
value
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module ETL #:nodoc:
|
2
|
+
module Transform #:nodoc:
|
3
|
+
# Transform from one type to another
|
4
|
+
class TypeTransform < ETL::Transform::Transform
|
5
|
+
def initialize(control, configuration={})
|
6
|
+
super
|
7
|
+
@type = configuration[:type]
|
8
|
+
end
|
9
|
+
# Transform the value using Time.parse
|
10
|
+
def transform(value)
|
11
|
+
case @type
|
12
|
+
when :string
|
13
|
+
value.to_s
|
14
|
+
when :number
|
15
|
+
value.to_i
|
16
|
+
else
|
17
|
+
raise "Unsupported type: #{@type}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/etl/version.rb
CHANGED
metadata
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
|
-
rubygems_version: 0.9.0
|
2
|
+
rubygems_version: 0.9.0.10
|
3
3
|
specification_version: 1
|
4
4
|
name: activewarehouse-etl
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.
|
7
|
-
date: 2007-
|
6
|
+
version: 0.5.0
|
7
|
+
date: 2007-02-17 00:00:00 -05:00
|
8
8
|
summary: Pure Ruby ETL package.
|
9
9
|
require_paths:
|
10
10
|
- lib
|
@@ -68,13 +68,18 @@ files:
|
|
68
68
|
- lib/etl/parser/delimited_parser.rb
|
69
69
|
- lib/etl/parser/fixed_width_parser.rb
|
70
70
|
- lib/etl/parser/parser.rb
|
71
|
+
- lib/etl/parser/sax_parser.rb
|
71
72
|
- lib/etl/parser/xml_parser.rb
|
72
73
|
- lib/etl/processor/bulk_import_processor.rb
|
73
74
|
- lib/etl/processor/processor.rb
|
74
75
|
- lib/etl/processor/truncate_processor.rb
|
76
|
+
- lib/etl/transform/date_to_string_transform.rb
|
75
77
|
- lib/etl/transform/decode_transform.rb
|
78
|
+
- lib/etl/transform/foreign_key_lookup_transform.rb
|
76
79
|
- lib/etl/transform/sha1_transform.rb
|
80
|
+
- lib/etl/transform/string_to_date_transform.rb
|
77
81
|
- lib/etl/transform/transform.rb
|
82
|
+
- lib/etl/transform/type_transform.rb
|
78
83
|
test_files: []
|
79
84
|
|
80
85
|
rdoc_options:
|