activewarehouse-etl 0.6.0 → 0.6.1

data/CHANGELOG

@@ -63,4 +63,8 @@
 * Added a depends_on directive that accepts a list of arguments of either strings or symbols. Each
   symbol is converted to a string and .ctl is appended; strings are passed through directly. The
   dependencies are executed in the order they are specified.
-* The default field separator in the bulk loader is now a comma (was a tab).
+* The default field separator in the bulk loader is now a comma (was a tab).
+
+0.6.1 - Mar 22, 2007
+* Added support for absolute paths in file sources
+* Added CopyFieldProcessor

data/lib/etl/control/source/file_source.rb

@@ -44,9 +44,10 @@ module ETL #:nodoc:
       private
       # Copy source data to a local directory structure
       def copy_sources
-        source_file = File.join(File.dirname(control.file), configuration[:file])
-        Dir.glob(source_file).each do |f|
-          next if File.directory?(f)
+        path = Pathname.new(configuration[:file])
+        path = path.absolute? ? path : Pathname.new(File.dirname(control.file)) + path
+        Pathname.glob(path).each do |f|
+          next if f.directory?
           FileUtils.cp(f, local_file)
         end
       end

data/lib/etl/engine.rb

@@ -18,13 +18,20 @@ module ETL #:nodoc:
         new().process(control_file)
       end
       
+      attr_accessor :timestamped_log
+      
       # A logger for the engine
       attr_accessor :logger
       
       def logger #:nodoc:
         unless @logger
-          @logger = Logger.new("etl_#{timestamp}.log")
-          @logger.level = Logger::WARN
+          if timestamped_log
+            @logger = Logger.new("etl_#{timestamp}.log")
+          else
+            @logger = Logger.new(File.open('etl.log', 'a'))
+          end
+          @logger.level = Logger::ERROR
+          @logger.formatter = Logger::Formatter.new
         end
         @logger
       end

data/lib/etl/parser/delimited_parser.rb

@@ -25,9 +25,9 @@ module ETL #:nodoc:
             line += 1
             row = {}
             validate_row(raw_row, line, file)
-            raw_row.each_with_index do |record, index|
+            raw_row.each_with_index do |value, index|
               f = fields[index]
-              row[f.name] = convert(f.name, record, f.type)
+              row[f.name] = value
             end
             yield row
           end
@@ -56,7 +56,7 @@ module ETL #:nodoc:
           when Symbol
             fields << Field.new(options)
           when Hash
-            fields << Field.new(options[:name], options[:type])
+            fields << Field.new(options[:name])
           else
             raise DefinitionError, "Each field definition must either be a symbol or a hash"
           end
@@ -64,10 +64,9 @@ module ETL #:nodoc:
       end
       
       class Field #:nodoc:
-        attr_reader :name, :type
-        def initialize(name, type=:string)
+        attr_reader :name
+        def initialize(name)
           @name = name
-          @type = type
         end
       end
     end

data/lib/etl/parser/fixed_width_parser.rb

@@ -22,7 +22,7 @@ module ETL #:nodoc:
                 next
               end
               # TODO make strip optional?
-              row[name] = convert(name, line[f.field_start, f.field_length].strip, f.type)
+              row[name] = line[f.field_start, f.field_length].strip
             end
             yield row
           end
@@ -37,16 +37,18 @@ module ETL #:nodoc:
       private
       def configure
         source.definition.each do |field, options|
-          fields[field] = FixedWidthField.new(options[:name], options[:start], options[:end], options[:length], options[:type])
+          fields[field] = FixedWidthField.new(
+            options[:name], options[:start], options[:end], options[:length]
+          )
         end
       end
     end
     
     class FixedWidthField #:nodoc:
-      attr_reader :name, :field_start, :field_end, :field_length, :type
-      def initialize(name, field_start, field_end=nil, field_length=nil, type=nil)
+      attr_reader :name, :field_start, :field_end, :field_length
+      # Initialize the field.
+      def initialize(name, field_start, field_end=nil, field_length=nil)
         @name = name
-        @type = type ||= :string
         @field_start = field_start - 1
         if field_end
           @field_end = field_end

data/lib/etl/parser/parser.rb

@@ -23,26 +23,11 @@ module ETL
         @options = options || {}
       end
       
-      # Convert the value to the specified type.
-      # 
-      # Parameters:
-      # * <tt>name</tt>: The name of the field
-      # * <tt>value</tt>: The value
-      # * <tt>type</tt>: The type name (:integer, :float, :string)
-      def convert(name, value, type)
-        case type
-        when :integer
-          value.to_i
-        when :float
-          value.to_f
-        else
-          value
-        end
-      end
-      
       protected
       def file
-        File.join(File.dirname(source.control.file), source.configuration[:file])
+        path = Pathname.new(source.configuration[:file])
+        path = path.absolute? ? path : Pathname.new(File.dirname(source.control.file)) + path
+        path
       end
       
       def raise_with_info(error, message, file, line)

data/lib/etl/parser/xml_parser.rb

@@ -23,7 +23,7 @@ module ETL
             row = {}
             fields.each do |f|
               value = element.text(f.xpath)
-              row[f.name] = convert(f.name, value, f.type)
+              row[f.name] = value
             end
             yield row
           end
@@ -46,7 +46,7 @@ module ETL
             fields << Field.new(options, options.to_s)
           when Hash
             options[:xpath] ||= options[:name]
-            fields << Field.new(options[:name], options[:xpath].to_s, options[:type])
+            fields << Field.new(options[:name], options[:xpath].to_s)
           else
             raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
           end
@@ -54,11 +54,10 @@ module ETL
       end
       
       class Field
-        attr_reader :name, :xpath, :type
-        def initialize(name, xpath, type=:string)
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
           @name = name
           @xpath = xpath
-          @type = type
         end
       end
     end

data/lib/etl/processor/copy_field_processor.rb

@@ -0,0 +1,10 @@
+module ETL
+  module Processor
+    class CopyField < ETL::Processor::RowProcessor
+      def process(row)
+        row[configuration[:destination]] = row[configuration[:source]].dup
+        row
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -5,14 +5,19 @@ module ETL #:nodoc:
       def initialize(control, configuration={})
         super
         @type = configuration[:type]
+        @significant = configuration[:significant] ||= 0
       end
       # Transform the value
       def transform(name, value, row)
         case @type
         when :string
           value.to_s
-        when :number
+        when :number, :integer
           value.to_i
+        when :float
+          value.to_f
+        when :decimal
+          BigDecimal.new(value.to_s, @significant)
         else
           raise "Unsupported type: #{@type}"
         end

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 6
-    TINY  = 0
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.6.0
-date: 2007-03-08 00:00:00 -05:00
+  version: 0.6.1
+date: 2007-03-22 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -69,6 +69,7 @@ files:
 - lib/etl/parser/sax_parser.rb
 - lib/etl/parser/xml_parser.rb
 - lib/etl/processor/bulk_import_processor.rb
+- lib/etl/processor/copy_field_processor.rb
 - lib/etl/processor/hierarchy_exploder_processor.rb
 - lib/etl/processor/processor.rb
 - lib/etl/processor/row_processor.rb