activewarehouse-etl 0.1.0 → 0.2.0

data/Rakefile

@@ -78,6 +78,8 @@ end
 
 Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
+  pkg.need_tar = true
+  pkg.need_zip = true
 end
 
 desc "Generate code statistics"

data/lib/etl.rb

@@ -47,6 +47,7 @@ require 'etl/control'
 require 'etl/parser'
 require 'etl/transform'
 require 'etl/processor'
+require 'etl/generator'
 
 module ETL #:nodoc:
   class ETLError < StandardError #:nodoc:

data/lib/etl/control/destination.rb

@@ -42,6 +42,10 @@ module ETL #:nodoc:
         @buffer ||= []
       end
       
+      def generators
+        @generators ||= {}
+      end
+      
       # Get the order of elements from the source order
       def order_from_source
         order = []

data/lib/etl/control/destination/database_destination.rb

@@ -10,6 +10,7 @@ module ETL
       end
       
       def flush
+        # TODO: add virtual fields and compound key constraint support like in the FileDestination
         conn = ActiveRecord::Base.connection
         conn.transaction do
           buffer.each do |row|
@@ -17,10 +18,10 @@ module ETL
             values = []
             order.each do |name|
               names << name
-              values << "'#{row[name]}'"
+              values << "'#{row[name]}'" # TODO: this is probably not database agnostic
             end
             q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
-            ETL::Engine.logger.debug("Query: #{q}")
+            # ETL::Engine.logger.debug("Query: #{q}")
             conn.execute(q, "Insert row #{current_row}")
             @current_row += 1
           end

data/lib/etl/control/destination/file_destination.rb

@@ -3,7 +3,7 @@ module ETL #:nodoc:
     # File as the final destination.
     class FileDestination < Destination
       attr_reader :file, :order
-      attr_accessor :append, :separator, :eol, :enclose
+      attr_accessor :append, :separator, :eol, :enclose, :unique
       
       # Initialize the object.
       # * <tt>control</tt>: The Control object
@@ -15,7 +15,8 @@ module ETL #:nodoc:
         @append = configuration[:append] ||= false
         @separator = configuration[:separator] ||= ','
         @eol = configuration[:eol] ||= "\n"
-        @enclose = configuration[:enclose] ||= nil
+        @enclose = configuration[:enclose]
+        @unique = configuration[:unique]
         
         @order = mapping[:order] || order_from_source
         raise ControlError, "Order required in mapping" unless @order
@@ -27,20 +28,49 @@ module ETL #:nodoc:
         f.close
       end
       
+      # Flush the destination buffer
       def flush
         buffer.each do |row|
-          add_virtuals(row)
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
           values = order.collect { |name| row[name] }
+          
+          # enclose the value if required
           if !enclose.nil?
             values.collect! { |v| enclose + v.to_s.gsub(/(#{enclose})/, '\\\\\1') + enclose }
           end
+          
+          # write the values joined by the separator defined in the configuration
           f.write(values.join(separator))
+          
+          # write the end-of-line
           f.write(eol)
         end
         buffer.clear
       end
       
       private
+      # Return true if the row is allowed. The row will not be allowed if the :unique option is specified
+      # in the configuration and the compound key already exists
+      def row_allowed?(row)
+        if unique
+          key = (unique.collect { |k| row[k] }).join('|')
+          return false if compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+        end
+        return true
+      end
+      
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
       # Get the open file stream
       def f
         @f ||= open(file, mode)
@@ -51,10 +81,16 @@ module ETL #:nodoc:
         append ? 'a' : 'w'
       end
       
-      def add_virtuals(row)
+      def add_virtuals!(row)
         if mapping[:virtual]
           mapping[:virtual].each do |key,value|
-            row[key] = value
+            case value
+            when Symbol
+              generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
+              row[key] = generators[key].next
+            else
+              row[key] = value
+            end
           end
         end
       end

data/lib/etl/generator.rb

@@ -0,0 +1,2 @@
+require 'etl/generator/generator'
+Dir[File.dirname(__FILE__) + "/generator/*.rb"].each { |file| require(file) }

data/lib/etl/generator/generator.rb

@@ -0,0 +1,11 @@
+module ETL
+  module Generator
+    class Generator
+      class << self
+        def class_for_name(name)
+          ETL::Generator.const_get("#{name.to_s.classify}Generator")
+        end
+      end
+    end
+  end
+end

data/lib/etl/generator/surrogate_key_generator.rb

@@ -0,0 +1,10 @@
+module ETL
+  module Generator
+    class SurrogateKeyGenerator < Generator
+      def next
+        @surrogate_key ||= 0
+        @surrogate_key += 1
+      end
+    end
+  end
+end

data/lib/etl/parser/xml_parser.rb

@@ -0,0 +1,66 @@
+require 'rexml/document'
+
+module ETL
+  module Parser
+    class XmlParser < ETL::Parser::Parser
+      include Enumerable
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      def initialize(source)
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          doc = nil
+          t = Benchmark.realtime do
+            doc = REXML::Document.new(File.new(file))
+          end
+          Engine.logger.info "XML #{file} parsed in #{t}s"
+          doc.elements.each(@collection_xpath) do |element|
+            row = {}
+            fields.each do |f|
+              value = element.text(f.xpath)
+              row[f.name] = convert(f.name, value, f.type)
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        raise "Collection XPath is required" if @collection_xpath.nil?
+        
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s, options[:type])
+          else
+            raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath, :type
+        def initialize(name, xpath, type=:string)
+          @name = name
+          @xpath = xpath
+          @type = type
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -1,18 +1,25 @@
 module ETL
   module Processor
     class BulkImportProcessor < ETL::Processor::Processor
-      attr_reader :file, :target
+      attr_reader :file, :target, :truncate, :columns
       def initialize(control, configuration)
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target]
+        @truncate = configuration[:truncate] ||= false
+        @columns = configuration[:columns]
         connect
       end
       def process
+        # columns = control.destinations.first.order.join(',') # TODO: support multiple destinations?
         conn = ActiveRecord::Base.connection
         conn.transaction do
+          # TODO: Support all database types
           # Since LOCAL is used this must be allowed by both the client and server
-          conn.execute("LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{target[:table]}")
+          conn.execute("TRUNCATE #{target[:table]}") if truncate
+          q = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{target[:table]}"
+          q << " (#{columns.join(',')})" if columns
+          conn.execute(q)
         end
       end
       private

data/lib/etl/version.rb

@@ -1,7 +1,7 @@
 module ETL
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 1
+    MINOR = 2
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2006-12-06 00:00:00 -05:00
+  version: 0.2.0
+date: 2006-12-07 00:00:00 -05:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -39,6 +39,8 @@ files:
 - lib/etl/control
 - lib/etl/control.rb
 - lib/etl/engine.rb
+- lib/etl/generator
+- lib/etl/generator.rb
 - lib/etl/parser
 - lib/etl/parser.rb
 - lib/etl/processor
@@ -56,9 +58,12 @@ files:
 - lib/etl/control/destination/file_destination.rb
 - lib/etl/control/source/database_source.rb
 - lib/etl/control/source/file_source.rb
+- lib/etl/generator/generator.rb
+- lib/etl/generator/surrogate_key_generator.rb
 - lib/etl/parser/delimited_parser.rb
 - lib/etl/parser/fixed_width_parser.rb
 - lib/etl/parser/parser.rb
+- lib/etl/parser/xml_parser.rb
 - lib/etl/processor/bulk_import_processor.rb
 - lib/etl/processor/processor.rb
 - lib/etl/transform/decode_transform.rb