activewarehouse-etl 0.1.0

data/CHANGELOG

@@ -0,0 +1,2 @@
+0.1.0 - Dec 6, 2006
+ * Initial release

data/README

@@ -0,0 +1,27 @@
+Ruby ETL parser.
+
+== Features
+
+Current supported features:
+
+* ETL Domain Specific Language (DSL) - Control files are specified in a Ruby-based DSL
+* Multiple source types - fixed-width and delimited text files currently supported
+* Multiple destination types - file and database destinations
+* Support for extracting from multiple sources
+* Support for loading to multiple destinations
+* Extensible transformations - comes with built in SHA1 one-way hash example
+* Pre/post processing - export to files and then post process with the bulk import processor for large amounts of data
+* Virtual fields - Add a field to the destination data which doesn't exist in the source data
+
+
+== Requirements
+
+* ActiveSupport Gem
+* ActiveRecord Gem
+* FasterCSV Gem
+
+== Examples
+Examples can be found in the test directory.
+
+== Feedback
+This is a work in progress. Comments should be made on the activewarehouse-discuss mailing list at the moment.

data/Rakefile

@@ -0,0 +1,117 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+
+require File.join(File.dirname(__FILE__), 'lib/etl', 'version')
+
+PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
+PKG_NAME        = 'activewarehouse-etl'
+PKG_VERSION     = ETL::VERSION::STRING + PKG_BUILD
+PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
+PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
+
+RELEASE_NAME  = "REL #{PKG_VERSION}"
+
+RUBY_FORGE_PROJECT = "activewarehouse"
+RUBY_FORGE_USER    = "aeden"
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the ETL application.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+  # TODO: reset the database
+end
+
+desc 'Generate documentation for the ETL application.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ActiveWarehouse ETL'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+PKG_FILES = FileList[
+  'CHANGELOG',
+  'README',
+  'Rakefile',
+  'bin/**/*',
+  'doc/**/*',
+  'lib/**/*',
+] - [ 'test' ]
+
+spec = Gem::Specification.new do |s|
+  s.name = 'activewarehouse-etl'
+  s.version = PKG_VERSION
+  s.summary = "Pure Ruby ETL package."
+  s.description = <<-EOF
+    ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
+  EOF
+
+  s.add_dependency('rake', '>= 0.7.1')
+  s.add_dependency('activesupport',    '>= 1.3.1.5618')
+  s.add_dependency('activerecord',     '>= 1.14.4.5618')
+  s.add_dependency('fastercsv',        '>= 1.0.0')
+
+  s.rdoc_options << '--exclude' << '.'
+  s.has_rdoc = false
+
+  s.files = PKG_FILES.to_a.delete_if {|f| f.include?('.svn')}
+  s.require_path = 'lib'
+
+  s.bindir = "bin" # Use these for applications.
+  s.executables = ['etl']
+  s.default_executable = "etl"
+
+  s.author = "Anthony Eden"
+  s.email = "anthonyeden@gmail.com"
+  s.homepage = "http://activewarehouse.rubyforge.org/etl"
+  s.rubyforge_project = "activewarehouse"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "Generate code statistics"
+task :lines do
+  lines, codelines, total_lines, total_codelines = 0, 0, 0, 0
+
+  for file_name in FileList["lib/**/*.rb"]
+    next if file_name =~ /vendor/
+    f = File.open(file_name)
+
+    while line = f.gets
+      lines += 1
+      next if line =~ /^\s*$/
+      next if line =~ /^\s*#/
+      codelines += 1
+    end
+    puts "L: #{sprintf("%4d", lines)}, LOC #{sprintf("%4d", codelines)} | #{file_name}"
+    
+    total_lines     += lines
+    total_codelines += codelines
+    
+    lines, codelines = 0, 0
+  end
+
+  puts "Total: Lines #{total_lines}, LOC #{total_codelines}"
+end
+
+desc "Publish the release files to RubyForge."
+task :release => [ :package ] do
+  `rubyforge login`
+
+  for ext in %w( gem tgz zip )
+    release_command = "rubyforge add_release activewarehouse #{PKG_NAME} 'REL #{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.#{ext}"
+    puts release_command
+    system(release_command)
+  end
+end

data/bin/etl

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.dirname(__FILE__) + "/../lib/etl/commands/etl"

data/lib/etl.rb

@@ -0,0 +1,58 @@
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+unless defined?(Logger)
+  require 'logger'
+end
+
+require 'rubygems'
+unless defined?(ActiveSupport)
+  require_gem 'activesupport'
+  require 'active_support'
+end
+
+unless defined?(ActiveRecord)
+  require_gem 'activerecord'
+  require 'active_record'
+end
+
+require_gem 'fastercsv'
+require 'faster_csv'
+
+$:.unshift(File.dirname(__FILE__))
+
+require 'etl/version'
+require 'etl/engine'
+require 'etl/control'
+require 'etl/parser'
+require 'etl/transform'
+require 'etl/processor'
+
+module ETL #:nodoc:
+  class ETLError < StandardError #:nodoc:
+  end
+  class ControlError < ETLError #:nodoc:
+  end
+  class DefinitionError < ControlError #:nodoc:
+  end
+end

data/lib/etl/commands/etl.rb

@@ -0,0 +1,45 @@
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'benchmark'
+require File.dirname(__FILE__) + '/../../etl'
+
+# Print a usage statement
+def usage #:nodoc:
+  puts "Usage: etl ctl_file [ctl_file2 ctl_file3 ...]"
+end
+
+if ARGV.length < 1
+  usage
+else
+  puts "Starting ETL process"
+
+  t = Benchmark.realtime do
+    ARGV.each do |f|
+      puts "Processing #{f}"
+      ETL::Engine.process(f)
+    end
+  end
+
+  puts "ETL process complete in #{sprintf('%.3f', t)} seconds"
+end

data/lib/etl/control.rb

@@ -0,0 +1,3 @@
+require 'etl/control/control'
+require 'etl/control/source'
+require 'etl/control/destination'

data/lib/etl/control/control.rb

@@ -0,0 +1,134 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Object representation of a control file
+    class Control
+      attr_reader :file
+      
+      class << self
+        # Parse a control file and return a Control instance
+        def parse(control_file)
+          control_file = control_file.path if control_file.instance_of?(File)
+          # logger.debug "Parsing control file #{control_file.path}"
+          control = ETL::Control::Control.new(control_file)
+          # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
+          eval(IO.readlines(control_file).join("\n"), control.get_binding)
+          control.validate
+          control
+        end
+        
+        def resolve(control)
+          case control
+          when String
+            ETL::Control::Control.parse(File.new(control))
+          when File
+            ETL::Control::Control.parse(control)
+          when ETL::Control::Control
+            control
+          else
+            raise ControlError, "Control must be a String, File or Control object"
+          end
+        end
+      end
+      
+      def initialize(file)
+        @file = file
+      end
+      
+      # Define a source
+      def source(name, configuration={}, definition={})
+        source_types = [:file, :db]
+        source_types.each do |source_type|
+          if configuration[source_type]
+            source_class = ETL::Control::Source.class_for_name(source_type)
+            sources << source_class.new(self, configuration, definition)
+          end
+        end
+      end
+      
+      # Get the defined source
+      def sources
+        @sources ||= []
+      end
+      
+      # Define a destination
+      def destination(name, configuration={}, mapping={})
+        destination_types.each do |dest_type|
+          if configuration[dest_type]
+            dest_class = ETL::Control::Destination.class_for_name(dest_type)
+            destinations << dest_class.new(self, configuration, mapping)
+          end
+        end
+      end
+      
+      # Get the defined destinations
+      def destinations
+        @destinations ||= []
+      end
+      
+      def transform(name, transformer=nil, configuration={}, &block)
+        transforms[name] ||= []
+        if transformer
+          transform_class = ETL::Transform.const_get("#{transformer.to_s.classify}Transform")
+          transforms[name] << transform_class.new(self, configuration)
+        elsif block_given?
+          transforms[name] << block
+        else
+          raise ControlError, "Either a transformer or a block must be specified"
+        end
+      end
+      
+      def get_transform(name)
+        transforms[name] ||= []
+      end
+      
+      def pre_process(name, configuration={})
+        processor_class = ETL::Processor.const_get("#{name.to_s.classify}Processor")
+        pre_processors << processor_class.new(self, configuration)
+      end
+      
+      def pre_processors
+        @pre_processors ||= []
+      end
+      
+      def post_process(name, configuration={})
+        processor_class = ETL::Processor.const_get("#{name.to_s.classify}Processor")
+        post_processors << processor_class.new(self, configuration)
+      end
+      
+      def post_processors
+        @post_processors ||= []
+      end
+      
+      def get_binding
+        binding
+      end
+      
+      # Get a map of all transforms for this control
+      def transforms
+        @transforms ||= {}
+      end
+      
+      # Validate the control file
+      def validate
+        unless sources.length > 0
+          raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
+        end
+        unless destinations.length > 0
+          raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
+        end
+      end
+      
+      protected
+      # Get an array of supported source types
+      def source_types
+        [:file, :database]
+      end
+      
+      # Get an array of supported destination types
+      def destination_types
+        [:file, :database]
+      end
+      
+    end
+  end
+end

data/lib/etl/control/destination.rb

@@ -0,0 +1,62 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class Destination
+      attr_reader :control, :configuration, :mapping
+      attr_accessor :buffer_size, :current_row
+      
+      class << self
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.classify}Destination")
+        end
+      end
+      
+      def initialize(control, configuration, mapping)
+        @control = control
+        @configuration = configuration
+        @mapping = mapping
+        @buffer_size = configuration[:buffer_size] ||= 1000
+      end
+      
+      def current_row
+        @current_row ||= 1
+      end
+      
+      # Abstract method
+      def write(row)
+        buffer << row
+        flush if buffer.length >= buffer_size
+      end
+      
+      # Abstract method
+      def flush
+        raise NotImplementedError, "flush method must be implemented by subclasses"
+      end
+      
+      # Abstract method
+      def close
+        raise NotImplementedError, "close method must be implemented by subclasses"
+      end
+      
+      protected
+      def buffer
+        @buffer ||= []
+      end
+      
+      # Get the order of elements from the source order
+      def order_from_source
+        order = []
+        control.sources.first.definition.each do |item|
+          case item
+          when Hash
+            order << item[:name]
+          else
+            order << item
+          end
+        end
+        order
+      end
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/destination/*.rb"].each { |file| require(file) }

data/lib/etl/control/destination/database_destination.rb

@@ -0,0 +1,47 @@
+module ETL
+  module Control
+    class DatabaseDestination < Destination
+      attr_reader :order
+      def initialize(control, configuration, mapping)
+        super
+        @order = mapping[:order] || order_from_source
+        raise ControlError, "Order required in mapping" unless @order
+        connect
+      end
+      
+      def flush
+        conn = ActiveRecord::Base.connection
+        conn.transaction do
+          buffer.each do |row|
+            names = []
+            values = []
+            order.each do |name|
+              names << name
+              values << "'#{row[name]}'"
+            end
+            q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
+            ETL::Engine.logger.debug("Query: #{q}")
+            conn.execute(q, "Insert row #{current_row}")
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      def close
+        flush
+        ActiveRecord::Base.connection.disconnect!
+      end
+      
+      private
+      def connect
+        ActiveRecord::Base.establish_connection(
+            :adapter  => (configuration[:adapter] || :mysql),
+            :username => (configuration[:username] || 'root'),
+            :host     => (configuration[:host] || 'localhost'),
+            :password => configuration[:password],
+            :database => configuration[:database] 
+        )
+      end
+    end
+  end
+end

data/lib/etl/control/destination/file_destination.rb

@@ -0,0 +1,63 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # File as the final destination.
+    class FileDestination < Destination
+      attr_reader :file, :order
+      attr_accessor :append, :separator, :eol, :enclose
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      def initialize(control, configuration, mapping)
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @append = configuration[:append] ||= false
+        @separator = configuration[:separator] ||= ','
+        @eol = configuration[:eol] ||= "\n"
+        @enclose = configuration[:enclose] ||= nil
+        
+        @order = mapping[:order] || order_from_source
+        raise ControlError, "Order required in mapping" unless @order
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        flush
+        f.close
+      end
+      
+      def flush
+        buffer.each do |row|
+          add_virtuals(row)
+          values = order.collect { |name| row[name] }
+          if !enclose.nil?
+            values.collect! { |v| enclose + v.to_s.gsub(/(#{enclose})/, '\\\\\1') + enclose }
+          end
+          f.write(values.join(separator))
+          f.write(eol)
+        end
+        buffer.clear
+      end
+      
+      private
+      # Get the open file stream
+      def f
+        @f ||= open(file, mode)
+      end
+      
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+      
+      def add_virtuals(row)
+        if mapping[:virtual]
+          mapping[:virtual].each do |key,value|
+            row[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -0,0 +1,27 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # ETL source. Subclasses must implement the <tt>each</tt> method.
+    class Source
+      include Enumerable
+      attr_accessor :control, :configuration, :definition
+      
+      class << self
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.classify}Source")
+        end
+      end
+      
+      # Initialize the Source instance
+      # * <tt>control</tt>: The control object
+      # * <tt>configuration</tt>: The configuration hash
+      # * <tt>definition</tt>: The source layout definition
+      def initialize(control, configuration, definition)
+        @control = control
+        @configuration = configuration
+        @definition = definition
+      end
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control/source/database_source.rb

@@ -0,0 +1,30 @@
+module ETL
+  module Control
+    class DatabaseSource < Source
+      def initialize(control, configuration, definition)
+        super
+        connect
+      end
+      
+      # Returns each row from the source
+      def each
+        conn = ActiveRecord::Base.connection
+        conn.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
+          yield row
+        end
+      end
+      
+      private
+      def connect
+        # set up the DB connection
+        ActiveRecord::Base.establish_connection(
+            :adapter  => (configuration[:adapter] || :mysql),
+            :username => (configuration[:username] || 'root'),
+            :host     => (configuration[:host] || 'localhost'),
+            :password => configuration[:password],
+            :database => configuration[:database] 
+        )
+      end
+    end
+  end
+end

data/lib/etl/control/source/file_source.rb

@@ -0,0 +1,19 @@
+module ETL
+  module Control
+    class FileSource < Source
+      def initialize(control, configuration, definition)
+        super
+        configure
+      end
+      # Returns each row from the source
+      def each
+        @parser.each { |row| yield row }
+      end
+      
+      private
+      def configure
+        @parser = ETL::Parser::Parser.class_for_name(@configuration[:parser]).new(self)
+      end
+    end
+  end
+end

data/lib/etl/engine.rb

@@ -0,0 +1,61 @@
+module ETL
+  class Engine
+  
+    class << self
+      def process(control_file)
+        new().process(control_file)
+      end
+      
+      attr_accessor :logger
+      
+      def logger
+        unless @logger
+          @logger = Logger.new('etl.log')
+          @logger.level = Logger::DEBUG
+        end
+        @logger
+      end
+    end
+  
+    # Process a control file or object.
+    def process(control)
+      control = ETL::Control::Control.resolve(control)
+      
+      pre_process(control)
+      
+      sources = control.sources
+      destinations = control.destinations
+      
+      sources.each do |source|
+        source.each_with_index do |row, index|
+          row.each do |name, value|
+            # execute transforms
+            row[name] = ETL::Transform::Transform.transform(name, value, control.get_transform(name))
+          end
+          # write the row to the destination
+          destinations.each do |destination|
+            destination.write(row)
+          end
+        end
+        destinations.each do |destination|
+          destination.close
+        end
+      end
+      
+      post_process(control)
+    end
+    
+    private
+    def pre_process(control)
+      control.pre_processors.each do |processor|
+        processor.process
+      end
+    end
+    
+    def post_process(control)
+      control.post_processors.each do |processor|
+        processor.process
+      end
+    end
+  end
+end

data/lib/etl/parser.rb

@@ -0,0 +1,2 @@
+require 'etl/parser/parser'
+Dir[File.dirname(__FILE__) + "/parser/*.rb"].each { |file| require(file) }

data/lib/etl/parser/delimited_parser.rb

@@ -0,0 +1,56 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parses delimited files
+    class DelimitedParser < ETL::Parser::Parser
+      include Enumerable
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      def initialize(source)
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        options = {}
+        Dir.glob(file).each do |file|
+          FasterCSV.foreach(file, options) do |raw_row|
+            row = {}
+            raw_row.each_with_index do |record, index|
+              f = fields[index]
+              row[f.name] = convert(f.name, record, f.type)
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        source.definition.each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name], options[:type])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :type
+        def initialize(name, type=:string)
+          @name = name
+          @type = type
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/fixed_width_parser.rb

@@ -0,0 +1,59 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser for fixed with files
+    class FixedWidthParser < ETL::Parser::Parser
+      include Enumerable
+      
+      # Initialize the parser
+      # * <tt>source</tt>: The source object
+      def initialize(source)
+        super
+        configure
+      end
+      
+      # Return each row
+      def each
+        Dir.glob(file).each do |file|
+          open(file).each do |line|
+            row = {}
+            fields.each do |name, f|
+              # TODO make strip optional?
+              row[name] = convert(name, line[f.field_start, f.field_length].strip, f.type)
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Return a map of defined fields
+      def fields
+        @fields ||= {}
+      end
+      
+      private
+      def configure
+        source.definition.each do |field, options|
+          fields[field] = FixedWidthField.new(options[:name], options[:start], options[:end], options[:length], options[:type])
+        end
+      end
+    end
+    
+    class FixedWidthField
+      attr_reader :name, :field_start, :field_end, :field_length, :type
+      def initialize(name, field_start, field_end=nil, field_length=nil, type=nil)
+        @name = name
+        @type = type ||= :string
+        @field_start = field_start - 1
+        if field_end
+          @field_end = field_end
+          @field_length = @field_end - @field_start
+        elsif field_length
+          @field_length = field_length
+          @field_end = @field_start + @field_length
+        else
+          raise DefinitionError, "Either field_end or field_length required"
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/parser.rb

@@ -0,0 +1,43 @@
+module ETL
+  module Parser
+    class Parser
+      class << self
+        # Convert the name (string or symbol) to a parser class.
+        #
+        # Example:
+        #   <tt>class_for_name(:fixed_width)</tt> returns a FixedWidthParser class
+        def class_for_name(name)
+          ETL::Parser.const_get("#{name.to_s.classify}Parser")
+        end
+      end
+      
+      attr_reader :source
+      
+      def initialize(source)
+        @source = source
+      end
+      
+      # Convert the value to the specified type.
+      # 
+      # Parameters:
+      # * <tt>name</tt>: The name of the field
+      # * <tt>value</tt>: The value
+      # * <tt>type</tt>: The type name (:integer, :float, :string)
+      def convert(name, value, type)
+        case type
+        when :integer
+          value.to_i
+        when :float
+          value.to_f
+        else
+          value
+        end
+      end
+      
+      protected
+      def file
+        File.join(File.dirname(source.control.file), source.configuration[:file])
+      end
+    end
+  end
+end

data/lib/etl/processor.rb

@@ -0,0 +1,2 @@
+require 'etl/processor/processor'
+Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }

data/lib/etl/processor/bulk_import_processor.rb

@@ -0,0 +1,39 @@
+module ETL
+  module Processor
+    class BulkImportProcessor < ETL::Processor::Processor
+      attr_reader :file, :target
+      def initialize(control, configuration)
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target]
+        connect
+      end
+      def process
+        conn = ActiveRecord::Base.connection
+        conn.transaction do
+          # Since LOCAL is used this must be allowed by both the client and server
+          conn.execute("LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{target[:table]}")
+        end
+      end
+      private
+      def log
+        unless @log
+          @log = Logger.new(STDOUT)
+          @log.level = Logger::DEBUG
+        end
+        @log
+      end
+      
+      # Connect to the database
+      def connect
+        ActiveRecord::Base.establish_connection(
+            :adapter  => (target[:adapter] || :mysql),
+            :username => (target[:username] || 'root'),
+            :host     => (target[:host] || 'localhost'),
+            :password => target[:password],
+            :database => target[:database] 
+        )
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -0,0 +1,18 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Base class for pre and post processors
+    class Processor
+      def initialize(control, configuration)
+        @control = control
+        @configuration = configuration
+      end
+      protected
+      def control
+        @control
+      end
+      def configuration
+        @configuration
+      end
+    end
+  end
+end

data/lib/etl/transform.rb

@@ -0,0 +1,2 @@
+require 'etl/transform/transform'
+Dir[File.dirname(__FILE__) + "/transform/*.rb"].each { |file| require(file) }

data/lib/etl/transform/decode_transform.rb

@@ -0,0 +1,37 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which decodes coded values
+    class DecodeTransform < ETL::Transform::Transform
+      attr_accessor :decode_table_path, :decode_table_delimiter, :default_value
+      def initialize(control, configuration={})
+        super
+        
+        if configuration[:decode_table_path]
+          configuration[:decode_table_path] = File.join(File.dirname(control.file), configuration[:decode_table_path])
+        end
+        
+        @decode_table_path = (configuration[:decode_table_path] || 'decode.txt')
+        @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
+        @default_value = (configuration[:default_value] || 'No Value')
+      end
+      def transform(value)
+        decode_table[value] || default_value
+      end
+      
+      def decode_table
+        unless @decode_table
+          @decode_table = {}
+          open(decode_table_path).each do |line|
+            code, value = line.strip.split(decode_table_delimiter)
+            if code && code.length > 0
+              @decode_table[code] = value
+            else
+              @default_value = value
+            end
+          end
+        end
+        @decode_table
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,15 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      def initialize(control, configuration={})
+        super
+      end
+      def transform(value)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -0,0 +1,29 @@
+module ETL
+  module Transform
+    class Transform
+      class << self
+        def transform(name, value, transforms)
+          # logger.debug "Transforming field #{name}" if transforms.length > 0
+          transforms.each do |transform|
+            case transform
+            when Proc
+              value = transform.call(value)
+            when Transform
+              value = transform.transform(value)
+            else
+              raise ControlError, "Unsupported transform configuration type: #{transform}"
+            end
+          end
+          value
+        end
+      end
+      
+      attr_reader :control, :configuration
+      
+      def initialize(control, configuration={})
+        @control = control
+        @configuration = configuration
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,9 @@
+module ETL
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: activewarehouse-etl
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-12-06 00:00:00 -05:00
+summary: Pure Ruby ETL package.
+require_paths: 
+- lib
+email: anthonyeden@gmail.com
+homepage: http://activewarehouse.rubyforge.org/etl
+rubyforge_project: activewarehouse
+description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
+autorequire: 
+default_executable: etl
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Anthony Eden
+files: 
+- CHANGELOG
+- README
+- Rakefile
+- bin/etl
+- lib/etl
+- lib/etl.rb
+- lib/etl/commands
+- lib/etl/control
+- lib/etl/control.rb
+- lib/etl/engine.rb
+- lib/etl/parser
+- lib/etl/parser.rb
+- lib/etl/processor
+- lib/etl/processor.rb
+- lib/etl/transform
+- lib/etl/transform.rb
+- lib/etl/version.rb
+- lib/etl/commands/etl.rb
+- lib/etl/control/control.rb
+- lib/etl/control/destination
+- lib/etl/control/destination.rb
+- lib/etl/control/source
+- lib/etl/control/source.rb
+- lib/etl/control/destination/database_destination.rb
+- lib/etl/control/destination/file_destination.rb
+- lib/etl/control/source/database_source.rb
+- lib/etl/control/source/file_source.rb
+- lib/etl/parser/delimited_parser.rb
+- lib/etl/parser/fixed_width_parser.rb
+- lib/etl/parser/parser.rb
+- lib/etl/processor/bulk_import_processor.rb
+- lib/etl/processor/processor.rb
+- lib/etl/transform/decode_transform.rb
+- lib/etl/transform/sha1_transform.rb
+- lib/etl/transform/transform.rb
+test_files: []
+
+rdoc_options: 
+- --exclude
+- .
+extra_rdoc_files: []
+
+executables: 
+- etl
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.7.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.1.5618
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.14.4.5618
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+    version: