activewarehouse-etl 0.9.5.rc1 → 1.0.0.rc1

data/lib/etl/commands/etl.rb

@@ -80,6 +80,7 @@ def execute
     ARGV.each do |f|
       ETL::Engine.realtime_activity = true
       ETL::Engine.process(f)
+      exit(ETL::Engine.exit_code) if ETL::Engine.exit_code
     end
   
     puts "ETL process complete\n\n"

data/lib/etl/control/control.rb

@@ -283,7 +283,7 @@ module ETL #:nodoc:
         end
         
         def parse_text(text)
-          control = ETL::Control::Control.new(nil)
+          control = ETL::Control::Control.new('no-file')
           eval(text, Context.create(control), 'inline')
           control.validate
           control

data/lib/etl/control/destination.rb

@@ -94,16 +94,7 @@ module ETL #:nodoc:
       
       # Get the order of elements from the source order
       def order_from_source
-        order = []
-        control.sources.first.definition.each do |item|
-          case item
-          when Hash
-            order << item[:name]
-          else
-            order << item
-          end
-        end
-        order
+        control.sources.first.order
       end
       
       # Return true if the row is allowed. The row will not be allowed if the
@@ -135,7 +126,7 @@ module ETL #:nodoc:
 
       # returns the fields that are required to identify an SCD
       def scd_required_fields
-        if scd?
+        if scd? and scd_type == 2
          [scd_effective_date_field, scd_end_date_field, scd_latest_version_field]
         else
          []
@@ -249,7 +240,7 @@ module ETL #:nodoc:
             when Symbol
               generator = generators[key] ||= ETL::Generator::Generator.class_for_name(value).new(options)
               row[key] = generator.next
-            when Proc
+            when Proc, Method
               row[key] = value.call(row)
             else
               if value.is_a?(ETL::Generator::Generator)
@@ -288,12 +279,10 @@ module ETL #:nodoc:
         statement = []
         values = []
         natural_key.each do |nk|
-          statement << "#{nk} = ?"
-          values << row[nk]
+          statement << "#{nk} = #{ActiveRecord::Base.send(:quote_bound_value, row[nk], connection)}"
         end
         statement = statement.join(" AND ")
-        x=ActiveRecord::Base.send(:sanitize_sql_array, [statement, *values])
-        return x
+        return statement
       end
       
       # Do all the steps required when a SCD *has* changed.  Exact steps

data/lib/etl/control/destination/csv_destination.rb

@@ -0,0 +1,122 @@
+# This source file contains the ETL::Control::CsvDestination
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # CSV File as the final destination.
+    class CsvDestination < Destination
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # The separator
+      attr_accessor :separator
+      
+      # The end of line marker
+      attr_accessor :eol
+      
+      # The enclosure character
+      attr_accessor :enclose
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:separator</tt>: Record separator (default is a comma)
+      # * <tt>:eol</tt>: End of line marker (default is \n)
+      # * <tt>:enclose</tt>: Set to true of false
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
+      def initialize(control, configuration, mapping={})
+        super
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+        @append = configuration[:append] ||= false
+        @separator = configuration[:separator] ||= ','
+        @eol = configuration[:eol] ||= "\n"
+        @enclose = true & configuration[:enclose]
+        @unique = configuration[:unique] ? configuration[:unique] + scd_required_fields : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @write_header = configuration[:write_header]
+        @order = mapping[:order] + scd_required_fields if mapping[:order]
+        @order.uniq! unless @order.nil?
+      end
+
+      def order
+        @order ||= order_from_source
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        buffer << append_rows if append_rows
+        flush
+        f.close
+      end
+      
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        if @write_header && !@header_written
+          f << order
+          @header_written = true
+        end
+
+        buffer.flatten.each do |row|
+          #puts "row change type: #{row.change_type}"
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          f << values
+        end
+        f.flush
+        buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
+      end
+      
+      private
+      # Get the open file stream
+      def f
+        @f ||= FasterCSV.open(file, mode, options)
+      end
+      
+      def options
+        @options ||= {
+          :col_sep => separator,
+          :row_sep => eol,
+          :force_quotes => enclose
+        }
+      end
+      
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/destination/excel_destination.rb

@@ -1,4 +1,4 @@
-require 'spreadsheet'
+optional_require 'spreadsheet'
 
 module ETL
   module Control

data/lib/etl/control/destination/insert_update_database_destination.rb

@@ -74,11 +74,14 @@ module ETL #:nodoc:
             res = conn.execute(q, "Select row #{current_row}")
             none = true
             
-            case conn
-              when ActiveRecord::ConnectionAdapters::PostgreSQLAdapter;
+            case conn.class.name
+              when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
                 res.each { none = false }
-              when ActiveRecord::ConnectionAdapters::MysqlAdapter;
+              when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
                 res.each_hash { none = false }
+                res.free
+              when "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+                res.each { none = false }
               else raise "Unsupported adapter #{conn.class} for this destination"
             end
 

data/lib/etl/control/destination/yaml_destination.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class YamlDestination < Destination
+      attr_reader :file, :append, :only, :except
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      #
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:only</tt>
+      # * <tt>:except</tt>
+      def initialize(control, configuration, mapping={})
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @append = configuration[:append] ||= false
+        @only = configuration[:only]
+        @except = configuration[:except]
+        raise ControlError, "the :only and :except options must be used seperately, do not specify both" if @only && @except
+      end
+
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        flush
+        f.close
+      end
+
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          # add any virtual fields
+          add_virtuals!(row)
+
+          yaml = {}
+          row.each do |key, value|
+            next if only && !only.include?(key)
+            next if except && except.include?(key)
+
+            case value
+            when Date, Time, DateTime
+              value = value.to_s(:db)
+            end
+
+            yaml[key] = value
+          end
+         
+          # write the values
+          YAML.dump(yaml, f)
+        end
+        f.flush
+        buffer.clear
+      end
+
+      private
+      # Get the open file stream
+      def f
+        @f ||= File.open(file, mode)
+      end
+
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -40,8 +40,7 @@ module ETL #:nodoc:
         @configuration = configuration
         @definition = definition
         
-        @store_locally = true
-        @store_locally = configuration[:store_locally] unless configuration[:store_locally].nil?
+        @store_locally = configuration[:store_locally].nil? ? true : configuration[:store_locally]
       end
       
       # Get an array of errors that occur during reading from the source
@@ -87,9 +86,32 @@ module ETL #:nodoc:
         File.join(local_directory, File.basename(last_local_file_trigger, '.trig'))
       end
       
-      # Get the last local file trigger
+      # Get the last local file trigger filename using timestamp in filenames.
+      # Filename is in the format YYYYMMDDHHMMSS.csv.trig, but in the case of a
+      # file source there is an unpadded sequence number before the file
+      # extension.  This code may not return the correct "last" file in that
+      # case (in particular when there are 10 or more source files).  However,
+      # at this point only the database source calls the method, and it wouldn't
+      # make sense for a file source to use it if multiple files are expected
       def last_local_file_trigger
-        Dir.glob(File.join(local_directory, '*.trig')).last
+        trig_files = []
+        trig_ext = '.csv.trig'
+
+        # Store the basename (without extension) of all files that end in the
+        # desired extension
+        Dir.glob(File.join(local_directory, "*" + trig_ext)) do |f|
+            # Extract the basename of each file with the extension snipped off
+            trig_files << File.basename(f, trig_ext) if File.file?(f)
+        end
+
+        # Throw an exception if no trigger files are available
+        raise "Local cache trigger file not found" if trig_files.empty?
+
+        # Sort trigger file strings and get the last one
+        last_trig = trig_files.sort {|a,b| a <=> b}.last
+
+        # Return the file path including extension
+        File.join(local_directory, last_trig + trig_ext)
       end
       
       # Get the local trigger file that is used to indicate that the file has
@@ -103,6 +125,19 @@ module ETL #:nodoc:
         Engine.read_locally
       end
       
+      # Get the order of fields that this source will present to the pipeline
+      def order
+        order = []
+        definition.each do |item|
+          case item
+          when Hash
+            order << item[:name]
+          else
+            order << item
+          end
+        end
+        order
+      end
     end
   end
 end

data/lib/etl/control/source/database_source.rb

@@ -203,7 +203,12 @@ module ETL #:nodoc:
       end
       
       def query_rows
-        @query_rows ||= connection.select_all(query)
+        return @query_rows if @query_rows
+        if (configuration[:mysqlstream] == true)
+          MySqlStreamer.new(query,@target)
+        else
+          connection.select_all(query)
+        end
       end
       
       # Get the database connection to use

data/lib/etl/control/source/file_source.rb

@@ -53,6 +53,10 @@ module ETL #:nodoc:
         end
       end
       
+      def order
+        @parser.fields.collect {|field| field.name}
+      end
+
       private
       # Copy source data to a local directory structure
       def copy_sources

data/lib/etl/control/source/mysql_streamer.rb

@@ -0,0 +1,31 @@
+require 'open3'
+
+class MySqlStreamer
+
+	def initialize(query, target)
+		@query = query
+		@name = target
+	end
+
+	def each
+		puts "Using the Streaming MySQL from the command line"
+		keys = nil
+		connection_configuration = ETL::Base.configurations[@name.to_s]
+		mysql_command = """mysql --quick -h #{connection_configuration["host"]} -u #{connection_configuration["username"]} -e \"#{@query.gsub("\n","")}\" -D #{connection_configuration["database"]} --password=#{connection_configuration["password"]} -B"""
+		Open3.popen3(mysql_command) do |stdin, out, err, external|
+			until (line = out.gets).nil? do
+				line = line.gsub("\n","")
+				if keys.nil?
+					keys = line.split("\t")
+				else
+					hash = Hash[keys.zip(line.split("\t"))]
+					yield hash
+				end
+			end
+			error = err.gets
+			if (!error.nil? && error.strip.length > 0)
+				throw error
+			end
+		end
+	end
+end

data/lib/etl/engine.rb

@@ -84,6 +84,9 @@ module ETL #:nodoc:
         Time.now.strftime("%Y%m%d%H%M%S")
       end
 
+      # exit code to be passed to the command line
+      attr_accessor :exit_code
+
       # The current source
       attr_accessor :current_source
 
@@ -200,9 +203,11 @@ module ETL #:nodoc:
       
       # Establish the named connection and return the database specific connection
       def establish_connection(name)
+        raise ETL::ETLError, "Connection with no name requested. Is there a missing :target parameter somewhere?" if name.blank?
+        
         logger.debug "Establishing connection to #{name}"
         conn_config = ETL::Base.configurations[name.to_s]
-        raise ETL::ETLError, "No connection found for #{name}" unless conn_config
+        raise ETL::ETLError, "Cannot find connection named #{name.inspect}" unless conn_config
         connection_method = "#{conn_config['adapter']}_connection"
         ETL::Base.send(connection_method, conn_config)
       end
@@ -255,15 +260,18 @@ module ETL #:nodoc:
     # * ETL::Batch::Batch instance
     def process(file)
       case file
-      when String
-        process(File.new(file))
-      when File
-        process_control(file) if file.path =~ /.ctl$/
-        process_batch(file) if file.path =~ /.ebf$/
-      when ETL::Control::Control
-        process_control(file)
-      when ETL::Batch::Batch
-        process_batch(file)
+        when String
+          process(File.new(file))
+        when File
+          case file.path
+            when /.ctl$/; process_control(file)
+            when /.ebf$/; process_batch(file)
+            else raise RuntimeError, "Unsupported file type - #{file.path}"
+          end
+        when ETL::Control::Control
+          process_control(file)
+        when ETL::Batch::Batch
+          process_batch(file)
       else
         raise RuntimeError, "Process object must be a String, File, Control 
         instance or Batch instance"
@@ -338,14 +346,15 @@ module ETL #:nodoc:
               control.after_read_processors.each do |processor|
                 processed_rows = []
                 rows.each do |row|
-                  processed_rows << processor.process(row)
+                  processed_rows << processor.process(row) unless empty_row?(row)
                 end
-                rows = processed_rows.flatten
+                rows = processed_rows.flatten.compact
               end
             rescue => e
               msg = "Error processing rows after read from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
               errors << msg
               Engine.logger.error(msg)
+              e.backtrace.each { |line| Engine.logger.error(line) }
               exceeded_error_threshold?(control) ? break : next
             end
           end
@@ -355,9 +364,12 @@ module ETL #:nodoc:
             begin
               Engine.logger.debug "Executing transforms"
               rows.each do |row|
-                control.transforms.each do |transform|
-                  name = transform.name.to_sym
-                  row[name] = transform.transform(name, row[name], row)
+                # only do the transform if there is a row
+                unless empty_row?(row)
+                  control.transforms.each do |transform|
+                    name = transform.name.to_sym
+                    row[name] = transform.transform(name, row[name], row)
+                  end
                 end
               end
             rescue ResolverError => e
@@ -384,7 +396,9 @@ module ETL #:nodoc:
               Engine.logger.debug "Processing before write"
               control.before_write_processors.each do |processor|
                 processed_rows = []
-                rows.each { |row| processed_rows << processor.process(row) }
+                rows.each do |row|
+                  processed_rows << processor.process(row) unless empty_row?(row)
+                end
                 rows = processed_rows.flatten.compact
               end
             rescue => e
@@ -420,7 +434,7 @@ module ETL #:nodoc:
         
         if exceeded_error_threshold?(control)
           say_on_own_line "Exiting due to exceeding error threshold: #{control.error_threshold}"
-          return
+          ETL::Engine.exit_code = 1
         end
         
       end
@@ -434,7 +448,7 @@ module ETL #:nodoc:
         execute_screens(control)
       rescue FatalScreenError => e
         say "Fatal screen error during job execution: #{e.message}"
-        exit
+        ETL::Engine.exit_code = 2
       rescue ScreenError => e
         say "Screen error during job execution: #{e.message}"
         return
@@ -456,7 +470,7 @@ module ETL #:nodoc:
         execute_screens(control, :after_post_process)
       rescue FatalScreenError => e
         say "Fatal screen error during job execution: #{e.message}"
-        exit
+        ETL::Engine.exit_code = 3
       rescue ScreenError => e
         say "Screen error during job execution: #{e.message}"
         return
@@ -477,12 +491,18 @@ module ETL #:nodoc:
       # ETL::Transform::Transform.benchmarks.each do |klass, t|
 #         say "Avg #{klass}: #{Engine.rows_read/t} rows/sec"
 #       end
-      
+
+      ActiveRecord::Base.verify_active_connections!
       ETL::Engine.job.completed_at = Time.now
       ETL::Engine.job.status = (errors.length > 0 ? 'completed with errors' : 'completed')
       ETL::Engine.job.save!
     end
     
+    def empty_row?(row)
+      # unsure about why it should respond to :[] - keeping it just in case for the moment
+      row.nil? || !row.respond_to?(:[])
+    end
+    
     private
     # Return true if the error threshold is exceeded
     def exceeded_error_threshold?(control)