activewarehouse-etl 0.7.0 → 0.7.1

data/CHANGELOG

@@ -95,4 +95,7 @@
 * Sources now provide a trigger file which can be used to indicate that the original source
   data has been completely extracted to the local file system. This is useful if you need to
   recover from a failed ETL process.
-* Updated README
+* Updated README
+
+0.7.1 - Apr 8, 2007
+* Fixed source caching

data/lib/etl/control/source/database_source.rb

@@ -85,11 +85,13 @@ module ETL #:nodoc:
       # error.
       def each(&block)
         if read_locally # Read from the last stored source
-          read_rows(&block)
+          ETL::Engine.logger.debug "Reading from local cache"
+          read_rows(local_file, &block)
         else # Read from the original source
           if store_locally
-            write_local
-            read_rows(&block)
+            file = local_file
+            write_local(file)
+            read_rows(file, &block)
           else
             connection.select_all(query).each do |row|
               row = Row.new(row.symbolize_keys)
@@ -101,11 +103,9 @@ module ETL #:nodoc:
       
       private
       # Read rows from the local cache
-      def read_rows
-        file = local_file
-        
-        File.exists?(file) or raise "Local cache file not found"
-        File.exists?(local_file_trigger(file)) or raise "Local cache trigger file not found"
+      def read_rows(file)
+        raise "Local cache file not found" unless File.exists?(file)
+        raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
         
         t = Benchmark.realtime do
           FasterCSV.open(file, :headers => true).each do |row|
@@ -120,9 +120,7 @@ module ETL #:nodoc:
       end
       
       # Write rows to the local cache
-      def write_local
-        file = local_file
-      
+      def write_local(file)
         lines = 0
         t = Benchmark.realtime do
           FasterCSV.open(file, 'w') do |f|

data/lib/etl/processor/check_exist_processor.rb

@@ -51,17 +51,18 @@ module ETL #:nodoc:
       # Process the row
       def process(row)
         return row unless should_check?
+        connection = ActiveRecord::Base.connection
         q = "SELECT * FROM #{table} WHERE "
         conditions = []
         row.each do |k,v| 
           if columns.nil? || columns.include?(k.to_sym)
-            conditions << "#{k} = '#{v}'" unless skip?(k)
+            conditions << "#{k} = '#{connection.quote_value(v)}'" unless skip?(k)
           end
         end
         q << conditions.join(" AND ")
       
         #puts "query: #{q}"
-        result = ActiveRecord::Base.connection.select_one(q)
+        result = connection.select_one(q)
         return row if result.nil?
       end
     end

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 7
-    TINY  = 0
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,7 +3,7 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.7.0
+  version: 0.7.1
 date: 2007-04-08 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: