activewarehouse-etl 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +4 -1
- data/lib/etl/control/source/database_source.rb +9 -11
- data/lib/etl/processor/check_exist_processor.rb +3 -2
- data/lib/etl/version.rb +1 -1
- metadata +1 -1
data/CHANGELOG
CHANGED
@@ -95,4 +95,7 @@
|
|
95
95
|
* Sources now provide a trigger file which can be used to indicate that the original source
|
96
96
|
data has been completely extracted to the local file system. This is useful if you need to
|
97
97
|
recover from a failed ETL process.
|
98
|
-
* Updated README
|
98
|
+
* Updated README
|
99
|
+
|
100
|
+
0.7.1 - Apr 8, 2007
|
101
|
+
* Fixed source caching
|
@@ -85,11 +85,13 @@ module ETL #:nodoc:
|
|
85
85
|
# error.
|
86
86
|
def each(&block)
|
87
87
|
if read_locally # Read from the last stored source
|
88
|
-
|
88
|
+
ETL::Engine.logger.debug "Reading from local cache"
|
89
|
+
read_rows(local_file, &block)
|
89
90
|
else # Read from the original source
|
90
91
|
if store_locally
|
91
|
-
|
92
|
-
|
92
|
+
file = local_file
|
93
|
+
write_local(file)
|
94
|
+
read_rows(file, &block)
|
93
95
|
else
|
94
96
|
connection.select_all(query).each do |row|
|
95
97
|
row = Row.new(row.symbolize_keys)
|
@@ -101,11 +103,9 @@ module ETL #:nodoc:
|
|
101
103
|
|
102
104
|
private
|
103
105
|
# Read rows from the local cache
|
104
|
-
def read_rows
|
105
|
-
file
|
106
|
-
|
107
|
-
File.exists?(file) or raise "Local cache file not found"
|
108
|
-
File.exists?(local_file_trigger(file)) or raise "Local cache trigger file not found"
|
106
|
+
def read_rows(file)
|
107
|
+
raise "Local cache file not found" unless File.exists?(file)
|
108
|
+
raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
|
109
109
|
|
110
110
|
t = Benchmark.realtime do
|
111
111
|
FasterCSV.open(file, :headers => true).each do |row|
|
@@ -120,9 +120,7 @@ module ETL #:nodoc:
|
|
120
120
|
end
|
121
121
|
|
122
122
|
# Write rows to the local cache
|
123
|
-
def write_local
|
124
|
-
file = local_file
|
125
|
-
|
123
|
+
def write_local(file)
|
126
124
|
lines = 0
|
127
125
|
t = Benchmark.realtime do
|
128
126
|
FasterCSV.open(file, 'w') do |f|
|
@@ -51,17 +51,18 @@ module ETL #:nodoc:
|
|
51
51
|
# Process the row
|
52
52
|
def process(row)
|
53
53
|
return row unless should_check?
|
54
|
+
connection = ActiveRecord::Base.connection
|
54
55
|
q = "SELECT * FROM #{table} WHERE "
|
55
56
|
conditions = []
|
56
57
|
row.each do |k,v|
|
57
58
|
if columns.nil? || columns.include?(k.to_sym)
|
58
|
-
conditions << "#{k} = '#{v}'" unless skip?(k)
|
59
|
+
conditions << "#{k} = '#{connection.quote_value(v)}'" unless skip?(k)
|
59
60
|
end
|
60
61
|
end
|
61
62
|
q << conditions.join(" AND ")
|
62
63
|
|
63
64
|
#puts "query: #{q}"
|
64
|
-
result =
|
65
|
+
result = connection.select_one(q)
|
65
66
|
return row if result.nil?
|
66
67
|
end
|
67
68
|
end
|
data/lib/etl/version.rb
CHANGED