salesforce_bulk_query 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37e885fe7f443a9c7ab9b3006f862e134d0b0fbb
|
4
|
+
data.tar.gz: 0ded5e51edde95927af934907f3a98326f9971f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 19f45a42a8b1f3be36f9681fecae0a70670baa5fe1eab5d39339ec0ced4a299071b5eaefbc04d8776b8eda9fbbb59dee0db2f5eadf477a13fce05e1021d363f0
|
7
|
+
data.tar.gz: b851dabb3da32e4ecdc693471dc9462c112a2488a5bdf78ede5c481cb025220c1032871476ae3f7c7bb98ca437c286c23070bf8fdccb782a229809aca7f5279f
|
data/.gitignore
CHANGED
@@ -60,6 +60,7 @@ module SalesforceBulkQuery
|
|
60
60
|
loop do
|
61
61
|
# get available results and check the status
|
62
62
|
results = query.get_available_results(options)
|
63
|
+
@logger.debug "get_available_results: #{results}"
|
63
64
|
|
64
65
|
# if finished get the result and we're done
|
65
66
|
if results[:succeeded]
|
@@ -93,7 +93,7 @@ module SalesforceBulkQuery
|
|
93
93
|
path = "job/#{@job_id}/batch/#{@batch_id}/result/#{@result_id}"
|
94
94
|
|
95
95
|
if !@result_id
|
96
|
-
raise "batch not finished yet, trying to get result: #{path}"
|
96
|
+
raise "batch #{@batch_id} not finished yet, trying to get result: #{path}"
|
97
97
|
end
|
98
98
|
|
99
99
|
directory_path ||= @@directory_path
|
@@ -106,16 +106,16 @@ module SalesforceBulkQuery
|
|
106
106
|
# count on the soql api
|
107
107
|
# maybe also verify
|
108
108
|
unless skip_verification
|
109
|
-
|
109
|
+
verify
|
110
110
|
end
|
111
|
-
|
111
|
+
@logger.debug "get_result :verification : #{@verification}" if @logger
|
112
112
|
return {
|
113
113
|
:filename => @filename,
|
114
|
-
:
|
114
|
+
:verification => @verification
|
115
115
|
}
|
116
116
|
end
|
117
117
|
|
118
|
-
def
|
118
|
+
def verify
|
119
119
|
api_count = @connection.query_count(@sobject, @start, @stop)
|
120
120
|
# if we weren't able to get the count, fail.
|
121
121
|
if api_count.nil?
|
@@ -125,13 +125,15 @@ module SalesforceBulkQuery
|
|
125
125
|
# count the records in the csv
|
126
126
|
@csv_record_count = Utils.line_count(@filename)
|
127
127
|
|
128
|
-
if @logger && @csv_record_count % 100 == 0
|
129
|
-
@logger.warn "The line count for batch #{@soql} is highly
|
128
|
+
if @logger && @csv_record_count > 0 && @csv_record_count % 100 == 0
|
129
|
+
@logger.warn "The line count for batch id #{@batch_id} soql #{@soql} is highly suspicious: #{@csv_record_count}"
|
130
130
|
end
|
131
131
|
if @logger && @csv_record_count != api_count
|
132
|
-
@logger.warn "The counts for batch #{@soql} don't match. Record count in downloaded csv #{@csv_record_count}, record count on api count(): #{api_count}"
|
132
|
+
@logger.warn "The counts for batch id #{@batch_id}, soql #{@soql} don't match. Record count in downloaded csv #{@csv_record_count}, record count on api count(): #{api_count}"
|
133
|
+
@logger.info "verify result: #{@csv_record_count >= api_count}"
|
134
|
+
|
133
135
|
end
|
134
|
-
|
136
|
+
@verification = (@csv_record_count >= api_count)
|
135
137
|
end
|
136
138
|
|
137
139
|
def to_log
|
@@ -144,6 +144,7 @@ module SalesforceBulkQuery
|
|
144
144
|
|
145
145
|
# download the result
|
146
146
|
result = batch.get_result(options)
|
147
|
+
@logger.info "get_result result: #{result}" if @logger
|
147
148
|
|
148
149
|
# if the verification failed, put it to failed
|
149
150
|
# will never ask about this one again.
|
@@ -173,9 +174,11 @@ module SalesforceBulkQuery
|
|
173
174
|
# cumulate filenames
|
174
175
|
@filenames += downloaded_filenames
|
175
176
|
|
177
|
+
@logger.info "unfinished batches: #{unfinished_batches}\nverification_fail_batches: #{verification_fail_batches}" if @logger
|
178
|
+
|
176
179
|
return {
|
177
180
|
:filenames => @filenames,
|
178
|
-
:unfinished_batches => unfinished_batches,
|
181
|
+
:unfinished_batches => @unfinished_batches,
|
179
182
|
:verification_fail_batches => verification_fail_batches
|
180
183
|
}
|
181
184
|
end
|
@@ -93,15 +93,19 @@ module SalesforceBulkQuery
|
|
93
93
|
|
94
94
|
# download what's available
|
95
95
|
job_results = job.get_available_results(options)
|
96
|
+
@logger.debug "job_results: #{job_results}" if @logger
|
97
|
+
|
96
98
|
unfinished_batches = job_results[:unfinished_batches]
|
99
|
+
verification_fail_batches = job_results[:verification_fail_batches]
|
100
|
+
|
97
101
|
unfinished_subqueries += unfinished_batches.map {|b| b.soql}
|
98
102
|
|
99
103
|
# split to subqueries what needs to be split
|
100
|
-
to_split =
|
104
|
+
to_split = verification_fail_batches
|
101
105
|
to_split += unfinished_batches if job_over_limit
|
102
106
|
|
103
107
|
# delete files associated with batches that failed verification
|
104
|
-
|
108
|
+
verification_fail_batches.each do |b|
|
105
109
|
@logger.info "Deleting #{b.filename}, verification failed."
|
106
110
|
File.delete(b.filename)
|
107
111
|
end
|
@@ -122,6 +126,8 @@ module SalesforceBulkQuery
|
|
122
126
|
if to_split.empty?
|
123
127
|
# done, nothing left
|
124
128
|
jobs_done.push(job)
|
129
|
+
|
130
|
+
@logger.info "#{job.job_id} finished. Nothing to split. unfinished_batches: #{unfinished_batches}, verification_fail_batches: #{verification_fail_batches}" if @logger
|
125
131
|
else
|
126
132
|
# done, some batches needed to be restarted
|
127
133
|
jobs_restarted.push(job)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: salesforce_bulk_query
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Petr Cvengros
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-01-
|
11
|
+
date: 2015-01-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|