salesforce_bulk_query 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 749b0a76eead05e1f7caf14a69107f54941fc05b
-  data.tar.gz: 49ca7c2a0ae1e4558f512d186db4ab394ca2e802
+  metadata.gz: 37e885fe7f443a9c7ab9b3006f862e134d0b0fbb
+  data.tar.gz: 0ded5e51edde95927af934907f3a98326f9971f9
 SHA512:
-  metadata.gz: e1b7e9377eaa96b7b6c6641aa581c625339c70b0d33a9420cf3ade36f2667d266db4df11997846642bee14ddb580c0eb0453d5382ab62cdf720309f70f0b551c
-  data.tar.gz: 4945bc91e29daa3e5deb36e51e8c7602d734a236476f9a7102e44a6c8411885e01cbfd039c50fca12d27a5cb28b052d8664254fbe0a08265607a7d29246653d3
+  metadata.gz: 19f45a42a8b1f3be36f9681fecae0a70670baa5fe1eab5d39339ec0ced4a299071b5eaefbc04d8776b8eda9fbbb59dee0db2f5eadf477a13fce05e1021d363f0
+  data.tar.gz: b851dabb3da32e4ecdc693471dc9462c112a2488a5bdf78ede5c481cb025220c1032871476ae3f7c7bb98ca437c286c23070bf8fdccb782a229809aca7f5279f

data/.gitignore

@@ -1,4 +1,4 @@
-env_setup.sh
+*env_setup.sh
 Gemfile.lock
 *.gem
 hh*

data/lib/salesforce_bulk_query.rb

@@ -60,6 +60,7 @@ module SalesforceBulkQuery
       loop do
         # get available results and check the status
         results = query.get_available_results(options)
+        @logger.debug "get_available_results: #{results}"
 
         # if finished get the result and we're done
         if results[:succeeded]

data/lib/salesforce_bulk_query/batch.rb

@@ -93,7 +93,7 @@ module SalesforceBulkQuery
       path = "job/#{@job_id}/batch/#{@batch_id}/result/#{@result_id}"
 
       if !@result_id
-        raise "batch not finished yet, trying to get result: #{path}"
+        raise "batch #{@batch_id} not finished yet, trying to get result: #{path}"
       end
 
       directory_path ||= @@directory_path
@@ -106,16 +106,16 @@ module SalesforceBulkQuery
       # count on the soql api
       # maybe also verify
       unless skip_verification
-        @verfication = verification
+        verify
       end
-
+      @logger.debug "get_result :verification : #{@verification}" if @logger
       return {
         :filename => @filename,
-        :verfication => @verfication
+        :verification => @verification
       }
     end
 
-    def verification
+    def verify
       api_count = @connection.query_count(@sobject, @start, @stop)
       # if we weren't able to get the count, fail.
       if api_count.nil?
@@ -125,13 +125,15 @@ module SalesforceBulkQuery
       # count the records in the csv
       @csv_record_count = Utils.line_count(@filename)
 
-      if @logger && @csv_record_count % 100 == 0
-        @logger.warn "The line count for batch #{@soql} is highly suspicius: #{@csv_record_count}"
+      if @logger && @csv_record_count > 0 && @csv_record_count % 100 == 0
+        @logger.warn "The line count for batch id #{@batch_id} soql #{@soql} is highly suspicious: #{@csv_record_count}"
       end
       if @logger && @csv_record_count != api_count
-        @logger.warn "The counts for batch #{@soql} don't match. Record count in downloaded csv #{@csv_record_count}, record count on api count(): #{api_count}"
+        @logger.warn "The counts for batch id #{@batch_id}, soql #{@soql} don't match. Record count in downloaded csv #{@csv_record_count}, record count on api count(): #{api_count}"
+        @logger.info "verify result: #{@csv_record_count >= api_count}"
+
       end
-      return @csv_record_count >= api_count
+      @verification = (@csv_record_count >= api_count)
     end
 
     def to_log

data/lib/salesforce_bulk_query/job.rb

@@ -144,6 +144,7 @@ module SalesforceBulkQuery
 
           # download the result
           result = batch.get_result(options)
+          @logger.info "get_result result: #{result}" if @logger
 
           # if the verification failed, put it to failed
           # will never ask about this one again.
@@ -173,9 +174,11 @@ module SalesforceBulkQuery
       # cumulate filenames
       @filenames += downloaded_filenames
 
+      @logger.info "unfinished batches: #{unfinished_batches}\nverification_fail_batches: #{verification_fail_batches}" if @logger
+
       return {
         :filenames => @filenames,
-        :unfinished_batches => unfinished_batches,
+        :unfinished_batches => @unfinished_batches,
         :verification_fail_batches => verification_fail_batches
       }
     end

data/lib/salesforce_bulk_query/query.rb

@@ -93,15 +93,19 @@ module SalesforceBulkQuery
 
         # download what's available
         job_results = job.get_available_results(options)
+        @logger.debug "job_results: #{job_results}" if @logger
+
         unfinished_batches = job_results[:unfinished_batches]
+        verification_fail_batches = job_results[:verification_fail_batches]
+
         unfinished_subqueries += unfinished_batches.map {|b| b.soql}
 
         # split to subqueries what needs to be split
-        to_split = job_results[:verification_fail_batches]
+        to_split = verification_fail_batches
         to_split += unfinished_batches if job_over_limit
 
         # delete files associated with batches that failed verification
-        job_results[:verification_fail_batches].each do |b|
+        verification_fail_batches.each do |b|
           @logger.info "Deleting #{b.filename}, verification failed."
           File.delete(b.filename)
         end
@@ -122,6 +126,8 @@ module SalesforceBulkQuery
           if to_split.empty?
             # done, nothing left
             jobs_done.push(job)
+
+            @logger.info "#{job.job_id} finished. Nothing to split. unfinished_batches: #{unfinished_batches}, verification_fail_batches: #{verification_fail_batches}" if @logger
           else
             # done, some batches needed to be restarted
             jobs_restarted.push(job)

data/lib/salesforce_bulk_query/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkQuery
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_bulk_query
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Petr Cvengros
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-27 00:00:00.000000000 Z
+date: 2015-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json