salesforce_chunker 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63428f8b047e970e10dab25118c83b9445e6c79b7d75cf3fd6552a37d8379cd5
-  data.tar.gz: 543260b73b9c91bed55e7788527553419bc83459d7917119e6b2286d74880a71
+  metadata.gz: b9aabe3e30bb91a83ba214f11c224f003869420e4f5e2bdaf0fd7cc89ad4b669
+  data.tar.gz: fcb13aaa42d91efbfffaa047b6c3f8e21d378fa2a318e5ec4ab4cb29736b52d2
 SHA512:
-  metadata.gz: 918d7303b522c79d351901073ac415fdb273b1e3582a9c6565cdb4878e76fb0744023766b6cacf7792a30f840f9bfe52c989790ceb7dad8e5de080503f34ef22
-  data.tar.gz: 28883b66685b1071a31b826ab23f166f4ec6dd42a86f98d5ae50ce590c3a38bb43ce6a374170408f522835edef009c20cdb01e5198d5091ba7e4c366eb95f7ca
+  metadata.gz: ff8e83600ff22d71455a5ff84fa1e997e7badbb2b4a6db5c885d12748ea8b6867a7adf3570dbc361bc9fbe233f1c22ed7bec9e411815dae8b38026082726609a
+  data.tar.gz: 12023fee018edb1550414fbb16fd918896142fed885970afed558b55fffcd50b13c692ec499ef07a6c7bd3a0c3d0ef50ffb9287fdaa9431e4e08032d76f9a388

data/.travis.yml

@@ -2,4 +2,4 @@ sudo: false
 language: ruby
 rvm:
   - 2.5.0
-before_install: gem install bundler -v 1.16.1
+before_install: gem install bundler -v 1.17.1

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 1.1.1 - 2018-11-26
+
+  - Reimplemented ManualChunkingQuery using CSV batch results.
+  - Changed sleeping and timeout error to only occur when no new results appear.
+  - Added more log info messages with regards to JSON parsing and yielding results.
+
 ## 1.1.0 - 2018-11-06
 
   - Added ManualChunkingQuery, which implements chunking within the gem for any Salesforce field.

data/Gemfile.lock

@@ -1,17 +1,21 @@
 PATH
   remote: .
   specs:
-    salesforce_chunker (1.1.0)
+    salesforce_chunker (1.1.1)
       httparty (~> 0.13)
 
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    httparty (0.16.2)
+    httparty (0.16.3)
+      mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     metaclass (0.0.4)
     method_source (0.9.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
     minitest (5.11.3)
     mocha (1.5.0)
       metaclass (~> 0.0.1)
@@ -27,10 +31,10 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.16)
   minitest (~> 5.0)
-  mocha (~> 1.5.0)
-  pry (~> 0.11.1)
+  mocha (~> 1.5)
+  pry (~> 0.11)
   rake (~> 10.0)
   salesforce_chunker!
 
 BUNDLED WITH
-   1.16.6
+   1.17.1

data/lib/salesforce_chunker.rb

@@ -4,6 +4,7 @@ require "salesforce_chunker/job.rb"
 require "salesforce_chunker/single_batch_job.rb"
 require "salesforce_chunker/primary_key_chunking_query.rb"
 require "salesforce_chunker/manual_chunking_query.rb"
+require "salesforce_chunker/manual_chunking_breakpoint_query.rb"
 require 'logger'
 
 module SalesforceChunker

data/lib/salesforce_chunker/connection.rb

@@ -43,6 +43,11 @@ module SalesforceChunker
       self.class.check_response_error(response.parsed_response)
     end
 
+    def get(url, headers={})
+      @log.info "GET: #{url}"
+      HTTParty.get(@base_url + url, headers: @default_headers.merge(headers)).body
+    end
+
     private
 
     def self.login_soap_request_body(username, password, security_token)

data/lib/salesforce_chunker/job.rb

@@ -1,3 +1,5 @@
+require "json"
+
 module SalesforceChunker
   class Job
     attr_reader :batches_count
@@ -27,20 +29,27 @@ module SalesforceChunker
       downloaded_batches = []
 
       loop do
+        results_downloaded = false
         @log.info "Retrieving batch status information"
         get_completed_batches.each do |batch|
           next if downloaded_batches.include?(batch["id"])
           @log.info "Batch #{downloaded_batches.length + 1} of #{@batches_count || '?'}: " \
             "retrieving #{batch["numberRecordsProcessed"]} records"
-          get_batch_results(batch["id"]) { |result| yield(result) } if batch["numberRecordsProcessed"] > 0
+          if batch["numberRecordsProcessed"].to_i > 0
+            get_batch_results(batch["id"]) { |result| yield(result) }
+            results_downloaded = true
+          end
           downloaded_batches.append(batch["id"])
         end
 
         break if @batches_count && downloaded_batches.length == @batches_count
-        raise TimeoutError, "Timeout during batch processing" if Time.now.utc > timeout_at
 
-        @log.info "Waiting #{retry_seconds} seconds"
-        sleep(retry_seconds)
+        unless results_downloaded
+          raise TimeoutError, "Timeout during batch processing" if Time.now.utc > timeout_at
+
+          @log.info "Waiting #{retry_seconds} seconds"
+          sleep(retry_seconds)
+        end
       end
       
       @log.info "Completed"
@@ -49,14 +58,20 @@ module SalesforceChunker
     def get_completed_batches
       get_batch_statuses.select do |batch|
         raise BatchError, "Batch failed: #{batch["stateMessage"]}" if batch["state"] == "Failed"
-        raise RecordError, "Failed records in batch" if batch["state"] == "Completed" && batch["numberRecordsFailed"] > 0
+        raise RecordError, "Failed records in batch" if batch["state"] == "Completed" && batch["numberRecordsFailed"].to_i > 0
         batch["state"] == "Completed"
       end
     end
 
     def get_batch_results(batch_id)
       retrieve_batch_results(batch_id).each do |result_id|
-        retrieve_results(batch_id, result_id).each do |result|
+        results = retrieve_raw_results(batch_id, result_id)
+
+        @log.info "Parsing JSON response"
+        parsed_results = JSON.parse(results)
+
+        @log.info "Yielding records"
+        parsed_results.each do |result|
           result.tap { |h| h.delete("attributes") }
           yield(result)
         end
@@ -85,6 +100,10 @@ module SalesforceChunker
       @connection.get_json("job/#{@job_id}/batch/#{batch_id}/result/#{result_id}")
     end
 
+    def retrieve_raw_results(batch_id, result_id)
+      @connection.get("job/#{@job_id}/batch/#{batch_id}/result/#{result_id}")
+    end
+
     def close
       body = {"state": "Closed"}
       @connection.post_json("job/#{@job_id}/", body)
@@ -96,7 +115,7 @@ module SalesforceChunker
       body = {
         "operation": @operation,
         "object": object,
-        "contentType": "JSON",
+        "contentType": options[:content_type] || "JSON",
       }
       body[:externalIdFieldName] = options[:external_id] if @operation == "upsert"
       @connection.post_json("job", body, options[:headers].to_h)["id"]

data/lib/salesforce_chunker/manual_chunking_breakpoint_query.rb

@@ -0,0 +1,60 @@
+module SalesforceChunker
+  class ManualChunkingBreakpointQuery < Job
+
+    def initialize(connection:, object:, operation:, query:, **options)
+      @batch_size = options[:batch_size] || 100000
+      super(connection: connection, object: object, operation: operation, **options)
+
+      create_batch(query)
+      @batches_count = 1
+
+      close
+    end
+
+    def get_batch_results(batch_id)
+      retrieve_batch_results(batch_id).each do |result_id|
+        results = retrieve_raw_results(batch_id, result_id)
+
+        @log.info "Generating breakpoints from CSV results"
+        process_csv_results(results) { |result| yield result }
+      end
+    end
+
+    def process_csv_results(result)
+      lines = result.each_line
+      headers = lines.next
+
+      loop do
+        @batch_size.times { lines.next }
+        yield(lines.peek.chomp.gsub("\"", ""))
+      end
+    rescue StopIteration
+      nil
+    end
+
+    def create_batch(payload)
+      @log.info "Creating Id Batch: \"#{payload.gsub(/\n/, " ").strip}\""
+      response = @connection.post("job/#{@job_id}/batch", payload.to_s, {"Content-Type": "text/csv"})
+      response["batchInfo"]["id"]
+    end
+
+    def retrieve_batch_results(batch_id)
+      # XML to JSON wrangling
+      response = super(batch_id)
+      if response["result_list"]["result"].is_a? Array
+        response["result_list"]["result"]
+      else
+        [response["result_list"]["result"]]
+      end
+    end
+
+    def get_batch_statuses
+      # XML to JSON wrangling
+      [@connection.get_json("job/#{@job_id}/batch")["batchInfoList"]["batchInfo"]]
+    end
+
+    def create_job(object, options)
+      super(object, options.merge(content_type: "CSV"))
+    end
+  end
+end

data/lib/salesforce_chunker/manual_chunking_query.rb

@@ -2,14 +2,23 @@ module SalesforceChunker
   class ManualChunkingQuery < Job
 
     def initialize(connection:, object:, operation:, query:, **options)
+      @log = options.delete(:logger) || Logger.new(options[:log_output])
+      @log.progname = "salesforce_chunker"
       batch_size = options[:batch_size] || 100000
       where_clause = self.class.query_where_clause(query)
 
-      super(connection: connection, object: object, operation: operation, **options)
       @log.info "Using Manual Chunking"
-
-      @log.info "Retrieving Ids from records"
-      breakpoints = breakpoints(object, where_clause, batch_size)
+      breakpoint_creation_job = SalesforceChunker::ManualChunkingBreakpointQuery.new(
+        connection: connection,
+        object: object,
+        operation: operation,
+        logger: @log,
+        batch_size: batch_size,
+        query: "Select Id From #{object} #{where_clause} Order By Id Asc",
+      )
+      breakpoints = breakpoint_creation_job.download_results(retry_seconds: 10).to_a
+
+      super(connection: connection, object: object, operation: operation, logger: @log, **options)
 
       @log.info "Creating Query Batches"
       create_batches(query, breakpoints, where_clause)
@@ -17,21 +26,6 @@ module SalesforceChunker
       close
     end
 
-    def get_batch_statuses
-      batches = super
-      batches.delete_if { |batch| batch["id"] == @initial_batch_id && batches.count > 1 }
-    end
-
-    def breakpoints(object, where_clause, batch_size)
-      @batches_count = 1
-      @initial_batch_id = create_batch("Select Id From #{object} #{where_clause} Order By Id Asc")
-
-      download_results(retry_seconds: 10)
-        .with_index
-        .select { |_, i| i % batch_size == 0 && i != 0 }
-        .map { |result, _| result["Id"] }
-    end
-
     def create_batches(query, breakpoints, where_clause)
       if breakpoints.empty?
         create_batch(query)

data/lib/salesforce_chunker/version.rb

@@ -1,3 +1,3 @@
 module SalesforceChunker
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/salesforce_chunker.gemspec

@@ -25,6 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "mocha", "~> 1.5.0"
-  spec.add_development_dependency "pry", "~> 0.11.1"
+  spec.add_development_dependency "mocha", "~> 1.5"
+  spec.add_development_dependency "pry", "~> 0.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_chunker
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Curtis Holmes
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-07 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -72,28 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.1
+        version: '0.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.1
+        version: '0.11'
 description: Salesforce client and extractor designed for handling large amounts of
   data
 email:
@@ -117,6 +117,7 @@ files:
 - lib/salesforce_chunker/connection.rb
 - lib/salesforce_chunker/exceptions.rb
 - lib/salesforce_chunker/job.rb
+- lib/salesforce_chunker/manual_chunking_breakpoint_query.rb
 - lib/salesforce_chunker/manual_chunking_query.rb
 - lib/salesforce_chunker/primary_key_chunking_query.rb
 - lib/salesforce_chunker/single_batch_job.rb