salesforce_chunker 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bce2e320c05d4d8a6cd855eeedc8eac3bb2fd02a0a2d90be3743d156fc326127
-  data.tar.gz: 88e0b977efdf78f298adceb9a09b5d31b67ecd59e58884e9ac2d649de15ac30b
+  metadata.gz: 8c051e8de05fca9caf7049b418d7366752889787689fe22502f0eb74131cdbf1
+  data.tar.gz: 0df0dffe1900e42f584d3e7c067e2c643582707f6468d6f988743fb6466f506c
 SHA512:
-  metadata.gz: 3294b08e552ac9dad461f13c9678ee48823f8fc3a708db6acabbf6574ad6f0715bcb0947fee78836d9f0d212ec713b555ab017268664d25ee40c1e75a78f939a
-  data.tar.gz: addb0a2b786fc330f1366582e616bf9e63677fe6664873d14189f923c9a96527ee50a17685aab2b8f021b548cbb0afaa20f59eaf6b85a7785633a4f62b5d599f
+  metadata.gz: 46ce51f398903355664620f9ee5274a28b5e74691d5ec5e1f5ff786410a530d547e07d7d35bfae77eb0de34ba690afc1a1732d2a8ec3eff81c87b954e4fe753b
+  data.tar.gz: bc1dc6ebb441f0a1b5e0253a8afd5b15a8d3440d8729228e0a62e136fbdfc1a22be5225908ec863461579709a826e16fa552ee7b3846e0ca4606844e749c6858

data/CHANGELOG.md

@@ -1,10 +1,16 @@
 # CHANGELOG
 
+## 1.2.1 - 2019-06-26
+
+  - Fixed bug in Manual Chunking that could result in larger batches.
+  - Added IOError to the types of errors that are retried.
+  - Removed circular reference and warning about it.
+
 ## 1.2.0 - 2019-06-14
 
   - Added an include_deleted flag to perform a queryAll operation.
   - Disabled explicit GZIP encoding to work with the latest versions of HTTParty.
-  - Added a retry for requests to recover from Net::ReadTimeout errors
+  - Added a retry for requests to recover from Net::ReadTimeout errors.
 
 ## 1.1.1 - 2018-11-26
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    salesforce_chunker (1.2.0)
+    salesforce_chunker (1.2.1)
       httparty (~> 0.15)
 
 GEM

data/README.md

@@ -55,6 +55,9 @@ client = SalesforceChunker::Client.new(
 | security_token | may be required depending on your Salesforce setup |
 | domain | optional. defaults to `"login"`. |
 | salesforce_version | optional. defaults to `"42.0"`. Must be >= `"33.0"` to use PK Chunking. |
+| logger | optional. logger to use. Must be instance of or similar to rails logger. Use here if you want to log all API page requests. |
+| log_output | optional. log output to use. i.e. `STDOUT`. |
+
 
 #### Functions
 
@@ -63,6 +66,7 @@ client = SalesforceChunker::Client.new(
 | query |
 | single_batch_query | calls `query(job_type: "single_batch", **options)`  |
 | primary_key_chunking_query | calls `query(job_type: "primary_key_chunking", **options)` |
+| manual_chunking_query | calls `query(job_type: "manual_chunking", **options)` |
 
 #### Query
 
@@ -88,12 +92,12 @@ end
 | --- | --- | --- |
 | query | required | SOQL query. |
 | object | required | Salesforce Object type. |
-| batch_size | optional | defaults to `100000`. Number of records to process in a batch. (Only for PK Chunking) |
+| batch_size | optional | defaults to `100000`. Number of records to process in a batch. (Not used in Single Batch jobs) |
 | retry_seconds | optional | defaults to `10`. Number of seconds to wait before querying API for updated results. |
-| timeout_seconds | optional | defaults to `3600`. Number of seconds to wait before query is killed. |
+| timeout_seconds | optional | defaults to `3600`. Number of seconds to wait for a batch to process before job is killed. |
 | logger | optional | logger to use. Must be instance of or similar to rails logger. |
 | log_output | optional | log output to use. i.e. `STDOUT`. |
-| job_type | optional | defaults to `"primary_key_chunking"`. Can also be set to `"single_batch"`. |
+| job_type | optional | defaults to `"primary_key_chunking"`. Can also be set to `"single_batch"` or `"manual_chunking`. |
 | include_deleted | optional | defaults to `false`. Whether to include deleted records. |
 
 `query` can either be called with a block, or will return an enumerator:
@@ -102,6 +106,34 @@ end
 names = client.query(query, object, options).map { |result| result["Name"] }
 ```
 
+### A discussion about Single Batch, Primary Key Chunking, and Manual Chunking job types.
+
+One of the advantages of the Salesforce Bulk API over the other Salesforce APIs is the ability for Salesforce to process a number of requests (either queries or uploads) in parallel on their servers. The request chunks are referred to as batches.
+
+#### Single Batch Query
+
+In a single batch query, one SOQL statement is executed as a single batch. This works best if the total number of records to return is fewer than around 100,000 depending on memory usage and the number of fields being returned.
+
+#### Primary Key Chunking Query
+
+In Primary Key Chunking, the internal Salesforce PK chunking flag is used. Salesforce will create a number of batches automatically based on an internal Id index. See https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/async_api_headers_enable_pk_chunking.htm
+
+#### Manual Chunking Query
+
+This approach is called "Manual" Chunking because it is our own implementation of PK Chunking in this gem. The gem downloads a CSV ordered list of all Ids it needs to download, and then uses this list to generate breakpoints that it uses to create batches.
+
+#### Primary Key Chunking Query vs Manual Chunking Query
+
+Advantages of Manual Chunking:
+
+- Manual Chunking takes into account the where clause in the SOQL statement. For example, if you are filtering a small number of a large object count, say 250k out of 20M Objects, then Manual Chunking will split this into 3 batches of max 100k while PK chunking will split this into 200 batches, which will use up batches and API requests against your account and take a longer amount of time.
+- Any object can use Manual Chunking (according to Salesforce, PK chunking is supported for the following objects: Account, Asset, Campaign, CampaignMember, Case, CaseHistory, Contact, Event, EventRelation, Lead, LoginHistory, Opportunity, Task, User, and custom objects.)
+
+Advantages of Primary Key Chunking:
+
+- Primary Key Chunking appears to be slightly faster, if using a PK Chunking eligible object and no where clause.
+- Primary Key Chunking may be less buggy because many more people depend on the Salesforce API than this gem.
+
 ### Under the hood: SalesforceChunker::Job
 
 Using `SalesforceChunker::Job`, you have more direct access to the Salesforce Bulk API functions, such as `create_batch`, `get_batch_statuses`, and `retrieve_batch_results`. This can be used to perform custom tasks, such as upserts or multiple batch queries.

data/lib/salesforce_chunker/connection.rb

@@ -32,7 +32,7 @@ module SalesforceChunker
 
     def post(url, body, headers={})
       @log.info "POST: #{url}"
-      response = self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+      response = self.class.retry_block(log: @log) do
         HTTParty.post(@base_url + url, headers: @default_headers.merge(headers), body: body)
       end
       self.class.check_response_error(response.parsed_response)
@@ -40,7 +40,7 @@ module SalesforceChunker
 
     def get_json(url, headers={})
       @log.info "GET: #{url}"
-      response = self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+      response = self.class.retry_block(log: @log) do
         HTTParty.get(@base_url + url, headers: @default_headers.merge(headers))
       end
       self.class.check_response_error(response.parsed_response)
@@ -48,7 +48,7 @@ module SalesforceChunker
 
     def get(url, headers={})
       @log.info "GET: #{url}"
-      self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+      self.class.retry_block(log: @log) do
         HTTParty.get(@base_url + url, headers: @default_headers.merge(headers)).body
       end
     end
@@ -85,8 +85,9 @@ module SalesforceChunker
 
     MAX_TRIES = 5
     SLEEP_DURATION = 10
+    RESCUED_EXCEPTIONS = [Net::ReadTimeout, IOError]
 
-    def self.retry_block(log: log, tries: MAX_TRIES, sleep_duration: SLEEP_DURATION, rescues:, &block)
+    def self.retry_block(log: Logger.new(nil), tries: MAX_TRIES, sleep_duration: SLEEP_DURATION, rescues: RESCUED_EXCEPTIONS, &block)
       attempt_number = 1
 
       begin

data/lib/salesforce_chunker/manual_chunking_breakpoint_query.rb

@@ -12,18 +12,20 @@ module SalesforceChunker
     end
 
     def get_batch_results(batch_id)
-      retrieve_batch_results(batch_id).each do |result_id|
+      retrieve_batch_results(batch_id).each_with_index do |result_id, result_index|
         results = retrieve_raw_results(batch_id, result_id)
 
         @log.info "Generating breakpoints from CSV results"
-        process_csv_results(results) { |result| yield result }
+        process_csv_results(results, result_index > 0) { |result| yield result }
       end
     end
 
-    def process_csv_results(result)
-      lines = result.each_line
+    def process_csv_results(input, include_first_element)
+      lines = input.each_line
       headers = lines.next
 
+      yield(lines.peek.chomp.gsub("\"", "")) if include_first_element
+
       loop do
         @batch_size.times { lines.next }
         yield(lines.peek.chomp.gsub("\"", ""))

data/lib/salesforce_chunker/version.rb

@@ -1,3 +1,3 @@
 module SalesforceChunker
-  VERSION = "1.2.0"
+  VERSION = "1.2.1"
 end

metadata

@@ -1,94 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_chunker
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Curtis Holmes
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-06-14 00:00:00.000000000 Z
+date: 2019-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
-  type: :runtime
+  name: httparty
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-  type: :development
+  name: minitest
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
-  type: :development
+  name: mocha
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.11'
-  type: :development
+  name: pry
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -127,7 +127,7 @@ homepage: https://github.com/Shopify/salesforce_chunker
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -143,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Salesforce Bulk API Client
 test_files: []