salesforce_chunker 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9aabe3e30bb91a83ba214f11c224f003869420e4f5e2bdaf0fd7cc89ad4b669
-  data.tar.gz: fcb13aaa42d91efbfffaa047b6c3f8e21d378fa2a318e5ec4ab4cb29736b52d2
+  metadata.gz: bce2e320c05d4d8a6cd855eeedc8eac3bb2fd02a0a2d90be3743d156fc326127
+  data.tar.gz: 88e0b977efdf78f298adceb9a09b5d31b67ecd59e58884e9ac2d649de15ac30b
 SHA512:
-  metadata.gz: ff8e83600ff22d71455a5ff84fa1e997e7badbb2b4a6db5c885d12748ea8b6867a7adf3570dbc361bc9fbe233f1c22ed7bec9e411815dae8b38026082726609a
-  data.tar.gz: 12023fee018edb1550414fbb16fd918896142fed885970afed558b55fffcd50b13c692ec499ef07a6c7bd3a0c3d0ef50ffb9287fdaa9431e4e08032d76f9a388
+  metadata.gz: 3294b08e552ac9dad461f13c9678ee48823f8fc3a708db6acabbf6574ad6f0715bcb0947fee78836d9f0d212ec713b555ab017268664d25ee40c1e75a78f939a
+  data.tar.gz: addb0a2b786fc330f1366582e616bf9e63677fe6664873d14189f923c9a96527ee50a17685aab2b8f021b548cbb0afaa20f59eaf6b85a7785633a4f62b5d599f

data/.travis.yml

@@ -2,4 +2,4 @@ sudo: false
 language: ruby
 rvm:
   - 2.5.0
-before_install: gem install bundler -v 1.17.1
+before_install: gem install bundler -v 1.17.3

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 1.2.0 - 2019-06-14
+
+  - Added an include_deleted flag to perform a queryAll operation.
+  - Disabled explicit GZIP encoding to work with the latest versions of HTTParty.
+  - Added a retry for requests to recover from Net::ReadTimeout errors
+
 ## 1.1.1 - 2018-11-26
 
   - Reimplemented ManualChunkingQuery using CSV batch results.

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    salesforce_chunker (1.1.1)
-      httparty (~> 0.13)
+    salesforce_chunker (1.2.0)
+      httparty (~> 0.15)
 
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    httparty (0.16.3)
+    httparty (0.17.0)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     metaclass (0.0.4)
     method_source (0.9.0)
     mime-types (3.2.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
+    mime-types-data (3.2019.0331)
     minitest (5.11.3)
     mocha (1.5.0)
       metaclass (~> 0.0.1)
@@ -37,4 +37,4 @@ DEPENDENCIES
   salesforce_chunker!
 
 BUNDLED WITH
-   1.17.1
+   1.17.3

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Shopify
+Copyright (c) 2018-2019 Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -76,6 +76,7 @@ options = {
   logger:           nil,
   log_output:       STDOUT,
   job_type:         "primary_key_chunking",
+  include_deleted:  true,
 }
 
 client.query(options) do |result|
@@ -93,6 +94,7 @@ end
 | logger | optional | logger to use. Must be instance of or similar to rails logger. |
 | log_output | optional | log output to use. i.e. `STDOUT`. |
 | job_type | optional | defaults to `"primary_key_chunking"`. Can also be set to `"single_batch"`. |
+| include_deleted | optional | defaults to `false`. Whether to include deleted records. |
 
 `query` can either be called with a block, or will return an enumerator:
 

data/lib/salesforce_chunker/connection.rb

@@ -21,7 +21,6 @@ module SalesforceChunker
       @default_headers = {
         "Content-Type": "application/json",
         "X-SFDC-Session": result["sessionId"],
-        "Accept-Encoding": "gzip",
       }
     rescue NoMethodError
       raise ConnectionError, response["Envelope"]["Body"]["Fault"]["faultstring"]
@@ -33,19 +32,25 @@ module SalesforceChunker
 
     def post(url, body, headers={})
       @log.info "POST: #{url}"
-      response = HTTParty.post(@base_url + url, headers: @default_headers.merge(headers), body: body)
+      response = self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+        HTTParty.post(@base_url + url, headers: @default_headers.merge(headers), body: body)
+      end
       self.class.check_response_error(response.parsed_response)
     end
 
     def get_json(url, headers={})
       @log.info "GET: #{url}"
-      response = HTTParty.get(@base_url + url, headers: @default_headers.merge(headers))
+      response = self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+        HTTParty.get(@base_url + url, headers: @default_headers.merge(headers))
+      end
       self.class.check_response_error(response.parsed_response)
     end
 
     def get(url, headers={})
       @log.info "GET: #{url}"
-      HTTParty.get(@base_url + url, headers: @default_headers.merge(headers)).body
+      self.class.retry_block(log: @log, rescues: Net::ReadTimeout) do
+        HTTParty.get(@base_url + url, headers: @default_headers.merge(headers)).body
+      end
     end
 
     private
@@ -77,5 +82,26 @@ module SalesforceChunker
         response
       end
     end
+
+    MAX_TRIES = 5
+    SLEEP_DURATION = 10
+
+    def self.retry_block(log: log, tries: MAX_TRIES, sleep_duration: SLEEP_DURATION, rescues:, &block)
+      attempt_number = 1
+
+      begin
+        return block.call
+      rescue *rescues => e
+        log.warn "Retry block attempt #{attempt_number} rescued exception: #{e}"
+        if attempt_number >= tries
+          log.error "Maximum number of tries (#{tries}) was reached in retry_block. Raising exception."
+          raise e
+        else
+          attempt_number += 1
+          sleep sleep_duration
+          retry
+        end
+      end
+    end
   end
 end

data/lib/salesforce_chunker/job.rb

@@ -4,7 +4,7 @@ module SalesforceChunker
   class Job
     attr_reader :batches_count
 
-    QUERY_OPERATIONS = ["query", "queryall"].freeze
+    QUERY_OPERATIONS = ["query", "queryAll"].freeze
     DEFAULT_RETRY_SECONDS = 10
     DEFAULT_TIMEOUT_SECONDS = 3600
 

data/lib/salesforce_chunker/version.rb

@@ -1,3 +1,3 @@
 module SalesforceChunker
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/lib/salesforce_chunker.rb

@@ -29,10 +29,12 @@ module SalesforceChunker
         job_class = SalesforceChunker::PrimaryKeyChunkingQuery
       end
 
+      operation = options[:include_deleted] ? "queryAll" : "query"
+
       job_params = {
         connection: @connection,
         object: object,
-        operation: "query",
+        operation: operation,
         query: query,
         **options.slice(:batch_size, :logger, :log_output)
       }

data/salesforce_chunker.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "httparty", "~> 0.13"
+  spec.add_dependency "httparty", "~> 0.15"
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_chunker
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Curtis Holmes
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2019-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -142,8 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Salesforce Bulk API Client