salesforce_bulk_client 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c82c0bb26673c09aac04c217c5da99e6a2a20774
-  data.tar.gz: 95a1a7bc0c8008850336d81f520b9c6d07d15425
+  metadata.gz: e8208d93127fa8709da263e47aed025abdc9409f
+  data.tar.gz: d75468507b491d8275ad99c571ba0588b1937d70
 SHA512:
-  metadata.gz: e9d725e0a9e1e3b1c33512258a6ec84edb6e8ad43eac6061ac5a352918e9e71e2fa8162cfc2b9c344611db351e87b6472f7bc071aaf8a940a8e39b2f6401a056
-  data.tar.gz: bae047540a388b2099c03ef687bd2e0ad5415047aa1eee77ffaa328c6c10c86a126c209409aff40e9a54cdaa782ce62582624e2750bf17b1cecefc1c30e236fa
+  metadata.gz: 84bb519249d07c71a5d4d071f625820689d706189f7fd64b3af818ee9e0595960c11de5b089b59208fa3986f3020ba8435a92f16c86d6ecf7d037d2f8abd6b32
+  data.tar.gz: ab2a737bf3be6d9fd7c2dbd81ab71b2a2f5fc28821b4d6ea3f17feb4bed6fa82d2351a5e7d21d3e5251f685940a2e11ea12e6d26b002234064d9c2796739edce

data/README.md

@@ -5,7 +5,8 @@ SalesforceBulkClient is a Ruby gem which allows for integration with the Salesfo
 Although there are many other gems to choose from for this purpose, this gem offers the following features:
 
 * JSON is used for all API requests and responses (instead of XML) in order to reduce message size.
-* Splitting data into batches for inserts, updates, and deletes are handled automatically.
+* Splitting data into batches for inserts, updates, and deletes are handled automatically. Batches will be split
+  further if the 10,000,000 character limit per batch would be exceeded.
 * Large sets of data can be processed without pre-loading multiple batches.
 * Connect using the Restforce client instance.
 
@@ -50,7 +51,7 @@ records = [ { Name: 'Test Account 1' }, { Name: 'Test Account 2' } ]
 result = bulk_client.insert(sobject, records)
 ```
 
-### Updating records
+### Upserting records
 
 ```ruby
 records = [ { Id: '00136000014tyyF', Name: 'Test Account 1' }, { Name: 'Test Account 2' } ]

data/lib/salesforce_bulk_client/job.rb

@@ -1,9 +1,12 @@
 require 'fire_poll'
+require 'multi_json'
 
 module SalesforceBulkClient
 
   class Job
 
+    BATCH_CHARACTER_LIMIT = 10000000
+
     attr_reader :job_id
 
     def initialize(args)
@@ -39,13 +42,46 @@ module SalesforceBulkClient
     def add_batches
       raise 'Records must be an array of hashes.' unless @records.is_a? Array
       @records.each_slice(@batch_size) do |batch|
-        @batch_ids << add_batch(batch)
+        @batch_ids.concat(add_batch(batch))
       end
     end
 
     def add_batch(batch)
-      add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch)
-      add_batch_result.id
+      batch_ids = []
+      batch_groups = []
+      batch_size = MultiJson.dump(batch).size
+      if batch_size <= BATCH_CHARACTER_LIMIT
+        batch_groups << batch
+      else
+
+        # Split batch into sub-batches
+        batch_group = []
+        batch_group_size = MultiJson.dump(batch_group).size
+        batch.each do |record|
+          record_size = MultiJson.dump(record).size
+          if batch_group_size + record_size + 1 > BATCH_CHARACTER_LIMIT
+            batch_groups << batch_group.dup
+            batch_group.clear
+            batch_group_size = MultiJson.dump(batch_group).size
+          end
+          batch_group << record.dup
+          batch_group_size += record_size + 1
+        end
+        
+        # Add remaining records
+        if !batch_group.empty?
+          batch_groups << batch_group.dup
+          batch_group.clear
+        end
+
+      end
+
+      batch_groups.each do |batch_group|
+        add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch_group)
+        batch_ids << add_batch_result.id
+      end
+
+      batch_ids
     end
 
     def check_job_status

data/lib/salesforce_bulk_client/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkClient
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_bulk_client
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - David Massad
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-02 00:00:00.000000000 Z
+date: 2017-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: restforce