salesforce_bulk_client 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c82c0bb26673c09aac04c217c5da99e6a2a20774
4
- data.tar.gz: 95a1a7bc0c8008850336d81f520b9c6d07d15425
3
+ metadata.gz: e8208d93127fa8709da263e47aed025abdc9409f
4
+ data.tar.gz: d75468507b491d8275ad99c571ba0588b1937d70
5
5
  SHA512:
6
- metadata.gz: e9d725e0a9e1e3b1c33512258a6ec84edb6e8ad43eac6061ac5a352918e9e71e2fa8162cfc2b9c344611db351e87b6472f7bc071aaf8a940a8e39b2f6401a056
7
- data.tar.gz: bae047540a388b2099c03ef687bd2e0ad5415047aa1eee77ffaa328c6c10c86a126c209409aff40e9a54cdaa782ce62582624e2750bf17b1cecefc1c30e236fa
6
+ metadata.gz: 84bb519249d07c71a5d4d071f625820689d706189f7fd64b3af818ee9e0595960c11de5b089b59208fa3986f3020ba8435a92f16c86d6ecf7d037d2f8abd6b32
7
+ data.tar.gz: ab2a737bf3be6d9fd7c2dbd81ab71b2a2f5fc28821b4d6ea3f17feb4bed6fa82d2351a5e7d21d3e5251f685940a2e11ea12e6d26b002234064d9c2796739edce
data/README.md CHANGED
@@ -5,7 +5,8 @@ SalesforceBulkClient is a Ruby gem which allows for integration with the Salesfo
5
5
  Although there are many other gems to choose from for this purpose, this gem offers the following features:
6
6
 
7
7
  * JSON is used for all API requests and responses (instead of XML) in order to reduce message size.
8
- * Splitting data into batches for inserts, updates, and deletes are handled automatically.
8
+ * Splitting data into batches for inserts, updates, and deletes are handled automatically. Batches will be split
9
+ further if the 10,000,000 character limit per batch would be exceeded.
9
10
  * Large sets of data can be processed without pre-loading multiple batches.
10
11
  * Connect using the Restforce client instance.
11
12
 
@@ -50,7 +51,7 @@ records = [ { Name: 'Test Account 1' }, { Name: 'Test Account 2' } ]
50
51
  result = bulk_client.insert(sobject, records)
51
52
  ```
52
53
 
53
- ### Updating records
54
+ ### Upserting records
54
55
 
55
56
  ```ruby
56
57
  records = [ { Id: '00136000014tyyF', Name: 'Test Account 1' }, { Name: 'Test Account 2' } ]
@@ -1,9 +1,12 @@
1
1
  require 'fire_poll'
2
+ require 'multi_json'
2
3
 
3
4
  module SalesforceBulkClient
4
5
 
5
6
  class Job
6
7
 
8
+ BATCH_CHARACTER_LIMIT = 10000000
9
+
7
10
  attr_reader :job_id
8
11
 
9
12
  def initialize(args)
@@ -39,13 +42,46 @@ module SalesforceBulkClient
39
42
  def add_batches
40
43
  raise 'Records must be an array of hashes.' unless @records.is_a? Array
41
44
  @records.each_slice(@batch_size) do |batch|
42
- @batch_ids << add_batch(batch)
45
+ @batch_ids.concat(add_batch(batch))
43
46
  end
44
47
  end
45
48
 
46
49
  def add_batch(batch)
47
- add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch)
48
- add_batch_result.id
50
+ batch_ids = []
51
+ batch_groups = []
52
+ batch_size = MultiJson.dump(batch).size
53
+ if batch_size <= BATCH_CHARACTER_LIMIT
54
+ batch_groups << batch
55
+ else
56
+
57
+ # Split batch into sub-batches
58
+ batch_group = []
59
+ batch_group_size = MultiJson.dump(batch_group).size
60
+ batch.each do |record|
61
+ record_size = MultiJson.dump(record).size
62
+ if batch_group_size + record_size + 1 > BATCH_CHARACTER_LIMIT
63
+ batch_groups << batch_group.dup
64
+ batch_group.clear
65
+ batch_group_size = MultiJson.dump(batch_group).size
66
+ end
67
+ batch_group << record.dup
68
+ batch_group_size += record_size + 1
69
+ end
70
+
71
+ # Add remaining records
72
+ if !batch_group.empty?
73
+ batch_groups << batch_group.dup
74
+ batch_group.clear
75
+ end
76
+
77
+ end
78
+
79
+ batch_groups.each do |batch_group|
80
+ add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch_group)
81
+ batch_ids << add_batch_result.id
82
+ end
83
+
84
+ batch_ids
49
85
  end
50
86
 
51
87
  def check_job_status
@@ -1,3 +1,3 @@
1
1
  module SalesforceBulkClient
2
- VERSION = '0.0.1'
2
+ VERSION = '0.0.2'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: salesforce_bulk_client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Massad
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-05-02 00:00:00.000000000 Z
11
+ date: 2017-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: restforce