salesforce_bulk_client 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ require 'salesforce_bulk_client/version'
2
+ require 'salesforce_bulk_client/client'
3
+
4
+ module SalesforceBulkClient
5
+ end
@@ -0,0 +1,62 @@
1
+ require 'salesforce_bulk_client/connection'
2
+ require 'salesforce_bulk_client/job'
3
+
4
+ module SalesforceBulkClient
5
+ class Client
6
+
7
+ DEFAULT_CLIENT_OPTIONS = { salesforce_api_version: '39.0' }
8
+
9
+ def initialize(restforce_client, options = {})
10
+ options = {}.merge(DEFAULT_CLIENT_OPTIONS).merge(options)
11
+ @connection = SalesforceBulkClient::Connection.new(options[:salesforce_api_version], restforce_client)
12
+ end
13
+
14
+ def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
15
+ do_operation('delete', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
16
+ end
17
+
18
+ def insert(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
19
+ do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
20
+ end
21
+
22
+ def query(sobject, query, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
23
+ do_operation('query', sobject, query,nil, get_response, timeout, batch_size, poll_delay)
24
+ end
25
+
26
+ def update(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
27
+ do_operation('update', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
28
+ end
29
+
30
+ def upsert(sobject, records, external_field, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
31
+ do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, poll_delay)
32
+ end
33
+
34
+ def job_from_id(job_id)
35
+ job = SalesforceBulkClient::Job.new(job_id: job_id, connection: @connection)
36
+ job_status = job.check_job_status
37
+ batches = job.list_batches
38
+ job.instance_variable_set(:@operation, job_status.operation)
39
+ job.instance_variable_set(:@sobject, job_status.object)
40
+ job.instance_variable_set(:@batch_ids, batches.map { |batch_info| batch_info.id })
41
+ job
42
+ end
43
+
44
+ private
45
+
46
+ def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, poll_delay)
47
+ job = SalesforceBulkClient::Job.new(
48
+ operation: operation,
49
+ sobject: sobject,
50
+ records: records,
51
+ external_field: external_field,
52
+ connection: @connection
53
+ )
54
+
55
+ job.create_job(batch_size)
56
+ operation.to_s == 'query' ? job.add_query : job.add_batches
57
+ job.close_job
58
+ job.get_job_result(get_response, timeout, poll_delay)
59
+ end
60
+
61
+ end
62
+ end
@@ -0,0 +1,35 @@
1
+ require 'multi_json'
2
+
3
+ module SalesforceBulkClient
4
+ class Connection
5
+
6
+ def initialize(api_version, restforce_client)
7
+ @restforce_client = restforce_client
8
+ @api_version = api_version
9
+ @path_prefix = "/services/async/#{@api_version}/"
10
+ @restforce_client.authenticate!
11
+ end
12
+
13
+ def post_request(path, post_data, as_json = true)
14
+ authenticate_results = @restforce_client.authenticate!
15
+ response = @restforce_client.post do |request|
16
+ request.url [ @path_prefix, path ].join('/')
17
+ request.headers['Content-Type'] = 'application/json'
18
+ request.headers['X-SFDC-Session'] = authenticate_results.access_token
19
+ request.body = as_json ? MultiJson.dump(post_data) : post_data
20
+ end
21
+ response.body
22
+ end
23
+
24
+ def get_request(path)
25
+ authenticate_results = @restforce_client.authenticate!
26
+ response = @restforce_client.get do |request|
27
+ request.url "#{@path_prefix}#{path}"
28
+ request.headers['Content-Type'] = 'application/json'
29
+ request.headers['X-SFDC-Session'] = authenticate_results.access_token
30
+ end
31
+ response.body
32
+ end
33
+
34
+ end
35
+ end
@@ -0,0 +1,129 @@
1
+ require 'fire_poll'
2
+
3
+ module SalesforceBulkClient
4
+
5
+ class Job
6
+
7
+ attr_reader :job_id
8
+
9
+ def initialize(args)
10
+ @job_id = args[:job_id]
11
+ @operation = args[:operation]
12
+ @sobject = args[:sobject]
13
+ @external_field = args[:external_field]
14
+ @records = args[:records]
15
+ @connection = args[:connection]
16
+ @batch_ids = []
17
+ end
18
+
19
+ def create_job(batch_size)
20
+ @batch_size = batch_size
21
+ create_job_request = { operation: @operation.to_s.downcase, object: @sobject, contentType: 'JSON' }
22
+ if !@external_field.nil?
23
+ create_job_request[:externalIdFieldName] = @external_field
24
+ end
25
+ create_job_result = @connection.post_request('job', create_job_request)
26
+ @job_id = create_job_result.id
27
+ end
28
+
29
+ def close_job
30
+ close_job_request = { state: 'Closed' }
31
+ @connection.post_request("job/#{@job_id}", close_job_request)
32
+ end
33
+
34
+ def add_query
35
+ add_query_result = @connection.post_request("job/#{@job_id}/batch", @records, false)
36
+ @batch_ids << add_query_result.id
37
+ end
38
+
39
+ def add_batches
40
+ raise 'Records must be an array of hashes.' unless @records.is_a? Array
41
+ @records.each_slice(@batch_size) do |batch|
42
+ @batch_ids << add_batch(batch)
43
+ end
44
+ end
45
+
46
+ def add_batch(batch)
47
+ add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch)
48
+ add_batch_result.id
49
+ end
50
+
51
+ def check_job_status
52
+ @connection.get_request("job/#{@job_id}")
53
+ end
54
+
55
+ def check_batch_status(batch_id)
56
+ @connection.get_request("job/#{@job_id}/batch/#{batch_id}")
57
+ end
58
+
59
+ def list_batches
60
+ @connection.get_request("job/#{@job_id}/batch")&.batchInfo
61
+ end
62
+
63
+ def get_job_result(return_result, timeout, poll_delay)
64
+ batch_infos = []
65
+ polling_started = false
66
+ polling_completed = false
67
+ FirePoll.poll("Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}.", timeout) do
68
+ sleep poll_delay if polling_started
69
+ polling_started = true
70
+ job_status = self.check_job_status
71
+ if job_status.state == 'Closed'
72
+ batch_info_map = {}
73
+
74
+ batches_ready = @batch_ids.all? do |batch_id|
75
+ batch_info = batch_info_map[batch_id] = self.check_batch_status(batch_id)
76
+ batch_info.state != 'Queued' && batch_info != 'InProgress'
77
+ end
78
+
79
+ if batches_ready
80
+ @batch_ids.each do |batch_id|
81
+ batch_infos.insert(0, batch_info_map[batch_id])
82
+ @batch_ids.delete(batch_id)
83
+ end
84
+ end
85
+ polling_completed = true if @batch_ids.empty?
86
+ else
87
+ polling_completed = true
88
+ end
89
+ polling_completed
90
+ end
91
+ job_status = self.check_job_status
92
+
93
+ batch_infos.each_with_index do |batch_state, i|
94
+ if batch_state.state == 'Completed' && return_result == true
95
+ batch_infos[i].merge!({ 'response' => self.get_batch_result(batch_state.id)})
96
+ end
97
+ end
98
+
99
+ job_status.merge!({ 'batches' => batch_infos })
100
+ job_status
101
+ end
102
+
103
+ def get_batch_result(batch_id)
104
+ batch_results = @connection.get_request("job/#{@job_id}/batch/#{batch_id}/result")
105
+ results = []
106
+ if @operation.to_s != 'query'
107
+ results = batch_results
108
+ else
109
+ batch_results.each do |batch_result_id|
110
+ results.concat(@connection.get_request("job/#{@job_id}/batch/#{batch_id}/result/#{batch_result_id}"))
111
+ end
112
+ end
113
+ results
114
+ end
115
+
116
+ def each_batch(timeout = 3600, poll_delay = 5)
117
+ job_result = self.get_job_result(false, timeout, poll_delay)
118
+ job_result.batches.each do |batch_info|
119
+ batch_result = nil
120
+ if batch_info.state == 'Completed'
121
+ batch_result = self.get_batch_result(batch_info.id)
122
+ end
123
+ yield(batch_info, batch_result)
124
+ end
125
+ end
126
+
127
+ end
128
+
129
+ end
@@ -0,0 +1,3 @@
1
+ module SalesforceBulkClient
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'salesforce_bulk_client/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'salesforce_bulk_client'
8
+ spec.version = SalesforceBulkClient::VERSION
9
+ spec.authors = ['David Massad']
10
+ spec.email = ['david.massad@fronteraconsulting.net']
11
+ spec.license = 'MIT'
12
+
13
+ spec.summary = 'Salesforce JSON-based Bulk API Client'
14
+ spec.homepage = 'https://github.com/FronteraConsulting/salesforce_bulk_client'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
17
+ f.match(%r{^(test|spec|features)/})
18
+ end
19
+ spec.bindir = 'exe'
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ['lib']
22
+
23
+ spec.add_dependency 'restforce', '~> 2.5', '>= 2.5.3'
24
+ spec.add_dependency 'multi_json', '~> 1.12', '>= 1.12.1'
25
+ spec.add_dependency 'fire_poll', '~> 1.2', '>= 1.2.0'
26
+
27
+ spec.add_development_dependency 'bundler', '~> 1.14'
28
+ spec.add_development_dependency 'rake', '~> 10.0'
29
+ spec.add_development_dependency 'rspec', '~> 3.5'
30
+ spec.add_development_dependency 'simplecov', '~> 0.14.1'
31
+ spec.add_development_dependency 'dotenv', '~> 2.2', '>= 2.2.1'
32
+ spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
33
+ spec.add_development_dependency 'webmock', '~> 3.0', '>= 3.0.1'
34
+ spec.add_development_dependency 'addressable', '~> 2.5', '>= 2.5.1'
35
+ end
metadata ADDED
@@ -0,0 +1,263 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: salesforce_bulk_client
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - David Massad
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2017-05-02 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: restforce
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.5'
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 2.5.3
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '2.5'
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 2.5.3
33
+ - !ruby/object:Gem::Dependency
34
+ name: multi_json
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '1.12'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 1.12.1
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '1.12'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 1.12.1
53
+ - !ruby/object:Gem::Dependency
54
+ name: fire_poll
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '1.2'
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: 1.2.0
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '1.2'
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: 1.2.0
73
+ - !ruby/object:Gem::Dependency
74
+ name: bundler
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '1.14'
80
+ type: :development
81
+ prerelease: false
82
+ version_requirements: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - "~>"
85
+ - !ruby/object:Gem::Version
86
+ version: '1.14'
87
+ - !ruby/object:Gem::Dependency
88
+ name: rake
89
+ requirement: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - "~>"
92
+ - !ruby/object:Gem::Version
93
+ version: '10.0'
94
+ type: :development
95
+ prerelease: false
96
+ version_requirements: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - "~>"
99
+ - !ruby/object:Gem::Version
100
+ version: '10.0'
101
+ - !ruby/object:Gem::Dependency
102
+ name: rspec
103
+ requirement: !ruby/object:Gem::Requirement
104
+ requirements:
105
+ - - "~>"
106
+ - !ruby/object:Gem::Version
107
+ version: '3.5'
108
+ type: :development
109
+ prerelease: false
110
+ version_requirements: !ruby/object:Gem::Requirement
111
+ requirements:
112
+ - - "~>"
113
+ - !ruby/object:Gem::Version
114
+ version: '3.5'
115
+ - !ruby/object:Gem::Dependency
116
+ name: simplecov
117
+ requirement: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - "~>"
120
+ - !ruby/object:Gem::Version
121
+ version: 0.14.1
122
+ type: :development
123
+ prerelease: false
124
+ version_requirements: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - "~>"
127
+ - !ruby/object:Gem::Version
128
+ version: 0.14.1
129
+ - !ruby/object:Gem::Dependency
130
+ name: dotenv
131
+ requirement: !ruby/object:Gem::Requirement
132
+ requirements:
133
+ - - "~>"
134
+ - !ruby/object:Gem::Version
135
+ version: '2.2'
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: 2.2.1
139
+ type: :development
140
+ prerelease: false
141
+ version_requirements: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '2.2'
146
+ - - ">="
147
+ - !ruby/object:Gem::Version
148
+ version: 2.2.1
149
+ - !ruby/object:Gem::Dependency
150
+ name: vcr
151
+ requirement: !ruby/object:Gem::Requirement
152
+ requirements:
153
+ - - "~>"
154
+ - !ruby/object:Gem::Version
155
+ version: '3.0'
156
+ - - ">="
157
+ - !ruby/object:Gem::Version
158
+ version: 3.0.3
159
+ type: :development
160
+ prerelease: false
161
+ version_requirements: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - "~>"
164
+ - !ruby/object:Gem::Version
165
+ version: '3.0'
166
+ - - ">="
167
+ - !ruby/object:Gem::Version
168
+ version: 3.0.3
169
+ - !ruby/object:Gem::Dependency
170
+ name: webmock
171
+ requirement: !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - "~>"
174
+ - !ruby/object:Gem::Version
175
+ version: '3.0'
176
+ - - ">="
177
+ - !ruby/object:Gem::Version
178
+ version: 3.0.1
179
+ type: :development
180
+ prerelease: false
181
+ version_requirements: !ruby/object:Gem::Requirement
182
+ requirements:
183
+ - - "~>"
184
+ - !ruby/object:Gem::Version
185
+ version: '3.0'
186
+ - - ">="
187
+ - !ruby/object:Gem::Version
188
+ version: 3.0.1
189
+ - !ruby/object:Gem::Dependency
190
+ name: addressable
191
+ requirement: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - "~>"
194
+ - !ruby/object:Gem::Version
195
+ version: '2.5'
196
+ - - ">="
197
+ - !ruby/object:Gem::Version
198
+ version: 2.5.1
199
+ type: :development
200
+ prerelease: false
201
+ version_requirements: !ruby/object:Gem::Requirement
202
+ requirements:
203
+ - - "~>"
204
+ - !ruby/object:Gem::Version
205
+ version: '2.5'
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: 2.5.1
209
+ description:
210
+ email:
211
+ - david.massad@fronteraconsulting.net
212
+ executables: []
213
+ extensions: []
214
+ extra_rdoc_files: []
215
+ files:
216
+ - ".gitignore"
217
+ - ".rspec"
218
+ - ".rspec_status"
219
+ - ".travis.yml"
220
+ - Gemfile
221
+ - LICENSE
222
+ - README.md
223
+ - Rakefile
224
+ - bin/console
225
+ - bin/setup
226
+ - fixtures/vcr_cassettes/salesforce/batch_processing.yml
227
+ - fixtures/vcr_cassettes/salesforce/delete.yml
228
+ - fixtures/vcr_cassettes/salesforce/insert.yml
229
+ - fixtures/vcr_cassettes/salesforce/login.yml
230
+ - fixtures/vcr_cassettes/salesforce/query.yml
231
+ - fixtures/vcr_cassettes/salesforce/update.yml
232
+ - fixtures/vcr_cassettes/salesforce/upsert.yml
233
+ - lib/salesforce_bulk_client.rb
234
+ - lib/salesforce_bulk_client/client.rb
235
+ - lib/salesforce_bulk_client/connection.rb
236
+ - lib/salesforce_bulk_client/job.rb
237
+ - lib/salesforce_bulk_client/version.rb
238
+ - salesforce_bulk_client.gemspec
239
+ homepage: https://github.com/FronteraConsulting/salesforce_bulk_client
240
+ licenses:
241
+ - MIT
242
+ metadata: {}
243
+ post_install_message:
244
+ rdoc_options: []
245
+ require_paths:
246
+ - lib
247
+ required_ruby_version: !ruby/object:Gem::Requirement
248
+ requirements:
249
+ - - ">="
250
+ - !ruby/object:Gem::Version
251
+ version: '0'
252
+ required_rubygems_version: !ruby/object:Gem::Requirement
253
+ requirements:
254
+ - - ">="
255
+ - !ruby/object:Gem::Version
256
+ version: '0'
257
+ requirements: []
258
+ rubyforge_project:
259
+ rubygems_version: 2.6.12
260
+ signing_key:
261
+ specification_version: 4
262
+ summary: Salesforce JSON-based Bulk API Client
263
+ test_files: []