salesforce_bulk_client 0.0.1

data/lib/salesforce_bulk_client.rb

@@ -0,0 +1,5 @@
+require 'salesforce_bulk_client/version'
+require 'salesforce_bulk_client/client'
+
+module SalesforceBulkClient
+end

data/lib/salesforce_bulk_client/client.rb

@@ -0,0 +1,62 @@
+require 'salesforce_bulk_client/connection'
+require 'salesforce_bulk_client/job'
+
+module SalesforceBulkClient
+  class Client
+
+    DEFAULT_CLIENT_OPTIONS = { salesforce_api_version: '39.0' }
+
+    def initialize(restforce_client, options = {})
+      options = {}.merge(DEFAULT_CLIENT_OPTIONS).merge(options)
+      @connection = SalesforceBulkClient::Connection.new(options[:salesforce_api_version], restforce_client)
+    end
+
+    def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
+      do_operation('delete', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
+    end
+
+    def insert(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
+      do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
+    end
+
+    def query(sobject, query, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
+      do_operation('query', sobject, query,nil, get_response, timeout, batch_size, poll_delay)
+    end
+
+    def update(sobject, records, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
+      do_operation('update', sobject, records, nil, get_response, timeout, batch_size, poll_delay)
+    end
+
+    def upsert(sobject, records, external_field, get_response = false, batch_size = 10000, timeout = 3600, poll_delay = 5)
+      do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, poll_delay)
+    end
+
+    def job_from_id(job_id)
+      job = SalesforceBulkClient::Job.new(job_id: job_id, connection: @connection)
+      job_status = job.check_job_status
+      batches = job.list_batches
+      job.instance_variable_set(:@operation, job_status.operation)
+      job.instance_variable_set(:@sobject, job_status.object)
+      job.instance_variable_set(:@batch_ids, batches.map { |batch_info| batch_info.id })
+      job
+    end
+
+    private
+
+    def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, poll_delay)
+      job = SalesforceBulkClient::Job.new(
+          operation: operation,
+          sobject: sobject,
+          records: records,
+          external_field: external_field,
+          connection: @connection
+      )
+
+      job.create_job(batch_size)
+      operation.to_s == 'query' ? job.add_query : job.add_batches
+      job.close_job
+      job.get_job_result(get_response, timeout, poll_delay)
+    end
+
+  end
+end

data/lib/salesforce_bulk_client/connection.rb

@@ -0,0 +1,35 @@
+require 'multi_json'
+
+module SalesforceBulkClient
+  class Connection
+
+    def initialize(api_version, restforce_client)
+      @restforce_client = restforce_client
+      @api_version = api_version
+      @path_prefix = "/services/async/#{@api_version}/"
+      @restforce_client.authenticate!
+    end
+
+    def post_request(path, post_data, as_json = true)
+      authenticate_results = @restforce_client.authenticate!
+      response = @restforce_client.post do |request|
+        request.url [ @path_prefix, path ].join('/')
+        request.headers['Content-Type'] = 'application/json'
+        request.headers['X-SFDC-Session'] = authenticate_results.access_token
+        request.body = as_json ? MultiJson.dump(post_data) : post_data
+      end
+      response.body
+    end
+
+    def get_request(path)
+      authenticate_results = @restforce_client.authenticate!
+      response = @restforce_client.get do |request|
+        request.url "#{@path_prefix}#{path}"
+        request.headers['Content-Type'] = 'application/json'
+        request.headers['X-SFDC-Session'] = authenticate_results.access_token
+      end
+      response.body
+    end
+
+  end
+end

data/lib/salesforce_bulk_client/job.rb

@@ -0,0 +1,129 @@
+require 'fire_poll'
+
+module SalesforceBulkClient
+
+  class Job
+
+    attr_reader :job_id
+
+    def initialize(args)
+      @job_id = args[:job_id]
+      @operation = args[:operation]
+      @sobject = args[:sobject]
+      @external_field = args[:external_field]
+      @records = args[:records]
+      @connection = args[:connection]
+      @batch_ids = []
+    end
+
+    def create_job(batch_size)
+      @batch_size = batch_size
+      create_job_request = { operation: @operation.to_s.downcase, object: @sobject, contentType: 'JSON' }
+      if !@external_field.nil?
+        create_job_request[:externalIdFieldName] = @external_field
+      end
+      create_job_result = @connection.post_request('job', create_job_request)
+      @job_id = create_job_result.id
+    end
+
+    def close_job
+      close_job_request = { state: 'Closed' }
+      @connection.post_request("job/#{@job_id}", close_job_request)
+    end
+
+    def add_query
+      add_query_result = @connection.post_request("job/#{@job_id}/batch", @records, false)
+      @batch_ids << add_query_result.id
+    end
+
+    def add_batches
+      raise 'Records must be an array of hashes.' unless @records.is_a? Array
+      @records.each_slice(@batch_size) do |batch|
+        @batch_ids << add_batch(batch)
+      end
+    end
+
+    def add_batch(batch)
+      add_batch_result = @connection.post_request("job/#{@job_id}/batch", batch)
+      add_batch_result.id
+    end
+
+    def check_job_status
+      @connection.get_request("job/#{@job_id}")
+    end
+
+    def check_batch_status(batch_id)
+      @connection.get_request("job/#{@job_id}/batch/#{batch_id}")
+    end
+    
+    def list_batches
+      @connection.get_request("job/#{@job_id}/batch")&.batchInfo
+    end
+
+    def get_job_result(return_result, timeout, poll_delay)
+      batch_infos = []
+      polling_started = false
+      polling_completed = false
+      FirePoll.poll("Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}.", timeout) do
+        sleep poll_delay if polling_started
+        polling_started = true
+        job_status = self.check_job_status
+        if job_status.state == 'Closed'
+          batch_info_map = {}
+
+          batches_ready = @batch_ids.all? do |batch_id|
+            batch_info = batch_info_map[batch_id] = self.check_batch_status(batch_id)
+            batch_info.state != 'Queued' && batch_info != 'InProgress'
+          end
+
+          if batches_ready
+            @batch_ids.each do |batch_id|
+              batch_infos.insert(0, batch_info_map[batch_id])
+              @batch_ids.delete(batch_id)
+            end
+          end
+          polling_completed = true if @batch_ids.empty?
+        else
+          polling_completed = true
+        end
+        polling_completed
+      end
+      job_status = self.check_job_status
+
+      batch_infos.each_with_index do |batch_state, i|
+        if batch_state.state == 'Completed' && return_result == true
+          batch_infos[i].merge!({ 'response' => self.get_batch_result(batch_state.id)})
+        end
+      end
+
+      job_status.merge!({ 'batches' => batch_infos })
+      job_status
+    end
+
+    def get_batch_result(batch_id)
+      batch_results = @connection.get_request("job/#{@job_id}/batch/#{batch_id}/result")
+      results = []
+      if @operation.to_s != 'query'
+        results = batch_results
+      else
+        batch_results.each do |batch_result_id|
+          results.concat(@connection.get_request("job/#{@job_id}/batch/#{batch_id}/result/#{batch_result_id}"))
+        end
+      end
+      results
+    end
+
+    def each_batch(timeout = 3600, poll_delay = 5)
+      job_result = self.get_job_result(false, timeout, poll_delay)
+      job_result.batches.each do |batch_info|
+        batch_result = nil
+        if batch_info.state == 'Completed'
+          batch_result = self.get_batch_result(batch_info.id)
+        end
+        yield(batch_info, batch_result)
+      end
+    end
+
+  end
+
+end

data/lib/salesforce_bulk_client/version.rb

@@ -0,0 +1,3 @@
+module SalesforceBulkClient
+  VERSION = '0.0.1'
+end

data/salesforce_bulk_client.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'salesforce_bulk_client/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'salesforce_bulk_client'
+  spec.version       = SalesforceBulkClient::VERSION
+  spec.authors       = ['David Massad']
+  spec.email         = ['david.massad@fronteraconsulting.net']
+  spec.license       = 'MIT'
+
+  spec.summary       = 'Salesforce JSON-based Bulk API Client'
+  spec.homepage      = 'https://github.com/FronteraConsulting/salesforce_bulk_client'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'restforce', '~> 2.5', '>= 2.5.3'
+  spec.add_dependency 'multi_json', '~> 1.12', '>= 1.12.1'
+  spec.add_dependency 'fire_poll', '~> 1.2', '>= 1.2.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'simplecov', '~> 0.14.1'
+  spec.add_development_dependency 'dotenv', '~> 2.2', '>= 2.2.1'
+  spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
+  spec.add_development_dependency 'webmock', '~> 3.0', '>= 3.0.1'
+  spec.add_development_dependency 'addressable', '~> 2.5', '>= 2.5.1'
+end

metadata

@@ -0,0 +1,263 @@
+--- !ruby/object:Gem::Specification
+name: salesforce_bulk_client
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- David Massad
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: restforce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.1
+- !ruby/object:Gem::Dependency
+  name: fire_poll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+description: 
+email:
+- david.massad@fronteraconsulting.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rspec_status"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- fixtures/vcr_cassettes/salesforce/batch_processing.yml
+- fixtures/vcr_cassettes/salesforce/delete.yml
+- fixtures/vcr_cassettes/salesforce/insert.yml
+- fixtures/vcr_cassettes/salesforce/login.yml
+- fixtures/vcr_cassettes/salesforce/query.yml
+- fixtures/vcr_cassettes/salesforce/update.yml
+- fixtures/vcr_cassettes/salesforce/upsert.yml
+- lib/salesforce_bulk_client.rb
+- lib/salesforce_bulk_client/client.rb
+- lib/salesforce_bulk_client/connection.rb
+- lib/salesforce_bulk_client/job.rb
+- lib/salesforce_bulk_client/version.rb
+- salesforce_bulk_client.gemspec
+homepage: https://github.com/FronteraConsulting/salesforce_bulk_client
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Salesforce JSON-based Bulk API Client
+test_files: []