salesforce_bulk_api 0.0.8 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e63ba9f2d9be3dfa89bf6d896ee94fe847bc1b57
-  data.tar.gz: e054528cb047b09383e98fe2d54b1e3ad628bb0e
+SHA256:
+  metadata.gz: bd5b061f299027dc456e2c65f6b33480c7192d9c5af15a5fecd3cd1b81b79589
+  data.tar.gz: 4e5a70fe4541df2b944bdb3f59c0acdc78f91d8f2100ae164c03a23e54e09031
 SHA512:
-  metadata.gz: 7d66df90281412848aba6eed1a5a5476d7883d1e774f28b21dcd2d3a3952027a9e527a474ec2cd702e5b1018c678a25892572206524d1cfbcbd064e7a7577495
-  data.tar.gz: 0326e9e4f4389d5052ce92aa04ff2daf65c26df997dc31d8b353f3cca60f1eb5634da45d126fb2db2d3221873353cd78df6ca8bab55d43f14199dd0d0de71c33
+  metadata.gz: b453618a4cddde3851d766ca672df92177bae019152192bf628ebd2f8e08e92f2662226daf85bf81ea92d64a7e9756ccd7a495e170f32924985f4bbbd5c4b727
+  data.tar.gz: f44136ac331cc22b288f8f27a87b8f5def897f4aca051dcc3c2e1dd860fb1d2f9fb0d337b40858f63cbdeabdc407644ea69ed3a1f129491ba6f50a12f4e893ac

data/.gitignore

@@ -5,3 +5,5 @@ Gemfile.lock
 .ruby-version
 pkg/*
 auth_credentials.yml
+*.swp
+.idea

data/LICENCE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Yatish Mehta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -1,91 +1,141 @@
 # Salesforce-Bulk-Api
 [![Gem Version](https://badge.fury.io/rb/salesforce_bulk_api.png)](http://badge.fury.io/rb/salesforce_bulk_api)
+
 ## Overview
 
-Salesforce bulk API is a simple ruby gem for connecting to and using the Salesforce Bulk API. It is actually a re-written code from [salesforce_bulk](https://github.com/jorgevaldivia/salesforce_bulk).Written to suit many more other features as well.
+`SalesforceBulkApi` is a ruby wrapper for the Salesforce Bulk API.
+It is rewritten from [salesforce_bulk](https://github.com/jorgevaldivia/salesforce_bulk).
+It adds some missing features of `salesforce_bulk`.
 
 ## How to use
 
 Using this gem is simple and straight forward.
 
-To initialize:
+### Install
 
-   `sudo gem install salesforce_bulk_api`
+   `gem install salesforce_bulk_api`
 
-or add
+or add it to your Gemfile
 
    `gem salesforce_bulk_api`
-   
-in your Gemfile
 
-There are two ways to authenticate with SalesForce to use the Bulk API: databasedotcom & restforce.
-Please check out the entire documentation of the gem you decide to use to learn the various ways of authentication.
+### Authenticate
+
+You can authenticate with Salesforce using two gems, `databasedotcom` & `restforce`.
+
+Please check the documentation of the respective gems to learn how to authenticate with Salesforce
 
 [Databasedotcom](https://github.com/heroku/databasedotcom)
 [Restforce](https://github.com/ejholmes/restforce)
 
-
 You can use username password combo, OmniAuth, Oauth2
 You can use as many records possible in the Array. Governor limits are taken care of inside the gem.
 
+```ruby
+require 'salesforce_bulk_api'
 
-	require 'salesforce_bulk_api'
-	client = Databasedotcom::Client.new :client_id =>  SFDC_APP_CONFIG["client_id"], :client_secret => SFDC_APP_CONFIG["client_secret"] #client_id and client_secret respectively
-	client.authenticate :token => "my-oauth-token", :instance_url => "http://na1.salesforce.com"  #=> "my-oauth-token"
+client = Databasedotcom::Client.new(
+  :client_id =>  SFDC_APP_CONFIG["client_id"],
+  :client_secret => SFDC_APP_CONFIG["client_secret"]
+)
+client.authenticate(
+  :token => " ",
+  :instance_url => "http://na1.salesforce.com"
+)
 
-    salesforce = SalesforceBulkApi::Api.new(client)
+salesforce = SalesforceBulkApi::Api.new(client)
+```
 
 OR
 
-	require 'salesforce_bulk_api'
-	client = Restforce.new(
-	  username:       SFDC_APP_CONFIG['SFDC_USERNAME'],
-	  password:       SFDC_APP_CONFIG['SFDC_PASSWORD'],
-	  security_token: SFDC_APP_CONFIG['SFDC_SECURITY_TOKEN'],
-	  client_id:      SFDC_APP_CONFIG['SFDC_CLIENT_ID'],
-	  client_secret:  SFDC_APP_CONFIG['SFDC_CLIENT_SECRET'].to_i,
-	  host:           SFDC_APP_CONFIG['SFDC_HOST']
-	)
-
-	salesforce = SalesforceBulkApi::Api.new(client)
-
-
-Sample operations:
-
-    # Insert/Create
-    # Add as many fields per record as needed.
-	new_account = Hash["name" => "Test Account", "type" => "Other"] 
-	records_to_insert = Array.new
-	# You can add as many records as you want here, just keep in mind that Salesforce has governor limits.
-	records_to_insert.push(new_account) 
-	result = salesforce.create("Account", records_to_insert)
-	puts "result is: #{result.inspect}"
-
-    # Update
-	updated_account = Hash["name" => "Test Account -- Updated", id => "a00A0001009zA2m"] # Nearly identical to an insert, but we need to pass the salesforce id.
-	records_to_update = Array.new
-	records_to_update.push(updated_account)
-	salesforce.update("Account", records_to_update)
-
-    # Upsert
-	upserted_account = Hash["name" => "Test Account -- Upserted", "External_Field_Name" => "123456"] # Fields to be updated. External field must be included
-	records_to_upsert = Array.new
-	records_to_upsert.push(upserted_account)
-	salesforce.upsert("Account", records_to_upsert, "External_Field_Name") # Note that upsert accepts an extra parameter for the external field name
-
-    # Delete
-	deleted_account = Hash["id" => "a00A0001009zA2m"] # We only specify the id of the records to delete
-	records_to_delete = Array.new
-	records_to_delete.push(deleted_account)
-	salesforce.delete("Account", records_to_delete)
-
-    # Query
-	res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string
-
-## Installation
-
-    sudo gem install salesforce_bulk_api
-	
+```ruby
+require 'salesforce_bulk_api'
+client = Restforce.new(
+  username:       SFDC_APP_CONFIG['SFDC_USERNAME'],
+  password:       SFDC_APP_CONFIG['SFDC_PASSWORD'],
+  security_token: SFDC_APP_CONFIG['SFDC_SECURITY_TOKEN'],
+  client_id:      SFDC_APP_CONFIG['SFDC_CLIENT_ID'],
+  client_secret:  SFDC_APP_CONFIG['SFDC_CLIENT_SECRET'].to_i,
+  host:           SFDC_APP_CONFIG['SFDC_HOST']
+)
+client.authenticate!
+
+salesforce = SalesforceBulkApi::Api.new(client)
+```
+
+### Sample operations:
+
+```ruby
+# Insert/Create
+# Add as many fields per record as needed.
+new_account = Hash["name" => "Test Account", "type" => "Other"]
+records_to_insert = Array.new
+# You can add as many records as you want here, just keep in mind that Salesforce has governor limits.
+records_to_insert.push(new_account)
+result = salesforce.create("Account", records_to_insert)
+puts "result is: #{result.inspect}"
+
+# Update
+updated_account = Hash["name" => "Test Account -- Updated", id => "a00A0001009zA2m"] # Nearly identical to an insert, but we need to pass the salesforce id.
+records_to_update = Array.new
+records_to_update.push(updated_account)
+salesforce.update("Account", records_to_update)
+
+# Upsert
+upserted_account = Hash["name" => "Test Account -- Upserted", "External_Field_Name" => "123456"] # Fields to be updated. External field must be included
+records_to_upsert = Array.new
+records_to_upsert.push(upserted_account)
+salesforce.upsert("Account", records_to_upsert, "External_Field_Name") # Note that upsert accepts an extra parameter for the external field name
+
+# Delete
+deleted_account = Hash["id" => "a00A0001009zA2m"] # We only specify the id of the records to delete
+records_to_delete = Array.new
+records_to_delete.push(deleted_account)
+salesforce.delete("Account", records_to_delete)
+
+# Query
+res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string
+```
+
+### Helpful methods:
+
+```ruby
+# Check status of a job via #job_from_id
+job = salesforce.job_from_id('a00A0001009zA2m') # Returns a SalesforceBulkApi::Job instance
+puts "status is: #{job.check_job_status.inspect}"
+```
+
+### Listening to events:
+
+```ruby
+# A job is created
+# Useful when you need to store the job_id before any work begins, then if you fail during a complex load scenario, you can wait for your
+# previous job(s) to finish.
+salesforce.on_job_created do |job|
+  puts "Job #{job.job_id} created!"
+end
+```
+
+### Fetching records from a batch
+
+```ruby
+job_id = 'l02A0231009Za8m'
+batch_id = 'H24a0708089zA2J'
+salesforce.get_batch_records(job_id, batch_id)
+# => [{"Id"=>["RECORD_ID_1"], "AField__c"=>["123123"]},
+      {"Id"=>["RECORD_ID_2"], "AField__c"=>["123123"]},
+      {"Id"=>["RECORD_ID_3"], "AField__c"=>["123123"]}]
+
+```
+
+### Throttling API calls:
+
+```ruby
+# By default, this gem (and maybe your app driving it) will query job/batch statuses at an unbounded rate.  We
+# can fix that, e.g.:
+salesforce.connection.set_status_throttle(30) # only check status of individual jobs/batches every 30 seconds
+```
+
 ## Contribute
 
 Feel to fork and send Pull request

data/auth_credentials.yml.example

@@ -0,0 +1,7 @@
+salesforce:
+  client_id: client_id_here
+  client_secret: client_secret_here
+  user: sf_user@example.com
+  passwordandtoken: passandtokenhere
+  test_account_id: 0013000000ymMBh
+  host: 'login.salesforce.com' # use test.salesforce.com if it is a sandbox

data/lib/salesforce_bulk_api.rb

@@ -1,53 +1,98 @@
 require 'rubygems'
 require 'bundler'
-Bundler.require()
-require "salesforce_bulk_api/version"
 require 'net/https'
 require 'xmlsimple'
 require 'csv'
+
+require 'salesforce_bulk_api/version'
+require 'salesforce_bulk_api/concerns/throttling'
 require 'salesforce_bulk_api/job'
 require 'salesforce_bulk_api/connection'
 
 module SalesforceBulkApi
-  
   class Api
+    attr_reader :connection
 
-    @@SALESFORCE_API_VERSION = '23.0'
+    SALESFORCE_API_VERSION = '46.0'
 
     def initialize(client)
-      @connection = SalesforceBulkApi::Connection.new(@@SALESFORCE_API_VERSION,client)
+      @connection = SalesforceBulkApi::Connection.new(SALESFORCE_API_VERSION, client)
+      @listeners = { job_created: [] }
     end
 
     def upsert(sobject, records, external_field, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      self.do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
+      do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
     end
 
     def update(sobject, records, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      self.do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
+      do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
     end
 
     def create(sobject, records, get_response = false, send_nulls = false, batch_size = 10000, timeout = 1500)
-      self.do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls)
+      do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls)
     end
 
     def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 1500)
-      self.do_operation('delete', sobject, records, nil, get_response, timeout, batch_size)
+      do_operation('delete', sobject, records, nil, get_response, timeout, batch_size)
     end
 
     def query(sobject, query, batch_size = 10000, timeout = 1500)
-      self.do_operation('query', sobject, query, nil, true, timeout, batch_size)
+      do_operation('query', sobject, query, nil, true, timeout, batch_size)
+    end
+
+    def counters
+      {
+        http_get: @connection.counters[:get],
+        http_post: @connection.counters[:post],
+        upsert: get_counters[:upsert],
+        update: get_counters[:update],
+        create: get_counters[:create],
+        delete: get_counters[:delete],
+        query: get_counters[:query]
+      }
     end
 
-    #private
+    # Allows you to attach a listener that accepts the created job (which has a useful #job_id field).  This is useful
+    # for recording a job ID persistently before you begin batch work (i.e. start modifying the salesforce database),
+    # so if the load process you are writing needs to recover, it can be aware of previous jobs it started and wait
+    # for them to finish.
+    #
+    def on_job_created(&block)
+      @listeners[:job_created] << block
+    end
+
+    def job_from_id(job_id)
+      SalesforceBulkApi::Job.new(job_id: job_id, connection: @connection)
+    end
 
     def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, send_nulls = false, no_null_list = [])
-      job = SalesforceBulkApi::Job.new(operation, sobject, records, external_field, @connection)
+      count operation.to_sym
+
+      job = SalesforceBulkApi::Job.new(
+        operation: operation,
+        sobject: sobject,
+        records: records,
+        external_field: external_field,
+        connection: @connection
+      )
 
       job.create_job(batch_size, send_nulls, no_null_list)
+      @listeners[:job_created].each {|callback| callback.call(job)}
       operation == "query" ? job.add_query() : job.add_batches()
       response = job.close_job
       response.merge!({'batches' => job.get_job_result(get_response, timeout)}) if get_response == true
       response
     end
-  end  
-end
+
+    private
+
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(name)
+      get_counters[name] += 1
+    end
+
+  end
+end

data/lib/salesforce_bulk_api/concerns/throttling.rb

@@ -0,0 +1,60 @@
+module SalesforceBulkApi::Concerns
+  module Throttling
+
+    def throttles
+      @throttles.dup
+    end
+
+    def add_throttle(&throttling_callback)
+      @throttles ||= []
+      @throttles << throttling_callback
+    end
+
+    def set_status_throttle(limit_seconds)
+      set_throttle_limit_in_seconds(limit_seconds, [:http_method, :path], ->(details) { details[:http_method] == :get })
+    end
+
+    def set_throttle_limit_in_seconds(limit_seconds, throttle_by_keys, only_if)
+      add_throttle do |details|
+        limit_log = get_limit_log(Time.now - limit_seconds)
+        key = extract_constraint_key_from(details, throttle_by_keys)
+        last_request = limit_log[key]
+
+        if !last_request.nil? && only_if.call(details)
+          seconds_since_last_request = Time.now.to_f - last_request.to_f
+          need_to_wait_seconds = limit_seconds - seconds_since_last_request
+          sleep(need_to_wait_seconds) if need_to_wait_seconds > 0
+        end
+
+        limit_log[key] = Time.now
+      end
+    end
+
+    private
+
+    def extract_constraint_key_from(details, throttle_by_keys)
+      hash = {}
+      throttle_by_keys.each { |k| hash[k] = details[k] }
+      hash
+    end
+
+    def get_limit_log(prune_older_than)
+      @limits ||= Hash.new(0)
+
+      @limits.delete_if do |k, v|
+        v < prune_older_than
+      end
+
+      @limits
+    end
+
+    def throttle(details={})
+      (@throttles || []).each do |callback|
+        args = [details]
+        args = args[0..callback.arity]
+        callback.call(*args)
+      end
+    end
+
+  end
+end

data/lib/salesforce_bulk_api/connection.rb

@@ -1,49 +1,43 @@
-module SalesforceBulkApi
 require 'timeout'
 
+module SalesforceBulkApi
   class Connection
+    include Concerns::Throttling
 
-    @@XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>'
-    @@API_VERSION = nil
-    @@LOGIN_HOST = 'login.salesforce.com'
-    @@INSTANCE_HOST = nil # Gets set in login()
+    LOGIN_HOST = 'login.salesforce.com'
 
-    def initialize(api_version,client)
-      @client=client
-      @session_id = nil
-      @server_url = nil
-      @instance = nil
-      @@API_VERSION = api_version
-      @@LOGIN_PATH = "/services/Soap/u/#{@@API_VERSION}"
-      @@PATH_PREFIX = "/services/async/#{@@API_VERSION}/"
+    def initialize(api_version, client)
+      @client = client
+      @api_version = api_version
+      @path_prefix = "/services/async/#{@api_version}/"
 
       login()
     end
 
-    #private
-
     def login()
       client_type = @client.class.to_s
       case client_type
       when "Restforce::Data::Client"
-        @session_id=@client.options[:oauth_token]
-        @server_url=@client.options[:instance_url]
+        @session_id = @client.options[:oauth_token]
+        @server_url = @client.options[:instance_url]
       else
-        @session_id=@client.oauth_token
-        @server_url=@client.instance_url
+        @session_id = @client.oauth_token
+        @server_url = @client.instance_url
       end
       @instance = parse_instance()
-      @@INSTANCE_HOST = "#{@instance}.salesforce.com"
+      @instance_host = "#{@instance}.salesforce.com"
     end
 
     def post_xml(host, path, xml, headers)
-      host = host || @@INSTANCE_HOST
-      if host != @@LOGIN_HOST # Not login, need to add session id to header
-        headers['X-SFDC-Session'] = @session_id;
-        path = "#{@@PATH_PREFIX}#{path}"
+      host = host || @instance_host
+      if host != LOGIN_HOST # Not login, need to add session id to header
+        headers['X-SFDC-Session'] = @session_id
+        path = "#{@path_prefix}#{path}"
       end
       i = 0
       begin
+        count :post
+        throttle(http_method: :post, path: path)
         https(host).post(path, xml, headers).body
       rescue
         i += 1
@@ -58,11 +52,14 @@ require 'timeout'
     end
 
     def get_request(host, path, headers)
-      host = host || @@INSTANCE_HOST
-      path = "#{@@PATH_PREFIX}#{path}"
-      if host != @@LOGIN_HOST # Not login, need to add session id to header
+      host = host || @instance_host
+      path = "#{@path_prefix}#{path}"
+      if host != LOGIN_HOST # Not login, need to add session id to header
         headers['X-SFDC-Session'] = @session_id;
       end
+
+      count :get
+      throttle(http_method: :get, path: path)
       https(host).get(path, headers).body
     end
 
@@ -73,10 +70,27 @@ require 'timeout'
       req
     end
 
+    def counters
+      {
+        get: get_counters[:get],
+        post: get_counters[:post]
+      }
+    end
+
+    private
+
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(http_method)
+      get_counters[http_method] += 1
+    end
+
     def parse_instance()
-      @instance=@server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}/).to_s.gsub("https://","")
-      @instance = @server_url.split(".salesforce.com")[0].split("://")[1] if @instance.blank?
-      return @instance
+      @instance = @server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}\./).to_s.gsub("https://","").split(".")[0]
+      @instance = @server_url.split(".salesforce.com")[0].split("://")[1] if @instance.nil? || @instance.empty?
+      @instance
     end
 
   end

data/lib/salesforce_bulk_api/job.rb

@@ -1,13 +1,17 @@
 module SalesforceBulkApi
 
   class Job
+    attr_reader :job_id
 
-    def initialize(operation, sobject, records, external_field, connection)
-      @operation      = operation
-      @sobject        = sobject
-      @external_field = external_field
-      @records        = records
-      @connection     = connection
+    class SalesforceException < StandardError; end
+
+    def initialize(args)
+      @job_id         = args[:job_id]
+      @operation      = args[:operation]
+      @sobject        = args[:sobject]
+      @external_field = args[:external_field]
+      @records        = args[:records]
+      @connection     = args[:connection]
       @batch_ids      = []
       @XML_HEADER     = '<?xml version="1.0" encoding="utf-8" ?>'
     end
@@ -20,7 +24,8 @@ module SalesforceBulkApi
       xml = "#{@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
       xml += "<operation>#{@operation}</operation>"
       xml += "<object>#{@sobject}</object>"
-      if !@external_field.nil? # This only happens on upsert
+      # This only happens on upsert
+      if !@external_field.nil?
         xml += "<externalIdFieldName>#{@external_field}</externalIdFieldName>"
       end
       xml += "<contentType>XML</contentType>"
@@ -31,6 +36,10 @@ module SalesforceBulkApi
 
       response = @connection.post_xml(nil, path, xml, headers)
       response_parsed = XmlSimple.xml_in(response)
+
+      # response may contain an exception, so raise it
+      raise SalesforceException.new("#{response_parsed['exceptionMessage'][0]} (#{response_parsed['exceptionCode'][0]})") if response_parsed['exceptionCode']
+
       @job_id = response_parsed['id'][0]
     end
 
@@ -58,12 +67,12 @@ module SalesforceBulkApi
 
     def add_batches
       raise 'Records must be an array of hashes.' unless @records.is_a? Array
-      keys = @records.reduce({}) {|h,pairs| pairs.each {|k,v| (h[k] ||= []) << v}; h}.keys
+      keys = @records.reduce({}) {|h, pairs| pairs.each {|k, v| (h[k] ||= []) << v}; h}.keys
 
       @records_dup = @records.clone
 
       super_records = []
-      (@records_dup.size/@batch_size).to_i.times do
+      (@records_dup.size / @batch_size).to_i.times do
         super_records << @records_dup.pop(@batch_size)
       end
       super_records << @records_dup unless @records_dup.empty?
@@ -86,18 +95,58 @@ module SalesforceBulkApi
       response_parsed['id'][0] if response_parsed['id']
     end
 
+    def build_sobject(data)
+      xml = '<sObject>'
+      data.keys.each do |k|
+        if k.is_a?(Hash)
+          xml += build_sobject(k)
+        elsif k.to_s.include? '.'
+          relations = k.to_s.split('.')
+          parent = relations[0]
+          child = relations[1..-1].join('.')
+          xml += "<#{parent}>#{build_sobject({ child => data[k] })}</#{parent}>"
+        elsif data[k] != :type
+          xml += "<#{k}>#{data[k]}</#{k}>"
+        end
+      end
+      xml += '</sObject>'
+    end
+
+    def build_relationship_sobject(key, value)
+      if key.to_s.include? '.'
+        relations = key.to_s.split('.')
+        parent = relations[0]
+        child = relations[1..-1].join('.')
+        xml = "<#{parent}>"
+        xml += "<sObject>"
+        xml += build_relationship_sobject(child, value)
+        xml += "</sObject>"
+        xml += "</#{parent}>"
+      else
+        xml = "<#{key}>#{value}</#{key}>"
+      end
+    end
+
     def create_sobject(keys, r)
       sobject_xml = '<sObject>'
       keys.each do |k|
-        if !r[k].to_s.empty?
+        if r[k].is_a?(Hash)
+          sobject_xml += "<#{k}>"
+          sobject_xml += build_sobject(r[k])
+          sobject_xml += "</#{k}>"
+        elsif k.to_s.include? '.'
+          sobject_xml += build_relationship_sobject(k, r[k])
+        elsif !r[k].to_s.empty?
           sobject_xml += "<#{k}>"
           if r[k].respond_to?(:encode)
             sobject_xml += r[k].encode(:xml => :text)
+          elsif r[k].respond_to?(:iso8601) # timestamps
+            sobject_xml += r[k].iso8601.to_s
           else
             sobject_xml += r[k].to_s
           end
           sobject_xml += "</#{k}>"
-        elsif @send_nulls && !@no_null_list.include?(k)
+        elsif @send_nulls && !@no_null_list.include?(k) && r.key?(k)
           sobject_xml += "<#{k} xsi:nil=\"true\"/>"
         end
       end
@@ -142,11 +191,18 @@ module SalesforceBulkApi
         state = []
         Timeout::timeout(timeout, SalesforceBulkApi::JobTimeout) do
           while true
-            if self.check_job_status['state'][0] == 'Closed'
-              @batch_ids.each do |batch_id|
-                batch_state = self.check_batch_status(batch_id)
-                if batch_state['state'][0] != "Queued" && batch_state['state'][0] != "InProgress"
-                  state << (batch_state)
+            job_status = self.check_job_status
+            if job_status && job_status['state'] && job_status['state'][0] == 'Closed'
+              batch_statuses = {}
+
+              batches_ready = @batch_ids.all? do |batch_id|
+                batch_state = batch_statuses[batch_id] = self.check_batch_status(batch_id)
+                batch_state && batch_state['state'] && batch_state['state'][0] && !['Queued', 'InProgress'].include?(batch_state['state'][0])
+              end
+
+              if batches_ready
+                @batch_ids.each do |batch_id|
+                  state.insert(0, batch_statuses[batch_id])
                   @batch_ids.delete(batch_id)
                 end
               end
@@ -190,6 +246,17 @@ module SalesforceBulkApi
       results
     end
 
+    def get_batch_records(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}/request"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+
+      response = @connection.get_request(nil, path, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      results = response_parsed['sObject']
+
+      results
+    end
+
   end
 
   class JobTimeout < StandardError

data/lib/salesforce_bulk_api/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkApi
-  VERSION = '0.0.8'
+  VERSION = '1.0.0'
 end

data/salesforce_bulk_api.gemspec

@@ -3,27 +3,26 @@ $:.push File.expand_path("../lib", __FILE__)
 require "salesforce_bulk_api/version"
 
 Gem::Specification.new do |s|
-  s.name        = "salesforce_bulk_api"
+  s.name        = 'salesforce_bulk_api'
   s.version     = SalesforceBulkApi::VERSION
-  s.authors     = ["Yatish Mehta"]
-  s.email       = ["yatishmehta27@gmail.com"]
+  s.authors     = ['Yatish Mehta']
+  s.email       = ['yatishmehta27@gmail.com']
 
-  s.homepage    = "https://github.com/yatishmehta27/salesforce_bulk_api"
+  s.homepage    = 'https://github.com/yatishmehta27/salesforce_bulk_api'
   s.summary     = %q{It uses the bulk api of salesforce to communicate with Salesforce CRM}
   s.description = %q{Salesforce Bulk API with governor limits taken care of}
 
-  s.rubyforge_project = "salesforce_bulk_api"
+  s.add_dependency('json', ['>= 0'])
+  s.add_dependency('xml-simple', ['>= 0'])
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'restforce', '~>  3.0.0'
+  s.add_development_dependency "rake", ">= 12.3.3"
+  s.add_development_dependency 'pry'
 
-  s.add_dependency(%q<json>, [">= 0"])
-  s.add_dependency(%q<xml-simple>, [">= 0"])
-  
-  s.add_development_dependency "rspec"
-  s.add_development_dependency("webmock", ["~> 1.13"])
-  s.add_development_dependency("vcr", ['~> 2.5'])
-  
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
 
 end

data/spec/salesforce_bulk_api/salesforce_bulk_api_spec.rb

@@ -1,63 +1,80 @@
 require 'spec_helper'
 require 'yaml'
-require 'databasedotcom'
+require 'restforce'
 
 describe SalesforceBulkApi do
 
   before :each do
-    auth_hash = YAML.load(File.read('auth_credentials.yml'))
-    @sf_client = Databasedotcom::Client.new(:client_id     => auth_hash['salesforce']['client_id'],
-                                            :client_secret => auth_hash['salesforce']['client_secret'])
-    @sf_client.authenticate(:username => auth_hash['salesforce']['user'], :password => auth_hash['salesforce']['passwordandtoken'])
+    auth_hash = YAML.load_file('auth_credentials.yml')
+    sfdc_auth_hash = auth_hash['salesforce']
+
+    @sf_client = Restforce.new(
+      username: sfdc_auth_hash['user'],
+      password: sfdc_auth_hash['passwordandtoken'],
+      client_id: sfdc_auth_hash['client_id'],
+      client_secret: sfdc_auth_hash['client_secret'],
+      host: sfdc_auth_hash['host'])
+    @sf_client.authenticate!
+
+    @account_id = auth_hash['salesforce']['test_account_id']
+
     @api = SalesforceBulkApi::Api.new(@sf_client)
   end
 
+  after :each do
+
+  end
+
   describe 'upsert' do
 
     context 'when not passed get_result' do
       it "doesn't return the batches array" do
-        res = @api.upsert('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'}], 'Id')
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => 'www.test.com'}], 'Id')
         res['batches'].should be_nil
       end
     end
 
     context 'when passed get_result = true' do
       it 'returns the batches array' do
-        res = @api.upsert('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'}], 'Id', true)
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => 'www.test.com'}], 'Id', true)
         res['batches'][0]['response'].is_a? Array
-        res['batches'][0]['response'][0].should eq({'id'=>['0013000000ymMBhAAM'], 'success'=>['true'], 'created'=>['false']})
+
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+
       end
     end
 
     context 'when passed send_nulls = true' do
       it 'sets the nil and empty attributes to NULL' do
-        @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'abc123', :Other_Phone__c => '5678', :Gold_Star__c => true}], true)
-        res = @api.query('Account', "SELECT Website, Other_Phone__c From Account WHERE Id = '0013000000ymMBh'")
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
         res['batches'][0]['response'][0]['Website'][0].should eq 'abc123'
-        res['batches'][0]['response'][0]['Other_Phone__c'][0].should eq '5678'
-        res = @api.upsert('Account', [{:Id => '0013000000ymMBh', :Website => '', :Other_Phone__c => nil, :Gold_Star__c => false, :CRM_Last_Modified__c => nil}], 'Id', true, true)
-        res['batches'][0]['response'][0].should eq({'id'=>['0013000000ymMBhAAM'], 'success'=>['true'], 'created'=>['false']})
-        res = @api.query('Account', "SELECT Website, Other_Phone__c, Gold_Star__c, CRM_Last_Modified__c From Account WHERE Id = '0013000000ymMBh'")
+        res['batches'][0]['response'][0]['Phone'][0].should eq '5678'
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => '', :Phone => nil}], 'Id', true, true)
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
         res['batches'][0]['response'][0]['Website'][0].should eq({"xsi:nil" => "true"})
-        res['batches'][0]['response'][0]['Other_Phone__c'][0].should eq({"xsi:nil" => "true"})
-        res['batches'][0]['response'][0]['Gold_Star__c'][0].should eq('false')
-        res['batches'][0]['response'][0]['CRM_Last_Modified__c'][0].should eq({"xsi:nil" => "true"})
+        res['batches'][0]['response'][0]['Phone'][0].should eq({"xsi:nil" => "true"})
       end
     end
 
     context 'when passed send_nulls = true and an array of fields not to null' do
       it 'sets the nil and empty attributes to NULL, except for those included in the list of fields to ignore' do
-        @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'abc123', :Other_Phone__c => '5678', :Gold_Star__c => true}], true)
-        res = @api.query('Account', "SELECT Website, Other_Phone__c From Account WHERE Id = '0013000000ymMBh'")
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
         res['batches'][0]['response'][0]['Website'][0].should eq 'abc123'
-        res['batches'][0]['response'][0]['Other_Phone__c'][0].should eq '5678'
-        res = @api.upsert('Account', [{:Id => '0013000000ymMBh', :Website => '', :Other_Phone__c => nil, :Gold_Star__c => false, :CRM_Last_Modified__c => nil}], 'Id', true, true, [:Website, :Other_Phone__c])
-        res['batches'][0]['response'][0].should eq({'id'=>['0013000000ymMBhAAM'], 'success'=>['true'], 'created'=>['false']})
-        res = @api.query('Account', "SELECT Website, Other_Phone__c, Gold_Star__c, CRM_Last_Modified__c From Account WHERE Id = '0013000000ymMBh'")
+        res['batches'][0]['response'][0]['Phone'][0].should eq '5678'
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => '', :Phone => nil}], 'Id', true, true, [:Website, :Phone])
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
         res['batches'][0]['response'][0]['Website'][0].should eq('abc123')
-        res['batches'][0]['response'][0]['Other_Phone__c'][0].should eq('5678')
-        res['batches'][0]['response'][0]['Gold_Star__c'][0].should eq('false')
-        res['batches'][0]['response'][0]['CRM_Last_Modified__c'][0].should eq({"xsi:nil" => "true"})
+        res['batches'][0]['response'][0]['Phone'][0].should eq('5678')
       end
     end
 
@@ -67,16 +84,18 @@ describe SalesforceBulkApi do
     context 'when there is not an error' do
       context 'when not passed get_result' do
         it "doesnt return the batches array" do
-          res = @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'}])
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}])
           res['batches'].should be_nil
         end
       end
 
       context 'when passed get_result = true' do
         it 'returns the batches array' do
-          res = @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'}], true)
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}], true)
           res['batches'][0]['response'].is_a? Array
-          res['batches'][0]['response'][0].should eq({'id'=>['0013000000ymMBhAAM'], 'success'=>['true'], 'created'=>['false']})
+          res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][0]['success'].should eq ['true']
+          res['batches'][0]['response'][0]['created'].should eq ['false']
         end
       end
     end
@@ -84,16 +103,26 @@ describe SalesforceBulkApi do
     context 'when there is an error' do
       context 'when not passed get_result' do
         it "doesn't return the results array" do
-          res = @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'},{:Id => 'abc123', :Website => 'www.test.com'}])
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'},{:Id => 'abc123', :Website => 'www.test.com'}])
           res['batches'].should be_nil
         end
       end
 
       context 'when passed get_result = true with batches' do
         it 'returns the results array' do
-          res = @api.update('Account', [{:Id => '0013000000ymMBh', :Website => 'www.test.com'}, {:Id => '0013000000ymMBh', :Website => 'www.test.com'}, {:Id => '0013000000ymMBh', :Website => 'www.test.com'}, {:Id => 'abc123', :Website => 'www.test.com'}], true, false, [], 2)
-          res['batches'][0]['response'].should eq([{"id"=>["0013000000ymMBhAAM"], "success"=>["true"], "created"=>["false"]}, {"errors"=>[{"fields"=>["Id"], "message"=>["Account ID: id value of incorrect type: abc123"], "statusCode"=>["MALFORMED_ID"]}], "success"=>["false"], "created"=>["false"]}])
-          res['batches'][1]['response'].should eq([{"id"=>["0013000000ymMBhAAM"], "success"=>["true"], "created"=>["false"]},{"id"=>["0013000000ymMBhAAM"], "success"=>["true"], "created"=>["false"]}])
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}, {:Id => @account_id, :Website => 'www.test.com'}, {:Id => @account_id, :Website => 'www.test.com'}, {:Id => 'abc123', :Website => 'www.test.com'}], true, false, [], 2)
+
+          res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][0]['success'].should eq ['true']
+          res['batches'][0]['response'][0]['created'].should eq ['false']
+          res['batches'][0]['response'][1]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][1]['success'].should eq ['true']
+          res['batches'][0]['response'][1]['created'].should eq ['false']
+
+          res['batches'][1]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][1]['response'][0]['success'].should eq ['true']
+          res['batches'][1]['response'][0]['created'].should eq ['false']
+          res['batches'][1]['response'][1].should eq({"errors"=>[{"fields"=>["Id"], "message"=>["Account ID: id value of incorrect type: abc123"], "statusCode"=>["MALFORMED_ID"]}], "success"=>["false"], "created"=>["false"]})
         end
       end
     end
@@ -116,13 +145,10 @@ describe SalesforceBulkApi do
         res['batches'][0]['response'].length.should > 1
         res['batches'][0]['response'][0]['Id'].should_not be_nil
       end
+
       context 'and there are multiple batches' do
-        it 'returns the query results in a merged hash' do
-          pending 'need dev to create > 10k records in dev organization'
-          res = @api.query('Account', "SELECT id, Name From Account WHERE Name LIKE 'Test%'")
-          res['batches'][0]['response'].length.should > 1
-          res['batches'][0]['response'][0]['Id'].should_not be_nil
-        end
+        # need dev to create > 10k records in dev organization
+        it 'returns the query results in a merged hash'
       end
     end
 
@@ -142,4 +168,26 @@ describe SalesforceBulkApi do
 
   end
 
+  describe 'counters' do
+    context 'when read operations are called' do
+      it 'increments operation count and http GET count' do
+        @api.counters[:http_get].should eq 0
+        @api.counters[:query].should eq 0
+        @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        @api.counters[:http_get].should eq 1
+        @api.counters[:query].should eq 1
+      end
+    end
+
+    context 'when update operations are called' do
+      it 'increments operation count and http POST count' do
+        @api.counters[:http_post].should eq 0
+        @api.counters[:update].should eq 0
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        @api.counters[:http_post].should eq 1
+        @api.counters[:update].should eq 1
+      end
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,16 +1,8 @@
 require 'rubygems'
 require 'bundler/setup'
-#require 'webmock/rspec'
-#require 'vcr'
 require 'salesforce_bulk_api'
 
 RSpec.configure do |c|
   c.filter_run :focus => true
   c.run_all_when_everything_filtered = true
 end
-
-# enable this and record the test requests using a SF developer org.
-# VCR.configure do |c|
-#   c.cassette_library_dir = 'spec/cassettes'
-#   c.hook_into :webmock
-# end

metadata

@@ -1,85 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_bulk_api
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 1.0.0
 platform: ruby
 authors:
 - Yatish Mehta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-13 00:00:00.000000000 Z
+date: 2020-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: xml-simple
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: restforce
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Salesforce Bulk API with governor limits taken care of
 email:
 - yatishmehta27@gmail.com
@@ -87,13 +101,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
+- ".gitignore"
+- ".rspec"
 - Gemfile
+- LICENCE
 - README.md
 - Rakefile
-- example_auth_credentials.yml
+- auth_credentials.yml.example
 - lib/salesforce_bulk_api.rb
+- lib/salesforce_bulk_api/concerns/throttling.rb
 - lib/salesforce_bulk_api/connection.rb
 - lib/salesforce_bulk_api/job.rb
 - lib/salesforce_bulk_api/version.rb
@@ -109,17 +125,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: salesforce_bulk_api
-rubygems_version: 2.1.11
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: It uses the bulk api of salesforce to communicate with Salesforce CRM

data/example_auth_credentials.yml

@@ -1,5 +0,0 @@
-salesforce:
-  client_id: client_id_here
-  client_secret: client_secret_here
-  user: sf_user@example.com
-  passwordandtoken: passandtokenhere