salesforce_bulk_api_serial_or_parallel 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b42acc44b480f4bc913f265ca0e5f1ee9d6eb27c
+  data.tar.gz: 2fae9dadedce4fea9d26be4828a27c0cbc662304
+SHA512:
+  metadata.gz: de7e1e445ecacd7b832830175fa501146de6e210f1b653a3f579f129135993b12a592802716ab3d523a214031f99643928f202bbe35d73d7deca0a7cac3ef32f
+  data.tar.gz: 9545be081a46a0f17a1e34ff20c8cf30fcfe9277dded5b21f7cc460fd0431c55edde2f74e751a145a0bec57e73fec52ebd9e6a1e8e9110bcff6ea6f3469eee49

data/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+.bundle
+Gemfile.lock
+.ruby-gemset
+.ruby-version
+pkg/*
+auth_credentials.yml
+*.swp
+.idea

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in salesforce_bulk_api_serial_or_parallel.gemspec
+gemspec

data/LICENCE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Yatish Mehta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# Salesforce-Bulk-Api
+[![Gem Version](https://badge.fury.io/rb/salesforce_bulk_api.png)](http://badge.fury.io/rb/salesforce_bulk_api)
+## Overview
+
+Salesforce bulk API is a simple ruby gem for connecting to and using the Salesforce Bulk API. It is actually a re-written code from [salesforce_bulk](https://github.com/jorgevaldivia/salesforce_bulk).Written to suit many more other features as well.
+
+## How to use
+
+Using this gem is simple and straight forward.
+
+To initialize:
+
+   `sudo gem install salesforce_bulk_api_serial_or_parallel`
+
+or add
+
+   `gem salesforce_bulk_api_serial_or_parallel`
+   
+in your Gemfile
+
+There are two ways to authenticate with SalesForce to use the Bulk API: databasedotcom & restforce.
+Please check out the entire documentation of the gem you decide to use to learn the various ways of authentication.
+
+[Databasedotcom](https://github.com/heroku/databasedotcom)
+[Restforce](https://github.com/ejholmes/restforce)
+
+
+You can use username password combo, OmniAuth, Oauth2
+You can use as many records possible in the Array. Governor limits are taken care of inside the gem.
+
+
+	require 'salesforce_bulk_api_serial_or_parallel'
+	client = Databasedotcom::Client.new :client_id =>  SFDC_APP_CONFIG["client_id"], :client_secret => SFDC_APP_CONFIG["client_secret"] #client_id and client_secret respectively
+	client.authenticate :token => "my-oauth-token", :instance_url => "http://na1.salesforce.com"  #=> "my-oauth-token"
+
+    salesforce = SalesforceBulkApi::Api.new(client)
+
+OR
+
+	require 'salesforce_bulk_api_serial_or_parallel'
+	client = Restforce.new(
+	  username:       SFDC_APP_CONFIG['SFDC_USERNAME'],
+	  password:       SFDC_APP_CONFIG['SFDC_PASSWORD'],
+	  security_token: SFDC_APP_CONFIG['SFDC_SECURITY_TOKEN'],
+	  client_id:      SFDC_APP_CONFIG['SFDC_CLIENT_ID'],
+	  client_secret:  SFDC_APP_CONFIG['SFDC_CLIENT_SECRET'].to_i,
+	  host:           SFDC_APP_CONFIG['SFDC_HOST']
+	)
+	client.authenticate!
+	salesforce = SalesforceBulkApi::Api.new(client)
+
+
+### Sample operations:
+
+    # Insert/Create
+    # Add as many fields per record as needed.
+	new_account = Hash["name" => "Test Account", "type" => "Other"] 
+	records_to_insert = Array.new
+	# You can add as many records as you want here, just keep in mind that Salesforce has governor limits.
+	records_to_insert.push(new_account) 
+	result = salesforce.create("Account", records_to_insert)
+	puts "result is: #{result.inspect}"
+
+    # Update
+	updated_account = Hash["name" => "Test Account -- Updated", id => "a00A0001009zA2m"] # Nearly identical to an insert, but we need to pass the salesforce id.
+	records_to_update = Array.new
+	records_to_update.push(updated_account)
+	salesforce.update("Account", records_to_update)
+
+    # Upsert
+	upserted_account = Hash["name" => "Test Account -- Upserted", "External_Field_Name" => "123456"] # Fields to be updated. External field must be included
+	records_to_upsert = Array.new
+	records_to_upsert.push(upserted_account)
+	salesforce.upsert("Account", records_to_upsert, "External_Field_Name") # Note that upsert accepts an extra parameter for the external field name
+
+    # Delete
+	deleted_account = Hash["id" => "a00A0001009zA2m"] # We only specify the id of the records to delete
+	records_to_delete = Array.new
+	records_to_delete.push(deleted_account)
+	salesforce.delete("Account", records_to_delete)
+
+    # Query
+	res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string
+
+### Helpful methods:
+
+    # Check status of a job via #job_from_id
+	job = salesforce.job_from_id('a00A0001009zA2m') # Returns a SalesforceBulkApi::Job instance
+	puts "status is: #{job.check_job_status.inspect}"
+
+### Listening to events:
+
+    # A job is created
+    # Useful when you need to store the job_id before any work begins, then if you fail during a complex load scenario, you can wait for your
+    # previous job(s) to finish.
+    salesforce.on_job_created do |job|
+      puts "Job #{job.job_id} created!"
+    end
+
+### Throttling API calls:
+
+    # By default, this gem (and maybe your app driving it) will query job/batch statuses at an unbounded rate.  We
+    # can fix that, e.g.:
+    salesforce.connection.set_status_throttle(30) # only check status of individual jobs/batches every 30 seconds
+
+## Installation
+
+    sudo gem install salesforce_bulk_api_serial_or_parallel
+	
+## Contribute
+
+Feel to fork and send Pull request

data/Rakefile

@@ -0,0 +1,3 @@
+require 'rspec/core/rake_task'
+task :default => :spec
+RSpec::Core::RakeTask.new

data/auth_credentials.yml.example

@@ -0,0 +1,7 @@
+salesforce:
+  client_id: client_id_here
+  client_secret: client_secret_here
+  user: sf_user@example.com
+  passwordandtoken: passandtokenhere
+  test_account_id: 0013000000ymMBh
+  host: 'login.salesforce.com' # use test.salesforce.com if it is a sandbox

data/lib/salesforce_bulk_api_serial_or_parallel.rb

@@ -0,0 +1,98 @@
+require 'rubygems'
+require 'bundler'
+require 'net/https'
+require 'xmlsimple'
+require 'csv'
+
+require 'salesforce_bulk_api_serial_or_parallel/version'
+require 'salesforce_bulk_api_serial_or_parallel/concerns/throttling'
+require 'salesforce_bulk_api_serial_or_parallel/job'
+require 'salesforce_bulk_api_serial_or_parallel/connection'
+
+module SalesforceBulkApiSerialOrParallel
+  class Api
+    attr_reader :connection
+
+    SALESFORCE_API_VERSION = '1.0.1'
+
+    def initialize(client)
+      @connection = SalesforceBulkApiSerialOrParallel::Connection.new(SALESFORCE_API_VERSION, client)
+      @listeners = { job_created: [] }
+    end
+
+    def upsert(sobject, records, external_field, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500, concurrency = nil)
+      do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list, concurrency)
+    end
+
+    def update(sobject, records, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500, concurrency = nil)
+      do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list, concurrency)
+    end
+
+    def create(sobject, records, get_response = false, send_nulls = false, batch_size = 10000, timeout = 1500, concurrency = nil)
+      do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls, concurrency)
+    end
+
+    def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 1500)
+      do_operation('delete', sobject, records, nil, get_response, timeout, batch_size, concurrency)
+    end
+
+    def query(sobject, query, batch_size = 10000, timeout = 1500)
+      do_operation('query', sobject, query, nil, true, timeout, batch_size)
+    end
+
+    def counters
+      {
+        http_get: @connection.counters[:get],
+        http_post: @connection.counters[:post],
+        upsert: get_counters[:upsert],
+        update: get_counters[:update],
+        create: get_counters[:create],
+        delete: get_counters[:delete],
+        query: get_counters[:query]
+      }
+    end
+
+
+    ##
+    # Allows you to attach a listener that accepts the created job (which has a useful #job_id field).  This is useful
+    # for recording a job ID persistently before you begin batch work (i.e. start modifying the salesforce database),
+    # so if the load process you are writing needs to recover, it can be aware of previous jobs it started and wait
+    # for them to finish.
+    def on_job_created(&block)
+      @listeners[:job_created] << block
+    end
+
+    def job_from_id(job_id)
+      SalesforceBulkApiSerialOrParallel::Job.new(job_id: job_id, connection: @connection)
+    end
+
+    def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, send_nulls = false, no_null_list = [], concurrency_mode = nil)
+      count operation.to_sym
+
+      job = SalesforceBulkApiSerialOrParallel::Job.new(
+          operation: operation,
+          sobject: sobject,
+          records: records,
+          external_field: external_field,
+          connection: @connection
+      )
+
+      job.create_job(batch_size, send_nulls, no_null_list, concurrency_mode)
+      @listeners[:job_created].each {|callback| callback.call(job)}
+      operation == "query" ? job.add_query() : job.add_batches()
+      response = job.close_job
+      response.merge!({'batches' => job.get_job_result(get_response, timeout)}) if get_response == true
+      response
+    end
+
+    private
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(name)
+      get_counters[name] += 1
+    end
+
+  end
+end

data/lib/salesforce_bulk_api_serial_or_parallel/concerns/throttling.rb

@@ -0,0 +1,60 @@
+module SalesforceBulkApiSerialOrParallel::Concerns
+  module Throttling
+
+    def throttles
+      @throttles.dup
+    end
+
+    def add_throttle(&throttling_callback)
+      @throttles ||= []
+      @throttles << throttling_callback
+    end
+
+    def set_status_throttle(limit_seconds)
+      set_throttle_limit_in_seconds(limit_seconds, [:http_method, :path], ->(details) { details[:http_method] == :get })
+    end
+
+    def set_throttle_limit_in_seconds(limit_seconds, throttle_by_keys, only_if)
+      add_throttle do |details|
+        limit_log = get_limit_log(Time.now - limit_seconds)
+        key = extract_constraint_key_from(details, throttle_by_keys)
+        last_request = limit_log[key]
+
+        if !last_request.nil? && only_if.call(details)
+          seconds_since_last_request = Time.now.to_f - last_request.to_f
+          need_to_wait_seconds = limit_seconds - seconds_since_last_request
+          sleep(need_to_wait_seconds) if need_to_wait_seconds > 0
+        end
+
+        limit_log[key] = Time.now
+      end
+    end
+
+    private
+
+    def extract_constraint_key_from(details, throttle_by_keys)
+      hash = {}
+      throttle_by_keys.each { |k| hash[k] = details[k] }
+      hash
+    end
+
+    def get_limit_log(prune_older_than)
+      @limits ||= Hash.new(0)
+
+      @limits.delete_if do |k, v|
+        v < prune_older_than
+      end
+
+      @limits
+    end
+
+    def throttle(details={})
+      (@throttles || []).each do |callback|
+        args = [details]
+        args = args[0..callback.arity]
+        callback.call(*args)
+      end
+    end
+
+  end
+end

data/lib/salesforce_bulk_api_serial_or_parallel/connection.rb

@@ -0,0 +1,98 @@
+module SalesforceBulkApiSerialOrParallel
+require 'timeout'
+
+  class Connection
+    include Concerns::Throttling
+
+    LOGIN_HOST = 'login.salesforce.com'
+
+    def initialize(api_version, client)
+      @client = client
+      @api_version = api_version
+      @path_prefix = "/services/async/#{@api_version}/"
+
+      login()
+    end
+
+    def login()
+      client_type = @client.class.to_s
+      case client_type
+      when "Restforce::Data::Client"
+        @session_id = @client.options[:oauth_token]
+        @server_url = @client.options[:instance_url]
+      else
+        @session_id = @client.oauth_token
+        @server_url = @client.instance_url
+      end
+      @instance = parse_instance()
+      @instance_host = "#{@instance}.salesforce.com"
+    end
+
+    def post_xml(host, path, xml, headers)
+      host = host || @instance_host
+      if host != LOGIN_HOST # Not login, need to add session id to header
+        headers['X-SFDC-Session'] = @session_id
+        path = "#{@path_prefix}#{path}"
+      end
+      i = 0
+      begin
+        count :post
+        throttle(http_method: :post, path: path)
+        https(host).post(path, xml, headers).body
+      rescue
+        i += 1
+        if i < 3
+          puts "Request fail #{i}: Retrying #{path}"
+          retry
+        else
+          puts "FATAL: Request to #{path} failed three times."
+          raise
+        end
+      end
+    end
+
+    def get_request(host, path, headers)
+      host = host || @instance_host
+      path = "#{@path_prefix}#{path}"
+      if host != LOGIN_HOST # Not login, need to add session id to header
+        headers['X-SFDC-Session'] = @session_id;
+      end
+
+      count :get
+      throttle(http_method: :get, path: path)
+      https(host).get(path, headers).body
+    end
+
+    def https(host)
+      req = Net::HTTP.new(host, 443)
+      req.use_ssl = true
+      req.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      req
+    end
+
+    def parse_instance()
+      @instance = @server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}/).to_s.gsub("https://","")
+      @instance = @server_url.split(".salesforce.com")[0].split("://")[1] if @instance.nil? || @instance.empty?
+      return @instance
+    end
+
+    def counters
+      {
+        get: get_counters[:get],
+        post: get_counters[:post]
+      }
+    end
+
+    private
+
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(http_method)
+      get_counters[http_method] += 1
+    end
+
+  end
+
+end

data/lib/salesforce_bulk_api_serial_or_parallel/job.rb

@@ -0,0 +1,235 @@
+module SalesforceBulkApiSerialOrParallel
+
+  class Job
+    attr_reader :job_id
+
+    class SalesforceException < StandardError; end
+
+    def initialize(args)
+      @job_id         = args[:job_id]
+      @operation      = args[:operation]
+      @sobject        = args[:sobject]
+      @external_field = args[:external_field]
+      @records        = args[:records]
+      @connection     = args[:connection]
+      @batch_ids      = []
+      @XML_HEADER     = '<?xml version="1.0" encoding="utf-8" ?>'
+    end
+
+    def create_job(batch_size, send_nulls, no_null_list, concurrency_mode)
+      @batch_size = batch_size
+      @send_nulls = send_nulls
+      @no_null_list = no_null_list
+      @concurrency_mode = concurrency_mode
+
+      xml = "#{@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml += "<operation>#{@operation}</operation>"
+      xml += "<object>#{@sobject}</object>"
+      if @concurrency_mode
+        xml += "<concurrencyMode>#{@concurrency_mode}</concurrencyMode>"
+      end
+      # This only happens on upsert
+      if !@external_field.nil?
+        xml += "<externalIdFieldName>#{@external_field}</externalIdFieldName>"
+      end
+      xml += "<contentType>XML</contentType>"
+      xml += "</jobInfo>"
+
+      path = "job"
+      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
+
+      response = @connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      # response may contain an exception, so raise it
+      raise SalesforceException.new("#{response_parsed['exceptionMessage'][0]} (#{response_parsed['exceptionCode'][0]})") if response_parsed['exceptionCode']
+
+      @job_id = response_parsed['id'][0]
+    end
+
+    def close_job()
+      xml = "#{@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml += "<state>Closed</state>"
+      xml += "</jobInfo>"
+
+      path = "job/#{@job_id}"
+      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
+
+      response = @connection.post_xml(nil, path, xml, headers)
+      XmlSimple.xml_in(response)
+    end
+
+    def add_query
+      path = "job/#{@job_id}/batch/"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+
+      response = @connection.post_xml(nil, path, @records, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      @batch_ids << response_parsed['id'][0]
+    end
+
+    def add_batches
+      raise 'Records must be an array of hashes.' unless @records.is_a? Array
+      keys = @records.reduce({}) {|h, pairs| pairs.each {|k, v| (h[k] ||= []) << v}; h}.keys
+
+      @records_dup = @records.clone
+
+      super_records = []
+      (@records_dup.size / @batch_size).to_i.times do
+        super_records << @records_dup.pop(@batch_size)
+      end
+      super_records << @records_dup unless @records_dup.empty?
+
+      super_records.each do |batch|
+        @batch_ids << add_batch(keys, batch)
+      end
+    end
+
+    def add_batch(keys, batch)
+      xml = "#{@XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">"
+      batch.each do |r|
+        xml += create_sobject(keys, r)
+      end
+      xml += '</sObjects>'
+      path = "job/#{@job_id}/batch/"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+      response = @connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      response_parsed['id'][0] if response_parsed['id']
+    end
+
+    def build_sobject(data)
+      xml = '<sObject>'
+      data.keys.each do |k|
+        if k.is_a?(Hash)
+          xml += build_sobject(k)
+        elsif data[k] != :type
+          xml += "<#{k}>#{data[k]}</#{k}>"
+        end
+      end
+      xml += '</sObject>'
+    end
+
+    def create_sobject(keys, r)
+      sobject_xml = '<sObject>'
+      keys.each do |k|
+        if r[k].is_a?(Hash)
+          sobject_xml += "<#{k}>"
+          sobject_xml += build_sobject(r[k])
+          sobject_xml += "</#{k}>"
+        elsif !r[k].to_s.empty?
+          sobject_xml += "<#{k}>"
+          if r[k].respond_to?(:encode)
+            sobject_xml += r[k].encode(:xml => :text)
+          elsif r[k].respond_to?(:iso8601) # timestamps
+            sobject_xml += r[k].iso8601.to_s
+          else
+            sobject_xml += r[k].to_s
+          end
+          sobject_xml += "</#{k}>"
+        elsif @send_nulls && !@no_null_list.include?(k)
+          sobject_xml += "<#{k} xsi:nil=\"true\"/>"
+        end
+      end
+      sobject_xml += '</sObject>'
+      sobject_xml
+    end
+
+    def check_job_status
+      path = "job/#{@job_id}"
+      headers = Hash.new
+      response = @connection.get_request(nil, path, headers)
+
+      begin
+        response_parsed = XmlSimple.xml_in(response) if response
+        response_parsed
+      rescue StandardError => e
+        puts "Error parsing XML response for #{@job_id}"
+        puts e
+        puts e.backtrace
+      end
+    end
+
+    def check_batch_status(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}"
+      headers = Hash.new
+
+      response = @connection.get_request(nil, path, headers)
+
+      begin
+        response_parsed = XmlSimple.xml_in(response) if response
+        response_parsed
+      rescue StandardError => e
+        puts "Error parsing XML response for #{@job_id}, batch #{batch_id}"
+        puts e
+        puts e.backtrace
+      end
+    end
+
+    def get_job_result(return_result, timeout)
+      # timeout is in seconds
+      begin
+        state = []
+        Timeout::timeout(timeout, SalesforceBulkApiSerialOrParallel::JobTimeout) do
+          while true
+            job_status = self.check_job_status
+            if job_status && job_status['state'] && job_status['state'][0] == 'Closed'
+              batch_statuses = {}
+
+              batches_ready = @batch_ids.all? do |batch_id|
+                batch_state = batch_statuses[batch_id] = self.check_batch_status(batch_id)
+                batch_state && batch_state['state'] && batch_state['state'][0] && !['Queued', 'InProgress'].include?(batch_state['state'][0])
+              end
+
+              if batches_ready
+                @batch_ids.each do |batch_id|
+                  state.insert(0, batch_statuses[batch_id])
+                  @batch_ids.delete(batch_id)
+                end
+              end
+              break if @batch_ids.empty?
+            else
+              break
+            end
+          end
+        end
+      rescue SalesforceBulkApiSerialOrParallel::JobTimeout => e
+        puts 'Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}.'
+        puts e
+        raise
+      end
+
+      state.each_with_index do |batch_state, i|
+        if batch_state['state'][0] == 'Completed' && return_result == true
+          state[i].merge!({'response' => self.get_batch_result(batch_state['id'][0])})
+        end
+      end
+      state
+    end
+
+    def get_batch_result(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}/result"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+
+      response = @connection.get_request(nil, path, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      results = response_parsed['result'] unless @operation == 'query'
+
+      if(@operation == 'query') # The query op requires us to do another request to get the results
+        result_id = response_parsed["result"][0]
+        path = "job/#{@job_id}/batch/#{batch_id}/result/#{result_id}"
+        headers = Hash.new
+        headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+        response = @connection.get_request(nil, path, headers)
+        response_parsed = XmlSimple.xml_in(response)
+        results = response_parsed['records']
+      end
+      results
+    end
+
+  end
+
+  class JobTimeout < StandardError
+  end
+end

data/lib/salesforce_bulk_api_serial_or_parallel/version.rb

@@ -0,0 +1,3 @@
+module SalesforceBulkApiSerialOrParallel
+  VERSION = '0.1'
+end

data/salesforce_bulk_api_serial_or_parallel.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "salesforce_bulk_api_serial_or_parallel/version"
+
+Gem::Specification.new do |s|
+  s.name        = 'salesforce_bulk_api_serial_or_parallel'
+  s.version     = SalesforceBulkApiSerialOrParallel::VERSION
+  s.authors     = ['Brendan Keogh']
+  s.email       = ['bkeogh123@gmail.com']
+
+  s.homepage    = 'https://github.com/beekermememe/salesforce_bulk_api_serial_or_parallel'
+  s.summary     = %q{It uses the bulk api of salesforce to communicate with Salesforce CRM}
+  s.description = %q{Salesforce Bulk API with governor limits taken care of, this is a fork off of yatish27/salesforce_bulk_api_serial_or_parallel that just adds serial/parallel concurrency support}
+
+  s.rubyforge_project = 'salesforce_bulk_api_serial_or_parallel'
+
+  s.add_dependency('json', ['>= 0'])
+  s.add_dependency('xml-simple', ['>= 0'])
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'restforce', '~> 1.5.1'
+  s.add_development_dependency 'rake', '~> 10.4.2'
+  s.add_development_dependency 'pry'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+end

data/spec/salesforce_bulk_api_serial_or_parallel/salesforce_bulk_api_serial_or_parallel.rb

@@ -0,0 +1,193 @@
+require 'spec_helper'
+require 'yaml'
+require 'restforce'
+
+describe SalesforceBulkApiSerialOrParallel do
+
+  before :each do
+    auth_hash = YAML.load_file('auth_credentials.yml')
+    sfdc_auth_hash = auth_hash['salesforce']
+
+    @sf_client = Restforce.new(
+      username: sfdc_auth_hash['user'],
+      password: sfdc_auth_hash['passwordandtoken'],
+      client_id: sfdc_auth_hash['client_id'],
+      client_secret: sfdc_auth_hash['client_secret'],
+      host: sfdc_auth_hash['host'])
+    @sf_client.authenticate!
+
+    @account_id = auth_hash['salesforce']['test_account_id']
+
+    @api = SalesforceBulkApiSerialOrParallel::Api.new(@sf_client)
+  end
+
+  after :each do
+
+  end
+
+  describe 'upsert' do
+
+    context 'when not passed get_result' do
+      it "doesn't return the batches array" do
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => 'www.test.com'}], 'Id')
+        res['batches'].should be_nil
+      end
+    end
+
+    context 'when passed get_result = true' do
+      it 'returns the batches array' do
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => 'www.test.com'}], 'Id', true)
+        res['batches'][0]['response'].is_a? Array
+
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+
+      end
+    end
+
+    context 'when passed send_nulls = true' do
+      it 'sets the nil and empty attributes to NULL' do
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        res['batches'][0]['response'][0]['Website'][0].should eq 'abc123'
+        res['batches'][0]['response'][0]['Phone'][0].should eq '5678'
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => '', :Phone => nil}], 'Id', true, true)
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        res['batches'][0]['response'][0]['Website'][0].should eq({"xsi:nil" => "true"})
+        res['batches'][0]['response'][0]['Phone'][0].should eq({"xsi:nil" => "true"})
+      end
+    end
+
+    context 'when passed send_nulls = true and an array of fields not to null' do
+      it 'sets the nil and empty attributes to NULL, except for those included in the list of fields to ignore' do
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        res['batches'][0]['response'][0]['Website'][0].should eq 'abc123'
+        res['batches'][0]['response'][0]['Phone'][0].should eq '5678'
+        res = @api.upsert('Account', [{:Id => @account_id, :Website => '', :Phone => nil}], 'Id', true, true, [:Website, :Phone])
+        res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+        res['batches'][0]['response'][0]['success'].should eq ['true']
+        res['batches'][0]['response'][0]['created'].should eq ['false']
+        res = @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        res['batches'][0]['response'][0]['Website'][0].should eq('abc123')
+        res['batches'][0]['response'][0]['Phone'][0].should eq('5678')
+      end
+    end
+
+  end
+
+  describe 'update' do
+    context 'when there is not an error' do
+      context 'when not passed get_result' do
+        it "doesnt return the batches array" do
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}])
+          res['batches'].should be_nil
+        end
+      end
+
+      context 'when passed get_result = true' do
+        it 'returns the batches array' do
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}], true)
+          res['batches'][0]['response'].is_a? Array
+          res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][0]['success'].should eq ['true']
+          res['batches'][0]['response'][0]['created'].should eq ['false']
+        end
+      end
+    end
+
+    context 'when there is an error' do
+      context 'when not passed get_result' do
+        it "doesn't return the results array" do
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'},{:Id => 'abc123', :Website => 'www.test.com'}])
+          res['batches'].should be_nil
+        end
+      end
+
+      context 'when passed get_result = true with batches' do
+        it 'returns the results array' do
+          res = @api.update('Account', [{:Id => @account_id, :Website => 'www.test.com'}, {:Id => @account_id, :Website => 'www.test.com'}, {:Id => @account_id, :Website => 'www.test.com'}, {:Id => 'abc123', :Website => 'www.test.com'}], true, false, [], 2)
+
+          res['batches'][0]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][0]['success'].should eq ['true']
+          res['batches'][0]['response'][0]['created'].should eq ['false']
+          res['batches'][0]['response'][1]['id'][0].should start_with(@account_id)
+          res['batches'][0]['response'][1]['success'].should eq ['true']
+          res['batches'][0]['response'][1]['created'].should eq ['false']
+
+          res['batches'][1]['response'][0]['id'][0].should start_with(@account_id)
+          res['batches'][1]['response'][0]['success'].should eq ['true']
+          res['batches'][1]['response'][0]['created'].should eq ['false']
+          res['batches'][1]['response'][1].should eq({"errors"=>[{"fields"=>["Id"], "message"=>["Account ID: id value of incorrect type: abc123"], "statusCode"=>["MALFORMED_ID"]}], "success"=>["false"], "created"=>["false"]})
+        end
+      end
+    end
+
+  end
+
+  describe 'create' do
+    pending
+  end
+
+  describe 'delete' do
+    pending
+  end
+
+  describe 'query' do
+
+    context 'when there are results' do
+      it 'returns the query results' do
+        res = @api.query('Account', "SELECT id, Name From Account WHERE Name LIKE 'Test%'")
+        res['batches'][0]['response'].length.should > 1
+        res['batches'][0]['response'][0]['Id'].should_not be_nil
+      end
+
+      context 'and there are multiple batches' do
+        # need dev to create > 10k records in dev organization
+        it 'returns the query results in a merged hash'
+      end
+    end
+
+    context 'when there are no results' do
+      it 'returns nil' do
+        res = @api.query('Account', "SELECT id From Account WHERE Name = 'ABC'")
+        res['batches'][0]['response'].should eq nil
+      end
+    end
+
+    context 'when there is an error' do
+      it 'returns nil' do
+        res = @api.query('Account', "SELECT id From Account WHERE Name = ''ABC'")
+        res['batches'][0]['response'].should eq nil
+      end
+    end
+
+  end
+
+  describe 'counters' do
+    context 'when read operations are called' do
+      it 'increments operation count and http GET count' do
+        @api.counters[:http_get].should eq 0
+        @api.counters[:query].should eq 0
+        @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        @api.counters[:http_get].should eq 1
+        @api.counters[:query].should eq 1
+      end
+    end
+
+    context 'when update operations are called' do
+      it 'increments operation count and http POST count' do
+        @api.counters[:http_post].should eq 0
+        @api.counters[:update].should eq 0
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        @api.counters[:http_post].should eq 1
+        @api.counters[:update].should eq 1
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'salesforce_bulk_api_serial_or_parallel'
+
+RSpec.configure do |c|
+  c.filter_run :focus => true
+  c.run_all_when_everything_filtered = true
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: salesforce_bulk_api_serial_or_parallel
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Brendan Keogh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: restforce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Salesforce Bulk API with governor limits taken care of, this is a fork
+  off of yatish27/salesforce_bulk_api_serial_or_parallel that just adds serial/parallel
+  concurrency support
+email:
+- bkeogh123@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENCE
+- README.md
+- Rakefile
+- auth_credentials.yml.example
+- lib/salesforce_bulk_api_serial_or_parallel.rb
+- lib/salesforce_bulk_api_serial_or_parallel/concerns/throttling.rb
+- lib/salesforce_bulk_api_serial_or_parallel/connection.rb
+- lib/salesforce_bulk_api_serial_or_parallel/job.rb
+- lib/salesforce_bulk_api_serial_or_parallel/version.rb
+- salesforce_bulk_api_serial_or_parallel.gemspec
+- spec/salesforce_bulk_api_serial_or_parallel/salesforce_bulk_api_serial_or_parallel.rb
+- spec/spec_helper.rb
+homepage: https://github.com/beekermememe/salesforce_bulk_api_serial_or_parallel
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: salesforce_bulk_api_serial_or_parallel
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: It uses the bulk api of salesforce to communicate with Salesforce CRM
+test_files:
+- spec/salesforce_bulk_api_serial_or_parallel/salesforce_bulk_api_serial_or_parallel.rb
+- spec/spec_helper.rb