salesforce_bulk_api 1.1.0 → 1.2.0

data/lib/salesforce_bulk_api/job.rb

@@ -1,19 +1,21 @@
-module SalesforceBulkApi
+require "timeout"
 
+module SalesforceBulkApi
   class Job
     attr_reader :job_id
 
     class SalesforceException < StandardError; end
 
+    XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>'.freeze
+
     def initialize(args)
-      @job_id         = args[:job_id]
-      @operation      = args[:operation]
-      @sobject        = args[:sobject]
+      @job_id = args[:job_id]
+      @operation = args[:operation]
+      @sobject = args[:sobject]
       @external_field = args[:external_field]
-      @records        = args[:records]
-      @connection     = args[:connection]
-      @batch_ids      = []
-      @XML_HEADER     = '<?xml version="1.0" encoding="utf-8" ?>'
+      @records = args[:records]
+      @connection = args[:connection]
+      @batch_ids = []
     end
 
     def create_job(batch_size, send_nulls, no_null_list)
@@ -21,244 +23,121 @@ module SalesforceBulkApi
       @send_nulls = send_nulls
       @no_null_list = no_null_list
 
-      xml = "#{@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
-      xml += "<operation>#{@operation}</operation>"
-      xml += "<object>#{@sobject}</object>"
-      # This only happens on upsert
-      if !@external_field.nil?
-        xml += "<externalIdFieldName>#{@external_field}</externalIdFieldName>"
-      end
-      xml += "<contentType>XML</contentType>"
-      xml += "</jobInfo>"
-
-      path = "job"
-      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
-
-      response = @connection.post_xml(nil, path, xml, headers)
-      response_parsed = XmlSimple.xml_in(response)
-
-      # response may contain an exception, so raise it
-      raise SalesforceException.new("#{response_parsed['exceptionMessage'][0]} (#{response_parsed['exceptionCode'][0]})") if response_parsed['exceptionCode']
-
-      @job_id = response_parsed['id'][0]
+      xml = build_job_xml
+      response = post_xml("job", xml)
+      parse_job_response(response)
     end
 
-    def close_job()
-      xml = "#{@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
-      xml += "<state>Closed</state>"
-      xml += "</jobInfo>"
-
-      path = "job/#{@job_id}"
-      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
-
-      response = @connection.post_xml(nil, path, xml, headers)
+    def close_job
+      xml = build_close_job_xml
+      response = post_xml("job/#{@job_id}", xml)
       XmlSimple.xml_in(response)
     end
 
     def add_query
-      path = "job/#{@job_id}/batch/"
-      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
-
-      response = @connection.post_xml(nil, path, @records, headers)
+      response = post_xml("job/#{@job_id}/batch/", @records)
       response_parsed = XmlSimple.xml_in(response)
-
-      @batch_ids << response_parsed['id'][0]
+      @batch_ids << response_parsed["id"][0]
     end
 
     def add_batches
-      raise 'Records must be an array of hashes.' unless @records.is_a? Array
-      keys = @records.reduce({}) {|h, pairs| pairs.each {|k, v| (h[k] ||= []) << v}; h}.keys
+      raise ArgumentError, "Records must be an array of hashes." unless @records.is_a?(Array)
 
-      @records_dup = @records.clone
+      keys = @records.each_with_object({}) { |pairs, h| pairs.each { |k, v| (h[k] ||= []) << v } }.keys
+      batches = @records.each_slice(@batch_size).to_a
 
-      super_records = []
-      (@records_dup.size / @batch_size).to_i.times do
-        super_records << @records_dup.pop(@batch_size)
-      end
-      super_records << @records_dup unless @records_dup.empty?
-
-      super_records.each do |batch|
+      batches.each do |batch|
         @batch_ids << add_batch(keys, batch)
       end
     end
 
-    def add_batch(keys, batch)
-      xml = "#{@XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">"
-      batch.each do |r|
-        xml += create_sobject(keys, r)
-      end
-      xml += '</sObjects>'
-      path = "job/#{@job_id}/batch/"
-      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
-      response = @connection.post_xml(nil, path, xml, headers)
-      response_parsed = XmlSimple.xml_in(response)
-      response_parsed['id'][0] if response_parsed['id']
-    end
-
-    def build_sobject(data)
-      xml = '<sObject>'
-      data.keys.each do |k|
-        if k.is_a?(Hash)
-          xml += build_sobject(k)
-        elsif k.to_s.include? '.'
-          relations = k.to_s.split('.')
-          parent = relations[0]
-          child = relations[1..-1].join('.')
-          xml += "<#{parent}>#{build_sobject({ child => data[k] })}</#{parent}>"
-        elsif data[k] != :type
-          xml += "<#{k}>#{data[k]}</#{k}>"
+    def get_job_result(return_result, timeout)
+      state = []
+      Timeout.timeout(timeout, JobTimeout) do
+        loop do
+          job_status = check_job_status
+          break unless job_closed_and_batches_completed?(job_status, state)
+          break if @batch_ids.empty?
         end
       end
-      xml += '</sObject>'
+    rescue JobTimeout => e
+      handle_timeout(e)
+    ensure
+      process_batch_results(state) if return_result
+      state
     end
 
-    def build_relationship_sobject(key, value)
-      if key.to_s.include? '.'
-        relations = key.to_s.split('.')
-        parent = relations[0]
-        child = relations[1..-1].join('.')
-        xml = "<#{parent}>"
-        xml += "<sObject>"
-        xml += build_relationship_sobject(child, value)
-        xml += "</sObject>"
-        xml += "</#{parent}>"
-      else
-        xml = "<#{key}>#{value}</#{key}>"
-      end
+    private
+
+    def build_job_xml
+      xml = "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml << "<operation>#{@operation}</operation>"
+      xml << "<object>#{@sobject}</object>"
+      xml << "<externalIdFieldName>#{@external_field}</externalIdFieldName>" if @external_field
+      xml << "<contentType>XML</contentType>"
+      xml << "</jobInfo>"
     end
 
-    def create_sobject(keys, r)
-      sobject_xml = '<sObject>'
-      keys.each do |k|
-        if r[k].is_a?(Hash)
-          sobject_xml += "<#{k}>"
-          sobject_xml += build_sobject(r[k])
-          sobject_xml += "</#{k}>"
-        elsif k.to_s.include? '.'
-          sobject_xml += build_relationship_sobject(k, r[k])
-        elsif !r[k].to_s.empty?
-          sobject_xml += "<#{k}>"
-          if r[k].respond_to?(:encode)
-            sobject_xml += r[k].encode(:xml => :text)
-          elsif r[k].respond_to?(:iso8601) # timestamps
-            sobject_xml += r[k].iso8601.to_s
-          else
-            sobject_xml += r[k].to_s
-          end
-          sobject_xml += "</#{k}>"
-        elsif @send_nulls && !@no_null_list.include?(k) && r.key?(k)
-          sobject_xml += "<#{k} xsi:nil=\"true\"/>"
-        end
-      end
-      sobject_xml += '</sObject>'
-      sobject_xml
+    def build_close_job_xml
+      "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\"><state>Closed</state></jobInfo>"
     end
 
-    def check_job_status
-      path = "job/#{@job_id}"
-      headers = Hash.new
-      response = @connection.get_request(nil, path, headers)
+    def post_xml(path, xml)
+      headers = {"Content-Type" => "application/xml; charset=utf-8"}
+      @connection.post_xml(nil, path, xml, headers)
+    end
 
-      begin
-        response_parsed = XmlSimple.xml_in(response) if response
-        response_parsed
-      rescue StandardError => e
-        puts "Error parsing XML response for #{@job_id}"
-        puts e
-        puts e.backtrace
+    def parse_job_response(response)
+      response_parsed = XmlSimple.xml_in(response)
+      if response_parsed["exceptionCode"]
+        raise SalesforceException, "#{response_parsed["exceptionMessage"][0]} (#{response_parsed["exceptionCode"][0]})"
       end
+      @job_id = response_parsed["id"][0]
     end
 
-    def check_batch_status(batch_id)
-      path = "job/#{@job_id}/batch/#{batch_id}"
-      headers = Hash.new
-
-      response = @connection.get_request(nil, path, headers)
+    def add_batch(keys, batch)
+      xml = "#{XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">"
+      batch.each { |r| xml << create_sobject(keys, r) }
+      xml << "</sObjects>"
 
-      begin
-        response_parsed = XmlSimple.xml_in(response) if response
-        response_parsed
-      rescue StandardError => e
-        puts "Error parsing XML response for #{@job_id}, batch #{batch_id}"
-        puts e
-        puts e.backtrace
-      end
+      response = post_xml("job/#{@job_id}/batch/", xml)
+      response_parsed = XmlSimple.xml_in(response)
+      response_parsed["id"]&.first
     end
 
-    def get_job_result(return_result, timeout)
-      # timeout is in seconds
-      begin
-        state = []
-        Timeout::timeout(timeout, SalesforceBulkApi::JobTimeout) do
-          while true
-            job_status = self.check_job_status
-            if job_status && job_status['state'] && job_status['state'][0] == 'Closed'
-              batch_statuses = {}
-
-              batches_ready = @batch_ids.all? do |batch_id|
-                batch_state = batch_statuses[batch_id] = self.check_batch_status(batch_id)
-                batch_state && batch_state['state'] && batch_state['state'][0] && !['Queued', 'InProgress'].include?(batch_state['state'][0])
-              end
+    def job_closed_and_batches_completed?(job_status, state)
+      return false unless job_status && job_status["state"] && job_status["state"][0] == "Closed"
 
-              if batches_ready
-                @batch_ids.each do |batch_id|
-                  state.insert(0, batch_statuses[batch_id])
-                  @batch_ids.delete(batch_id)
-                end
-              end
-              break if @batch_ids.empty?
-            else
-              break
-            end
-          end
-        end
-      rescue SalesforceBulkApi::JobTimeout => e
-        puts 'Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}.'
-        puts e
-        raise
+      batch_statuses = {}
+      batches_ready = @batch_ids.all? do |batch_id|
+        batch_state = batch_statuses[batch_id] = check_batch_status(batch_id)
+        batch_state && batch_state["state"] && batch_state["state"][0] && !["Queued", "InProgress"].include?(batch_state["state"][0])
       end
 
-      state.each_with_index do |batch_state, i|
-        if batch_state['state'][0] == 'Completed' && return_result == true
-          state[i].merge!({'response' => self.get_batch_result(batch_state['id'][0])})
+      if batches_ready
+        @batch_ids.each do |batch_id|
+          state.unshift(batch_statuses[batch_id])
+          @batch_ids.delete(batch_id)
         end
       end
-      state
-    end
 
-    def get_batch_result(batch_id)
-      path = "job/#{@job_id}/batch/#{batch_id}/result"
-      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
-
-      response = @connection.get_request(nil, path, headers)
-      response_parsed = XmlSimple.xml_in(response)
-      results = response_parsed['result'] unless @operation == 'query'
-
-      if(@operation == 'query') # The query op requires us to do another request to get the results
-        result_id = response_parsed["result"][0]
-        path = "job/#{@job_id}/batch/#{batch_id}/result/#{result_id}"
-        headers = Hash.new
-        headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
-        response = @connection.get_request(nil, path, headers)
-        response_parsed = XmlSimple.xml_in(response)
-        results = response_parsed['records']
-      end
-      results
+      true
     end
 
-    def get_batch_records(batch_id)
-      path = "job/#{@job_id}/batch/#{batch_id}/request"
-      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
-
-      response = @connection.get_request(nil, path, headers)
-      response_parsed = XmlSimple.xml_in(response)
-      results = response_parsed['sObject']
-
-      results
+    def handle_timeout(error)
+      puts "Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}."
+      puts error
+      raise
     end
 
+    def process_batch_results(state)
+      state.each_with_index do |batch_state, i|
+        if batch_state["state"][0] == "Completed"
+          state[i].merge!("response" => get_batch_result(batch_state["id"][0]))
+        end
+      end
+    end
   end
 
-  class JobTimeout < StandardError
-  end
+  class JobTimeout < StandardError; end
 end

data/lib/salesforce_bulk_api/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkApi
-  VERSION = '1.1.0'
+  VERSION = "1.2.0"
 end

data/lib/salesforce_bulk_api.rb

@@ -1,13 +1,13 @@
-require 'rubygems'
-require 'bundler'
-require 'net/https'
-require 'xmlsimple'
-require 'csv'
+require "rubygems"
+require "bundler"
+require "net/https"
+require "xmlsimple"
+require "csv"
 
-require 'salesforce_bulk_api/version'
-require 'salesforce_bulk_api/concerns/throttling'
-require 'salesforce_bulk_api/job'
-require 'salesforce_bulk_api/connection'
+require "salesforce_bulk_api/version"
+require "salesforce_bulk_api/concerns/throttling"
+require "salesforce_bulk_api/job"
+require "salesforce_bulk_api/connection"
 
 module SalesforceBulkApi
   class Api
@@ -15,43 +15,34 @@ module SalesforceBulkApi
 
     def initialize(client, salesforce_api_version = "46.0")
       @connection = SalesforceBulkApi::Connection.new(salesforce_api_version, client)
-      @listeners = { job_created: [] }
+      @listeners = {job_created: []}
+      @counters = Hash.new(0)
     end
 
-    def upsert(sobject, records, external_field, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
+    %w[upsert update create delete].each do |operation|
+      define_method(operation) do |sobject, records, external_field = nil, **options|
+        do_operation(operation, sobject, records, external_field, **options)
+      end
     end
 
-    def update(sobject, records, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
-    end
-
-    def create(sobject, records, get_response = false, send_nulls = false, batch_size = 10000, timeout = 1500)
-      do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls)
-    end
-
-    def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 1500)
-      do_operation('delete', sobject, records, nil, get_response, timeout, batch_size)
-    end
-
-    def query(sobject, query, batch_size = 10000, timeout = 1500)
-      do_operation('query', sobject, query, nil, true, timeout, batch_size)
+    def query(sobject, query, **)
+      do_operation("query", sobject, query, nil, get_response: true, **)
     end
 
     def counters
       {
         http_get: @connection.counters[:get],
         http_post: @connection.counters[:post],
-        upsert: get_counters[:upsert],
-        update: get_counters[:update],
-        create: get_counters[:create],
-        delete: get_counters[:delete],
-        query: get_counters[:query]
+        upsert: @counters[:upsert],
+        update: @counters[:update],
+        create: @counters[:create],
+        delete: @counters[:delete],
+        query: @counters[:query]
       }
     end
 
-    # Allows you to attach a listener that accepts the created job (which has a useful #job_id field).  This is useful
-    # for recording a job ID persistently before you begin batch work (i.e. start modifying the salesforce database),
+    # Allows you to attach a listener that accepts the created job (which has a useful #job_id field).
+    # This is useful for recording a job ID persistently before you begin batch work (i.e. start modifying the salesforce database),
     # so if the load process you are writing needs to recover, it can be aware of previous jobs it started and wait
     # for them to finish.
     #
@@ -63,8 +54,10 @@ module SalesforceBulkApi
       SalesforceBulkApi::Job.new(job_id: job_id, connection: @connection)
     end
 
-    def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, send_nulls = false, no_null_list = [])
-      count operation.to_sym
+    private
+
+    def do_operation(operation, sobject, records, external_field, **options)
+      count(operation.to_sym)
 
       job = SalesforceBulkApi::Job.new(
         operation: operation,
@@ -74,23 +67,18 @@ module SalesforceBulkApi
         connection: @connection
       )
 
-      job.create_job(batch_size, send_nulls, no_null_list)
-      @listeners[:job_created].each {|callback| callback.call(job)}
-      operation == "query" ? job.add_query() : job.add_batches()
-      response = job.close_job
-      response.merge!({'batches' => job.get_job_result(get_response, timeout)}) if get_response == true
-      response
-    end
+      job.create_job(options[:batch_size], options[:send_nulls], options[:no_null_list])
+      @listeners[:job_created].each { |callback| callback.call(job) }
 
-    private
+      (operation == "query") ? job.add_query : job.add_batches
 
-    def get_counters
-      @counters ||= Hash.new(0)
+      response = job.close_job
+      response.merge!("batches" => job.get_job_result(options[:get_response], options[:timeout])) if options[:get_response]
+      response
     end
 
     def count(name)
-      get_counters[name] += 1
+      @counters[name] += 1
     end
-
   end
 end

data/salesforce_bulk_api.gemspec

@@ -1,28 +1,27 @@
-# -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 require "salesforce_bulk_api/version"
 
 Gem::Specification.new do |s|
-  s.name        = 'salesforce_bulk_api'
-  s.version     = SalesforceBulkApi::VERSION
-  s.authors     = ['Yatish Mehta']
-  s.email       = ['hi@example.com']
+  s.name = "salesforce_bulk_api"
+  s.version = SalesforceBulkApi::VERSION
+  s.authors = ["Yatish Mehta"]
+  s.email = ["yatish27@users.noreply.github.com"]
 
-  s.homepage    = 'https://github.com/yatishmehta27/salesforce_bulk_api'
-  s.summary     = %q{It uses the bulk api of salesforce to communicate with Salesforce CRM}
-  s.description = %q{Salesforce Bulk API with governor limits taken care of}
+  s.homepage = "https://github.com/yatishmehta27/salesforce_bulk_api"
+  s.summary = "It uses the bulk api of salesforce to communicate with Salesforce CRM"
+  s.description = "Salesforce Bulk API with governor limits taken care of"
 
-  s.add_dependency('json', ['>= 0'])
-  s.add_dependency('xml-simple', ['>= 0'])
+  s.add_dependency("json", [">= 0"])
+  s.add_dependency("xml-simple", [">= 0"])
+  s.add_dependency("csv", [">= 0"])
 
-  s.add_development_dependency 'rspec'
-  s.add_development_dependency 'restforce', '~>  3.0.0'
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "restforce", "~>  3.0.0"
   s.add_development_dependency "rake", ">= 12.3.3"
-  s.add_development_dependency 'pry'
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ['lib']
+  s.add_development_dependency "pry"
+  s.add_development_dependency "standardrb"
 
+  s.files = `git ls-files`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ["lib"]
 end