salesforce_bulk_api 1.2.0 → 1.3.0

data/lib/salesforce_bulk_api/job.rb

@@ -1,5 +1,3 @@
-require "timeout"
-
 module SalesforceBulkApi
   class Job
     attr_reader :job_id
@@ -23,121 +21,242 @@ module SalesforceBulkApi
       @send_nulls = send_nulls
       @no_null_list = no_null_list
 
-      xml = build_job_xml
-      response = post_xml("job", xml)
-      parse_job_response(response)
+      xml = "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml << "<operation>#{@operation}</operation>"
+      xml << "<object>#{@sobject}</object>"
+      # This only happens on upsert
+      if !@external_field.nil?
+        xml << "<externalIdFieldName>#{@external_field}</externalIdFieldName>"
+      end
+      xml << "<contentType>XML</contentType>"
+      xml << "</jobInfo>"
+
+      path = "job"
+      headers = {"Content-Type" => "application/xml; charset=utf-8"}
+
+      response = @connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      # response may contain an exception, so raise it
+      raise SalesforceException.new("#{response_parsed["exceptionMessage"][0]} (#{response_parsed["exceptionCode"][0]})") if response_parsed["exceptionCode"]
+
+      @job_id = response_parsed["id"][0]
     end
 
     def close_job
-      xml = build_close_job_xml
-      response = post_xml("job/#{@job_id}", xml)
+      xml = "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml << "<state>Closed</state>"
+      xml << "</jobInfo>"
+
+      path = "job/#{@job_id}"
+      headers = {"Content-Type" => "application/xml; charset=utf-8"}
+
+      response = @connection.post_xml(nil, path, xml, headers)
       XmlSimple.xml_in(response)
     end
 
     def add_query
-      response = post_xml("job/#{@job_id}/batch/", @records)
+      path = "job/#{@job_id}/batch/"
+      headers = {"Content-Type" => "application/xml; charset=UTF-8"}
+
+      response = @connection.post_xml(nil, path, @records, headers)
       response_parsed = XmlSimple.xml_in(response)
+
       @batch_ids << response_parsed["id"][0]
     end
 
     def add_batches
-      raise ArgumentError, "Records must be an array of hashes." unless @records.is_a?(Array)
+      raise ArgumentError, "Records must be an array of hashes." unless @records.is_a? Array
+      keys = @records.each_with_object({}) { |pairs, h|
+        pairs.each { |k, v| (h[k] ||= []) << v }
+      }.keys
 
-      keys = @records.each_with_object({}) { |pairs, h| pairs.each { |k, v| (h[k] ||= []) << v } }.keys
-      batches = @records.each_slice(@batch_size).to_a
+      @records_dup = @records.clone
 
-      batches.each do |batch|
+      super_records = []
+      (@records_dup.size / @batch_size).to_i.times do
+        super_records << @records_dup.pop(@batch_size)
+      end
+      super_records << @records_dup unless @records_dup.empty?
+
+      super_records.each do |batch|
         @batch_ids << add_batch(keys, batch)
       end
     end
 
-    def get_job_result(return_result, timeout)
-      state = []
-      Timeout.timeout(timeout, JobTimeout) do
-        loop do
-          job_status = check_job_status
-          break unless job_closed_and_batches_completed?(job_status, state)
-          break if @batch_ids.empty?
-        end
+    def add_batch(keys, batch)
+      xml = "#{XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">"
+      batch.each do |r|
+        xml << create_sobject(keys, r)
       end
-    rescue JobTimeout => e
-      handle_timeout(e)
-    ensure
-      process_batch_results(state) if return_result
-      state
+      xml << "</sObjects>"
+      path = "job/#{@job_id}/batch/"
+      headers = {"Content-Type" => "application/xml; charset=UTF-8"}
+      response = @connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      response_parsed["id"][0] if response_parsed["id"]
     end
 
-    private
-
-    def build_job_xml
-      xml = "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
-      xml << "<operation>#{@operation}</operation>"
-      xml << "<object>#{@sobject}</object>"
-      xml << "<externalIdFieldName>#{@external_field}</externalIdFieldName>" if @external_field
-      xml << "<contentType>XML</contentType>"
-      xml << "</jobInfo>"
+    def build_sobject(data)
+      xml = "<sObject>"
+      data.keys.each do |k|
+        if k.is_a?(Hash)
+          xml << build_sobject(k)
+        elsif k.to_s.include? "."
+          relations = k.to_s.split(".")
+          parent = relations[0]
+          child = relations[1..].join(".")
+          xml << "<#{parent}>#{build_sobject({child => data[k]})}</#{parent}>"
+        elsif data[k] != :type
+          xml << "<#{k}>#{data[k]}</#{k}>"
+        end
+      end
+      xml << "</sObject>"
     end
 
-    def build_close_job_xml
-      "#{XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\"><state>Closed</state></jobInfo>"
+    def build_relationship_sobject(key, value)
+      if key.to_s.include? "."
+        relations = key.to_s.split(".")
+        parent = relations[0]
+        child = relations[1..].join(".")
+        xml = "<#{parent}>"
+        xml << "<sObject>"
+        xml << build_relationship_sobject(child, value)
+        xml << "</sObject>"
+        xml << "</#{parent}>"
+      else
+        "<#{key}>#{value}</#{key}>"
+      end
     end
 
-    def post_xml(path, xml)
-      headers = {"Content-Type" => "application/xml; charset=utf-8"}
-      @connection.post_xml(nil, path, xml, headers)
+    def create_sobject(keys, r)
+      sobject_xml = "<sObject>"
+      keys.each do |k|
+        if r[k].is_a?(Hash)
+          sobject_xml << "<#{k}>"
+          sobject_xml << build_sobject(r[k])
+          sobject_xml << "</#{k}>"
+        elsif k.to_s.include? "."
+          sobject_xml << build_relationship_sobject(k, r[k])
+        elsif !r[k].to_s.empty?
+          sobject_xml << "<#{k}>"
+          sobject_xml << if r[k].respond_to?(:encode)
+            r[k].encode(xml: :text)
+          elsif r[k].respond_to?(:iso8601) # timestamps
+            r[k].iso8601.to_s
+          else
+            r[k].to_s
+          end
+          sobject_xml << "</#{k}>"
+        elsif @send_nulls && !@no_null_list.include?(k) && r.key?(k)
+          sobject_xml << "<#{k} xsi:nil=\"true\"/>"
+        end
+      end
+      sobject_xml << "</sObject>"
+      sobject_xml
     end
 
-    def parse_job_response(response)
-      response_parsed = XmlSimple.xml_in(response)
-      if response_parsed["exceptionCode"]
-        raise SalesforceException, "#{response_parsed["exceptionMessage"][0]} (#{response_parsed["exceptionCode"][0]})"
+    def check_job_status
+      path = "job/#{@job_id}"
+      headers = {}
+      response = @connection.get_request(nil, path, headers)
+
+      begin
+        response_parsed = XmlSimple.xml_in(response) if response
+        response_parsed
+      rescue => e
+        puts "Error parsing XML response for #{@job_id}"
+        puts e
+        puts e.backtrace
       end
-      @job_id = response_parsed["id"][0]
     end
 
-    def add_batch(keys, batch)
-      xml = "#{XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">"
-      batch.each { |r| xml << create_sobject(keys, r) }
-      xml << "</sObjects>"
+    def check_batch_status(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}"
+      headers = {}
 
-      response = post_xml("job/#{@job_id}/batch/", xml)
-      response_parsed = XmlSimple.xml_in(response)
-      response_parsed["id"]&.first
+      response = @connection.get_request(nil, path, headers)
+
+      begin
+        response_parsed = XmlSimple.xml_in(response) if response
+        response_parsed
+      rescue => e
+        puts "Error parsing XML response for #{@job_id}, batch #{batch_id}"
+        puts e
+        puts e.backtrace
+      end
     end
 
-    def job_closed_and_batches_completed?(job_status, state)
-      return false unless job_status && job_status["state"] && job_status["state"][0] == "Closed"
+    def get_job_result(return_result, timeout)
+      # timeout is in seconds
+      begin
+        state = []
+        Timeout.timeout(timeout, SalesforceBulkApi::JobTimeout) do
+          loop do
+            job_status = check_job_status
+            if job_status && job_status["state"] && job_status["state"][0] == "Closed"
+              batch_statuses = {}
 
-      batch_statuses = {}
-      batches_ready = @batch_ids.all? do |batch_id|
-        batch_state = batch_statuses[batch_id] = check_batch_status(batch_id)
-        batch_state && batch_state["state"] && batch_state["state"][0] && !["Queued", "InProgress"].include?(batch_state["state"][0])
-      end
+              batches_ready = @batch_ids.all? do |batch_id|
+                batch_state = batch_statuses[batch_id] = check_batch_status(batch_id)
+                batch_state && batch_state["state"] && batch_state["state"][0] && !["Queued", "InProgress"].include?(batch_state["state"][0])
+              end
 
-      if batches_ready
-        @batch_ids.each do |batch_id|
-          state.unshift(batch_statuses[batch_id])
-          @batch_ids.delete(batch_id)
+              if batches_ready
+                @batch_ids.each do |batch_id|
+                  state.insert(0, batch_statuses[batch_id])
+                  @batch_ids.delete(batch_id)
+                end
+              end
+              break if @batch_ids.empty?
+            else
+              break
+            end
+          end
         end
+      rescue SalesforceBulkApi::JobTimeout => e
+        puts "Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}."
+        puts e
+        raise
       end
 
-      true
+      state.each_with_index do |batch_state, i|
+        if batch_state["state"][0] == "Completed" && return_result == true
+          state[i].merge!({"response" => get_batch_result(batch_state["id"][0])})
+        end
+      end
+      state
     end
 
-    def handle_timeout(error)
-      puts "Timeout waiting for Salesforce to process job batches #{@batch_ids} of job #{@job_id}."
-      puts error
-      raise
-    end
+    def get_batch_result(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}/result"
+      headers = {"Content-Type" => "application/xml; charset=UTF-8"}
 
-    def process_batch_results(state)
-      state.each_with_index do |batch_state, i|
-        if batch_state["state"][0] == "Completed"
-          state[i].merge!("response" => get_batch_result(batch_state["id"][0]))
-        end
+      response = @connection.get_request(nil, path, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      results = response_parsed["result"] unless @operation == "query"
+
+      if @operation == "query" # The query op requires us to do another request to get the results
+        result_id = response_parsed["result"][0]
+        path = "job/#{@job_id}/batch/#{batch_id}/result/#{result_id}"
+        headers = {"Content-Type" => "application/xml; charset=UTF-8"}
+        response = @connection.get_request(nil, path, headers)
+        response_parsed = XmlSimple.xml_in(response)
+        results = response_parsed["records"]
       end
+      results
+    end
+
+    def get_batch_records(batch_id)
+      path = "job/#{@job_id}/batch/#{batch_id}/request"
+      headers = {"Content-Type" => "application/xml; charset=UTF-8"}
+
+      response = @connection.get_request(nil, path, headers)
+      response_parsed = XmlSimple.xml_in(response)
+      response_parsed["sObject"]
     end
   end
 
-  class JobTimeout < StandardError; end
+  class JobTimeout < StandardError
+  end
 end

data/lib/salesforce_bulk_api/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkApi
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/salesforce_bulk_api.rb

@@ -16,28 +16,37 @@ module SalesforceBulkApi
     def initialize(client, salesforce_api_version = "46.0")
       @connection = SalesforceBulkApi::Connection.new(salesforce_api_version, client)
       @listeners = {job_created: []}
-      @counters = Hash.new(0)
     end
 
-    %w[upsert update create delete].each do |operation|
-      define_method(operation) do |sobject, records, external_field = nil, **options|
-        do_operation(operation, sobject, records, external_field, **options)
-      end
+    def upsert(sobject, records, external_field, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
+      do_operation("upsert", sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
     end
 
-    def query(sobject, query, **)
-      do_operation("query", sobject, query, nil, get_response: true, **)
+    def update(sobject, records, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
+      do_operation("update", sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
+    end
+
+    def create(sobject, records, get_response = false, send_nulls = false, batch_size = 10000, timeout = 1500)
+      do_operation("insert", sobject, records, nil, get_response, timeout, batch_size, send_nulls)
+    end
+
+    def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 1500)
+      do_operation("delete", sobject, records, nil, get_response, timeout, batch_size)
+    end
+
+    def query(sobject, query, batch_size = 10000, timeout = 1500)
+      do_operation("query", sobject, query, nil, true, timeout, batch_size)
     end
 
     def counters
       {
         http_get: @connection.counters[:get],
         http_post: @connection.counters[:post],
-        upsert: @counters[:upsert],
-        update: @counters[:update],
-        create: @counters[:create],
-        delete: @counters[:delete],
-        query: @counters[:query]
+        upsert: get_counters[:upsert],
+        update: get_counters[:update],
+        create: get_counters[:create],
+        delete: get_counters[:delete],
+        query: get_counters[:query]
       }
     end
 
@@ -54,10 +63,8 @@ module SalesforceBulkApi
       SalesforceBulkApi::Job.new(job_id: job_id, connection: @connection)
     end
 
-    private
-
-    def do_operation(operation, sobject, records, external_field, **options)
-      count(operation.to_sym)
+    def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, send_nulls = false, no_null_list = [])
+      count operation.to_sym
 
       job = SalesforceBulkApi::Job.new(
         operation: operation,
@@ -67,18 +74,22 @@ module SalesforceBulkApi
         connection: @connection
       )
 
-      job.create_job(options[:batch_size], options[:send_nulls], options[:no_null_list])
+      job.create_job(batch_size, send_nulls, no_null_list)
       @listeners[:job_created].each { |callback| callback.call(job) }
-
       (operation == "query") ? job.add_query : job.add_batches
-
       response = job.close_job
-      response.merge!("batches" => job.get_job_result(options[:get_response], options[:timeout])) if options[:get_response]
+      response["batches"] = job.get_job_result(get_response, timeout) if get_response == true
       response
     end
 
+    private
+
+    def get_counters
+      @counters ||= Hash.new { |hash, key| hash[key] = 0 }
+    end
+
     def count(name)
-      @counters[name] += 1
+      get_counters[name] += 1
     end
   end
 end

data/salesforce_bulk_api.gemspec

@@ -14,12 +14,15 @@ Gem::Specification.new do |s|
   s.add_dependency("json", [">= 0"])
   s.add_dependency("xml-simple", [">= 0"])
   s.add_dependency("csv", [">= 0"])
+  s.add_dependency("logger", [">= 0"])
 
   s.add_development_dependency "rspec"
-  s.add_development_dependency "restforce", "~>  3.0.0"
+  s.add_development_dependency "restforce", "~> 8.0.0"
   s.add_development_dependency "rake", ">= 12.3.3"
   s.add_development_dependency "pry"
-  s.add_development_dependency "standardrb"
+  s.add_development_dependency "rubocop"
+  s.add_development_dependency "rubocop-rake"
+  s.add_development_dependency "dotenv"
 
   s.files = `git ls-files`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }