salesforce_bulk2 0.5.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/README.md

@@ -0,0 +1,143 @@
+# salesforce_bulk
+
+## Overview
+
+Salesforce Bulk is a simple Ruby gem for connecting to and using the [Salesforce Bulk API](http://www.salesforce.com/us/developer/docs/api_asynch/index.htm).
+
+## Installation
+
+Install SalesforceBulk from RubyGems:
+
+    gem install salesforce_bulk
+
+Or include it in your project's `Gemfile` with Bundler:
+
+    gem 'salesforce_bulk'
+
+## Contribute
+
+To contribute, fork this repo, create a topic branch, make changes, then send a pull request. Pull requests without accompanying tests will *not* be accepted. To run tests in your fork, just do:
+
+    bundle install
+    rake
+
+## Configuration and Initialization
+
+### Basic Configuration
+
+    require 'salesforce_bulk'
+    
+    client = SalesforceBulk::Client.new(username: 'MyUsername', password: 'MyPassword', token: 'MySecurityToken')
+    client.authenticate
+
+Optional keys include host (default: login.salesforce.com), version (default: 24.0) and debugging (default: false).
+
+### Configuring from a YAML file
+
+The optional keys mentioned in the Basic Configuration section can also be used here.
+
+    ---
+    username: MyUsername
+    password: MyPassword
+    token: MySecurityToken
+
+Then in a Ruby script:
+
+    require 'salesforce_bulk'
+    
+    client = SalesforceBulk::Client.new("config/salesforce_bulk.yml")
+    client.authenticate
+
+## Usage Examples
+
+Some requirements if you are moving from an older version of the gem. You must specify every key even if it has no value for each hash in the data array for a batch.
+
+### Basic Example
+
+    data1 = [{:Name__c => 'Test 1'}, {:Name__c => 'Test 2'}]
+    data2 = [{:Name__c => 'Test 3'}, {:Name__c => 'Test 4'}]
+    
+    job = client.add_job(:insert, :MyObject__c)
+    
+    # easily add multiple batches to a job
+    batch = client.add_batch(job.id, data1)
+    batch = client.add_batch(job.id, data2)
+    
+    job = client.close_job(job.id) # or use the abort_job(id) method
+
+### Adding a Job
+
+When adding a job you can specify the following operations for the first argument:
+- :delete
+- :insert
+- :update
+- :upsert
+- :query
+
+When using the :upsert operation you must specify an external ID field name:
+
+    job = client.add_job(:upsert, :MyObject__c, :external_id_field_name => :MyId__c)
+
+For any operation you should be able to specify a concurrency mode. The default is Parallel. The other choice is Serial.
+
+    job = client.add_job(:upsert, :MyObject__c, :concurrency_mode => :Serial, :external_id_field_name => :MyId__c)
+
+### Retrieving Info for a Job
+
+    job = client.job_info(jobId) # returns a Job object
+    
+    puts "Job #{job.id} is closed." if job.closed? # other: open?, aborted?
+
+### Retrieving Info for all Batches
+
+    batches = client.batch_info_list(jobId) # returns an Array of Batch objects
+    
+    batches.each do |batch|
+      puts "Batch #{batch.id} failed." if batch.failed? # other: completed?, failed?, in_progress?, queued?
+    end
+
+### Retrieving Info for a single Batch
+
+    batch = client.batch_info(jobId, batchId) # returns a Batch object
+    
+    puts "Batch #{batch.id} is in progress." if batch.in_progress?
+
+### Retrieving Batch Results (for Delete, Insert, Update and Upsert)
+
+To verify that a batch completed successfully or failed call the `batch_info` or `batch_info_list` methods first, otherwise if you call `batch_result` without verifying and the batch failed the method will raise an error.
+
+The object returned from the following example only applies to the operations: delete, insert, update and upsert. Query results are handled differently.
+
+    results = client.batch_result(jobId, batchId) # returns an Array of BatchResult objects
+    
+    results.each do |result|
+      puts "Item #{result.id} had an error of: #{result.error}" if result.error?
+    end
+
+### Retrieving Query based Batch Results
+
+To verify that a batch completed successfully or failed call the `batch_info` or `batch_info_list` methods first, otherwise if you call `batch_result` without verifying and the batch failed the method will raise an error.
+
+Query results are handled differently as the response will not contain the full result set. You'll have to page through sets if you added multiple batches to a job.
+
+    # returns a QueryResultCollection object (an Array)
+    results = client.batch_result(jobId, batchId)
+    
+    while results.any?
+      
+      # Assuming query was: SELECT Id, Name, CustomField__c FROM Account
+      results.each do |result|
+        puts result[:Id], result[:Name], result[:CustomField__c]
+      end
+      
+      puts "Another set is available." if results.next?
+      
+      results.next
+      
+    end
+
+## Copyright
+
+Copyright (c) 2011 Jorge Valdivia.
+
+Copyright (c) 2012 Javier Julio.

data/Rakefile

@@ -0,0 +1,22 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+namespace :doc do
+  require 'rdoc/task'
+  require File.expand_path('../lib/salesforce_bulk/version', __FILE__)
+  RDoc::Task.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = "SalesforceBulk #{SalesforceBulk::VERSION}"
+    rdoc.main = 'README.md'
+    rdoc.rdoc_files.include('README.md', 'LICENSE.md', 'lib/**/*.rb')
+  end
+end

data/lib/salesforce_bulk/batch.rb

@@ -0,0 +1,86 @@
+module SalesforceBulk
+  class Batch
+    attr_accessor :connection
+
+    attr_reader :apex_processing_time
+    attr_reader :api_active_processing_time
+    attr_reader :completed_at
+    attr_reader :created_at
+    attr_reader :failed_records
+    attr_reader :id
+    attr_reader :job_id
+    attr_reader :processed_records
+    attr_reader :state
+    attr_reader :total_processing_time
+    attr_reader :data
+
+    @@max_records = 10000
+
+    def self.max_records
+      @@max_records
+    end
+    
+    #This is not the way this is going to run
+    def initialize(data, job_id, client = nil)
+      update(data)
+      @client = client
+    end
+
+    def update(data)
+      @data = data
+
+      @id = data['id']
+      @job_id = data['jobId']
+      @state = data['state']
+      @created_at = DateTime.parse(data['createdDate'])
+      @completed_at = DateTime.parse(data['systemModstamp'])
+      @processed_records = data['numberRecordsProcessed'].to_i
+      @failed_records = data['numberRecordsFailed'].to_i
+      @total_processing_time = data['totalProcessingTime'].to_i
+      @api_active_processing_time = data['apiActiveProcessingTime'].to_i
+      @apex_processing_time = data['apex_processing_time'].to_i
+    end
+
+    ### State Information ###
+    def in_progress?
+      state? 'InProgress'
+    end
+    
+    def queued?
+      state? 'Queued'
+    end
+    
+    def completed?
+      state? 'Completed'
+    end
+    
+    def failed?
+      state? 'Failed'
+    end
+    
+    def finished?
+      completed? or finished?
+    end
+
+    def state?(value)
+      self.state.present? && self.state.casecmp(value) == 0
+    end
+
+    def errors?
+      @number_records_failed > 0
+    end
+
+    def result
+      @client.get_batch_result(@job_id, @batch_id)
+    end
+
+    def request
+      @client.get_batch_request(@job_id, @batch_id)
+    end
+
+    def refresh
+      xml_data = http_get_xml("job/#{jobId}/batch/#{batchId}")
+      update(xml_data)
+    end
+  end
+end

data/lib/salesforce_bulk/batch_result.rb

@@ -0,0 +1,39 @@
+module SalesforceBulk
+  class BatchResult
+    
+    # A boolean indicating if record was created. If updated value is false.
+    attr_reader :created
+    
+    # The error message.
+    attr_reader :error
+    
+    # The record's unique id.
+    attr_reader :id
+    
+    # If record was created successfully. If false then an error message is provided. 
+    attr_reader :success
+    
+    def initialize(id, success, created, error)
+      @id = id
+      @success = success
+      @created = created
+      @error = error
+    end
+    
+    def error?
+      error.present?
+    end
+    
+    def created?
+      created
+    end
+    
+    def successful?
+      success
+    end
+    
+    def updated?
+      !created && success
+    end
+  end
+end

data/lib/salesforce_bulk/batch_result_collection.rb

@@ -0,0 +1,29 @@
+module SalesforceBulk
+  class BatchResultCollection < Array
+    
+    attr_reader :batch_id
+    attr_reader :job_id
+    
+    def initialize(job_id, batch_id)
+      @job_id = job_id
+      @batch_id = batch_id
+    end
+    
+    def any_failures?
+      self.any? { |result| result.error.length > 0 }
+    end
+    
+    def failed
+      self.select { |result| result.error.length > 0 }
+    end
+    
+    def completed
+      self.select { |result| result.success }
+    end
+    
+    def created
+      self.select { |result| result.success && result.created }
+    end
+    
+  end
+end

data/lib/salesforce_bulk/client.rb

@@ -0,0 +1,209 @@
+module SalesforceBulk
+  # Interface for operating the Salesforce Bulk REST API
+  class Client
+    # The HTTP connection we will be using to connect to Salesforce.com
+    attr_accessor :connection
+
+    def initialize(options={})
+      @connection = Connection.new(options)
+    end
+
+    def connected?
+      @connection.connected?
+    end
+
+    def disconnect
+      @connection.disconnect
+    end
+
+    def connect
+      @connection.connect
+    end
+
+    def create_job operation, sobject, options = {}
+      Job.new add_job(operation, sobject, options), self
+    end
+
+    def add_job operation, sobject, options={}
+      operation = operation.to_sym.downcase
+      
+      raise ArgumentError.new("Invalid operation: #{operation}") unless Job.valid_operation?(operation)
+      
+      options.assert_valid_keys(:external_id_field_name, :concurrency_mode)
+      
+      if options[:concurrency_mode]
+        concurrency_mode = options[:concurrency_mode].capitalize
+        raise ArgumentError.new("Invalid concurrency mode: #{concurrency_mode}") unless Job.valid_concurrency_mode?(concurrency_mode)
+      end
+      
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += "  <operation>#{operation}</operation>"
+      xml += "  <object>#{sobject}</object>"
+      xml += "  <externalIdFieldName>#{options[:external_id_field_name]}</externalIdFieldName>" if options[:external_id_field_name]
+      xml += "  <concurrencyMode>#{options[:concurrency_mode]}</concurrencyMode>" if options[:concurrency_mode]
+      xml += "  <contentType>CSV</contentType>"
+      xml += "</jobInfo>"
+      
+      @connection.http_post_xml("job", xml)
+    end
+
+    def abort_job job_id
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += '  <state>Aborted</state>'
+      xml += '</jobInfo>'
+      
+      @connection.http_post_xml("job/#{job_id}", xml)
+    end
+
+    def close_job job_id
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += '  <state>Closed</state>'
+      xml += '</jobInfo>'
+      
+      @connection.http_post_xml("job/#{job_id}", xml)
+    end
+
+    def get_job job_id
+      @connection.http_get_xml("job/#{job_id}")
+    end
+
+    def get_batch job_id, batch_id
+      @connection.http_get_xml("job/#{jobId}/batch/#{batchId}")
+    end
+
+    def find_job job_id
+      Job.new get_job(job_id)
+    end
+
+    def find_batch job_id, batch_id
+      Batch.new get_batch(job_id, batch_id)
+    end
+
+    def create_batch job_id, data
+      Batch.new add_batch(job_id, data)
+    end
+
+    def add_batch job_id, data
+      body = data
+      
+      if data.is_a?(Array)
+        raise ArgumentError, "Batch data set exceeds #{Batch.max_records} record limit by #{data.length - Batch.max_records}" if data.length > Batch.max_records
+        raise ArgumentError, "Batch data set is empty" if data.length < 1
+        
+        keys = data.first.keys
+        body = keys.to_csv
+        
+        data.each do |item|
+          item_values = keys.map { |key| item[key] }
+          body += item_values.to_csv
+        end
+      end
+      
+      # Despite the content for a query operation batch being plain text we 
+      # still have to specify CSV content type per API docs.
+      @connection.http_post_xml("job/#{job_id}/batch", body, "Content-Type" => "text/csv; charset=UTF-8")
+    end
+    
+    def get_batch_list(job_id)
+      result = @connection.http_get_xml("job/#{job_id}/batch")
+      
+      if result['batchInfo'].is_a?(Array)
+        result['batchInfo'].collect { |info| Batch.new(info) }
+      else
+        [Batch.new(result['batchInfo'])]
+      end
+    end
+
+    def get_batch_request(job_id, batch_id)
+      response = http_get("job/#{job_id}/batch/#{batch_id}/request")
+        
+      CSV.parse(response.body, :headers => true) do |row|
+        result << BatchResult.new(row[0], row[1].to_b, row[2].to_b, row[3])
+      end
+    end
+    
+    def get_batch_result(job_id, batch_id)
+      response = http_get("job/#{job_id}/batch/#{batch_id}/result")
+      
+      #Query Result
+      if response.body =~ /<.*?>/m
+        result = XmlSimple.xml_in(response.body)
+        
+        if result['result'].present?
+          results = get_query_result(job_id, batch_id, result['result'].first)
+          
+          collection = QueryResultCollection.new(self, job_id, batch_id, result['result'].first, result['result'])
+          collection.replace(results)
+        end
+
+      #Batch Result
+      else
+        result = BatchResultCollection.new(job_id, batch_id)
+        
+        CSV.parse(response.body, :headers => true) do |row|
+          result << BatchResult.new(row[0], row[1].to_b, row[2].to_b, row[3])
+        end
+        
+        result
+      end
+    end
+    
+    def get_query_result(job_id, batch_id, result_id)
+      headers = {"Content-Type" => "text/csv; charset=UTF-8"}
+      response = http_get("job/#{job_id}/batch/#{batch_id}/result/#{result_id}", headers)
+      
+      lines = response.body.lines.to_a
+      headers = CSV.parse_line(lines.shift).collect { |header| header.to_sym }
+      
+      result = []
+      
+      #CSV.parse(lines.join, :headers => headers, :converters => [:all, lambda{|s| s.to_b if s.kind_of? String }]) do |row|
+      CSV.parse(lines.join, :headers => headers) do |row|
+        result << Hash[row.headers.zip(row.fields)]
+      end
+      
+      result
+    end
+
+    ## Operations
+    def delete(sobject, data)
+      perform_operation(:delete, sobject, data)
+    end
+    
+    def insert(sobject, data)
+      perform_operation(:insert, sobject, data)
+    end
+    
+    def query(sobject, data)
+      perform_operation(:query, sobject, data)
+    end
+    
+    def update(sobject, data)
+      perform_operation(:update, sobject, data)
+    end
+    
+    def upsert(sobject, external_id, data)
+      perform_operation(:upsert, sobject, data, external_id)
+    end
+    
+    def perform_operation(operation, sobject, data, external_id = nil, batch_size = nil)
+      job = create_job(operation, sobject, :external_id_field_name => external_id)
+      
+      data.each_slice(batch_size || Batch.max_records) do |records|
+        job.add_batch(records)
+      end
+
+      job.close
+      
+      until job.finished?
+        job.refresh
+        sleep 2
+      end
+      
+      job.get_results
+    end
+  end
+end

data/lib/salesforce_bulk/connection.rb

@@ -0,0 +1,155 @@
+module SalesforceBulk
+  class Connection
+    # If true, print API debugging information to stdout. Defaults to false.
+    attr_accessor :debugging
+
+    # The host to use for authentication. Defaults to login.salesforce.com.
+    attr_reader :host
+    
+    # The instance host to use for API calls. Determined from login response.
+    attr_reader :instance_host
+    
+    # The Salesforce password
+    attr_reader :password
+    
+    # The Salesforce security token
+    attr_reader :token
+    
+    # The Salesforce username
+    attr_reader :username
+    
+    # The API version the client is using. Defaults to 24.0.
+    attr_reader :version
+
+
+    # Defaults
+    @@host = 'login.salesforce.com'
+    @@version = 24.0
+    @@debugging = false
+    @@api_path_prefix = "/services/async/"
+
+    def initialize options
+      if options.is_a?(String)
+        options = YAML.load_file(options)
+        options.symbolize_keys!
+      end
+      
+      options.assert_valid_keys(:username, :password, :token, :debugging, :host, :version)
+      
+      @username = options[:username]
+      @password = "#{options[:password]}#{options[:token]}"
+      @token    = options[:token]
+      @host     = options[:host] || @@host
+      @version  = options[:version] || @@version
+      @debugging = options[:debugging] || @@debugging
+    end
+
+    def connect
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<env:Envelope xmlns:xsd="http://www.w3.org/2001/XMLSchema"'
+      xml += ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
+      xml += ' xmlns:env="http://schemas.xmlsoap.org/soap/envelope/">'
+      xml += '  <env:Body>'
+      xml += '    <n1:login xmlns:n1="urn:partner.soap.sforce.com">'
+      xml += "      <n1:username>#{@username}</n1:username>"
+      xml += "      <n1:password>#{@password}</n1:password>"
+      xml += "    </n1:login>"
+      xml += "  </env:Body>"
+      xml += "</env:Envelope>"
+      
+      data = http_post_xml("/services/Soap/u/#{@version}", xml, 'Content-Type' => 'text/xml', 'SOAPAction' => 'login')
+      result = data['Body']['loginResponse']['result']
+      
+      @session_id = result['sessionId']
+      @server_url = result['serverUrl']
+      @instance_id = instance_id(@server_url)
+      @instance_host = "#{@instance_id}.salesforce.com"
+      
+      @api_path_prefix = "#{@@api_path_prefix}/#{@version}/"
+
+      result
+    end
+
+    def disconnect
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<env:Envelope xmlns:xsd="http://www.w3.org/2001/XMLSchema"'
+      xml += ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
+      xml += ' xmlns:env="http://schemas.xmlsoap.org/soap/envelope/">'
+      xml += '  <env:Body>'
+      xml += '    <n1:logout xmlns:n1="urn:partner.soap.sforce.com" />'
+      xml += '  </env:Body>'
+      xml += '</env:Envelope>'
+      
+      result = http_post_xml("/services/Soap/u/#{@version}", xml, 'Content-Type' => 'text/xml', 'SOAPAction' => 'logout')
+
+      @session_id = nil
+      @server_url = nil
+      @instance_id = nil
+      @instance_host = nil
+      @api_path_prefix = nil
+
+      result
+    end
+
+    def connected?
+      !!@session_id
+    end
+
+    def http_post(path, body, headers={})
+      headers = {'Content-Type' => 'application/xml'}.merge(headers)
+      
+      #Are we connected?
+      if connected?
+        headers['X-SFDC-Session'] = @session_id
+        host = @instance_host
+        path = "#{@api_path_prefix}#{path}"
+      else
+        host = @host
+      end
+
+      response = https_request(host).post(path, body, headers)
+      
+      if response.is_a?(Net::HTTPSuccess)
+        response
+      else
+        raise SalesforceError.new(response)
+      end
+    end
+    
+    def http_get(path, headers={})
+      path = "#{@api_path_prefix}#{path}"
+      
+      headers = {'Content-Type' => 'application/xml'}.merge(headers)
+      
+      headers['X-SFDC-Session'] = @session_id if @session_id
+      
+      response = https_request(@instance_host).get(path, headers)
+      
+      if response.is_a?(Net::HTTPSuccess)
+        response
+      else
+        raise SalesforceError.new(response)
+      end
+    end
+
+    def http_post_xml(path, body, headers = {})
+      XmlSimple.xml_in(http_post(path, body, headers).body, :ForceArray => false)
+    end
+
+    def http_get_xml(path, headers = {})
+      XmlSimple.xml_in(http_get(path, headers).body, :ForceArray => false)
+    end
+    
+    def https_request(host)
+      req = Net::HTTP.new(host, 443)
+      req.use_ssl = true
+      req.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      req
+    end
+
+  private
+    def instance_id(url)
+      url.match(/:\/\/([a-zA-Z0-9-]{2,}).salesforce/)[1]
+    end
+  end
+end