salesforce_bulk2 0.5.1 → 0.5.2

data/README.md

@@ -76,11 +76,11 @@ When adding a job you can specify the following operations for the first argumen
 
 When using the :upsert operation you must specify an external ID field name:
 
-    job = client.add_job(:upsert, :MyObject__c, :external_id_field_name => :MyId__c)
+    job = client.add_job(:upsert, :MyObject__c, :external_id => :MyId__c)
 
 For any operation you should be able to specify a concurrency mode. The default is Parallel. The other choice is Serial.
 
-    job = client.add_job(:upsert, :MyObject__c, :concurrency_mode => :Serial, :external_id_field_name => :MyId__c)
+    job = client.add_job(:upsert, :MyObject__c, :concurrency_mode => :Serial, :external_id => :MyId__c)
 
 ### Retrieving Info for a Job
 

data/lib/salesforce_bulk2/batch.rb

@@ -0,0 +1,186 @@
+module SalesforceBulk2
+  class Batch
+    attr_accessor :session_id
+    attr_accessor :batch_size
+
+    attr_reader :apex_processing_time
+    attr_reader :api_active_processing_time
+    attr_reader :completed_at
+    attr_reader :created_at
+    attr_reader :failed_records
+    attr_reader :id
+    attr_reader :job_id
+    attr_reader :processed_records
+    attr_reader :state
+    attr_reader :total_processing_time
+    attr_reader :data
+
+    @@batch_size = 10000
+
+    def self.batch_size
+      @@batch_size
+    end
+
+    def self.batch_size= batch_size
+      @@batch_size = batch_size
+    end
+
+    def batch_size
+      @batch_size || @@batch_size
+    end
+
+
+    def initialize job, data = nil
+      @job = job
+      @job_id = job.id
+      @client = job.client
+
+      update(data) if data
+    end
+
+    def self.create job, data
+      batch = Batch.new(job)
+      batch.execute(data)
+    end
+
+    def self.find job, batch_id
+      batch = Batch.new(job)
+      batch.id = batch_id
+      batch.refresh
+      batch
+    end
+
+    def self.find job, batch_id
+      @job = job
+      @client = job.client
+    end
+
+    def execute data
+      raise Exception.new "Already executed" if @data
+
+      @data = data
+      body = data
+      
+      if data.is_a?(Array)
+        raise ArgumentError, "Batch data set exceeds #{@@batch_size} record limit by #{data.length - @@batch_size}" if data.length > @@batch_size
+        raise ArgumentError, "Batch data set is empty" if data.length < 1
+        
+        keys = data.first.keys
+        body = keys.to_csv
+        
+        data.each do |item|
+          item_values = keys.map { |key| item[key] }
+          body += item_values.to_csv
+        end
+      end
+
+      # Despite the content for a query operation batch being plain text we 
+      # still have to specify CSV content type per API docs.
+      @client.http_post_xml("job/#{@job_id}/batch", body, "Content-Type" => "text/csv; charset=UTF-8")
+    end
+
+    def get_request
+      response = @client.http_get("job/#{@job_id}/batch/#{@id}/request")
+
+      CSV.parse(response.body, :headers => true)
+    end
+    
+    def get_result
+      response = @client.http_get("job/#{@job_id}/batch/#{@id}/result")
+      
+      #Query Result
+      if response.body =~ /<.*?>/m
+        result = XmlSimple.xml_in(response.body)
+        
+        if result['result'].present?
+          results = get_query_result(@id, result['result'].first)
+          
+          collection = QueryResultCollection.new(self, @id, result['result'].first, result['result'])
+          collection.replace(results)
+        end
+
+      #Batch Result
+      else
+        results = BatchResultCollection.new
+        requests = get_request
+        
+        i = 0
+        CSV.parse(response.body, :headers => true) do |row|
+          result = BatchResult.new(row[0], row[1].to_b, row[2].to_b, row[3])
+          result['request'] = requests[i]
+          results << result
+
+          i += 1
+        end
+        
+        return results
+      end
+    end
+    
+    def get_query_result(batch_id, result_id)
+      headers = {"Content-Type" => "text/csv; charset=UTF-8"}
+      response = @client.http_get("job/#{@job_id}/batch/#{batch_id}/result/#{result_id}", headers)
+      
+      lines = response.body.lines.to_a
+      headers = CSV.parse_line(lines.shift).collect { |header| header.to_sym }
+      
+      result = []
+      
+      #CSV.parse(lines.join, :headers => headers, :converters => [:all, lambda{|s| s.to_b if s.kind_of? String }]) do |row|
+      CSV.parse(lines.join, :headers => headers) do |row|
+        result << Hash[row.headers.zip(row.fields)]
+      end
+      
+      result
+    end
+
+    def update(data)
+      @data = data
+
+      @id = data['id']
+      @job_id = data['jobId']
+      @state = data['state']
+      @created_at = DateTime.parse(data['createdDate']) rescue nil
+      @completed_at = DateTime.parse(data['systemModstamp']) rescue nil
+      @processed_records = data['numberRecordsProcessed'].to_i
+      @failed_records = data['numberRecordsFailed'].to_i
+      @total_processing_time = data['totalProcessingTime'].to_i
+      @api_active_processing_time = data['apiActiveProcessingTime'].to_i
+      @apex_processing_time = data['apex_processing_time'].to_i
+    end
+
+    ### State Information ###
+    def in_progress?
+      state? 'InProgress'
+    end
+    
+    def queued?
+      state? 'Queued'
+    end
+    
+    def completed?
+      state? 'Completed'
+    end
+    
+    def failed?
+      state? 'Failed'
+    end
+    
+    def finished?
+      completed? or finished?
+    end
+
+    def state?(value)
+      self.state.present? && self.state.casecmp(value) == 0
+    end
+
+    def errors?
+      @number_records_failed > 0
+    end
+
+    def refresh
+      xml_data = @client.http_get_xml("job/#{@job_id}/batch/#{@batch_id}")
+      update(xml_data)
+    end
+  end
+end

data/lib/salesforce_bulk2/batch_result.rb

@@ -0,0 +1,42 @@
+module SalesforceBulk2
+  class BatchResult < Hash
+    def initialize(id, success, created, error)
+      self['id'] = id
+      self['success'] = success
+      self['created'] = created
+      self['error'] = error
+    end
+    
+    def error?
+      error.present?
+    end
+    
+    def created?
+      created
+    end
+    
+    def successful?
+      success
+    end
+    
+    def updated?
+      !created && success
+    end
+
+    def method_missing method, *args, &block
+      if has_key? method.to_s
+        self[method.to_s] 
+      else
+        super method, *args, &block
+      end
+    end
+
+    def respond_to? method
+      if has_key? method.to_sym
+        return true
+      else
+        super
+      end
+    end
+  end
+end

data/lib/salesforce_bulk2/batch_result_collection.rb

@@ -0,0 +1,20 @@
+module SalesforceBulk2
+  class BatchResultCollection < Array
+    
+    def any_failures?
+      self.any? { |result| result.error? }
+    end
+    
+    def failed
+      self.select { |result| result.error? }
+    end
+    
+    def completed
+      self.select { |result| result.successful? }
+    end
+    
+    def created
+      self.select { |result| result.successful? && result.created? }
+    end
+  end
+end

data/lib/{salesforce_bulk/connection.rb → salesforce_bulk2/client.rb}

@@ -1,5 +1,6 @@
-module SalesforceBulk
-  class Connection
+module SalesforceBulk2
+  # Interface for operating the Salesforce Bulk REST API
+  class Client
     # If true, print API debugging information to stdout. Defaults to false.
     attr_accessor :debugging
 
@@ -18,9 +19,12 @@ module SalesforceBulk
     # The Salesforce username
     attr_reader :username
     
-    # The API version the client is using. Defaults to 24.0.
+    # The API version the client is using
     attr_reader :version
 
+    #List of jobs associatd with this client
+    attr_accessor :jobs
+
 
     # Defaults
     @@host = 'login.salesforce.com'
@@ -38,10 +42,12 @@ module SalesforceBulk
       
       @username = options[:username]
       @password = "#{options[:password]}#{options[:token]}"
-      @token    = options[:token]
+      @token    = options[:token] || ''
       @host     = options[:host] || @@host
       @version  = options[:version] || @@version
       @debugging = options[:debugging] || @@debugging
+
+      @jobs = []
     end
 
     def connect options = {}
@@ -153,9 +159,66 @@ module SalesforceBulk
       req
     end
 
-  private
     def instance_id(url)
       url.match(/:\/\/([a-zA-Z0-9-]{2,}).salesforce/)[1]
     end
+
+
+    #Job related
+    def new_job options = {}
+      job = Job.create(self, options)
+      @jobs << job
+      job
+    end
+
+    def find_job id
+      job = Job.find(connection, id)
+      @jobs << job
+      job
+    end
+
+    def close_jobs
+      @jobs.map(&:close)
+    end
+
+    def abort_jobs
+      @jobs.map(&:abort)
+    end
+
+
+    ## Operations
+    def delete(sobject, data, batch_size = nil)
+      perform_operation(:delete, sobject, data, :batch_size => nil)
+    end
+    
+    def insert(sobject, data, batch_size = nil)
+      perform_operation(:insert, sobject, data, :batch_size => nil)
+    end
+    
+    def query(sobject, data, batch_size = nil)
+      perform_operation(:query, sobject, data, :batch_size => nil)
+    end
+    
+    def update(sobject, data, batch_size = nil)
+      perform_operation(:update, sobject, data, :batch_size => nil)
+    end
+    
+    def upsert(sobject, data, external_id, batch_size = nil)
+      perform_operation(:upsert, sobject, data, :external_id => external_id, :batch_size => batch_size)
+    end
+    
+    def perform_operation(operation, sobject, data, options = {})
+      job = new_job(operation: operation, object: sobject, :external_id => options[:external_id])
+
+      job.add_data(data, options[:batch_size])
+      job.close
+      
+      until job.finished?
+        job.refresh
+        sleep 2
+      end
+      
+      return job.get_results
+    end
   end
-end
+end

data/lib/salesforce_bulk2/job.rb

@@ -0,0 +1,193 @@
+module SalesforceBulk2
+  class Job
+    attr_reader :client
+
+    attr_reader :concurrency_mode
+    attr_reader :external_id
+    attr_reader :data
+    attr_reader :xml_data
+
+    @@fields = [:id, :operation, :object, :createdById, :state, :createdDate, 
+      :systemModstamp, :externalIdFieldName, :concurrencyMode, :contentType, 
+      :numberBatchesQueued, :numberBatchesInProgress, :numberBatchesCompleted, 
+      :numberBatchesFailed, :totalBatches, :retries, :numberRecordsProcessed, 
+      :numberRecordsFailed, :totalProcessingTime, :apiActiveProcessingTime, 
+      :apexProcessingTime, :apiVersion]
+    
+    @@valid_operations = [:delete, :insert, :update, :upsert, :query]
+    @@valid_concurrency_modes = ['Parallel', 'Serial']
+
+    @@fields.each do |field|
+      attr_reader field.to_s.underscore.to_sym
+    end
+
+    def self.valid_operation? operation
+      @@valid_operations.include?(operation)
+    end
+
+    def self.valid_concurrency_mode? mode
+      @@valid_concurrency_modes.include?(concurrency_mode)
+    end
+
+    def self.create client, options = {}
+      job = Job.new(client)
+
+      options.assert_valid_keys(:external_id, :concurrency_mode, :object, :operation)
+
+      operation = options[:operation].to_sym.downcase
+      raise ArgumentError.new("Invalid operation: #{operation}") unless Job.valid_operation?(operation)
+
+      external_id = options[:external_id]
+      concurrency_mode = options[:concurrency_mode]
+      object = options[:object]
+      
+      if concurrency_mode
+        concurrency_mode = concurrency_mode.capitalize
+        raise ArgumentError.new("Invalid concurrency mode: #{concurrency_mode}") unless Job.valid_concurrency_mode?(concurrency_mode)
+      end
+      
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += "  <operation>#{operation}</operation>"
+      xml += "  <object>#{object}</object>" if object
+      xml += "  <externalIdFieldName>#{external_id}</externalIdFieldName>" if external_id
+      xml += "  <concurrencyMode>#{concurrency_mode}</concurrencyMode>" if concurrency_mode
+      xml += "  <contentType>CSV</contentType>"
+      xml += "</jobInfo>"
+
+      job.update(client.http_post_xml("job", xml))
+      job
+    end
+
+    def self.find client, id
+      Job.new(client)
+      job.id = id
+      job.refresh
+      job
+    end
+
+    def refresh
+      xml_data = @client.http_get_xml("job/#{@id}")
+      update(xml_data)
+    end
+
+    def initialize client
+      @client = client
+    end
+
+    def new_batch data
+      Batch.create(self, data)
+    end
+
+    def get_batches
+      result = @client.http_get_xml("job/#{@id}/batch")
+
+      if result['batchInfo'].is_a?(Array)
+        result['batchInfo'].collect { |info| Batch.new(self, info) }
+      elsif result['batchInfo']
+        [Batch.new(self, result['batchInfo'])]
+      else
+        []
+      end
+    end
+
+    def abort
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += '  <state>Aborted</state>'
+      xml += '</jobInfo>'
+      
+      @client.http_post_xml("job/#{@id}", xml)
+    end
+
+    def close
+      xml  = '<?xml version="1.0" encoding="utf-8"?>'
+      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
+      xml += '  <state>Closed</state>'
+      xml += '</jobInfo>'
+      
+      @client.http_post_xml("job/#{@id}", xml)
+    end
+
+    def update xml_data
+      #Assign object
+      @xml_data = xml_data
+
+      #Mass assign the defaults
+      @@fields.each do |field|
+        instance_variable_set(:"@#{field}", xml_data[field.to_s])
+      end
+
+      #Special cases and data formats
+      @created_date = DateTime.parse(xml_data['createdDate'])
+      @system_modstamp = DateTime.parse(xml_data['systemModstamp'])
+
+      @retries = xml_data['retries'].to_i
+      @api_version = xml_data['apiVersion'].to_i
+      @number_batches_queued = xml_data['numberBatchesQueued'].to_i
+      @number_batches_in_progress = xml_data['numberBatchesInProgress'].to_i
+      @number_batches_completed = xml_data['numberBatchesCompleted'].to_i
+      @number_batches_failed = xml_data['numberBatchesFailed'].to_i
+      @total_batches = xml_data['totalBatches'].to_i
+      @number_records_processed = xml_data['numberRecordsProcessed'].to_i
+      @number_records_failed = xml_data['numberRecordsFailed'].to_i
+      @total_processing_time = xml_data['totalProcessingTime'].to_i
+      @api_active_processing_time = xml_data['apiActiveProcessingTime'].to_i
+      @apex_processing_time = xml_data['apexProcessingTime'].to_i
+    end
+
+    def add_data data, batch_size = nil
+      data.each_slice(batch_size || Batch.batch_size) do |records|
+        new_batch(records)
+      end
+    end
+
+    def get_results
+      results = BatchResultCollection.new
+
+      get_batches.each { |batch| results << batch.get_result }
+
+      results.flatten
+    end
+
+    # def get_requests
+    #   results = BatchResultCollection.new
+
+    #   get_batches.each { |batch| results << batch.get_request }
+
+    #   results.flatten
+    # end
+
+    #Statuses
+    def batches_finished?
+      (@number_batches_queued == 0) and
+      (@number_batches_in_progress == 0)
+    end
+    
+    def finished?
+      failed?  or 
+      aborted? or 
+      (closed? and batches_finished?)
+    end
+
+    def failed?
+      state? 'Failed'
+    end
+
+    def aborted?
+      state? 'Aborted'
+    end
+    
+    def closed?
+      state? 'Closed'
+    end
+    
+    def open?
+      state? 'Open'
+    end
+    
+    def state?(value)
+      @state.present? && @state.casecmp(value) == 0
+    end
+  end
+end

data/lib/{salesforce_bulk → salesforce_bulk2}/query_result_collection.rb

@@ -1,4 +1,4 @@
-module SalesforceBulk
+module SalesforceBulk2
   class QueryResultCollection < Array
     
     attr_reader :client

data/lib/{salesforce_bulk → salesforce_bulk2}/salesforce_error.rb

@@ -1,4 +1,4 @@
-module SalesforceBulk
+module SalesforceBulk2
   # An exception raised when any non successful request is made through the Salesforce Bulk API.
   class SalesforceError < StandardError
     # The Net::HTTPResponse instance from the API call.

data/lib/salesforce_bulk2/version.rb

@@ -0,0 +1,3 @@
+module SalesforceBulk2
+  VERSION = "0.5.2"
+end

data/lib/salesforce_bulk2.rb

@@ -0,0 +1,19 @@
+require 'net/https'
+require 'xmlsimple'
+require 'csv'
+require 'active_support'
+require 'active_support/inflector'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/hash/keys'
+require 'salesforce_bulk2/version'
+require 'salesforce_bulk2/core_extensions/string'
+require 'salesforce_bulk2/salesforce_error'
+require 'salesforce_bulk2/client'
+require 'salesforce_bulk2/job'
+require 'salesforce_bulk2/batch'
+require 'salesforce_bulk2/batch_result'
+require 'salesforce_bulk2/batch_result_collection'
+require 'salesforce_bulk2/query_result_collection'
+
+module SalesforceBulk2
+end

data/salesforce_bulk2.gemspec

@@ -1,10 +1,10 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
-require "salesforce_bulk/version"
+require "salesforce_bulk2/version"
 
 Gem::Specification.new do |s|
   s.name        = "salesforce_bulk2"
-  s.version     = SalesforceBulk::VERSION
+  s.version     = SalesforceBulk2::VERSION
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Adam Kerr", "Jorge Valdivia", "Javier Julio"]
   s.email       = ["ajrkerr@gmail.com", "jorge@valdivia.me", "jjfutbol@gmail.com"]

data/test/lib/test_job.rb

@@ -27,7 +27,7 @@ class TestJob < Test::Unit::TestCase
     assert_equal job.created_at, DateTime.parse('2012-05-30T04:08:30.000Z')
     assert_equal job.completed_at, DateTime.parse('2012-05-30T04:08:30.000Z')
     assert_equal job.state, 'Open'
-    assert_equal job.external_id_field_name, 'Id__c'
+    assert_equal job.external_id, 'Id__c'
     assert_equal job.concurrency_mode, 'Parallel'
     assert_equal job.content_type, 'CSV'
     assert_equal job.queued_batches, 0
@@ -87,7 +87,7 @@ class TestJob < Test::Unit::TestCase
       .with(:body => request, :headers => @headers)
       .to_return(:body => response, :status => 200)
     
-    job = @client.add_job(:upsert, :VideoEvent__c, :external_id_field_name => :Id__c)
+    job = @client.add_job(:upsert, :VideoEvent__c, :external_id => :Id__c)
     
     assert_requested :post, "#{api_url(@client)}job", :body => request, :headers => @headers, :times => 1
     
@@ -98,7 +98,7 @@ class TestJob < Test::Unit::TestCase
     assert_equal job.created_at, DateTime.parse('2012-05-29T21:50:47.000Z')
     assert_equal job.completed_at, DateTime.parse('2012-05-29T21:50:47.000Z')
     assert_equal job.state, 'Open'
-    assert_equal job.external_id_field_name, 'Id__c'
+    assert_equal job.external_id, 'Id__c'
     assert_equal job.concurrency_mode, 'Parallel'
     assert_equal job.content_type, 'CSV'
     assert_equal job.queued_batches, 0
@@ -153,7 +153,7 @@ class TestJob < Test::Unit::TestCase
     assert_equal job.created_at, DateTime.parse('2012-05-29T23:51:53.000Z')
     assert_equal job.completed_at, DateTime.parse('2012-05-29T23:51:53.000Z')
     assert_equal job.state, 'Closed'
-    assert_equal job.external_id_field_name, 'Id__c'
+    assert_equal job.external_id, 'Id__c'
     assert_equal job.concurrency_mode, 'Parallel'
     assert_equal job.content_type, 'CSV'
     assert_equal job.queued_batches, 0
@@ -190,7 +190,7 @@ class TestJob < Test::Unit::TestCase
     assert_equal job.created_at, DateTime.parse('2012-05-30T00:16:04.000Z')
     assert_equal job.completed_at, DateTime.parse('2012-05-30T00:16:04.000Z')
     assert_equal job.state, 'Aborted'
-    assert_equal job.external_id_field_name, 'Id__c'
+    assert_equal job.external_id, 'Id__c'
     assert_equal job.concurrency_mode, 'Parallel'
     assert_equal job.content_type, 'CSV'
     assert_equal job.queued_batches, 0
@@ -226,7 +226,7 @@ class TestJob < Test::Unit::TestCase
     assert_equal job.created_at, DateTime.parse('2012-05-30T04:08:30.000Z')
     assert_equal job.completed_at, DateTime.parse('2012-05-30T04:08:30.000Z')
     assert_equal job.state, 'Open'
-    assert_equal job.external_id_field_name, 'Id__c'
+    assert_equal job.external_id, 'Id__c'
     assert_equal job.concurrency_mode, 'Parallel'
     assert_equal job.content_type, 'CSV'
     assert_equal job.queued_batches, 0
@@ -249,7 +249,7 @@ class TestJob < Test::Unit::TestCase
     stub_request(:post, "#{api_url(@client)}job").to_return(:body => response, :status => 500)
     
     assert_raise SalesforceBulk::SalesforceError do
-      job = @client.add_job(:upsert, :SomeNonExistingObject__c, :external_id_field_name => :Id__c)
+      job = @client.add_job(:upsert, :SomeNonExistingObject__c, :external_id => :Id__c)
     end
   end
   

data/test/lib/test_simple_api.rb

@@ -19,7 +19,7 @@ class TestSimpleApi < Test::Unit::TestCase
   test "delete" do
     data = [{:Id => '123123'}, {:Id => '234234'}]
     
-    @client.expects(:add_job).once.with(:delete, :VideoEvent__c, :external_id_field_name => nil).returns(@job)
+    @client.expects(:add_job).once.with(:delete, :VideoEvent__c, :external_id => nil).returns(@job)
     @client.expects(:add_batch).once.with(@job.id, data).returns(@batch)
     @client.expects(:close_job).once.with(@job.id).returns(@job)
     @client.expects(:batch_info).at_least_once.returns(@batch)
@@ -31,7 +31,7 @@ class TestSimpleApi < Test::Unit::TestCase
   test "insert" do
     data = [{:Title__c => 'Test Title'}, {:Title__c => 'Test Title'}]
     
-    @client.expects(:add_job).once.with(:insert, :VideoEvent__c, :external_id_field_name => nil).returns(@job)
+    @client.expects(:add_job).once.with(:insert, :VideoEvent__c, :external_id => nil).returns(@job)
     @client.expects(:add_batch).once.with(@job.id, data).returns(@batch)
     @client.expects(:close_job).once.with(@job.id).returns(@job)
     @client.expects(:batch_info).at_least_once.returns(@batch)
@@ -43,7 +43,7 @@ class TestSimpleApi < Test::Unit::TestCase
   test "query" do
     data = 'SELECT Id, Name FROM Account'
     
-    @client.expects(:add_job).once.with(:query, :VideoEvent__c, :external_id_field_name => nil).returns(@job)
+    @client.expects(:add_job).once.with(:query, :VideoEvent__c, :external_id => nil).returns(@job)
     @client.expects(:add_batch).once.with(@job.id, data).returns(@batch)
     @client.expects(:close_job).once.with(@job.id).returns(@job)
     @client.expects(:batch_info).at_least_once.returns(@batch)
@@ -55,7 +55,7 @@ class TestSimpleApi < Test::Unit::TestCase
   test "update" do
     data = [{:Id => '123123', :Title__c => 'Test Title'}, {:Id => '234234', :Title__c => 'A Second Title'}]
     
-    @client.expects(:add_job).once.with(:update, :VideoEvent__c, :external_id_field_name => nil).returns(@job)
+    @client.expects(:add_job).once.with(:update, :VideoEvent__c, :external_id => nil).returns(@job)
     @client.expects(:add_batch).once.with(@job.id, data).returns(@batch)
     @client.expects(:close_job).once.with(@job.id).returns(@job)
     @client.expects(:batch_info).at_least_once.returns(@batch)
@@ -67,7 +67,7 @@ class TestSimpleApi < Test::Unit::TestCase
   test "upsert" do
     data = [{:Id__c => '123123', :Title__c => 'Test Title'}, {:Id__c => '234234', :Title__c => 'A Second Title'}]
     
-    @client.expects(:add_job).once.with(:upsert, :VideoEvent__c, :external_id_field_name => :Id__c).returns(@job)
+    @client.expects(:add_job).once.with(:upsert, :VideoEvent__c, :external_id => :Id__c).returns(@job)
     @client.expects(:add_batch).once.with(@job.id, data).returns(@batch)
     @client.expects(:close_job).once.with(@job.id).returns(@job)
     @client.expects(:batch_info).at_least_once.returns(@batch)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_bulk2
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-01 00:00:00.000000000 Z
+date: 2012-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -123,17 +123,16 @@ files:
 - Gemfile
 - README.md
 - Rakefile
-- lib/salesforce_bulk.rb
-- lib/salesforce_bulk/batch.rb
-- lib/salesforce_bulk/batch_result.rb
-- lib/salesforce_bulk/batch_result_collection.rb
-- lib/salesforce_bulk/client.rb
-- lib/salesforce_bulk/connection.rb
-- lib/salesforce_bulk/core_extensions/string.rb
-- lib/salesforce_bulk/job.rb
-- lib/salesforce_bulk/query_result_collection.rb
-- lib/salesforce_bulk/salesforce_error.rb
-- lib/salesforce_bulk/version.rb
+- lib/salesforce_bulk2.rb
+- lib/salesforce_bulk2/batch.rb
+- lib/salesforce_bulk2/batch_result.rb
+- lib/salesforce_bulk2/batch_result_collection.rb
+- lib/salesforce_bulk2/client.rb
+- lib/salesforce_bulk2/core_extensions/string.rb
+- lib/salesforce_bulk2/job.rb
+- lib/salesforce_bulk2/query_result_collection.rb
+- lib/salesforce_bulk2/salesforce_error.rb
+- lib/salesforce_bulk2/version.rb
 - salesforce_bulk2.gemspec
 - test/fixtures/batch_create_request.csv
 - test/fixtures/batch_create_response.xml

data/lib/salesforce_bulk/batch.rb

@@ -1,101 +0,0 @@
-module SalesforceBulk
-  class Batch
-    attr_accessor :session_id
-
-    attr_reader :apex_processing_time
-    attr_reader :api_active_processing_time
-    attr_reader :completed_at
-    attr_reader :created_at
-    attr_reader :failed_records
-    attr_reader :id
-    attr_reader :job_id
-    attr_reader :processed_records
-    attr_reader :state
-    attr_reader :total_processing_time
-    attr_reader :data
-
-    @@batch_size = 10000
-
-    def self.batch_size
-      @@batch_size
-    end
-
-    def self.new_from_xml xml_data, session_id = nil
-      batch = Batch.new
-      batch.update(data)
-      batch.session_id = session_id
-      batch
-    end
-
-    def self.find job_id, batch_id, session_id
-      batch = Batch.new
-      batch.id = batch_id
-      batch.job_id = job_id
-      batch.session_id = session_id
-      batch.refresh
-      batch
-    end
-
-
-    def update(data)
-      @data = data
-
-      @id = data['id']
-      @job_id = data['jobId']
-      @state = data['state']
-      @created_at = DateTime.parse(data['createdDate'])
-      @completed_at = DateTime.parse(data['systemModstamp'])
-      @processed_records = data['numberRecordsProcessed'].to_i
-      @failed_records = data['numberRecordsFailed'].to_i
-      @total_processing_time = data['totalProcessingTime'].to_i
-      @api_active_processing_time = data['apiActiveProcessingTime'].to_i
-      @apex_processing_time = data['apex_processing_time'].to_i
-    end
-
-    def job
-      @job ||= Job.find(@job_id, @session_id) if @session_id
-    end
-
-    ### State Information ###
-    def in_progress?
-      state? 'InProgress'
-    end
-    
-    def queued?
-      state? 'Queued'
-    end
-    
-    def completed?
-      state? 'Completed'
-    end
-    
-    def failed?
-      state? 'Failed'
-    end
-    
-    def finished?
-      completed? or finished?
-    end
-
-    def state?(value)
-      self.state.present? && self.state.casecmp(value) == 0
-    end
-
-    def errors?
-      @number_records_failed > 0
-    end
-
-    def result
-      @client.get_batch_result(@job_id, @batch_id)
-    end
-
-    def request
-      @client.get_batch_request(@job_id, @batch_id)
-    end
-
-    def refresh
-      xml_data = @connection.http_get_xml("job/#{jobId}/batch/#{batchId}")
-      update(xml_data)
-    end
-  end
-end

data/lib/salesforce_bulk/batch_result.rb

@@ -1,39 +0,0 @@
-module SalesforceBulk
-  class BatchResult
-    
-    # A boolean indicating if record was created. If updated value is false.
-    attr_reader :created
-    
-    # The error message.
-    attr_reader :error
-    
-    # The record's unique id.
-    attr_reader :id
-    
-    # If record was created successfully. If false then an error message is provided. 
-    attr_reader :success
-    
-    def initialize(id, success, created, error)
-      @id = id
-      @success = success
-      @created = created
-      @error = error
-    end
-    
-    def error?
-      error.present?
-    end
-    
-    def created?
-      created
-    end
-    
-    def successful?
-      success
-    end
-    
-    def updated?
-      !created && success
-    end
-  end
-end

data/lib/salesforce_bulk/batch_result_collection.rb

@@ -1,29 +0,0 @@
-module SalesforceBulk
-  class BatchResultCollection < Array
-    
-    attr_reader :batch_id
-    attr_reader :job_id
-    
-    def initialize(job_id, batch_id)
-      @job_id = job_id
-      @batch_id = batch_id
-    end
-    
-    def any_failures?
-      self.any? { |result| result.error.length > 0 }
-    end
-    
-    def failed
-      self.select { |result| result.error.length > 0 }
-    end
-    
-    def completed
-      self.select { |result| result.success }
-    end
-    
-    def created
-      self.select { |result| result.success && result.created }
-    end
-    
-  end
-end

data/lib/salesforce_bulk/client.rb

@@ -1,209 +0,0 @@
-module SalesforceBulk
-  # Interface for operating the Salesforce Bulk REST API
-  class Client
-    # The HTTP connection we will be using to connect to Salesforce.com
-    attr_accessor :connection
-
-    def initialize(options={})
-      @connection = Connection.new(options)
-    end
-
-    def connected?
-      @connection.connected?
-    end
-
-    def disconnect
-      @connection.disconnect
-    end
-
-    def connect options = {}
-      @connection.connect(options)
-    end
-
-    def new_job operation, sobject, options = {}
-      Job.new(add_job(operation, sobject, options), self)
-    end
-
-    def add_job operation, sobject, options={}
-      operation = operation.to_sym.downcase
-      
-      raise ArgumentError.new("Invalid operation: #{operation}") unless Job.valid_operation?(operation)
-      
-      options.assert_valid_keys(:external_id_field_name, :concurrency_mode)
-      
-      if options[:concurrency_mode]
-        concurrency_mode = options[:concurrency_mode].capitalize
-        raise ArgumentError.new("Invalid concurrency mode: #{concurrency_mode}") unless Job.valid_concurrency_mode?(concurrency_mode)
-      end
-      
-      xml  = '<?xml version="1.0" encoding="utf-8"?>'
-      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
-      xml += "  <operation>#{operation}</operation>"
-      xml += "  <object>#{sobject}</object>" if sobject
-      xml += "  <externalIdFieldName>#{options[:external_id_field_name]}</externalIdFieldName>" if options[:external_id_field_name]
-      xml += "  <concurrencyMode>#{options[:concurrency_mode]}</concurrencyMode>" if options[:concurrency_mode]
-      xml += "  <contentType>CSV</contentType>"
-      xml += "</jobInfo>"
-      
-      @connection.http_post_xml("job", xml)
-    end
-
-    def abort_job job_id
-      xml  = '<?xml version="1.0" encoding="utf-8"?>'
-      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
-      xml += '  <state>Aborted</state>'
-      xml += '</jobInfo>'
-      
-      @connection.http_post_xml("job/#{job_id}", xml)
-    end
-
-    def close_job job_id
-      xml  = '<?xml version="1.0" encoding="utf-8"?>'
-      xml += '<jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">'
-      xml += '  <state>Closed</state>'
-      xml += '</jobInfo>'
-      
-      @connection.http_post_xml("job/#{job_id}", xml)
-    end
-
-    def get_job_info job_id
-      @connection.http_get_xml("job/#{job_id}")
-    end
-
-    def get_batch_info job_id, batch_id
-      @connection.http_get_xml("job/#{jobId}/batch/#{batchId}")
-    end
-
-    def find_job job_id
-      Job.new get_job(job_id)
-    end
-
-    def find_batch job_id, batch_id
-      Batch.new get_batch(job_id, batch_id)
-    end
-
-    def create_batch job_id, data
-      Batch.new add_batch(job_id, data)
-    end
-
-    def add_batch job_id, data
-      body = data
-      
-      if data.is_a?(Array)
-        raise ArgumentError, "Batch data set exceeds #{Batch.max_records} record limit by #{data.length - Batch.max_records}" if data.length > Batch.max_records
-        raise ArgumentError, "Batch data set is empty" if data.length < 1
-        
-        keys = data.first.keys
-        body = keys.to_csv
-        
-        data.each do |item|
-          item_values = keys.map { |key| item[key] }
-          body += item_values.to_csv
-        end
-      end
-      
-      # Despite the content for a query operation batch being plain text we 
-      # still have to specify CSV content type per API docs.
-      @connection.http_post_xml("job/#{job_id}/batch", body, "Content-Type" => "text/csv; charset=UTF-8")
-    end
-    
-    def get_batch_list(job_id)
-      result = @connection.http_get_xml("job/#{job_id}/batch")
-      
-      if result['batchInfo'].is_a?(Array)
-        result['batchInfo'].collect { |info| Batch.new(info) }
-      else
-        [Batch.new(result['batchInfo'])]
-      end
-    end
-
-    def get_batch_request(job_id, batch_id)
-      response = http_get("job/#{job_id}/batch/#{batch_id}/request")
-        
-      CSV.parse(response.body, :headers => true) do |row|
-        result << BatchResult.new(row[0], row[1].to_b, row[2].to_b, row[3])
-      end
-    end
-    
-    def get_batch_result(job_id, batch_id)
-      response = http_get("job/#{job_id}/batch/#{batch_id}/result")
-      
-      #Query Result
-      if response.body =~ /<.*?>/m
-        result = XmlSimple.xml_in(response.body)
-        
-        if result['result'].present?
-          results = get_query_result(job_id, batch_id, result['result'].first)
-          
-          collection = QueryResultCollection.new(self, job_id, batch_id, result['result'].first, result['result'])
-          collection.replace(results)
-        end
-
-      #Batch Result
-      else
-        result = BatchResultCollection.new(job_id, batch_id)
-        
-        CSV.parse(response.body, :headers => true) do |row|
-          result << BatchResult.new(row[0], row[1].to_b, row[2].to_b, row[3])
-        end
-        
-        result
-      end
-    end
-    
-    def get_query_result(job_id, batch_id, result_id)
-      headers = {"Content-Type" => "text/csv; charset=UTF-8"}
-      response = http_get("job/#{job_id}/batch/#{batch_id}/result/#{result_id}", headers)
-      
-      lines = response.body.lines.to_a
-      headers = CSV.parse_line(lines.shift).collect { |header| header.to_sym }
-      
-      result = []
-      
-      #CSV.parse(lines.join, :headers => headers, :converters => [:all, lambda{|s| s.to_b if s.kind_of? String }]) do |row|
-      CSV.parse(lines.join, :headers => headers) do |row|
-        result << Hash[row.headers.zip(row.fields)]
-      end
-      
-      result
-    end
-
-    ## Operations
-    def delete(sobject, data)
-      perform_operation(:delete, sobject, data)
-    end
-    
-    def insert(sobject, data)
-      perform_operation(:insert, sobject, data)
-    end
-    
-    def query(sobject, data)
-      perform_operation(:query, sobject, data)
-    end
-    
-    def update(sobject, data)
-      perform_operation(:update, sobject, data)
-    end
-    
-    def upsert(sobject, external_id, data)
-      perform_operation(:upsert, sobject, data, external_id)
-    end
-    
-    def perform_operation(operation, sobject, data, external_id = nil, batch_size = nil)
-      job = new_job(operation, sobject, :external_id_field_name => external_id)
-      
-      data.each_slice(batch_size || Batch.batch_size) do |records|
-        job.add_batch(records)
-      end
-
-      job.close
-      
-      until job.finished?
-        job.refresh
-        sleep 2
-      end
-      
-      job.get_results
-    end
-  end
-end

data/lib/salesforce_bulk/job.rb

@@ -1,128 +0,0 @@
-module SalesforceBulk
-  class Job
-    attr_accessor :client
-
-    attr_reader :concurrency_mode
-    attr_reader :external_id_field_name
-    attr_reader :data
-
-    @@fields = [:id, :operation, :object, :createdById, :state, :createdDate, 
-      :systemModstamp, :externalIdFieldName, :concurrencyMode, :contentType, 
-      :numberBatchesQueued, :numberBatchesInProgress, :numberBatchesCompleted, 
-      :numberBatchesFailed, :totalBatches, :retries, :numberRecordsProcessed, 
-      :numberRecordsFailed, :totalProcessingTime, :apiActiveProcessingTime, 
-      :apexProcessingTime, :apiVersion]
-    
-    @@valid_operations = [:delete, :insert, :update, :upsert, :query]
-    @@valid_concurrency_modes = ['Parallel', 'Serial']
-
-
-    @@fields.each do |field|
-      attr_reader field.to_s.underscore.to_sym
-    end
-
-    def self.valid_operation? operation
-      @@valid_operations.include?(operation)
-    end
-
-    def self.valid_concurrency_mode? mode
-      @@valid_concurrency_modes.include?(concurrency_mode)
-    end
-
-    def new_from_xml xml_data, client = nil
-      job = Job.new
-      job.update(xml_data)
-      job.client = client
-    end
-
-    def update xml_data
-      #Check fields
-      xml_data.assert_valid_keys(@@fields)
-
-      #Assign object
-      @xml_data = xml_data
-
-      #Mass assign the defaults
-      @@fields.each do |field|
-        instance_variable_set(field, xml_data[field])
-      end
-
-      #Special cases and data formats
-      @created_date = DateTime.parse(xml_data['createdDate'])
-      @system_modstamp = DateTime.parse(xml_data['systemModstamp'])
-
-      @retries = xml_data['retries'].to_i
-      @api_version = xml_data['apiVersion'].to_i
-      @number_batches_queued = xml_data['numberBatchesQueued'].to_i
-      @number_batches_in_progress = xml_data['numberBatchesInProgress'].to_i
-      @number_batches_completed = xml_data['numberBatchesCompleted'].to_i
-      @number_batches_failed = xml_data['numberBatchesFailed'].to_i
-      @total_batches = xml_data['totalBatches'].to_i
-      @number_records_processed = xml_data['numberRecordsProcessed'].to_i
-      @number_records_failed = xml_data['numberRecordsFailed'].to_i
-      @total_processing_time = xml_data['totalProcessingTime'].to_i
-      @api_active_processing_time = xml_data['apiActiveProcessingTime'].to_i
-      @apex_processing_time = xml_data['apexProcessingTime'].to_i
-    end
-
-    def batch_list
-      @client.get_batch_list(@id)
-    end
-    
-    def create_batch data
-      @client.create_batch(data)
-    end
-
-    def add_batch data
-      @client.add_batch(data)
-    end
-
-    def close
-      update(@client.close_job(@id))
-    end
-
-    def abort
-      update(@client.abort_job(@id))
-    end
-
-    def refresh
-      update(@client.get_job_info(@id))
-    end
-
-    def get_results
-      batch_list.map(&:result).flatten
-    end
-
-    #Statuses
-    def batches_finished?
-      (@number_batches_queued == 0) and
-      (@number_batches_in_progress == 0)
-    end
-    
-    def finished?
-      failed?  or 
-      aborted? or 
-      (closed? and batches_finished?)
-    end
-
-    def failed?
-      state? 'Failed'
-    end
-
-    def aborted?
-      state? 'Aborted'
-    end
-    
-    def closed?
-      state? 'Closed'
-    end
-    
-    def open?
-      state? 'Open'
-    end
-    
-    def state?(value)
-      @state.present? && @state.casecmp(value) == 0
-    end
-  end
-end

data/lib/salesforce_bulk/version.rb

@@ -1,3 +0,0 @@
-module SalesforceBulk
-  VERSION = "0.5.1"
-end

data/lib/salesforce_bulk.rb

@@ -1,20 +0,0 @@
-require 'net/https'
-require 'xmlsimple'
-require 'csv'
-require 'active_support'
-require 'active_support/inflector'
-require 'active_support/core_ext/object/blank'
-require 'active_support/core_ext/hash/keys'
-require 'salesforce_bulk/version'
-require 'salesforce_bulk/core_extensions/string'
-require 'salesforce_bulk/salesforce_error'
-require 'salesforce_bulk/connection'
-require 'salesforce_bulk/client'
-require 'salesforce_bulk/job'
-require 'salesforce_bulk/batch'
-require 'salesforce_bulk/batch_result'
-require 'salesforce_bulk/batch_result_collection'
-require 'salesforce_bulk/query_result_collection'
-
-module SalesforceBulk
-end