salesforce_bulk_api 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99c3f558cf30c25184d78f67c158fac104d3deca
-  data.tar.gz: 4298337ac2a86b50e959eee7c9707a8de423778d
+  metadata.gz: ec07d0cc7d5f0c5795459e444948d45ed308dde1
+  data.tar.gz: 0faae2875e7ec75bf9a8bb8dfe54025b607cedb6
 SHA512:
-  metadata.gz: 035074901c3cbc3aeb06cb47b008c094e1e02b62b0c784021d9b434e8608c8d9bd89472580bf993397420c5849122821c3df3c540ac1d420f97d57178c5b74a4
-  data.tar.gz: 6b804db79f5ca0b6aa8be4233f72d842c08ab0e1465edfbc606c25615d33a3a5233419cae594b990fcbcefc7c4660afef43aca6b94417f907f7748e18dbe5274
+  metadata.gz: c68f01bc8d22feb69025bed933e52049e7227e70a8df2e4496ac89f224e6a7571199510ac2a5c61e9764e8f828a6d776a8b05bd78d88560f8baf1bde5ccc18a2
+  data.tar.gz: d91276c381a0f188f105af7442a95ef5c4e75f399f4d55a37906b3e5c1bfa0f6bcaecc6c2d8fde967a10ac8ea15e01913b71a426f7e71a216e18023d9e7d7ecb

data/.gitignore

@@ -5,3 +5,4 @@ Gemfile.lock
 .ruby-version
 pkg/*
 auth_credentials.yml
+*.swp

data/README.md

@@ -50,7 +50,7 @@ OR
 	salesforce = SalesforceBulkApi::Api.new(client)
 
 
-Sample operations:
+### Sample operations:
 
     # Insert/Create
     # Add as many fields per record as needed.
@@ -82,21 +82,26 @@ Sample operations:
     # Query
 	res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string
 
-Helpful methods:
+### Helpful methods:
 
     # Check status of a job via #job_from_id
 	job = salesforce.job_from_id('a00A0001009zA2m') # Returns a SalesforceBulkApi::Job instance
 	puts "status is: #{job.check_job_status.inspect}"
 
-
-Listening to events:
+### Listening to events:
 
     # A job is created
     # Useful when you need to store the job_id before any work begins, then if you fail during a complex load scenario, you can wait for your
     # previous job(s) to finish.
-  salesforce.on_job_created do |job|
-    puts "Job #{job.job_id} created!"
-  end
+    salesforce.on_job_created do |job|
+      puts "Job #{job.job_id} created!"
+    end
+
+### Throttling API calls:
+
+    # By default, this gem (and maybe your app driving it) will query job/batch statuses at an unbounded rate.  We
+    # can fix that, e.g.:
+    salesforce.connection.set_status_throttle(30) # only check status of individual jobs/batches every 30 seconds
 
 ## Installation
 

data/example_auth_credentials.yml

@@ -3,4 +3,5 @@ salesforce:
   client_secret: client_secret_here
   user: sf_user@example.com
   passwordandtoken: passandtokenhere
-  test_account_id: 0013000000ymMBh
+  test_account_id: 0013000000ymMBh
+  host: 'login.salesforce.com' # use test.salesforce.com if it is a sandbox

data/lib/salesforce_bulk_api.rb

@@ -1,44 +1,59 @@
 require 'rubygems'
 require 'bundler'
 Bundler.require()
-require "salesforce_bulk_api/version"
+require 'salesforce_bulk_api/version'
 require 'net/https'
 require 'xmlsimple'
 require 'csv'
+require 'salesforce_bulk_api/concerns/throttling'
 require 'salesforce_bulk_api/job'
 require 'salesforce_bulk_api/connection'
 
 module SalesforceBulkApi
 
   class Api
+    attr_reader :connection
 
     @@SALESFORCE_API_VERSION = '32.0'
 
     def initialize(client)
-      @connection = SalesforceBulkApi::Connection.new(@@SALESFORCE_API_VERSION,client)
+      @connection = SalesforceBulkApi::Connection.new(@@SALESFORCE_API_VERSION, client)
       @listeners = { job_created: [] }
     end
 
     def upsert(sobject, records, external_field, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      self.do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
+      do_operation('upsert', sobject, records, external_field, get_response, timeout, batch_size, send_nulls, no_null_list)
     end
 
     def update(sobject, records, get_response = false, send_nulls = false, no_null_list = [], batch_size = 10000, timeout = 1500)
-      self.do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
+      do_operation('update', sobject, records, nil, get_response, timeout, batch_size, send_nulls, no_null_list)
     end
 
     def create(sobject, records, get_response = false, send_nulls = false, batch_size = 10000, timeout = 1500)
-      self.do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls)
+      do_operation('insert', sobject, records, nil, get_response, timeout, batch_size, send_nulls)
     end
 
     def delete(sobject, records, get_response = false, batch_size = 10000, timeout = 1500)
-      self.do_operation('delete', sobject, records, nil, get_response, timeout, batch_size)
+      do_operation('delete', sobject, records, nil, get_response, timeout, batch_size)
     end
 
     def query(sobject, query, batch_size = 10000, timeout = 1500)
-      self.do_operation('query', sobject, query, nil, true, timeout, batch_size)
+      do_operation('query', sobject, query, nil, true, timeout, batch_size)
     end
 
+    def counters
+      {
+        http_get: @connection.counters[:get],
+        http_post: @connection.counters[:post],
+        upsert: get_counters[:upsert],
+        update: get_counters[:update],
+        create: get_counters[:create],
+        delete: get_counters[:delete],
+        query: get_counters[:query]
+      }
+    end
+
+
     ##
     # Allows you to attach a listener that accepts the created job (which has a useful #job_id field).  This is useful
     # for recording a job ID persistently before you begin batch work (i.e. start modifying the salesforce database),
@@ -52,9 +67,9 @@ module SalesforceBulkApi
       SalesforceBulkApi::Job.new(job_id: job_id, connection: @connection)
     end
 
-    #private
-
     def do_operation(operation, sobject, records, external_field, get_response, timeout, batch_size, send_nulls = false, no_null_list = [])
+      count operation.to_sym
+
       job = SalesforceBulkApi::Job.new(operation: operation, sobject: sobject, records: records, external_field: external_field, connection: @connection)
 
       job.create_job(batch_size, send_nulls, no_null_list)
@@ -64,5 +79,15 @@ module SalesforceBulkApi
       response.merge!({'batches' => job.get_job_result(get_response, timeout)}) if get_response == true
       response
     end
+
+    private
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(name)
+      get_counters[name] += 1
+    end
+
   end
 end

data/lib/salesforce_bulk_api/concerns/throttling.rb

@@ -0,0 +1,60 @@
+module SalesforceBulkApi::Concerns
+  module Throttling
+
+    def throttles
+      @throttles.dup
+    end
+
+    def add_throttle(&throttling_callback)
+      @throttles ||= []
+      @throttles << throttling_callback
+    end
+
+    def set_status_throttle(limit_seconds)
+      set_throttle_limit_in_seconds(limit_seconds, [:http_method, :path], ->(details) { details[:path] == :get })
+    end
+
+    def set_throttle_limit_in_seconds(limit_seconds, throttle_by_keys, only_if)
+      add_throttle do |details|
+        limit_log = get_limit_log(Time.now - limit_seconds)
+        key = extract_constraint_key_from(details, throttle_by_keys)
+        last_request = limit_log[key]
+
+        if !last_request.nil? && only_if.call(details)
+          seconds_since_last_request = Time.now.to_f - last_request.to_f
+          need_to_wait_seconds = limit_seconds - seconds_since_last_request
+          sleep(need_to_wait_seconds) if need_to_wait_seconds > 0
+        end
+
+        limit_log[key] = Time.now
+      end
+    end
+
+    private
+
+    def extract_constraint_key_from(details, throttle_by_keys)
+      hash = {}
+      throttle_by_keys.each { |k| hash[k] = details[k] }
+      hash
+    end
+
+    def get_limit_log(prune_older_than)
+      @limits ||= Hash.new(0)
+
+      @limits.delete_if do |k, v|
+        v < prune_older_than
+      end
+
+      @limits
+    end
+
+    def throttle(details={})
+      (@throttles || []).each do |callback|
+        args = [details]
+        args = args[0..callback.arity]
+        callback.call(*args)
+      end
+    end
+
+  end
+end

data/lib/salesforce_bulk_api/connection.rb

@@ -2,6 +2,7 @@ module SalesforceBulkApi
 require 'timeout'
 
   class Connection
+    include Concerns::Throttling
 
     @@XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>'
     @@API_VERSION = nil
@@ -20,8 +21,6 @@ require 'timeout'
       login()
     end
 
-    #private
-
     def login()
       client_type = @client.class.to_s
       case client_type
@@ -39,11 +38,13 @@ require 'timeout'
     def post_xml(host, path, xml, headers)
       host = host || @@INSTANCE_HOST
       if host != @@LOGIN_HOST # Not login, need to add session id to header
-        headers['X-SFDC-Session'] = @session_id;
+        headers['X-SFDC-Session'] = @session_id
         path = "#{@@PATH_PREFIX}#{path}"
       end
       i = 0
       begin
+        count :post
+        throttle(http_method: :post, path: path)
         https(host).post(path, xml, headers).body
       rescue
         i += 1
@@ -63,6 +64,9 @@ require 'timeout'
       if host != @@LOGIN_HOST # Not login, need to add session id to header
         headers['X-SFDC-Session'] = @session_id;
       end
+
+      count :get
+      throttle(http_method: :get, path: path)
       https(host).get(path, headers).body
     end
 
@@ -74,11 +78,28 @@ require 'timeout'
     end
 
     def parse_instance()
-      @instance=@server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}/).to_s.gsub("https://","")
+      @instance = @server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}/).to_s.gsub("https://","")
       @instance = @server_url.split(".salesforce.com")[0].split("://")[1] if @instance.nil? || @instance.empty?
       return @instance
     end
 
+    def counters
+      {
+        get: get_counters[:get],
+        post: get_counters[:post]
+      }
+    end
+
+    private
+
+    def get_counters
+      @counters ||= Hash.new(0)
+    end
+
+    def count(http_method)
+      get_counters[http_method] += 1
+    end
+
   end
 
 end

data/lib/salesforce_bulk_api/job.rb

@@ -5,13 +5,13 @@ module SalesforceBulkApi
 
     class SalesforceException < StandardError; end
 
-    def initialize(operation: nil, sobject: nil, records: nil, external_field: nil, connection: nil, job_id: nil)
-      @job_id         = job_id
-      @operation      = operation
-      @sobject        = sobject
-      @external_field = external_field
-      @records        = records
-      @connection     = connection
+    def initialize(args)
+      @job_id         = args[:job_id]
+      @operation      = args[:operation]
+      @sobject        = args[:sobject]
+      @external_field = args[:external_field]
+      @records        = args[:records]
+      @connection     = args[:connection]
       @batch_ids      = []
       @XML_HEADER     = '<?xml version="1.0" encoding="utf-8" ?>'
     end
@@ -69,7 +69,7 @@ module SalesforceBulkApi
 
     def add_batches
       raise 'Records must be an array of hashes.' unless @records.is_a? Array
-      keys = @records.reduce({}) {|h,pairs| pairs.each {|k,v| (h[k] ||= []) << v}; h}.keys
+      keys = @records.reduce({}) {|h, pairs| pairs.each {|k, v| (h[k] ||= []) << v}; h}.keys
 
       @records_dup = @records.clone
 
@@ -103,7 +103,6 @@ module SalesforceBulkApi
         if k.is_a?(Hash)
           xml += build_sobject(k)
         elsif data[k] != :type
-          #xml += "<type>#{data[:type]}</type>"
           xml += "<#{k}>#{data[k]}</#{k}>"
         end
       end

data/lib/salesforce_bulk_api/version.rb

@@ -1,3 +1,3 @@
 module SalesforceBulkApi
-  VERSION = '0.0.11'
+  VERSION = '0.0.12'
 end

data/salesforce_bulk_api.gemspec

@@ -14,15 +14,14 @@ Gem::Specification.new do |s|
 
   s.rubyforge_project = 'salesforce_bulk_api'
 
-  s.add_dependency(%q<json>, ['>= 0'])
-  s.add_dependency(%q<xml-simple>, ['>= 0'])
-  
+  s.add_dependency('json', ['>= 0'])
+  s.add_dependency('xml-simple', ['>= 0'])
+
   s.add_development_dependency 'rspec'
-  s.add_development_dependency('webmock', ['~> 1.13'])
-  s.add_development_dependency('vcr', ['~> 2.5'])
-  s.add_development_dependency 'databasedotcom'
-  
-  
+  s.add_development_dependency 'restforce', '~> 1.5.1'
+  s.add_development_dependency 'rake', '~> 10.4.2'
+  s.add_development_dependency 'pry'
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }

data/spec/salesforce_bulk_api/salesforce_bulk_api_spec.rb

@@ -1,14 +1,20 @@
 require 'spec_helper'
 require 'yaml'
-require 'databasedotcom'
+require 'restforce'
 
 describe SalesforceBulkApi do
 
   before :each do
-    auth_hash = YAML.load(File.read('auth_credentials.yml'))
-    @sf_client = Databasedotcom::Client.new(:client_id     => auth_hash['salesforce']['client_id'],
-                                            :client_secret => auth_hash['salesforce']['client_secret'])
-    @sf_client.authenticate(:username => auth_hash['salesforce']['user'], :password => auth_hash['salesforce']['passwordandtoken'])
+    auth_hash = YAML.load_file('auth_credentials.yml')
+    sfdc_auth_hash = auth_hash['salesforce']
+
+    @sf_client = Restforce.new(
+      username: sfdc_auth_hash['user'],
+      password: sfdc_auth_hash['passwordandtoken'],
+      client_id: sfdc_auth_hash['client_id'],
+      client_secret: sfdc_auth_hash['client_secret'],
+      host: sfdc_auth_hash['host'])
+    @sf_client.authenticate!
 
     @account_id = auth_hash['salesforce']['test_account_id']
 
@@ -22,7 +28,7 @@ describe SalesforceBulkApi do
   describe 'upsert' do
 
     context 'when not passed get_result' do
-      it "doesn't return the batches array" do        
+      it "doesn't return the batches array" do
         res = @api.upsert('Account', [{:Id => @account_id, :Website => 'www.test.com'}], 'Id')
         res['batches'].should be_nil
       end
@@ -139,13 +145,10 @@ describe SalesforceBulkApi do
         res['batches'][0]['response'].length.should > 1
         res['batches'][0]['response'][0]['Id'].should_not be_nil
       end
+
       context 'and there are multiple batches' do
-        it 'returns the query results in a merged hash' do
-          pending 'need dev to create > 10k records in dev organization'
-          res = @api.query('Account', "SELECT id, Name From Account WHERE Name LIKE 'Test%'")
-          res['batches'][0]['response'].length.should > 1
-          res['batches'][0]['response'][0]['Id'].should_not be_nil
-        end
+        # need dev to create > 10k records in dev organization
+        it 'returns the query results in a merged hash'
       end
     end
 
@@ -165,4 +168,26 @@ describe SalesforceBulkApi do
 
   end
 
+  describe 'counters' do
+    context 'when read operations are called' do
+      it 'increments operation count and http GET count' do
+        @api.counters[:http_get].should eq 0
+        @api.counters[:query].should eq 0
+        @api.query('Account', "SELECT Website, Phone From Account WHERE Id = '#{@account_id}'")
+        @api.counters[:http_get].should eq 1
+        @api.counters[:query].should eq 1
+      end
+    end
+
+    context 'when update operations are called' do
+      it 'increments operation count and http POST count' do
+        @api.counters[:http_post].should eq 0
+        @api.counters[:update].should eq 0
+        @api.update('Account', [{:Id => @account_id, :Website => 'abc123', :Phone => '5678'}], true)
+        @api.counters[:http_post].should eq 1
+        @api.counters[:update].should eq 1
+      end
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,16 +1,8 @@
 require 'rubygems'
 require 'bundler/setup'
-#require 'webmock/rspec'
-#require 'vcr'
 require 'salesforce_bulk_api'
 
 RSpec.configure do |c|
   c.filter_run :focus => true
   c.run_all_when_everything_filtered = true
 end
-
-# enable this and record the test requests using a SF developer org.
-# VCR.configure do |c|
-#   c.cassette_library_dir = 'spec/cassettes'
-#   c.hook_into :webmock
-# end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: salesforce_bulk_api
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Yatish Mehta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-10 00:00:00.000000000 Z
+date: 2015-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -53,35 +53,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: restforce
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 1.5.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: 10.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: 10.4.2
 - !ruby/object:Gem::Dependency
-  name: databasedotcom
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -108,6 +108,7 @@ files:
 - Rakefile
 - example_auth_credentials.yml
 - lib/salesforce_bulk_api.rb
+- lib/salesforce_bulk_api/concerns/throttling.rb
 - lib/salesforce_bulk_api/connection.rb
 - lib/salesforce_bulk_api/job.rb
 - lib/salesforce_bulk_api/version.rb