salesforce_bulk_api 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in salesforce_bulk_api.gemspec
+gemspec

data/README.rdoc

@@ -0,0 +1,64 @@
+= Salesforce-Bulk-Api
+
+==Overview
+
+Salesforce bulk Api is a simple ruby gem for connecting to and using the Salesforce Bulk API. It is actually a re-written code from salesforce_bulk[https://github.com/jorgevaldivia/salesforce_bulk].Written to suit many more other features as well.
+ 
+==How to use
+
+Using this gem is simple and straight forward.
+
+To initialize:
+
+Pl do check the entire documentation of the databasedotcom gem and it various ways of authentication
+Databasedotcom[https://github.com/heroku/databasedotcom]
+
+You can use username password combo,OmniAuth,Oauth2
+
+  require 'salesforce_bulk_api'
+  client = Databasedotcom::Client.new :client_id =>  $SFDC_APP_CONFIG["client_id"], :client_secret => $SFDC_APP_CONFIG["client_secret"] #client_id and client_secret respectively
+  client.authenticate :token => "my-oauth-token", :instance_url => "http://na1.salesforce.com"  #=> "my-oauth-token"
+  salesforce = SalesforceBulk::Api.new(client)
+
+Sample operations:
+
+  # Insert/Create
+  new_account = Hash["name" => "Test Account", "type" => "Other"] # Add as many fields per record as needed.
+  records_to_insert = Array.new
+  records_to_insert.push(new_account) # You can add as many records as you want here, just keep in mind that Salesforce has governor limits.
+  result = salesforce.create("Account", records_to_insert)
+  puts "result is: #{result.inspect}"
+
+  # Update
+  updated_account = Hash["name" => "Test Account -- Updated", id => "a00A0001009zA2m"] # Nearly identical to an insert, but we need to pass the salesforce id.
+  records_to_update = Array.new
+  records_to_update.push(updated_account)
+  salesforce.update("Account", records_to_update)
+
+  # Upsert
+  upserted_account = Hash["name" => "Test Account -- Upserted", "External_Field_Name" => "123456"] # Fields to be updated. External field must be included
+  records_to_upsert = Array.new
+  records_to_upsert.push(upserted_account)
+  salesforce.upsert("Account", records_to_upsert, "External_Field_Name") # Note that upsert accepts an extra parameter for the external field name
+
+  # Delete
+  deleted_account = Hash["id" => "a00A0001009zA2m"] # We only specify the id of the records to delete
+  records_to_delete = Array.new
+  records_to_delete.push(deleted_account)
+  salesforce.delete("Account", records_to_delete)
+
+  # Query
+  res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string
+
+==Installation
+sudo gem install salesforce_bulk_api
+
+==TODO
+This is a rough early version of the gem. Immediate plans include better error reporting as there currently is none.
+
+== Copyright
+
+Copyright (c) 2012 Yatish Mehta
+
+===end
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/salesforce_bulk_api/connection.rb

@@ -0,0 +1,64 @@
+module SalesforceBulkApi
+
+  class Connection
+
+    @@XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>'
+    @@API_VERSION = nil
+    @@LOGIN_HOST = 'login.salesforce.com'
+    @@INSTANCE_HOST = nil # Gets set in login()
+
+    def initialize(api_version,client)
+      @client=client
+      @session_id = nil
+      @server_url = nil
+      @instance = nil
+      @@API_VERSION = api_version
+      @@LOGIN_PATH = "/services/Soap/u/#{@@API_VERSION}"
+      @@PATH_PREFIX = "/services/async/#{@@API_VERSION}/"
+
+      login()
+    end
+
+    #private
+
+    def login()
+      @session_id=@client.oauth_token
+      @server_url=@client.instance_url
+      @instance = parse_instance()
+      puts @instance
+      @@INSTANCE_HOST = "#{@instance}.salesforce.com"
+      puts @@INSTANCE_HOST
+    end
+
+    def post_xml(host, path, xml, headers)
+      host = host || @@INSTANCE_HOST
+      if host != @@LOGIN_HOST # Not login, need to add session id to header
+        headers['X-SFDC-Session'] = @session_id;
+        path = "#{@@PATH_PREFIX}#{path}"
+      end
+      https(host).post(path, xml, headers).body
+    end
+
+    def get_request(host, path, headers)
+      host = host || @@INSTANCE_HOST
+      path = "#{@@PATH_PREFIX}#{path}"
+      if host != @@LOGIN_HOST # Not login, need to add session id to header
+        headers['X-SFDC-Session'] = @session_id;
+      end
+      https(host).get(path, headers).body
+    end
+
+    def https(host)
+      req = Net::HTTP.new(host, 443)
+      req.use_ssl = true
+      req.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      req
+    end
+
+    def parse_instance()
+      @instance=@server_url.match(/https:\/\/[a-z]{2}[0-9]{1,2}/).to_s.gsub("https://","")
+    end
+
+  end
+
+end

data/lib/salesforce_bulk_api/job.rb

@@ -0,0 +1,129 @@
+module SalesforceBulkApi
+
+  class Job
+
+    def initialize(operation, sobject, records, external_field, connection)
+
+      @@operation = operation
+      @@sobject = sobject
+      @@external_field = external_field
+      @@records = records
+      @@connection = connection
+      @@XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>'
+
+    end
+
+    def create_job()
+      xml = "#{@@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml += "<operation>#{@@operation}</operation>"
+      xml += "<object>#{@@sobject}</object>"
+      if !@@external_field.nil? # This only happens on upsert
+        xml += "<externalIdFieldName>#{@@external_field}</externalIdFieldName>"
+      end
+      xml += "<contentType>XML</contentType>"
+      xml += "</jobInfo>"
+
+      path = "job"
+      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
+
+      response = @@connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)    
+      puts response
+      @@job_id = response_parsed['id'][0]
+    end
+
+    def close_job()
+      xml = "#{@@XML_HEADER}<jobInfo xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+      xml += "<state>Closed</state>"
+      xml += "</jobInfo>"
+
+      path = "job/#{@@job_id}"
+      headers = Hash['Content-Type' => 'application/xml; charset=utf-8']
+
+      response = @@connection.post_xml(nil, path, xml, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      #job_id = response_parsed['id'][0]
+    end
+
+    def add_query
+      path = "job/#{@@job_id}/batch/"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+      
+      response = @@connection.post_xml(nil, path, @@records, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      @@batch_id = response_parsed['id'][0]
+    end
+
+    def add_batch()
+      keys = @@records.reduce({}) {|h,pairs| pairs.each {|k,v| (h[k] ||= []) << v}; h}.keys
+      headers = keys
+      @@records_dup=@@records.clone
+      super_records=[]
+      (@@records_dup.size/10000).times do
+        super_records<<@@records_dup.pop(10000)
+      end
+      super_records<<@@records_dup
+      
+      super_records.each do|batch|
+        xml = "#{@@XML_HEADER}<sObjects xmlns=\"http://www.force.com/2009/06/asyncapi/dataload\">"
+        batch.each do |r|
+          xml += "<sObject>"
+          keys.each do |k|
+            xml += "<#{k}>#{r[k]}</#{k}>"
+          end
+          xml += "</sObject>"
+        end
+        xml += "</sObjects>"
+    
+
+        path = "job/#{@@job_id}/batch/"
+        headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+        response = @@connection.post_xml(nil, path, xml, headers)
+        response_parsed = XmlSimple.xml_in(response)
+      
+        @@batch_id = response_parsed['id'][0]
+        puts @@batch_id
+      end
+    end
+
+    def check_batch_status()
+      path = "job/#{@@job_id}/batch/#{@@batch_id}"
+      headers = Hash.new
+
+      response = @@connection.get_request(nil, path, headers)
+      response_parsed = XmlSimple.xml_in(response)
+
+      puts response_parsed
+      begin
+        #puts "check: #{response_parsed.inspect}\n"
+        response_parsed['state'][0]
+      rescue Exception => e
+        #puts "check: #{response_parsed.inspect}\n"
+
+        nil
+      end
+    end
+
+    def get_batch_result()
+      path = "job/#{@@job_id}/batch/#{@@batch_id}/result"
+      headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+
+      response = @@connection.get_request(nil, path, headers)
+
+      if(@@operation == "query") # The query op requires us to do another request to get the results
+        response_parsed = XmlSimple.xml_in(response)
+        result_id = response_parsed["result"][0]
+
+        path = "job/#{@@job_id}/batch/#{@@batch_id}/result/#{result_id}"
+        headers = Hash.new
+        headers = Hash["Content-Type" => "application/xml; charset=UTF-8"]
+        response = @@connection.get_request(nil, path, headers)
+      end
+
+
+    end
+
+  end
+end

data/lib/salesforce_bulk_api/version.rb

@@ -0,0 +1,3 @@
+module SalesforceBulkApi
+  VERSION = "0.0.1"
+end

data/lib/salesforce_bulk_api.rb

@@ -0,0 +1,73 @@
+require "salesforce_bulk_api/version"
+require 'net/https'
+require 'rubygems'
+require 'xmlsimple'
+require 'csv'
+require "salesforce_bulk_api/version"
+require 'salesforce_bulk_api/job'
+require 'salesforce_bulk_api/connection'
+module SalesforceBulkApi
+  # Your code goes here...
+  class Api
+
+    @@SALESFORCE_API_VERSION = '23.0'
+
+    def initialize(client)
+      @connection = SalesforceBulk::Connection.new(@@SALESFORCE_API_VERSION,client)
+    end
+
+    def upsert(sobject, records, external_field)
+      self.do_operation('upsert', sobject, records, external_field)
+    end
+
+    def update(sobject, records)
+      self.do_operation('update', sobject, records, nil)
+    end
+    
+    def create(sobject, records)
+      self.do_operation('insert', sobject, records, nil)
+    end
+
+    def delete(sobject, records)
+      self.do_operation('delete', sobject, records, nil)
+    end
+
+    def query(sobject, query)
+      self.do_operation('query', sobject, query, nil)
+    end
+
+    #private
+
+    def do_operation(operation, sobject, records, external_field)
+      job = SalesforceBulk::Job.new(operation, sobject, records, external_field, @connection)
+
+      # TODO: put this in one function
+      job_id = job.create_job()
+      if(operation == "query")
+        batch_id = job.add_query()
+      else
+        batch_id = job.add_batch()
+      end
+      job.close_job()
+
+      while true
+        state = job.check_batch_status()
+        #puts "\nstate is #{state}\n"
+        if state != "Queued" && state != "InProgress"
+          break
+        end
+        sleep(2) # wait x seconds and check again
+      end
+
+      if state == 'Completed'
+        job.get_batch_result()
+      else
+        return "error"
+      end
+
+    end
+
+  end  # End class
+end
+
+

data/salesforce_bulk_api.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "salesforce_bulk_api/version"
+
+Gem::Specification.new do |s|
+  s.name        = "salesforce_bulk_api"
+  s.version     = SalesforceBulkApi::VERSION
+  s.authors     = ["Yatish Mehta"]
+  s.email       = ["yatishmehta27@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{It uses the bulk api of salesforce to communicate with teh salesforce CRM}
+  s.description = %q{Salesforce Bulk API with governor limits taken care of}
+
+  s.rubyforge_project = "salesforce_bulk_api"
+  s.add_dependency(%q<oauth2>, ["0.4.1"])
+  s.add_dependency(%q<databasedotcom>, [">= 0"])
+  s.add_dependency(%q<json>, [">= 0"])
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: salesforce_bulk_api
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Yatish Mehta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-12 00:00:00.000000000 +05:30
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oauth2
+  requirement: &21802940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *21802940
+- !ruby/object:Gem::Dependency
+  name: databasedotcom
+  requirement: &21802320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21802320
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &21801740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21801740
+description: Salesforce Bulk API with governor limits taken care of
+email:
+- yatishmehta27@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.rdoc
+- Rakefile
+- lib/salesforce_bulk_api.rb
+- lib/salesforce_bulk_api/connection.rb
+- lib/salesforce_bulk_api/job.rb
+- lib/salesforce_bulk_api/version.rb
+- salesforce_bulk_api.gemspec
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: salesforce_bulk_api
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: It uses the bulk api of salesforce to communicate with teh salesforce CRM
+test_files: []