bulkforce 1.0.0

data/lib/bulkforce/batch.rb

@@ -0,0 +1,61 @@
+class Bulkforce
+  class Batch
+    attr_reader :job_id
+
+    def initialize connection, job_id, batch_id
+      @connection = connection
+      @job_id = job_id
+      @batch_id = batch_id
+
+      if @batch_id == -1
+        @final_status = {
+          state: "Completed",
+          state_message: "Empty Request"
+        }
+      end
+    end
+
+    def final_status poll_interval=2
+      return @final_status if @final_status
+
+      @final_status = self.status
+      while ["Queued", "InProgress"].include?(@final_status[:state])
+        sleep poll_interval
+        @final_status = self.status
+        yield @final_status if block_given?
+      end
+
+      raise @final_status[:state_message] if @final_status[:state] == "Failed"
+
+      @final_status.merge({
+          results: results
+        })
+    end
+
+    def status
+      @connection.query_batch @job_id, @batch_id
+    end
+
+    # results returned from Salesforce can be a single page id, or an array of ids.
+    # if it"s an array of ids, we will fetch the results from each, and concatenate them.
+    def results
+      Array(query_result_id).map do |result_id|
+        @connection.query_batch_result_data(@job_id, @batch_id, result_id)
+      end.flatten
+    end
+
+    def raw_request
+      @connection.raw_request
+    end
+
+    def raw_result
+      @connection.raw_result
+    end
+
+    private
+    def query_result_id
+      result_raw = @connection.query_batch_result_id(@job_id, @batch_id)
+      result_raw[:result] if result_raw
+    end
+  end
+end

data/lib/bulkforce/configuration.rb

@@ -0,0 +1,42 @@
+class Bulkforce
+  class Configuration
+    attr_accessor :api_version
+    attr_accessor :username
+    attr_accessor :password
+    attr_accessor :security_token
+    attr_accessor :host
+    attr_accessor :session_id
+    attr_accessor :instance
+    attr_accessor :client_id
+    attr_accessor :client_secret
+    attr_accessor :refresh_token
+
+    def initialize
+      @api_version = ENV["SALESFORCE_API_VERSION"] || "33.0"
+      @username = ENV["SALESFORCE_USERNAME"]
+      @password = ENV["SALESFORCE_PASSWORD"]
+      @security_token = ENV["SALESFORCE_SECURITY_TOKEN"]
+      @host = ENV["SALESFORCE_HOST"] || "login.salesforce.com"
+      @session_id = ENV["SALESFORCE_SESSION_ID"]
+      @instance = ENV["SALESFORCE_INSTANCE"]
+      @client_id = ENV["SALESFORCE_CLIENT_ID"]
+      @client_secret = ENV["SALESFORCE_CLIENT_SECRET"]
+      @refresh_token = ENV["SALESFORCE_REFRESH_TOKEN"]
+    end
+
+    def to_h
+      {
+        api_version: api_version,
+        username: username,
+        password: password,
+        security_token: security_token,
+        host: host,
+        session_id: session_id,
+        instance: instance,
+        client_id: client_id,
+        client_secret: client_secret,
+        refresh_token: refresh_token,
+      }.reject { |_, v| v.nil? }.to_h
+    end
+  end
+end

data/lib/bulkforce/connection.rb

@@ -0,0 +1,114 @@
+class Bulkforce
+  class Connection
+    attr_reader :raw_request
+    attr_reader :raw_result
+
+    def initialize(session_id:, instance:, api_version:)
+      @api_version = api_version
+      @session_id = session_id
+      @instance = instance
+    end
+
+    def login
+      response = Bulkforce::Http.login(
+        @host,
+        @username,
+        @password,
+        @api_version)
+
+      @session_id = response[:session_id]
+      @instance = response[:instance]
+      self
+    end
+
+    def org_id
+      @session_id.split("!").first
+    end
+
+    def create_job operation, sobject, content_type, external_field
+      Bulkforce::Http.create_job(
+        @instance,
+        @session_id,
+        operation,
+        sobject,
+        content_type,
+        @api_version,
+        external_field)[:id]
+    end
+
+    def close_job job_id
+      Bulkforce::Http.close_job(
+        @instance,
+        @session_id,
+        job_id,
+        @api_version)[:id]
+    end
+
+    def add_query job_id, data_or_soql
+      Bulkforce::Http.add_batch(
+        @instance,
+        @session_id,
+        job_id,
+        data_or_soql,
+        @api_version)[:id]
+    end
+
+    def query_batch job_id, batch_id
+      Bulkforce::Http.query_batch(
+        @instance,
+        @session_id,
+        job_id,
+        batch_id,
+        @api_version,
+      )
+    end
+
+    def query_batch_result_id job_id, batch_id
+      Bulkforce::Http.query_batch_result_id(
+        @instance,
+        @session_id,
+        job_id,
+        batch_id,
+        @api_version,
+      )
+    end
+
+    def query_batch_result_data job_id, batch_id, result_id
+      @raw_result = Bulkforce::Http.query_batch_result_data(
+        @instance,
+        @session_id,
+        job_id,
+        batch_id,
+        result_id,
+        @api_version,
+      )
+      Bulkforce::Helper.parse_csv @raw_result
+    end
+
+    def add_file_upload_batch job_id, filename
+      @raw_request = File.read(filename)
+      Bulkforce::Http.add_file_upload_batch(
+        @instance,
+        @session_id,
+        job_id,
+        @raw_request,
+        @api_version)[:id]
+    end
+
+    def add_batch job_id, records
+      return -1 if records.nil? || records.empty?
+      @raw_request = Bulkforce::Helper.records_to_csv(records)
+
+      Bulkforce::Http.add_batch(
+        @instance,
+        @session_id,
+        job_id,
+        @raw_request,
+        @api_version)[:id]
+    end
+
+    def self.connect(username, password, api_version, host)
+      self.new(username, password, api_version, host).login
+    end
+  end
+end

data/lib/bulkforce/connection_builder.rb

@@ -0,0 +1,40 @@
+class Bulkforce
+  class ConnectionBuilder
+    attr_reader :host
+    attr_reader :api_version
+    attr_reader :credentials
+
+    def initialize(host:, api_version:, **credentials)
+      @host = host
+      @api_version = api_version
+      @credentials = credentials
+    end
+
+    def build
+      base_options = { api_version: api_version }
+
+      session_options = if credentials[:session_id]
+        {
+          session_id: credentials[:session_id],
+          instance: credentials.fetch(:instance),
+        }
+      elsif credentials[:refresh_token]
+        Bulkforce::Http.oauth_login(
+          host,
+          credentials.fetch(:client_id),
+          credentials.fetch(:client_secret),
+          credentials[:refresh_token]
+        )
+      else
+        Bulkforce::Http.login(
+          host,
+          credentials.fetch(:username),
+          "#{credentials.fetch(:password)}#{credentials.fetch(:security_token)}",
+          api_version
+        )
+      end.select { |k, _|  [:session_id, :instance].include?(k) }.to_h
+
+      Bulkforce::Connection.new(base_options.merge(session_options))
+    end
+  end
+end

data/lib/bulkforce/helper.rb

@@ -0,0 +1,77 @@
+require "csv"
+
+class Bulkforce
+  module Helper
+    extend self
+
+    CSV_OPTIONS = {
+      col_sep: ",",
+      quote_char: "\"",
+      force_quotes: true,
+    }
+
+    def records_to_csv records
+      file_mock = StringIO.new
+      csv_client = CSV.new(file_mock, CSV_OPTIONS)
+      all_headers = []
+      all_rows = []
+      records.each do |hash|
+        row = CSV::Row.new([],[],false)
+        to_store = hash.inject({}) do |h, (k, v)|
+          if v == nil || v == "" || v == []
+            h[k] = "#N/A"
+          else
+            h[k] = v.class == Array ? v.join(";") : v
+          end
+          h
+        end
+        row << to_store
+        all_headers << row.headers
+        all_rows << row
+      end
+      all_headers.flatten!.uniq!
+      csv_client << all_headers
+      all_rows.each do |row|
+        csv_client << row.fields(*all_headers)
+      end
+      file_mock.string
+    end
+
+    def fetch_instance_from_server_url server_url
+      before_sf = server_url[/^https?:\/\/(.+)\.salesforce\.com/, 1]
+      before_sf.gsub(/-api$/,"")
+    end
+
+    def attachment_keys records
+      records.map do |record|
+        record.select do |key, value|
+          value.class == File
+        end.keys
+      end.flatten.uniq
+    end
+
+    def transform_values! records, keys
+      keys.each do |key|
+        records.each do |record|
+          file_handle = record[key]
+          if file_handle
+            file_path = File.absolute_path(file_handle)
+            record
+              .merge!({
+                key => Bulkforce::Helper.absolute_to_relative_path(file_path,"#")
+              })
+            yield file_path if block_given?
+          end
+        end
+      end
+    end
+
+    def absolute_to_relative_path input, replacement
+      input.gsub(/(^C:[\/\\])|(^\/)/,replacement)
+    end
+
+    def parse_csv csv_string
+      CSV.parse(csv_string, headers: true).map{|r| r.to_hash}
+    end
+  end
+end

data/lib/bulkforce/http.rb

@@ -0,0 +1,272 @@
+require "net/https"
+require "nori"
+require "csv"
+
+class Bulkforce
+  module Http
+    extend self
+
+    def login *args
+      r = Http::Request.login(*args)
+      process_soap_response(nori.parse(process_http_request(r)))
+    end
+
+    def oauth_login *args
+      r = Http::Request.oauth_login(*args)
+      process_oauth_response(nori.parse(process_http_request(r)))
+    end
+
+    def create_job *args
+      r = Http::Request.create_job(*args)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def close_job *args
+      r = Http::Request.close_job(*args)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def add_batch *args
+      r = Http::Request.add_batch(*args)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def query_batch *args
+      r = Http::Request.query_batch(*args)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def query_batch_result_id *args
+      r = Http::Request.query_batch_result_id(*args)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def query_batch_result_data *args
+      r = Http::Request.query_batch_result_data(*args)
+      normalize_csv(process_http_request(r))
+    end
+
+    def add_file_upload_batch instance, session_id, job_id, data, api_version
+      headers = {
+        "Content-Type" => "zip/csv",
+        "X-SFDC-Session" => session_id}
+      r = Http::Request.new(
+        :post,
+        Http::Request.instance_host(instance),
+        "/services/async/#{api_version}/job/#{job_id}/batch",
+        data,
+        headers)
+      process_xml_response(nori.parse(process_http_request(r)))
+    end
+
+    def process_http_request(r)
+      http = Net::HTTP.new(r.host, 443)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http_request = Net::HTTP.
+      const_get(r.http_method.capitalize).
+        new(r.path, r.headers)
+      http_request.body = r.body if r.body
+      http.request(http_request).body
+    end
+
+    private
+    def nori
+      Nori.new(
+        :advanced_typecasting => true,
+        :strip_namespaces => true,
+        :convert_tags_to => lambda { |tag| tag.snakecase.to_sym })
+    end
+
+    def process_xml_response res
+      if res[:error]
+        raise "#{res[:error][:exception_code]}: #{res[:error][:exception_message]}"
+      end
+
+      res.values.first
+    end
+
+    def normalize_csv res
+      res.gsub(/\n\s+/, "\n")
+    end
+
+    def process_soap_response res
+      raw_result = res.fetch(:body){ res.fetch(:envelope).fetch(:body) }
+      raise raw_result[:fault][:faultstring] if raw_result[:fault]
+
+      login_result = raw_result[:login_response][:result]
+      instance = Helper.fetch_instance_from_server_url(login_result[:server_url])
+      login_result.merge(instance: instance)
+    end
+
+    def process_oauth_response res
+      inner = res.fetch(:o_auth)
+
+      if inner[:error]
+        raise "#{inner[:error]}: #{inner[:error_description]}"
+      end
+
+      {
+        server_url: inner.fetch(:instance_url),
+        session_id: inner.fetch(:access_token),
+        instance: Helper.fetch_instance_from_server_url(inner.fetch(:instance_url)),
+      }
+    end
+
+    class Request
+      attr_reader :path
+      attr_reader :host
+      attr_reader :body
+      attr_reader :headers
+      attr_reader :http_method
+
+      def initialize http_method, host, path, body, headers
+        @http_method  = http_method
+        @host         = host
+        @path         = path
+        @body         = body
+        @headers      = headers
+      end
+
+      def self.login host, username, password, api_version
+        body =  %Q{<?xml version="1.0" encoding="utf-8" ?>
+        <env:Envelope xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xmlns:env="http://schemas.xmlsoap.org/soap/envelope/">
+          <env:Body>
+            <n1:login xmlns:n1="urn:partner.soap.sforce.com">
+              <n1:username>#{username}</n1:username>
+              <n1:password>#{password}</n1:password>
+            </n1:login>
+          </env:Body>
+        </env:Envelope>}
+        headers = {
+          "Content-Type" => "text/xml; charset=utf-8",
+          "SOAPAction" => "login"
+        }
+        Http::Request.new(
+          :post,
+          host,
+          "/services/Soap/u/#{api_version}",
+          body,
+          headers)
+      end
+
+      def self.oauth_login(host, client_id, client_secret, refresh_token)
+        headers = {
+          "Content-Type" => "application/x-www-form-urlencoded",
+          "Accept" => "application/xml",
+        }
+
+        body = {
+          grant_type: "refresh_token",
+          client_id: client_id,
+          client_secret: client_secret,
+          refresh_token: refresh_token
+        }.inject("") do |string, (k,v)|
+          string += "#{k}=#{v}&"
+        end
+
+        Http::Request.new(
+          :post,
+          host,
+          "/services/oauth2/token",
+          body,
+          headers)
+      end
+
+      def self.create_job instance, session_id, operation, sobject, content_type, api_version, external_field = nil
+        external_field_line = external_field ?
+          "<externalIdFieldName>#{external_field}</externalIdFieldName>" : nil
+        body = %Q{<?xml version="1.0" encoding="utf-8" ?>
+          <jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">
+            <operation>#{operation}</operation>
+            <object>#{sobject}</object>
+            #{external_field_line}
+            <contentType>#{content_type}</contentType>
+          </jobInfo>
+        }
+        headers = {
+          "Content-Type" => "application/xml; charset=utf-8",
+          "X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :post,
+          instance_host(instance),
+          "/services/async/#{api_version}/job",
+          body,
+          headers)
+      end
+
+      def self.close_job instance, session_id, job_id, api_version
+        body = %Q{<?xml version="1.0" encoding="utf-8" ?>
+          <jobInfo xmlns="http://www.force.com/2009/06/asyncapi/dataload">
+            <state>Closed</state>
+          </jobInfo>
+        }
+        headers = {
+          "Content-Type" => "application/xml; charset=utf-8",
+          "X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :post,
+          instance_host(instance),
+          "/services/async/#{api_version}/job/#{job_id}",
+          body,
+          headers)
+      end
+
+      def self.add_batch instance, session_id, job_id, data, api_version
+        headers = {"Content-Type" => "text/csv; charset=UTF-8", "X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :post,
+          instance_host(instance),
+          "/services/async/#{api_version}/job/#{job_id}/batch",
+          data,
+          headers)
+      end
+
+      def self.query_batch instance, session_id, job_id, batch_id, api_version
+        headers = {"X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :get,
+          instance_host(instance),
+          "/services/async/#{api_version}/job/#{job_id}/batch/#{batch_id}",
+          nil,
+          headers)
+      end
+
+      def self.query_batch_result_id instance, session_id, job_id, batch_id, api_version
+        headers = {
+          "Content-Type" => "application/xml; charset=utf-8",
+          "X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :get,
+          instance_host(instance),
+          "/services/async/#{api_version}/job/#{job_id}/batch/#{batch_id}/result",
+          nil,
+          headers)
+      end
+
+      def self.query_batch_result_data(instance,
+        session_id,
+        job_id,
+        batch_id,
+        result_id,
+        api_version)
+        headers = {
+          "Content-Type" => "text/csv; charset=UTF-8",
+          "X-SFDC-Session" => session_id}
+        Http::Request.new(
+          :get,
+          instance_host(instance),
+          "/services/async/#{api_version}" \
+            "/job/#{job_id}/batch/#{batch_id}/result/#{result_id}",
+          nil,
+          headers)
+      end
+
+      def self.instance_host instance
+        "#{instance}.salesforce.com"
+      end
+    end
+  end
+end