salesforce_bulk_query-edge 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a22424006fca0ee3da13b2edea99a629199d708
+  data.tar.gz: 561cf3fe12a4a240e70c5140dae51b0d72d86617
+SHA512:
+  metadata.gz: bda06a6eef7746b4ba76679057ca36d2232de8e0c6158c7de7af3b852bb0a9af255e45d9dae2f6fc4a93e32d65646f932e516461462652f1749854480778c2b8
+  data.tar.gz: ef399db80fa4dc12ace6ccb135fcffc7e83524efeed5d6b11fbe8580826bcbc485cd2201f96d0798f36bad11f113f07498ba0ccf87a8fee8374f501e576b49b5

data/.gitignore

@@ -0,0 +1,4 @@
+*env_setup.sh
+Gemfile.lock
+*.gem
+hh*

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,26 @@
+language: ruby
+sudo: false
+os:
+  - linux
+  - osx
+branches:
+except:
+  - gh-pages
+  - next_doc_release
+env:
+  global:
+    - LOGGING=true
+    - secure: EVku6JcQcp69DksaZUo8Sluk2Mk45CHjs/7w2xDG6n58Px3zxG3i0DSxm5SYBaDvcIQCv+LLt/LyjTRs5t51GbQrFLZGxGLMHDkJ/eS4K2q27P8NGqM/Yj4WtxB7BYQNLQf00eCo9HeTyjY7AsAqZG8wn6W9DUF5Gtu7AUO0aiQ=
+    - secure: GwO5mPCUhaeC1fHuX86T3INRrrpqb2raWS+M83Ko4fnwKUryGiMOVYtzu5NSiTfN2YRHd3gvAsisempqCHndTPvuWcJhSCIGEZdIYgdxXAz4Meb78t0w/5rY7O9BOJ5+Zin098SmB/uiPvW/HDSQw+T+bBHO3ETw+h8VvFDmHM4=
+    - secure: TbguXDUV3wZhZ9ZHGYP2X0zo8gJejeuawmHQ6xey5gZzYrAH92r6wgddQ7KM2Vc+UCw0Gpnuy2AKM6EhEGbRShc5u6LmwAJIqZ6iG48ALOxha6JOoICuSbahRmnNSVOP1utzAXjOB20YO8lao5PGNfqyacFnr3VhqG22Kl5EJhc=
+    - secure: a2wb07pS1Cl6aiKN0tSVjW4Zsc0K0yQ3mI+nrR3KQpF1vaMu43gKYRGLz3NDWcaPrIgKfFyeKjucxd8o52dLn5Mi8OW7hpPWy3d9Y9umx0PpTYHl16tOw2PLpWedqGTM/eZn5Y8UtzrsAdVJKIcCXE7SA28qGmaR5KKuaJjC+3w=
+    - secure: JvH3Nr6N9GbSWDNc8vE0cwDZPQcUgwXB2MjtkPZ/fnfrR/x/cNmCyc2lzZqfExmDNn17YbR2fPP8zKBczEDvY8q03v42LfnpDSCR2F73yJzQe9dxHwACroAT8VgAz3o23ejgry31SzZz+IoUvZ2fzuoZs+uAt1yr48I2V03PqfM=
+rvm:
+  - 1.9.3
+  - jruby-19mode
+  - 2.1
+  - 2.2
+before_install:
+  - gem update --system
+  - gem update bundler
+script: rake ci

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in salesforce_bulk_api.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+(BSD License)
+
+Copyright (c) 2014 Yatish Mehta & GoodData Corporation. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided
+that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and
+       the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions
+       and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the GoodData Corporation nor the names of its contributors may be used to endorse
+       or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,168 @@
+Salesforce Bulk Query
+=====================
+A library for downloading data from Salesforce Bulk API. We only focus on querying, other operations of the API aren't supported. Designed to handle a lot of data.
+
+Derived from [Salesforce Bulk API](https://github.com/yatish27/salesforce_bulk_api)
+
+## Status
+
+[![Gem Version](https://badge.fury.io/rb/salesforce_bulk_query.png)](http://badge.fury.io/rb/salesforce_bulk_query)
+[![Downloads](http://img.shields.io/gem/dt/salesforce_bulk_query.svg)](http://rubygems.org/gems/salesforce_bulk_query)
+[![Dependency Status](https://gemnasium.com/cvengros/salesforce_bulk_query.png)](https://gemnasium.com/cvengros/salesforce_bulk_query)
+[![Code Climate](https://codeclimate.com/github/cvengros/salesforce_bulk_query.png)](https://codeclimate.com/github/cvengros/salesforce_bulk_query)
+[![Build Status](https://travis-ci.org/cvengros/salesforce_bulk_query.png)](https://travis-ci.org/cvengros/salesforce_bulk_query)
+[![Coverage Status](https://coveralls.io/repos/cvengros/salesforce_bulk_query/badge.png)](https://coveralls.io/r/cvengros/salesforce_bulk_query)
+
+## Basic Usage
+To install, run:
+
+    gem install salesforce_bulk_query
+
+or add
+
+    gem salesforce_bulk_query
+
+to your Gemfile.
+
+Before using the library, make sure you have the right account in your Salesforce organization that has access to API and that you won't run out of the [API limits](http://www.salesforce.com/us/developer/docs/api_asynchpre/Content/asynch_api_concepts_limits.htm#batch_proc_time_title). 
+
+You will also need a Salesforce connected app for the `client_id` and `client_sercret` params, see [the guide](https://help.salesforce.com/HTViewHelpDoc?id=connected_app_create.htm&language=en_US). The app needs to have OAuth settings enabled (even if you plan to use just username-password-token authentication). The required permissions are _Access and manage your data (api)_, _Perform requests on your behalf at any time (refresh token, offline access)_. The other parameters such as redirect url don't need to be set.
+
+For doing most of the API calls, the library uses [Restforce](https://github.com/ejholmes/restforce) Code example:
+
+```ruby
+require 'restforce'
+require 'salesforce_bulk_query'
+
+# Create a restforce client instance
+# with basic auth
+restforce = Restforce.new(
+  :username => 'me',
+  :password => 'password',
+  :security_token => 'token',
+  :client_id => "my sfdc app client id",
+  :client_secret => "my sfdc app client secret"
+)
+
+# or OAuth
+restforce = Restforce.new(
+  :refresh_token => "xyz",
+  :client_id => "my sfdc app client id",
+  :client_secret => "my sfdc app client secret"
+)
+
+bulk_api = SalesforceBulkQuery::Api.new(restforce)
+
+# query the api
+result = bulk_api.query("Task", "SELECT Id, Name FROM Task")
+
+# the result is files 
+puts "All the downloaded stuff is in csvs: #{result[:filenames]}"
+
+# query is a blocking call and can take several hours
+# if you want to just start the query asynchronously, use 
+query = start_query("Task", "SELECT Id, Name FROM Task")
+
+# get a coffee
+sleep(1234)
+
+# get what's available and check the status
+results = query.get_available_results
+if results[:succeeded]
+  puts "All the downloaded stuff is in csvs: #{results[:filenames]}"
+else
+  puts "This is going to take a while, get another coffee"
+end
+```
+
+## How it works
+
+The library uses the [Salesforce Bulk API](https://www.salesforce.com/us/developer/docs/api_asynch/index_Left.htm#CSHID=asynch_api_bulk_query.htm|StartTopic=Content%2Fasynch_api_bulk_query.htm|SkinName=webhelp). The given query is divided into 15 subqueries, according to the [limits](http://www.salesforce.com/us/developer/docs/api_asynchpre/Content/asynch_api_concepts_limits.htm#batch_proc_time_title). Each subquery is an interval based on the CreatedDate Salesforce field (date field can be customized). The limits are passed to the API in SOQL queries. Subqueries are sent to the API as batches and added to a job. 
+
+The first interval starts with the date the first Salesforce object was created, we query Salesforce REST API for that. If this query times out, we use a constant. The last interval ends a few minutes before now to avoid consistency issues. Custom start and end can be passed - see Options.
+
+Job has a fixed time limit to process all the subqueries. Batches that finish in time are downloaded to CSVs, batches that don't are divided to 15 subqueries each and added to new jobs.
+
+CSV results are downloaded by chunks, so that we don't run into memory related issues. All other requests are made through the Restforce client that is passed when instantiating the Api class. Restforce is not in the dependencies, so theoretically you can pass another object with the same set of methods as Restforce client.
+
+## Options
+There are a few optional settings you can pass to the `Api` methods:
+* `api_version`: Which Salesforce api version should be used
+* `logger`: Where logs should go, see the example.
+* `filename_prefix`: Prefix applied to csv files.
+* `directory_path`: Custom direcotory path for CSVs, if omitted, a new temp directory is created.
+* `check_interval`: How often the results should be checked in secs. 
+* `time_limit`: Maximum time the query can take. If this time limit is exceeded, available results are downloaded and the list of subqueries that didn't finished is returned. In seconds. The limti should be understood as limit for waiting. When the limit is reached the function downloads data that is ready which can take some additonal time. If no limit is given the query runs until it finishes.
+* `date_field`: Salesforce date field that will be used for splitting the query, and optionally limiting the results. Must be a date field on the queried sobject. Default is `CreatedDate`
+* `date_from`, `date_to`: limits for the `date_field` (`CreatedDate` by default). Note that queries can't contain any WHERE statements as we're doing some manipulations to create subqueries and we don't want things to get too difficult. So this is the way to limit the query yourself. The format is like `"1999-01-01T00:00:00.000Z"`
+* `single_batch`: If true, the queries are not divided into subqueries as described above. Instead one batch job is created with the given query. This is faster for small amount of data, but will fail with a timeout if you have a lot of data. 
+
+See specs for exact usage.
+
+## Logging
+
+```ruby
+require 'logger'
+require 'restforce'
+
+# create the restforce client
+restforce = Restforce.new(...)
+
+# instantiate a logger and pass it to the Api constructor
+logger = Logger.new(STDOUT)
+bulk_api = SalesforceBulkQuery::Api.new(restforce, :logger => logger)
+
+# switch off logging in Restforce so you don't get every message twice
+Restforce.log = false
+```
+
+If you're using Restforce as a client (which you probably are) and you want to do logging, Salesforce Bulk Query will use a custom logging middleware for Restforce. This is because the original logging middleware puts all API responses to log, which is not something you would like to do for a few gigabytes CSVs. When you use the :logger parameter it's recommended you switch off the default logging in Restforce, otherwise you'll get all messages twice. 
+
+## Notes
+
+Query (user given) -> Job (Salesforce construct that encapsulates 15 batches) -> Batch (1 SOQL with Date constraints)
+
+At the beginning the query is divided into 15 subqueries and put into a single job. When one of the subqueries fails, a new job with 15 subqueries is created, the range of the failed query is divided into 15 sub-subqueries.
+
+## Change policy
+
+The gem is trying to follow [semantic versioning](http://semver.org/). All  methods and options described in this readme document are considered public API. If any of these change, at least the minor version is bumped. Methods and their params not described in this document can change even in patches.
+
+## Running tests locally
+Travis CI is set up for this repository to make sure all the tests are passing with each commit.
+
+To run the tests locally:
+
+* Copy the env_setup-example.sh file
+```
+cp  env_setup-example.sh env_setup.sh
+```
+* Setup all the params in env_setup. USERNAME, PASSWORD and TOKEN are your salesforce account credentials. You can get those by [registering for a free developer account](https://developer.salesforce.com/signup). You might need to [reset your security token](https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm) to put it to TOKEN variable. CLIENT_ID and CLIENT_SECRET belong to your Salesforce connected app. You can create one by following the steps outlined in [the tutorial](https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm). Make sure you check the 'api' permission.
+* Run the env_setup
+```
+. env_setup.sh
+```
+* Run the tests
+```
+bundle exec rspec
+```
+
+Note that env_setup.sh is ignored from git in .gitignore so that you don't commit your credentials by accident.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/salesforce_bulk_query/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Run the tests (see above), fix if they fail.
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+Make sure you run all the tests and they pass. If you create a new feature, write a test for it. 
+
+## Copyright
+
+Copyright (c) 2014 Yatish Mehta & GoodData Corporation. See [LICENSE](LICENSE) for details.
+
+
+

data/Rakefile

@@ -0,0 +1,20 @@
+require 'coveralls/rake/task'
+require 'rake/testtask'
+require 'rspec/core/rake_task'
+
+Coveralls::RakeTask.new
+
+task default: %w[ci]
+
+desc 'Run continuous integration test'
+task :ci do
+  Rake::Task['test:unit'].invoke
+  Rake::Task['coveralls:push'].invoke
+end
+
+namespace :test do
+  desc "Run unit tests"
+  RSpec::Core::RakeTask.new(:unit) do |t|
+    t.pattern = 'spec/**/*.rb'
+  end
+end

data/env_setup-example.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# mandatory
+export CLIENT_ID=""
+export CLIENT_SECRET=""
+
+export USERNAME=""
+export PASSWORD=""
+export TOKEN=""
+
+# optional
+export LOGGING='true' # unset to switch off logging
+export ENTITY='Task'

data/lib/salesforce_bulk_query.rb

@@ -0,0 +1,108 @@
+require 'csv'
+
+require 'salesforce_bulk_query/connection'
+require 'salesforce_bulk_query/query'
+require 'salesforce_bulk_query/logger'
+require 'salesforce_bulk_query/utils'
+require 'salesforce_bulk_query/version'
+
+
+# Module where all the stuff is happening
+module SalesforceBulkQuery
+
+  # Abstracts the whole library, class the user interacts with
+  class Api
+    @@DEFAULT_API_VERSION = '29.0'
+
+    Encoding.default_external = Encoding::UTF_8
+    Encoding.default_internal = Encoding::UTF_8
+
+    # Constructor
+    # @param client [Restforce] An instance of the Restforce client, that is used internally to access Salesforce api
+    # @param options
+    def initialize(client, options={})
+      @logger = options[:logger]
+
+      api_version = options[:api_version] || @@DEFAULT_API_VERSION
+      ssl_version = options[:ssl_version]
+      # use our own logging middleware if logger passed
+      if @logger && client.respond_to?(:middleware)
+        client.middleware.use(SalesforceBulkQuery::Logger, @logger, options)
+      end
+
+      # initialize connection
+      @connection = SalesforceBulkQuery::Connection.new(client, api_version, @logger, options[:filename_prefix],ssl_version)
+    end
+
+    # Get the Salesforce instance URL
+    def instance_url
+      # make sure it ends with /
+      url = @connection.client.instance_url
+      url += '/' if url[-1] != '/'
+      return url
+    end
+
+    CHECK_INTERVAL = 30
+
+    # Query the Salesforce API. It's a blocking method - waits until the query is resolved
+    # can take quite some time
+    # @param sobject Salesforce object, e.g. "Opportunity"
+    # @param soql SOQL query, e.g. "SELECT Name FROM Opportunity"
+    # @return hash with :filenames and other useful stuff
+    def query(sobject, soql, options={})
+      @logger.info "Running query: #{soql}. Gem version salesforce_bulk_query: #{SalesforceBulkQuery::VERSION}" if @logger
+      check_interval = options[:check_interval] || CHECK_INTERVAL
+      time_limit = options[:time_limit] # in seconds
+
+      start_time = Time.now
+
+      # start the machinery
+      query = start_query(sobject, soql, options)
+      results = nil
+
+      loop do
+        # get available results and check the status
+        results = query.get_available_results(options)
+        @logger.debug "get_available_results: #{results}"
+
+        # if finished get the result and we're done
+        if results[:succeeded]
+
+          # we're done
+          @logger.info "Query succeeded. Results: #{results}" if @logger
+          break
+        end
+
+        # if we've run out of time limit, go away
+        if time_limit && (Time.now - start_time > time_limit)
+          @logger.warn "Ran out of time limit, downloading what's available and terminating" if @logger
+
+          @logger.info "Downloaded the following files: #{results[:filenames]} The following didn't finish in time: #{results[:unfinished_subqueries]}." if @logger
+          break
+        end
+
+        @logger.info "Sleeping #{check_interval}" if @logger
+        @logger.info "Downloaded files: #{results[:filenames].length} Jobs in progress: #{query.jobs_in_progress.length}"
+        sleep(check_interval)
+      end
+
+      # nice list of files to log
+      if @logger && ! results[:filenames].empty?
+
+        @logger.info "Download finished. Downloaded files in #{File.dirname(results[:filenames][0])}. Filename size [line count]:"
+        @logger.info "\n" + results[:filenames].sort.map{|f| "#{File.basename(f)} #{File.size(f)} #{Utils.line_count(f) if options[:count_lines]}"}.join("\n")
+      end
+      return results
+    end
+
+    # Start the query (synchronous method)
+    # @params see #query
+    # @return Query instance with the running query
+    def start_query(sobject, soql, options={})
+      # create the query, start it and return it
+      query = SalesforceBulkQuery::Query.new(sobject, soql, @connection, {:logger => @logger}.merge(options))
+      query.start(options)
+      return query
+    end
+  end
+end

data/lib/salesforce_bulk_query/batch.rb

@@ -0,0 +1,153 @@
+require 'tmpdir'
+
+require 'salesforce_bulk_query/utils'
+
+
+module SalesforceBulkQuery
+  # Represents a Salesforce api batch. Batch contains a single subquery.
+  # Many batches are contained in a Job.
+  class Batch
+    def initialize(options)
+      @sobject = options[:sobject]
+      @soql = options[:soql]
+      @job_id = options[:job_id]
+      @connection = options[:connection]
+      @date_field = options[:date_field] or fail "date_field must be given when creating a batch"
+      @start = options[:start]
+      @stop = options[:stop]
+      @logger = options[:logger]
+      @@directory_path ||= Dir.mktmpdir
+      @filename = nil
+    end
+
+    attr_reader :soql, :start, :stop, :filename, :fail_message, :batch_id, :csv_record_count
+
+    # Do the api request
+    def create
+      path = "job/#{@job_id}/batch/"
+
+      response_parsed = @connection.post_xml(path, @soql, {:csv_content_type => true})
+
+      @batch_id = response_parsed['id'][0]
+    end
+
+    # check status of the batch
+    # if it fails, don't throw an error now, let the job above collect all fails and raise it at once
+    def check_status
+      succeeded = nil
+      failed = nil
+
+      # get the status of the batch
+      # https://www.salesforce.com/us/developer/docs/api_asynch/Content/asynch_api_batches_get_info.htm
+      status_path = "job/#{@job_id}/batch/#{@batch_id}"
+      status_response = @connection.get_xml(status_path)
+
+      # interpret the status
+      @status = status_response['state'][0]
+
+      # https://www.salesforce.com/us/developer/docs/api_asynch/Content/asynch_api_batches_interpret_status.htm
+      case @status
+        when 'Failed'
+          failed = true
+          @fail_message = status_response['stateMessage']
+        when 'InProgress', 'Queued'
+          succeeded = false
+        when 'Completed'
+          succeeded = true
+          failed = false
+        else
+          fail "Something weird happened, #{@batch_id} has status #{@status}."
+      end
+
+      if succeeded
+        # request to get the result id
+        # https://www.salesforce.com/us/developer/docs/api_asynch/Content/asynch_api_batches_get_results.htm
+        path = "job/#{@job_id}/batch/#{@batch_id}/result"
+
+        response_parsed = @connection.get_xml(path)
+
+        @result_id = response_parsed["result"] ? response_parsed["result"][0] : nil
+      end
+
+      return {
+        :failed => failed,
+        :fail_message => @fail_message,
+        :succeeded => succeeded,
+        :result_id => @result_id
+      }
+    end
+
+    def get_filename
+      return "#{@sobject}_#{@date_field}_#{@start}_#{@stop}_#{@batch_id}.csv"
+    end
+
+    def get_result(options={})
+      # if it was already downloaded, no one should ask about it
+      if @filename
+        raise "This batch was already downloaded once: #{@filename}, #{@batch_id}"
+      end
+
+      directory_path = options[:directory_path]
+      skip_verification = options[:skip_verification]
+
+      # request to get the actual results
+      path = "job/#{@job_id}/batch/#{@batch_id}/result/#{@result_id}"
+
+      if !@result_id
+        raise "batch #{@batch_id} not finished yet, trying to get result: #{path}"
+      end
+
+      directory_path ||= @@directory_path
+
+      # write it to a file
+      @filename = File.join(directory_path, get_filename)
+      @connection.get_to_file(path, @filename)
+
+      # Verify the number of downloaded records is roughly the same as
+      # count on the soql api
+      # maybe also verify
+      @verification = true
+      unless skip_verification
+        verify
+      end
+      @logger.debug "get_result :verification : #{@verification}" if @logger
+      return {
+        :filename => @filename,
+        :verification => @verification
+      }
+    end
+
+    def verify
+      api_count = @connection.query_count(@sobject, @date_field, @start, @stop)
+      # if we weren't able to get the count, fail.
+      if api_count.nil?
+        return @verification = false
+      end
+
+      # count the records in the csv
+      @csv_record_count = Utils.line_count(@filename)
+
+      if @logger && @csv_record_count > 0 && @csv_record_count % 100 == 0
+        @logger.warn "The line count for batch id #{@batch_id} soql #{@soql} is highly suspicious: #{@csv_record_count}"
+      end
+      if @logger &&  (api_count - @csv_record_count).abs > 2
+        @logger.warn "The counts for batch id #{@batch_id}, soql #{@soql} don't match. Record count in downloaded csv #{@csv_record_count}, record count on api count(): #{api_count}"
+        @logger.info "verify result: #{(api_count - @csv_record_count).abs < 2}"
+
+      end
+      @verification = ((api_count - @csv_record_count).abs < 2)
+    end
+
+    def to_log
+      return {
+        :sobject => @sobject,
+        :soql => @soql,
+        :job_id => @job_id,
+        :connection => @connection.to_log,
+        :start => @start,
+        :stop => @stop,
+        :directory_path => @@directory_path
+      }
+    end
+  end
+end