salesforce_bulk_query-edge 0.2.1

data/lib/salesforce_bulk_query/utils.rb

@@ -0,0 +1,16 @@
+require 'csv'
+
+module SalesforceBulkQuery
+  class Utils
+    # record count if they want to
+    def self.line_count(f)
+      i = 0
+      CSV.foreach(f, :headers => true,:encoding => "UTF-8") {|_| i += 1}
+      i
+    end
+
+    def self.header(f)
+      File.open(f, &:readline).split(',').map{ |c| c.strip.delete('"') }
+    end
+  end
+end

data/lib/salesforce_bulk_query/version.rb

@@ -0,0 +1,3 @@
+module SalesforceBulkQuery
+  VERSION = '0.2.1'
+end

data/new-version.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# get the new version
+VERSION=`bundle exec ruby <<-EORUBY
+
+  require 'salesforce_bulk_query'
+  puts SalesforceBulkQuery::VERSION
+
+EORUBY`
+
+# create tag and push it
+TAG="v$VERSION"
+git tag $TAG
+git push origin $TAG
+
+# build and push the gem
+gem build salesforce_bulk_query.gemspec
+gem push "salesforce_bulk_query-$VERSION.gem"
+
+# update the gem after a few secs
+sleep 30
+gem update salesforce_bulk_query

data/salesforce_bulk_query.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib/", __FILE__)
+require "salesforce_bulk_query/version"
+
+Gem::Specification.new do |s|
+  s.name = 'salesforce_bulk_query-edge'
+  s.version = SalesforceBulkQuery::VERSION
+  s.authors = ['Petr Cvengros']
+  s.email = ['petr.cvengros@gooddata.com']
+
+  s.required_ruby_version = '>= 1.9'
+
+  s.homepage = 'https://github.com/cvengros/salesforce_bulk_query'
+  s.summary = %q{Downloading data from Salesforce Bulk API made easy and scalable.}
+  s.description = %q{A library for downloading data from Salesforce Bulk API. We only focus on querying, other operations of the API aren't supported. Designed to handle a lot of data.}
+  s.license = 'BSD'
+
+  s.add_dependency 'json', '<= 2'
+  s.add_dependency 'xml-simple', '~> 1.1'
+
+  s.add_development_dependency 'multi_json', '~> 1.9'
+  s.add_development_dependency 'restforce', '~>1.4'
+  s.add_development_dependency 'rspec', '~>2.14'
+  s.add_development_dependency 'pry', '~>0.9'
+  s.add_development_dependency 'pry-stack_explorer', '~>0.4' if RUBY_PLATFORM != 'java'
+  s.add_development_dependency 'rake', '~> 10.3'
+  s.add_development_dependency 'coveralls', '~> 0.7'
+  s.add_development_dependency 'webmock', '~> 1.20'
+
+  s.files = `git ls-files`.split($/)
+  s.require_paths = ['lib']
+
+  s.rubygems_version = "1.3.7"
+end

data/spec/salesforce_bulk_query_spec.rb

@@ -0,0 +1,227 @@
+require 'multi_json'
+require 'csv'
+require 'tmpdir'
+require 'logger'
+require 'set'
+require 'salesforce_bulk_query'
+require 'restforce'
+require 'webmock/rspec'
+
+class Helper
+  DEFAULT_API_VERSION = '30.0'
+  def self.create_default_restforce
+    Restforce.new(
+      :username => ENV['USERNAME'],
+      :password => ENV['PASSWORD'],
+      :security_token => ENV['TOKEN'],
+      :client_id => ENV['CLIENT_ID'],
+      :client_secret => ENV['CLIENT_SECRET'],
+      :api_version => api_version
+    )
+  end
+
+  def self.api_version
+    ENV['API_VERSION'] || DEFAULT_API_VERSION
+  end
+
+  def self.create_default_api(restforce)
+    # switch off the normal logging
+    Restforce.log = false
+
+    SalesforceBulkQuery::Api.new(restforce,
+      :api_version => ENV['API_VERSION'] || DEFAULT_API_VERSION,
+      :logger => ENV['LOGGING'] ? Logger.new(STDOUT): nil
+    )
+  end
+end
+
+# test co nejak nafakuje tu situaci v twc
+describe SalesforceBulkQuery do
+  before :all do
+    WebMock.allow_net_connect!
+
+    @client = Helper.create_default_restforce
+    @api = Helper.create_default_api(@client)
+    @entity = ENV['ENTITY'] || 'Opportunity'
+    @field_list = (ENV['FIELD_LIST'] || "Id,CreatedDate").split(',')
+    @api_version = Helper.api_version
+  end
+
+  describe "instance_url" do
+    it "gives you some reasonable url" do
+      url = @api.instance_url
+      url.should_not be_empty
+      url.should match(/salesforce\.com\//)
+    end
+  end
+
+  describe "query" do
+    context "if you give it an invalid SOQL" do
+      it "fails with argument error" do
+        expect{@api.query(@entity, "SELECT Id, SomethingInvalid FROM #{@entity}")}.to raise_error(ArgumentError)
+      end
+    end
+    context "when you give it no options" do
+      it "downloads the data to a few files", :constraint => 'slow'  do
+        result = @api.query(@entity, "SELECT #{@field_list.join(', ')} FROM #{@entity}", :count_lines => true)
+        filenames = result[:filenames]
+        filenames.should have_at_least(2).items
+        result[:jobs_done].should_not be_empty
+
+        # no duplicate filenames
+        expect(Set.new(filenames).length).to eq(filenames.length)
+
+        filenames.each do |filename|
+          File.size?(filename).should be_true
+
+          lines = CSV.read(filename)
+
+          if lines.length > 1
+            # first line should be the header
+            lines[0].should eql(@field_list)
+
+            # first id shouldn't be emtpy
+            lines[1][0].should_not be_empty
+          end
+        end
+      end
+    end
+    context "when we want to mock things" do
+      before(:each) do
+        WebMock.allow_net_connect!
+      end
+      after(:each) do
+        WebMock.allow_net_connect!
+      end
+      it "catches the timeout error for query" do
+        # stub the timeout on query
+        host = URI.parse(@api.instance_url).host
+        query_url = "#{host}/services/data/v#{@api_version}/query"
+        query_regexp = Regexp.new(query_url)
+        # 4 timeouts (first get the oldest record), then fake a
+        # 0 count query response
+        stub_request(:get, query_regexp).to_timeout.times(4).then.to_return(
+          :body => "{\"totalSize\":0,\"done\":true,\"records\":[]}",
+          :headers => {
+            "date"=>"Wed, 04 Feb 2015 01:18:45 GMT",
+            "set-cookie"=>"BrowserId=hahaha;Path=/;Domain=.salesforce.com;Expires=never",
+            "expires"=>"Thu, 01 Jan 1970 00:00:00 GMT",
+            "sforce-limit-info"=>"api-usage=6666/15000",
+            "content-type"=>"application/json;charset=UTF-8",
+            "transfer-encoding"=>"chunked"}
+        )
+
+        # do the actual request
+        WebMock.allow_net_connect!
+        result = @api.query(
+          @entity,
+          "SELECT #{@field_list.join(', ')} FROM #{@entity}",
+          :count_lines => true,
+          :single_batch => true
+        )
+
+        # check it
+        expect(result[:succeeded]).to be_true
+        expect(result[:unfinished_subqueries]).to be_empty
+        expect(result[:filenames]).not_to be_empty
+        expect(result[:jobs_done]).not_to be_empty
+      end
+    end
+    context "when you give it all the options" do
+      it "downloads a single file" do
+        tmp = Dir.mktmpdir
+        frm = "2000-01-01"
+        from = "#{frm}T00:00:00.000Z"
+        t = "2020-01-01"
+        to = "#{t}T00:00:00.000Z"
+        field = 'SystemModstamp'
+        result = @api.query(
+          "Account",
+          "SELECT Id, Name, Industry, Type FROM Account",
+          :check_interval => 30,
+          :directory_path => tmp,
+          :date_from => from,
+          :date_to => to,
+          :single_batch => true,
+          :count_lines => true,
+          :date_field => field
+        )
+
+        result[:filenames].should have(1).items
+        result[:jobs_done].should_not be_empty
+
+        filename = result[:filenames][0]
+
+        File.size?(filename).should be_true
+        lines = CSV.read(filename)
+
+        # first line should be the header
+        lines[0].should eql(["Id", "Name", "Industry", "Type"])
+
+        # first id shouldn't be emtpy
+        lines[1][0].should_not be_empty
+
+        filename.should match(tmp)
+        filename.should match(frm)
+        filename.should match(t)
+        filename.should match(field)
+      end
+    end
+    context "when you give it a bad date_field" do
+      it "fails with argument error with no from date" do
+        expect{@api.query(@entity, "SELECT Id, CreatedDate FROM #{@entity}", :date_field => 'SomethingInvalid')}.to raise_error(ArgumentError)
+      end
+      it "fails with argument error with given from date" do
+        from = "2000-01-01T00:00:00.000Z"
+        expect{
+          @api.query(
+            @entity,
+            "SELECT Id, CreatedDate FROM #{@entity}",
+            :date_field => 'SomethingInvalid',
+            :date_from => from
+          )
+        }.to raise_error(ArgumentError)
+      end
+
+    end
+    context "when you give it a short time limit" do
+      it "downloads some stuff is unfinished" do
+        result = @api.query(
+          "Opportunity",
+          "SELECT Id, Name, CreatedDate FROM Opportunity",
+          :time_limit => 15
+        )
+        # one of them should be non-empty
+        expect((! result[:unfinished_subqueries].empty?) || (! result[:filenames].empty?)).to eq true
+      end
+    end
+    context "when you pass a short job time limit" do
+      it "creates quite a few jobs quickly", :skip => true do
+        # development only
+        result = @api.query(
+          @entity,
+          "SELECT Id, CreatedDate FROM #{@entity}",
+          :count_lines => true,
+          :job_time_limit => 60
+        )
+        require 'pry'; binding.pry
+      end
+    end
+  end
+
+  describe "start_query" do
+    it "starts a query that finishes some time later" do
+      query = @api.start_query("Opportunity",  "SELECT Id, Name, CreatedDate FROM Opportunity", :single_batch => true)
+
+      # get a cofee
+      sleep(60*2)
+
+      # check the status
+      result = query.get_available_results
+      expect(result[:succeeded]).to eq true
+      result[:filenames].should have_at_least(1).items
+      result[:jobs_done].should_not be_empty
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'coveralls'
+Coveralls.wear!
+
+RSpec.configure do |c|
+  c.filter_run :focus => true
+  c.run_all_when_everything_filtered = true
+  c.filter_run_excluding :skip => true
+end
+

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: salesforce_bulk_query-edge
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+platform: ruby
+authors:
+- Petr Cvengros
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: restforce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+description: A library for downloading data from Salesforce Bulk API. We only focus
+  on querying, other operations of the API aren't supported. Designed to handle a
+  lot of data.
+email:
+- petr.cvengros@gooddata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- env_setup-example.sh
+- lib/salesforce_bulk_query.rb
+- lib/salesforce_bulk_query/batch.rb
+- lib/salesforce_bulk_query/connection.rb
+- lib/salesforce_bulk_query/job.rb
+- lib/salesforce_bulk_query/logger.rb
+- lib/salesforce_bulk_query/query.rb
+- lib/salesforce_bulk_query/utils.rb
+- lib/salesforce_bulk_query/version.rb
+- new-version.sh
+- salesforce_bulk_query.gemspec
+- spec/salesforce_bulk_query_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/cvengros/salesforce_bulk_query
+licenses:
+- BSD
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Downloading data from Salesforce Bulk API made easy and scalable.
+test_files: []
+has_rdoc: