salesforce-sql 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bdbae01f13b8d65322ceee413d9fe5fda6414ae0
+  data.tar.gz: 20dea60817eeeea2686de725bb2ce985304c2615
+SHA512:
+  metadata.gz: 619f77b5a3941790c970aa083441c425f4dc5e66f858e9eaa13b07c134a37c3385fe2de577946dc48f6fd4b888734dfcb18a46b831b9cae57848827876c28a6a
+  data.tar.gz: 19848c066cf11ca33edc1099997a4c059aa27b78271ae61605158286ce1801f18ace3e074c424aed4c2f6e1dd63519d2036b5a8d80f00db658ca7c2e7d0c31ed

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in salesforce-sql.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Juan Breinlinger
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# Salesforce::Sql
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'salesforce-sql'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install salesforce-sql
+
+## Usage
+
+This gem is in a very early stage, please use under your own risk:
+
+copy a table:
+
+
+```
+require 'salesforce/sql'
+
+# Crednetials:
+src_credentials = {
+  :host           => 'test.salesforce.com',
+  :username       => 'user@example.com.source',
+  :password       => 'pass',
+  :client_id      => 'client_id',
+  :client_secret  => 'client_secret'
+}
+
+dst_credentials  = {
+  :host           => 'test.salesforce.com',
+  :username       => 'user@example.com.destination',
+  :password       => 'pass',
+  :client_id      => 'client_id',
+  :client_secret  => 'client_secret'
+}
+
+# Initialize sql objects
+src = Salesforce::Sql::App.new src_credentials
+dst = Salesforce::Sql::App.new dst_credentials
+
+account_ignore_fields = [ 
+  'Status__c',
+  'Other__c',
+]
+
+# Delete current Account content
+dst.delete 'Account'
+
+# Copy src Accounts to Destination
+dst.copy_object source: src, 
+  object: 'Account',
+  ignore_fields: account_ignore_fields
+
+# Delete contact table
+dst.delete 'Contact'
+
+# Copy contacts preserving IDs
+dst.copy_object source: src, 
+  object: 'Contact',
+  object_ids: contact_ids,
+  ignore_fields: contact_ignore_fields,
+  dependencies: [
+    {
+      dependency_object: 'Account',
+      dependency_object_pk: 'Name',
+      object_fk_field: 'AccountId',
+    },
+  ]
+
+```
+
+You could also do partial copy specifying ids:
+
+```
+account_ids = File.read('ids.txt').split("\n").sort.uniq
+dst.copy_object source: src,
+  object: 'Account',
+  object_ids: account_ids,
+  ignore_fields: account_ignore_fields
+
+
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/salesforce-sql/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/salesforce/sql.rb

@@ -0,0 +1,11 @@
+require 'salesforce/sql/version'
+require 'salesforce/sql/app'
+require 'salesforce_bulk'
+require 'restforce'
+require 'terminfo'
+require 'pry'
+
+module Salesforce
+  module Sql
+  end
+end

data/lib/salesforce/sql/app.rb

@@ -0,0 +1,226 @@
+module Salesforce
+  module Sql
+    class App
+
+      attr_accessor :debug
+      attr_accessor :username
+      attr_accessor :step
+
+      def initialize credentials
+
+        # Login credentials
+        @restforce_client = restforce_rest_login credentials
+        @salesforce_bulk_client = salesforce_bulk_login credentials[:username], credentials[:password], credentials[:host]
+        @username = credentials[:username]
+
+        # Default variables:
+        @bulk_api_step = 10000
+        @debug = false
+        @step = 100
+        @default_ignore_fields = [
+          'Id',
+          'IsDeleted',
+          'MasterRecordId',
+          'RecordTypeId',
+          'ParentId',
+          'OwnerId',
+          'CreatedById',
+          'CreatedDate',
+          'LastModifiedDate',
+          'LastModifiedById', 
+          'SystemModstamp',
+          'LastActivityDate',
+        ]
+
+      end
+
+      def get_fields object
+
+        fields = @restforce_client.describe(object)['fields'].select do |field| 
+          field['name'][-4..-1] != '__pc' && field['calculated'] != true  
+        end
+        fields.map {|f| f['name']}
+
+      end
+
+      def query_select_in query, ids = []
+        retval = []
+        if ids.empty?
+          retval = normalize_query @restforce_client.query(query)
+        else
+          (0..(ids.size-1)).step(@step).each do |n|
+
+            # Create the query
+            ids_string = ids[n..(n+@step-1)].map {|i| "'#{i}'"}.join ','
+            stepped_query =  query + " IN (#{ids_string})" 
+
+            # run it and add it to the result
+            retval += normalize_query @restforce_client.query(stepped_query)
+          end
+        end
+
+        retval
+      end
+
+      def query query, ids = []
+
+        retval = []
+        if ids.empty?
+          retval = normalize_query @restforce_client.query(query)
+        else
+          (0..(ids.size-1)).step(@step).each do |n|
+
+            # Create the query
+            ids_string = ids[n..(n+@step-1)].map {|i| "'#{i}'"}.join ','
+            stepped_query =  query + " WHERE Id IN (#{ids_string})" 
+
+            # run it and add it to the result
+            retval += normalize_query @restforce_client.query(stepped_query)
+          end
+        end
+
+        retval
+
+      end
+
+      def delete object
+
+        query = "Select Id FROM #{object}" 
+        bulk_delete_records = normalize_query @restforce_client.query(query)
+
+        print_debug "#{bulk_delete_records.size} #{object} records added to delete on #{self.username}"
+
+        bulk_delete object, bulk_delete_records if !bulk_delete_records.empty?
+
+      end
+
+      # Description: Copy an object from one source to target
+      # Parameters:
+      #   object: The object to copy
+      #   object_ids: An array of object ids to limit the query
+      #   ignore_fields: An array of fields to ignore
+      #   dependencies: When trying to copy a TABLE with DEPENDENCIES
+      #     dependency_object: The DEPENDENCY object name
+      #     dependency_object_pk: the DEPENENCY field to be used as primary key 
+      #     object_fk_field: The foreign key field name in the TABLE
+
+      def copy_object source:, object:, object_ids: [], ignore_fields: [], dependencies: []
+
+        # Remove well known problematic fields and merge them with user requirements:
+        ignore_fields = (ignore_fields + @default_ignore_fields).uniq
+        
+        # Get all the fields from source and destination removing __pc and calculated ones
+        source_object_fields = get_fields object
+        target_object_fields = source.get_fields object
+
+        # Get common fields
+        object_fields = source_object_fields & target_object_fields
+
+        # Get all the records from the source sandbox and store them in bulk_import_records
+        bulk_import_records = source.query "Select #{object_fields.join(',')} FROM #{object}", object_ids
+
+        # Dependencies ID Matching:
+        dependencies.each do |dep|
+
+          # Export the dependency object ids from the source sandbox
+          dependency_ids = bulk_import_records.map{|row| row[dep[:object_fk_field]]}.sort.uniq
+
+          # Use those Ids to get the object records from source including the dependency_object_pk
+          source_object = source.query "Select Id,#{dep[:dependency_object_pk]} FROM #{dep[:dependency_object]}", dependency_ids
+
+          # Get the dependency_object_pk values and export the IDs from the target object
+          dependency_object_pk_values = source_object.map {|row| row[dep[:dependency_object_pk]].gsub("'", %q(\\\'))}
+          target_object = self.query_select_in "Select Id,#{dep[:dependency_object_pk]} FROM #{dep[:dependency_object]} WHERE #{dep[:dependency_object_pk]}", dependency_object_pk_values
+
+          # Now we have source_object and target_object ids and values, we can do the mapping on bulk_import_records
+          source_object.each do |row|
+            source_id = row['Id']
+            target_id = target_object.select{|r| r[dep[:dependency_object_pk]] == row[dep[:dependency_object_pk]]}.first['Id']
+            bulk_import_records.each {|r| r[dep[:object_fk_field]] = target_id if r[dep[:object_fk_field]] == source_id}
+          end
+
+        end
+
+        # Remove ignored fields
+        bulk_import_records.each do |row|
+          ignore_fields.each {|f| row.delete f}
+        end
+
+        print_debug "#{bulk_import_records.size} #{object} records added to import on #{self.username}"
+
+        # Import the data using salesforce_bulk
+        if !bulk_import_records.empty?
+          bulk_insert object, bulk_import_records
+        end
+
+      end
+
+      private
+
+      def bulk_insert object, records
+        (0..(records.size-1)).step(@bulk_api_step).each do |n|
+          job = @salesforce_bulk_client.create object, records[n..n+@bulk_api_step-1]
+          salesforce_bulk_job_status job
+        end
+      end
+
+      def bulk_delete object, records
+        (0..(records.size-1)).step(@bulk_api_step).each do |n|
+          job = @salesforce_bulk_client.delete object, records[n..n+@bulk_api_step-1]
+          salesforce_bulk_job_status job
+        end
+      end
+
+      def normalize_query records
+        records.to_a.each do |row| 
+          row.delete 'attributes'
+          row.delete 'IsPersonAccount'
+        end
+      end
+
+      def delete_ignored_fields set
+        set.to_a.each do |row|
+          @default_ignore_fields.each {|field| row.delete field }
+        end
+      end
+
+      def salesforce_bulk_job_status job
+        sleep 1 while job.check_batch_status.match(/InProgress|Queued/)
+        print_debug "Bulk job status: #{job.check_batch_status}"
+        job.check_batch_status == "Completed" ? true : false
+      end
+
+      def print_debug message
+        puts "DEBUG: #{message[0..TermInfo.screen_size.last]}" if @debug
+      end
+
+      def restforce_rest_login credentials
+        begin
+          client = Restforce.new :host          => credentials[:host],
+                                 :username      => credentials[:username],
+                                 :password      => credentials[:password],
+                                 :client_id     => credentials[:client_id],
+                                 :client_secret => credentials[:client_secret]
+          client.authenticate!
+          client
+        rescue => e
+          puts "Error trying to login rest API using: #{credentials[:username]} "
+          puts e
+          exit 1
+        end
+      end
+
+      def salesforce_bulk_login user, pass, host
+        begin
+          sandbox = host.match(/login/) ? nil : true
+          SalesforceBulk::Api.new(user,pass,sandbox)
+        rescue => e
+          puts "Error trying to login bulk API using: #{user} "
+          puts e
+          exit 1
+        end
+      end
+
+    end
+  end
+end

data/lib/salesforce/sql/version.rb

@@ -0,0 +1,5 @@
+module Salesforce
+  module Sql
+    VERSION = "0.0.1"
+  end
+end

data/salesforce-sql.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'salesforce/sql/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "salesforce-sql"
+  spec.version       = Salesforce::Sql::VERSION
+  spec.authors       = ["Juan Breinlinger"]
+  spec.email         = ["<juan.brein@breins.net>"]
+  spec.summary       = %q{}
+  spec.description   = %q{}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+
+  spec.add_runtime_dependency 'salesforce_bulk'
+  spec.add_runtime_dependency 'restforce'
+  spec.add_runtime_dependency 'pry'
+  spec.add_runtime_dependency 'ruby-terminfo'
+
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: salesforce-sql
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Juan Breinlinger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: salesforce_bulk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: restforce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-terminfo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- "<juan.brein@breins.net>"
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/salesforce/sql.rb
+- lib/salesforce/sql/app.rb
+- lib/salesforce/sql/version.rb
+- salesforce-sql.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: ''
+test_files: []