draisine 0.7.10

data/lib/draisine/util/salesforce_comparisons.rb

@@ -0,0 +1,53 @@
+module Draisine
+  module SalesforceComparisons
+    JSON_TIME_REGEX = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{3})?[+\-]\d{2}:\d{2}\z/
+    EPSILON = 1e-10
+    module_function
+
+    def salesforce_equals?(value, other)
+      value = salesforce_cleanup(value)
+      other = salesforce_cleanup(other)
+      value = salesforce_coerce(value)
+      other = salesforce_coerce(other)
+
+      return fp_equals?(value, other) if numeric?(value) && numeric?(other)
+      return value == other if value.class != other.class
+
+      case value
+      when String
+        normalize_string(value) == normalize_string(other)
+      else
+        value == other
+      end
+    end
+
+    def salesforce_cleanup(value)
+      case value
+      when String
+        value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+      else
+        value
+      end
+    end
+
+    def salesforce_coerce(value)
+      if value.kind_of?(DateTime) || value.kind_of?(Time) || value.kind_of?(Date) ||
+         value =~ JSON_TIME_REGEX
+        value = value.to_time.utc.change(usec: 0)
+      end
+      value
+    end
+
+    def normalize_string(string)
+      string.gsub(/\W/, "")
+    end
+
+    def fp_equals?(value, other)
+      (value - other).abs <= EPSILON
+    end
+
+    def numeric?(value)
+      value.kind_of?(Numeric)
+    end
+  end
+end

data/lib/draisine/version.rb

@@ -0,0 +1,3 @@
+module Draisine
+  VERSION = "0.7.10"
+end

data/lib/draisine.rb

@@ -0,0 +1,48 @@
+require "active_support"
+require "active_support/core_ext"
+require "active_support/concern"
+require "active_job/base"
+
+require "databasedotcom"
+require "ext/databasedotcom"
+
+require "draisine/version"
+require "draisine/setup"
+
+require "draisine/jobs/job_base"
+require "draisine/jobs/inbound_update_job"
+require "draisine/jobs/inbound_delete_job"
+require "draisine/jobs/soap_update_job"
+require "draisine/jobs/soap_delete_job"
+require "draisine/jobs/outbound_create_job"
+require "draisine/jobs/outbound_update_job"
+require "draisine/jobs/outbound_delete_job"
+
+require "draisine/registry"
+require "draisine/ip_checker"
+
+require "draisine/util/hash_diff"
+require "draisine/util/parse_time"
+require "draisine/util/salesforce_comparisons"
+require "draisine/util/caching_client"
+
+require "draisine/query_mechanisms"
+require "draisine/partitioner"
+require "draisine/auditor"
+require "draisine/conflict_detector"
+require "draisine/conflict_resolver"
+require "draisine/syncer"
+require "draisine/type_mapper"
+require "draisine/soap_handler"
+require "draisine/engine"
+require "draisine/importer"
+require "draisine/poller"
+
+require "draisine/concerns/array_setter"
+require "draisine/concerns/attributes_mapping"
+require "draisine/concerns/import"
+require "draisine/active_record"
+
+
+module Draisine
+end

data/lib/ext/databasedotcom.rb

@@ -0,0 +1,98 @@
+module Databasedotcom
+  class Client
+    # Fetches a collection of sobjects with given ids
+    # Useful in conjunction with get_updated / get_deleted calls
+    def fetch_multiple(classname, ids, batch_size = 100, field_list = nil)
+      return [] unless ids.present?
+      klass = find_or_materialize(classname)
+      field_list ||= klass.field_list.split(",")
+      field_list = field_list | ["Id"]
+      field_list = field_list.join(",")
+      ids.in_groups_of(batch_size).flat_map do |ids|
+        query <<-EOQ
+        SELECT #{field_list}
+        FROM #{klass.sobject_name}
+        WHERE id IN (#{ids.map {|id| "'%s'" % id}.join(',')})
+        EOQ
+      end
+    end
+
+    # Returns a list of updated sobject ids for provided date range
+    # reference: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_getupdated.htm?search_text=getUpdated
+    def get_updated(classname, start_date, end_date = Time.current)
+      result = http_get("/services/data/v#{self.version}/sobjects/#{classname}/updated",
+                        start: prepare_date_arg(start_date),
+                        end: prepare_date_arg(end_date))
+      JSON.parse(result.body)
+    rescue Databasedotcom::SalesForceError => e
+      if e.message.include?("is not replicable")
+        {}
+      else
+        raise
+      end
+    end
+
+    def get_updated_ids(classname, start_date, end_date = Time.current)
+      get_updated(classname, start_date, end_date).fetch("ids", [])
+    end
+
+    # Returns a list of updated sobject ids for provided date range
+    # reference: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_getdeleted.htm?search_text=getUpdated
+    def get_deleted(classname, start_date, end_date = Time.current)
+      result = http_get("/services/data/v#{self.version}/sobjects/#{classname}/deleted",
+                        start: prepare_date_arg(start_date),
+                        end: prepare_date_arg(end_date))
+      JSON.parse(result.body)
+    rescue Databasedotcom::SalesForceError => e
+      if e.message.include?("is not replicable")
+        {}
+      else
+        raise
+      end
+    end
+
+    def get_deleted_ids(classname, start_date, end_date = Time.current)
+      get_deleted(classname, start_date, end_date).fetch("deletedRecords", []).map {|r| r["id"] }
+    end
+
+    def count(classname)
+      query("SELECT COUNT() FROM #{classname}").total_size
+    end
+
+    protected
+
+    def prepare_date_arg(date)
+      return date.to_time.iso8601 if date.respond_to?(:to_time)
+      date
+    end
+
+    def find_or_materialize(class_or_classname)
+      if class_or_classname.is_a?(Class)
+        clazz = class_or_classname
+      else
+        match = class_or_classname.match(/(?:(.+)::)?(\w+)$/)
+        preceding_namespace = match[1]
+        classname = match[2]
+        raise ArgumentError if preceding_namespace && preceding_namespace != module_namespace.name
+        clazz = module_namespace.const_get(classname.to_sym, false) rescue nil # fix inherited namespace
+        clazz ||= self.materialize(classname)
+      end
+      clazz
+    end
+  end
+
+  module Sobject
+    module NumericFieldsExtensions
+      def register_field(name, field)
+        super
+        %w[scale precision length byteLength].each do |attr|
+          self.type_map[name][attr.underscore.to_sym] = field[attr] if field[attr]
+        end
+      end
+    end
+
+    class <<Sobject
+      prepend NumericFieldsExtensions
+    end
+  end
+end

data/lib/generators/draisine/delta_migration_generator.rb

@@ -0,0 +1,77 @@
+require 'rails/generators/active_record'
+
+module Draisine
+  class DeltaMigrationGenerator < Rails::Generators::Base
+    self.source_root(File.expand_path('../templates/', __FILE__))
+
+    desc <<-DESC
+    Generates a migration for missing columns for given model
+    DESC
+
+    argument :model,
+      type: :string,
+      banner: "ModelName",
+      desc: "Model name for active record (e.g. Lead)",
+      required: true
+
+    argument :salesforce_object_name,
+      type: :string,
+      banner: "SalesforceObjectName",
+      desc: "Salesforce object name (inferred from ModelName by default)",
+      required: false
+
+    class_option :fields,
+      type: :array,
+      banner: "Field1,Field2",
+      desc: "Fields to generate (would otherwise generate all missing fields)",
+      required: false
+
+    def generate
+      @model_name = model.classify.singularize
+      @model_file = "app/models/#{model.underscore.singularize}.rb"
+      @table_name = model.underscore.gsub("/", "_").pluralize
+      @migration_title = "DeltaUpdate#{model.classify.gsub('::', '').pluralize}#{migration_uid}"
+      @migration_file = existing_migration_name(@migration_title) ||
+                        "db/migrate/#{migration_number}_#{@migration_title.underscore}.rb"
+      @fields = (options["fields"] || []).map(&:downcase)
+      @salesforce_object_name = salesforce_object_name || model.classify.demodulize
+      @materialized_model = Draisine.salesforce_client.materialize(@salesforce_object_name)
+      @mapper = Draisine::TypeMapper.new(@materialized_model.type_map)
+      @existing_columns = ActiveRecord::Base.connection
+        .columns(@table_name)
+        .each_with_object({}) {|col, h| h[col.name] = Draisine::TypeMapper::ActiveRecordColumnDef.from_ar_column(col) }
+      @new_ar_col_defs = @mapper.active_record_column_defs
+        .reject {|col_def| @existing_columns.key?(col_def.column_name) }
+        .select {|col_def| column_in_required_fields?(col_def.column_name) }
+      @changed_ar_col_defs = @mapper.active_record_column_defs
+        .select {|col_def| @existing_columns.key?(col_def.column_name) }
+        .select {|col_def| different_column?(@existing_columns[col_def.column_name], col_def) }
+        .select {|col_def| column_in_required_fields?(col_def.column_name) }
+      template "delta_migration.rb", @migration_file
+    end
+
+    protected
+
+    def migration_number
+      Time.current.utc.strftime("%Y%m%d%H%M%S")
+    end
+
+    def migration_uid
+      Time.current.utc.strftime("%Y%m%d")
+    end
+
+    def column_in_required_fields?(column)
+      @fields.empty? || @fields.include?(column.to_s.downcase)
+    end
+
+    # This is needed for rails destroy to work, since every time we generate a new name for our migration
+    def existing_migration_name(migration_title)
+      Dir.glob("#{Rails.root}/db/migrate/[0-9]*_#{migration_title.underscore}.rb").first
+    end
+
+    def different_column?(first, second)
+      first.column_type != second.column_type ||
+        (first.options[:limit] && second.options[:limit] && first.options[:limit] != second.options[:limit])
+    end
+  end
+end

data/lib/generators/draisine/integration_generator.rb

@@ -0,0 +1,53 @@
+require 'rails/generators/active_record'
+
+module Draisine
+  class IntegrationGenerator < Rails::Generators::Base
+    self.source_root(File.expand_path('../templates/', __FILE__))
+
+    desc <<-DESC
+    Uses materialized models from salesforce.
+    Generates a migration creating an integration table for a salesforce model and
+    an active_record model under app/models/salesforce/
+    DESC
+
+    argument :model, type: :string,
+                     banner: "ModelName",
+                     desc: "Model name for active record (e.g. Lead)",
+                     required: true
+    argument :salesforce_object_name, type: :string,
+                     banner: "SalesforceObjectName",
+                     desc: "Salesforce object name (inferred from ModelName by default)",
+                     required: false
+    def create_salesforce_integrations
+      @model_name = model.classify.singularize
+      @model_file = "app/models/#{model.underscore.singularize}.rb"
+      @table_name = model.underscore.gsub("/", "_").pluralize
+      @migration_title = "CreateSalesforce#{model.classify.gsub('::', '').pluralize}"
+      @migration_file = existing_migration_name(@migration_title) ||
+                        "db/migrate/#{migration_number}_#{@migration_title.underscore}.rb"
+
+      @salesforce_object_name = salesforce_object_name || model.classify.demodulize
+      @materialized_model = Draisine.salesforce_client.materialize(@salesforce_object_name)
+      @mapper = Draisine::TypeMapper.new(@materialized_model.type_map)
+      @ar_col_defs = @mapper.active_record_column_defs
+      @columns = @mapper.active_record_column_defs.map(&:column_name)
+      @serialized_columns = @mapper.serialized_columns
+      @array_columns = @mapper.array_columns
+      @non_audited_columns = @mapper.columns - @mapper.updateable_columns
+
+      template "migration.rb", @migration_file
+      template "model.rb", @model_file
+    end
+
+    protected
+
+    def migration_number
+      Time.current.utc.strftime("%Y%m%d%H%M%S")
+    end
+
+    # This is needed for rails destroy to work, since every time we generate a new name for our migration
+    def existing_migration_name(migration_title)
+      Dir.glob("#{Rails.root}/db/migrate/[0-9]*_#{migration_title.underscore}.rb").first
+    end
+  end
+end

data/lib/generators/draisine/templates/delta_migration.rb

@@ -0,0 +1,24 @@
+class <%= @migration_title %> < ActiveRecord::Migration
+  def up
+    change_table :<%= @table_name %>, bulk: true do |t|
+      <%- @new_ar_col_defs.each do |col_def| -%>
+      t.column :<%= col_def.column_name %>, :<%= col_def.column_type %>, <%= col_def.options.inspect %>
+      <%- end -%>
+      <%- @changed_ar_col_defs.each do |col_def| -%>
+      t.change :<%= col_def.column_name %>, :<%= col_def.column_type %>, <%= col_def.options.inspect %>
+      <%- end -%>
+    end
+  end
+
+  def down
+    change_table :<%= @table_name %>, bulk: true do |t|
+      <%- @new_ar_col_defs.each do |col_def| -%>
+      t.remove :<%= col_def.column_name %>
+      <%- end -%>
+      <%- @changed_ar_col_defs.each do |col_def| -%>
+      <%- ex_col = @existing_columns[col_def.column_name] -%>
+      t.change :<%= ex_col.column_name %>, :<%= ex_col.column_type %>, <%= ex_col.options.inspect %>
+      <%- end -%>
+    end
+  end
+end

data/lib/generators/draisine/templates/migration.rb

@@ -0,0 +1,21 @@
+class <%= @migration_title %> < ActiveRecord::Migration
+  def up
+    create_table :<%= @table_name %> do |t|
+      <%- @ar_col_defs.each do |col_def| -%>
+      t.column :<%= col_def.column_name %>, :<%= col_def.column_type %>, <%= col_def.options.inspect %>
+      <%- end -%>
+
+      t.timestamps
+      t.index :updated_at
+
+      # These are draisine-specific columns
+      t.column :salesforce_id, :string, :limit => 18
+      t.column :salesforce_updated_at, :datetime
+      t.index :salesforce_id, :unique => true
+    end
+  end
+
+  def down
+    drop_table :<%= @table_name %>
+  end
+end

data/lib/generators/draisine/templates/model.rb

@@ -0,0 +1,11 @@
+class <%= @model_name %> < ActiveRecord::Base
+  self.table_name = <%= @table_name.inspect %>
+  <%- @serialized_columns.each do |col| -%>
+  serialize :<%= col %>, JSON
+  <%- end -%>
+
+  salesforce_syncable synced_attributes: <%= @columns.map(&:to_sym).inspect %>,
+      non_audited_attributes: <%= @non_audited_columns.map(&:to_sym).inspect %>,
+      array_attributes: <%= @array_columns.map(&:to_sym).inspect %>,
+      salesforce_object_name: <%= @salesforce_object_name.inspect %>
+end

data/salesforce/sample_delete_trigger.apex

@@ -0,0 +1,7 @@
+trigger Create_Deleted_Object on Lead (after delete) {
+  List <Lead> leads = Trigger.old;
+  for (Lead lead: leads) {
+    Deleted_Object__c delObj = new Deleted_Object__c(Object_Id__c = lead.Id, Object_Type__c = 'Salesforce::Lead');
+    insert delObj;
+  }
+}

data/salesforce/sample_test_class_for_delete_trigger.apex

@@ -0,0 +1,15 @@
+@isTest(seeAllData=false)
+
+private class CreateDeletedObjectAfterLeadDeleteTest {
+  static testMethod void testCreatedDeletedObject() {
+    Test.startTest();
+    Lead lead = new Lead(FirstName = 'John', LastName = 'Snow');
+    insert lead;
+    System.assertEquals([SELECT COUNT() FROM Deleted_Object__c], 0);
+    String id = lead.Id;
+    delete lead;
+    System.assertEquals([SELECT COUNT() FROM Deleted_Object__c], 1);
+    System.assertEquals([SELECT Object_Id__c FROM Deleted_Object__c LIMIT 1].Object_Id__c, id);
+    Test.stopTest();
+  }
+}

metadata

@@ -0,0 +1,242 @@
+--- !ruby/object:Gem::Specification
+name: draisine
+version: !ruby/object:Gem::Version
+  version: 0.7.10
+platform: ruby
+authors:
+- Mark Abramov
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: databasedotcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Bidirectional synchronization for salesforce / activerecord
+email:
+- mark.abramov@chloeandisabel.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/controllers/draisine/soap_controller.rb
+- bin/console
+- bin/setup
+- config/routes.rb
+- draisine.gemspec
+- lib/draisine.rb
+- lib/draisine/active_record.rb
+- lib/draisine/auditor.rb
+- lib/draisine/auditor/result.rb
+- lib/draisine/concerns/array_setter.rb
+- lib/draisine/concerns/attributes_mapping.rb
+- lib/draisine/concerns/import.rb
+- lib/draisine/conflict_detector.rb
+- lib/draisine/conflict_resolver.rb
+- lib/draisine/engine.rb
+- lib/draisine/importer.rb
+- lib/draisine/ip_checker.rb
+- lib/draisine/jobs/inbound_delete_job.rb
+- lib/draisine/jobs/inbound_update_job.rb
+- lib/draisine/jobs/job_base.rb
+- lib/draisine/jobs/outbound_create_job.rb
+- lib/draisine/jobs/outbound_delete_job.rb
+- lib/draisine/jobs/outbound_update_job.rb
+- lib/draisine/jobs/soap_delete_job.rb
+- lib/draisine/jobs/soap_update_job.rb
+- lib/draisine/partitioner.rb
+- lib/draisine/poller.rb
+- lib/draisine/query_mechanisms.rb
+- lib/draisine/query_mechanisms/base.rb
+- lib/draisine/query_mechanisms/default.rb
+- lib/draisine/query_mechanisms/last_modified_date.rb
+- lib/draisine/query_mechanisms/system_modstamp.rb
+- lib/draisine/registry.rb
+- lib/draisine/setup.rb
+- lib/draisine/soap_handler.rb
+- lib/draisine/syncer.rb
+- lib/draisine/type_mapper.rb
+- lib/draisine/util/caching_client.rb
+- lib/draisine/util/hash_diff.rb
+- lib/draisine/util/parse_time.rb
+- lib/draisine/util/salesforce_comparisons.rb
+- lib/draisine/version.rb
+- lib/ext/databasedotcom.rb
+- lib/generators/draisine/delta_migration_generator.rb
+- lib/generators/draisine/integration_generator.rb
+- lib/generators/draisine/templates/delta_migration.rb
+- lib/generators/draisine/templates/migration.rb
+- lib/generators/draisine/templates/model.rb
+- salesforce/sample_delete_trigger.apex
+- salesforce/sample_test_class_for_delete_trigger.apex
+homepage: https://github.com/chloeandisabel/draisine
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Synchronization machinery for salesforce
+test_files: []