draisine 0.7.10

data/lib/draisine/poller.rb

@@ -0,0 +1,101 @@
+module Draisine
+  class Poller
+    Result = Struct.new(:created_count, :updated_count, :deleted_count)
+
+    class <<self
+      def run(model_class:, mechanism: :default, start_date:, end_date: Time.current, **run_args)
+        partitions = partition(
+          model_class: model_class,
+          mechanism: mechanism,
+          start_date: start_date,
+          end_date: end_date,
+          partition_size: 10**12)
+        run_partition(partitions.first, **run_args)
+      end
+      alias_method :poll, :run
+
+      def partition(model_class:, mechanism: :default, start_date:, end_date: Time.current, partition_size: 100)
+        Partitioner.partition(
+          model_class: model_class,
+          mechanism: mechanism,
+          start_date: start_date,
+          end_date: end_date,
+          partition_size: partition_size)
+      end
+
+      def run_partition(partition, **run_args)
+        new(partition).run(**run_args)
+      end
+      alias_method :poll_partition, :run_partition
+    end
+
+
+    attr_reader :partition, :model_class, :start_date, :end_date
+    def initialize(partition)
+      @partition = partition
+      @model_class = partition.model_class
+      @start_date = partition.start_date
+      @end_date = partition.end_date
+    end
+
+    def run(import_created: true, import_updated: false, import_deleted: true)
+      created_count = updated_count = deleted_count = 0
+      if import_created || import_updated
+        created_count, updated_count = import_changes(import_created, import_updated)
+      end
+
+      deleted_count = import_deletes if import_deleted
+
+      Result.new(
+        created_count,
+        updated_count,
+        deleted_count)
+    end
+    alias_method :poll, :run
+
+    protected
+
+    def import_changes(import_created, import_updated)
+      updated_ids = partition.updated_ids
+      return [0, 0] unless updated_ids.present?
+
+      created_count = updated_count = 0
+      changed_objects = client.fetch_multiple(salesforce_object_name, updated_ids)
+
+      existing_models = model_class
+        .where(salesforce_id: updated_ids)
+        .each_with_object({}) { |model, rs| rs[model.salesforce_id] = model }
+
+      changed_objects.each do |object|
+        id = object.attributes.fetch('Id')
+        model = existing_models[id]
+        is_new = !model
+        attrs = object.attributes
+        if is_new && import_created
+          model_class.import_or_update_with_attrs(id, attrs)
+          created_count += 1
+        elsif !is_new && import_updated
+          if model.salesforce_update_without_sync(attrs, true)
+            updated_count += 1
+          end
+        end
+      end
+
+      [created_count, updated_count]
+    end
+
+    def import_deletes
+      deleted_ids = partition.deleted_ids
+      return 0 unless deleted_ids.present?
+      model_class.where(salesforce_id: deleted_ids).delete_all
+    end
+
+    def client
+      Draisine.salesforce_client
+    end
+
+    def salesforce_object_name
+      model_class.salesforce_object_name
+    end
+  end
+end

data/lib/draisine/query_mechanisms/base.rb

@@ -0,0 +1,15 @@
+module Draisine
+  module QueryMechanisms
+    class Base
+      attr_reader :model_class, :client
+      def initialize(model_class, client = Draisine.salesforce_client)
+        @model_class = model_class
+        @client = client
+      end
+
+      def salesforce_object_name
+        model_class.salesforce_object_name
+      end
+    end
+  end
+end

data/lib/draisine/query_mechanisms/default.rb

@@ -0,0 +1,13 @@
+module Draisine
+  module QueryMechanisms
+    class Default < Base
+      def get_updated_ids(start_date, end_date)
+        client.get_updated_ids(salesforce_object_name, start_date, end_date)
+      end
+
+      def get_deleted_ids(start_date, end_date)
+        client.get_deleted_ids(salesforce_object_name, start_date, end_date)
+      end
+    end
+  end
+end

data/lib/draisine/query_mechanisms/last_modified_date.rb

@@ -0,0 +1,18 @@
+module Draisine
+  module QueryMechanisms
+    class LastModifiedDate < Base
+      def get_updated_ids(start_date, end_date)
+        response = client.query <<-EOQ
+        SELECT Id FROM #{salesforce_object_name}
+        WHERE LastModifiedDate >= #{start_date.iso8601}
+        AND LastModifiedDate <= #{end_date.iso8601}
+        EOQ
+        response.map(&:Id)
+      end
+
+      def get_deleted_ids(start_date, end_date)
+        []
+      end
+    end
+  end
+end

data/lib/draisine/query_mechanisms/system_modstamp.rb

@@ -0,0 +1,18 @@
+module Draisine
+  module QueryMechanisms
+    class SystemModstamp < Base
+      def get_updated_ids(start_date, end_date)
+        response = client.query <<-EOQ
+        SELECT Id FROM #{salesforce_object_name}
+        WHERE SystemModstamp >= #{start_date.iso8601}
+        AND SystemModstamp <= #{end_date.iso8601}
+        EOQ
+        response.map(&:Id)
+      end
+
+      def get_deleted_ids(start_date, end_date)
+        []
+      end
+    end
+  end
+end

data/lib/draisine/query_mechanisms.rb

@@ -0,0 +1,18 @@
+module Draisine
+  module QueryMechanisms
+    require "draisine/query_mechanisms/base"
+    require "draisine/query_mechanisms/default"
+    require "draisine/query_mechanisms/system_modstamp"
+    require "draisine/query_mechanisms/last_modified_date"
+
+    MAP = {
+      default: Default,
+      system_modstamp: SystemModstamp,
+      last_modified_date: LastModifiedDate
+    }
+
+    def self.fetch(name)
+      MAP.fetch(name)
+    end
+  end
+end

data/lib/draisine/registry.rb

@@ -0,0 +1,22 @@
+module Draisine
+  class Registry
+    attr_reader :models
+
+    def initialize
+      @models = {}
+    end
+
+    def find(name)
+      models.fetch(name)
+    end
+
+    def register(model, name)
+      models[name] = model
+      models[model.name] ||= model
+    end
+  end
+
+  def self.registry
+    @registry ||= Registry.new
+  end
+end

data/lib/draisine/setup.rb

@@ -0,0 +1,97 @@
+require "logger"
+
+module Draisine
+  def self.salesforce_client=(client)
+    @salesforce_client = client
+  end
+
+  def self.salesforce_client
+    unless @salesforce_client
+      fail <<-EOM
+DatabaseDotcom client was not properly set up. You can set it up as follows:
+sf_client = Databasedotcom::Client.new("config/databasedotcom.yml")
+sf_client.authenticate :username => <username>, :password => <password>
+Draisine.salesforce_client = sf_client
+EOM
+    end
+    @salesforce_client
+  end
+
+  def self.organization_id
+    unless @organization_id
+      fail <<-EOM
+Draisine.organization_id was not properly set up.
+You can use Draisine.organization_id= method to set it.
+See https://cloudjedi.wordpress.com/no-fuss-salesforce-id-converter/ if
+you need to convert your 15-char id into 18-char.
+EOM
+    end
+    @organization_id
+  end
+
+  def self.organization_id=(id)
+    unless id.kind_of?(String) && id.length == 18
+      fail ArgumentError, "You should set organization id to an 18 character string"
+    end
+    @organization_id = id
+  end
+
+  def self.job_error_handler
+    @job_error_handler ||= proc {|error, job_instance, args| raise error }
+  end
+
+  def self.job_error_handler=(handler)
+    @job_error_handler = handler
+  end
+
+  def self.sync_callback
+    @sync_callback ||= proc {|type, salesforce_id, options| }
+  end
+
+  def self.sync_callback=(callback)
+    @sync_callback = callback
+  end
+
+  def self.job_retry_attempts
+    @job_retry_attempts ||= 0
+  end
+
+  def self.sync_soap_operations?
+    @sync_soap_operations = true if @sync_soap_operations.nil?
+    @sync_soap_operations
+  end
+
+  def self.sync_soap_operations=(value)
+    @sync_soap_operations = value
+  end
+
+  def self.job_retry_attempts=(count)
+    @job_retry_attempts = count
+  end
+
+  def self.invalid_organization_handler
+    @invalid_organization_handler ||= proc {|message| fail Draisine::SoapHandler::InvalidOrganizationError, "invalid organization id in the inbound message from salesforce" }
+  end
+
+  def self.invalid_organization_handler=(handler)
+    @invalid_organization_handler = handler
+  end
+
+  # https://help.salesforce.com/apex/HTViewSolution?language=en_US&id=000003652
+  def self.allowed_ip_ranges
+    @allowed_ip_ranges ||= [
+      '96.43.144.0/20',
+      '136.146.210.8/15',
+      '204.14.232.0/21',
+      '85.222.128.0/19',
+      '185.79.140.0/22',
+      '182.50.76.0/22',
+      '202.129.242.0/23',
+      '127.0.0.1'
+    ]
+  end
+
+  def self.allowed_ip_ranges=(ranges)
+    @allowed_ip_ranges = ranges
+  end
+end

data/lib/draisine/soap_handler.rb

@@ -0,0 +1,79 @@
+require 'active_support/core_ext/hash/conversions'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Draisine
+  class SoapHandler
+    InvalidOrganizationError = Class.new(StandardError)
+
+    def initialize
+    end
+
+    def update(message_xml)
+      message = parse(message_xml)
+
+      assert_valid_message!(message)
+      extract_sobjects(message).each do |sobject|
+        type = sobject.fetch('xsi:type').sub('sf:', '')
+        klass = Draisine.registry.find(type)
+        klass.salesforce_on_inbound_update(sobject)
+      end
+    rescue InvalidOrganizationError => e
+      Draisine.invalid_organization_handler.call(message)
+    end
+
+    def delete(message_xml)
+      message = parse(message_xml)
+
+      assert_valid_message!(message)
+      extract_sobjects(message).each do |sobject|
+        type = sobject.fetch('Object_Type__c')
+        id = sobject.fetch('Object_Id__c')
+        klass = Draisine.registry.find(type)
+        klass.salesforce_on_inbound_delete(id)
+      end
+    rescue InvalidOrganizationError => e
+      Draisine.invalid_organization_handler.call(message)
+    end
+
+    protected
+
+    def parse(message_xml)
+      case message_xml
+      when Hash
+        message_xml
+      when String
+        Hash.from_xml(message_xml)
+      else
+        raise ArgumentError
+      end
+    end
+
+    def extract_sobjects(message)
+      Array.wrap(message['Envelope']['Body']['notifications']['Notification']).map do |sobject|
+        sobject.fetch('sObject')
+      end
+    end
+
+    def assert_valid_organization_id!(message)
+      unless diggable_to?(message, ['Envelope', 'Body', 'notifications', 'OrganizationId']) &&
+             message['Envelope']['Body']['notifications']['OrganizationId'] == Draisine.organization_id
+        fail InvalidOrganizationError, "a message from invalid organization id received"
+      end
+    end
+
+    def assert_valid_message!(message)
+      unless diggable_to?(message, ['Envelope', 'Body', 'notifications', 'Notification'])
+        fail ArgumentError, "malformed xml inbound message from salesforce"
+      end
+      assert_valid_organization_id!(message)
+    end
+
+    def diggable_to?(hash, path)
+      path.each do |key|
+        return false unless hash.respond_to?(:key?) && hash.key?(key)
+        hash = hash[key]
+      end
+      true
+    end
+  end
+end

data/lib/draisine/syncer.rb

@@ -0,0 +1,52 @@
+module Draisine
+  # Wrapper around salesforce client implementation
+  # Might have pluggable adapters in the future.
+  class Syncer
+    attr_reader :salesforce_object_name
+
+    def initialize(salesforce_object_name, client = nil)
+      @salesforce_object_name ||= salesforce_object_name
+      @client = client
+    end
+
+    def get(id, options = {})
+      raise ArgumentError unless id.present?
+      response = client.http_get(build_sobject_url(id), options)
+      JSON.parse(response.body)
+    end
+
+    def create(attrs)
+      response = client.http_post(build_sobject_url(nil), attrs.to_json)
+      JSON.parse(response.body)
+    end
+
+    def update(id, attrs)
+      raise ArgumentError unless id.present?
+      return unless attrs.present?
+      client.http_patch(build_sobject_url(id), attrs.to_json)
+    end
+
+    def delete(id)
+      raise ArgumentError unless id.present?
+      client.http_delete(build_sobject_url(id))
+    end
+
+    def get_system_modstamp(id)
+      raise ArgumentError unless id.present?
+      time = get(id, fields: "SystemModstamp")['SystemModstamp']
+      time && Time.parse(time)
+    end
+
+    protected
+
+    def client
+      @client || Draisine.salesforce_client
+    end
+
+    def build_sobject_url(id)
+      url = "/services/data/v#{client.version}/sobjects/#{salesforce_object_name}"
+      url << "/#{id}" if url
+      url
+    end
+  end
+end

data/lib/draisine/type_mapper.rb

@@ -0,0 +1,105 @@
+module Draisine
+  class TypeMapper
+    Type = Struct.new(:ar_type, :ar_options, :serialized, :array)
+
+    ActiveRecordColumnDef = Struct.new(:column_name, :column_type, :options) do
+      def self.from_ar_column(ar_col)
+        new(ar_col.name, ar_col.type, { limit: ar_col.limit }.compact)
+      end
+    end
+
+    MAX_ALLOWED_STRING_TYPE_LENGTH = 40
+
+    def self.type(ar_type, ar_options: {}, serialized: false, array: false)
+      Type.new(ar_type, ar_options, serialized, array)
+    end
+
+    def self.determine_type_for_float(name, sf_schema)
+      if (precision = sf_schema[:precision]) && (scale = sf_schema[:scale]) && scale == 0
+        type(:integer, ar_options: { limit: 8 })
+      else
+        type(:float, ar_options: { limit: 53 })
+      end
+    end
+
+    def self.determine_type_for_string(name, sf_schema)
+      if (length = sf_schema[:length]) && length > 0 && length <= MAX_ALLOWED_STRING_TYPE_LENGTH
+        type(:string, ar_options: { limit: length })
+      else
+        type(:text)
+      end
+    end
+
+    # Apparently, mysql has a hard limit of 64k per row.
+    # That's why we're using text types where we could also use strings.
+    TYPES_MAP = {
+      "boolean" => type(:boolean),
+      "string" => method(:determine_type_for_string),
+      "reference" => type(:string, ar_options: { limit: 20 }),
+      "picklist" => type(:binary, serialized: true),
+      "textarea" => method(:determine_type_for_string),
+      "phone" => method(:determine_type_for_string),
+      "email" => method(:determine_type_for_string),
+      "url" => method(:determine_type_for_string),
+      "int" => type(:integer),
+      "date" => type(:date),
+      "time" => type(:time),
+      "multipicklist" => type(:binary, serialized: true, array: true),
+      "double" => method(:determine_type_for_float),
+      "datetime" => type(:datetime),
+      "anyType" => type(:binary, serialized: true),
+      "combobox" => type(:text),
+      "currency" => type(:decimal, ar_options: { precision: 18, scale: 6 }),
+      "percent" => type(:decimal, ar_options: { precision: 18, scale: 6 })
+      # Leave this one for now
+      # "encrypted_string" => :string,
+    }
+
+    EXCLUDED_COLUMNS = ["Id"]
+
+    attr_reader :sf_type_map, :type_map
+    def initialize(sf_type_map)
+      @sf_type_map = sf_type_map
+      @type_map = sf_type_map.reject {|name, schema| ignored_column?(name, schema) }.
+                              select {|name, schema| type_for(name, schema) }.
+                              map {|name, schema| [name, type_for(name, schema)] }.
+                              to_h
+    end
+
+    def active_record_column_defs
+      type_map.map {|name, type| active_record_column_def(name, type) }
+    end
+
+    def columns
+      @columns ||= type_map.keys
+    end
+
+    def updateable_columns
+      @updateable_columns ||= sf_type_map.select {|_, type| type[:updateable?] }.keys
+    end
+
+    def serialized_columns
+      @serialized_columns ||= type_map.select {|_, type| type.serialized }.keys
+    end
+
+    def array_columns
+      @array_columns ||= type_map.select {|_, type| type.array }.keys
+    end
+
+    protected
+
+    def type_for(sf_column_name, sf_column_schema)
+      sf_type = sf_column_schema.fetch(:type)
+      type = TYPES_MAP.fetch(sf_type) { warn "Unknown column type #{sf_type} for column #{sf_column_name}, ignoring it" }
+      type.respond_to?(:call) ? type.call(sf_column_name, sf_column_schema) : type
+    end
+
+    def ignored_column?(sf_column_name, sf_column_schema)
+      EXCLUDED_COLUMNS.include?(sf_column_name)
+    end
+
+    def active_record_column_def(column_name, type)
+      ActiveRecordColumnDef.new(column_name, type.ar_type, type.ar_options)
+    end
+  end
+end

data/lib/draisine/util/caching_client.rb

@@ -0,0 +1,73 @@
+module Draisine
+  class CachingClient
+    class Cache
+      attr_reader :cache
+
+      def initialize
+        @cache = {}
+      end
+
+      def [](key)
+        cache[key]
+      end
+
+      def fetch(key, &block)
+        cache.fetch(key) { cache[key] = yield }
+      end
+
+      def []=(key, value)
+        cache[key] = value
+      end
+
+      def add(record)
+        self[record.attributes.fetch('Id')] = record
+      end
+
+      def add_multiple(records)
+        records.each {|record| add(record) }
+      end
+
+      def has_ids?(ids)
+        (ids - cache.keys).empty?
+      end
+
+      def fetch_multiple(ids, &block)
+        if has_ids?(ids)
+          cache.values_at(*ids)
+        else
+          yield.tap do |records|
+            add_multiple(records)
+          end
+        end
+      end
+    end
+
+    attr_reader :cache_map, :client
+
+    def initialize(client = Draisine.salesforce_client)
+      @cache_map = Hash.new {|h,k| h[k] = Cache.new }
+      @client = client
+    end
+
+    def find(salesforce_object_name, id)
+      cache_map[salesforce_object_name].fetch(id) do
+        client.find(salesforce_object_name, id)
+      end
+    end
+
+    def fetch_multiple(salesforce_object_name, ids)
+      cache_map[salesforce_object_name].fetch_multiple(ids) do
+        client.fetch_multiple(salesforce_object_name, ids)
+      end
+    end
+    alias_method :prefetch, :fetch_multiple
+
+    def method_missing(method, *args, &block)
+      if client.respond_to?(method)
+        client.__send__(method, *args, &block)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/draisine/util/hash_diff.rb

@@ -0,0 +1,39 @@
+module Draisine
+  HashDiff = Struct.new(:added, :removed, :changed, :unchanged) do
+    def self.diff(hash1, hash2, equality = -> (a, b) { a == b })
+      unless hash1.respond_to?(:key?) && hash2.respond_to?(:key?)
+        fail ArgumentError, "both arguments should be hashes"
+      end
+
+      added = []
+      removed = []
+      changed = []
+      unchanged = []
+
+      (hash1.keys | hash2.keys).each do |key|
+        if hash1.key?(key) && hash2.key?(key)
+          if equality.call(hash1[key], hash2[key])
+            unchanged << key
+          else
+            changed << key
+          end
+        elsif hash1.key?(key)
+          removed << key
+        else
+          added << key
+        end
+      end
+
+      new(added, removed, changed, unchanged)
+    end
+
+    def self.sf_diff(hash1, hash2)
+      diff(hash1, hash2, SalesforceComparisons.method(:salesforce_equals?))
+    end
+
+    def diff_keys
+      changed | added | removed
+    end
+  end
+
+end

data/lib/draisine/util/parse_time.rb

@@ -0,0 +1,14 @@
+module Draisine
+  def self.parse_time(time_or_string)
+    case time_or_string
+    when Time, DateTime
+      time_or_string
+    when String
+      Time.parse(time_or_string)
+    else
+      Time.parse(time_or_string.to_s)
+    end
+  rescue => e
+    nil
+  end
+end