draisine 0.7.10

data/lib/draisine/auditor.rb

@@ -0,0 +1,130 @@
+require "draisine/auditor/result"
+
+module Draisine
+  class Auditor
+    def self.run(model_class:, start_date: Time.current.beginning_of_day, end_date: Time.current, mechanism: :default)
+      # TODO: instead of using one huge partition, combine multiple results into one
+      partitions = partition(model_class: model_class, start_date: start_date, end_date: end_date, partition_size: 10**12, mechanism: mechanism)
+      run_partition(partitions.first)
+    end
+
+    def self.run_partition(partition)
+      new(partition).run
+    end
+
+    def self.partition(model_class:, start_date:, end_date:, partition_size: 100, mechanism: :default)
+      Partitioner.partition(
+        model_class: model_class,
+        start_date: start_date,
+        end_date: end_date,
+        partition_size: partition_size,
+        mechanism: mechanism)
+    end
+
+    attr_reader :partition, :model_class, :start_date, :end_date, :result
+    def initialize(partition)
+      @partition = partition
+      @model_class = partition.model_class
+      @start_date = partition.start_date
+      @end_date = partition.end_date
+    end
+
+    def run
+      @result = Result.new
+
+      check_unpersisted_records
+      check_deletes
+      check_modifications
+
+      result.calculate_result!
+    rescue => e
+      result.error!(e)
+      raise
+    end
+
+    def check_unpersisted_records
+      return unless partition.unpersisted_ids.present?
+
+      bad_records = model_class.where(id: partition.unpersisted_ids)
+      bad_records.each do |record|
+        result.discrepancy(
+          type: :local_record_without_salesforce_id,
+          salesforce_type: salesforce_object_name,
+          salesforce_id: nil,
+          local_id: record.id,
+          local_type: record.class.name,
+          local_attributes: record.attributes)
+      end
+    end
+
+    def check_deletes
+      return unless partition.deleted_ids.present?
+
+      ghost_models = model_class.where(salesforce_id: partition.deleted_ids).all
+      ghost_models.each do |ghost_model|
+        result.discrepancy(
+          type: :remote_delete_kept_locally,
+          salesforce_type: salesforce_object_name,
+          salesforce_id: ghost_model.salesforce_id,
+          local_id: ghost_model.id,
+          local_type: ghost_model.class.name,
+          local_attributes: ghost_model.attributes)
+      end
+    end
+
+    def check_modifications
+      updated_ids = partition.updated_ids
+      return unless updated_ids.present?
+
+      local_records = model_class.where(salesforce_id: updated_ids).to_a
+      remote_records = client.fetch_multiple(salesforce_object_name, updated_ids)
+
+      local_records_map = build_map(local_records) {|record| record.salesforce_id }
+      remote_records_map = build_map(remote_records) {|record| record.Id }
+
+      missing_ids = updated_ids - local_records_map.keys
+      missing_ids.each do |id|
+        result.discrepancy(
+          type: :remote_record_missing_locally,
+          salesforce_type: salesforce_object_name,
+          salesforce_id: id,
+          remote_attributes: remote_records_map.fetch(id))
+      end
+
+      attr_list = model_class.salesforce_audited_attributes
+      local_records_map.each do |salesforce_id, local_record|
+        remote_record = remote_records_map[salesforce_id]
+        next unless remote_record
+        conflict_detector = ConflictDetector.new(local_record, remote_record, attr_list)
+
+        if conflict_detector.conflict?
+          result.discrepancy(
+            type: :mismatching_records,
+            salesforce_type: salesforce_object_name,
+            salesforce_id: salesforce_id,
+            local_id: local_record.id,
+            local_type: local_record.class.name,
+            local_attributes: local_record.salesforce_attributes,
+            remote_attributes: remote_record.attributes,
+            diff_keys: conflict_detector.diff.diff_keys)
+        end
+      end
+    end
+
+    protected
+
+    def client
+      Draisine.salesforce_client
+    end
+
+    def build_map(list_of_hashes, &key_block)
+      list_of_hashes.each_with_object({}) do |item, rs|
+        rs[key_block.call(item)] = item
+      end
+    end
+
+    def salesforce_object_name
+      model_class.salesforce_object_name
+    end
+  end
+end

data/lib/draisine/concerns/array_setter.rb

@@ -0,0 +1,23 @@
+require "active_support/concern"
+
+module Draisine
+  module Concerns
+    module ArraySetter
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def salesforce_array_setter(attr)
+          mod = Module.new do
+            define_method "#{attr}=" do |value|
+              value = [] if value.nil?
+              value = value.split(';') if value.kind_of?(String)
+              super(value)
+            end
+          end
+          prepend mod
+          attr
+        end
+      end
+    end
+  end
+end

data/lib/draisine/concerns/attributes_mapping.rb

@@ -0,0 +1,46 @@
+require "active_support/concern"
+
+module Draisine
+  module Concerns
+    module AttributesMapping
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        attr_accessor :salesforce_mapping
+
+        def salesforce_synced_attributes
+          @salesforce_synced_attributes ||= salesforce_mapping.keys
+        end
+
+        def salesforce_reverse_mapping
+          @salesforce_reverse_mapping ||= salesforce_mapping.map(&:reverse).to_h
+        end
+      end
+
+      def salesforce_mapped_attributes(attributes, mapping = self.class.salesforce_mapping)
+        attributes.slice(*mapping.keys).each_with_object({}) do |(key, value), acc|
+          acc[mapping.fetch(key)] = value
+        end
+      end
+
+      def salesforce_assign_attributes(attributes)
+        salesforce_mapped_attributes(attributes.with_indifferent_access).each do |key, value|
+          method_name = "#{key}="
+          if respond_to?(method_name)
+            value = Draisine::SalesforceComparisons.salesforce_cleanup(value)
+            __send__(method_name, value)
+          end
+        end
+      end
+
+      def salesforce_reverse_mapped_attributes(attributes)
+        salesforce_mapped_attributes(attributes, self.class.salesforce_reverse_mapping)
+      end
+
+      def salesforce_attributes
+        salesforce_reverse_mapped_attributes(attributes)
+          .with_indifferent_access
+      end
+    end
+  end
+end

data/lib/draisine/concerns/import.rb

@@ -0,0 +1,36 @@
+require "active_support/concern"
+
+module Draisine
+  module Concerns
+    module Import
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        # Doesn't update record if found
+        def import_with_attrs(sf_id, attrs)
+          find_or_initialize_by(salesforce_id: sf_id) do |model|
+            model.salesforce_update_without_sync(attrs)
+          end
+        end
+
+        # Does update record if found
+        def import_or_update_with_attrs(sf_id, attrs, check_modstamp = false)
+          find_or_initialize_by(salesforce_id: sf_id).tap do |model|
+            model.salesforce_update_without_sync(attrs, check_modstamp)
+          end
+        end
+      end
+
+      def salesforce_update_without_sync(attributes, check_modstamp = false)
+        salesforce_skipping_sync do
+          modstamp = attributes["SystemModstamp"]
+          own_modstamp = self.attributes["SystemModstamp"]
+          if !check_modstamp || !modstamp || !own_modstamp || own_modstamp < modstamp
+            salesforce_assign_attributes(attributes)
+            save!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/draisine/conflict_detector.rb

@@ -0,0 +1,38 @@
+module Draisine
+  class ConflictDetector
+    attr_reader :model, :remote_model, :attributes_list
+    def initialize(model, remote_model, attributes_list = model.class.salesforce_audited_attributes)
+      @model = model
+      @remote_model = remote_model
+      @attributes_list = attributes_list
+    end
+
+    def conflict?
+      conflict_type != :no_conflict
+    end
+
+    def conflict_type
+      if model && remote_model
+        if diff.diff_keys.empty?
+          :no_conflict
+        else
+          :mismatching_records
+        end
+      elsif model
+        :remote_record_missing
+      elsif remote_model
+        :local_record_missing
+      else
+        :no_conflict
+      end
+    end
+
+    def diff
+      return unless model && remote_model
+
+      @diff ||= HashDiff.sf_diff(
+          model.salesforce_attributes.slice(*attributes_list).compact,
+          remote_model.attributes.slice(*attributes_list).compact)
+    end
+  end
+end

data/lib/draisine/conflict_resolver.rb

@@ -0,0 +1,97 @@
+module Draisine
+  class ConflictResolver
+    ALLOWED_RESOLUTIONS = %w[
+      remote_push remote_pull local_delete merge
+    ]
+
+    attr_reader :model_class, :client, :salesforce_object_name,
+                :local_id, :salesforce_id
+
+    def initialize(model_class, client, local_id, salesforce_id)
+      @model_class = model_class
+      @client = client
+      @salesforce_object_name = model_class.salesforce_object_name
+      @local_id = local_id
+      @salesforce_id = salesforce_id
+    end
+
+    def conflict?
+      ConflictDetector.new(model, remote_model, model_class.salesforce_synced_attributes).conflict?
+    end
+
+    def resolve(resolution_type, options = {})
+      resolution_type = resolution_type.to_s
+      fail ArgumentError, "Unknown resolution type '#{resolution_type}'" unless allowed_resolution?(resolution_type)
+
+      __send__(resolution_type, options)
+    end
+
+    def allowed_resolution?(resolution_type)
+      ALLOWED_RESOLUTIONS.include?(resolution_type)
+    end
+
+    def remote_pull(_options = {})
+      fail ArgumentError, "remote model is required for remote pull" unless remote_model
+
+      model_class.salesforce_inbound_update(remote_model.attributes)
+    end
+
+    def remote_push(_options = {})
+      fail ArgumentError, "local model is required for remote push" unless model
+
+      if model.salesforce_id.present?
+        model.salesforce_outbound_update(model.salesforce_attributes)
+      else
+        model.salesforce_outbound_create
+      end
+    end
+
+    def remote_delete(_options = {})
+      fail ArgumentError, "local model is required for remote delete" unless model
+
+      model.salesforce_outbound_delete
+    end
+
+    def local_delete(_options = {})
+      model_class.salesforce_inbound_delete(salesforce_id)
+    end
+
+    def merge(options)
+      fail ArgumentError unless model && remote_model
+      assert_required_options!(options, [:local_attributes, :remote_attributes])
+
+      local_attrs_to_merge = options.fetch(:local_attributes)
+      remote_attrs_to_merge = options.fetch(:remote_attributes)
+
+      model.salesforce_outbound_update(
+        model.salesforce_attributes.slice(*local_attrs_to_merge))
+      model.salesforce_inbound_update(
+        remote_model.attributes.slice(*remote_attrs_to_merge), false)
+    end
+
+    def model
+      @model ||= if local_id
+        model_class.find_by(id: local_id)
+      else
+        model_class.find_by(salesforce_id: salesforce_id)
+      end
+    end
+
+    def remote_model
+      return @remote_model unless @remote_model.nil?
+      @remote_model = begin
+        client.find(salesforce_object_name, salesforce_id)
+      rescue Databasedotcom::SalesForceError
+        false
+      end
+    end
+
+    protected
+
+    def assert_required_options!(options, keys)
+      keys.each do |key|
+        fail ArgumentError, "missing required option #{key}" unless options.key?(key)
+      end
+    end
+  end
+end

data/lib/draisine/engine.rb

@@ -0,0 +1,6 @@
+require 'rails'
+
+module Draisine
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/draisine/importer.rb

@@ -0,0 +1,111 @@
+module Draisine
+  class Importer
+    attr_reader :model_class
+
+    def initialize(model_class)
+      @model_class = model_class
+    end
+
+    def import(start_id: nil, start_date: nil, batch_size: 500)
+      find_each(batch_size: batch_size, start_id: start_id, start_date: start_date) do |sobj|
+        attrs = sobj.attributes
+        model_class.import_with_attrs(
+          attrs.fetch("Id"),
+          attrs.slice(*model_class.salesforce_synced_attributes))
+      end
+    end
+
+    def import_new(batch_size: 500, start_date_window_size: 20.minutes)
+      last_model = model_class.order("salesforce_id DESC").first
+      start_id = last_model.try(:salesforce_id)
+      start_date = last_model.try(:CreatedDate)
+      start_date -= start_date_window_size if start_date
+
+      import(start_id: start_id, batch_size: batch_size, start_date: start_date)
+    end
+
+    def import_fields(batch_size: 500, fields:)
+      model_class.find_in_batches(batch_size: batch_size) do |batch|
+        attempt do
+          sobjs = client.fetch_multiple(model_class.salesforce_object_name, batch.map(&:salesforce_id), batch_size, fields)
+          sobjs_map = sobjs.map {|sobj| [sobj.Id, sobj] }.to_h
+          batch.each do |model|
+            sobject = sobjs_map[model.salesforce_id]
+            next unless sobject
+            model.salesforce_assign_attributes(sobject.attributes.slice(*fields))
+            model.salesforce_skipping_sync { model.save! }
+          end
+        end
+      end
+    end
+
+    protected
+
+    def find_each(batch_size:, start_id:, start_date: nil, &block)
+      salesforce_model = client.materialize(salesforce_object_name)
+      # if we have start_date set, only use id starting from the second query
+      last_id = start_id unless start_date
+
+      counter = 0
+      loop do
+        query = import_query(salesforce_model, salesforce_object_name, batch_size, last_id, start_date)
+        collection = attempt { client.query(query) }
+        break unless collection.count > 0
+
+        model_class.transaction do
+          collection.each do |sobj|
+            yield sobj
+          end
+        end
+
+        counter += collection.count
+        last_id = collection.last.attributes.fetch("Id")
+        logger.info "[#{model_class} import] Imported #{counter} records, last record id #{last_id}"
+      end
+      logger.info "[#{model_class} import] Finished, imported a total of #{counter} records"
+    end
+
+    def client
+      Draisine.salesforce_client
+    end
+
+    def salesforce_object_name
+      model_class.salesforce_object_name
+    end
+
+    def import_query(salesforce_model, salesforce_object_name, batch_size, start_id = nil, start_date = nil)
+      conds = [
+        start_id && "Id > '#{start_id}'",
+        start_date && "CreatedDate >= #{start_date.iso8601}"
+      ].compact
+      where_clause = conds.presence && "WHERE #{conds.join(" AND ")}"
+
+      <<-QUERY
+      SELECT #{salesforce_model.field_list}
+      FROM #{salesforce_object_name}
+      #{where_clause}
+      ORDER BY Id ASC
+      LIMIT #{batch_size}
+      QUERY
+    end
+
+    def attempt(times = 5)
+      attempts ||= 0
+      yield
+    rescue => e
+      attempts += 1
+      logger.error "#{e.class}: #{e.message}"
+      if attempts < times
+        logger.error "Retrying... (attempt ##{attempts})"
+        retry
+      else
+        logger.error "Too many attempts, failing..."
+        raise
+      end
+    end
+
+    def logger
+      @logger ||= Rails.logger || Logger.new($stdout)
+    end
+  end
+end

data/lib/draisine/ip_checker.rb

@@ -0,0 +1,15 @@
+require "ipaddr"
+
+module Draisine
+  class IpChecker
+    attr_reader :ip_ranges
+    def initialize(ip_ranges)
+      @ip_ranges = ip_ranges.map {|net| IPAddr.new(net) }
+    end
+
+    def check(ip)
+      addr = IPAddr.new(ip)
+      ip_ranges.any? {|range| range.include?(addr) }
+    end
+  end
+end

data/lib/draisine/jobs/inbound_delete_job.rb

@@ -0,0 +1,8 @@
+module Draisine
+  class InboundDeleteJob < Draisine::JobBase
+    def _perform(class_name, salesforce_id)
+      klass = class_name.constantize
+      klass.salesforce_inbound_delete(salesforce_id)
+    end
+  end
+end

data/lib/draisine/jobs/inbound_update_job.rb

@@ -0,0 +1,8 @@
+module Draisine
+  class InboundUpdateJob < Draisine::JobBase
+    def _perform(class_name, attributes)
+      klass = class_name.constantize
+      klass.salesforce_inbound_update(attributes)
+    end
+  end
+end

data/lib/draisine/jobs/job_base.rb

@@ -0,0 +1,39 @@
+module Draisine
+  class JobBase < ActiveJob::Base
+    queue_as :draisine_job
+
+    def perform(*args)
+      _perform(*args)
+    rescue Exception => ex
+      logger.error "#{ex.class}: #{ex}\n#{ex.backtrace.join("\n")}"
+
+      if retry_attempt < retries_count
+        @retry_attempt = retry_attempt + 1
+        logger.error "Retrying (attempt #{retry_attempt})"
+        retry_job
+      else
+        logger.error "Too many attempts, no more retries"
+        Draisine.job_error_handler.call(ex, self, arguments)
+      end
+    end
+
+    def _perform(*args)
+    end
+
+    def retries_count
+      Draisine.job_retry_attempts
+    end
+
+    def retry_attempt
+      @retry_attempt ||= 0
+    end
+
+    def serialize
+      super.merge('_retry_attempt' => retry_attempt)
+    end
+
+    def deserialize(job_data)
+      @retry_attempt = job_data.fetch('_retry_attempt', 0)
+    end
+  end
+end

data/lib/draisine/jobs/outbound_create_job.rb

@@ -0,0 +1,7 @@
+module Draisine
+  class OutboundCreateJob < Draisine::JobBase
+    def _perform(instance)
+      instance.salesforce_outbound_create
+    end
+  end
+end

data/lib/draisine/jobs/outbound_delete_job.rb

@@ -0,0 +1,7 @@
+module Draisine
+  class OutboundDeleteJob < Draisine::JobBase
+    def _perform(instance)
+      instance.salesforce_outbound_delete
+    end
+  end
+end

data/lib/draisine/jobs/outbound_update_job.rb

@@ -0,0 +1,7 @@
+module Draisine
+  class OutboundUpdateJob < Draisine::JobBase
+    def _perform(instance, changed_attributes)
+      instance.salesforce_outbound_update(changed_attributes)
+    end
+  end
+end

data/lib/draisine/jobs/soap_delete_job.rb

@@ -0,0 +1,7 @@
+module Draisine
+  class SoapDeleteJob < Draisine::JobBase
+    def _perform(message)
+      SoapHandler.new.delete(message)
+    end
+  end
+end

data/lib/draisine/jobs/soap_update_job.rb

@@ -0,0 +1,7 @@
+module Draisine
+  class SoapUpdateJob < Draisine::JobBase
+    def _perform(message)
+      SoapHandler.new.update(message)
+    end
+  end
+end

data/lib/draisine/partitioner.rb

@@ -0,0 +1,73 @@
+module Draisine
+  Partition = Struct.new(:model_class, :start_date, :end_date, :updated_ids, :deleted_ids, :unpersisted_ids) do
+    def initialize(model_class, *args)
+      model_class = model_class.constantize if model_class.is_a?(String)
+      super(model_class, *args)
+    end
+
+    def as_json(*)
+      model_class, *fields = to_a
+      [model_class.name, *fields].as_json
+    end
+
+    def self.from_json(fields)
+      new(*fields)
+    end
+  end
+
+  class Partitioner
+    def self.partition(model_class:, start_date:, end_date:, partition_size: 100, mechanism: :default)
+      new(model_class, mechanism).partition(start_date, end_date, partition_size: partition_size)
+    end
+
+    attr_reader :model_class, :mechanism
+    def initialize(model_class, mechanism = :default)
+      @model_class = model_class
+      @mechanism = QueryMechanisms.fetch(mechanism).new(model_class)
+    end
+
+    def partition(start_date, end_date, partition_size: 100)
+      updated_ids = get_updated_ids(start_date, end_date)
+      deleted_ids = get_deleted_ids(start_date, end_date)
+      unpersisted_ids = get_unpersisted_ids(start_date, end_date)
+
+      # if anyone knows how to do this packing procedure better, please tell me
+      all_ids = updated_ids.map {|id| [:updated, id] } +
+                deleted_ids.map {|id| [:deleted, id] } +
+                unpersisted_ids.map {|id| [:unpersisted, id] }
+
+      if all_ids.present?
+        all_ids.each_slice(partition_size).map do |slice|
+          part = slice.group_by(&:first).map {|k,v| [k, v.map(&:last)] }.to_h
+          Partition.new(model_class.name, start_date, end_date, part[:updated], part[:deleted], part[:unpersisted])
+        end
+      else
+        [Partition.new(model_class.name, start_date, end_date)]
+      end
+    end
+
+    protected
+
+    def get_updated_ids(start_date, end_date)
+      mechanism.get_updated_ids(start_date, end_date) |
+        model_class
+          .where("updated_at >= ? AND updated_at <= ?", start_date, end_date)
+          .uniq.pluck(:salesforce_id).compact
+    end
+
+    def get_deleted_ids(start_date, end_date)
+      mechanism.get_deleted_ids(start_date, end_date)
+    end
+
+    def get_unpersisted_ids(start_date, end_date)
+      model_class
+        .where("salesforce_id IS NULL OR salesforce_id = ?", '')
+        .where("updated_at >= ? and updated_at <= ?", start_date, end_date)
+        .pluck(:id)
+    end
+
+    def client
+      Draisine.salesforce_client
+    end
+  end
+end