advanced_ar 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 89ee5d16158505c78a2daf264fdbe578f97c81c14be8722d489aede8759d8306
+  data.tar.gz: 87d105eca9cece82a02fabdcc83ea641549fa6243c400561e089a104e380a0b1
+SHA512:
+  metadata.gz: 9a8fea0f12d6bb2246910c61d6870065b6af7a31300a49fe815c555cdbf0d540a4401911450d0da7e6c52520a514d0e482855f540f68bd526625624cb99df3e8
+  data.tar.gz: 99765c594830908e98a36a870bd7cc75fcb80ea1d3828af496c470459ad6e17c919c2f0fe51e0e50fe4be4835a296dc6ede2595c40d32bd9e3c43013fb6fe045

data/README.md

@@ -0,0 +1,3 @@
+# AdvancedAR
+
+Gem for adding advanced features into ActiveRecord.

data/config/initializers/01_custom_preloaders.rb

@@ -0,0 +1,2 @@
+# This Preloader needs to run before any that will load Models
+AdvancedAR::CustomPreloaders.install

data/config/initializers/arbitrary_prefetch.rb

@@ -0,0 +1 @@
+AdvancedAR::ArbitraryPrefetch.install

data/config/initializers/cancancan.rb

@@ -0,0 +1,34 @@
+# This patch fixes an issue where CanCanCan wasn't properly handling
+# existence checks for AR Scopes - instead of do a DB Query, it loaded the entire relation
+module CanCan
+  module ConditionsMatcher
+    def condition_match?(attribute, value)
+      case value
+      when Hash
+        hash_condition_match?(attribute, value)
+      when Range
+        value.cover?(attribute)
+      when ActiveRecord::Relation
+        ar_condition_match?(attribute, value)
+      when Enumerable
+        value.include?(attribute)
+      else
+        attribute == value
+      end
+    end
+
+    def ar_condition_match?(attribute, value)
+      return false if attribute.nil?
+
+      if attribute.is_a?(Array) || (defined?(ActiveRecord) && attribute.is_a?(ActiveRecord::Relation) && attribute.loaded?)
+        value.where(id: attribute.pluck(:id)).exists?
+      elsif (defined?(ActiveRecord) && attribute.is_a?(ActiveRecord::Relation))
+        value.where(id: attribute.select(:id)).exists?
+      elsif (defined?(ActiveRecord) && attribute.is_a?(ActiveRecord::Base))
+        value.where(id: attribute.id).exists?
+      else
+        value.where(id: attribute).exists?
+      end
+    end
+  end
+end

data/lib/advanced_ar.rb

@@ -0,0 +1,6 @@
+
+Dir[File.dirname(__FILE__) +  "/advanced_ar/*.rb"].each { |file| require file }
+
+module AdvancedAR
+
+end

data/lib/advanced_ar/arbitrary_prefetch.rb

@@ -0,0 +1,161 @@
+# Add support for creating arbitrary associations when using ActiveRecord
+# Adds a `prefetch` method to ActiveRecord Queries.
+#   This method accepts a Hash. The keys of the Hash represent how the Association will be made available.
+#   The values of the Hash may be an array of [Symbol, Relation] or another (filtered) Relation.
+#   Objects are queried from an existing Association on the model. This Association is detemrined
+#   by either the Symbol when an array is passed, or by finding an Assoication for the passed Relation's model
+#
+# NOTICE: This implementation is NOT COMPLETE by itself - it depends on Goldiloader
+#   to detect the use of the virtual associations and prevent N+1s. We were already using
+#   Goldiloader, so this made sense. If this module is ever needed stand-alone,
+#   the following options have been identified:
+#     1. Extend ActiveRecordRelationPatch#exec_queries to execute an ActiveRecord::Associations::Preloader
+#        that will load the related objects
+#     2. Duplicates the relevant snippets from Goldiloader into this module. See Goldiloader::AutoIncludeContext
+#   The current Goldiloader implementation uses Option 1 internally, but also makes the relations lazy - even
+#     if you define a prefetch, it won't actually be loaded until you attempt to access it on one of the models.
+module AdvancedAR::ArbitraryPrefetch
+  class PrefetcherContext
+    attr_accessor :model, :target_attribute
+    attr_reader :options
+
+    def initialize(model, opts)
+      @options = opts
+      @model = model
+      @source_key = opts[:relation]
+      @target_attribute = opts[:attribute]
+      @queryset = opts[:queryset]
+      @models = []
+    end
+
+    def link_models(models)
+      Array(models).each do |m|
+        @models << m
+
+        # assoc = PrefetchAssociation.new(m, self, reflection)
+        assoc = reflection.association_class.new(m, reflection)
+        m.send(:association_instance_set, target_attribute, assoc)
+
+        m.instance_eval <<-CODE, __FILE__, __LINE__ + 1
+          def #{target_attribute}
+            association(:#{target_attribute}).reader
+          end
+        CODE
+      end
+    end
+
+    def reflection
+      @reflection ||= begin
+        queryset = @queryset
+        source_refl = model.reflections[@source_key.to_s]
+        scope = lambda { |*_args|
+          qs = queryset
+          qs = qs.merge(source_refl.scope_for(model.unscoped)) if source_refl.scope
+          qs
+        }
+        ActiveRecord::Reflection.create(
+          options[:type],
+          @target_attribute,
+          scope,
+          source_refl.options.merge(
+            class_name: source_refl.class_name,
+            inverse_of: nil
+          ),
+          model
+        )
+      end
+    end
+  end
+
+  module ActiveRecordBasePatch
+    extend ActiveSupport::Concern
+
+    included do
+      class << self
+        delegate :prefetch, to: :all
+      end
+    end
+  end
+
+  module ActiveRecordRelationPatch
+    def exec_queries
+      return super if loaded?
+
+      records = super
+      preloader = nil
+      (@values[:prefetches] || {}).each do |_key, opts|
+        pfc = PrefetcherContext.new(model, opts)
+        pfc.link_models(records)
+
+        unless defined?(Goldiloader)
+          preloader ||= build_preloader
+          preloader.preload(records, opts[:attribute])
+        end
+      end
+      records
+    end
+
+    def prefetch(**kwargs)
+      spawn.add_prefetches!(kwargs)
+    end
+
+    def add_prefetches!(kwargs)
+      return unless kwargs.present?
+
+      assert_mutability!
+      @values[:prefetches] ||= {}
+      kwargs.each do |attr, opts|
+        @values[:prefetches][attr] = normalize_options(attr, opts)
+      end
+      self
+    end
+
+    def normalize_options(attr, opts)
+      norm = if opts.is_a?(Array)
+          { relation: opts[0], queryset: opts[1] }
+        elsif opts.is_a?(ActiveRecord::Relation)
+          rel_name = opts.model.name.underscore
+          rel = (model.reflections[rel_name] || model.reflections[rel_name.pluralize])&.name
+          { relation: rel, queryset: opts }
+        else
+          opts
+      end
+
+      norm[:attribute] = attr
+      norm[:type] ||= (attr.to_s.pluralize == attr.to_s) ? :has_many : :has_one
+
+      norm
+    end
+  end
+
+  module ActiveRecordMergerPatch
+    def merge
+      super.tap do
+        merge_prefetches
+      end
+    end
+
+    private
+
+    def merge_prefetches
+      relation.add_prefetches!(other.values[:prefetches])
+    end
+  end
+
+  def self.install
+    ::ActiveRecord::Base.include(ActiveRecordBasePatch)
+    ::ActiveRecord::Relation.prepend(ActiveRecordRelationPatch)
+    ::ActiveRecord::Relation::Merger.prepend(ActiveRecordMergerPatch)
+
+    return unless defined? ::Goldiloader
+
+    ::Goldiloader::AssociationLoader.module_eval do
+      def self.has_association?(model, association_name) # rubocop:disable Naming/PredicateName
+        model.association(association_name)
+        true
+      rescue ::ActiveRecord::AssociationNotFoundError => _err
+        false
+      end
+    end
+  end
+end

data/lib/advanced_ar/batch_matcher.rb

@@ -0,0 +1,199 @@
+module AdvancedAR
+  # Matches a set of rows (eg from a CSV) against Rows in the Database.
+  #
+  # Examples:
+  #   rule_matcher = BatchMatcher.new(
+  #     Rule, rows,
+  #     validate_all: false,
+  #     columns: [[:rule_id, :id, 'ID'], [:rule_import_id, :import_id, 'Import ID']]
+  #   )
+  #   context_matcher = BatchMatcher.new(
+  #     [Account, Course], rows,
+  #     polymorphic_on: :rule_context,
+  #     columns: [[:canvas_context_id, :canvas_id, 'Canvas ID'], [:sis_context_id, :sis_id, 'SIS ID']]
+  #   )
+  #   role_matcher = BatchMatcher.new(
+  #     Role, rows,
+  #     columns: [[:canvas_role_id, :canvas_id, 'Canvas Role ID'], [:role_label, :label, 'Role Label']]
+  #   )
+  #
+  # Params:
+  # @param clazz - Model or ActiveRecord::Realtion
+  # @param rows - Data to search the DB for matches
+  # @param columns - [[CSV Header Key, Model Key, (Human Name)], ...]
+  class BatchMatcher
+    attr_reader :rows, :maps, :columns, :primary_column
+
+    def initialize(clazz, rows, columns:, polymorphic_on: false, validate_all: true)
+      # columns: [csv_key, db_key, human_name]
+      @options = {
+        mode: :eager,
+        polymorphic_on: polymorphic_on,
+        validate_all: validate_all
+      }
+      @clazz = clazz
+      @columns = columns
+      @primary_column = columns[0]
+      @rows = rows
+      @maps = {}
+      @loaded_columns = {}
+      @mode = :eager
+    end
+
+    def get_for_row!(row)
+      resolve_row_value(row, :get_by_column)
+    end
+
+    def get_for_row(row)
+      get_for_row!(row)
+    rescue ActiveRecord::RecordNotFound
+      nil
+    end
+
+    def get_primary_for_row!(row)
+      resolve_row_value(row, :column_to_primary_key)
+    end
+
+    def get_primary_for_row(row)
+      get_primary_for_row!(row)
+    rescue ActiveRecord::RecordNotFound
+      nil
+    end
+
+    # Returns True if the row has a value for any of this Matcher's Columns
+    def should_match?(row)
+      @columns.any? { |col| row[col[0]].present? }
+    end
+
+    protected
+
+    def get_by_column(column, row)
+      if column == @primary_column
+        column_value(column, row)
+      else
+        mapped_id = column_value(column, row)
+        get_column_map(primary_column, row, via_column: column)[transform_key(mapped_id)]
+      end
+    end
+
+    def column_value(column, row)
+      get_column_map(column, row)[transform_key(row[column[0]])]
+    end
+
+    def column_to_primary_key(column, row)
+      if column == @primary_column
+        transform_key(row[column[0]])
+      else
+        transform_key(column_value(column, row))
+      end
+    end
+
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def resolve_row_value(row, resolver)
+      found_values = []
+      @columns.each do |c|
+        next unless row[c[0]].present?
+
+        inst = send(resolver, c, row)
+        if inst.nil? && (!found_values.present? || @options[:validate_all])
+          clazz = as_class(get_base_query(row))
+          raise ActiveRecord::RecordNotFound, "could not find #{clazz.name} with #{c[2] || c[0]} #{row[c[0]]}"
+        end
+        found_values << inst
+      end
+
+      if @options[:validate_all] && found_values.uniq.count > 1
+        raise ActiveRecord::RecordNotFound, "multiple of [#{@columns.pluck(2).join(', ')}] were supplied, but resolved to different objects" # rubocop:disable Metrics/LineLength
+      end
+
+      found_values[0]
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+    def get_column_map(column, row, via_column: nil)
+      base_query = get_base_query(row)
+      clazz = as_class(base_query)
+      raise ActiveRecord::RecordNotFound, "invalid #{@options[:polymorphic_on]}: #{row[@options[:polymorphic_on]]}" if clazz.nil? # rubocop:disable Metrics/LineLength
+
+      load_column(clazz, column, via_column) do
+        relevant_rows = rows
+        relevant_rows = relevant_rows.select { |r| r[@options[:polymorphic_on]] == clazz.name } if @options[:polymorphic_on]
+        row_keys = Set.new(relevant_rows.pluck(column[0]).compact)
+
+        # In :eager mode, requesting the primary column triggers loading of all columns
+        row_keys |= eager_load_columns(base_query, relevant_rows) if @options[:mode] == :eager && column == @primary_column
+
+        # Load data for the requested column
+        loaded_hash = load_column_data(column, base_query, row_keys)
+
+        # In :lazy mode, the corresponding primary_column data is loaded with each column
+        load_column_data(primary_column, base_query, loaded_hash.values) if @options[:mode] == :lazy && column != @primary_column # rubocop:disable Metrics/LineLength
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
+
+    def load_column_data(column, base_query, keys)
+      data = if column == @primary_column
+              base_query.where(column[1] => keys).index_by do |row|
+                transform_key(row[column[1]])
+              end
+            else
+              base_query.where(column[1] => keys).pluck(column[1], @primary_column[1]).map do |k, v|
+                [transform_key(k), v]
+              end.to_h
+            end
+      add_to_column(column, as_class(base_query), data)
+      data
+    end
+
+    def add_to_column(column, clazz, entries)
+      @maps[column] ||= {}
+      cache = @maps[column][clazz] ||= {}
+      cache.merge! entries
+      cache
+    end
+
+    def as_class(clazz)
+      clazz.is_a?(ActiveRecord::Relation) ? clazz.model : clazz
+    end
+
+    def get_base_query(row)
+      return row if row.is_a?(Class) || row.is_a?(ActiveRecord::Relation)
+      return @clazz unless @options[:polymorphic_on]
+
+      @clazz.find { |cls| cls.name == row[@options[:polymorphic_on]] }
+    end
+
+    def transform_key(key)
+      key.nil? ? key : key.to_s
+    end
+
+    private
+
+    def load_column(clazz, column, request_column)
+      lkey = loaded_key(clazz, column, request_column)
+      unless @loaded_columns[lkey]
+        yield
+        @loaded_columns[lkey] = true
+      end
+      @maps[column][clazz]
+    end
+
+    def loaded_key(clazz, column, request_column)
+      key_column = @options[:mode] == :eager ? column : request_column || column
+      [*key_column, clazz]
+    end
+
+    def eager_load_columns(base_query, relevant_rows)
+      additional_keys = Set.new
+      @columns.each do |column|
+        next if column == primary_column
+
+        loaded_col = load_column_data(column, base_query, relevant_rows.pluck(column[0]).compact.uniq)
+        additional_keys |= loaded_col.values
+      end
+      additional_keys
+    end
+  end
+end

data/lib/advanced_ar/batch_processor.rb

@@ -0,0 +1,41 @@
+module AdvancedAR
+  # An array that "processes" after so many items are added.
+  #
+  # Example Usage:
+  #   batches = BatchProcessor.new(of: 1000) do |batch|
+  #     # Process the batch somehow
+  #   end
+  #   enumerator_of_some_kind.each { |item| batches << item }
+  #   batches.flush
+  class BatchProcessor
+    attr_reader :batch_size
+
+    def initialize(of: 1000, &blk)
+      @batch_size = of
+      @block = blk
+      @current_batch = []
+    end
+
+    def <<(item)
+      @current_batch << item
+      process_batch if @current_batch.count >= batch_size
+    end
+
+    def add_all(items)
+      items.each do |i|
+        self << i
+      end
+    end
+
+    def flush
+      process_batch if @current_batch.present?
+    end
+
+    protected
+
+    def process_batch
+      @block.call(@current_batch)
+      @current_batch = []
+    end
+  end
+end