upkeep-rails 0.2.5-aarch64-linux-gnu

data/lib/upkeep/subscriptions/layered_reverse_index.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require_relative "lookup_instrumentation"
+
+module Upkeep
+  module Subscriptions
+    class LayeredReverseIndex
+      include LookupInstrumentation
+
+      LOOKUP_NOTIFICATION = LookupInstrumentation::LOOKUP_NOTIFICATION
+
+      def initialize(active_index:, persistent_index:, persistent_count:, store:, pending_index: nil)
+        @active_index = active_index
+        @persistent_index = persistent_index
+        @persistent_count = persistent_count
+        @store = store
+        @pending_index = pending_index
+      end
+
+      def entries_for_without_payload(changes)
+        active_entries = active_index.entries_for(changes)
+        return persistent_index.entries_for(changes) if active_index.count.zero?
+        return active_entries if active_index.covers?(persistent_subscription_count)
+
+        merge_entries(active_entries, persistent_index.entries_for(changes))
+      end
+
+      def entries_for_with_payload(changes, payload)
+        persistent_summary = persistent_index.summary
+        active_entries = active_index.entries_for(changes)
+        active_count = active_index.count
+        pending_entries = pending_entries_for(changes)
+        pending_count = pending_count_for_payload
+        payload[:active_entries] = active_entries.size
+        payload[:active_subscriptions] = active_count
+        payload[:pending_entries] = pending_entries.size
+        payload[:pending_subscriptions] = pending_count
+        payload[:persistent_direct_index_entries] = persistent_summary.fetch(:direct).fetch(:entries)
+        payload[:persistent_shape_index_entries] = persistent_summary.fetch(:shape).fetch(:entries)
+        payload[:persistent_direct_lookup_keys] = persistent_summary.fetch(:direct).fetch(:lookup_keys)
+        payload[:persistent_shape_lookup_keys] = persistent_summary.fetch(:shape).fetch(:lookup_keys)
+        payload[:persistent_shape_keys] = persistent_summary.fetch(:shape).fetch(:shape_keys)
+        payload[:persistent_shape_subscriptions] = persistent_summary.fetch(:shape).fetch(:subscriptions)
+
+        if active_count.zero?
+          persistent_entries = persistent_index.entries_for(changes)
+          payload[:mode] = persistent_entries.empty? && pending_entries.any? ? "pending_activation" : "persistent"
+          payload[:persistent_entries] = persistent_entries.size
+          apply_miss_reason(payload, active_entries: active_entries, persistent_entries: persistent_entries, pending_entries: pending_entries)
+          return persistent_entries
+        end
+
+        if active_index.covers?(persistent_subscription_count)
+          payload[:mode] = "active"
+          payload[:persistent_entries] = 0
+          apply_miss_reason(payload, active_entries: active_entries, persistent_entries: [], pending_entries: pending_entries)
+          return active_entries
+        end
+
+        persistent_entries = persistent_index.entries_for(changes)
+        payload[:mode] = "active_and_persistent"
+        payload[:persistent_entries] = persistent_entries.size
+        apply_miss_reason(payload, active_entries: active_entries, persistent_entries: persistent_entries, pending_entries: pending_entries)
+        merge_entries(active_entries, persistent_entries)
+      end
+
+      def summary
+        persistent = persistent_index.summary
+        active = active_index.summary
+        pending = pending_index&.summary || { lookup_keys: 0, entries: 0, subscriptions: 0 }
+        mode = active_index.covers?(persistent_subscription_count) ? :active : :active_and_persistent
+        totals = if mode == :active
+          active
+        else
+          {
+            lookup_keys: active.fetch(:lookup_keys) + persistent.fetch(:lookup_keys),
+            entries: active.fetch(:entries) + persistent.fetch(:entries)
+          }
+        end
+
+        {
+          lookup_keys: totals.fetch(:lookup_keys),
+          entries: totals.fetch(:entries),
+          mode: mode,
+          active: active,
+          pending: pending,
+          persistent: persistent
+        }
+      end
+
+      private
+
+      attr_reader :active_index, :persistent_index, :persistent_count, :store, :pending_index
+
+      def lookup_store
+        store
+      end
+
+      def persistent_subscription_count
+        persistent_count.call
+      end
+
+      def merge_entries(active_entries, persistent_entries)
+        (active_entries + persistent_entries).uniq do |entry|
+          [entry.subscription_id, entry.owner_id, entry.dependency_cache_key]
+        end
+      end
+
+      def pending_entries_for(changes)
+        pending_index ? pending_index.entries_for(changes) : []
+      end
+
+      def pending_count_for_payload
+        pending_index&.count || 0
+      end
+
+      def apply_miss_reason(payload, active_entries:, persistent_entries:, pending_entries:)
+        return if active_entries.any? || persistent_entries.any?
+
+        payload[:miss_reason] = miss_reason(pending_entries)
+      end
+    end
+  end
+end

data/lib/upkeep/subscriptions/lookup_instrumentation.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+
+module Upkeep
+  module Subscriptions
+    # Shared lookup instrumentation for the memory and layered reverse indexes:
+    # one notification name, one entries_for dispatch, one miss-reason rule.
+    # Including classes provide #lookup_store, #entries_for_with_payload and
+    # #entries_for_without_payload.
+    module LookupInstrumentation
+      LOOKUP_NOTIFICATION = "lookup_subscription_index.upkeep"
+
+      def entries_for(changes)
+        if ActiveSupport::Notifications.notifier.listening?(LOOKUP_NOTIFICATION)
+          payload = { changes: Array(changes).size, store: lookup_store }
+          ActiveSupport::Notifications.instrument(LOOKUP_NOTIFICATION, payload) do
+            entries_for_with_payload(changes, payload)
+          end
+        else
+          entries_for_without_payload(changes)
+        end
+      end
+
+      private
+
+      def miss_reason(pending_entries)
+        pending_entries.any? ? "not_activated_yet" : "no_matching_subscriber"
+      end
+    end
+  end
+end

data/lib/upkeep/subscriptions/persistent_reverse_index.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require_relative "../dependencies"
+require_relative "json_snapshot"
+require_relative "reverse_index"
+
+module Upkeep
+  module Subscriptions
+    class PersistentReverseIndex
+      LOOKUP_COLUMNS = [
+        :subscription_id,
+        :lookup_key_digest,
+        :dependency_source,
+        :lookup_table,
+        :lookup_record_id_snapshot,
+        :lookup_attribute,
+        :dependency_table,
+        :dependency_predicate_digest,
+        :dependency_metadata_snapshot,
+        :owner_ids_snapshot
+      ].freeze
+
+      SHAPE_LOOKUP_COLUMNS = [
+        :subscription_shape_key,
+        :lookup_key_digest,
+        :dependency_source,
+        :lookup_table,
+        :lookup_record_id_snapshot,
+        :lookup_attribute,
+        :dependency_table,
+        :dependency_predicate_digest,
+        :dependency_metadata_snapshot,
+        :owner_ids_snapshot
+      ].freeze
+
+      def initialize(reverse_index:, index_record:, shape_index_record:, subscription_record:)
+        @reverse_index = reverse_index
+        @index_record = index_record
+        @shape_index_record = shape_index_record
+        @subscription_record = subscription_record
+      end
+
+      def entries_for(changes)
+        persistent_entries_for(changes)
+      end
+
+      def summary
+        direct_lookup_key_digests = index_record.distinct.pluck(:lookup_key_digest)
+        shape_lookup_key_digests = shape_index_record.distinct.pluck(:lookup_key_digest)
+        direct_entries = index_record.count
+        shape_entries = shape_index_record.count
+        {
+          lookup_keys: (direct_lookup_key_digests + shape_lookup_key_digests).uniq.size,
+          entries: direct_entries + shape_entries,
+          direct: {
+            lookup_keys: direct_lookup_key_digests.uniq.size,
+            entries: direct_entries
+          },
+          shape: {
+            lookup_keys: shape_lookup_key_digests.uniq.size,
+            entries: shape_entries,
+            shape_keys: shape_index_record.distinct.count(:subscription_shape_key),
+            subscriptions: subscription_record.where.not(subscription_shape_key: nil).count
+          }
+        }
+      end
+
+      def self.digest(value)
+        Digest::SHA256.hexdigest(JSON.generate(canonical_lookup_value(value)))
+      end
+
+      def self.canonical_lookup_value(value)
+        case value
+        when Array
+          value.map { |item| canonical_lookup_value(item) }
+        when Hash
+          value.keys.sort_by(&:to_s).map do |key|
+            [canonical_lookup_value(key), canonical_lookup_value(value.fetch(key))]
+          end
+        when Symbol
+          ["symbol", value.to_s]
+        when String
+          ["string", value.encode(Encoding::UTF_8)]
+        else
+          value
+        end
+      end
+
+      private
+
+      attr_reader :reverse_index, :index_record, :shape_index_record, :subscription_record
+
+      def persistent_entries_for(changes)
+        lookup_keys = Array(changes).flat_map { |change| reverse_index.lookup_keys_for_change(change) }.uniq
+        lookup_keys_by_digest = Hash.new { |hash, digest| hash[digest] = [] }
+        lookup_keys.each do |lookup_key|
+          lookup_keys_by_digest[self.class.digest(lookup_key)] << lookup_key
+        end
+        lookup_key_digests = lookup_keys_by_digest.keys
+
+        direct_entries = index_record
+          .where(lookup_key_digest: lookup_key_digests)
+          .pluck(*LOOKUP_COLUMNS)
+          .flat_map { |row| entries_for_row(row, lookup_keys_by_digest) }
+
+        shape_entries = shape_entries_for(lookup_key_digests, lookup_keys_by_digest)
+
+        (direct_entries + shape_entries).uniq { |entry| [entry.subscription_id, entry.owner_id, entry.dependency_cache_key] }
+      end
+
+      def shape_entries_for(lookup_key_digests, lookup_keys_by_digest)
+        shape_rows = shape_index_record
+          .where(lookup_key_digest: lookup_key_digests)
+          .pluck(*SHAPE_LOOKUP_COLUMNS)
+        return [] if shape_rows.empty?
+
+        subscription_ids_by_shape_key = subscription_ids_by_shape_key(
+          shape_rows.map { |row| row.fetch(0) }.uniq
+        )
+        shape_rows.flat_map do |row|
+          entries_for_shape_row(row, lookup_keys_by_digest, subscription_ids_by_shape_key)
+        end
+      end
+
+      def entries_for_row(row, lookup_keys_by_digest)
+        attributes = LOOKUP_COLUMNS.zip(row).to_h
+        lookup_keys = lookup_keys_by_digest.fetch(attributes.fetch(:lookup_key_digest)) { return [] }
+        return [] unless lookup_keys.any? { |lookup_key| lookup_key_matches_row?(lookup_key, attributes) }
+
+        dependency = dependency_for_row(attributes)
+        dependency_cache_key = dependency.cache_key
+        JsonSnapshot.load(attributes.fetch(:owner_ids_snapshot)).map do |owner_id|
+          ReverseIndex::Entry.new(
+            attributes.fetch(:subscription_id),
+            owner_id,
+            dependency_cache_key,
+            dependency,
+            nil,
+            nil,
+            nil
+          )
+        end
+      end
+
+      def entries_for_shape_row(row, lookup_keys_by_digest, subscription_ids_by_shape_key)
+        attributes = SHAPE_LOOKUP_COLUMNS.zip(row).to_h
+        lookup_keys = lookup_keys_by_digest.fetch(attributes.fetch(:lookup_key_digest)) { return [] }
+        return [] unless lookup_keys.any? { |lookup_key| lookup_key_matches_row?(lookup_key, attributes) }
+
+        dependency = dependency_for_row(attributes)
+        dependency_cache_key = dependency.cache_key
+        subscription_ids = subscription_ids_by_shape_key.fetch(attributes.fetch(:subscription_shape_key), [])
+        subscription_ids.flat_map do |subscription_id|
+          JsonSnapshot.load(attributes.fetch(:owner_ids_snapshot)).map do |owner_id|
+            ReverseIndex::Entry.new(
+              subscription_id,
+              owner_id,
+              dependency_cache_key,
+              dependency,
+              nil,
+              nil,
+              nil
+            )
+          end
+        end
+      end
+
+      def lookup_key_matches_row?(lookup_key, attributes)
+        case lookup_key.fetch(0)
+        when :active_record_attribute
+          lookup_key.fetch(1).to_s == attributes.fetch(:lookup_table).to_s &&
+            lookup_key.fetch(2) == JsonSnapshot.load(attributes.fetch(:lookup_record_id_snapshot)) &&
+            lookup_key.fetch(3).to_s == attributes.fetch(:lookup_attribute).to_s
+        when :active_record_attribute_any_id
+          lookup_key.fetch(1).to_s == attributes.fetch(:lookup_table).to_s &&
+            attributes.fetch(:lookup_record_id_snapshot).nil? &&
+            lookup_key.fetch(2).to_s == attributes.fetch(:lookup_attribute).to_s
+        when :active_record_collection_column
+          lookup_key.fetch(1).to_s == attributes.fetch(:lookup_table).to_s &&
+            attributes.fetch(:lookup_record_id_snapshot).nil? &&
+            lookup_key.fetch(2).to_s == attributes.fetch(:lookup_attribute).to_s
+        else
+          false
+        end
+      end
+
+      def dependency_for_row(attributes)
+        source = attributes.fetch(:dependency_source).to_sym
+        case source
+        when :active_record_attribute
+          metadata_snapshot = attributes[:dependency_metadata_snapshot]
+          metadata = metadata_snapshot && JsonSnapshot.load(metadata_snapshot)
+          Dependencies::ActiveRecordAttribute.new(
+            table: attributes.fetch(:dependency_table),
+            id: attributes[:lookup_record_id_snapshot] && JsonSnapshot.load(attributes.fetch(:lookup_record_id_snapshot)),
+            attribute: attributes.fetch(:lookup_attribute),
+            scope: metadata && metadata[:scope]
+          )
+        when :active_record_collection, :active_record_query
+          metadata = JsonSnapshot.load(attributes.fetch(:dependency_metadata_snapshot))
+          dependency_class = source == :active_record_query ? Dependencies::ActiveRecordQuery : Dependencies::ActiveRecordCollection
+          dependency_class.new(
+            primary_table: attributes.fetch(:dependency_table),
+            table_columns: metadata.fetch(:table_columns),
+            coverage: metadata.fetch(:coverage),
+            sql: metadata.fetch(:sql),
+            predicates: metadata.fetch(:predicates)
+          )
+        else
+          raise ArgumentError, "unsupported persistent dependency source: #{source.inspect}"
+        end
+      end
+
+      def subscription_ids_by_shape_key(shape_keys)
+        shape_key_lookup = shape_keys.to_h { |shape_key| [shape_key, []] }
+        subscription_record
+          .where(subscription_shape_key: shape_keys)
+          .pluck(:subscription_shape_key, :id)
+          .each do |shape_key, id|
+            shape_key_lookup[shape_key] << id if shape_key_lookup.key?(shape_key)
+          end
+        shape_key_lookup.transform_values { |ids| ids.sort_by(&:to_s) }
+      end
+    end
+  end
+end

data/lib/upkeep/subscriptions/registrar.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "shape"
+
+module Upkeep
+  module Subscriptions
+    class Registrar
+      Registration = Data.define(:identity, :decision, :subscription, :shape)
+
+      def initialize(store:, shape_cache: ShapeCache.new)
+        @store = store
+        @shape_cache = shape_cache
+      end
+
+      def register(identity:, decision:, recorder:, metadata: {}, signature: nil)
+        shape = shape_cache.resolve(recorder: recorder, decision: decision, signature: signature)
+        subscription = store.register(
+          subscriber_id: identity.subscriber_id,
+          recorder: recorder,
+          metadata: metadata.merge(
+            shared_stream_names: shape.shared_stream_names,
+            subscription_shape_key: shape.key,
+            subscription_shape_cache: shape.cache_state
+          ).compact,
+          entries: shape.entries
+        )
+
+        Registration.new(identity, decision, subscription, shape)
+      end
+
+      private
+
+      attr_reader :store, :shape_cache
+    end
+  end
+end