upkeep-rails 0.1.9

data/lib/upkeep/subscriptions/store.rb

@@ -0,0 +1,375 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+require "active_support/notifications"
+require_relative "active_registry"
+
+module Upkeep
+  module Subscriptions
+    class NotFound < KeyError; end
+
+    Subscription = Data.define(:id, :subscriber_id, :recorder, :graph, :metadata) do
+      def explain
+        dependency_nodes = graph.dependency_nodes
+        dependencies = dependency_nodes.map(&:payload)
+
+        {
+          id: id,
+          subscriber_id: subscriber_id,
+          tables: active_record_tables(dependencies),
+          identity: identity_dependencies(dependencies),
+          frame_count: graph.frame_nodes.size,
+          dependency_count: dependency_nodes.size,
+          lookup_keys: lookup_keys_for(dependencies),
+          metadata: explain_metadata
+        }
+      end
+
+      def identity_signature(frame_id)
+        recorder.identity_signature(frame_id)
+      end
+
+      def replay_recipe(frame_id)
+        graph.node(frame_id).payload[:recipe]
+      end
+
+      def to_h
+        {
+          id: id,
+          subscriber_id: subscriber_id,
+          recorder: recorder.to_h,
+          metadata: metadata
+        }
+      end
+
+      def to_persistent_h
+        {
+          id: id,
+          subscriber_id: subscriber_id,
+          recorder: recorder.to_persistent_h,
+          metadata: metadata
+        }
+      end
+
+      def self.from_h(snapshot)
+        snapshot = Dependencies.symbolize_keys(snapshot)
+        recorder = Runtime::Recorder.from_h(snapshot.fetch(:recorder))
+
+        new(
+          snapshot.fetch(:id),
+          snapshot.fetch(:subscriber_id),
+          recorder,
+          recorder.graph,
+          snapshot.fetch(:metadata)
+        )
+      end
+
+      private
+
+      def active_record_tables(dependencies)
+        tables = Hash.new { |hash, table| hash[table] = [] }
+
+        dependencies.each do |dependency|
+          case dependency.source
+          when :active_record_attribute
+            tables[dependency.key.fetch(:table).to_s] << dependency.key.fetch(:attribute).to_s
+          when :active_record_collection, :active_record_query
+            dependency.metadata.fetch(:table_columns, {}).each do |table, columns|
+              tables[table.to_s].concat(Array(columns).map(&:to_s))
+            end
+          end
+        end
+
+        tables.transform_values { |columns| columns.uniq.sort }.sort.to_h
+      end
+
+      def identity_dependencies(dependencies)
+        dependencies.select(&:identity?).map do |dependency|
+          {
+            source: dependency.source.to_s,
+            key: dependency.key.fetch(:key),
+            value: dependency.key.fetch(:value),
+            partitioning: Dependencies.partitioning_identity?(dependency)
+          }
+        end.sort_by(&:inspect)
+      end
+
+      def lookup_keys_for(dependencies)
+        index = ReverseIndex.new
+        dependencies.flat_map { |dependency| index.lookup_keys_for_dependency(dependency) }.uniq.sort_by(&:inspect)
+      end
+
+      def explain_metadata
+        keys = [
+          :path,
+          "path",
+          :stream_name,
+          "stream_name",
+          :shared_stream_names,
+          "shared_stream_names",
+          :identity_mode,
+          "identity_mode",
+          :identity_sources,
+          "identity_sources",
+          :identity_names,
+          "identity_names",
+          :subscription_shape_key,
+          "subscription_shape_key"
+        ]
+
+        metadata.to_h.select { |key, _value| keys.include?(key) }
+      end
+    end
+
+    class MemoryReverseIndex
+      LOOKUP_NOTIFICATION = "lookup_subscription_index.upkeep"
+
+      def initialize(active_registry:, pending_registry:)
+        @active_registry = active_registry
+        @pending_registry = pending_registry
+      end
+
+      def entries_for(changes)
+        if ActiveSupport::Notifications.notifier.listening?(LOOKUP_NOTIFICATION)
+          payload = { changes: Array(changes).size, store: "memory" }
+          ActiveSupport::Notifications.instrument(LOOKUP_NOTIFICATION, payload) do
+            entries_for_with_payload(changes, payload)
+          end
+        else
+          active_registry.entries_for(changes)
+        end
+      end
+
+      def summary
+        active_registry.summary
+      end
+
+      private
+
+      attr_reader :active_registry, :pending_registry
+
+      def entries_for_with_payload(changes, payload)
+        active_entries = active_registry.entries_for(changes)
+        pending_entries = pending_registry.entries_for(changes)
+
+        payload.merge!(
+          active_entries: active_entries.size,
+          active_subscriptions: active_registry.count,
+          pending_entries: pending_entries.size,
+          pending_subscriptions: pending_registry.count,
+          persistent_entries: 0,
+          persistent_direct_index_entries: 0,
+          persistent_shape_index_entries: 0,
+          persistent_direct_lookup_keys: 0,
+          persistent_shape_lookup_keys: 0,
+          persistent_shape_keys: 0,
+          persistent_shape_subscriptions: 0,
+          mode: active_entries.empty? && pending_entries.any? ? "pending_activation" : "active"
+        )
+
+        payload[:miss_reason] = active_entries.empty? ? miss_reason(pending_entries) : nil
+        payload.delete(:miss_reason) unless payload[:miss_reason]
+
+        active_entries
+      end
+
+      def miss_reason(pending_entries)
+        pending_entries.any? ? "not_activated_yet" : "no_matching_subscriber"
+      end
+    end
+
+    class Store
+      PERSIST_NOTIFICATION = "persist_subscription_store.upkeep"
+
+      attr_reader :reverse_index
+
+      def initialize(reverse_index: ReverseIndex.new)
+        @active_registry = ActiveRegistry.new(reverse_index: reverse_index)
+        @pending_registry = ActiveRegistry.new
+        @pending_index_entries = {}
+        @reverse_index = MemoryReverseIndex.new(active_registry: active_registry, pending_registry: pending_registry)
+        @next_id = 0
+      end
+
+      def register(subscriber_id:, recorder:, metadata: {}, entries: nil)
+        if ActiveSupport::Notifications.notifier.listening?(PERSIST_NOTIFICATION)
+          payload = memory_persist_payload(operation: :persist_subscription)
+          ActiveSupport::Notifications.instrument(PERSIST_NOTIFICATION, payload) do
+            register_subscription(subscriber_id: subscriber_id, recorder: recorder, metadata: metadata, entries: entries, payload: payload)
+          end
+        else
+          register_subscription(subscriber_id: subscriber_id, recorder: recorder, metadata: metadata, entries: entries)
+        end
+      end
+
+      def touch(id, now: Time.now)
+        fetch(id)
+        metadata = { "last_seen_at" => now.utc.iso8601 }
+        pending_registry.touch(id, metadata: metadata)
+        active_registry.touch(id, metadata: metadata)
+        true
+      end
+
+      def prune_stale!(older_than:)
+        stale_ids = subscriptions.filter_map do |subscription|
+          id = subscription.id
+          id if last_seen_at(subscription) && last_seen_at(subscription) < older_than
+        end
+
+        unregister(stale_ids)
+        stale_ids.size
+      end
+
+      def unregister(ids)
+        ids = Array(ids)
+        ids.each { |id| @pending_index_entries.delete(id) }
+        pending_registry.unregister(ids)
+        active_registry.unregister(ids)
+        ids.size
+      end
+
+      def activate(id)
+        if ActiveSupport::Notifications.notifier.listening?(PERSIST_NOTIFICATION)
+          payload = memory_persist_payload(operation: :persist_index)
+          ActiveSupport::Notifications.instrument(PERSIST_NOTIFICATION, payload) do
+            activate_subscription(id, payload: payload)
+          end
+        else
+          activate_subscription(id)
+        end
+      end
+
+      def drain
+        true
+      end
+
+      def shutdown
+        true
+      end
+
+      def fetch(id)
+        active_registry.fetch(id) || pending_registry.fetch(id) || raise(NotFound, id)
+      end
+
+      def explain(id)
+        fetch(id).explain
+      end
+
+      def subscriptions
+        active_registry.subscriptions + pending_registry.subscriptions
+      end
+
+      def reset
+        @active_registry = ActiveRegistry.new
+        @pending_registry = ActiveRegistry.new
+        @pending_index_entries = {}
+        @reverse_index = MemoryReverseIndex.new(active_registry: active_registry, pending_registry: pending_registry)
+        @next_id = 0
+      end
+
+      def summary
+        active = active_registry.summary
+        pending = pending_registry.summary
+        {
+          subscriptions: subscriptions.size,
+          pending_subscriptions: pending_registry.count,
+          active_subscriptions: active_registry.count,
+          deferred_index_subscriptions: 0,
+          reverse_index: active.merge(
+            mode: :active,
+            active: active,
+            pending: pending,
+            persistent: { lookup_keys: 0, entries: 0 }
+          )
+        }
+      end
+
+      private
+
+      attr_reader :pending_registry, :active_registry
+
+      def register_subscription(subscriber_id:, recorder:, metadata: {}, entries: nil, payload: nil)
+        recorder.flush_pending_dependencies if recorder.respond_to?(:flush_pending_dependencies)
+        subscription = Subscription.new(
+          next_subscription_id,
+          subscriber_id,
+          recorder,
+          recorder.graph,
+          metadata
+        )
+
+        if payload
+          entry_count = dependency_entry_count(subscription, entries)
+          payload[:pending_index_entries] = entry_count
+          payload[:subscription_rows] = 1
+          payload[:index_rows] = 0
+          payload[:direct_index_rows] = 0
+          payload[:shape_index_rows] = 0
+        end
+
+        pending_registry.register(subscription, entries: entries)
+        @pending_index_entries[subscription.id] = entries if entries
+        subscription
+      end
+
+      def activate_subscription(id, payload: nil)
+        if active_registry.fetch(id)
+          payload.merge!(subscription_rows: 0, index_rows: 0, direct_index_rows: 0, shape_index_rows: 0) if payload
+          return true
+        end
+
+        subscription = pending_registry.fetch(id)
+        unless subscription
+          payload.merge!(subscription_rows: 0, index_rows: 0, direct_index_rows: 0, shape_index_rows: 0) if payload
+          return false
+        end
+
+        entries = @pending_index_entries.delete(id)
+        if payload
+          entry_count = dependency_entry_count(subscription, entries)
+          payload[:dependency_entries] = entry_count
+          payload[:subscription_rows] = 0
+          payload[:index_rows] = entry_count
+          payload[:direct_index_rows] = entry_count
+          payload[:shape_index_rows] = 0
+        end
+
+        active_registry.register(subscription, entries: entries)
+        pending_registry.unregister(id)
+        true
+      end
+
+      def memory_persist_payload(operation:)
+        {
+          store: "memory",
+          jobs: 1,
+          subscriptions: operation == :persist_subscription ? 1 : 0,
+          index_jobs: operation == :persist_index ? 1 : 0,
+          dependency_entries: 0,
+          pending_index_entries: 0,
+          operations: { operation.to_s => 1 }
+        }
+      end
+
+      def dependency_entry_count(subscription, entries)
+        index = ReverseIndex.new
+        materialized = if entries
+          index.entries_for_subscription_instance(entries, subscription)
+        else
+          index.entries_for_subscription(subscription)
+        end
+        materialized.uniq { |entry| [entry.owner_id, entry.dependency_cache_key] }.size
+      end
+
+      def last_seen_at(subscription)
+        value = subscription.metadata["last_seen_at"] || subscription.metadata[:last_seen_at]
+        Time.parse(value.to_s) if value
+      end
+
+      def next_subscription_id
+        "subscription-#{SecureRandom.uuid}"
+      end
+    end
+  end
+end

data/lib/upkeep/subscriptions.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative "subscriptions/reverse_index"
+require_relative "subscriptions/shape"
+require_relative "subscriptions/registrar"
+require_relative "subscriptions/store"
+require_relative "subscriptions/active_record_store"

data/lib/upkeep/targeting.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "digest"
+require "nokogiri"
+
+module Upkeep
+  module Targeting
+    Target = Data.define(:kind, :id, :reason)
+    Patch = Data.define(:target, :html)
+
+    class Selector
+      def select(recorder, changes)
+        graph = recorder.graph
+
+        frame_nodes =
+          graph.dependency_node_ids_matching(changes)
+            .flat_map { |dependency_id| graph.dependency_owner_ids(dependency_id) }
+            .flat_map { |owner_id| graph.nearest_frame_nodes_from(owner_id) }
+
+        uniq_targets(remove_contained_frames(graph, frame_nodes).filter_map { |frame| target_for_frame(frame) })
+      end
+
+      private
+
+      def remove_contained_frames(graph, frames)
+        frames.uniq(&:id).reject do |frame|
+          frames.any? { |candidate| candidate.id != frame.id && graph.contained_by?(frame.id, candidate.id) }
+        end
+      end
+
+      def target_for_frame(frame)
+        case frame.payload.fetch(:kind)
+        when "page"
+          Target.new("page", frame.id, "page frame dependency matched committed change")
+        when "render_site"
+          Target.new("render_site", frame.payload.fetch(:site_id), "render-site dependency matched committed change")
+        when "fragment"
+          Target.new("fragment", frame.id, "record attribute read matched committed attributes")
+        end
+      end
+
+      def uniq_targets(targets)
+        targets.uniq { |target| [target.kind, target.id] }
+      end
+    end
+
+    class Patcher
+      def initialize(html)
+        @fragment = Extraction.parse_html(html)
+      end
+
+      def apply(patches)
+        patches.each { |patch| apply_patch(patch) }
+        fragment.to_html
+      end
+
+      private
+
+      attr_reader :fragment
+
+      def apply_patch(patch)
+        node = node_for(patch.target)
+        raise "target not found in current DOM: #{patch.target.inspect}" unless node
+
+        replacement = replacement_for(patch)
+        node.replace(replacement)
+      end
+
+      def node_for(target)
+        Extraction.node_for(fragment, target)
+      end
+
+      def replacement_for(patch)
+        parsed = Extraction.parse_html(patch.html)
+        Extraction.node_for(parsed, patch.target) ||
+          parsed.children.find { |child| child.element? } ||
+          parsed.at_css("body > *")
+      end
+    end
+
+    module Extraction
+      module_function
+
+      def patches_from_full_rerender(full_html, targets)
+        targets.map { |target| Patch.new(target, extract_target_html(full_html, target)) }
+      end
+
+      def extract_target_html(html, target)
+        fragment = parse_html(html)
+        node = node_for(fragment, target)
+
+        raise "target not found in full rerender: #{target.inspect}" unless node
+
+        node.to_html
+      end
+
+      def node_for(fragment, target)
+        case target.kind
+        when "page"
+          fragment.at_css(%([data-upkeep-page-frame="#{css_escape(target.id)}"]))
+        when "fragment"
+          fragment.at_css(%([data-upkeep-frame="#{css_escape(target.id)}"]))
+        when "render_site"
+          fragment.at_css(%([data-upkeep-render-site="#{css_escape(target.id)}"]))
+        end
+      end
+
+      def normalize_html(html)
+        parse_html(html).to_html
+      end
+
+      def digest_html(html)
+        Digest::SHA256.hexdigest(normalize_html(html))
+      end
+
+      def frame_id_for(target)
+        target.kind == "render_site" ? "site:#{target.id}" : target.id
+      end
+
+      def css_escape(value)
+        value.to_s.gsub("\\", "\\\\\\").gsub('"', '\"')
+      end
+
+      def parse_html(html)
+        source = html.to_s
+
+        if source.match?(/\A\s*(?:<!doctype\b[^>]*>\s*)?<html[\s>]/i)
+          Nokogiri::HTML5.parse(source)
+        else
+          Nokogiri::HTML5.fragment(source)
+        end
+      end
+    end
+  end
+end

data/lib/upkeep/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Upkeep
+  VERSION = "0.1.9"
+end

data/lib/upkeep-rails.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "upkeep"

data/lib/upkeep.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "upkeep/version"
+require_relative "upkeep/dependencies"
+require_relative "upkeep/dag"
+require_relative "upkeep/replay"
+require_relative "upkeep/active_record_query"
+require_relative "upkeep/runtime"
+require_relative "upkeep/targeting"
+require_relative "upkeep/subscriptions"
+require_relative "upkeep/shared_streams"
+require_relative "upkeep/invalidation"
+require_relative "upkeep/delivery"
+require_relative "upkeep/rails"

data/upkeep-rails.gemspec

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "lib/upkeep/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "upkeep-rails"
+  spec.version = Upkeep::VERSION
+  spec.authors = [ "Felipe dos Anjos" ]
+  spec.email = [ "felipe.cavalheiro.anjos@gmail.com" ]
+  spec.license = "MIT"
+
+  spec.summary = "Dependency-tracked live updates for Rails views"
+  spec.description = "Upkeep records the data and identity dependencies used while Rails renders a view, then updates subscribed frames when matching application data changes."
+  spec.homepage = "https://github.com/fc-anjos/upkeep-rails"
+  spec.required_ruby_version = ">= 3.2.0"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "#{spec.homepage}/tree/v#{spec.version}"
+  spec.metadata["bug_tracker_uri"] = "#{spec.homepage}/issues"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  internal_files = Dir[
+    "lib/upkeep/probes/**/*.rb",
+    "lib/upkeep/proofs/**/*.rb"
+  ] + %w[
+    lib/upkeep/domain.rb
+    lib/upkeep/herb/fallback_analyzer.rb
+    lib/upkeep/herb/performance_gate.rb
+    lib/upkeep/herb/runtime_alignment.rb
+    lib/upkeep/proof_support.rb
+    lib/upkeep/rendering.rb
+    lib/upkeep/templates.rb
+  ]
+
+  spec.files = (Dir[
+    "README.md",
+    "docs/**/*.md",
+    "LICENSE.txt",
+    "upkeep-rails.gemspec",
+    "lib/**/*.rb",
+    "lib/generators/**/templates/**/*"
+  ] - internal_files).sort
+  spec.require_paths = [ "lib" ]
+
+  spec.add_dependency "actionview", ">= 7.1", "< 9.0"
+  spec.add_dependency "actionpack", ">= 7.1", "< 9.0"
+  spec.add_dependency "actioncable", ">= 7.1", "< 9.0"
+  spec.add_dependency "activejob", ">= 7.1", "< 9.0"
+  spec.add_dependency "activerecord", ">= 7.1", "< 9.0"
+  spec.add_dependency "activesupport", ">= 7.1", "< 9.0"
+  spec.add_dependency "herb", ">= 0.10.1", "< 0.11"
+  spec.add_dependency "nokogiri", ">= 1.15", "< 2.0"
+  spec.add_dependency "railties", ">= 7.1", "< 9.0"
+  spec.add_dependency "turbo-rails", ">= 2.0", "< 3.0"
+end