upkeep-rails 0.1.9

data/lib/upkeep/dag/subscription_shape.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../shared_streams"
+require_relative "../version"
+
+module Upkeep
+  module DAG
+    class SubscriptionShape
+      DIGEST_SCOPE = "upkeep-subscription-shape"
+      FRAME_PAYLOAD_SHAPE_IGNORED_KEYS = %i[manifest recipe].freeze
+
+      attr_reader :signature
+
+      def self.from_graph(graph, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms(graph)))
+      end
+
+      def self.from_components(graph_component, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, terms_for_component(graph_component)))
+      end
+
+      def self.from_terms(graph_terms, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms))
+      end
+
+      def self.from_trace_digest(trace_digest, request_signature: nil)
+        new(signature: signature_for_trace_digest(request_signature, trace_digest))
+      end
+
+      def self.signature_for_terms(request_signature, graph_terms)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        %i[frames dependencies contains].each do |group|
+          digest.update("\0")
+          digest.update(group.to_s)
+          Array(graph_terms[group]).each do |term|
+            digest.update("\0")
+            digest.update(term)
+          end
+        end
+        digest.hexdigest
+      end
+
+      def self.signature_for_trace_digest(request_signature, trace_digest)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        digest.update("\0")
+        digest.update(trace_digest)
+        digest.hexdigest
+      end
+
+      def self.request_signature_component(signature)
+        return nil unless signature
+
+        signature.respond_to?(:to_h) ? signature.to_h : signature
+      end
+
+      def self.graph_component(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_component(node.id, node.payload) }.sort_by { |component| component.fetch(:id).to_s },
+          dependencies: graph.dependency_nodes.map { |node| dependency_component(graph, node) }.sort_by { |component| component.fetch(:id).inspect },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| [edge.from, edge.to] }
+            .sort_by(&:inspect)
+        }
+      end
+
+      def self.graph_terms(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_term(node.id, node.payload) },
+          dependencies: graph.dependency_nodes.map { |node| dependency_term(node.id, node.payload, graph.dependency_owner_ids(node.id)) },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| contains_term(edge.from, edge.to) }
+        }
+      end
+
+      def self.terms_for_component(graph_component)
+        {
+          frames: graph_component.fetch(:frames).map { |component| canonical_term(:frame, component.fetch(:id), component.fetch(:payload)) },
+          dependencies: graph_component.fetch(:dependencies).map do |component|
+            canonical_term(:dependency, component.fetch(:id), component.fetch(:dependency), component.fetch(:owners))
+          end,
+          contains: graph_component.fetch(:contains).map { |from, to| contains_term(from, to) }
+        }
+      end
+
+      def self.frame_component(id, payload)
+        {
+          id: id,
+          payload: frame_payload_component(payload)
+        }
+      end
+
+      def self.frame_term(id, payload)
+        canonical_term(:frame, id, frame_payload_component(payload))
+      end
+
+      def self.frame_payload_component(payload)
+        component = payload.reject do |key, _value|
+          key.respond_to?(:to_sym) && FRAME_PAYLOAD_SHAPE_IGNORED_KEYS.include?(key.to_sym)
+        end
+        component = shape_value(component)
+        recipe = payload[:recipe] || payload["recipe"]
+        kind = payload[:kind] || payload["kind"]
+        if recipe && kind.to_s == "render_site"
+          component[:shared_stream_signature] = SharedStreams.signature_for(recipe)
+        end
+        component
+      end
+
+      def self.dependency_component(graph, node)
+        {
+          id: node.id,
+          dependency: shape_value(node.payload.to_h),
+          owners: graph.dependency_owner_ids(node.id).sort_by(&:to_s)
+        }
+      end
+
+      def self.dependency_term(id, dependency, owners)
+        canonical_term(:dependency, id, dependency.to_h, owners.sort_by(&:to_s))
+      end
+
+      def self.contains_term(from, to)
+        canonical_term(:contains, from, to)
+      end
+
+      def self.shape_value(value)
+        case value
+        when Hash
+          value.keys.sort_by(&:to_s).to_h { |key| [key, shape_value(value.fetch(key))] }
+        when Array
+          value.map { |item| shape_value(item) }
+        else
+          value.respond_to?(:to_h) ? shape_value(value.to_h) : value
+        end
+      end
+
+      def self.canonical_term(*parts)
+        parts.map { |part| canonical_value(part) }.join("\0")
+      end
+
+      def self.canonical_value(value)
+        shape_value(value).inspect
+      end
+
+      def initialize(signature:)
+        @signature = signature
+      end
+
+      class Trace
+        def initialize(graph_version:)
+          @graph_version = graph_version
+          @seen_frame_ids = {}
+          @seen_dependency_keys = {}
+          @seen_dependency_owner_ids_by_key = Hash.new { |owners, dependency_key| owners[dependency_key] = {} }
+          @seen_contains_edges = {}
+          @digest = Digest::SHA256.new
+          @digest.update("subscription-shape-trace")
+          @invalid = false
+          @recorded = false
+        end
+
+        def synchronized_with?(graph)
+          !@invalid && @graph_version == graph.version
+        end
+
+        def invalidate!
+          @invalid = true
+        end
+
+        def record_frame(frame_id, metadata, parent_id:, graph_version:)
+          return if @invalid
+
+          unless @seen_frame_ids.key?(frame_id)
+            @seen_frame_ids[frame_id] = true
+            record_digest_term(:frame, SubscriptionShape.frame_term(frame_id, metadata))
+          end
+          edge_key = [parent_id, frame_id]
+          unless @seen_contains_edges.key?(edge_key)
+            @seen_contains_edges[edge_key] = true
+            record_digest_term(:contains, SubscriptionShape.contains_term(parent_id, frame_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def record_dependency(owner_id, dependency, graph_version:)
+          return if @invalid
+
+          dependency_cache_key = dependency.cache_key
+          unless @seen_dependency_keys.key?(dependency_cache_key)
+            @seen_dependency_keys[dependency_cache_key] = true
+            dependency_payload = SubscriptionShape.shape_value(dependency.to_h)
+            record_digest_term(:dependency, SubscriptionShape.canonical_term(:dependency, dependency_cache_key, dependency_payload))
+          end
+          unless @seen_dependency_owner_ids_by_key[dependency_cache_key].key?(owner_id)
+            @seen_dependency_owner_ids_by_key[dependency_cache_key][owner_id] = true
+            record_digest_term(:dependency_owner, SubscriptionShape.canonical_term(:dependency_owner, dependency_cache_key, owner_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def covers?(graph)
+          synchronized_with?(graph) && (recorded? || graph_shape_empty?(graph))
+        end
+
+        def subscription_shape(request_signature: nil)
+          SubscriptionShape.from_trace_digest(@digest.hexdigest, request_signature: request_signature)
+        end
+
+        private
+
+        def recorded?
+          @recorded
+        end
+
+        def graph_shape_empty?(graph)
+          graph.frame_nodes.empty? &&
+            graph.dependency_nodes.empty? &&
+            graph.edges.none? { |edge| edge.reason == :contains }
+        end
+
+        def record_digest_term(kind, term)
+          @digest.update("\0")
+          @digest.update(kind.to_s)
+          @digest.update("\0")
+          @digest.update(term)
+        end
+      end
+    end
+  end
+end

data/lib/upkeep/dag.rb

@@ -0,0 +1,370 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Upkeep
+  module DAG
+    Node = Data.define(:id, :kind, :payload)
+    Edge = Data.define(:from, :to, :reason)
+
+    class Graph
+      attr_reader :nodes, :edges, :version
+
+      def initialize
+        @nodes = {}
+        @edges = []
+        @version = 0
+        reset_indexes!
+      end
+
+      def add_node(id, kind:, payload: {})
+        return nodes.fetch(id) if nodes.key?(id)
+
+        @version += 1
+        nodes[id] = Node.new(id, kind, payload)
+      end
+
+      def add_edge(from, to, reason:)
+        key = edge_key(from, to, reason)
+        return false if @edge_keys[key]
+
+        @edge_keys[key] = true
+        @version += 1
+        edge = Edge.new(from, to, reason)
+        edges << edge
+        @outgoing_edges_by_from[from] << edge
+        @incoming_edges_by_to[to] << edge
+        @dependency_owner_ids_by_node[to] << from if reason == :depends_on
+        true
+      end
+
+      def add_dependency(owner_id, dependency)
+        dependency_cache_key = dependency.cache_key
+        dependency_cache_keys = @dependency_cache_keys_by_node[owner_id]
+        return false if dependency_cache_keys.key?(dependency_cache_key)
+
+        add_node(owner_id, kind: :unknown) unless nodes.key?(owner_id)
+        add_node(dependency_cache_key, kind: :dependency, payload: dependency)
+        add_edge(owner_id, dependency_cache_key, reason: :depends_on)
+
+        dependency_cache_keys[dependency_cache_key] = true
+        @dependencies_by_node[owner_id] << dependency
+        true
+      end
+
+      def dependencies_for(node_id)
+        @dependencies_by_node[node_id]
+      end
+
+      def node(id)
+        nodes.fetch(id)
+      end
+
+      def node?(id)
+        nodes.key?(id)
+      end
+
+      def outgoing_edges(from, reason: nil)
+        indexed_edges = @outgoing_edges_by_from.fetch(from, [])
+        return indexed_edges.dup unless reason
+
+        indexed_edges.select { |edge| edge.reason == reason }
+      end
+
+      def incoming_edges(to, reason: nil)
+        indexed_edges = @incoming_edges_by_to.fetch(to, [])
+        return indexed_edges.dup unless reason
+
+        indexed_edges.select { |edge| edge.reason == reason }
+      end
+
+      def dependency_owner_ids(dependency_node_id)
+        @dependency_owner_ids_by_node.fetch(dependency_node_id, []).dup
+      end
+
+      def dependency_node_ids_matching(changes)
+        dependency_nodes.filter_map do |node|
+          node.id if changes.any? { |change| node.payload.matches_change?(change) }
+        end
+      end
+
+      def nearest_frame_nodes_from(node_id)
+        current = node(node_id)
+        return [current] if current.kind == :frame
+
+        queue = outgoing_edges(node_id, reason: :contains).map(&:to)
+        visited = {}
+        frames = []
+
+        until queue.empty?
+          id = queue.shift
+          next if visited[id]
+
+          visited[id] = true
+          current = node(id)
+          if current.kind == :frame
+            frames << current
+          else
+            queue.concat(outgoing_edges(id, reason: :contains).map(&:to))
+          end
+        end
+
+        frames
+      end
+
+      def ancestor_node_ids(node_id)
+        ancestors = []
+        current = node_id
+
+        while (edge = incoming_edges(current, reason: :contains).first)
+          ancestors << edge.from
+          current = edge.from
+        end
+
+        ancestors
+      end
+
+      def contained_by?(descendant_id, ancestor_id)
+        ancestor_node_ids(descendant_id).include?(ancestor_id)
+      end
+
+      def contained_node_ids(node_id)
+        ids = []
+        queue = [node_id]
+        visited = {}
+
+        until queue.empty?
+          id = queue.shift
+          next if visited[id]
+
+          visited[id] = true
+          ids << id
+          queue.concat(outgoing_edges(id, reason: :contains).map(&:to))
+        end
+
+        ids
+      end
+
+      def frame_nodes
+        nodes.values.select { |node| node.kind == :frame }
+      end
+
+      def dependency_nodes
+        nodes.values.select { |node| node.kind == :dependency }
+      end
+
+      def summary
+        {
+          nodes: nodes.size,
+          edges: edges.size,
+          frames: frame_nodes.size,
+          manifest_attached_frames: frame_nodes.count { |node| node.payload[:manifest_path] },
+          dependencies: dependency_nodes.size,
+          containment_edges: edges.count { |edge| edge.reason == :contains },
+          dependency_edges: edges.count { |edge| edge.reason == :depends_on },
+          replay_recipes: frame_nodes.count { |node| node.payload[:recipe] },
+          replay_recipe_kinds: frame_nodes.filter_map { |node| node.payload[:recipe]&.kind }.map(&:to_s).uniq.sort,
+          dependency_sources: dependency_nodes.map { |node| node.payload.source.to_s }.uniq.sort
+        }
+      end
+
+      def report
+        {
+          summary: summary,
+          frames: frame_reports,
+          dependencies: dependency_reports,
+          edges: edges.map(&:to_h)
+        }
+      end
+
+      def to_h(dependencies: :all)
+        serialized_nodes = serializable_nodes(dependencies: dependencies)
+        node_ids = serialized_nodes.to_h { |node| [node.id, true] }
+
+        {
+          nodes: serialized_nodes.map { |node| serialize_node(node) },
+          edges: edges.select { |edge| node_ids[edge.from] && node_ids[edge.to] }.map(&:to_h)
+        }
+      end
+
+      def self.from_h(snapshot)
+        snapshot = symbolize_keys(snapshot)
+        graph = new
+        graph.nodes.clear
+        graph.edges.clear
+        graph.send(:reset_indexes!)
+
+        snapshot.fetch(:nodes).each do |node_snapshot|
+          node_snapshot = symbolize_keys(node_snapshot)
+          kind = node_snapshot.fetch(:kind).to_sym
+          graph.nodes[node_snapshot.fetch(:id)] = Node.new(
+            node_snapshot.fetch(:id),
+            kind,
+            deserialize_payload(kind, node_snapshot.fetch(:payload))
+          )
+        end
+
+        snapshot.fetch(:edges).each do |edge_snapshot|
+          edge_snapshot = symbolize_keys(edge_snapshot)
+          graph.add_edge(
+            edge_snapshot.fetch(:from),
+            edge_snapshot.fetch(:to),
+            reason: edge_snapshot.fetch(:reason).to_sym
+          )
+        end
+
+        graph.send(:rebuild_dependency_index)
+        graph
+      end
+
+      def frame_reports
+        frame_nodes.map do |node|
+          {
+            id: node.id,
+            kind: node.payload.fetch(:kind),
+            template: node.payload[:template],
+            site_id: node.payload[:site_id],
+            manifest_path: node.payload[:manifest_path],
+            manifest_fingerprint: node.payload[:manifest_fingerprint],
+            locals: node.payload[:locals],
+            contains: outgoing_edges(node.id, reason: :contains).map(&:to),
+            dependencies: dependencies_for(node.id).map(&:to_h),
+            replay_recipe: recipe_report(node.payload[:recipe])
+          }.compact
+        end
+      end
+
+      def recipe_report(recipe)
+        return unless recipe
+
+        snapshot = recipe.to_h
+        replay = snapshot[:replay] || snapshot["replay"] || {}
+        replay_json = JSON.generate(replay)
+        {
+          kind: recipe.kind.to_s,
+          target_kind: recipe.target_kind,
+          target_id: recipe.target_id,
+          runtime: recipe.runtime,
+          template: recipe.template,
+          replay: {
+            type: recipe.replay.respond_to?(:type) ? recipe.replay.type : nil,
+            keys: replay.keys.map(&:to_s).sort,
+            bytes: replay_json.bytesize,
+            digest: Digest::SHA256.hexdigest(replay_json)
+          }.compact
+        }.compact
+      end
+
+      def dependency_reports
+        dependency_nodes.map do |node|
+          {
+            id: node.id,
+            dependency: node.payload.to_h,
+            owners: dependency_owner_ids(node.id)
+          }
+        end
+      end
+
+      private
+
+      def serializable_nodes(dependencies:)
+        case dependencies
+        when :all
+          nodes.values
+        when :identity
+          nodes.values.select { |node| node.kind != :dependency || node.payload.identity? }
+        else
+          raise ArgumentError, "unsupported graph dependency serialization mode: #{dependencies.inspect}"
+        end
+      end
+
+      def reset_indexes!
+        @edge_keys = {}
+        @outgoing_edges_by_from = Hash.new { |hash, key| hash[key] = [] }
+        @incoming_edges_by_to = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_owner_ids_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependencies_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_cache_keys_by_node = Hash.new { |hash, key| hash[key] = {} }
+      end
+
+      def rebuild_dependency_index
+        @dependencies_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_cache_keys_by_node = Hash.new { |hash, key| hash[key] = {} }
+
+        edges.each do |edge|
+          next unless edge.reason == :depends_on
+
+          dependency = nodes.fetch(edge.to).payload
+          dependency_cache_keys = @dependency_cache_keys_by_node[edge.from]
+          next if dependency_cache_keys.key?(dependency.cache_key)
+
+          dependency_cache_keys[dependency.cache_key] = true
+          @dependencies_by_node[edge.from] << dependency
+        end
+      end
+
+      def edge_key(from, to, reason)
+        [from, to, reason]
+      end
+
+      class << self
+        def deserialize_payload(kind, payload)
+          payload = symbolize_keys(payload)
+
+          case kind
+          when :dependency
+            Dependencies.from_h(payload)
+          when :frame
+            deserialize_frame_payload(payload)
+          else
+            payload
+          end
+        end
+
+        def deserialize_frame_payload(payload)
+          payload.each_with_object({}) do |(key, value), frame_payload|
+            frame_payload[key] = key == :recipe && value ? Replay::Recipe.from_h(value) : value
+          end
+        end
+
+        def symbolize_keys(value)
+          case value
+          when Hash
+            value.each_with_object({}) do |(key, nested_value), result|
+              normalized_key = key.respond_to?(:to_sym) ? key.to_sym : key
+              result[normalized_key] = symbolize_keys(nested_value)
+            end
+          when Array
+            value.map { |nested_value| symbolize_keys(nested_value) }
+          else
+            value
+          end
+        end
+      end
+
+      def serialize_node(node)
+        {
+          id: node.id,
+          kind: node.kind,
+          payload: serialize_payload(node)
+        }
+      end
+
+      def serialize_payload(node)
+        case node.kind
+        when :dependency
+          node.payload.to_h
+        when :frame
+          node.payload.each_with_object({}) do |(key, value), frame_payload|
+            frame_payload[key] = key == :recipe && value ? value.to_h : value
+          end
+        else
+          node.payload
+        end
+      end
+    end
+  end
+end
+
+require_relative "dag/subscription_shape"

data/lib/upkeep/delivery/action_cable_adapter.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "digest"
+
+module Upkeep
+  module Delivery
+    class ActionCableAdapter
+      STREAM_PREFIX = "upkeep:subscriber"
+
+      def self.stream_name_for(subscriber_id)
+        "#{STREAM_PREFIX}:#{Digest::SHA256.hexdigest(subscriber_id.to_s)[0, 32]}"
+      end
+
+      def initialize(server: default_server)
+        @server = server
+      end
+
+      def deliver(envelope)
+        stream_name = envelope.stream_name || self.class.stream_name_for(envelope.subscriber_id)
+        payload = {
+          subscriber_id: envelope.subscriber_id,
+          stream_name: stream_name,
+          envelope_digest: Transport.envelope_digest(envelope),
+          bytesize: envelope.body.bytesize
+        }
+
+        ActiveSupport::Notifications.instrument("deliver.upkeep", payload) do
+          server.broadcast(stream_name, envelope.body)
+        end
+      end
+
+      private
+
+      attr_reader :server
+
+      def default_server
+        require "action_cable"
+        ::ActionCable.server
+      end
+    end
+  end
+end