upkeep-rails 0.2.5-aarch64-linux-gnu

data/lib/upkeep/dag.rb

@@ -0,0 +1,454 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Upkeep
+  module DAG
+    Node = Data.define(:id, :kind, :payload)
+    Edge = Data.define(:from, :to, :reason)
+
+    class Graph
+      attr_reader :nodes, :edges, :version
+
+      def initialize
+        @nodes = {}
+        @edges = []
+        @version = 0
+        reset_indexes!
+      end
+
+      def add_node(id, kind:, payload: {})
+        return nodes.fetch(id) if nodes.key?(id)
+
+        @version += 1
+        nodes[id] = Node.new(id, kind, payload)
+      end
+
+      def add_edge(from, to, reason:)
+        key = edge_key(from, to, reason)
+        return false if @edge_keys[key]
+
+        @edge_keys[key] = true
+        @version += 1
+        edge = Edge.new(from, to, reason)
+        edges << edge
+        @outgoing_edges_by_from[from] << edge
+        @incoming_edges_by_to[to] << edge
+        @dependency_owner_ids_by_node[to] << from if reason == :depends_on
+        true
+      end
+
+      def add_dependency(owner_id, dependency)
+        dependency_cache_key = dependency.cache_key
+        dependency_cache_keys = @dependency_cache_keys_by_node[owner_id]
+        return false if dependency_cache_keys.key?(dependency_cache_key)
+
+        add_node(owner_id, kind: :unknown) unless nodes.key?(owner_id)
+        add_node(dependency_cache_key, kind: :dependency, payload: dependency)
+        add_edge(owner_id, dependency_cache_key, reason: :depends_on)
+
+        dependency_cache_keys[dependency_cache_key] = true
+        @dependencies_by_node[owner_id] << dependency
+        true
+      end
+
+      def dependencies_for(node_id)
+        @dependencies_by_node[node_id]
+      end
+
+      def node(id)
+        nodes.fetch(id)
+      end
+
+      def node?(id)
+        nodes.key?(id)
+      end
+
+      def outgoing_edges(from, reason: nil)
+        indexed_edges = @outgoing_edges_by_from.fetch(from, [])
+        return indexed_edges.dup unless reason
+
+        indexed_edges.select { |edge| edge.reason == reason }
+      end
+
+      def incoming_edges(to, reason: nil)
+        indexed_edges = @incoming_edges_by_to.fetch(to, [])
+        return indexed_edges.dup unless reason
+
+        indexed_edges.select { |edge| edge.reason == reason }
+      end
+
+      def dependency_owner_ids(dependency_node_id)
+        @dependency_owner_ids_by_node.fetch(dependency_node_id, []).dup
+      end
+
+      def dependency_node_ids_matching(changes)
+        dependency_nodes.filter_map do |node|
+          node.id if changes.any? { |change| node.payload.matches_change?(change) }
+        end
+      end
+
+      def nearest_frame_nodes_from(node_id)
+        current = node(node_id)
+        return [current] if current.kind == :frame
+
+        queue = outgoing_edges(node_id, reason: :contains).map(&:to)
+        visited = {}
+        frames = []
+
+        until queue.empty?
+          id = queue.shift
+          next if visited[id]
+
+          visited[id] = true
+          current = node(id)
+          if current.kind == :frame
+            frames << current
+          else
+            queue.concat(outgoing_edges(id, reason: :contains).map(&:to))
+          end
+        end
+
+        frames
+      end
+
+      def ancestor_node_ids(node_id)
+        ancestors = []
+        current = node_id
+
+        while (edge = incoming_edges(current, reason: :contains).first)
+          ancestors << edge.from
+          current = edge.from
+        end
+
+        ancestors
+      end
+
+      def contained_by?(descendant_id, ancestor_id)
+        ancestor_node_ids(descendant_id).include?(ancestor_id)
+      end
+
+      def contained_node_ids(node_id)
+        ids = []
+        queue = [node_id]
+        visited = {}
+
+        until queue.empty?
+          id = queue.shift
+          next if visited[id]
+
+          visited[id] = true
+          ids << id
+          queue.concat(outgoing_edges(id, reason: :contains).map(&:to))
+        end
+
+        ids
+      end
+
+      def replace_subgraph(scope_id, replacement_graph, replacement_scope_id: scope_id)
+        raise KeyError, scope_id unless node?(scope_id)
+        raise KeyError, replacement_scope_id unless replacement_graph.node?(replacement_scope_id)
+
+        removed_ids = contained_node_ids(scope_id).to_h { |id| [id, true] }
+        replacement_ids = replacement_graph.contained_node_ids(replacement_scope_id).to_h { |id| [id, true] }
+        composed = self.class.new
+
+        nodes.each_value do |node|
+          next if node.kind == :dependency || removed_ids[node.id]
+
+          composed.add_node(node.id, kind: node.kind, payload: node.payload)
+        end
+        replacement_graph.nodes.each_value do |node|
+          next if node.kind == :dependency || !replacement_ids[node.id]
+
+          composed.add_node(node.id, kind: node.kind, payload: node.payload)
+        end
+
+        edges.each do |edge|
+          next unless edge.reason == :contains
+          next unless composed.node?(edge.from) && composed.node?(edge.to)
+          next if removed_ids[edge.from] || edge.to != scope_id && removed_ids[edge.to]
+
+          composed.add_edge(edge.from, edge.to, reason: :contains)
+        end
+        replacement_graph.edges.each do |edge|
+          next unless edge.reason == :contains
+          next unless replacement_ids[edge.from] && replacement_ids[edge.to]
+
+          composed.add_edge(edge.from, edge.to, reason: :contains)
+        end
+
+        composed.nodes.keys.each do |owner_id|
+          source = replacement_ids[owner_id] ? replacement_graph : self
+          source.dependencies_for(owner_id).each { |dependency| composed.add_dependency(owner_id, dependency) }
+        end
+
+        composed
+      end
+
+      def attach_subgraph(scope_id, replacement_graph, parent_id:, replacement_scope_id: scope_id)
+        raise KeyError, scope_id if node?(scope_id)
+        raise KeyError, replacement_scope_id unless replacement_graph.node?(replacement_scope_id)
+        raise KeyError, parent_id unless node?(parent_id)
+
+        replacement_ids = replacement_graph.contained_node_ids(replacement_scope_id).to_h { |id| [id, true] }
+        composed = self.class.new
+
+        nodes.each_value do |node|
+          next if node.kind == :dependency
+
+          composed.add_node(node.id, kind: node.kind, payload: node.payload)
+        end
+        edges.each do |edge|
+          next unless edge.reason == :contains
+
+          composed.add_edge(edge.from, edge.to, reason: :contains)
+        end
+        composed.nodes.keys.each do |owner_id|
+          dependencies_for(owner_id).each { |dependency| composed.add_dependency(owner_id, dependency) }
+        end
+
+        replacement_graph.nodes.each_value do |node|
+          next if node.kind == :dependency || !replacement_ids[node.id]
+
+          composed.add_node(node.id, kind: node.kind, payload: node.payload)
+        end
+        replacement_graph.edges.each do |edge|
+          next unless edge.reason == :contains
+          next unless replacement_ids[edge.from] && replacement_ids[edge.to]
+
+          composed.add_edge(edge.from, edge.to, reason: :contains)
+        end
+        composed.add_edge(parent_id, replacement_scope_id, reason: :contains)
+        replacement_ids.each_key do |owner_id|
+          replacement_graph.dependencies_for(owner_id).each do |dependency|
+            composed.add_dependency(owner_id, dependency)
+          end
+        end
+
+        composed
+      end
+
+      def frame_nodes
+        nodes.values.select { |node| node.kind == :frame }
+      end
+
+      def dependency_nodes
+        nodes.values.select { |node| node.kind == :dependency }
+      end
+
+      def summary
+        {
+          nodes: nodes.size,
+          edges: edges.size,
+          frames: frame_nodes.size,
+          manifest_attached_frames: frame_nodes.count { |node| node.payload[:manifest_path] },
+          dependencies: dependency_nodes.size,
+          containment_edges: edges.count { |edge| edge.reason == :contains },
+          dependency_edges: edges.count { |edge| edge.reason == :depends_on },
+          replay_recipes: frame_nodes.count { |node| node.payload[:recipe] },
+          replay_recipe_kinds: frame_nodes.filter_map { |node| node.payload[:recipe]&.kind }.map(&:to_s).uniq.sort,
+          dependency_sources: dependency_nodes.map { |node| node.payload.source.to_s }.uniq.sort
+        }
+      end
+
+      def report
+        {
+          summary: summary,
+          frames: frame_reports,
+          dependencies: dependency_reports,
+          edges: edges.map(&:to_h)
+        }
+      end
+
+      def to_h(dependencies: :all)
+        serialized_nodes = serializable_nodes(dependencies: dependencies)
+        node_ids = serialized_nodes.to_h { |node| [node.id, true] }
+
+        {
+          nodes: serialized_nodes.map { |node| serialize_node(node) },
+          edges: edges.select { |edge| node_ids[edge.from] && node_ids[edge.to] }.map(&:to_h)
+        }
+      end
+
+      def self.from_h(snapshot)
+        snapshot = symbolize_keys(snapshot)
+        graph = new
+        graph.nodes.clear
+        graph.edges.clear
+        graph.send(:reset_indexes!)
+
+        snapshot.fetch(:nodes).each do |node_snapshot|
+          node_snapshot = symbolize_keys(node_snapshot)
+          kind = node_snapshot.fetch(:kind).to_sym
+          graph.nodes[node_snapshot.fetch(:id)] = Node.new(
+            node_snapshot.fetch(:id),
+            kind,
+            deserialize_payload(kind, node_snapshot.fetch(:payload))
+          )
+        end
+
+        snapshot.fetch(:edges).each do |edge_snapshot|
+          edge_snapshot = symbolize_keys(edge_snapshot)
+          graph.add_edge(
+            edge_snapshot.fetch(:from),
+            edge_snapshot.fetch(:to),
+            reason: edge_snapshot.fetch(:reason).to_sym
+          )
+        end
+
+        graph.send(:rebuild_dependency_index)
+        graph
+      end
+
+      def frame_reports
+        frame_nodes.map do |node|
+          {
+            id: node.id,
+            kind: node.payload.fetch(:kind),
+            template: node.payload[:template],
+            site_id: node.payload[:site_id],
+            manifest_path: node.payload[:manifest_path],
+            manifest_fingerprint: node.payload[:manifest_fingerprint],
+            locals: node.payload[:locals],
+            contains: outgoing_edges(node.id, reason: :contains).map(&:to),
+            dependencies: dependencies_for(node.id).map(&:to_h),
+            replay_recipe: recipe_report(node.payload[:recipe])
+          }.compact
+        end
+      end
+
+      def recipe_report(recipe)
+        return unless recipe
+
+        snapshot = recipe.to_h
+        replay = snapshot[:replay] || snapshot["replay"] || {}
+        replay_json = JSON.generate(replay)
+        {
+          kind: recipe.kind.to_s,
+          target_kind: recipe.target_kind,
+          target_id: recipe.target_id,
+          runtime: recipe.runtime,
+          template: recipe.template,
+          replay: {
+            type: recipe.replay.respond_to?(:type) ? recipe.replay.type : nil,
+            keys: replay.keys.map(&:to_s).sort,
+            bytes: replay_json.bytesize,
+            digest: Digest::SHA256.hexdigest(replay_json)
+          }.compact
+        }.compact
+      end
+
+      def dependency_reports
+        dependency_nodes.map do |node|
+          {
+            id: node.id,
+            dependency: node.payload.to_h,
+            owners: dependency_owner_ids(node.id)
+          }
+        end
+      end
+
+      private
+
+      def serializable_nodes(dependencies:)
+        case dependencies
+        when :all
+          nodes.values
+        when :identity
+          nodes.values.select { |node| node.kind != :dependency || node.payload.identity? }
+        else
+          raise ArgumentError, "unsupported graph dependency serialization mode: #{dependencies.inspect}"
+        end
+      end
+
+      def reset_indexes!
+        @edge_keys = {}
+        @outgoing_edges_by_from = Hash.new { |hash, key| hash[key] = [] }
+        @incoming_edges_by_to = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_owner_ids_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependencies_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_cache_keys_by_node = Hash.new { |hash, key| hash[key] = {} }
+      end
+
+      def rebuild_dependency_index
+        @dependencies_by_node = Hash.new { |hash, key| hash[key] = [] }
+        @dependency_cache_keys_by_node = Hash.new { |hash, key| hash[key] = {} }
+
+        edges.each do |edge|
+          next unless edge.reason == :depends_on
+
+          dependency = nodes.fetch(edge.to).payload
+          dependency_cache_keys = @dependency_cache_keys_by_node[edge.from]
+          next if dependency_cache_keys.key?(dependency.cache_key)
+
+          dependency_cache_keys[dependency.cache_key] = true
+          @dependencies_by_node[edge.from] << dependency
+        end
+      end
+
+      def edge_key(from, to, reason)
+        [from, to, reason]
+      end
+
+      class << self
+        def deserialize_payload(kind, payload)
+          payload = symbolize_keys(payload)
+
+          case kind
+          when :dependency
+            Dependencies.from_h(payload)
+          when :frame
+            deserialize_frame_payload(payload)
+          else
+            payload
+          end
+        end
+
+        def deserialize_frame_payload(payload)
+          payload.each_with_object({}) do |(key, value), frame_payload|
+            frame_payload[key] = key == :recipe && value ? Replay::Recipe.from_h(value) : value
+          end
+        end
+
+        def symbolize_keys(value)
+          case value
+          when Hash
+            value.each_with_object({}) do |(key, nested_value), result|
+              normalized_key = key.respond_to?(:to_sym) ? key.to_sym : key
+              result[normalized_key] = symbolize_keys(nested_value)
+            end
+          when Array
+            value.map { |nested_value| symbolize_keys(nested_value) }
+          else
+            value
+          end
+        end
+      end
+
+      def serialize_node(node)
+        {
+          id: node.id,
+          kind: node.kind,
+          payload: serialize_payload(node)
+        }
+      end
+
+      def serialize_payload(node)
+        case node.kind
+        when :dependency
+          node.payload.to_h
+        when :frame
+          node.payload.each_with_object({}) do |(key, value), frame_payload|
+            frame_payload[key] = key == :recipe && value ? value.to_h : value
+          end
+        else
+          node.payload
+        end
+      end
+    end
+  end
+end
+
+require_relative "dag/subscription_shape"

data/lib/upkeep/delivery/action_cable_adapter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "digest"
+
+module Upkeep
+  module Delivery
+    class ActionCableAdapter
+      STREAM_PREFIX = "upkeep:subscriber"
+      SUBSCRIPTION_STREAM_PREFIX = "upkeep:subscription"
+
+      def self.stream_name_for(subscriber_id)
+        "#{STREAM_PREFIX}:#{Digest::SHA256.hexdigest(subscriber_id.to_s)[0, 32]}"
+      end
+
+      def self.subscription_stream_name_for(subscription_id)
+        "#{SUBSCRIPTION_STREAM_PREFIX}:#{Digest::SHA256.hexdigest(subscription_id.to_s)[0, 32]}"
+      end
+
+      def initialize(server: default_server)
+        @server = server
+      end
+
+      def deliver(envelope)
+        stream_name = envelope.stream_name || self.class.stream_name_for(envelope.subscriber_id)
+        payload = {
+          subscriber_id: envelope.subscriber_id,
+          stream_name: stream_name,
+          envelope_digest: Transport.envelope_digest(envelope),
+          bytesize: envelope.body.bytesize
+        }
+
+        ActiveSupport::Notifications.instrument("deliver.upkeep", payload) do
+          server.broadcast(stream_name, envelope.body)
+        end
+      end
+
+      private
+
+      attr_reader :server
+
+      def default_server
+        require "action_cable"
+        ::ActionCable.server
+      end
+    end
+  end
+end

data/lib/upkeep/delivery/async_dispatcher.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Upkeep
+  module Delivery
+    class AsyncDispatcher
+      def initialize(batch_window: 0.01, &deliver)
+        @deliver = deliver
+        @batch_window = batch_window
+        @jobs = []
+        @mutex = Mutex.new
+        @available = ConditionVariable.new
+        @idle = ConditionVariable.new
+        @pending_jobs = 0
+        @last_error = nil
+        @stopping = false
+        @worker = Thread.new { work_loop }
+      end
+
+      def enqueue(changes)
+        changes = Array(changes)
+        return Transport::DispatchReport.new([]) if changes.empty?
+
+        @mutex.synchronize do
+          raise @last_error if @last_error
+
+          @pending_jobs += 1
+          @jobs << changes
+          @available.signal
+        end
+
+        Transport::DispatchReport.new([])
+      end
+
+      def drain
+        @mutex.synchronize do
+          @idle.wait(@mutex) until @pending_jobs.zero?
+          raise @last_error if @last_error
+        end
+      end
+
+      def shutdown
+        error = nil
+        begin
+          drain
+        rescue StandardError => shutdown_error
+          error = shutdown_error
+        ensure
+          @mutex.synchronize do
+            @stopping = true
+            @available.signal
+          end
+          @worker.join
+        end
+
+        raise error if error
+      end
+
+      private
+
+      attr_reader :deliver, :batch_window
+
+      def work_loop
+        loop do
+          batch = next_batch
+          break unless batch
+
+          begin
+            deliver.call(batch)
+          rescue StandardError => error
+            @mutex.synchronize { @last_error = error }
+          ensure
+            complete_jobs(batch.size)
+          end
+        end
+      end
+
+      def next_batch
+        @mutex.synchronize do
+          @available.wait(@mutex) while @jobs.empty? && !@stopping
+          return nil if @jobs.empty? && @stopping
+
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + batch_window
+          while !@stopping
+            remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            break unless remaining.positive?
+
+            @available.wait(@mutex, remaining)
+          end
+
+          @jobs.shift(@jobs.length)
+        end
+      end
+
+      def complete_jobs(count)
+        @mutex.synchronize do
+          @pending_jobs -= count
+          @idle.broadcast if @pending_jobs.zero?
+        end
+      end
+    end
+  end
+end

data/lib/upkeep/delivery/broadcast_transport.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Upkeep
+  module Delivery
+    class BroadcastTransport
+      def initialize(adapter: ActionCableAdapter.new, max_queue_size: 100, retry_limit: 3)
+        @adapter = adapter
+        @max_queue_size = max_queue_size
+        @retry_limit = retry_limit
+        @adapter_overrides = {}
+        @queue = []
+      end
+
+      def connect(subscriber_id:, adapter:)
+        adapter_overrides[subscriber_id] = adapter
+      end
+
+      def disconnect(subscriber_id)
+        adapter_overrides.delete(subscriber_id)
+        retained, dropped = queue.partition { |item| item.envelope.subscriber_id != subscriber_id }
+        @queue = retained
+
+        Transport::Cleanup.new(subscriber_id, :disconnected, dropped.size)
+      end
+
+      def deliver(batch)
+        Transport::DispatchReport.new(batch.envelopes.map { |envelope| deliver_envelope(envelope, attempts: 0) })
+      end
+
+      def retry_pending(subscriber_id: nil)
+        selected, retained = queue.partition do |item|
+          subscriber_id.nil? || item.envelope.subscriber_id == subscriber_id
+        end
+        @queue = retained
+
+        Transport::DispatchReport.new(selected.map { |item| deliver_envelope(item.envelope, attempts: item.attempts) })
+      end
+
+      def connected?(subscriber_id)
+        adapter_overrides.key?(subscriber_id)
+      end
+
+      def summary
+        {
+          adapter_overrides: adapter_overrides.size,
+          queued_envelopes: queue.size,
+          max_queue_size: max_queue_size,
+          retry_limit: retry_limit
+        }
+      end
+
+      private
+
+      attr_reader :adapter, :adapter_overrides, :max_queue_size, :retry_limit, :queue
+
+      def deliver_envelope(envelope, attempts:)
+        next_attempt = attempts + 1
+        adapter_for(envelope.subscriber_id).deliver(envelope)
+
+        outcome(:delivered, envelope, attempts: next_attempt)
+      rescue StandardError => error
+        if next_attempt >= retry_limit
+          outcome(:dropped_retry_exhausted, envelope, attempts: next_attempt, error: error)
+        elsif queue.size >= max_queue_size
+          outcome(:backpressured, envelope, attempts: next_attempt, error: error)
+        else
+          queue << Transport::RetryItem.new(envelope, next_attempt, error)
+          outcome(:queued_retry, envelope, attempts: next_attempt, error: error)
+        end
+      end
+
+      def adapter_for(subscriber_id)
+        adapter_overrides.fetch(subscriber_id, adapter)
+      end
+
+      def outcome(status, envelope, attempts:, error: nil)
+        Transport::Outcome.new(
+          envelope.subscriber_id,
+          status,
+          attempts,
+          queue.size,
+          Transport.envelope_digest(envelope),
+          error&.class&.name,
+          error&.message
+        )
+      end
+    end
+  end
+end