upkeep-rails 0.1.6

data/lib/upkeep/delivery/transport.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Upkeep
+  module Delivery
+    class Transport
+      Outcome = Data.define(
+        :subscriber_id,
+        :status,
+        :attempts,
+        :queue_depth,
+        :envelope_digest,
+        :error_class,
+        :error_message
+      ) do
+        def delivered?
+          status == :delivered
+        end
+
+        def report
+          {
+            subscriber_id: subscriber_id,
+            status: status,
+            attempts: attempts,
+            queue_depth: queue_depth,
+            envelope_digest: envelope_digest,
+            error_class: error_class,
+            error_message: error_message
+          }.compact
+        end
+      end
+
+      DispatchReport = Data.define(:outcomes) do
+        def summary
+          outcomes.each_with_object(Hash.new(0)) { |outcome, counts| counts[outcome.status] += 1 }
+        end
+
+        def report
+          {
+            summary: summary,
+            outcomes: outcomes.map(&:report)
+          }
+        end
+      end
+
+      Cleanup = Data.define(:subscriber_id, :status, :dropped_envelopes)
+      RetryItem = Data.define(:envelope, :attempts, :error)
+
+      class Connection
+        attr_reader :subscriber_id
+
+        def initialize(subscriber_id:, adapter:, max_queue_size:, retry_limit:)
+          @subscriber_id = subscriber_id
+          @adapter = adapter
+          @max_queue_size = max_queue_size
+          @retry_limit = retry_limit
+          @queue = []
+        end
+
+        def deliver(envelope)
+          deliver_envelope(envelope, attempts: 0)
+        end
+
+        def retry_pending
+          pending = @queue
+          @queue = []
+
+          pending.map { |item| deliver_envelope(item.envelope, attempts: item.attempts) }
+        end
+
+        def disconnect
+          dropped = @queue.size
+          @queue = []
+          dropped
+        end
+
+        def queue_depth
+          @queue.size
+        end
+
+        private
+
+        attr_reader :adapter, :max_queue_size, :retry_limit
+
+        def deliver_envelope(envelope, attempts:)
+          next_attempt = attempts + 1
+          adapter.deliver(envelope)
+
+          outcome(:delivered, envelope, attempts: next_attempt)
+        rescue StandardError => error
+          if next_attempt >= retry_limit
+            outcome(:dropped_retry_exhausted, envelope, attempts: next_attempt, error: error)
+          elsif queue_depth >= max_queue_size
+            outcome(:backpressured, envelope, attempts: next_attempt, error: error)
+          else
+            @queue << RetryItem.new(envelope, next_attempt, error)
+            outcome(:queued_retry, envelope, attempts: next_attempt, error: error)
+          end
+        end
+
+        def outcome(status, envelope, attempts:, error: nil)
+          Outcome.new(
+            subscriber_id,
+            status,
+            attempts,
+            queue_depth,
+            Transport.envelope_digest(envelope),
+            error&.class&.name,
+            error&.message
+          )
+        end
+      end
+
+      class << self
+        def envelope_digest(envelope)
+          Digest::SHA256.hexdigest(envelope.body)
+        end
+      end
+
+      def initialize(max_queue_size: 100, retry_limit: 3)
+        @max_queue_size = max_queue_size
+        @retry_limit = retry_limit
+        @connections = {}
+      end
+
+      def connect(subscriber_id:, adapter:)
+        Connection.new(
+          subscriber_id: subscriber_id,
+          adapter: adapter,
+          max_queue_size: max_queue_size,
+          retry_limit: retry_limit
+        ).tap { |connection| connections[subscriber_id] = connection }
+      end
+
+      def disconnect(subscriber_id)
+        connection = connections.delete(subscriber_id)
+        dropped = connection&.disconnect || 0
+
+        Cleanup.new(subscriber_id, :disconnected, dropped)
+      end
+
+      def deliver(batch)
+        DispatchReport.new(batch.envelopes.map { |envelope| deliver_envelope(envelope) })
+      end
+
+      def retry_pending(subscriber_id: nil)
+        selected_connections = if subscriber_id
+          Array(connections[subscriber_id])
+        else
+          connections.values
+        end
+
+        DispatchReport.new(selected_connections.flat_map(&:retry_pending))
+      end
+
+      def connected?(subscriber_id)
+        connections.key?(subscriber_id)
+      end
+
+      def summary
+        {
+          connections: connections.size,
+          queued_envelopes: connections.values.sum(&:queue_depth),
+          max_queue_size: max_queue_size,
+          retry_limit: retry_limit
+        }
+      end
+
+      private
+
+      attr_reader :connections, :max_queue_size, :retry_limit
+
+      def deliver_envelope(envelope)
+        connection = connections[envelope.subscriber_id]
+        return disconnected_outcome(envelope) unless connection
+
+        connection.deliver(envelope)
+      end
+
+      def disconnected_outcome(envelope)
+        Outcome.new(
+          envelope.subscriber_id,
+          :disconnected,
+          0,
+          0,
+          self.class.envelope_digest(envelope),
+          nil,
+          nil
+        )
+      end
+    end
+  end
+end

data/lib/upkeep/delivery/turbo_streams.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require "cgi"
+
+module Upkeep
+  module Delivery
+    class TurboStreams
+      DELIVERY_ERROR = "delivery_error.upkeep"
+
+      Stream = Data.define(
+        :action,
+        :target,
+        :target_selector,
+        :html,
+        :html_digest,
+        :identity_signature,
+        :shared_stream_name,
+        :subscriber_ids,
+        :matched_dependency_keys,
+        :deoptimization_reason,
+        :render_duration_ms
+      ) do
+        def to_html
+          return %(<turbo-stream action="refresh" method="morph" scroll="preserve"></turbo-stream>) if action == "refresh"
+
+          attributes = %(action="#{CGI.escapeHTML(action)}" targets="#{CGI.escapeHTML(target_selector)}")
+          attributes = %(#{attributes} method="morph") if morph_action?
+          return %(<turbo-stream #{attributes}></turbo-stream>) if action == "remove"
+
+          %(<turbo-stream #{attributes}><template>#{html}</template></turbo-stream>)
+        end
+
+        def rendered?
+          !%w[remove refresh].include?(action)
+        end
+
+        def morph_action?
+          %w[replace update].include?(action)
+        end
+
+        def for_subscriber?(subscriber_id)
+          subscriber_ids.include?(subscriber_id)
+        end
+
+        def report
+          {
+            action: action,
+            target: target.to_h,
+            target_selector: target_selector,
+            identity_signature: identity_signature,
+            shared_stream_name: shared_stream_name,
+            html_digest: html_digest,
+            subscriber_ids: subscriber_ids,
+            matched_dependency_keys: matched_dependency_keys,
+            deoptimization_reason: deoptimization_reason,
+            render_duration_ms: render_duration_ms
+          }
+        end
+      end
+
+      Envelope = Data.define(:subscriber_id, :streams, :stream_name) do
+        def self.subscriber(subscriber_id, streams)
+          new(subscriber_id, streams, nil)
+        end
+
+        def self.shared(stream_name, streams)
+          new("shared:#{stream_name}", streams, stream_name)
+        end
+
+        def body
+          streams.map(&:to_html).join("\n")
+        end
+
+        def report
+          {
+            subscriber_id: subscriber_id,
+            streams: streams.map(&:report)
+          }
+        end
+      end
+
+      Batch = Data.define(:streams) do
+        def envelopes
+          return [] if streams.empty?
+
+          direct_subscriber_id = single_direct_subscriber_id
+          return [Envelope.subscriber(direct_subscriber_id, streams)] if direct_subscriber_id
+
+          shared_envelopes + subscriber_envelopes
+        end
+
+        def envelope_for(subscriber_id)
+          Envelope.subscriber(subscriber_id, streams.select { |stream| stream.for_subscriber?(subscriber_id) })
+        end
+
+        def report
+          {
+            streams: streams.map(&:report),
+            envelopes: envelopes.map(&:report)
+          }
+        end
+
+        private
+
+        def single_direct_subscriber_id
+          return if streams.any?(&:shared_stream_name)
+
+          subscriber_ids = streams.flat_map(&:subscriber_ids).uniq
+          subscriber_ids.first if subscriber_ids.one?
+        end
+
+        def shared_envelopes
+          streams
+            .select(&:shared_stream_name)
+            .group_by(&:shared_stream_name)
+            .map { |stream_name, shared_streams| Envelope.shared(stream_name, shared_streams) }
+        end
+
+        def subscriber_envelopes
+          direct_streams = streams.reject(&:shared_stream_name)
+          direct_streams
+            .flat_map(&:subscriber_ids)
+            .uniq
+            .sort_by(&:to_s)
+            .map { |subscriber_id| Envelope.subscriber(subscriber_id, direct_streams.select { |stream| stream.for_subscriber?(subscriber_id) }) }
+        end
+      end
+
+      def build(plan)
+        build_many([plan])
+      end
+
+      def build_many(plans)
+        payload = {
+          plans: plans.size,
+          planned_targets: plans.sum { |plan| plan.targets.size }
+        }
+
+        ActiveSupport::Notifications.instrument("build_turbo_streams.upkeep", payload) do
+          streams = plans.flat_map { |plan| stream_targets(plan.targets) }.compact
+          batch = Batch.new(merge_streams(streams))
+          payload.merge!(payload_for(batch, rendered_streams: streams))
+          batch
+        end
+      end
+
+      private
+
+      def payload_for(batch, rendered_streams:)
+        envelopes = batch.envelopes
+
+        {
+          streams: batch.streams.size,
+          envelopes: envelopes.size,
+          actions: batch.streams.map(&:action).tally,
+          deoptimizations: batch.streams.filter_map(&:deoptimization_reason).tally,
+          renders: rendered_streams.count(&:rendered?),
+          render_duration_ms: sum_render_duration(rendered_streams),
+          payload_bytes: envelopes.sum { |envelope| envelope.body.bytesize }
+        }
+      end
+
+      def stream_targets(planned_targets)
+        return [] if planned_targets.empty?
+        return [stream_for(planned_targets.first)] if planned_targets.one?
+
+        planned_targets.group_by { |planned_target| render_group_key(planned_target) }.map do |_key, targets|
+          stream_for(
+            targets.first,
+            subscriber_ids: targets.flat_map(&:subscriber_ids),
+            matched_dependency_keys: targets.flat_map(&:matched_dependency_keys)
+          )
+        end
+      end
+
+      # The write that produced these changes has already committed; an isolated render/targeting
+      # failure for one target must never propagate back into the writer's request. Rescue per
+      # target, surface the failure via instrumentation, and keep delivering the other targets.
+      def stream_for(planned_target, subscriber_ids: planned_target.subscriber_ids, matched_dependency_keys: planned_target.matched_dependency_keys)
+        build_stream(planned_target, subscriber_ids: subscriber_ids, matched_dependency_keys: matched_dependency_keys)
+      rescue StandardError => error
+        ActiveSupport::Notifications.instrument(
+          DELIVERY_ERROR,
+          target: planned_target.target.to_h,
+          action: planned_target.action,
+          subscription_id: planned_target.subscription_id,
+          subscriber_ids: subscriber_ids.uniq.sort_by(&:to_s),
+          error_class: error.class.name,
+          error_message: error.message
+        )
+        nil
+      end
+
+      def build_stream(planned_target, subscriber_ids:, matched_dependency_keys:)
+        html, render_duration_ms = render_target(planned_target)
+
+        Stream.new(
+          planned_target.action,
+          planned_target.target,
+          target_selector_for(planned_target.target),
+          html,
+          Targeting::Extraction.digest_html(html),
+          planned_target.identity_signature,
+          shared_stream_name_for(planned_target, subscriber_ids: subscriber_ids),
+          subscriber_ids.uniq.sort_by(&:to_s),
+          matched_dependency_keys.uniq,
+          planned_target.deoptimization_reason,
+          render_duration_ms
+        )
+      end
+
+      def render_target(planned_target)
+        return ["", 0.0] if %w[remove refresh].include?(planned_target.action)
+
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        html = planned_target.render
+        finished_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        [html, ((finished_at - started_at) * 1000.0).round(3)]
+      end
+
+      def render_group_key(planned_target)
+        [
+          planned_target.action,
+          planned_target.target.kind,
+          planned_target.target.id,
+          planned_target.identity_signature,
+          planned_target.sharing_signature,
+          SharedStreams.signature_for(planned_target.recipe),
+          planned_target.deoptimization_reason
+        ]
+      end
+
+      def merge_streams(streams)
+        streams.each_with_object({}) do |stream, indexed_streams|
+          key = [
+            stream.action,
+            stream.target.kind,
+            stream.target.id,
+            stream.identity_signature,
+            stream.shared_stream_name,
+            stream.html_digest,
+            stream.deoptimization_reason
+          ]
+          indexed_streams[key] = merge_stream(indexed_streams[key], stream)
+        end.values
+      end
+
+      def merge_stream(existing, stream)
+        return stream unless existing
+
+        Stream.new(
+          existing.action,
+          existing.target,
+          existing.target_selector,
+          existing.html,
+          existing.html_digest,
+          existing.identity_signature,
+          existing.shared_stream_name,
+          (existing.subscriber_ids + stream.subscriber_ids).uniq.sort_by(&:to_s),
+          (existing.matched_dependency_keys + stream.matched_dependency_keys).uniq,
+          existing.deoptimization_reason,
+          (existing.render_duration_ms + stream.render_duration_ms).round(3)
+        )
+      end
+
+      def sum_render_duration(streams)
+        streams.sum(&:render_duration_ms).round(3)
+      end
+
+      def shared_stream_name_for(planned_target, subscriber_ids:)
+        return unless planned_target.sharing_signature
+        return unless subscriber_ids.uniq.size > 1
+
+        SharedStreams.stream_name(
+          target: planned_target.shared_stream_target,
+          identity_signature: planned_target.identity_signature,
+          sharing_signature: planned_target.sharing_signature
+        )
+      end
+
+      def target_selector_for(target)
+        case target.kind
+        when "page"
+          %([data-upkeep-page-frame="#{css_escape(target.id)}"])
+        when "fragment"
+          %([data-upkeep-frame="#{css_escape(target.id)}"])
+        when "render_site"
+          %([data-upkeep-render-site="#{css_escape(target.id)}"])
+        when "dom_id"
+          %[##{css_escape(target.id)}]
+        else
+          raise "unknown delivery target kind: #{target.kind.inspect}"
+        end
+      end
+
+      def css_escape(value)
+        value.to_s.gsub("\\", "\\\\\\").gsub('"', '\"')
+      end
+    end
+  end
+end

data/lib/upkeep/delivery.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative "delivery/turbo_streams"
+require_relative "delivery/transport"
+require_relative "delivery/action_cable_adapter"
+require_relative "delivery/broadcast_transport"
+require_relative "delivery/async_dispatcher"