upkeep-rails 0.1.0

data/lib/upkeep/active_record_query.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Upkeep
+  module ActiveRecordQuery
+    class OpaqueRelationError < StandardError
+      attr_reader :model_name, :table_name, :sql, :reasons
+
+      def initialize(relation, reasons:)
+        @model_name = relation.klass.name
+        @table_name = relation.klass.table_name
+        @sql = relation.to_sql
+        @reasons = reasons
+
+        super(build_message)
+      rescue StandardError => error
+        super("Upkeep cannot prove this Active Record relation's structural dependencies: #{error.message}")
+      end
+
+      private
+
+      def build_message
+        <<~MESSAGE
+          Upkeep cannot make this Active Record relation reactive because its query shape is opaque.
+
+          Relation:
+            #{model_name} (#{table_name})
+
+          SQL:
+            #{sql}
+
+          Why:
+          #{reasons.map { |reason| "            - #{reason}" }.join("\n")}
+
+          What to do:
+            - Rewrite raw SQL predicates with structural Active Record hash or Arel predicates.
+            - Rewrite raw SQL joins or FROM sources with structural Active Record/Arel joins.
+            - Render this boundary outside Upkeep reactivity when the query cannot expose its sources.
+        MESSAGE
+      end
+
+      public
+
+      def suggestions
+        [
+          "Rewrite raw SQL predicates with structural Active Record hash or Arel predicates.",
+          "Rewrite raw SQL joins or FROM sources with structural Active Record/Arel joins.",
+          "Render this boundary outside Upkeep reactivity when the query cannot expose its sources."
+        ]
+      end
+    end
+
+    Result = Data.define(
+      :primary_table,
+      :table_columns,
+      :coverage,
+      :sql,
+      :primary_key,
+      :appendable,
+      :limit_value,
+      :predicates
+    ) do
+      def tables = table_columns.keys.sort
+
+      def appendable?
+        appendable
+      end
+    end
+
+    module_function
+
+    def analyze(relation, opaque_table_policy: :raise)
+      collector = Collector.new(relation, opaque_table_policy: opaque_table_policy)
+      collector.analyze
+    end
+
+    class Collector
+      def initialize(relation, opaque_table_policy:)
+        @relation = relation
+        @opaque_table_policy = opaque_table_policy
+        @primary_table = relation.klass.table_name
+        @primary_key = relation.klass.primary_key
+        @table_columns = Hash.new { |hash, table| hash[table] = [] }
+        @table_aliases = {}
+        @opaque_columns = false
+        @opaque_tables = false
+        @opaque_table_reasons = []
+        @opaque_column_reasons = []
+        @predicates = []
+      end
+
+      def analyze
+        table(@primary_table)
+        collect_relation_shape
+        raise_opaque_relation! if opaque_relation? && @opaque_table_policy == :raise
+
+        Result.new(
+          primary_table: @primary_table,
+          table_columns: normalized_table_columns,
+          coverage: coverage,
+          sql: safe_sql,
+          primary_key: @primary_key,
+          appendable: appendable_relation?,
+          limit_value: @relation.limit_value,
+          predicates: normalized_predicates
+        )
+      end
+
+      private
+
+      def collect_relation_shape
+        ast = @relation.arel.ast
+
+        ast.cores.each do |core|
+          walk(core.source, source: true)
+          walk(core.wheres)
+          walk(core.groups)
+          walk(core.havings)
+        end
+
+        walk(ast.orders)
+        walk(ast.with) if ast.respond_to?(:with)
+      rescue StandardError => error
+        opaque_table!("relation AST could not be inspected (#{error.class}: #{error.message})")
+      end
+
+      def coverage
+        return :tables if @opaque_tables || @opaque_columns
+
+        :columns
+      end
+
+      def normalized_table_columns
+        table(@primary_table)
+        column(@primary_table, @primary_key) if @primary_key
+
+        @table_columns.transform_values { |columns| columns.compact.uniq.sort }.sort.to_h
+      end
+
+      def appendable_relation?
+        return false unless coverage == :columns
+        return false if @opaque_tables
+        return false if @relation.limit_value || @relation.offset_value
+        return false if @relation.distinct_value
+        return false if @relation.group_values.any?
+        return false if !@relation.having_clause.empty?
+
+        true
+      end
+
+      def walk(value, source: false)
+        case value
+        when nil, true, false, Numeric, Symbol, Class, Module
+          nil
+        when Array
+          value.each { |entry| walk(entry, source: source) }
+        when Hash
+          value.each_value { |entry| walk(entry, source: source) }
+        when Arel::Attributes::Attribute
+          attribute(value)
+        when Arel::Nodes::Equality
+          equality_predicate(value)
+          walk_arel_node(value, source: source)
+        when Arel::Nodes::HomogeneousIn
+          homogeneous_in_predicate(value)
+          walk_arel_node(value, source: source)
+        when Arel::Table
+          table(value.name)
+        when Arel::Nodes::TableAlias
+          table_alias(value)
+        when Arel::Nodes::StringJoin
+          opaque_table!("raw SQL join")
+        when Arel::Nodes::BoundSqlLiteral, Arel::Nodes::SqlLiteral
+          source ? opaque_table!("raw SQL source") : opaque_column!("raw SQL predicate or order expression")
+        when String
+          source ? opaque_table!("string SQL source") : opaque_column!("string SQL predicate or order expression")
+        else
+          walk_arel_node(value, source: source)
+        end
+      end
+
+      def walk_arel_node(value, source:)
+        return unless value.is_a?(Arel::Nodes::Node)
+
+        value.instance_variables.each do |ivar|
+          walk(value.instance_variable_get(ivar), source: source)
+        end
+      end
+
+      def attribute(value)
+        table_name = table_name_for(value.relation)
+        return opaque_table!("attribute references an unknown table source") unless table_name
+        return if value.name.to_s == "*"
+
+        column(table_name, value.name)
+      end
+
+      def table_name_for(relation)
+        if relation.is_a?(Arel::Nodes::TableAlias)
+          table_name_for(relation.left)
+        elsif relation.respond_to?(:name)
+          name = relation.name.to_s
+          @table_aliases.fetch(name, name)
+        elsif relation.respond_to?(:left)
+          table_name_for(relation.left)
+        end
+      end
+
+      def table_alias(value)
+        table_name = table_name_for(value.left)
+        return opaque_table!("table alias references an unknown table source") unless table_name
+
+        @table_aliases[value.right.to_s] = table_name
+        table(table_name)
+      end
+
+      def opaque_table!(reason)
+        @opaque_tables = true
+        @opaque_table_reasons << reason
+      end
+
+      def opaque_column!(reason)
+        @opaque_columns = true
+        @opaque_column_reasons << reason
+      end
+
+      def opaque_relation?
+        @opaque_tables || @opaque_columns
+      end
+
+      def raise_opaque_relation!
+        raise OpaqueRelationError.new(@relation, reasons: (@opaque_table_reasons + @opaque_column_reasons).uniq)
+      end
+
+      def table(name)
+        @table_columns[name.to_s]
+      end
+
+      def column(table_name, column_name)
+        @table_columns[table_name.to_s] << column_name.to_s
+      end
+
+      def equality_predicate(node)
+        predicate = predicate_for(node.left, "eq", [predicate_value(node.right)])
+        @predicates << predicate if predicate
+      end
+
+      def homogeneous_in_predicate(node)
+        predicate = predicate_for(node.attribute, "in", Array(node.values).map { |value| predicate_value(value) })
+        @predicates << predicate if predicate
+      end
+
+      def predicate_for(attribute, operator, values)
+        return unless attribute.is_a?(Arel::Attributes::Attribute)
+
+        table_name = table_name_for(attribute.relation)
+        return unless table_name
+
+        values = values.compact
+        return if values.empty?
+
+        {
+          table: table_name.to_s,
+          column: attribute.name.to_s,
+          operator: operator,
+          values: values.uniq
+        }
+      end
+
+      def predicate_value(value)
+        if value.respond_to?(:value_for_database)
+          value.value_for_database
+        elsif value.respond_to?(:value)
+          value.value
+        elsif value.nil? || value.is_a?(String) || value.is_a?(Numeric) || value == true || value == false || value.is_a?(Symbol)
+          value
+        end
+      end
+
+      def normalized_predicates
+        @predicates
+          .uniq
+          .sort_by { |predicate| [predicate.fetch(:table), predicate.fetch(:column), predicate.fetch(:operator), predicate.fetch(:values).inspect] }
+      end
+
+      def safe_sql
+        @relation.to_sql
+      rescue StandardError => error
+        "#{error.class}: #{error.message}"
+      end
+    end
+  end
+end

data/lib/upkeep/capture/request.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Upkeep
+  module Capture
+    RequestSignature = Data.define(:controller, :action, :method, :fullpath)
+
+    RequestResult = Data.define(
+      :action_result,
+      :html,
+      :recorder,
+      :response_status,
+      :response_content_type,
+      :response_media_type,
+      :response_successful,
+      :signature,
+      :timings,
+      :counters
+    ) do
+      def successful?
+        !!response_successful
+      end
+
+      def html_response?
+        response_media_type == "text/html" ||
+          response_content_type.to_s.start_with?("text/html")
+      end
+    end
+
+    module Request
+      module_function
+
+      def call(controller, profile: false)
+        timings = {}
+        counters = {}
+        action_result, recorder = measure(timings, :action_ms) do
+          if profile
+            profile_action(timings, counters) do
+              Runtime::Observation.capture_request(profile: true) { yield }
+            end
+          else
+            Runtime::Observation.capture_request { yield }
+          end
+        end
+        timings.merge!(recorder.profile_timings)
+        counters.merge!(recorder.profile_counts)
+        html = measure(timings, :response_body_ms) { response_body_html(controller.response.body) }
+        signature = measure(timings, :signature_ms) { signature_for(controller) }
+        RequestResult.new(
+          action_result,
+          html,
+          recorder,
+          controller.response.status,
+          controller.response.content_type,
+          controller.response.media_type,
+          controller.response.successful?,
+          signature,
+          timings,
+          counters
+        )
+      end
+
+      def profile_action(timings, counters)
+        collector = ActionProfiler.new
+        collector.capture { yield }.tap do
+          timings.merge!(collector.timings)
+          counters.merge!(collector.counters)
+        end
+      end
+
+      def signature_for(controller)
+        request = controller.request
+        RequestSignature.new(
+          controller.class.name,
+          controller.action_name,
+          request.request_method,
+          request.fullpath
+        )
+      end
+
+      def response_body_html(body)
+        case body
+        when String
+          body
+        when Array
+          body.join
+        else
+          return body.body.join if body.respond_to?(:body) && body.body.respond_to?(:join)
+          return body.to_a.join if body.respond_to?(:to_a)
+
+          body.to_s
+        end
+      end
+
+      def measure(timings, key)
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        yield
+      ensure
+        timings[key] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000.0).round(3)
+      end
+
+      class ActionProfiler
+        EVENT_MAP = {
+          "sql.active_record" => :sql,
+          "render_template.action_view" => :render_template,
+          "render_partial.action_view" => :render_partial,
+          "render_collection.action_view" => :render_collection
+        }.freeze
+
+        attr_reader :timings, :counters
+
+        def initialize
+          @thread = Thread.current
+          @timings = Hash.new(0.0)
+          @counters = Hash.new(0)
+        end
+
+        def capture
+          callback = lambda do |name, started, finished, unique_id, payload|
+            next unless Thread.current.equal?(@thread)
+
+            event = ActiveSupport::Notifications::Event.new(name, started, finished, unique_id, payload)
+            record(event)
+          end
+
+          ActiveSupport::Notifications.subscribed(callback, /\A(sql\.active_record|render_(template|partial|collection)\.action_view)\z/) do
+            yield
+          end
+        ensure
+          @timings.transform_values! { |value| value.round(3) }
+        end
+
+        private
+
+        def record(event)
+          key = EVENT_MAP[event.name]
+          return unless key
+          return if ignored_sql?(event)
+
+          @timings[:"#{key}_ms"] += event.duration
+          @counters[:"#{key}_count"] += 1
+          @timings[:view_ms] += event.duration if event.name.end_with?(".action_view")
+        end
+
+        def ignored_sql?(event)
+          event.name == "sql.active_record" && event.payload[:name] == "SCHEMA"
+        end
+      end
+    end
+  end
+end

data/lib/upkeep/dag/subscription_shape.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../shared_streams"
+require_relative "../version"
+
+module Upkeep
+  module DAG
+    class SubscriptionShape
+      DIGEST_SCOPE = "upkeep-subscription-shape"
+      FRAME_PAYLOAD_SHAPE_IGNORED_KEYS = %i[manifest recipe].freeze
+
+      attr_reader :signature
+
+      def self.from_graph(graph, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms(graph)))
+      end
+
+      def self.from_components(graph_component, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, terms_for_component(graph_component)))
+      end
+
+      def self.from_terms(graph_terms, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms))
+      end
+
+      def self.from_trace_digest(trace_digest, request_signature: nil)
+        new(signature: signature_for_trace_digest(request_signature, trace_digest))
+      end
+
+      def self.signature_for_terms(request_signature, graph_terms)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        %i[frames dependencies contains].each do |group|
+          digest.update("\0")
+          digest.update(group.to_s)
+          Array(graph_terms[group]).each do |term|
+            digest.update("\0")
+            digest.update(term)
+          end
+        end
+        digest.hexdigest
+      end
+
+      def self.signature_for_trace_digest(request_signature, trace_digest)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        digest.update("\0")
+        digest.update(trace_digest)
+        digest.hexdigest
+      end
+
+      def self.request_signature_component(signature)
+        return nil unless signature
+
+        signature.respond_to?(:to_h) ? signature.to_h : signature
+      end
+
+      def self.graph_component(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_component(node.id, node.payload) }.sort_by { |component| component.fetch(:id).to_s },
+          dependencies: graph.dependency_nodes.map { |node| dependency_component(graph, node) }.sort_by { |component| component.fetch(:id).inspect },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| [edge.from, edge.to] }
+            .sort_by(&:inspect)
+        }
+      end
+
+      def self.graph_terms(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_term(node.id, node.payload) },
+          dependencies: graph.dependency_nodes.map { |node| dependency_term(node.id, node.payload, graph.dependency_owner_ids(node.id)) },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| contains_term(edge.from, edge.to) }
+        }
+      end
+
+      def self.terms_for_component(graph_component)
+        {
+          frames: graph_component.fetch(:frames).map { |component| canonical_term(:frame, component.fetch(:id), component.fetch(:payload)) },
+          dependencies: graph_component.fetch(:dependencies).map do |component|
+            canonical_term(:dependency, component.fetch(:id), component.fetch(:dependency), component.fetch(:owners))
+          end,
+          contains: graph_component.fetch(:contains).map { |from, to| contains_term(from, to) }
+        }
+      end
+
+      def self.frame_component(id, payload)
+        {
+          id: id,
+          payload: frame_payload_component(payload)
+        }
+      end
+
+      def self.frame_term(id, payload)
+        canonical_term(:frame, id, frame_payload_component(payload))
+      end
+
+      def self.frame_payload_component(payload)
+        component = payload.reject do |key, _value|
+          key.respond_to?(:to_sym) && FRAME_PAYLOAD_SHAPE_IGNORED_KEYS.include?(key.to_sym)
+        end
+        component = shape_value(component)
+        recipe = payload[:recipe] || payload["recipe"]
+        kind = payload[:kind] || payload["kind"]
+        if recipe && kind.to_s == "render_site"
+          component[:shared_stream_signature] = SharedStreams.signature_for(recipe)
+        end
+        component
+      end
+
+      def self.dependency_component(graph, node)
+        {
+          id: node.id,
+          dependency: shape_value(node.payload.to_h),
+          owners: graph.dependency_owner_ids(node.id).sort_by(&:to_s)
+        }
+      end
+
+      def self.dependency_term(id, dependency, owners)
+        canonical_term(:dependency, id, dependency.to_h, owners.sort_by(&:to_s))
+      end
+
+      def self.contains_term(from, to)
+        canonical_term(:contains, from, to)
+      end
+
+      def self.shape_value(value)
+        case value
+        when Hash
+          value.keys.sort_by(&:to_s).to_h { |key| [key, shape_value(value.fetch(key))] }
+        when Array
+          value.map { |item| shape_value(item) }
+        else
+          value.respond_to?(:to_h) ? shape_value(value.to_h) : value
+        end
+      end
+
+      def self.canonical_term(*parts)
+        parts.map { |part| canonical_value(part) }.join("\0")
+      end
+
+      def self.canonical_value(value)
+        shape_value(value).inspect
+      end
+
+      def initialize(signature:)
+        @signature = signature
+      end
+
+      class Trace
+        def initialize(graph_version:)
+          @graph_version = graph_version
+          @seen_frame_ids = {}
+          @seen_dependency_keys = {}
+          @seen_dependency_owner_ids_by_key = Hash.new { |owners, dependency_key| owners[dependency_key] = {} }
+          @seen_contains_edges = {}
+          @digest = Digest::SHA256.new
+          @digest.update("subscription-shape-trace")
+          @invalid = false
+          @recorded = false
+        end
+
+        def synchronized_with?(graph)
+          !@invalid && @graph_version == graph.version
+        end
+
+        def invalidate!
+          @invalid = true
+        end
+
+        def record_frame(frame_id, metadata, parent_id:, graph_version:)
+          return if @invalid
+
+          unless @seen_frame_ids.key?(frame_id)
+            @seen_frame_ids[frame_id] = true
+            record_digest_term(:frame, SubscriptionShape.frame_term(frame_id, metadata))
+          end
+          edge_key = [parent_id, frame_id]
+          unless @seen_contains_edges.key?(edge_key)
+            @seen_contains_edges[edge_key] = true
+            record_digest_term(:contains, SubscriptionShape.contains_term(parent_id, frame_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def record_dependency(owner_id, dependency, graph_version:)
+          return if @invalid
+
+          dependency_cache_key = dependency.cache_key
+          unless @seen_dependency_keys.key?(dependency_cache_key)
+            @seen_dependency_keys[dependency_cache_key] = true
+            dependency_payload = SubscriptionShape.shape_value(dependency.to_h)
+            record_digest_term(:dependency, SubscriptionShape.canonical_term(:dependency, dependency_cache_key, dependency_payload))
+          end
+          unless @seen_dependency_owner_ids_by_key[dependency_cache_key].key?(owner_id)
+            @seen_dependency_owner_ids_by_key[dependency_cache_key][owner_id] = true
+            record_digest_term(:dependency_owner, SubscriptionShape.canonical_term(:dependency_owner, dependency_cache_key, owner_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def covers?(graph)
+          synchronized_with?(graph) && (recorded? || graph_shape_empty?(graph))
+        end
+
+        def subscription_shape(request_signature: nil)
+          SubscriptionShape.from_trace_digest(@digest.hexdigest, request_signature: request_signature)
+        end
+
+        private
+
+        def recorded?
+          @recorded
+        end
+
+        def graph_shape_empty?(graph)
+          graph.frame_nodes.empty? &&
+            graph.dependency_nodes.empty? &&
+            graph.edges.none? { |edge| edge.reason == :contains }
+        end
+
+        def record_digest_term(kind, term)
+          @digest.update("\0")
+          @digest.update(kind.to_s)
+          @digest.update("\0")
+          @digest.update(term)
+        end
+      end
+    end
+  end
+end