upkeep-rails 0.2.5-aarch64-linux-gnu

data/lib/upkeep/active_record_query.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "active_record"
+require_relative "sqlglot"
+require_relative "sql_dependency_analysis"
+
+module Upkeep
+  module ActiveRecordQuery
+    class OpaqueRelationError < StandardError
+      attr_reader :model_name, :table_name, :sql, :reasons
+
+      def initialize(relation, reasons:)
+        @model_name = relation.klass.name
+        @table_name = relation.klass.table_name
+        @sql = relation.to_sql
+        @reasons = reasons
+
+        super(build_message)
+      rescue StandardError => error
+        super("Upkeep could not analyze this Active Record relation: #{error.message}")
+      end
+
+      def suggestions
+        [
+          "Check that the relation emits valid SQL for the configured database adapter.",
+          "Report unsupported SQL with the query, adapter, and SQLGlot version.",
+          "Render this boundary outside Upkeep reactivity until the SQL is supported."
+        ]
+      end
+
+      private
+
+      def build_message
+        <<~MESSAGE
+          Upkeep cannot make this Active Record relation reactive because SQLGlot could not prove its dependencies.
+
+          Relation:
+            #{model_name} (#{table_name})
+
+          SQL:
+            #{sql}
+
+          Why:
+          #{reasons.map { |reason| "            - #{reason}" }.join("\n")}
+
+          What to do:
+          #{suggestions.map { |suggestion| "            - #{suggestion}" }.join("\n")}
+        MESSAGE
+      end
+    end
+
+    Result = Data.define(
+      :primary_table,
+      :table_columns,
+      :coverage,
+      :sql,
+      :primary_key,
+      :appendable,
+      :limit_value,
+      :predicates
+    ) do
+      def tables = table_columns.keys.sort
+      def appendable? = appendable
+    end
+
+    module_function
+
+    def analyze(relation)
+      sql = relation.to_sql
+      dialect = dialect_for(relation.klass.connection)
+      statement = SQLGlot.parse(sql, dialect: dialect)
+      schema = Schema.for(relation.klass.connection, statement)
+      mapping_schema = SQLGlot::MappingSchema.new(schema, dialect: dialect)
+      qualified_statement = SQLGlot.qualify_columns(statement, mapping_schema)
+      dependency_statement = SQLDependencyAnalysis.preserve_wildcard_projections(
+        statement,
+        qualified_statement
+      )
+      dependency = SQLDependencyAnalysis.analyze(
+        dependency_statement,
+        schema: schema,
+        scope: SQLGlot.build_scope(dependency_statement)
+      )
+
+      Result.new(
+        primary_table: relation.klass.table_name,
+        table_columns: with_primary_key(
+          dependency.table_columns,
+          relation.klass.table_name,
+          relation.klass.primary_key
+        ),
+        coverage: :columns,
+        sql: sql,
+        primary_key: relation.klass.primary_key,
+        appendable: dependency.appendable?,
+        limit_value: dependency.limit_value,
+        predicates: dependency.predicates
+      )
+    rescue ActiveRecord::StatementInvalid,
+      SQLGlot::Error,
+      SQLDependencyAnalysis::UnsupportedError,
+      KeyError => error
+      raise OpaqueRelationError.new(
+        relation,
+        reasons: [
+          "#{error.class}: #{error.message}",
+          "dialect: #{safe_dialect(relation)}",
+          "SQLGlot: #{SQLGlot.version}"
+        ]
+      )
+    end
+
+    def analyze_for_write(relation)
+      analyze(relation)
+    rescue OpaqueRelationError
+      table_only_result(relation)
+    end
+
+    def dialect_for(connection)
+      adapter = connection.adapter_name.to_s.downcase
+
+      case adapter
+      when /postgres/
+        :postgres
+      when /mysql|trilogy/
+        :mysql
+      when /sqlite/
+        :sqlite
+      when /sqlserver/
+        :tsql
+      when /oracle/
+        :oracle
+      else
+        raise SQLDependencyAnalysis::UnsupportedError,
+          "unsupported Active Record adapter: #{connection.adapter_name}"
+      end
+    end
+
+    def with_primary_key(table_columns, primary_table, primary_key)
+      columns = table_columns.transform_values(&:dup)
+      columns[primary_table.to_s] ||= []
+      columns[primary_table.to_s] << primary_key.to_s if primary_key
+      columns.transform_values { |names| names.uniq.sort }.sort.to_h
+    end
+    private_class_method :with_primary_key
+
+    def table_only_result(relation)
+      primary_table = relation.klass.table_name
+      primary_key = relation.klass.primary_key
+
+      Result.new(
+        primary_table: primary_table,
+        table_columns: {
+          primary_table => [primary_key].compact.map(&:to_s)
+        },
+        coverage: :tables,
+        sql: relation.to_sql,
+        primary_key: primary_key,
+        appendable: false,
+        limit_value: relation.limit_value,
+        predicates: []
+      )
+    end
+    private_class_method :table_only_result
+
+    def safe_dialect(relation)
+      dialect_for(relation.klass.connection)
+    rescue StandardError
+      relation.klass.connection.adapter_name
+    end
+    private_class_method :safe_dialect
+
+    module Schema
+      module_function
+
+      def for(connection, statement)
+        physical_tables(statement).each_with_object({}) do |table, schema|
+          lookup_name = table.split(".").last
+          columns = begin
+            connection.schema_cache.columns(lookup_name).to_h do |column|
+              [column.name, column.sql_type]
+            end
+          rescue ActiveRecord::StatementInvalid
+            {}
+          end
+          schema[table] = columns.freeze unless columns.empty?
+        end.freeze
+      end
+
+      def physical_tables(statement)
+        collect_sources(statement, visible_ctes: Set.new).uniq.sort
+      end
+
+      def collect_sources(node, visible_ctes:)
+        case node
+        when Array
+          node.flat_map { |child| collect_sources(child, visible_ctes: visible_ctes) }
+        when Hash
+          return collect_select(node.fetch("Select"), visible_ctes: visible_ctes) if node.key?("Select")
+          return collect_table(node.fetch("Table"), visible_ctes: visible_ctes) if node.key?("Table")
+
+          node.each_value.flat_map do |child|
+            collect_sources(child, visible_ctes: visible_ctes)
+          end
+        else
+          []
+        end
+      end
+      private_class_method :collect_sources
+
+      def collect_select(select, visible_ctes:)
+        sources = []
+        local_ctes = visible_ctes.dup
+
+        Array(select["ctes"]).each do |cte|
+          cte_scope = local_ctes.dup
+          cte_scope << cte["name"] if cte["recursive"] && cte["name"]
+          sources.concat(collect_sources(cte["query"], visible_ctes: cte_scope))
+          local_ctes << cte["name"] if cte["name"]
+        end
+
+        select
+          .reject { |key, _value| key == "ctes" }
+          .each_value do |child|
+            sources.concat(collect_sources(child, visible_ctes: local_ctes))
+          end
+
+        sources
+      end
+      private_class_method :collect_select
+
+      def collect_table(table, visible_ctes:)
+        name = [table["catalog"], table["schema"], table["name"]]
+          .compact
+          .reject(&:empty?)
+          .join(".")
+        unqualified = [table["catalog"], table["schema"]].all? do |part|
+          part.nil? || part.empty?
+        end
+
+        return [] if unqualified && visible_ctes.include?(name)
+
+        [name]
+      end
+      private_class_method :collect_table
+    end
+  end
+end

data/lib/upkeep/capture/request.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Upkeep
+  module Capture
+    RequestSignature = Data.define(:controller, :action, :method, :fullpath)
+
+    RequestResult = Data.define(
+      :action_result,
+      :html,
+      :recorder,
+      :response_status,
+      :response_content_type,
+      :response_media_type,
+      :response_successful,
+      :signature,
+      :timings,
+      :counters
+    ) do
+      def successful?
+        !!response_successful
+      end
+
+      def html_response?
+        response_media_type == "text/html" ||
+          response_content_type.to_s.start_with?("text/html")
+      end
+    end
+
+    module Request
+      module_function
+
+      def call(controller, profile: false)
+        timings = {}
+        counters = {}
+        action_result, recorder = measure(timings, :action_ms) do
+          if profile
+            profile_action(timings, counters) do
+              Runtime::Observation.capture_request(profile: true) { yield }
+            end
+          else
+            Runtime::Observation.capture_request { yield }
+          end
+        end
+        timings.merge!(recorder.profile_timings)
+        counters.merge!(recorder.profile_counts)
+        html = measure(timings, :response_body_ms) { response_body_html(controller.response.body) }
+        signature = measure(timings, :signature_ms) { signature_for(controller) }
+        RequestResult.new(
+          action_result,
+          html,
+          recorder,
+          controller.response.status,
+          controller.response.content_type,
+          controller.response.media_type,
+          controller.response.successful?,
+          signature,
+          timings,
+          counters
+        )
+      end
+
+      def profile_action(timings, counters)
+        collector = ActionProfiler.new
+        collector.capture { yield }.tap do
+          timings.merge!(collector.timings)
+          counters.merge!(collector.counters)
+        end
+      end
+
+      def signature_for(controller)
+        request = controller.request
+        RequestSignature.new(
+          controller.class.name,
+          controller.action_name,
+          request.request_method,
+          request.fullpath
+        )
+      end
+
+      def response_body_html(body)
+        case body
+        when String
+          body
+        when Array
+          body.join
+        else
+          return body.body.join if body.respond_to?(:body) && body.body.respond_to?(:join)
+          return body.to_a.join if body.respond_to?(:to_a)
+
+          body.to_s
+        end
+      end
+
+      def measure(timings, key)
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        yield
+      ensure
+        timings[key] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000.0).round(3)
+      end
+
+      class ActionProfiler
+        EVENT_MAP = {
+          "sql.active_record" => :sql,
+          "render_template.action_view" => :render_template,
+          "render_partial.action_view" => :render_partial,
+          "render_collection.action_view" => :render_collection
+        }.freeze
+
+        attr_reader :timings, :counters
+
+        def initialize
+          @thread = Thread.current
+          @timings = Hash.new(0.0)
+          @counters = Hash.new(0)
+        end
+
+        def capture
+          callback = lambda do |name, started, finished, unique_id, payload|
+            next unless Thread.current.equal?(@thread)
+
+            event = ActiveSupport::Notifications::Event.new(name, started, finished, unique_id, payload)
+            record(event)
+          end
+
+          ActiveSupport::Notifications.subscribed(callback, /\A(sql\.active_record|render_(template|partial|collection)\.action_view)\z/) do
+            yield
+          end
+        ensure
+          @timings.transform_values! { |value| value.round(3) }
+        end
+
+        private
+
+        def record(event)
+          key = EVENT_MAP[event.name]
+          return unless key
+          return if ignored_sql?(event)
+
+          @timings[:"#{key}_ms"] += event.duration
+          @counters[:"#{key}_count"] += 1
+          @timings[:view_ms] += event.duration if event.name.end_with?(".action_view")
+        end
+
+        def ignored_sql?(event)
+          event.name == "sql.active_record" && event.payload[:name] == "SCHEMA"
+        end
+      end
+    end
+  end
+end

data/lib/upkeep/dag/subscription_shape.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../shared_streams"
+require_relative "../version"
+
+module Upkeep
+  module DAG
+    class SubscriptionShape
+      DIGEST_SCOPE = "upkeep-subscription-shape"
+      FRAME_PAYLOAD_SHAPE_IGNORED_KEYS = %i[manifest recipe].freeze
+
+      attr_reader :signature
+
+      def self.from_graph(graph, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms(graph)))
+      end
+
+      def self.from_components(graph_component, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, terms_for_component(graph_component)))
+      end
+
+      def self.from_terms(graph_terms, request_signature: nil)
+        new(signature: signature_for_terms(request_signature, graph_terms))
+      end
+
+      def self.from_trace_digest(trace_digest, request_signature: nil)
+        new(signature: signature_for_trace_digest(request_signature, trace_digest))
+      end
+
+      def self.signature_for_terms(request_signature, graph_terms)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        %i[frames dependencies contains].each do |group|
+          digest.update("\0")
+          digest.update(group.to_s)
+          Array(graph_terms[group]).each do |term|
+            digest.update("\0")
+            digest.update(term)
+          end
+        end
+        digest.hexdigest
+      end
+
+      def self.signature_for_trace_digest(request_signature, trace_digest)
+        digest = Digest::SHA256.new
+        digest.update(DIGEST_SCOPE)
+        digest.update("\0")
+        digest.update(Upkeep::VERSION)
+        digest.update("\0")
+        digest.update(canonical_value(request_signature_component(request_signature)))
+        digest.update("\0")
+        digest.update(trace_digest)
+        digest.hexdigest
+      end
+
+      def self.request_signature_component(signature)
+        return nil unless signature
+
+        signature.respond_to?(:to_h) ? signature.to_h : signature
+      end
+
+      def self.graph_component(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_component(node.id, node.payload) }.sort_by { |component| component.fetch(:id).to_s },
+          dependencies: graph.dependency_nodes.map { |node| dependency_component(graph, node) }.sort_by { |component| component.fetch(:id).inspect },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| [edge.from, edge.to] }
+            .sort_by(&:inspect)
+        }
+      end
+
+      def self.graph_terms(graph)
+        {
+          frames: graph.frame_nodes.map { |node| frame_term(node.id, node.payload) },
+          dependencies: graph.dependency_nodes.map { |node| dependency_term(node.id, node.payload, graph.dependency_owner_ids(node.id)) },
+          contains: graph.edges
+            .select { |edge| edge.reason == :contains }
+            .map { |edge| contains_term(edge.from, edge.to) }
+        }
+      end
+
+      def self.terms_for_component(graph_component)
+        {
+          frames: graph_component.fetch(:frames).map { |component| canonical_term(:frame, component.fetch(:id), component.fetch(:payload)) },
+          dependencies: graph_component.fetch(:dependencies).map do |component|
+            canonical_term(:dependency, component.fetch(:id), component.fetch(:dependency), component.fetch(:owners))
+          end,
+          contains: graph_component.fetch(:contains).map { |from, to| contains_term(from, to) }
+        }
+      end
+
+      def self.frame_component(id, payload)
+        {
+          id: id,
+          payload: frame_payload_component(payload)
+        }
+      end
+
+      def self.frame_term(id, payload)
+        canonical_term(:frame, id, frame_payload_component(payload))
+      end
+
+      def self.frame_payload_component(payload)
+        component = payload.reject do |key, _value|
+          key.respond_to?(:to_sym) && FRAME_PAYLOAD_SHAPE_IGNORED_KEYS.include?(key.to_sym)
+        end
+        component = shape_value(component)
+        recipe = payload[:recipe] || payload["recipe"]
+        kind = payload[:kind] || payload["kind"]
+        if recipe && kind.to_s == "render_site"
+          component[:shared_stream_signature] = SharedStreams.signature_for(recipe)
+        end
+        component
+      end
+
+      def self.dependency_component(graph, node)
+        {
+          id: node.id,
+          dependency: shape_value(node.payload.to_h),
+          owners: graph.dependency_owner_ids(node.id).sort_by(&:to_s)
+        }
+      end
+
+      def self.dependency_term(id, dependency, owners)
+        canonical_term(:dependency, id, dependency.to_h, owners.sort_by(&:to_s))
+      end
+
+      def self.contains_term(from, to)
+        canonical_term(:contains, from, to)
+      end
+
+      def self.shape_value(value)
+        case value
+        when Hash
+          value.keys.sort_by(&:to_s).to_h { |key| [key, shape_value(value.fetch(key))] }
+        when Array
+          value.map { |item| shape_value(item) }
+        else
+          value.respond_to?(:to_h) ? shape_value(value.to_h) : value
+        end
+      end
+
+      def self.canonical_term(*parts)
+        parts.map { |part| canonical_value(part) }.join("\0")
+      end
+
+      def self.canonical_value(value)
+        shape_value(value).inspect
+      end
+
+      def initialize(signature:)
+        @signature = signature
+      end
+
+      class Trace
+        def initialize(graph_version:)
+          @graph_version = graph_version
+          @seen_frame_ids = {}
+          @seen_dependency_keys = {}
+          @seen_dependency_owner_ids_by_key = Hash.new { |owners, dependency_key| owners[dependency_key] = {} }
+          @seen_contains_edges = {}
+          @digest = Digest::SHA256.new
+          @digest.update("subscription-shape-trace")
+          @invalid = false
+          @recorded = false
+        end
+
+        def synchronized_with?(graph)
+          !@invalid && @graph_version == graph.version
+        end
+
+        def invalidate!
+          @invalid = true
+        end
+
+        def record_frame(frame_id, metadata, parent_id:, graph_version:)
+          return if @invalid
+
+          unless @seen_frame_ids.key?(frame_id)
+            @seen_frame_ids[frame_id] = true
+            record_digest_term(:frame, SubscriptionShape.frame_term(frame_id, metadata))
+          end
+          edge_key = [parent_id, frame_id]
+          unless @seen_contains_edges.key?(edge_key)
+            @seen_contains_edges[edge_key] = true
+            record_digest_term(:contains, SubscriptionShape.contains_term(parent_id, frame_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def record_dependency(owner_id, dependency, graph_version:)
+          return if @invalid
+
+          dependency_cache_key = dependency.cache_key
+          unless @seen_dependency_keys.key?(dependency_cache_key)
+            @seen_dependency_keys[dependency_cache_key] = true
+            dependency_payload = SubscriptionShape.shape_value(dependency.to_h)
+            record_digest_term(:dependency, SubscriptionShape.canonical_term(:dependency, dependency_cache_key, dependency_payload))
+          end
+          unless @seen_dependency_owner_ids_by_key[dependency_cache_key].key?(owner_id)
+            @seen_dependency_owner_ids_by_key[dependency_cache_key][owner_id] = true
+            record_digest_term(:dependency_owner, SubscriptionShape.canonical_term(:dependency_owner, dependency_cache_key, owner_id))
+          end
+          @recorded = true
+          @graph_version = graph_version
+        end
+
+        def covers?(graph)
+          synchronized_with?(graph) && (recorded? || graph_shape_empty?(graph))
+        end
+
+        def subscription_shape(request_signature: nil)
+          SubscriptionShape.from_trace_digest(@digest.hexdigest, request_signature: request_signature)
+        end
+
+        private
+
+        def recorded?
+          @recorded
+        end
+
+        def graph_shape_empty?(graph)
+          graph.frame_nodes.empty? &&
+            graph.dependency_nodes.empty? &&
+            graph.edges.none? { |edge| edge.reason == :contains }
+        end
+
+        def record_digest_term(kind, term)
+          @digest.update("\0")
+          @digest.update(kind.to_s)
+          @digest.update("\0")
+          @digest.update(term)
+        end
+      end
+    end
+  end
+end