graph_weaver 0.4.6 → 0.5.1

data/lib/graph_weaver/testing/cassette.rb

@@ -7,92 +7,205 @@ require "yaml"
 
 module GraphWeaver
   module Testing
-    # Raised by Replayer when a request has no recording.
+    # Raised by Replayer when a request has no recording. The query
+    # usually matches and the variables don't, so the variables lead and
+    # the recorded ones for the same query come next — the diff you'd
+    # otherwise do by eye against the YAML.
     class MissingRecording < GraphWeaver::Error
-      def initialize(path:, query:)
-        super(<<~MSG.strip)
-          no recording for this request in #{path} — re-record it
-          (Recorder / Cassette.use with a live client, or delete
-          the cassette to start over). Query:
-          #{query.strip[0, 200]}
-        MSG
+      # how many recorded variable sets to print before summarizing
+      SHOWN = 5
+
+      def initialize(path:, query:, variables:, recorded:, size:)
+        super([
+          "no recording for this request in #{path}",
+          "  variables: #{Cassette.normalize_variables(variables).inspect}",
+          "  #{self.class.recorded_summary(recorded, size)}",
+          "  query: #{Cassette.summarize(query)}",
+          "re-record it (GRAPHWEAVER_RECORD=1 with a client:), or delete the cassette to start over.",
+        ].join("\n"))
+      end
+
+      def self.recorded_summary(recorded, size)
+        return "no entry recorded for this query (#{size} in the cassette)" if recorded.empty?
+
+        more = recorded.size > SHOWN ? " (+#{recorded.size - SHOWN} more)" : ""
+        "#{recorded.size} #{(recorded.size == 1) ? "entry" : "entries"} recorded for this query, " \
+          "with variables #{recorded.first(SHOWN).map(&:inspect).join(", ")}#{more}"
+      end
+    end
+
+    class << self
+      # The cassette-backed client: replays spec/cassettes/<name>.yml when
+      # it exists, records it through client: when it doesn't (VCR's once
+      # mode). Record mode (GRAPHWEAVER_RECORD=1 / config.record) always
+      # records, so it needs a client: too.
+      #
+      #      client = GraphWeaver::Testing.cassette("github", client: live)
+      #      result = RepoQuery.execute!(client:, owner: "dpep")
+      #
+      def cassette(name, client: nil)
+        file = Cassette.new(name)
+
+        if client && (config.record || !file.exist?)
+          Recorder.new(client, file)
+        elsif config.record
+          # record mode without a client would quietly serve the stale
+          # recording — the one thing "re-record everything" didn't ask for
+          raise GraphWeaver::Error, "record mode is on but no `client:` was given for #{file.path} " \
+            "— pass a live `client:` to re-record it, or turn record mode off " \
+            "(GRAPHWEAVER_RECORD / Testing.config.record)."
+        elsif file.exist?
+          Replayer.new(file)
+        else
+          # a first run, not a missing recording: there is no request yet
+          raise GraphWeaver::Error, "#{file.path} doesn't exist and no `client:` was given to " \
+            "record with — pass `client:` on the first run, or commit the cassette."
+        end
       end
     end
 
-    # Capture/replay above the transport (no HTTP interception): a
-    # cassette is a YAML file of {query, variables, response} entries,
-    # keyed on the normalized query + variables.
-    #
-    #      # record against a real client, replay when the file exists:
-    #      client = GraphWeaver::Testing::Cassette.use("github", client: real)
-    #
-    # Cassettes hold real responses — anonymize before committing:
-    #
-    #      Cassette.new("spec/cassettes/github.yml").anonymize!(schema:)
-    #
-    # keeps every shape (list lengths, null positions, enums, __typename,
-    # id relationships via a consistent mapping) while replacing values
-    # with fakes, semantically matched where field names allow.
+    # The cassette file itself: a YAML list of {query, variables, response}
+    # entries. Testing.cassette wraps one in a record/replay client; this is
+    # the file object behind it — and what the anonymize rake task rewrites.
     class Cassette
+      # What replaying one cassette through the current generated modules
+      # found. `checked` is how many recordings a module claimed: a run that
+      # claimed none proved nothing, which is a different answer from "all
+      # good" — the same distinction `federation:diff` draws.
+      Check = Struct.new(:path, :checked, :skipped, :stale, keyword_init: true) do
+        def ok? = stale.empty?
+
+        def report
+          counted = ["#{checked} checked"]
+          counted << "#{skipped} not sent by any query module" if skipped.positive?
+
+          ["#{path}: #{stale.size} stale (#{counted.join(", ")})"] +
+            stale.flat_map do |entry|
+              ["  #{entry.module_name} #{entry.variables.inspect}", "    #{entry.message}"]
+            end
+        end
+      end
+
+      # One recording the generated structs can no longer read.
+      Stale = Struct.new(:module_name, :variables, :message, keyword_init: true)
+
+      # Shapes that are a credential whatever the field around them is
+      # called. Anonymization can't cover everything a cassette holds — the
+      # variables ARE the replay key, so they're written verbatim — so the
+      # bytes that reach disk get one look before anyone commits them.
+      # Deliberately narrow: a false alarm costs a glance, while a password
+      # like "hunter2" has no shape at all, so a quiet run is not a clean
+      # bill of health.
+      CREDENTIAL_SHAPES = {
+        "a JWT" => /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\./,
+        "an AWS access key" => /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/,
+        "a GitHub token" => /\b(?:gh[opusr]|github_pat)_[A-Za-z0-9_]{20,}/,
+        "a Slack token" => /\bxox[baprs]-[A-Za-z0-9-]{10,}/,
+        "a Stripe key" => /\bsk_(?:live|test)_[A-Za-z0-9]{10,}/,
+        "a private key" => /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+        "an Authorization header" => /\bBearer\s+\S{16,}/,
+      }.freeze
+
       attr_reader :path
 
       def initialize(path)
         @path = Testing.cassette_path(path)
         @entries = File.exist?(@path) ? YAML.safe_load_file(@path, aliases: true) : []
+        @flagged = []
       end
 
       def exist? = File.exist?(@path)
       def size = @entries.size
 
-      # Replay when recorded, record when not (VCR's once mode).
-      # client: is required to record; omit it to replay-or-raise.
-      # With Testing.config.record on (or GRAPHWEAVER_RECORD=1), always
-      # records — the "just re-record everything" switch.
-      def self.use(path, client: nil)
-        cassette = new(path)
-        if Testing.config.record && client
-          Recorder.new(client, cassette)
-        elsif cassette.exist?
-          Replayer.new(cassette)
-        elsif client
-          Recorder.new(client, cassette)
-        else
-          raise MissingRecording.new(path: cassette.path, query: "(no client to record with)")
-        end
+      def lookup(query, variables, operation_name = nil)
+        wanted = self.class.key(query, variables, operation_name)
+        @entries.find { |entry| self.class.entry_key(entry) == wanted }
       end
 
-      def lookup(query, variables)
-        wanted = self.class.key(query, variables)
-      @entries.find { |entry| entry["key"] == wanted }
+      # every variables hash recorded for this query — what a miss needs
+      # to show, since the variables are what usually differ
+      def variants(query, operation_name = nil)
+        normalized = self.class.normalize_query(query)
+        @entries.select do |entry|
+          self.class.normalize_query(entry["query"]) == normalized && entry["operationName"] == operation_name
+        end.map { |entry| entry["variables"] || {} }
       end
 
-      def record(query, variables, response)
-        entry = {
-          "key" => self.class.key(query, variables),
-          "query" => query,
-          "variables" => self.class.normalize_variables(variables),
-          "response" => response,
-        }
-        @entries.reject! { |existing| existing["key"] == entry["key"] }
+      def record(query, variables, response, operation_name = nil)
+        entry = { "query" => query }
+        entry["operationName"] = operation_name if operation_name
+        entry["variables"] = self.class.normalize_variables(variables)
+        entry["response"] = response
+
+        wanted = self.class.key(query, variables, operation_name)
+        @entries.reject! { |existing| self.class.entry_key(existing) == wanted }
         @entries << entry
         save
       end
 
+      # Replay every recording through `modules` — the generated query
+      # modules — and report the ones that no longer cast.
+      #
+      # A cassette is the one artifact here recorded from a *foreign* server,
+      # and nothing else notices when that server's answers drift out of the
+      # shape the structs were generated for: `verify`, `queries:check` and
+      # `schema:diff` all ask about the local side. Without this the drift
+      # surfaces mid-spec as a `TypeError` naming a struct and a sorbet
+      # frame, with nothing pointing at the stale file.
+      #
+      # Matching is on the query text, which is the module that sent it — a
+      # recording no module sends is skipped rather than guessed at.
+      def check(modules)
+        index = modules.to_h { |mod| [self.class.normalize_query(mod.const_get(:QUERY)), mod] }
+        checked = 0
+        stale = @entries.filter_map do |entry|
+          mod = index[self.class.normalize_query(entry["query"])] or next
+          checked += 1
+
+          begin
+            mod.from_response(entry["response"])
+            nil
+          rescue GraphWeaver::Error => e
+            Stale.new(module_name: mod.name, variables: entry["variables"] || {}, message: e.message)
+          end
+        end
+
+        Check.new(path: @path, checked:, skipped: @entries.size - checked, stale:)
+      end
+
       # Replace recorded response values with fakes, preserving structure.
       # Walks each entry's query against the schema (like FakeClient,
       # but transforming what's there instead of generating from scratch).
       def anonymize!(schema:, seed: nil, mode: nil)
         anonymizer = Anonymizer.new(schema:, seed:, mode:)
         @entries.each do |entry|
-          data = entry.dig("response", "data")
-          entry["response"]["data"] = anonymizer.anonymize(entry["query"], data) if data
+          entry["response"] = anonymizer.anonymize(entry["query"], entry["response"]) if entry["response"]
         end
         save
         self
       end
 
-      def self.key(query, variables)
-        { "query" => query.gsub(/\s+/, " ").strip, "variables" => normalize_variables(variables) }
+      # The request's identity, exactly as the server sees it. operationName
+      # is part of that: it picks the operation the document runs, so two
+      # requests with identical text but different names are different
+      # requests. Derived, never stored — the file holds the request once,
+      # so a hand-edited entry can't disagree with what replay matches on.
+      def self.key(query, variables, operation_name = nil)
+        key = { "query" => normalize_query(query), "variables" => normalize_variables(variables) }
+        key["operationName"] = operation_name if operation_name
+        key
+      end
+
+      def self.entry_key(entry)
+        key(entry["query"], entry["variables"], entry["operationName"])
+      end
+
+      def self.normalize_query(query) = query.gsub(/\s+/, " ").strip
+
+      # one readable line: an error naming a 60-line query is a wall, not a hint
+      def self.summarize(query, limit: 160)
+        normalized = normalize_query(query)
+        (normalized.length > limit) ? "#{normalized[0, limit]}…" : normalized
       end
 
       # JSON round-trip so symbol keys become strings — otherwise YAML.dump
@@ -105,22 +218,39 @@ module GraphWeaver
       private
 
       def save
+        yaml = YAML.dump(@entries)
         FileUtils.mkdir_p(File.dirname(@path))
-        File.write(@path, YAML.dump(@entries))
+        File.write(@path, yaml)
+        flag_credentials(yaml)
+      end
+
+      # Once per shape per cassette: a recording run saves after every
+      # request, and one line is a warning where forty is noise. On stderr
+      # rather than GraphWeaver.logger — the logger is silent by default,
+      # and this has to reach whoever is about to commit the file.
+      def flag_credentials(yaml)
+        found = CREDENTIAL_SHAPES.reject { |name, _| @flagged.include?(name) }
+          .select { |_, pattern| pattern.match?(yaml) }.keys
+        return if found.empty?
+
+        @flagged.concat(found)
+        warn "graph_weaver: #{@path} contains #{found.join(", ")} — a cassette is committed as " \
+          "written, so review this one first. Testing.config.anonymize scrubs the response; the " \
+          "query and variables are the replay key and are recorded verbatim."
       end
     end
 
     # Tees requests through a live client and records every response.
-    # With Testing.config.anonymize (or anonymize: true), responses are
-    # anonymized as they're recorded — and the anonymized version is what
-    # the caller sees too, so assertions written now hold on replay.
+    # With Testing.config.anonymize, responses are anonymized as they're
+    # recorded — and the anonymized version is what the caller sees too,
+    # so assertions written now hold on replay.
     class Recorder
-      def initialize(client, cassette, anonymize: nil)
+      def initialize(client, cassette)
         @client = client
         @cassette = cassette.is_a?(Cassette) ? cassette : Cassette.new(cassette)
 
         config = Testing.config
-        if anonymize.nil? ? config.anonymize : anonymize
+        if config.anonymize
           unless config.schema
             raise ArgumentError, "anonymizing recordings needs GraphWeaver::Testing.config.schema"
           end
@@ -129,13 +259,11 @@ module GraphWeaver
         end
       end
 
-      def execute(query, variables: {})
-        response = @client.execute(query, variables:).to_h
-        if @anonymizer && (data = response["data"])
-          response = response.merge("data" => @anonymizer.anonymize(query, data))
-        end
+      def execute(query, variables: {}, operation_name: nil)
+        response = @client.execute(query, variables:, operation_name:).to_h
+        response = @anonymizer.anonymize(query, response) if @anonymizer
 
-        @cassette.record(query, variables, response)
+        @cassette.record(query, variables, response, operation_name)
         response
       end
     end
@@ -147,9 +275,12 @@ module GraphWeaver
         @cassette = cassette.is_a?(Cassette) ? cassette : Cassette.new(cassette)
       end
 
-      def execute(query, variables: {})
-        entry = @cassette.lookup(query, variables)
-        raise MissingRecording.new(path: @cassette.path, query:) unless entry
+      def execute(query, variables: {}, operation_name: nil)
+        entry = @cassette.lookup(query, variables, operation_name)
+        unless entry
+          raise MissingRecording.new(path: @cassette.path, query:, variables:,
+            recorded: @cassette.variants(query, operation_name), size: @cassette.size)
+        end
 
         entry["response"]
       end
@@ -161,18 +292,52 @@ module GraphWeaver
     class Anonymizer
       include GraphWeaver::Selection
 
+      # Keys under `errors`/`extensions` whose value describes the request
+      # rather than carrying data: `path` and `locations` point into the
+      # document, and `code` is the errors-world enum — call sites branch on
+      # it exactly as they branch on an enum in `data`, which is preserved
+      # for the same reason.
+      VERBATIM_KEYS = %w[path locations code].freeze
+
       def initialize(schema:, seed: nil, mode: nil)
         @schema = schema
         @values = Values.new(seed:, mode:)
       end
 
-      def anonymize(query, data)
-        operation = load_operation(query)
+      # The whole response, not just `data`: an error message routinely
+      # quotes the input that caused it, and `extensions` is whatever the
+      # server felt like attaching. One rule — `data` is walked against the
+      # schema, everything else by shape.
+      def anonymize(query, response)
+        response.to_h do |key, value|
+          [key, (key == "data") ? data_value(query, value) : untyped_value(key, value)]
+        end
+      end
+
+      private
 
+      def data_value(query, data)
+        return if data.nil?
+
+        operation = load_operation(query)
         object_value(operation_root_type(operation), operation.selections, data)
       end
 
-      private
+      # No schema stands behind errors or extensions, so shape is all there
+      # is to preserve: keys, nesting, list lengths, nulls and booleans
+      # survive; every string and number is replaced.
+      def untyped_value(key, value)
+        return value if VERBATIM_KEYS.include?(key)
+
+        case value
+        when Hash then value.to_h { |name, nested| [name, untyped_value(name, nested)] }
+        when Array then value.map { |element| untyped_value(key, element) }
+        when String then @values.scalar("String", key)
+        when Integer then @values.scalar("Int", key)
+        when Float then @values.scalar("Float", key)
+        else value
+        end
+      end
 
       # Anonymization walks recorded data, not a live dispatch. When the query
       # narrows an abstract type without selecting __typename (`named { name
@@ -196,42 +361,45 @@ module GraphWeaver
         end
 
         result = {}
-        each_field(type, selections) do |key, node|
+        # gather (not each_field) so a key selected twice — `a { x } a { y }` —
+        # keeps the MERGED shape codegen's struct expects, not last-writer-wins
+        gather(type, selections).each do |key, nodes|
           next unless data.key?(key)
 
+          node = nodes.first
           result[key] = if node.name == "__typename"
             data[key]
           else
-            field_value(type, node, data[key])
+            field_value(type, node.name, nodes.flat_map(&:selections), data[key])
           end
         end
 
         result
       end
 
-      def field_value(parent_type, node, value)
+      def field_value(parent_type, name, selections, value)
         # a field from a `... on Member` fragment lives on the member, not the
         # abstract type we're walking (no __typename to narrow by), so fall back
         # to whichever possible type declares it
-        field = @schema.get_field(parent_type.graphql_name, node.name) ||
-          @schema.possible_types(parent_type).filter_map { |t| @schema.get_field(t.graphql_name, node.name) }.first
-        type_value(field.type, node, value)
+        field = @schema.get_field(parent_type.graphql_name, name) ||
+          @schema.possible_types(parent_type).filter_map { |t| @schema.get_field(t.graphql_name, name) }.first
+        type_value(field.type, name, selections, value)
       end
 
-      def type_value(type, node, value)
+      def type_value(type, name, selections, value)
         return if value.nil? # preserve null positions
 
         case type.kind.name
         when "NON_NULL"
-          type_value(type.of_type, node, value)
+          type_value(type.of_type, name, selections, value)
         when "LIST"
-          value.map { |element| type_value(type.of_type, node, element) }
+          value.map { |element| type_value(type.of_type, name, selections, element) }
         when "SCALAR"
-          scalar_value(type.graphql_name, node.name, value)
+          scalar_value(type.graphql_name, name, value)
         when "ENUM"
           value # enums aren't PII; preserving them keeps semantics
         when "OBJECT", "UNION", "INTERFACE"
-          object_value(type, node.selections, value)
+          object_value(type, selections, value)
         else
           value
         end

data/lib/graph_weaver/testing/coverage.rb

@@ -0,0 +1,160 @@
+# typed: true
+# frozen_string_literal: true
+
+require "graphql"
+
+require_relative "router"
+
+module GraphWeaver
+  module Testing
+    # How much of a query set the local {Router} can plan, and — for the rest
+    # — exactly what stopped it:
+    #
+    #      rake graph_weaver:federation:coverage SUPERGRAPH=supergraph.graphql
+    #
+    # The router refuses every query that crosses a subgraph boundary, so its
+    # worth to a suite is one number: the fraction of *your* queries it can
+    # answer. Nobody can guess that from outside — it depends on the shape of
+    # your graph and the shape of your queries — so measure it before
+    # deciding the router is (or isn't) enough.
+    #
+    # Planning needs the supergraph and nothing else, so this runs without
+    # any subgraph being loadable, in CI or on a laptop with the SDL alone.
+    class Coverage
+      # One query file's verdict: `category` nil means the router can plan
+      # it, and `subgraph` is where it runs — "accounts+reviews" when the
+      # plan stitches across a boundary. `absent` names the subgraphs that
+      # plan reaches which nothing in this process serves — plannable and
+      # runnable-here are different questions.
+      Result = Struct.new(:path, :subgraph, :absent, :category, :detail) do
+        def servable? = category.nil? && absent.empty?
+      end
+
+      # the label each verdict groups under
+      LABELS = Unplannable::CATEGORIES
+        .transform_values(&:first)
+        .merge(invalid: "doesn't validate against the supergraph")
+        .freeze
+
+      attr_reader :results
+
+      def initialize(supergraph:, queries: GraphWeaver.queries_paths, fragments: GraphWeaver.fragments_paths)
+        source = supergraph.to_s
+        table = GraphWeaver::SchemaLoader.routing_table(source)
+        # with the table incomplete, every number this would report is a guess
+        Unplannable.unsupported!(table)
+
+        # Planning still runs with `absent` empty — a query is plannable or
+        # not whoever is serving. Which subgraphs are *here* is asked
+        # separately, from evidence (a loaded schema defining what the table
+        # says one resolves), and never refuses: with nothing loaded the
+        # answer is simply "none", which is the SDL-alone CI run.
+        @planner = Router::Planner.new(table:, schema: GraphWeaver::SchemaLoader.load(source))
+        @absent = table.subgraphs.reject { |name| Subgraphs.served?(table, name) }
+        @local = @absent.size < table.subgraphs.size
+        @shared = GraphWeaver::Codegen.load_fragments(fragments)
+        @results = GraphWeaver.query_files(queries).map { |path| measure(path) }
+      end
+
+      def plannable = @results.count { |result| result.category.nil? }
+
+      # plannable *and* every subgraph the plan reaches is served here
+      def servable = @results.count(&:servable?)
+
+      # plannable, but reaching a subgraph another process serves
+      def elsewhere = @results.select { |result| result.category.nil? && result.absent.any? }
+
+      def refused = @results.reject { |result| result.category.nil? }
+
+      # whole percent: this is a count of a handful of files, and a decimal
+      # place would claim precision the sample doesn't have
+      def percent = @results.empty? ? 0 : (plannable * 100.0 / @results.size).round
+
+      def report
+        return "no queries found" if @results.empty?
+
+        [headline, *breakdown, *served_here, *refusals].join("\n")
+      end
+      alias to_s report
+
+      def inspect = "#<#{self.class.name} #{plannable}/#{@results.size} plannable>"
+
+      private
+
+      def headline
+        counted = "#{plannable}/#{@results.size} #{(@results.size == 1) ? "query" : "queries"} " \
+          "plannable locally (#{percent}%)"
+        @local ? "#{counted}, #{servable} servable here" : counted
+      end
+
+      # Plan-only is by design — a query is plannable whoever serves it — but
+      # the question this report exists to answer is whether wiring the router
+      # up is worth it, and a suite can only *run* what this process serves.
+      # A partly-local supergraph is the usual migration shape, so the second
+      # number has to be here rather than inferred from a refusal later.
+      def served_here
+        unless @local
+          return ["", "nothing here serves any of this supergraph's subgraphs " \
+            "(#{@absent.join(", ")}), so this counts planning only"]
+        end
+        return [] if elsewhere.empty?
+
+        example = elsewhere.first.absent.first
+        ["", "plannable, but nothing here serves what they reach (#{elsewhere.size}) — name a " \
+          "schema for those subgraphs, fake them (subgraphs: { #{example.inspect} => :fake }), " \
+          "or run these against a real router:"] +
+          elsewhere.map { |result| "  #{name(result)}  #{result.absent.join(", ")}" }
+      end
+
+      # where the plannable ones land — a graph whose queries all sit in one
+      # subgraph is a different situation from one that's evenly spread
+      def breakdown
+        by_subgraph = @results.filter_map(&:subgraph).tally.sort_by { |name, count| [-count, name] }
+        return [] if by_subgraph.empty?
+
+        ["  " + by_subgraph.map { |name, count| "#{name} #{count}" }.join(", ")]
+      end
+
+      def refusals
+        return [] if refused.empty?
+
+        groups = refused.group_by(&:category).sort_by { |category, group| [-group.size, category.to_s] }
+
+        ["", "refused (#{refused.size})"] + groups.flat_map do |category, group|
+          ["", "  #{LABELS.fetch(category, category)} (#{group.size})"] +
+            group.map { |result| "    #{name(result)}  #{result.detail}" }
+        end
+      end
+
+      # the file column: filenames when every query came from one directory,
+      # padded so what sits beside them lines up
+      def name(result) = basename(result).ljust(width)
+
+      def basename(result) = result.path.delete_prefix(shared_dir)
+
+      def width = @width ||= @results.map { |result| basename(result).length }.max
+
+      # the directory every query came from, so the report's file column is
+      # filenames rather than the same path repeated
+      def shared_dir
+        @shared_dir ||= begin
+          dirs = @results.map { |result| File.dirname(result.path) }.uniq
+          dirs.one? ? "#{dirs.first}/" : ""
+        end
+      end
+
+      def measure(path)
+        document = GraphQL.parse(GraphWeaver::Codegen.inline_fragments(File.read(path), @shared, path))
+        errors = @planner.validate(document)
+        return Result.new(path, nil, [], :invalid, errors.first["message"]) if errors.any?
+
+        plan = @planner.plan(document)
+        Result.new(path, plan.where, plan.subgraphs & @absent, nil, nil)
+      rescue Unplannable => e
+        Result.new(path, nil, [], e.category, e.detail)
+      rescue GraphQL::ParseError, GraphWeaver::Error => e
+        Result.new(path, nil, [], :invalid, e.message)
+      end
+    end
+  end
+end