graph_weaver 0.4.6 → 0.5.0

data/lib/graph_weaver/testing/cassette.rb

@@ -7,33 +7,88 @@ require "yaml"
 
 module GraphWeaver
   module Testing
-    # Raised by Replayer when a request has no recording.
+    # Raised by Replayer when a request has no recording. The query
+    # usually matches and the variables don't, so the variables lead and
+    # the recorded ones for the same query come next — the diff you'd
+    # otherwise do by eye against the YAML.
     class MissingRecording < GraphWeaver::Error
-      def initialize(path:, query:)
-        super(<<~MSG.strip)
-          no recording for this request in #{path} — re-record it
-          (Recorder / Cassette.use with a live client, or delete
-          the cassette to start over). Query:
-          #{query.strip[0, 200]}
-        MSG
+      # how many recorded variable sets to print before summarizing
+      SHOWN = 5
+
+      def initialize(path:, query:, variables:, recorded:, size:)
+        super([
+          "no recording for this request in #{path}",
+          "  variables: #{Cassette.normalize_variables(variables).inspect}",
+          "  #{self.class.recorded_summary(recorded, size)}",
+          "  query: #{Cassette.summarize(query)}",
+          "re-record it (GRAPHWEAVER_RECORD=1 with a client:), or delete the cassette to start over.",
+        ].join("\n"))
+      end
+
+      def self.recorded_summary(recorded, size)
+        return "no entry recorded for this query (#{size} in the cassette)" if recorded.empty?
+
+        more = recorded.size > SHOWN ? " (+#{recorded.size - SHOWN} more)" : ""
+        "#{recorded.size} #{(recorded.size == 1) ? "entry" : "entries"} recorded for this query, " \
+          "with variables #{recorded.first(SHOWN).map(&:inspect).join(", ")}#{more}"
       end
     end
 
-    # Capture/replay above the transport (no HTTP interception): a
-    # cassette is a YAML file of {query, variables, response} entries,
-    # keyed on the normalized query + variables.
-    #
-    #      # record against a real client, replay when the file exists:
-    #      client = GraphWeaver::Testing::Cassette.use("github", client: real)
-    #
-    # Cassettes hold real responses — anonymize before committing:
-    #
-    #      Cassette.new("spec/cassettes/github.yml").anonymize!(schema:)
-    #
-    # keeps every shape (list lengths, null positions, enums, __typename,
-    # id relationships via a consistent mapping) while replacing values
-    # with fakes, semantically matched where field names allow.
+    class << self
+      # The cassette-backed client: replays spec/cassettes/<name>.yml when
+      # it exists, records it through client: when it doesn't (VCR's once
+      # mode). Record mode (GRAPHWEAVER_RECORD=1 / config.record) always
+      # records, so it needs a client: too.
+      #
+      #      client = GraphWeaver::Testing.cassette("github", client: live)
+      #      result = RepoQuery.execute!(client:, owner: "dpep")
+      #
+      def cassette(name, client: nil)
+        file = Cassette.new(name)
+
+        if client && (config.record || !file.exist?)
+          Recorder.new(client, file)
+        elsif config.record
+          # record mode without a client would quietly serve the stale
+          # recording — the one thing "re-record everything" didn't ask for
+          raise GraphWeaver::Error, "record mode is on but no `client:` was given for #{file.path} " \
+            "— pass a live `client:` to re-record it, or turn record mode off " \
+            "(GRAPHWEAVER_RECORD / Testing.config.record)."
+        elsif file.exist?
+          Replayer.new(file)
+        else
+          # a first run, not a missing recording: there is no request yet
+          raise GraphWeaver::Error, "#{file.path} doesn't exist and no `client:` was given to " \
+            "record with — pass `client:` on the first run, or commit the cassette."
+        end
+      end
+    end
+
+    # The cassette file itself: a YAML list of {query, variables, response}
+    # entries. Testing.cassette wraps one in a record/replay client; this is
+    # the file object behind it — and what the anonymize rake task rewrites.
     class Cassette
+      # What replaying one cassette through the current generated modules
+      # found. `checked` is how many recordings a module claimed: a run that
+      # claimed none proved nothing, which is a different answer from "all
+      # good" — the same distinction `federation:diff` draws.
+      Check = Struct.new(:path, :checked, :skipped, :stale, keyword_init: true) do
+        def ok? = stale.empty?
+
+        def report
+          counted = ["#{checked} checked"]
+          counted << "#{skipped} not sent by any query module" if skipped.positive?
+
+          ["#{path}: #{stale.size} stale (#{counted.join(", ")})"] +
+            stale.flat_map do |entry|
+              ["  #{entry.module_name} #{entry.variables.inspect}", "    #{entry.message}"]
+            end
+        end
+      end
+
+      # One recording the generated structs can no longer read.
+      Stale = Struct.new(:module_name, :variables, :message, keyword_init: true)
+
       attr_reader :path
 
       def initialize(path)
@@ -44,40 +99,62 @@ module GraphWeaver
       def exist? = File.exist?(@path)
       def size = @entries.size
 
-      # Replay when recorded, record when not (VCR's once mode).
-      # client: is required to record; omit it to replay-or-raise.
-      # With Testing.config.record on (or GRAPHWEAVER_RECORD=1), always
-      # records — the "just re-record everything" switch.
-      def self.use(path, client: nil)
-        cassette = new(path)
-        if Testing.config.record && client
-          Recorder.new(client, cassette)
-        elsif cassette.exist?
-          Replayer.new(cassette)
-        elsif client
-          Recorder.new(client, cassette)
-        else
-          raise MissingRecording.new(path: cassette.path, query: "(no client to record with)")
-        end
+      def lookup(query, variables, operation_name = nil)
+        wanted = self.class.key(query, variables, operation_name)
+        @entries.find { |entry| self.class.entry_key(entry) == wanted }
       end
 
-      def lookup(query, variables)
-        wanted = self.class.key(query, variables)
-      @entries.find { |entry| entry["key"] == wanted }
+      # every variables hash recorded for this query — what a miss needs
+      # to show, since the variables are what usually differ
+      def variants(query, operation_name = nil)
+        normalized = self.class.normalize_query(query)
+        @entries.select do |entry|
+          self.class.normalize_query(entry["query"]) == normalized && entry["operationName"] == operation_name
+        end.map { |entry| entry["variables"] || {} }
       end
 
-      def record(query, variables, response)
-        entry = {
-          "key" => self.class.key(query, variables),
-          "query" => query,
-          "variables" => self.class.normalize_variables(variables),
-          "response" => response,
-        }
-        @entries.reject! { |existing| existing["key"] == entry["key"] }
+      def record(query, variables, response, operation_name = nil)
+        entry = { "query" => query }
+        entry["operationName"] = operation_name if operation_name
+        entry["variables"] = self.class.normalize_variables(variables)
+        entry["response"] = response
+
+        wanted = self.class.key(query, variables, operation_name)
+        @entries.reject! { |existing| self.class.entry_key(existing) == wanted }
         @entries << entry
         save
       end
 
+      # Replay every recording through `modules` — the generated query
+      # modules — and report the ones that no longer cast.
+      #
+      # A cassette is the one artifact here recorded from a *foreign* server,
+      # and nothing else notices when that server's answers drift out of the
+      # shape the structs were generated for: `verify`, `queries:check` and
+      # `schema:diff` all ask about the local side. Without this the drift
+      # surfaces mid-spec as a `TypeError` naming a struct and a sorbet
+      # frame, with nothing pointing at the stale file.
+      #
+      # Matching is on the query text, which is the module that sent it — a
+      # recording no module sends is skipped rather than guessed at.
+      def check(modules)
+        index = modules.to_h { |mod| [self.class.normalize_query(mod.const_get(:QUERY)), mod] }
+        checked = 0
+        stale = @entries.filter_map do |entry|
+          mod = index[self.class.normalize_query(entry["query"])] or next
+          checked += 1
+
+          begin
+            mod.from_response(entry["response"])
+            nil
+          rescue GraphWeaver::Error => e
+            Stale.new(module_name: mod.name, variables: entry["variables"] || {}, message: e.message)
+          end
+        end
+
+        Check.new(path: @path, checked:, skipped: @entries.size - checked, stale:)
+      end
+
       # Replace recorded response values with fakes, preserving structure.
       # Walks each entry's query against the schema (like FakeClient,
       # but transforming what's there instead of generating from scratch).
@@ -91,8 +168,27 @@ module GraphWeaver
         self
       end
 
-      def self.key(query, variables)
-        { "query" => query.gsub(/\s+/, " ").strip, "variables" => normalize_variables(variables) }
+      # The request's identity, exactly as the server sees it. operationName
+      # is part of that: it picks the operation the document runs, so two
+      # requests with identical text but different names are different
+      # requests. Derived, never stored — the file holds the request once,
+      # so a hand-edited entry can't disagree with what replay matches on.
+      def self.key(query, variables, operation_name = nil)
+        key = { "query" => normalize_query(query), "variables" => normalize_variables(variables) }
+        key["operationName"] = operation_name if operation_name
+        key
+      end
+
+      def self.entry_key(entry)
+        key(entry["query"], entry["variables"], entry["operationName"])
+      end
+
+      def self.normalize_query(query) = query.gsub(/\s+/, " ").strip
+
+      # one readable line: an error naming a 60-line query is a wall, not a hint
+      def self.summarize(query, limit: 160)
+        normalized = normalize_query(query)
+        (normalized.length > limit) ? "#{normalized[0, limit]}…" : normalized
       end
 
       # JSON round-trip so symbol keys become strings — otherwise YAML.dump
@@ -111,16 +207,16 @@ module GraphWeaver
     end
 
     # Tees requests through a live client and records every response.
-    # With Testing.config.anonymize (or anonymize: true), responses are
-    # anonymized as they're recorded — and the anonymized version is what
-    # the caller sees too, so assertions written now hold on replay.
+    # With Testing.config.anonymize, responses are anonymized as they're
+    # recorded — and the anonymized version is what the caller sees too,
+    # so assertions written now hold on replay.
     class Recorder
-      def initialize(client, cassette, anonymize: nil)
+      def initialize(client, cassette)
         @client = client
         @cassette = cassette.is_a?(Cassette) ? cassette : Cassette.new(cassette)
 
         config = Testing.config
-        if anonymize.nil? ? config.anonymize : anonymize
+        if config.anonymize
           unless config.schema
             raise ArgumentError, "anonymizing recordings needs GraphWeaver::Testing.config.schema"
           end
@@ -129,13 +225,13 @@ module GraphWeaver
         end
       end
 
-      def execute(query, variables: {})
-        response = @client.execute(query, variables:).to_h
+      def execute(query, variables: {}, operation_name: nil)
+        response = @client.execute(query, variables:, operation_name:).to_h
         if @anonymizer && (data = response["data"])
           response = response.merge("data" => @anonymizer.anonymize(query, data))
         end
 
-        @cassette.record(query, variables, response)
+        @cassette.record(query, variables, response, operation_name)
         response
       end
     end
@@ -147,9 +243,12 @@ module GraphWeaver
         @cassette = cassette.is_a?(Cassette) ? cassette : Cassette.new(cassette)
       end
 
-      def execute(query, variables: {})
-        entry = @cassette.lookup(query, variables)
-        raise MissingRecording.new(path: @cassette.path, query:) unless entry
+      def execute(query, variables: {}, operation_name: nil)
+        entry = @cassette.lookup(query, variables, operation_name)
+        unless entry
+          raise MissingRecording.new(path: @cassette.path, query:, variables:,
+            recorded: @cassette.variants(query, operation_name), size: @cassette.size)
+        end
 
         entry["response"]
       end

data/lib/graph_weaver/testing/coverage.rb

@@ -0,0 +1,165 @@
+# typed: true
+# frozen_string_literal: true
+
+require "graphql"
+
+require_relative "router"
+
+module GraphWeaver
+  module Testing
+    # How much of a query set the local {Router} can plan, and — for the rest
+    # — exactly what stopped it:
+    #
+    #      rake graph_weaver:federation:coverage SUPERGRAPH=supergraph.graphql
+    #
+    # The router refuses every query that crosses a subgraph boundary, so its
+    # worth to a suite is one number: the fraction of *your* queries it can
+    # answer. Nobody can guess that from outside — it depends on the shape of
+    # your graph and the shape of your queries — so measure it before
+    # deciding the router is (or isn't) enough.
+    #
+    # Planning needs the supergraph and nothing else, so this runs without
+    # any subgraph being loadable, in CI or on a laptop with the SDL alone.
+    class Coverage
+      # One query file's verdict: `category` nil means the router can plan
+      # it, and `subgraph` is where it runs — "accounts+reviews" when the
+      # plan stitches across a boundary. `absent` names the subgraphs that
+      # plan reaches which nothing in this process serves — plannable and
+      # runnable-here are different questions.
+      Result = Struct.new(:path, :subgraph, :absent, :category, :detail) do
+        def servable? = category.nil? && absent.empty?
+      end
+
+      # the label each verdict groups under
+      LABELS = Unplannable::CATEGORIES
+        .transform_values(&:first)
+        .merge(invalid: "doesn't validate against the supergraph")
+        .freeze
+
+      attr_reader :results
+
+      def initialize(supergraph:, queries: GraphWeaver.queries_paths, fragments: GraphWeaver.fragments_paths)
+        source = supergraph.to_s
+        table = GraphWeaver::SchemaLoader.routing_table(source)
+        unless table.unsupported.empty?
+          # the same refusal the Router makes at construction: with the table
+          # incomplete, every number this would report is a guess
+          raise Unplannable.new(
+            "this supergraph uses federation constructs the local router doesn't read: " +
+              table.unsupported.join("; "),
+            category: :unsupported_federation,
+          )
+        end
+
+        # Planning still runs with `absent` empty — a query is plannable or
+        # not whoever is serving. Which subgraphs are *here* is asked
+        # separately, from evidence (a loaded schema defining what the table
+        # says one resolves), and never refuses: with nothing loaded the
+        # answer is simply "none", which is the SDL-alone CI run.
+        @planner = Router::Planner.new(table:, schema: GraphWeaver::SchemaLoader.load(source))
+        @absent = table.subgraphs.reject { |name| Subgraphs.candidates(table, name).any? }
+        @local = @absent.size < table.subgraphs.size
+        @shared = GraphWeaver::Codegen.load_fragments(fragments)
+        @results = GraphWeaver.query_files(queries).map { |path| measure(path) }
+      end
+
+      def plannable = @results.count { |result| result.category.nil? }
+
+      # plannable *and* every subgraph the plan reaches is served here
+      def servable = @results.count(&:servable?)
+
+      # plannable, but reaching a subgraph another process serves
+      def elsewhere = @results.select { |result| result.category.nil? && result.absent.any? }
+
+      def refused = @results.reject { |result| result.category.nil? }
+
+      # whole percent: this is a count of a handful of files, and a decimal
+      # place would claim precision the sample doesn't have
+      def percent = @results.empty? ? 0 : (plannable * 100.0 / @results.size).round
+
+      def report
+        return "no queries found" if @results.empty?
+
+        [headline, *breakdown, *served_here, *refusals].join("\n")
+      end
+      alias to_s report
+
+      def inspect = "#<#{self.class.name} #{plannable}/#{@results.size} plannable>"
+
+      private
+
+      def headline
+        counted = "#{plannable}/#{@results.size} #{(@results.size == 1) ? "query" : "queries"} " \
+          "plannable locally (#{percent}%)"
+        @local ? "#{counted}, #{servable} servable here" : counted
+      end
+
+      # Plan-only is by design — a query is plannable whoever serves it — but
+      # the question this report exists to answer is whether wiring the router
+      # up is worth it, and a suite can only *run* what this process serves.
+      # A partly-local supergraph is the usual migration shape, so the second
+      # number has to be here rather than inferred from a refusal later.
+      def served_here
+        unless @local
+          return ["", "nothing here serves any of this supergraph's subgraphs " \
+            "(#{@absent.join(", ")}), so this counts planning only"]
+        end
+        return [] if elsewhere.empty?
+
+        example = elsewhere.first.absent.first
+        ["", "plannable, but nothing here serves what they reach (#{elsewhere.size}) — name a " \
+          "schema for those subgraphs, fake them (subgraphs: { #{example.inspect} => :fake }), " \
+          "or run these against a real router:"] +
+          elsewhere.map { |result| "  #{name(result)}  #{result.absent.join(", ")}" }
+      end
+
+      # where the plannable ones land — a graph whose queries all sit in one
+      # subgraph is a different situation from one that's evenly spread
+      def breakdown
+        by_subgraph = @results.filter_map(&:subgraph).tally.sort_by { |name, count| [-count, name] }
+        return [] if by_subgraph.empty?
+
+        ["  " + by_subgraph.map { |name, count| "#{name} #{count}" }.join(", ")]
+      end
+
+      def refusals
+        return [] if refused.empty?
+
+        groups = refused.group_by(&:category).sort_by { |category, group| [-group.size, category.to_s] }
+
+        ["", "refused (#{refused.size})"] + groups.flat_map do |category, group|
+          ["", "  #{LABELS.fetch(category, category)} (#{group.size})"] +
+            group.map { |result| "    #{name(result)}  #{result.detail}" }
+        end
+      end
+
+      # the file column: filenames when every query came from one directory,
+      # padded so what sits beside them lines up
+      def name(result) = basename(result).ljust(width)
+
+      def basename(result) = result.path.delete_prefix(shared_dir)
+
+      def width = @width ||= @results.map { |result| basename(result).length }.max
+
+      # the directory every query came from, so the report's file column is
+      # filenames rather than the same path repeated
+      def shared_dir
+        dirs = @results.map { |result| File.dirname(result.path) }.uniq
+        dirs.one? ? "#{dirs.first}/" : ""
+      end
+
+      def measure(path)
+        document = GraphQL.parse(GraphWeaver::Codegen.inline_fragments(File.read(path), @shared, path))
+        errors = @planner.validate(document)
+        return Result.new(path, nil, [], :invalid, errors.first["message"]) if errors.any?
+
+        plan = @planner.plan(document)
+        Result.new(path, plan.where, plan.subgraphs & @absent, nil, nil)
+      rescue Unplannable => e
+        Result.new(path, nil, [], e.category, e.detail)
+      rescue GraphQL::ParseError, GraphWeaver::Error => e
+        Result.new(path, nil, [], :invalid, e.message)
+      end
+    end
+  end
+end

data/lib/graph_weaver/testing/failure.rb

@@ -9,10 +9,10 @@ module GraphWeaver
     # transports produce, so error-handling paths are testable without a
     # server that misbehaves on cue:
     #
-    #      PersonQuery.execute(id: "1", Failure.transport)   # TransportError
-    #      PersonQuery.execute(id: "1", Failure.server(status: 502))
-    #      PersonQuery.execute(id: "1", Failure.throttled)   # QueryError, code THROTTLED
-    #      PersonQuery.execute(id: "1", Failure.stale_schema) # schema_stale? => true
+    #      PersonQuery.execute(client: Failure.transport, id: "1")   # TransportError
+    #      PersonQuery.execute(client: Failure.server(status: 502), id: "1")
+    #      PersonQuery.execute(client: Failure.throttled, id: "1")    # QueryError, code THROTTLED
+    #      PersonQuery.execute(client: Failure.stale_schema, id: "1") # schema_stale? => true
     #
     # For type mismatches, corrupt the wire with a FakeClient override:
     #      FakeClient.new(schema:, overrides: { "Person.birthday" => 123 })
@@ -56,24 +56,11 @@ module GraphWeaver
       end
 
       # A validation-shaped rejection — trips schema_stale? and its
-      # regenerate hint, as if the schema changed under the module. Name
-      # the casualty explicitly, or pass schema: to sample a real
-      # type/field (as if the server just dropped it):
+      # regenerate hint, as if the schema changed under the module. Name the
+      # casualty when the message matters:
       #
       #      Failure.stale_schema(type: "Person", field: "name")
-      #      Failure.stale_schema(schema: MySchema)             # random real field
-      #      Failure.stale_schema(schema: MySchema, seed: 42)   # reproducibly random
-      def stale_schema(field: nil, type: nil, schema: nil, seed: nil)
-        if schema && (field.nil? || type.nil?)
-          rng = Random.new(seed || GraphWeaver::Testing.config.seed || Random.new_seed)
-          candidates = schema.types.values.select do |candidate|
-            candidate.kind.name == "OBJECT" && !candidate.graphql_name.start_with?("__")
-          end
-          chosen = candidates.sort_by(&:graphql_name).sample(random: rng)
-          type ||= chosen.graphql_name
-          field ||= chosen.fields.keys.sort.sample(random: rng)
-        end
-
+      def stale_schema(field: nil, type: nil)
         graphql("Field '#{field || "someField"}' doesn't exist on type '#{type || "SomeType"}'")
       end
     end
@@ -84,7 +71,7 @@ module GraphWeaver
         @response = response
       end
 
-      def execute(_query, variables: {})
+      def execute(_query, variables: {}, operation_name: nil)
         @response.call
       end
     end
@@ -99,10 +86,10 @@ module GraphWeaver
         @calls = 0
       end
 
-      def execute(query, variables: {})
+      def execute(query, variables: {}, operation_name: nil)
         client = @clients[[@calls, @clients.size - 1].min]
         @calls += 1
-        client.execute(query, variables:)
+        client.execute(query, variables:, operation_name:)
       end
     end
   end