@canonical/pragma-cli 0.36.0 → 0.38.0

package/src/kernel/packs/cursor.ts

@@ -0,0 +1,135 @@
+/**
+ * The cursor a caller spends to ask for the page after this one.
+ *
+ * WHY AN OFFSET. The candidate alternative is a keyset cursor encoding the last
+ * row's ordering key, which does not drift when the store changes underneath a
+ * caller. It cannot be general, and `block list` is the counterexample: it
+ * orders by `?tierRank`, a variable its SELECT does not project, so the
+ * ordering key is not in the rows at all — and a pagination mechanism that
+ * works only for stories whose ordering keys happen to be columns is not a
+ * pagination mechanism. An offset needs nothing from the rows.
+ *
+ * The drift an offset admits is bounded by what the store is: a read-only,
+ * content-addressed pack that changes only when `sources update` rebuilds it.
+ * Between two pages of one browse there is no writer.
+ *
+ * What an offset cannot survive is the caller changing the QUESTION between
+ * pages — different filters, a different search term, another story — which is
+ * why the cursor carries a fingerprint of the read it was issued for and
+ * refuses to be spent on another. That is the failure worth catching, and the
+ * one a bare integer would have answered wrongly and in silence.
+ *
+ * The encoding is opaque on purpose: a caller who cannot read it cannot come to
+ * depend on it, so the day a story needs a keyset cursor the cursor's contents
+ * change and its contract does not.
+ *
+ * The fingerprint does not cover the OFFSET, and cannot: a cursor is a token
+ * this build issued, not a signed one, so its `o` is editable by anyone who
+ * base64-decodes it. That is why the offset is BOUNDED here as well as parsed
+ * here — an `o` of 2^53 is a legal JSON integer that the store answers with a
+ * raw parse error, so {@link ./paging.MAX_LIST_WINDOW} is checked before the
+ * number can reach a query.
+ */
+
+import { createHash } from "node:crypto";
+import { PragmaError } from "../error/index.js";
+import { MAX_LIST_WINDOW } from "./paging.js";
+
+/** The cursor encoding's own version, so a future keyset form is tellable apart. */
+const CURSOR_VERSION = 1;
+
+/**
+ * The identity of the read a cursor belongs to — the author query and the
+ * arguments that narrow it, hashed short.
+ *
+ * Each part is LENGTH-PREFIXED rather than merely joined. A separator is only
+ * unambiguous if it cannot occur inside a part, and these parts are SPARQL
+ * text and JSON — there is no such character. Two different argument lists
+ * hashing the same would let a cursor be spent on a read it was not issued
+ * for, which is the single thing this function exists to prevent.
+ *
+ * @param parts - Everything that decides which rows the read returns.
+ * @returns A short, stable fingerprint.
+ */
+export function pageFingerprint(parts: readonly string[]): string {
+  const hash = createHash("sha256");
+  for (const part of parts) hash.update(`${part.length}:${part}`);
+  return hash.digest("base64url").slice(0, 12);
+}
+
+/**
+ * Encode the cursor that asks for the rows after this page.
+ *
+ * @param offset - The row index the next page starts at.
+ * @param fingerprint - The read this cursor may be spent on.
+ * @returns An opaque, URL-safe token.
+ */
+export function encodeCursor(offset: number, fingerprint: string): string {
+  const payload = JSON.stringify({
+    v: CURSOR_VERSION,
+    o: offset,
+    q: fingerprint,
+  });
+  return Buffer.from(payload, "utf8").toString("base64url");
+}
+
+/**
+ * Read an `--after`/`after` cursor back into an offset.
+ *
+ * @param provided - The raw argument, if any.
+ * @param fingerprint - The read the cursor must have been issued for.
+ * @returns The row index to start at (0 when no cursor was given).
+ * @throws PragmaError INVALID_INPUT when the token is not a cursor this build
+ *   issued, or was issued for a different read — a mismatch answered with rows
+ *   would be a page of the wrong question, with nothing to show it.
+ */
+export function readCursor(provided: unknown, fingerprint: string): number {
+  if (provided === undefined) return 0;
+  const decoded = decodeCursor(provided);
+  if (!decoded) {
+    throw PragmaError.invalidInput("after", String(provided), {
+      recovery: {
+        message:
+          "--after takes the cursor from a previous page of this same read; run the read without it to start from the first page.",
+      },
+    });
+  }
+  if (decoded.q !== fingerprint) {
+    throw PragmaError.invalidInput("after", String(provided), {
+      recovery: {
+        message:
+          "This cursor was issued for a different read — the filters or the search term changed. Run the new read without --after to start from its first page.",
+      },
+    });
+  }
+  return decoded.o;
+}
+
+/**
+ * The cursor payload, or `undefined` for anything that is not one.
+ *
+ * The offset is checked against {@link MAX_LIST_WINDOW} here rather than at the
+ * store: an unbounded `o` is emitted straight into the generated query's
+ * `OFFSET`, where a number the engine cannot express comes back as a parse
+ * error dressed as an internal one. Refused here, a hand-edited offset is the
+ * same calm INVALID_INPUT as every other malformed cursor.
+ */
+function decodeCursor(
+  provided: unknown,
+): { readonly o: number; readonly q: string } | undefined {
+  if (typeof provided !== "string" || provided === "") return undefined;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(Buffer.from(provided, "base64url").toString("utf8"));
+  } catch {
+    return undefined;
+  }
+  if (typeof parsed !== "object" || parsed === null) return undefined;
+  const payload = parsed as Record<string, unknown>;
+  if (payload.v !== CURSOR_VERSION) return undefined;
+  if (!Number.isInteger(payload.o)) return undefined;
+  const offset = payload.o as number;
+  if (offset < 0 || offset > MAX_LIST_WINDOW) return undefined;
+  if (typeof payload.q !== "string") return undefined;
+  return { o: offset, q: payload.q };
+}

package/src/kernel/packs/filteredEmpty.test.ts

@@ -0,0 +1,213 @@
+/**
+ * A zero-row answer names the FILTER and still explains the POPULATION.
+ *
+ * Two facts can empty a list and the rows cannot tell them apart: nothing was
+ * there, or a filter missed what was. So the page says both — the kernel's
+ * sentence names the filters in force (`token list --search zzzznotreal` used
+ * to answer "No token symbols in the store … run `pragma sources update`",
+ * telling a reader with 745 present symbols to rebuild for nothing), and the
+ * story's own `emptyRecovery` follows it, exactly as it does when nothing
+ * narrowed the read.
+ *
+ * Which puts one requirement on the stories rather than on the kernel: a
+ * recovery is worded to hold in BOTH cases, or the story declares none. The
+ * fixture below is worded that way on purpose.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { buildFixtureRuntime } from "../../testing/helpers/packRuntime.js";
+import { executeVerb } from "../project/cli/dispatch.js";
+import type { GlobalFlags, PragmaRuntime } from "../runtime/types.js";
+import type { VerbSpec } from "../spec/types.js";
+import { compilePack } from "./compile.js";
+import type { PackDefinition } from "./types.js";
+import { distributionSource } from "./types.js";
+import { verbKey } from "./uniqueness.js";
+
+const PREFIXES = {
+  ex: "https://example.org/widgets#",
+  owl: "http://www.w3.org/2002/07/owl#",
+  rdfs: "http://www.w3.org/2000/01/rdf-schema#",
+  xsd: "http://www.w3.org/2001/XMLSchema#",
+};
+
+const TTL = `
+@prefix ex: <https://example.org/widgets#> .
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+ex:Widget a owl:Class .
+ex:Gadget a owl:Class .
+ex:name a owl:DatatypeProperty ; rdfs:domain ex:Widget ; rdfs:range xsd:string .
+ex:kind a owl:DatatypeProperty ; rdfs:domain ex:Widget ; rdfs:range xsd:string .
+
+ex:button a ex:Widget ; ex:name "Button" ; ex:kind "input" .
+ex:label  a ex:Widget ; ex:name "Label"  ; ex:kind "display" .
+ex:slider a ex:Widget ; ex:name "Slider" ; ex:kind "input" .
+`;
+
+/**
+ * The recovery a story authors — where the rows come from, said so it holds
+ * over a populated table a filter missed as well as over an empty one.
+ *
+ * "No widgets in the store." would not: it is a claim about the population, and
+ * the kernel prints this under a filtered miss too, where three widgets are
+ * sitting in the store.
+ */
+const EMPTY_RECOVERY = {
+  message:
+    "Widgets are recorded by the catalogue build, and a store built before it ran records none.",
+  cli: "sources update",
+} as const;
+
+const listShape = (uriClass: string) => ({
+  query: [
+    "SELECT ?uri ?name ?kind WHERE {",
+    `  ?uri a ${uriClass} ; ex:name ?name .`,
+    "  OPTIONAL { ?uri ex:kind ?kind }",
+    "} ORDER BY ?name",
+  ].join("\n"),
+  columns: [
+    { field: "uri", label: "IRI" },
+    { field: "name", label: "Name" },
+    { field: "kind", label: "Kind" },
+  ],
+  filters: [{ param: "kind", variable: "kind", values: ["input", "display"] }],
+  search: { variables: ["name"] },
+  emptyRecovery: EMPTY_RECOVERY,
+});
+
+/** Three rows — a POPULATED table, so a filter that misses really is the news. */
+const WIDGET_PACK: PackDefinition = {
+  noun: "widget",
+  description: "List widgets.",
+  list: listShape("ex:Widget"),
+};
+
+/** A declared class with no individuals — the genuinely empty population. */
+const GADGET_PACK: PackDefinition = {
+  noun: "gadget",
+  description: "List gadgets.",
+  list: { ...listShape("ex:Gadget"), emptyRecovery: EMPTY_RECOVERY },
+};
+
+const listVerb = (pack: PackDefinition): VerbSpec =>
+  compilePack(pack, distributionSource("test:list"), PREFIXES).find(
+    (v) => verbKey(v.path) === `${pack.noun} list`,
+  ) as VerbSpec;
+
+const REAL = { dryRun: false, undo: false, yes: false };
+
+const PLAIN: GlobalFlags = {
+  llm: false,
+  autoLlm: false,
+  format: "plain",
+  verbose: false,
+};
+const LLM: GlobalFlags = { ...PLAIN, llm: true, format: "llm" };
+const JSON_FLAGS: GlobalFlags = { ...PLAIN, format: "json" };
+
+let rt: PragmaRuntime;
+/** The same runtime, reading in another format. */
+const as = (flags: GlobalFlags): PragmaRuntime => ({
+  ...rt,
+  globalFlags: flags,
+});
+
+beforeAll(async () => {
+  ({ rt } = await buildFixtureRuntime({ ttl: TTL, prefixes: PREFIXES }));
+});
+afterAll(async () => {
+  (await rt.store.get()).store.dispose();
+});
+
+describe("a zero-row answer names the filter and keeps the recovery", () => {
+  it("a search that matched nothing names the search, and still explains", async () => {
+    const outcome = await executeVerb(
+      listVerb(WIDGET_PACK),
+      { search: "zzzznotreal" },
+      REAL,
+      as(PLAIN),
+    );
+    expect(outcome.exitCode).toBe(0);
+    const said = `${outcome.stdout ?? ""}${outcome.stderr ?? ""}`;
+    expect(said).toContain("No widget matches `--search zzzznotreal`.");
+    // The filter is the news; the story's account of where widgets come from is
+    // the other half, and a reader who mistyped nothing needs it.
+    expect(said).toContain("recorded by the catalogue build");
+    expect(said).toContain("pragma sources update");
+  });
+
+  it("every filter in force is named", async () => {
+    const outcome = await executeVerb(
+      listVerb(WIDGET_PACK),
+      { kind: "display", search: "zzzznotreal" },
+      REAL,
+      as(PLAIN),
+    );
+    const said = `${outcome.stdout ?? ""}${outcome.stderr ?? ""}`;
+    expect(said).toContain(
+      "No widget matches `--kind display` and `--search zzzznotreal`.",
+    );
+  });
+
+  it("the llm body says the same thing", async () => {
+    const outcome = await executeVerb(
+      listVerb(WIDGET_PACK),
+      { search: "zzzznotreal" },
+      REAL,
+      as(LLM),
+    );
+    expect(outcome.stdout).toContain(
+      "No widget matches `--search zzzznotreal`.",
+    );
+    expect(outcome.stdout).toContain("recorded by the catalogue build");
+  });
+
+  it("json says the same thing on meta.notice", async () => {
+    const outcome = await executeVerb(
+      listVerb(WIDGET_PACK),
+      { search: "zzzznotreal" },
+      REAL,
+      as(JSON_FLAGS),
+    );
+    const envelope = JSON.parse(outcome.stdout as string);
+    expect(envelope.data).toEqual([]);
+    expect(envelope.meta.notice).toContain(
+      "No widget matches `--search zzzznotreal`.",
+    );
+    expect(envelope.meta.notice).toContain("recorded by the catalogue build");
+  });
+
+  it("an EMPTY POPULATION under a filter keeps the story's own recovery", async () => {
+    // The regression: `token consumers --symbol color.text` narrows a table
+    // that records nothing at all, and answering only "no match for that
+    // symbol" read as a typo while withholding the one account the verb has.
+    // No probe tells the two apart — the sentence does, by saying both.
+    const outcome = await executeVerb(
+      listVerb(GADGET_PACK),
+      { kind: "input" },
+      REAL,
+      as(PLAIN),
+    );
+    expect(outcome.exitCode).toBe(0);
+    const said = `${outcome.stderr ?? ""}`;
+    expect(said).toContain("No gadget matches `--kind input`.");
+    expect(said).toContain("recorded by the catalogue build");
+    expect(said).toContain("pragma sources update");
+  });
+
+  it("an EMPTY POPULATION unfiltered gets the kernel's own sentence", async () => {
+    const outcome = await executeVerb(
+      listVerb(GADGET_PACK),
+      {},
+      REAL,
+      as(PLAIN),
+    );
+    expect(outcome.exitCode).toBe(0);
+    expect(outcome.stderr).toContain("No gadget entries found.");
+    expect(outcome.stderr).toContain("recorded by the catalogue build");
+    expect(outcome.stderr).toContain("pragma sources update");
+  });
+});

package/src/kernel/packs/graphql/buildLookupDocument.ts

@@ -31,7 +31,7 @@ import type {
   PackLookup,
   PackNestedExpand,
 } from "../types.js";
-import { isNestedExpand } from "../types.js";
+import { expandIsSparql, isNestedExpand } from "../types.js";
 import { MAX_PAGE_SIZE, pluralize, stripVerbPrefix } from "./nameMap.js";
 
 type GraphQLSchema = import("graphql").GraphQLSchema;
@@ -367,16 +367,23 @@ export function buildLookupDocument(
         ),
       ),
     ),
-    ...activeExpands(lookup, level).map((expand, index) =>
-      buildScopedSelection(schema, fragmentType, (container) =>
-        buildCollectionSelection(
-          container,
-          expand,
-          `lookup.expand[${index}]`,
-          source,
+    // An expand that declares the SPARQL lane is not this document's — the
+    // entity fetcher runs its sub-SELECT beside this query and merges the rows
+    // in. Skipped rather than attempted: its terms are chosen for a lane where
+    // property paths read and every defined property is reachable, and asking
+    // for them here would either drop them in silence or raise.
+    ...activeExpands(lookup, level)
+      .filter((expand) => !expandIsSparql(lookup, expand))
+      .map((expand, index) =>
+        buildScopedSelection(schema, fragmentType, (container) =>
+          buildCollectionSelection(
+            container,
+            expand,
+            `lookup.expand[${index}]`,
+            source,
+          ),
         ),
       ),
-    ),
   ].filter((selection): selection is Selection => selection !== undefined);
 
   const body = selections.map((selection) => `      ${selection.text}`);

package/src/kernel/packs/graphqlSource.test.ts

@@ -130,6 +130,158 @@ describe("source rule — zod refinements reject dual/ill-sourced lookups (PROTE
     ).toThrow(/single-hop|graphql/i);
   });
 
+  it("accepts a SPARQL-lane expand inside a graphql lookup", () => {
+    // The two lanes are not interchangeable and the block lookup needs both:
+    // a nested expand only a document can make, beside a table whose columns
+    // the compiled schema has no field for. So the lane is per-expand, and the
+    // rules that follow are about the lane that will actually resolve it.
+    const definition = {
+      noun: "x",
+      lookup: {
+        source: "graphql",
+        by: "ds:name",
+        type: "ds:Component",
+        graphqlType: "Component",
+        expand: [
+          {
+            name: "t",
+            relation: "ds:hasTokenBinding",
+            source: "sparql",
+            orderBy: ["node"],
+            note: "How to read the rows.",
+            select: [
+              { name: "symbol", property: "ds:consumesSymbol/rdfs:label" },
+              {
+                name: "via",
+                property: "ds:viaBlock/ds:name",
+                blankWhenSelf: true,
+              },
+              { name: "node", property: "ds:node" },
+            ],
+          },
+        ],
+      },
+    };
+    expect(
+      parsePackDefinition(JSON.parse(JSON.stringify(definition)), "t"),
+    ).toEqual(definition);
+  });
+
+  it("rejects a nested expand on a SPARQL-lane expand of a graphql lookup", () => {
+    // The single-hop rule follows the EXPAND's lane, not the lookup's — it
+    // used to read the lookup's, so an expand that opted out of the document
+    // could still declare a second hop no sub-SELECT can make.
+    expect(() =>
+      parsePackDefinition(
+        {
+          noun: "x",
+          lookup: {
+            source: "graphql",
+            by: "ds:name",
+            type: "ds:Component",
+            graphqlType: "Component",
+            expand: [
+              {
+                name: "f",
+                relation: "ds:hasModifierFamily",
+                source: "sparql",
+                select: [
+                  {
+                    name: "v",
+                    relation: "ds:hasModifier",
+                    select: [{ name: "n", property: "ds:name" }],
+                  },
+                ],
+              },
+            ],
+          },
+        },
+        "t",
+      ),
+    ).toThrow(/single-hop|graphql/i);
+  });
+
+  it("rejects orderBy and blankWhenSelf on a lane that cannot express them", () => {
+    // A GraphQL collection's order is the schema's and a document cannot
+    // filter a cell against the entity's own name, so both are refused where
+    // they are written rather than ignored at runtime — a silent no-op is how
+    // a declaration that never applied survives review.
+    const base = {
+      noun: "x",
+      lookup: {
+        source: "graphql",
+        by: "ds:name",
+        type: "ds:Component",
+        graphqlType: "Component",
+      },
+    };
+    expect(() =>
+      parsePackDefinition(
+        {
+          ...base,
+          lookup: {
+            ...base.lookup,
+            expand: [
+              {
+                name: "t",
+                relation: "ds:hasTokenBinding",
+                orderBy: ["node"],
+                select: [{ name: "node", property: "ds:node" }],
+              },
+            ],
+          },
+        },
+        "t",
+      ),
+    ).toThrow(/orderBy/);
+    expect(() =>
+      parsePackDefinition(
+        {
+          ...base,
+          lookup: {
+            ...base.lookup,
+            expand: [
+              {
+                name: "t",
+                relation: "ds:hasTokenBinding",
+                select: [
+                  { name: "via", property: "ds:viaBlock", blankWhenSelf: true },
+                ],
+              },
+            ],
+          },
+        },
+        "t",
+      ),
+    ).toThrow(/blankWhenSelf/);
+  });
+
+  it("rejects an orderBy naming a field the select does not declare", () => {
+    // It would order by an unbound variable, which SPARQL permits and which
+    // orders by nothing at all — the rows would come back in the store's order
+    // with a declaration claiming otherwise.
+    expect(() =>
+      parsePackDefinition(
+        {
+          noun: "x",
+          lookup: {
+            by: "ds:name",
+            type: "ds:Component",
+            expand: [
+              {
+                name: "t",
+                relation: "ds:hasTokenBinding",
+                orderBy: ["rank"],
+                select: [{ name: "node", property: "ds:node" }],
+              },
+            ],
+          },
+        },
+        "t",
+      ),
+    ).toThrow(/orders by "rank"/);
+  });
+
   it("rejects a graphql field addressed by a property path", () => {
     expect(() =>
       parsePackDefinition(