@canonical/pragma-cli 0.36.0 → 0.38.0

package/src/kernel/packs/sparql/authorQuery.ts

@@ -0,0 +1,257 @@
+/**
+ * Read a list story's author query as the three parts a page has to keep apart:
+ * the PROLOGUE it declares, the VARIABLES it projects, and the BODY a wrapping
+ * select can hold.
+ *
+ * Every list answer is one page now, and a page is a wrapping select over the
+ * story's own query ({@link ./buildListQuery.buildListQuery}). Three properties
+ * of the author's text decide whether that wrap is legal, and none of them can
+ * be asked of the store:
+ *
+ * - **The prologue cannot be wrapped.** `PREFIX` and `BASE` belong to a QUERY;
+ *   a group graph pattern admits only patterns, so splicing author text that
+ *   opens with `PREFIX` inside one is a parse error (`expected OPTIONAL`). The
+ *   declarations are lifted to the wrapper's own prologue instead, in order, and
+ *   the body from the `SELECT` keyword on is what gets wrapped.
+ * - **The projection has to be reproduced.** `SELECT *` in the wrapper hands
+ *   back alphabetised binding keys, so `block list`'s rows would go from
+ *   `{uri,name,type,tier}` to `{name,tier,type,uri}` and every `--format json`
+ *   answer would change shape for no reason a caller asked for. The store cannot
+ *   answer this either: `SelectResult.variables` is derived from the rows that
+ *   came back, not from the query's header — over the shipped pack `block list`
+ *   reports four of its five, because the fifth is unbound on the first row. So
+ *   the projection is read from the query TEXT, which is the only place that
+ *   states it. An author who wrote `SELECT *` is wrapped with `SELECT *`, which
+ *   preserves their key order because it is the same alphabetisation twice.
+ * - **A dataset clause cannot be wrapped either.** `FROM` sits between the
+ *   select clause and the where clause, and `SubSelect` has no place for it. It
+ *   is REFUSED rather than lifted, because lifting it would mean cutting a span
+ *   out of the middle of the author's text — the prologue is a prefix, which is
+ *   why lifting that one is a split and not an edit.
+ *
+ * This is a lexer for the two clauses that precede the where clause, not a
+ * SPARQL parser. It covers the whole grammar of both — `Prologue ::= (BaseDecl
+ * | PrefixDecl)*` and `SelectClause ::= 'SELECT' ('DISTINCT'|'REDUCED')?
+ * ((Var | '(' Expression 'AS' Var ')')+ | '*')` — and it skips comments and
+ * string literals everywhere it scans, so a `#` inside a prefix IRI, a `# select
+ * the blocks` heading, and a `WHERE {` inside a `REPLACE` literal are all read
+ * as what they are. What it cannot read it says it cannot read, naming the
+ * shape it found: a refusal that misdiagnoses sends the author to the wrong
+ * file.
+ */
+
+/** A SPARQL variable as this reader recognises it (`?name`, `$name`). */
+const VARIABLE = /^[?$]([A-Za-z_][A-Za-z0-9_]*)/;
+
+/** The `AS ?var` binding that names a projection expression's result. */
+const ALIAS = /\bAS\s+[?$]([A-Za-z_][A-Za-z0-9_]*)\s*$/i;
+
+/** Whitespace and `#` comments, which may sit between any two tokens. */
+const IGNORABLE = /^(?:\s+|#[^\n\r]*)+/;
+
+/** `PNAME_NS ::= PN_PREFIX? ':'` — the name a `PREFIX` declaration binds. */
+const PREFIX_NAME = /^[A-Za-z][\w.-]*:|^:/;
+
+/** An author query split where a wrapping select can take it over. */
+export interface AuthorQuery {
+  /**
+   * The author's own `PREFIX`/`BASE` declarations, verbatim and in order, or
+   * `""` when there are none. Emitted as the WRAPPER's prologue.
+   */
+  readonly prologue: string;
+  /**
+   * The projected variables (without `?`), in author order — or `undefined`
+   * for `SELECT *`, which names no variables to reproduce.
+   */
+  readonly projection: readonly string[] | undefined;
+  /** The query from its `SELECT` keyword on: what a sub-select can hold. */
+  readonly body: string;
+}
+
+/** A read of an author query: the parts, or what stopped the reader. */
+export type AuthorQueryRead =
+  | { readonly ok: true; readonly query: AuthorQuery }
+  | { readonly ok: false; readonly reason: string };
+
+/**
+ * Split an author query into its prologue, its projection and its body.
+ *
+ * @param text - The author's SPARQL SELECT text.
+ * @returns The three parts, or the reason the text cannot be wrapped — a
+ *   sentence fragment naming what the reader FOUND, for a caller to quote.
+ */
+export function readAuthorQuery(text: string): AuthorQueryRead {
+  const prologue = readPrologue(text);
+  const start = skipIgnorable(text, prologue.end);
+  if (!/^SELECT\b/i.test(text.slice(start))) {
+    return { ok: false, reason: "its text declares no SPARQL SELECT" };
+  }
+  const clause = readSelectClause(text, start + "SELECT".length);
+  if (!clause.ok) return clause;
+  return {
+    ok: true,
+    query: {
+      prologue: text.slice(prologue.start, prologue.end),
+      projection: clause.projection,
+      body: text.slice(start),
+    },
+  };
+}
+
+/** Where the prologue's declarations begin and end (equal when there are none). */
+function readPrologue(text: string): { start: number; end: number } {
+  const start = skipIgnorable(text, 0);
+  let end = start;
+  let at = start;
+  for (;;) {
+    const declaration = readDeclaration(text, at);
+    if (declaration === undefined) return { start, end };
+    end = declaration;
+    at = skipIgnorable(text, declaration);
+  }
+}
+
+/**
+ * The index just past one `PREFIX`/`BASE` declaration, or `undefined` when the
+ * text at {@link at} is not one.
+ *
+ * The IRI is scanned to its closing `>`, which is exact: `IRIREF` excludes `>`
+ * from its character class, so the first one closes it — and it does NOT
+ * exclude `#`, which is why a comment-aware scan has to read the IRI as an IRI
+ * rather than skip to end of line.
+ */
+function readDeclaration(text: string, at: number): number | undefined {
+  const keyword = text.slice(at).match(/^(BASE|PREFIX)\b/i);
+  if (!keyword?.[1]) return undefined;
+  let index = skipIgnorable(text, at + keyword[0].length);
+  if (keyword[1].toUpperCase() === "PREFIX") {
+    const name = text.slice(index).match(PREFIX_NAME);
+    if (!name) return undefined;
+    index = skipIgnorable(text, index + name[0].length);
+  }
+  if (text[index] !== "<") return undefined;
+  const close = text.indexOf(">", index + 1);
+  return close === -1 ? undefined : close + 1;
+}
+
+/** Read the select clause that starts just past the `SELECT` keyword. */
+function readSelectClause(
+  text: string,
+  from: number,
+):
+  | { readonly ok: true; readonly projection: readonly string[] | undefined }
+  | { readonly ok: false; readonly reason: string } {
+  const projected: string[] = [];
+  let star = false;
+  let at = from;
+  while (at < text.length) {
+    at = skipIgnorable(text, at);
+    const rest = text.slice(at);
+    // The clause ends at the WHERE keyword or at the group graph pattern's
+    // brace, whichever the author wrote (`WHERE` is optional in SPARQL).
+    if (rest === "" || /^(?:WHERE\b|\{)/i.test(rest)) break;
+    if (/^FROM\b/i.test(rest)) {
+      return {
+        ok: false,
+        reason:
+          "it carries a FROM dataset clause, and a sub-select cannot carry one",
+      };
+    }
+    // DISTINCT/REDUCED sit between the keyword and the projection.
+    const modifier = rest.match(/^(?:DISTINCT|REDUCED)\b/i);
+    if (modifier && projected.length === 0 && !star) {
+      at += modifier[0].length;
+      continue;
+    }
+    if (rest.startsWith("*")) {
+      star = true;
+      at += 1;
+      continue;
+    }
+    const variable = rest.match(VARIABLE);
+    if (variable?.[1]) {
+      projected.push(variable[1]);
+      at += variable[0].length;
+      continue;
+    }
+    if (rest.startsWith("(")) {
+      const end = matchingParen(text, at);
+      const alias = end === -1 ? null : text.slice(at + 1, end).match(ALIAS);
+      if (!alias?.[1]) {
+        return {
+          ok: false,
+          reason:
+            "its SELECT clause carries a parenthesised expression that names no result variable (`(… AS ?name)`)",
+        };
+      }
+      projected.push(alias[1]);
+      at = end + 1;
+      continue;
+    }
+    return {
+      ok: false,
+      reason: `its SELECT clause carries ${quoted(rest)}, which is neither a variable nor an \`(expression AS ?variable)\``,
+    };
+  }
+  if (star) {
+    return projected.length === 0
+      ? { ok: true, projection: undefined }
+      : {
+          ok: false,
+          reason: "its SELECT clause mixes `*` with named variables",
+        };
+  }
+  return projected.length > 0
+    ? { ok: true, projection: projected }
+    : { ok: false, reason: "its SELECT clause projects nothing" };
+}
+
+/** The first token of {@link rest}, quoted short enough to read in a message. */
+function quoted(rest: string): string {
+  const token = rest.match(/^\S{1,24}/)?.[0] ?? rest.slice(0, 24);
+  return `"${token}"`;
+}
+
+/** Whitespace and comments consumed: the next index that is neither. */
+function skipIgnorable(text: string, at: number): number {
+  const skip = text.slice(at).match(IGNORABLE);
+  return skip ? at + skip[0].length : at;
+}
+
+/**
+ * The index of the `)` closing the `(` at {@link open}, or -1 when unbalanced.
+ *
+ * Depth-counted rather than regex-matched: a projection expression nests
+ * (`(COALESCE(IF(…), 0) AS ?rank)`), and the first `)` is rarely the right one.
+ * Quoted strings are skipped so a `)` inside a literal cannot close the
+ * expression.
+ */
+function matchingParen(text: string, open: number): number {
+  let depth = 0;
+  for (let index = open; index < text.length; index += 1) {
+    const char = text[index];
+    if (char === '"' || char === "'") {
+      index = skipLiteral(text, index);
+      continue;
+    }
+    if (char === "(") depth += 1;
+    if (char === ")") {
+      depth -= 1;
+      if (depth === 0) return index;
+    }
+  }
+  return -1;
+}
+
+/** The index of the quote closing the literal opened at {@link open}. */
+function skipLiteral(text: string, open: number): number {
+  const quote = text[open];
+  for (let index = open + 1; index < text.length; index += 1) {
+    if (text[index] === "\\") {
+      index += 1;
+      continue;
+    }
+    if (text[index] === quote) return index;
+  }
+  return text.length;
+}

package/src/kernel/packs/sparql/buildListQuery.test.ts

@@ -0,0 +1,344 @@
+/**
+ * The generated list query — the SHAPE it composes, and the one property that
+ * shape exists to guarantee: a caller's value never reaches query text.
+ *
+ * These are text assertions on purpose. The behavioural half — that the
+ * compiled predicates answer exactly what the row predicates answered — is
+ * pinned against a real store in `compile.test.ts` and, over the shipped
+ * corpus, in `listQuery.shipped.exec.test.ts`. What can only be asserted here
+ * is what the text does NOT contain: no author query edited, no user string
+ * spliced into a predicate, no page bounds inside the author's own modifiers.
+ */
+
+import { describe, expect, it } from "vitest";
+import { PragmaError } from "../../error/index.js";
+import { buildListQuery } from "./buildListQuery.js";
+
+/** The author query every case below wraps: aggregate, grouped, ordered. */
+const AUTHORED = [
+  "SELECT ?uri ?name ?category",
+  '       (GROUP_CONCAT(DISTINCT ?slug; SEPARATOR=" ") AS ?categories)',
+  "WHERE {",
+  "  ?uri a cs:CodeStandard ; rdfs:label ?name .",
+  "  OPTIONAL { ?uri cs:hasCategory/cs:slug ?category . }",
+  "  OPTIONAL { ?uri cs:hasCategory/skos:broader*/cs:slug ?slug . }",
+  "}",
+  "GROUP BY ?uri ?name ?category",
+  "ORDER BY ?name",
+].join("\n");
+
+const window = { limit: 10, offset: 0 };
+const label = "pragma.conf.ts";
+
+describe("buildListQuery — nothing to filter", () => {
+  const paged = buildListQuery({
+    query: AUTHORED,
+    predicates: [],
+    window: { limit: 10, offset: 20 },
+    label,
+  });
+
+  it("still wraps, because appending a page to author text cannot be safe", () => {
+    // A story whose own text ends in `LIMIT`, `OFFSET` or a trailing `VALUES`
+    // is a syntax error the moment a page is appended to it — and the page has
+    // a default, so that error fires on every call rather than on paged ones.
+    // The wrap is the only shape that holds for all of them.
+    expect(paged).toContain(`  {\n${AUTHORED}\n  }`);
+    expect(paged.startsWith("SELECT ?uri ?name ?category ?categories\n")).toBe(
+      true,
+    );
+    expect(paged.trimEnd().endsWith("LIMIT 10 OFFSET 20")).toBe(true);
+  });
+
+  it("carries no predicate at all — an unfiltered page filters nothing", () => {
+    expect(paged).not.toContain("FILTER");
+    expect(paged).not.toContain("VALUES");
+  });
+
+  it("omits OFFSET 0, which is the no-op it looks like", () => {
+    const first = buildListQuery({
+      query: AUTHORED,
+      predicates: [],
+      window,
+      label,
+    });
+    expect(first.trimEnd().endsWith("LIMIT 10")).toBe(true);
+    expect(first).not.toContain("OFFSET");
+  });
+
+  it("pages an author query that already ends in its own LIMIT", () => {
+    // The shape the appended modifier broke: `LIMIT 5 LIMIT 10` is not a query.
+    // Inside a sub-select the author's own cap is legal, and the page's cap
+    // applies to what that cap returned.
+    const authored = "SELECT ?s WHERE { ?s ?p ?o } ORDER BY ?s LIMIT 5";
+    const query = buildListQuery({
+      query: authored,
+      predicates: [],
+      window,
+      label,
+    });
+    expect(query).toContain(`  {\n${authored}\n  }`);
+    expect(query.trimEnd().endsWith("LIMIT 10")).toBe(true);
+  });
+
+  it("pages an author query that already ends in a trailing VALUES block", () => {
+    const authored = 'SELECT ?s WHERE { ?s ?p ?o } VALUES ?s { "a" }';
+    const query = buildListQuery({
+      query: authored,
+      predicates: [],
+      window,
+      label,
+    });
+    expect(query).toContain(`  {\n${authored}\n  }`);
+  });
+
+  it("keeps an author's SELECT * as SELECT *, which is its own key order", () => {
+    // `SELECT *` in the wrapper over an author `SELECT *` is the same
+    // alphabetisation twice, so the rows a caller gets keep their shape.
+    const query = buildListQuery({
+      query: "SELECT * WHERE { ?s ?p ?o }",
+      predicates: [],
+      window,
+      label,
+    });
+    expect(query).toBe(
+      [
+        "SELECT *",
+        "WHERE {",
+        "  {",
+        "SELECT * WHERE { ?s ?p ?o }",
+        "  }",
+        "}",
+        "LIMIT 10",
+      ].join("\n"),
+    );
+  });
+});
+
+describe("buildListQuery — the author's own prologue", () => {
+  const AUTHORED_WITH_PROLOGUE = [
+    "PREFIX ex: <https://example.test/x#>",
+    "PREFIX ds: <https://example.test/ds#>",
+    "SELECT ?uri ?kind WHERE { ?uri ex:kind ?kind }",
+  ].join("\n");
+
+  it("lifts PREFIX lines to the wrapper's prologue, not into the pattern", () => {
+    // Spliced inside the group graph pattern — where only patterns may appear —
+    // a `PREFIX` line is a parse error, and a story shaped this way died the
+    // moment somebody passed a filter.
+    const query = buildListQuery({
+      query: AUTHORED_WITH_PROLOGUE,
+      predicates: [{ variable: "kind", match: "exact", terms: ["a"] }],
+      window,
+      label,
+    });
+    expect(query.startsWith("PREFIX ex: <https://example.test/x#>\n")).toBe(
+      true,
+    );
+    // The body is a SUFFIX of the author's text, byte for byte: a split, not
+    // an edit.
+    expect(query).toContain(
+      "  {\nSELECT ?uri ?kind WHERE { ?uri ex:kind ?kind }\n  }",
+    );
+    // And the prologue is above the wrapper's own SELECT, exactly once.
+    expect(query.indexOf("PREFIX ds:")).toBeLessThan(
+      query.indexOf("SELECT ?uri ?kind\nWHERE {"),
+    );
+    expect(query.match(/PREFIX ds:/g)).toHaveLength(1);
+  });
+
+  it("lifts it for an unfiltered page too", () => {
+    const query = buildListQuery({
+      query: AUTHORED_WITH_PROLOGUE,
+      predicates: [],
+      window,
+      label,
+    });
+    expect(query.startsWith("PREFIX ex:")).toBe(true);
+    expect(query).not.toContain("  {\nPREFIX");
+  });
+
+  it("emits nothing extra for a story that declares no prologue", () => {
+    const query = buildListQuery({
+      query: AUTHORED,
+      predicates: [],
+      window,
+      label,
+    });
+    expect(query.startsWith("SELECT ?uri ?name ?category ?categories\n")).toBe(
+      true,
+    );
+    expect(query).not.toContain("PREFIX");
+  });
+});
+
+describe("buildListQuery — something to filter", () => {
+  const filtered = buildListQuery({
+    query: AUTHORED,
+    predicates: [{ variable: "categories", match: "set", terms: ["testing"] }],
+    search: { variables: ["name", "description"], term: "unit" },
+    window: { limit: 10, offset: 30 },
+    label,
+  });
+
+  it("wraps the author query in a sub-select, verbatim", () => {
+    expect(filtered).toContain(`  {\n${AUTHORED}\n  }`);
+    // The author's own GROUP BY and ORDER BY survive inside the sub-select,
+    // which is the whole reason a `set` filter can be expressed at all: the
+    // cell it reads is what that GROUP BY computed.
+    expect(filtered).toContain("GROUP BY ?uri ?name ?category");
+    expect(filtered).toContain("ORDER BY ?name");
+  });
+
+  it("projects the author's own variables, in the author's own order", () => {
+    expect(
+      filtered.startsWith("SELECT ?uri ?name ?category ?categories\n"),
+    ).toBe(true);
+    // `SELECT *` here would return alphabetised binding keys, changing the
+    // shape of every JSON answer for no reason a caller asked for.
+    expect(filtered).not.toContain("SELECT *");
+  });
+
+  it("carries the page on the WRAPPER, after the predicates", () => {
+    expect(filtered.trimEnd().endsWith("LIMIT 10 OFFSET 30")).toBe(true);
+    // Not inside the author query, where it would cut before the filter ran.
+    expect(filtered.indexOf("FILTER EXISTS")).toBeLessThan(
+      filtered.lastIndexOf("LIMIT 10 OFFSET 30"),
+    );
+  });
+
+  it("binds each value through VALUES rather than into the predicate", () => {
+    expect(filtered).toContain('VALUES ?__pragmaFilter0 { "testing" }');
+    expect(filtered).toContain('VALUES ?__pragmaSearch { "unit" }');
+    // The comparison names the bound variable — the literal appears once, in
+    // the VALUES row, and nowhere inside an expression.
+    expect(filtered).toContain("LCASE(?__pragmaFilter0)");
+    expect(filtered.match(/"testing"/g)).toHaveLength(1);
+  });
+
+  it("wraps each VALUES in FILTER EXISTS, so a union cannot duplicate a row", () => {
+    // Joined into the group, `VALUES` with two rows would pair every row with
+    // every requested value, and a `set` cell carrying both would come back
+    // twice. EXISTS makes it a per-row boolean instead.
+    const union = buildListQuery({
+      query: AUTHORED,
+      predicates: [
+        { variable: "categories", match: "set", terms: ["testing", "react"] },
+      ],
+      window,
+      label,
+    });
+    expect(union).toContain(
+      'FILTER EXISTS { VALUES ?__pragmaFilter0 { "testing" "react" }',
+    );
+  });
+});
+
+/**
+ * The generated query with every string LITERAL blanked out — what is left is
+ * the query TEXT, which is the only place a caller's value could do harm.
+ *
+ * Asserting on the raw text is not enough: an escaped value legitimately
+ * CONTAINS the words a caller hoped to inject, so counting `UNION` over the
+ * whole string counts their own string back at them.
+ */
+function queryTextOnly(query: string): string {
+  return query.replace(/"(?:[^"\\]|\\.)*"/g, '""');
+}
+
+describe("buildListQuery — injection safety", () => {
+  it("escapes a value that tries to close its own literal and add a clause", () => {
+    const hostile = '" } UNION { ?uri a ds:Secret . VALUES ?x { "';
+    const query = buildListQuery({
+      query: AUTHORED,
+      predicates: [{ variable: "category", match: "exact", terms: [hostile] }],
+      window,
+      label,
+    });
+    // The quote is escaped, so the literal never closes and the injected
+    // pattern stays a string. One VALUES block, one FILTER EXISTS, no UNION.
+    expect(query).toContain(
+      '{ "\\" } UNION { ?uri a ds:Secret . VALUES ?x { \\"" }',
+    );
+    // With the literals blanked, the query text carries exactly the one
+    // VALUES block the builder minted, and no UNION at all.
+    const text = queryTextOnly(query);
+    expect(text.match(/VALUES/g)).toHaveLength(1);
+    expect(text).not.toContain("UNION");
+    expect(text).not.toContain("ds:Secret");
+  });
+
+  it("escapes a newline, so a value cannot start a line of query text", () => {
+    const query = buildListQuery({
+      query: AUTHORED,
+      predicates: [
+        { variable: "category", match: "exact", terms: ["a\nLIMIT 1"] },
+      ],
+      window,
+      label,
+    });
+    expect(query).toContain('{ "a\\nLIMIT 1" }');
+    expect(queryTextOnly(query).match(/LIMIT/g)).toHaveLength(1);
+  });
+
+  it("escapes a search term the same way", () => {
+    const query = buildListQuery({
+      query: AUTHORED,
+      predicates: [],
+      search: { variables: ["name"], term: '"' },
+      window,
+      label,
+    });
+    expect(query).toContain('{ "\\"" }');
+  });
+});
+
+describe("buildListQuery — a story it cannot serve", () => {
+  const build = (query: string, filtered = true) =>
+    buildListQuery({
+      query,
+      predicates: filtered
+        ? [{ variable: "kind", match: "exact", terms: ["a"] }]
+        : [],
+      window,
+      label,
+    });
+
+  it("refuses a filterable story whose SELECT projects *", () => {
+    expect(() =>
+      build("SELECT * WHERE { ?uri a ds:Thing ; ds:kind ?kind }"),
+    ).toThrow(PragmaError);
+    expect(() =>
+      build("SELECT * WHERE { ?uri a ds:Thing ; ds:kind ?kind }"),
+    ).toThrow(/project its variables by name/);
+  });
+
+  it("refuses a story whose own query uses the generated variable prefix", () => {
+    // Shadowing would leave the predicate comparing the caller's value against
+    // the author's variable — a filter that matches nothing, silently.
+    expect(() =>
+      build("SELECT ?kind ?__pragmaFilter0 WHERE { ?u ds:kind ?kind }"),
+    ).toThrow(/reserved variable prefix/);
+  });
+
+  it("refuses a dataset clause, and says it found one", () => {
+    // Not the `SELECT *` message it used to get: the author DOES project by
+    // name, and a wrong reason sends them to the wrong line of their own file.
+    // A sub-select has no place for `FROM`, and lifting it would mean cutting a
+    // span out of the middle of the author's text.
+    for (const filtered of [true, false]) {
+      expect(() =>
+        build(
+          "SELECT ?kind FROM <https://example.test/g> WHERE { ?u ds:kind ?kind }",
+          filtered,
+        ),
+      ).toThrow(/cannot be paged.*FROM dataset clause/s);
+    }
+  });
+
+  it("refuses a text that is not a SELECT, and says that", () => {
+    expect(() => build("ASK { ?s ?p ?o }", false)).toThrow(
+      /cannot be paged.*no SPARQL SELECT/s,
+    );
+  });
+});