@m6d/cortex-cli 0.14.0 → 0.16.0

package/src/contracts/datasets/datasets.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { isRecord } from "../json";
+import { isRecord, JSON_NUMBER_PATTERN } from "../json";
 
 /*
  * Datasets: governed tabular reads over a host's own data. A dataset host
@@ -25,11 +25,36 @@ export const datasetColumnRefSchema = z.string().regex(DATASET_COLUMN_REF_PATTER
 export const DATASET_COLUMN_TYPES = ["string", "number", "boolean", "date"] as const;
 export type DatasetColumnType = (typeof DATASET_COLUMN_TYPES)[number];
 
-export const datasetColumnSchema = z.object({
-    name: datasetIdentifierSchema,
-    type: z.enum(DATASET_COLUMN_TYPES),
-    description: z.string().max(200).default(""),
-});
+/** How long a definition's texts run, how many columns and listed values it holds, and the most rows a query may answer. */
+export const DATASET_DEFINITION_LIMITS = {
+    description: 500,
+    columns: 200,
+    columnDescription: 200,
+    values: 100,
+    valueLength: 200,
+    maxRows: 5000,
+} as const;
+
+/**
+ * One column. `values`, on a string column, lists every value it can hold:
+ * the agent reads them, and a query comparing the column to anything else is
+ * refused before it runs.
+ */
+export const datasetColumnSchema = z
+    .object({
+        name: datasetIdentifierSchema,
+        type: z.enum(DATASET_COLUMN_TYPES),
+        description: z.string().max(DATASET_DEFINITION_LIMITS.columnDescription).default(""),
+        values: z
+            .array(z.string().min(1).max(DATASET_DEFINITION_LIMITS.valueLength))
+            .min(1)
+            .max(DATASET_DEFINITION_LIMITS.values)
+            .optional(),
+    })
+    .refine((column) => column.values === undefined || column.type === "string", {
+        message: "only a string column lists its values",
+        path: ["values"],
+    });
 export type DatasetColumn = z.infer<typeof datasetColumnSchema>;
 
 /**
@@ -54,11 +79,16 @@ export const DATASET_MAX_ROWS_DEFAULT = 500;
 export const datasetDefinitionSchema = z.object({
     name: datasetIdentifierSchema,
     /** What the model reads in the catalog; the columns come only on request. */
-    description: z.string().max(500),
-    columns: z.array(datasetColumnSchema).min(1).max(200),
+    description: z.string().max(DATASET_DEFINITION_LIMITS.description),
+    columns: z.array(datasetColumnSchema).min(1).max(DATASET_DEFINITION_LIMITS.columns),
     relations: z.record(datasetIdentifierSchema, datasetRelationSchema).default({}),
     /** The most rows one query may answer with; the host clamps to it whatever the caller sent. */
-    maxRows: z.number().int().min(1).max(5000).default(DATASET_MAX_ROWS_DEFAULT),
+    maxRows: z
+        .number()
+        .int()
+        .min(1)
+        .max(DATASET_DEFINITION_LIMITS.maxRows)
+        .default(DATASET_MAX_ROWS_DEFAULT),
 });
 export type DatasetDefinition = z.infer<typeof datasetDefinitionSchema>;
 export type DatasetDefinitionInput = z.input<typeof datasetDefinitionSchema>;
@@ -208,19 +238,22 @@ export const datasetConditionSchema: z.ZodType<DatasetCondition, DatasetConditio
  * A condition, a group of filters that must all hold (`all`) or of which one
  * must hold (`any`), or the negation of one (`not`, with SQL's null
  * semantics: a row whose comparison is unknown is left out either way). A
- * filter list is an `all`.
+ * filter list is an `all`. An object carrying `col` is a condition, whatever
+ * else it carries; any other is an `any`, else an `all`, else a `not` group.
  */
 export type DatasetFilter =
     | DatasetCondition
     | { any: DatasetFilter[] }
     | { all: DatasetFilter[] }
     | { not: DatasetFilter };
+/** A group refuses `col`, so an object with one is read as a condition or not at all. */
+const notACondition = { col: z.never().optional() };
 export const datasetFilterSchema: z.ZodType<DatasetFilter, DatasetFilter> = z.lazy(() =>
     z.union([
         datasetConditionSchema,
-        z.object({ any: z.array(datasetFilterSchema).min(1).max(20) }),
-        z.object({ all: z.array(datasetFilterSchema).min(1).max(20) }),
-        z.object({ not: datasetFilterSchema }),
+        z.object({ any: z.array(datasetFilterSchema).min(1).max(20), ...notACondition }),
+        z.object({ all: z.array(datasetFilterSchema).min(1).max(20), ...notACondition }),
+        z.object({ not: datasetFilterSchema, ...notACondition }),
     ]),
 );
 
@@ -316,13 +349,18 @@ const datasetQueryObjectSchema = z.object({
         .describe(
             "By a column or select alias, a group key, an aggregate alias or a computed alias.",
         ),
-    limit: z.number().int().min(1).optional(),
+    /** A 32-bit integer, as `offset` is; the dataset's `maxRows` caps it. */
+    limit: z.int32().min(1).optional(),
     /** Rows to skip before `limit`; only with a `sort`, since the row order is undefined without one. */
-    offset: z.number().int().min(0).optional(),
+    offset: z.int32().min(0).optional(),
 });
 
 /** How deep a query's JSON may nest, objects and arrays alike; far past what a statement needs. */
 export const DATASET_QUERY_MAX_DEPTH = 32;
+/** The largest body `POST {host}/{name}` takes: a query is a few kilobytes of JSON, so a host refuses anything far past that before reading it. */
+export const DATASET_REQUEST_MAX_BYTES = 256 * 1024;
+/** How many times a query may name its computed aliases once each is written out where it is named; far past any query a model writes. */
+export const DATASET_COMPUTE_REFERENCES_MAX = 100;
 
 /** Whether `value` nests at most `budget` objects and arrays deep; the walk stops at the budget, so a deeper value costs no more than an allowed one. */
 function nestsWithin(value: unknown, budget: number): boolean {
@@ -341,12 +379,19 @@ export const datasetQuerySchema = z
     .pipe(datasetQueryObjectSchema);
 export type DatasetQuery = z.infer<typeof datasetQuerySchema>;
 
-/** What `POST {host}/{name}` receives: the validated query, capped, plus what the host may want in its own logs. */
+/**
+ * What `POST {host}/{name}` receives: the validated query, capped, plus what
+ * the host may want in its own logs, and the IANA zone of the user the query
+ * runs for. A host reads that zone's calendar into the query: a bare date
+ * names its midnight there, a period or part of an instant is taken on its
+ * clock (shifted by the zone's offset now), and an instant answers on it.
+ */
 export const datasetRequestSchema = z.object({
     query: datasetQuerySchema,
     context: z.object({
         threadId: z.string().max(128),
         requestId: z.string().max(128).optional(),
+        timeZone: z.string().max(64).optional(),
     }),
 });
 export type DatasetRequest = z.infer<typeof datasetRequestSchema>;
@@ -396,10 +441,59 @@ export function resolveDatasetColumn(catalog: DatasetCatalog, from: string, ref:
         dataset = target;
     }
     const column = dataset.columns.find((candidate) => candidate.name === columnName);
-    if (!column) return missing(`"${dataset.name}" has no column "${columnName}"`);
+    if (!column) return missing(unknownColumnError(dataset, columnName));
     return { ok: true as const, column, path: segments.join(".") };
 }
 
+/**
+ * Why `dataset` has no column `name`, with the columns it has and up to
+ * three closest ones: names containing it or contained in it (ignoring
+ * case, and only for names longer than two characters), else names two
+ * edits or fewer away. A query that guessed `name` reads `departments has
+ * no column "name". Closest: nameAr, nameEn. Columns: id, nameAr, ...`
+ */
+export function unknownColumnError(dataset: DatasetDefinition, name: string) {
+    const columns = dataset.columns.map((column) => column.name);
+    const wanted = name.toLowerCase();
+    const containing = columns.filter((column) => {
+        const lower = column.toLowerCase();
+        return (
+            (wanted.length > 2 && lower.includes(wanted)) ||
+            (lower.length > 2 && wanted.includes(lower))
+        );
+    });
+    const closest = (
+        containing.length > 0
+            ? containing
+            : // Lengths more than two apart are more than two edits apart, so a
+              // name of any length costs only its own columns' comparisons.
+              columns.filter(
+                  (column) =>
+                      Math.abs(column.length - wanted.length) <= 2 &&
+                      editDistance(column.toLowerCase(), wanted) <= 2,
+              )
+    ).slice(0, 3);
+    const hint = closest.length > 0 ? ` Closest: ${closest.join(", ")}.` : "";
+    return `${dataset.name} has no column "${name}".${hint} Columns: ${columns.join(", ")}.`;
+}
+
+/** Levenshtein distance: the fewest single-character edits from `a` to `b`. */
+function editDistance(a: string, b: string) {
+    let previous = Array.from({ length: b.length + 1 }, (_, index) => index);
+    for (let i = 1; i <= a.length; i++) {
+        const current = [i];
+        for (let j = 1; j <= b.length; j++) {
+            current[j] = Math.min(
+                previous[j]! + 1,
+                current[j - 1]! + 1,
+                previous[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
+            );
+        }
+        previous = current;
+    }
+    return previous[b.length]!;
+}
+
 function refError(kind: string, ref: string, reason: string) {
     return `${kind} "${ref}": ${reason}`;
 }
@@ -424,8 +518,9 @@ function filterValueIssue(filter: DatasetCondition) {
  * The query as the host runs it: habits the model brings from SQL are
  * absorbed here rather than refused. A reference prefixed with the dataset's
  * own name (`orders.total` on `orders`) drops the prefix when no relation
- * has that name; a relation any reference names is joined without being
- * listed; a `select` on a grouped query that only repeats the group keys is
+ * has that name; a relation a reference names is joined without being
+ * listed, except in `compute` and `having`, which read the answered row; a
+ * `select` on a grouped query that only repeats the group keys is
  * dropped, and an ungrouped query without one selects the definition's
  * columns. Every host receives this shape, so all of them agree on it.
  */
@@ -624,8 +719,9 @@ export function boundDatasetResult(
 /**
  * Every way a query can be wrong given the catalog, as messages the model
  * can act on; an empty list means the host will understand it. Checked
- * before anything leaves Cortex, so a host can trust the shape and only
- * has to translate.
+ * before anything leaves Cortex, and again by every host against its own
+ * catalog. Computed aliases written out where they are named are capped
+ * here; how large the rest grows as SQL or LINQ is each host's own bound.
  */
 export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuery) {
     if (!catalog.has(query.from)) return [`Unknown dataset "${query.from}"`];
@@ -654,6 +750,23 @@ export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuer
     } satisfies QueryCheck;
 
     for (const filter of query.filter) checkFilter(check, filter);
+    const row = checkAnsweredRow(check, catalog, query);
+    checkFinishedRow(check, query, row);
+    if (computeReferences(query) > DATASET_COMPUTE_REFERENCES_MAX) {
+        check.issue(
+            `computed aliases are named more than ${DATASET_COMPUTE_REFERENCES_MAX} times once each is written out where it is named; name earlier computed aliases fewer times`,
+        );
+    }
+    return issues;
+}
+
+/**
+ * The row a query answers, checked as it is built: on a grouped query its
+ * keys and aggregate aliases (never one name for both, and no select), on a
+ * plain one its select aliases; each with the names that are a column as it
+ * is, which keep what the column says.
+ */
+function checkAnsweredRow(check: QueryCheck, catalog: DatasetCatalog, query: DatasetQuery) {
     const groupKeys = checkGroupBy(check, query.groupBy);
     const aliases = checkAggregates(check, query.aggregate);
     for (const alias of aliases.keys()) {
@@ -671,181 +784,48 @@ export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuer
         );
     }
     const selectAliases = checkSelect(check, query.select ?? []);
-    const row = grouped
-        ? { names: new Map([...groupKeys, ...aliases]), kind: "grouped" as const }
-        : { names: selectAliases, kind: "plain" as const };
-    checkFinishedRow(check, query, row);
-    checkExpansion(check, query, catalog.get(query.from)?.columns.length ?? 0);
-    return issues;
+    return grouped
+        ? {
+              names: new Map([...groupKeys, ...aliases]),
+              kind: "grouped" as const,
+              columns: passedColumns(catalog, query, query.groupBy),
+          }
+        : {
+              names: selectAliases,
+              kind: "plain" as const,
+              columns: passedColumns(catalog, query, query.select ?? []),
+          };
 }
 
-/** How large any one expression a host translates may grow once every alias it names is written out in place. */
-export const DATASET_EXPANSION_BUDGET = 2_000;
-
 /**
- * Hosts write a computed alias out again wherever a later calculation or
- * HAVING names it, as SQL requires, so ten aliases each naming the previous
- * one ten times grow to 10^10 nodes from a few hundred bytes, with or without
- * a column among them. Each alias costs what it expands to, worked out once in
- * order, so the expansion is measured without being performed: a named select
- * value or group key, an aggregate and a window cost their own trees wherever
- * they are named, since hosts write them out. Every expression a host
- * translates is held to the budget: select values, group keys, WHERE,
- * aggregates and their filters, windows, calculations, HAVING and QUALIFY.
- * Sort and rank name values and add none. A window is a correlated subquery
- * that carries the row it counts over, so it also costs that row: WHERE and
- * every value the row projects (`columnCount` of them for a plain query with
- * no select).
+ * How many times the query names its computed aliases once each is written
+ * out where it is named, as SQL needs: every name counts once plus what the
+ * named value itself names, so ten values each naming the previous one ten
+ * times count past a billion. Worked out once per alias, capped just past
+ * the limit.
  */
-function checkExpansion(check: QueryCheck, query: DatasetQuery, columnCount: number) {
-    // The source row: what WHERE, select values, group keys and aggregates read, each column once.
-    const source = new Map<string, number>();
-    const named = [...(query.select ?? []), ...query.groupBy].flatMap((entry) =>
-        typeof entry === "string" ? [] : [[entry.as, operandCost(entry.expr, source)] as const],
-    );
-    const where = query.filter.map((filter) => filterCost(filter, source));
-    const costs = [...named.map(([, total]) => total), ...where];
-    const expanded = new Map(named);
-    const aggregates = query.aggregate.map((entry) => {
-        const parts = [
-            ...(entry.col === undefined ? [] : [operandCost(entry.col, source)]),
-            ...entry.filter.map((filter) => filterCost(filter, source)),
-        ];
-        costs.push(...parts);
-        expanded.set(entry.as, nodeCost(parts));
-        return nodeCost(parts);
-    });
-    const grouped = query.groupBy.length > 0 || query.aggregate.length > 0;
-    const projected = grouped ? query.groupBy : (query.select ?? []);
-    const carried = Math.min(
-        DATASET_EXPANSION_BUDGET + 1,
-        [
-            ...where,
-            ...projected.map((entry) =>
-                typeof entry === "string" ? 1 : (expanded.get(entry.as) ?? 1),
-            ),
-            ...(grouped || projected.length ? [] : Array<number>(columnCount).fill(1)),
-            ...aggregates,
-        ].reduce((total, cost) => total + cost, 0),
-    );
-    // Every window reads the row as it was before any window, its partition and the row it carries included.
-    const windows = query.window.map((entry) => {
-        const total = nodeCost([
-            ...(entry.col === undefined ? [] : [operandCost(entry.col, expanded)]),
-            ...entry.filter.map((filter) => filterCost(filter, expanded)),
-            ...entry.partitionBy.map((name) => operandCost(name, expanded)),
-            carried,
-        ]);
-        costs.push(total);
-        return [entry.as, total] as const;
-    });
-    for (const [as, total] of windows) expanded.set(as, total);
-    for (const entry of query.compute) {
-        const total = operandCost(entry.expr, expanded);
-        expanded.set(entry.as, total);
-        costs.push(total);
-    }
-    for (const filter of query.having) costs.push(filterCost(filter, expanded));
-    // A grouped row is materialized before QUALIFY reads it; without group keys the
-    // .NET host writes the calculations QUALIFY names out in full.
-    const qualifying = query.groupBy.length ? source : expanded;
-    for (const filter of query.qualify) costs.push(filterCost(filter, qualifying));
-    if (costs.some((total) => total > DATASET_EXPANSION_BUDGET)) {
-        check.issue(
-            `Computed aliases expand to more than ${DATASET_EXPANSION_BUDGET} references; name intermediate results and reuse them less`,
+function computeReferences(query: DatasetQuery) {
+    const written = new Map<string, number>();
+    const count = (refs: string[]) =>
+        Math.min(
+            refs.reduce((sum, ref) => {
+                const inner = written.get(ref);
+                return inner === undefined ? sum : sum + 1 + inner;
+            }, 0),
+            DATASET_COMPUTE_REFERENCES_MAX + 1,
         );
+    let total = 0;
+    for (const entry of query.compute) {
+        const own = count(operandRefs(entry.expr));
+        written.set(entry.as, own);
+        total += own;
     }
-}
-
-/**
- * How many times a host writes one part out when it spells the node holding
- * it, the most either host does; every other part is written once. A divisor
- * is tested for zero before the division, ROUND's digits scale the value
- * before and after, and LENGTH's text is tested for NULL besides measured. A
- * date is read once per part a truncation or part needs (Zeller's congruence
- * for the week and weekday), and a truncation other than the week, packed as
- * a number, is read 3 more times when it meets a real date in a COALESCE, a
- * CASE or a comparison, so those factors are the product. A condition guards
- * its sides against NULL, and a negated condition with no opposite operator
- * (`between`, the text matches) tests its left side once more.
- */
-const EXPANSION_WEIGHTS = {
-    divisor: 2,
-    roundDigits: 2,
-    lengthText: 2,
-    conditionLeft: 3,
-    conditionRight: 2,
-    dateTrunc: { day: 9, month: 6, quarter: 6, year: 3, week: 17 },
-    datePart: { day: 1, month: 1, quarter: 1, year: 1, dayOfWeek: 16, week: 17 },
-} as const;
-
-/** One node plus its children's costs, held at one past the budget so no sum can overflow. */
-function nodeCost(children: number[]) {
-    return Math.min(
-        DATASET_EXPANSION_BUDGET + 1,
-        children.reduce((total, cost) => total + cost, 1),
-    );
-}
-
-/** How often a scalar function's argument is written out. */
-function argumentWeight(fn: DatasetFunction, index: number) {
-    if (fn === "length") return EXPANSION_WEIGHTS.lengthText;
-    return fn === "round" && index === 1 ? EXPANSION_WEIGHTS.roundDigits : 1;
-}
-
-/**
- * An operand's size written out: a literal or a column is 1, a reference to an
- * earlier computed alias is that alias's cost (a self or forward one, refused
- * elsewhere, is 1), and arithmetic, a date function, a scalar function, a CASE
- * and each of its branches are 1 plus their parts, each part times its weight.
- */
-function operandCost(operand: DatasetOperand, expanded: ReadonlyMap<string, number>): number {
-    if (typeof operand === "string") return expanded.get(operand) ?? 1;
-    if (typeof operand === "number" || "value" in operand) return 1;
-    if ("op" in operand) {
-        const divisor = operand.op === "/" ? EXPANSION_WEIGHTS.divisor : 1;
-        return nodeCost([
-            operandCost(operand.left, expanded),
-            operandCost(operand.right, expanded) * divisor,
-        ]);
-    }
-    if ("case" in operand) {
-        return nodeCost([
-            ...operand.case.map((branch) =>
-                nodeCost([filterCost(branch.when, expanded), operandCost(branch.then, expanded)]),
-            ),
-            ...(operand.else === undefined ? [] : [operandCost(operand.else, expanded)]),
-        ]);
-    }
-    if ("args" in operand) {
-        return nodeCost(
-            operand.args.map(
-                (arg, index) => operandCost(arg, expanded) * argumentWeight(operand.fn, index),
-            ),
-        );
-    }
-    const weight =
-        operand.fn === "dateTrunc"
-            ? EXPANSION_WEIGHTS.dateTrunc[operand.unit]
-            : EXPANSION_WEIGHTS.datePart[operand.unit];
-    return nodeCost([operandCost(operand.of, expanded) * weight]);
-}
-
-/** A filter's size written out: 1 per condition or group plus its parts; a condition's left side is weighted, each literal on its right is 1, and a `{ col }` right side is its weighted operand cost. */
-function filterCost(filter: DatasetFilter, expanded: ReadonlyMap<string, number>): number {
-    if ("not" in filter) return nodeCost([filterCost(filter.not, expanded)]);
-    if ("any" in filter || "all" in filter) {
-        const entries = "any" in filter ? filter.any : filter.all;
-        return nodeCost(entries.map((entry) => filterCost(entry, expanded)));
-    }
-    const right = isColumnValue(filter.value)
-        ? operandCost(filter.value.col, expanded) * EXPANSION_WEIGHTS.conditionRight
-        : Array.isArray(filter.value)
-          ? filter.value.length
-          : filter.value === undefined
-            ? 0
-            : 1;
-    return nodeCost([operandCost(filter.col, expanded) * EXPANSION_WEIGHTS.conditionLeft, right]);
+    const rowNames = [
+        ...[...query.having, ...query.qualify].flatMap(filterRefs),
+        ...[...query.sort, ...(query.rank?.sort ?? [])].map((entry) => entry.col),
+        ...(query.rank?.partitionBy ?? []),
+    ];
+    return total + count(rowNames);
 }
 
 /**
@@ -856,7 +836,7 @@ function filterCost(filter: DatasetFilter, expanded: ReadonlyMap<string, number>
 function checkFinishedRow(
     check: QueryCheck,
     query: DatasetQuery,
-    row: { names: Map<string, DatasetColumnType>; kind: "grouped" | "plain" },
+    row: RowNames & { names: Map<string, DatasetColumnType> },
 ) {
     const rowCheck = rowScope(check, row);
     // On a plain row a selected column is a name too; a window or calculation may not take it.
@@ -881,7 +861,7 @@ function rowScope(check: QueryCheck, row: RowNames) {
         issue: (message: string) => check.issue(message),
         columnOf(kind, ref) {
             const type = row.names.get(ref);
-            if (type) return { name: ref, type, description: "" };
+            if (type) return row.columns.get(ref) ?? { name: ref, type, description: "" };
             if (row.kind === "plain") return check.columnOf(kind, ref);
             check.issue(
                 refError(
@@ -1203,10 +1183,31 @@ function checkFilter(check: QueryCheck, filter: DatasetFilter, kind = "filter")
         ? columnValueIssue(check, kind, type, filter.op, filter.value.col)
         : (filterValueIssue(filter) ??
           filterValueTypeIssue(type, filter) ??
-          truncatedDateIssue(filter));
+          truncatedDateIssue(filter) ??
+          listedValueIssue(check, kind, filter));
     if (issue) check.issue(refError(kind, shown, issue));
 }
 
+const LISTED_VALUE_OPS = new Set<DatasetFilterOp>(["=", "!=", "in", "notIn"]);
+
+/**
+ * A column that lists its values is only compared with one of them: any
+ * other literal would match no row, or every row, and the host would answer
+ * as if the data said so.
+ */
+function listedValueIssue(check: QueryCheck, kind: string, filter: DatasetCondition) {
+    if (typeof filter.col !== "string" || !LISTED_VALUE_OPS.has(filter.op)) return undefined;
+    if (filter.value === undefined || isColumnValue(filter.value)) return undefined;
+    const listed = check.columnOf(kind, filter.col)?.values;
+    if (!listed) return undefined;
+    const values = Array.isArray(filter.value) ? filter.value : [filter.value];
+    // A number or a boolean is compared as the text the host turns it into.
+    const odd = values.find((value) => !listed.includes(String(value)));
+    return odd === undefined
+        ? undefined
+        : `${JSON.stringify(odd)} is not one of its values: ${listed.join(", ")}`;
+}
+
 /** A truncated date is a whole day, so a literal against it is one too ("2024-03-01"); a time or a bare month would never match. */
 function truncatedDateIssue(filter: DatasetCondition) {
     const truncated =
@@ -1242,7 +1243,7 @@ function filterValueTypeIssue(type: DatasetColumnType, filter: DatasetCondition)
                 return (
                     typeof value === "number" ||
                     (typeof value === "string" &&
-                        value.trim() !== "" &&
+                        JSON_NUMBER_PATTERN.test(value) &&
                         Number.isFinite(Number(value)))
                 );
             case "date":
@@ -1314,7 +1315,31 @@ function aggregateType(
 }
 
 /** What sort and rank may name: on a grouped query a group key or an alias; on a plain one a select alias or any reachable column. */
-type RowNames = { names: ReadonlyMap<string, DatasetColumnType>; kind: "grouped" | "plain" };
+type RowNames = {
+    names: ReadonlyMap<string, DatasetColumnType>;
+    kind: "grouped" | "plain";
+    /** The names that are a column as it is (group keys, select entries), so what the column says (its values) still holds. */
+    columns: ReadonlyMap<string, DatasetColumn>;
+};
+
+/**
+ * The row names that are a column as it is, each with the column: a bare
+ * column (`status`), or one under an alias (`{ expr: "status", as: "state" }`).
+ */
+function passedColumns(
+    catalog: DatasetCatalog,
+    query: DatasetQuery,
+    entries: ReadonlyArray<string | { expr: DatasetOperand; as: string }>,
+) {
+    return new Map(
+        entries.flatMap((entry) => {
+            const [name, ref] = typeof entry === "string" ? [entry, entry] : [entry.as, entry.expr];
+            if (typeof ref !== "string") return [];
+            const resolved = resolveDatasetColumn(catalog, query.from, ref);
+            return resolved.ok ? [[name, resolved.column] as const] : [];
+        }),
+    );
+}
 
 function checkSort(check: QueryCheck, sort: DatasetQuery["sort"], kind: string, row: RowNames) {
     for (const { col } of sort) checkRowName(check, kind, col, row);

package/src/contracts/egress-policy/scrub.ts

@@ -1,4 +1,4 @@
-import { isRecord } from "../json";
+import { isRecord, JSON_NUMBER_PATTERN } from "../json";
 
 const REDACTION_MARKER = "[REDACTED]";
 
@@ -7,19 +7,22 @@ const REDACTION_MARKER = "[REDACTED]";
  * itself and, after a leading scheme word (`Bearer xyz`, `Basic abc`), the
  * bare secret the scheme merely labels; each of those raw as in a header,
  * percent-encoded as in a path slot (`encodeURIComponent`) or a query value
- * (`URLSearchParams`, which spells a space as `+`), and JSON-escaped as inside
- * a request body. The scrubber replaces longer forms first.
+ * (`URLSearchParams`, which spells a space as `+`), JSON-escaped as inside
+ * a request body, and JSON-escaped then percent-encoded as a form field sent
+ * as JSON text is. The scrubber replaces longer forms first.
  */
 export function credentialForms(value: string) {
     const bare = /^[A-Za-z][\w.-]* +(\S.*)$/.exec(value)?.[1];
+    const percentEncoded = (form: string) => [
+        encodeURIComponent(form),
+        new URLSearchParams({ v: form }).toString().slice(2),
+    ];
     return [
         ...new Set(
-            (bare ? [value, bare] : [value]).flatMap((form) => [
-                form,
-                encodeURIComponent(form),
-                new URLSearchParams({ v: form }).toString().slice(2),
-                JSON.stringify(form).slice(1, -1),
-            ]),
+            (bare ? [value, bare] : [value]).flatMap((form) => {
+                const json = JSON.stringify(form).slice(1, -1);
+                return [form, ...percentEncoded(form), json, ...percentEncoded(json)];
+            }),
         ),
     ];
 }
@@ -31,7 +34,9 @@ export function credentialForms(value: string) {
  * transcript alike, then the credentials the request presented are
  * redacted. Dropping first, on the patterns as well, means a credential
  * an upstream spelled with U+0000 inside it still matches; redacting first
- * would let the strip reassemble it.
+ * would let the strip reassemble it. A number is matched whole, by value,
+ * against each credential read as a number, so a digit credential echoed
+ * as a JSON number is redacted even when parsing rounded it.
  */
 export function scrubCredentialStrings(value: unknown, credentials: string[]): unknown {
     const dropU0000 = (text: string) => text.replaceAll("\u0000", "");
@@ -48,6 +53,13 @@ export function scrubCredentialStrings(value: unknown, credentials: string[]): u
         );
 
     if (typeof value === "string") return scrub(value);
+    if (typeof value === "number") {
+        return ordered.some(
+            (credential) => JSON_NUMBER_PATTERN.test(credential) && Number(credential) === value,
+        )
+            ? REDACTION_MARKER
+            : value;
+    }
     if (Array.isArray(value)) {
         return value.map((item) => scrubCredentialStrings(item, credentials));
     }

package/src/contracts/errors/index.ts

@@ -36,3 +36,10 @@ export const backendErrorSchema = z.object({
     }),
 });
 export type BackendError = z.infer<typeof backendErrorSchema>;
+
+/** The envelope a backend answers a refusal in, its detail cut to the length Cortex accepts. */
+export function errorEnvelope(kind: BackendErrorKind, detail: string) {
+    return {
+        error: { kind, detail: detail.slice(0, BACKEND_ERROR_DETAIL_MAX) },
+    } satisfies BackendError;
+}