@m6d/cortex-cli 0.15.0 → 0.17.0

package/src/contracts/datasets/datasets.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { isRecord } from "../json";
+import { isRecord, JSON_NUMBER_PATTERN } from "../json";
 
 /*
  * Datasets: governed tabular reads over a host's own data. A dataset host
@@ -25,6 +25,16 @@ export const datasetColumnRefSchema = z.string().regex(DATASET_COLUMN_REF_PATTER
 export const DATASET_COLUMN_TYPES = ["string", "number", "boolean", "date"] as const;
 export type DatasetColumnType = (typeof DATASET_COLUMN_TYPES)[number];
 
+/** How long a definition's texts run, how many columns and listed values it holds, and the most rows a query may answer. */
+export const DATASET_DEFINITION_LIMITS = {
+    description: 500,
+    columns: 200,
+    columnDescription: 200,
+    values: 100,
+    valueLength: 200,
+    maxRows: 5000,
+} as const;
+
 /**
  * One column. `values`, on a string column, lists every value it can hold:
  * the agent reads them, and a query comparing the column to anything else is
@@ -34,8 +44,12 @@ export const datasetColumnSchema = z
     .object({
         name: datasetIdentifierSchema,
         type: z.enum(DATASET_COLUMN_TYPES),
-        description: z.string().max(200).default(""),
-        values: z.array(z.string().min(1).max(200)).min(1).max(100).optional(),
+        description: z.string().max(DATASET_DEFINITION_LIMITS.columnDescription).default(""),
+        values: z
+            .array(z.string().min(1).max(DATASET_DEFINITION_LIMITS.valueLength))
+            .min(1)
+            .max(DATASET_DEFINITION_LIMITS.values)
+            .optional(),
     })
     .refine((column) => column.values === undefined || column.type === "string", {
         message: "only a string column lists its values",
@@ -65,11 +79,16 @@ export const DATASET_MAX_ROWS_DEFAULT = 500;
 export const datasetDefinitionSchema = z.object({
     name: datasetIdentifierSchema,
     /** What the model reads in the catalog; the columns come only on request. */
-    description: z.string().max(500),
-    columns: z.array(datasetColumnSchema).min(1).max(200),
+    description: z.string().max(DATASET_DEFINITION_LIMITS.description),
+    columns: z.array(datasetColumnSchema).min(1).max(DATASET_DEFINITION_LIMITS.columns),
     relations: z.record(datasetIdentifierSchema, datasetRelationSchema).default({}),
     /** The most rows one query may answer with; the host clamps to it whatever the caller sent. */
-    maxRows: z.number().int().min(1).max(5000).default(DATASET_MAX_ROWS_DEFAULT),
+    maxRows: z
+        .number()
+        .int()
+        .min(1)
+        .max(DATASET_DEFINITION_LIMITS.maxRows)
+        .default(DATASET_MAX_ROWS_DEFAULT),
 });
 export type DatasetDefinition = z.infer<typeof datasetDefinitionSchema>;
 export type DatasetDefinitionInput = z.input<typeof datasetDefinitionSchema>;
@@ -219,19 +238,22 @@ export const datasetConditionSchema: z.ZodType<DatasetCondition, DatasetConditio
  * A condition, a group of filters that must all hold (`all`) or of which one
  * must hold (`any`), or the negation of one (`not`, with SQL's null
  * semantics: a row whose comparison is unknown is left out either way). A
- * filter list is an `all`.
+ * filter list is an `all`. An object carrying `col` is a condition, whatever
+ * else it carries; any other is an `any`, else an `all`, else a `not` group.
  */
 export type DatasetFilter =
     | DatasetCondition
     | { any: DatasetFilter[] }
     | { all: DatasetFilter[] }
     | { not: DatasetFilter };
+/** A group refuses `col`, so an object with one is read as a condition or not at all. */
+const notACondition = { col: z.never().optional() };
 export const datasetFilterSchema: z.ZodType<DatasetFilter, DatasetFilter> = z.lazy(() =>
     z.union([
         datasetConditionSchema,
-        z.object({ any: z.array(datasetFilterSchema).min(1).max(20) }),
-        z.object({ all: z.array(datasetFilterSchema).min(1).max(20) }),
-        z.object({ not: datasetFilterSchema }),
+        z.object({ any: z.array(datasetFilterSchema).min(1).max(20), ...notACondition }),
+        z.object({ all: z.array(datasetFilterSchema).min(1).max(20), ...notACondition }),
+        z.object({ not: datasetFilterSchema, ...notACondition }),
     ]),
 );
 
@@ -327,13 +349,18 @@ const datasetQueryObjectSchema = z.object({
         .describe(
             "By a column or select alias, a group key, an aggregate alias or a computed alias.",
         ),
-    limit: z.number().int().min(1).optional(),
+    /** A 32-bit integer, as `offset` is; the dataset's `maxRows` caps it. */
+    limit: z.int32().min(1).optional(),
     /** Rows to skip before `limit`; only with a `sort`, since the row order is undefined without one. */
-    offset: z.number().int().min(0).optional(),
+    offset: z.int32().min(0).optional(),
 });
 
 /** How deep a query's JSON may nest, objects and arrays alike; far past what a statement needs. */
 export const DATASET_QUERY_MAX_DEPTH = 32;
+/** The largest body `POST {host}/{name}` takes: a query is a few kilobytes of JSON, so a host refuses anything far past that before reading it. */
+export const DATASET_REQUEST_MAX_BYTES = 256 * 1024;
+/** How many times a query may name its computed aliases once each is written out where it is named; far past any query a model writes. */
+export const DATASET_COMPUTE_REFERENCES_MAX = 100;
 
 /** Whether `value` nests at most `budget` objects and arrays deep; the walk stops at the budget, so a deeper value costs no more than an allowed one. */
 function nestsWithin(value: unknown, budget: number): boolean {
@@ -352,12 +379,19 @@ export const datasetQuerySchema = z
     .pipe(datasetQueryObjectSchema);
 export type DatasetQuery = z.infer<typeof datasetQuerySchema>;
 
-/** What `POST {host}/{name}` receives: the validated query, capped, plus what the host may want in its own logs. */
+/**
+ * What `POST {host}/{name}` receives: the validated query, capped, plus what
+ * the host may want in its own logs, and the IANA zone of the user the query
+ * runs for. A host reads that zone's calendar into the query: a bare date
+ * names its midnight there, a period or part of an instant is taken on its
+ * clock (shifted by the zone's offset now), and an instant answers on it.
+ */
 export const datasetRequestSchema = z.object({
     query: datasetQuerySchema,
     context: z.object({
         threadId: z.string().max(128),
         requestId: z.string().max(128).optional(),
+        timeZone: z.string().max(64).optional(),
     }),
 });
 export type DatasetRequest = z.infer<typeof datasetRequestSchema>;
@@ -484,8 +518,9 @@ function filterValueIssue(filter: DatasetCondition) {
  * The query as the host runs it: habits the model brings from SQL are
  * absorbed here rather than refused. A reference prefixed with the dataset's
  * own name (`orders.total` on `orders`) drops the prefix when no relation
- * has that name; a relation any reference names is joined without being
- * listed; a `select` on a grouped query that only repeats the group keys is
+ * has that name; a relation a reference names is joined without being
+ * listed, except in `compute` and `having`, which read the answered row; a
+ * `select` on a grouped query that only repeats the group keys is
  * dropped, and an ungrouped query without one selects the definition's
  * columns. Every host receives this shape, so all of them agree on it.
  */
@@ -684,8 +719,9 @@ export function boundDatasetResult(
 /**
  * Every way a query can be wrong given the catalog, as messages the model
  * can act on; an empty list means the host will understand it. Checked
- * before anything leaves Cortex, so a host can trust the shape and only
- * has to translate.
+ * before anything leaves Cortex, and again by every host against its own
+ * catalog. Computed aliases written out where they are named are capped
+ * here; how large the rest grows as SQL or LINQ is each host's own bound.
  */
 export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuery) {
     if (!catalog.has(query.from)) return [`Unknown dataset "${query.from}"`];
@@ -716,7 +752,11 @@ export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuer
     for (const filter of query.filter) checkFilter(check, filter);
     const row = checkAnsweredRow(check, catalog, query);
     checkFinishedRow(check, query, row);
-    checkExpansion(check, query, catalog.get(query.from)?.columns.length ?? 0);
+    if (computeReferences(query) > DATASET_COMPUTE_REFERENCES_MAX) {
+        check.issue(
+            `computed aliases are named more than ${DATASET_COMPUTE_REFERENCES_MAX} times once each is written out where it is named; name earlier computed aliases fewer times`,
+        );
+    }
     return issues;
 }
 
@@ -757,173 +797,35 @@ function checkAnsweredRow(check: QueryCheck, catalog: DatasetCatalog, query: Dat
           };
 }
 
-/** How large any one expression a host translates may grow once every alias it names is written out in place. */
-export const DATASET_EXPANSION_BUDGET = 2_000;
-
-/**
- * Hosts write a computed alias out again wherever a later calculation or
- * HAVING names it, as SQL requires, so ten aliases each naming the previous
- * one ten times grow to 10^10 nodes from a few hundred bytes, with or without
- * a column among them. Each alias costs what it expands to, worked out once in
- * order, so the expansion is measured without being performed: a named select
- * value or group key, an aggregate and a window cost their own trees wherever
- * they are named, since hosts write them out. Every expression a host
- * translates is held to the budget: select values, group keys, WHERE,
- * aggregates and their filters, windows, calculations, HAVING and QUALIFY.
- * Sort and rank name values and add none. A window is a correlated subquery
- * that carries the row it counts over, so it also costs that row: WHERE and
- * every value the row projects (`columnCount` of them for a plain query with
- * no select).
- */
-function checkExpansion(check: QueryCheck, query: DatasetQuery, columnCount: number) {
-    // The source row: what WHERE, select values, group keys and aggregates read, each column once.
-    const source = new Map<string, number>();
-    const named = [...(query.select ?? []), ...query.groupBy].flatMap((entry) =>
-        typeof entry === "string" ? [] : [[entry.as, operandCost(entry.expr, source)] as const],
-    );
-    const where = query.filter.map((filter) => filterCost(filter, source));
-    const costs = [...named.map(([, total]) => total), ...where];
-    const expanded = new Map(named);
-    const aggregates = query.aggregate.map((entry) => {
-        const parts = [
-            ...(entry.col === undefined ? [] : [operandCost(entry.col, source)]),
-            ...entry.filter.map((filter) => filterCost(filter, source)),
-        ];
-        costs.push(...parts);
-        expanded.set(entry.as, nodeCost(parts));
-        return nodeCost(parts);
-    });
-    const grouped = query.groupBy.length > 0 || query.aggregate.length > 0;
-    const projected = grouped ? query.groupBy : (query.select ?? []);
-    const carried = Math.min(
-        DATASET_EXPANSION_BUDGET + 1,
-        [
-            ...where,
-            ...projected.map((entry) =>
-                typeof entry === "string" ? 1 : (expanded.get(entry.as) ?? 1),
-            ),
-            ...(grouped || projected.length ? [] : Array<number>(columnCount).fill(1)),
-            ...aggregates,
-        ].reduce((total, cost) => total + cost, 0),
-    );
-    // Every window reads the row as it was before any window, its partition and the row it carries included.
-    const windows = query.window.map((entry) => {
-        const total = nodeCost([
-            ...(entry.col === undefined ? [] : [operandCost(entry.col, expanded)]),
-            ...entry.filter.map((filter) => filterCost(filter, expanded)),
-            ...entry.partitionBy.map((name) => operandCost(name, expanded)),
-            carried,
-        ]);
-        costs.push(total);
-        return [entry.as, total] as const;
-    });
-    for (const [as, total] of windows) expanded.set(as, total);
-    for (const entry of query.compute) {
-        const total = operandCost(entry.expr, expanded);
-        expanded.set(entry.as, total);
-        costs.push(total);
-    }
-    for (const filter of query.having) costs.push(filterCost(filter, expanded));
-    // A grouped row is materialized before QUALIFY reads it; without group keys the
-    // .NET host writes the calculations QUALIFY names out in full.
-    const qualifying = query.groupBy.length ? source : expanded;
-    for (const filter of query.qualify) costs.push(filterCost(filter, qualifying));
-    if (costs.some((total) => total > DATASET_EXPANSION_BUDGET)) {
-        check.issue(
-            `Computed aliases expand to more than ${DATASET_EXPANSION_BUDGET} references; name intermediate results and reuse them less`,
-        );
-    }
-}
-
-/**
- * How many times a host writes one part out when it spells the node holding
- * it, the most either host does; every other part is written once. A divisor
- * is tested for zero before the division, ROUND's digits scale the value
- * before and after, and LENGTH's text is tested for NULL besides measured. A
- * date is read once per part a truncation or part needs (Zeller's congruence
- * for the week and weekday), and a truncation other than the week, packed as
- * a number, is read 3 more times when it meets a real date in a COALESCE, a
- * CASE or a comparison, so those factors are the product. A condition guards
- * its sides against NULL, and a negated condition with no opposite operator
- * (`between`, the text matches) tests its left side once more.
- */
-const EXPANSION_WEIGHTS = {
-    divisor: 2,
-    roundDigits: 2,
-    lengthText: 2,
-    conditionLeft: 3,
-    conditionRight: 2,
-    dateTrunc: { day: 9, month: 6, quarter: 6, year: 3, week: 17 },
-    datePart: { day: 1, month: 1, quarter: 1, year: 1, dayOfWeek: 16, week: 17 },
-} as const;
-
-/** One node plus its children's costs, held at one past the budget so no sum can overflow. */
-function nodeCost(children: number[]) {
-    return Math.min(
-        DATASET_EXPANSION_BUDGET + 1,
-        children.reduce((total, cost) => total + cost, 1),
-    );
-}
-
-/** How often a scalar function's argument is written out. */
-function argumentWeight(fn: DatasetFunction, index: number) {
-    if (fn === "length") return EXPANSION_WEIGHTS.lengthText;
-    return fn === "round" && index === 1 ? EXPANSION_WEIGHTS.roundDigits : 1;
-}
-
 /**
- * An operand's size written out: a literal or a column is 1, a reference to an
- * earlier computed alias is that alias's cost (a self or forward one, refused
- * elsewhere, is 1), and arithmetic, a date function, a scalar function, a CASE
- * and each of its branches are 1 plus their parts, each part times its weight.
+ * How many times the query names its computed aliases once each is written
+ * out where it is named, as SQL needs: every name counts once plus what the
+ * named value itself names, so ten values each naming the previous one ten
+ * times count past a billion. Worked out once per alias, capped just past
+ * the limit.
  */
-function operandCost(operand: DatasetOperand, expanded: ReadonlyMap<string, number>): number {
-    if (typeof operand === "string") return expanded.get(operand) ?? 1;
-    if (typeof operand === "number" || "value" in operand) return 1;
-    if ("op" in operand) {
-        const divisor = operand.op === "/" ? EXPANSION_WEIGHTS.divisor : 1;
-        return nodeCost([
-            operandCost(operand.left, expanded),
-            operandCost(operand.right, expanded) * divisor,
-        ]);
-    }
-    if ("case" in operand) {
-        return nodeCost([
-            ...operand.case.map((branch) =>
-                nodeCost([filterCost(branch.when, expanded), operandCost(branch.then, expanded)]),
-            ),
-            ...(operand.else === undefined ? [] : [operandCost(operand.else, expanded)]),
-        ]);
-    }
-    if ("args" in operand) {
-        return nodeCost(
-            operand.args.map(
-                (arg, index) => operandCost(arg, expanded) * argumentWeight(operand.fn, index),
-            ),
+function computeReferences(query: DatasetQuery) {
+    const written = new Map<string, number>();
+    const count = (refs: string[]) =>
+        Math.min(
+            refs.reduce((sum, ref) => {
+                const inner = written.get(ref);
+                return inner === undefined ? sum : sum + 1 + inner;
+            }, 0),
+            DATASET_COMPUTE_REFERENCES_MAX + 1,
         );
+    let total = 0;
+    for (const entry of query.compute) {
+        const own = count(operandRefs(entry.expr));
+        written.set(entry.as, own);
+        total += own;
     }
-    const weight =
-        operand.fn === "dateTrunc"
-            ? EXPANSION_WEIGHTS.dateTrunc[operand.unit]
-            : EXPANSION_WEIGHTS.datePart[operand.unit];
-    return nodeCost([operandCost(operand.of, expanded) * weight]);
-}
-
-/** A filter's size written out: 1 per condition or group plus its parts; a condition's left side is weighted, each literal on its right is 1, and a `{ col }` right side is its weighted operand cost. */
-function filterCost(filter: DatasetFilter, expanded: ReadonlyMap<string, number>): number {
-    if ("not" in filter) return nodeCost([filterCost(filter.not, expanded)]);
-    if ("any" in filter || "all" in filter) {
-        const entries = "any" in filter ? filter.any : filter.all;
-        return nodeCost(entries.map((entry) => filterCost(entry, expanded)));
-    }
-    const right = isColumnValue(filter.value)
-        ? operandCost(filter.value.col, expanded) * EXPANSION_WEIGHTS.conditionRight
-        : Array.isArray(filter.value)
-          ? filter.value.length
-          : filter.value === undefined
-            ? 0
-            : 1;
-    return nodeCost([operandCost(filter.col, expanded) * EXPANSION_WEIGHTS.conditionLeft, right]);
+    const rowNames = [
+        ...[...query.having, ...query.qualify].flatMap(filterRefs),
+        ...[...query.sort, ...(query.rank?.sort ?? [])].map((entry) => entry.col),
+        ...(query.rank?.partitionBy ?? []),
+    ];
+    return total + count(rowNames);
 }
 
 /**
@@ -1341,7 +1243,7 @@ function filterValueTypeIssue(type: DatasetColumnType, filter: DatasetCondition)
                 return (
                     typeof value === "number" ||
                     (typeof value === "string" &&
-                        value.trim() !== "" &&
+                        JSON_NUMBER_PATTERN.test(value) &&
                         Number.isFinite(Number(value)))
                 );
             case "date":

package/src/contracts/egress-policy/scrub.ts

@@ -1,4 +1,4 @@
-import { isRecord } from "../json";
+import { isRecord, JSON_NUMBER_PATTERN } from "../json";
 
 const REDACTION_MARKER = "[REDACTED]";
 
@@ -34,7 +34,9 @@ export function credentialForms(value: string) {
  * transcript alike, then the credentials the request presented are
  * redacted. Dropping first, on the patterns as well, means a credential
  * an upstream spelled with U+0000 inside it still matches; redacting first
- * would let the strip reassemble it.
+ * would let the strip reassemble it. A number is matched whole, by value,
+ * against each credential read as a number, so a digit credential echoed
+ * as a JSON number is redacted even when parsing rounded it.
  */
 export function scrubCredentialStrings(value: unknown, credentials: string[]): unknown {
     const dropU0000 = (text: string) => text.replaceAll("\u0000", "");
@@ -51,6 +53,13 @@ export function scrubCredentialStrings(value: unknown, credentials: string[]): u
         );
 
     if (typeof value === "string") return scrub(value);
+    if (typeof value === "number") {
+        return ordered.some(
+            (credential) => JSON_NUMBER_PATTERN.test(credential) && Number(credential) === value,
+        )
+            ? REDACTION_MARKER
+            : value;
+    }
     if (Array.isArray(value)) {
         return value.map((item) => scrubCredentialStrings(item, credentials));
     }

package/src/contracts/errors/index.ts

@@ -36,3 +36,10 @@ export const backendErrorSchema = z.object({
     }),
 });
 export type BackendError = z.infer<typeof backendErrorSchema>;
+
+/** The envelope a backend answers a refusal in, its detail cut to the length Cortex accepts. */
+export function errorEnvelope(kind: BackendErrorKind, detail: string) {
+    return {
+        error: { kind, detail: detail.slice(0, BACKEND_ERROR_DETAIL_MAX) },
+    } satisfies BackendError;
+}

package/src/contracts/json/index.ts

@@ -21,27 +21,50 @@ export function escapePointerToken(token: string) {
     return token.replaceAll("~", "~0").replaceAll("/", "~1");
 }
 
-export function unescapePointerToken(token: string) {
+function unescapePointerToken(token: string) {
     return token.replaceAll("~1", "/").replaceAll("~0", "~");
 }
 
-/** An RFC 6901 array index: digits with no leading zero. */
-export const ARRAY_INDEX = /^(0|[1-9]\d*)$/;
-
-/** RFC 6901 lookup; `undefined` when the pointer names nothing (an inherited key, or an array index not written canonically, included). */
-export function readJsonPointer(value: unknown, pointer: string) {
-    if (pointer === "") return value;
-    let current: unknown = value;
-    for (const raw of pointer.slice(1).split("/")) {
-        const key = unescapePointerToken(raw);
-        if (Array.isArray(current)) {
-            if (!ARRAY_INDEX.test(key)) return undefined;
-            current = current[Number(key)];
-        } else if (isRecord(current) && Object.hasOwn(current, key)) {
-            current = current[key];
-        } else {
+/**
+ * The reference tokens of a JSON Pointer, bare (`/a/b`) or as a URI fragment (`#/a/b`, which
+ * is percent-decoded before it splits, as RFC 6901 section 6 has it); `[]` names the whole
+ * document. Undefined for text that is neither, or a fragment whose percent-encoding is
+ * malformed.
+ */
+export function pointerTokens(pointer: string) {
+    let path = pointer;
+    if (pointer.startsWith("#")) {
+        try {
+            path = decodeURIComponent(pointer.slice(1));
+        } catch {
             return undefined;
         }
     }
-    return current;
+    if (path === "") return [];
+    if (!path.startsWith("/")) return undefined;
+    return path.slice(1).split("/").map(unescapePointerToken);
+}
+
+/** A JSON Pointer token that names an array index: digits with no leading zero. */
+export const ARRAY_INDEX = /^(?:0|[1-9]\d*)$/;
+
+/** A number spelled as JSON writes one: decimal, in ASCII digits so every regex engine reads the pattern alike. */
+export const JSON_NUMBER_PATTERN = /^-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?$/;
+
+/**
+ * The value one reference token names under `container`: an own key of an object, or an
+ * `ARRAY_INDEX` of an array; undefined when there is none, so an inherited key such as
+ * `constructor` never resolves.
+ */
+export function pointerChild(container: unknown, token: string) {
+    if (Array.isArray(container)) {
+        const items: unknown[] = container;
+        return ARRAY_INDEX.test(token) ? items[Number(token)] : undefined;
+    }
+    return isRecord(container) && Object.hasOwn(container, token) ? container[token] : undefined;
+}
+
+/** The value a JSON Pointer names in `document`, one `pointerChild` per token; undefined when it names nothing. */
+export function readJsonPointer(document: unknown, pointer: string) {
+    return pointerTokens(pointer)?.reduce<unknown>(pointerChild, document);
 }