@m6d/cortex-cli 0.13.0 → 0.15.0

package/src/contracts/datasets/datasets.ts

@@ -25,11 +25,22 @@ export const datasetColumnRefSchema = z.string().regex(DATASET_COLUMN_REF_PATTER
 export const DATASET_COLUMN_TYPES = ["string", "number", "boolean", "date"] as const;
 export type DatasetColumnType = (typeof DATASET_COLUMN_TYPES)[number];
 
-export const datasetColumnSchema = z.object({
-    name: datasetIdentifierSchema,
-    type: z.enum(DATASET_COLUMN_TYPES),
-    description: z.string().max(200).default(""),
-});
+/**
+ * One column. `values`, on a string column, lists every value it can hold:
+ * the agent reads them, and a query comparing the column to anything else is
+ * refused before it runs.
+ */
+export const datasetColumnSchema = z
+    .object({
+        name: datasetIdentifierSchema,
+        type: z.enum(DATASET_COLUMN_TYPES),
+        description: z.string().max(200).default(""),
+        values: z.array(z.string().min(1).max(200)).min(1).max(100).optional(),
+    })
+    .refine((column) => column.values === undefined || column.type === "string", {
+        message: "only a string column lists its values",
+        path: ["values"],
+    });
 export type DatasetColumn = z.infer<typeof datasetColumnSchema>;
 
 /**
@@ -396,10 +407,59 @@ export function resolveDatasetColumn(catalog: DatasetCatalog, from: string, ref:
         dataset = target;
     }
     const column = dataset.columns.find((candidate) => candidate.name === columnName);
-    if (!column) return missing(`"${dataset.name}" has no column "${columnName}"`);
+    if (!column) return missing(unknownColumnError(dataset, columnName));
     return { ok: true as const, column, path: segments.join(".") };
 }
 
+/**
+ * Why `dataset` has no column `name`, with the columns it has and up to
+ * three closest ones: names containing it or contained in it (ignoring
+ * case, and only for names longer than two characters), else names two
+ * edits or fewer away. A query that guessed `name` reads `departments has
+ * no column "name". Closest: nameAr, nameEn. Columns: id, nameAr, ...`
+ */
+export function unknownColumnError(dataset: DatasetDefinition, name: string) {
+    const columns = dataset.columns.map((column) => column.name);
+    const wanted = name.toLowerCase();
+    const containing = columns.filter((column) => {
+        const lower = column.toLowerCase();
+        return (
+            (wanted.length > 2 && lower.includes(wanted)) ||
+            (lower.length > 2 && wanted.includes(lower))
+        );
+    });
+    const closest = (
+        containing.length > 0
+            ? containing
+            : // Lengths more than two apart are more than two edits apart, so a
+              // name of any length costs only its own columns' comparisons.
+              columns.filter(
+                  (column) =>
+                      Math.abs(column.length - wanted.length) <= 2 &&
+                      editDistance(column.toLowerCase(), wanted) <= 2,
+              )
+    ).slice(0, 3);
+    const hint = closest.length > 0 ? ` Closest: ${closest.join(", ")}.` : "";
+    return `${dataset.name} has no column "${name}".${hint} Columns: ${columns.join(", ")}.`;
+}
+
+/** Levenshtein distance: the fewest single-character edits from `a` to `b`. */
+function editDistance(a: string, b: string) {
+    let previous = Array.from({ length: b.length + 1 }, (_, index) => index);
+    for (let i = 1; i <= a.length; i++) {
+        const current = [i];
+        for (let j = 1; j <= b.length; j++) {
+            current[j] = Math.min(
+                previous[j]! + 1,
+                current[j - 1]! + 1,
+                previous[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
+            );
+        }
+        previous = current;
+    }
+    return previous[b.length]!;
+}
+
 function refError(kind: string, ref: string, reason: string) {
     return `${kind} "${ref}": ${reason}`;
 }
@@ -654,6 +714,19 @@ export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuer
     } satisfies QueryCheck;
 
     for (const filter of query.filter) checkFilter(check, filter);
+    const row = checkAnsweredRow(check, catalog, query);
+    checkFinishedRow(check, query, row);
+    checkExpansion(check, query, catalog.get(query.from)?.columns.length ?? 0);
+    return issues;
+}
+
+/**
+ * The row a query answers, checked as it is built: on a grouped query its
+ * keys and aggregate aliases (never one name for both, and no select), on a
+ * plain one its select aliases; each with the names that are a column as it
+ * is, which keep what the column says.
+ */
+function checkAnsweredRow(check: QueryCheck, catalog: DatasetCatalog, query: DatasetQuery) {
     const groupKeys = checkGroupBy(check, query.groupBy);
     const aliases = checkAggregates(check, query.aggregate);
     for (const alias of aliases.keys()) {
@@ -671,12 +744,17 @@ export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuer
         );
     }
     const selectAliases = checkSelect(check, query.select ?? []);
-    const row = grouped
-        ? { names: new Map([...groupKeys, ...aliases]), kind: "grouped" as const }
-        : { names: selectAliases, kind: "plain" as const };
-    checkFinishedRow(check, query, row);
-    checkExpansion(check, query, catalog.get(query.from)?.columns.length ?? 0);
-    return issues;
+    return grouped
+        ? {
+              names: new Map([...groupKeys, ...aliases]),
+              kind: "grouped" as const,
+              columns: passedColumns(catalog, query, query.groupBy),
+          }
+        : {
+              names: selectAliases,
+              kind: "plain" as const,
+              columns: passedColumns(catalog, query, query.select ?? []),
+          };
 }
 
 /** How large any one expression a host translates may grow once every alias it names is written out in place. */
@@ -856,7 +934,7 @@ function filterCost(filter: DatasetFilter, expanded: ReadonlyMap<string, number>
 function checkFinishedRow(
     check: QueryCheck,
     query: DatasetQuery,
-    row: { names: Map<string, DatasetColumnType>; kind: "grouped" | "plain" },
+    row: RowNames & { names: Map<string, DatasetColumnType> },
 ) {
     const rowCheck = rowScope(check, row);
     // On a plain row a selected column is a name too; a window or calculation may not take it.
@@ -881,7 +959,7 @@ function rowScope(check: QueryCheck, row: RowNames) {
         issue: (message: string) => check.issue(message),
         columnOf(kind, ref) {
             const type = row.names.get(ref);
-            if (type) return { name: ref, type, description: "" };
+            if (type) return row.columns.get(ref) ?? { name: ref, type, description: "" };
             if (row.kind === "plain") return check.columnOf(kind, ref);
             check.issue(
                 refError(
@@ -1203,10 +1281,31 @@ function checkFilter(check: QueryCheck, filter: DatasetFilter, kind = "filter")
         ? columnValueIssue(check, kind, type, filter.op, filter.value.col)
         : (filterValueIssue(filter) ??
           filterValueTypeIssue(type, filter) ??
-          truncatedDateIssue(filter));
+          truncatedDateIssue(filter) ??
+          listedValueIssue(check, kind, filter));
     if (issue) check.issue(refError(kind, shown, issue));
 }
 
+const LISTED_VALUE_OPS = new Set<DatasetFilterOp>(["=", "!=", "in", "notIn"]);
+
+/**
+ * A column that lists its values is only compared with one of them: any
+ * other literal would match no row, or every row, and the host would answer
+ * as if the data said so.
+ */
+function listedValueIssue(check: QueryCheck, kind: string, filter: DatasetCondition) {
+    if (typeof filter.col !== "string" || !LISTED_VALUE_OPS.has(filter.op)) return undefined;
+    if (filter.value === undefined || isColumnValue(filter.value)) return undefined;
+    const listed = check.columnOf(kind, filter.col)?.values;
+    if (!listed) return undefined;
+    const values = Array.isArray(filter.value) ? filter.value : [filter.value];
+    // A number or a boolean is compared as the text the host turns it into.
+    const odd = values.find((value) => !listed.includes(String(value)));
+    return odd === undefined
+        ? undefined
+        : `${JSON.stringify(odd)} is not one of its values: ${listed.join(", ")}`;
+}
+
 /** A truncated date is a whole day, so a literal against it is one too ("2024-03-01"); a time or a bare month would never match. */
 function truncatedDateIssue(filter: DatasetCondition) {
     const truncated =
@@ -1314,7 +1413,31 @@ function aggregateType(
 }
 
 /** What sort and rank may name: on a grouped query a group key or an alias; on a plain one a select alias or any reachable column. */
-type RowNames = { names: ReadonlyMap<string, DatasetColumnType>; kind: "grouped" | "plain" };
+type RowNames = {
+    names: ReadonlyMap<string, DatasetColumnType>;
+    kind: "grouped" | "plain";
+    /** The names that are a column as it is (group keys, select entries), so what the column says (its values) still holds. */
+    columns: ReadonlyMap<string, DatasetColumn>;
+};
+
+/**
+ * The row names that are a column as it is, each with the column: a bare
+ * column (`status`), or one under an alias (`{ expr: "status", as: "state" }`).
+ */
+function passedColumns(
+    catalog: DatasetCatalog,
+    query: DatasetQuery,
+    entries: ReadonlyArray<string | { expr: DatasetOperand; as: string }>,
+) {
+    return new Map(
+        entries.flatMap((entry) => {
+            const [name, ref] = typeof entry === "string" ? [entry, entry] : [entry.as, entry.expr];
+            if (typeof ref !== "string") return [];
+            const resolved = resolveDatasetColumn(catalog, query.from, ref);
+            return resolved.ok ? [[name, resolved.column] as const] : [];
+        }),
+    );
+}
 
 function checkSort(check: QueryCheck, sort: DatasetQuery["sort"], kind: string, row: RowNames) {
     for (const { col } of sort) checkRowName(check, kind, col, row);

package/src/contracts/egress-policy/scrub.ts

@@ -7,19 +7,22 @@ const REDACTION_MARKER = "[REDACTED]";
  * itself and, after a leading scheme word (`Bearer xyz`, `Basic abc`), the
  * bare secret the scheme merely labels; each of those raw as in a header,
  * percent-encoded as in a path slot (`encodeURIComponent`) or a query value
- * (`URLSearchParams`, which spells a space as `+`), and JSON-escaped as inside
- * a request body. The scrubber replaces longer forms first.
+ * (`URLSearchParams`, which spells a space as `+`), JSON-escaped as inside
+ * a request body, and JSON-escaped then percent-encoded as a form field sent
+ * as JSON text is. The scrubber replaces longer forms first.
  */
 export function credentialForms(value: string) {
     const bare = /^[A-Za-z][\w.-]* +(\S.*)$/.exec(value)?.[1];
+    const percentEncoded = (form: string) => [
+        encodeURIComponent(form),
+        new URLSearchParams({ v: form }).toString().slice(2),
+    ];
     return [
         ...new Set(
-            (bare ? [value, bare] : [value]).flatMap((form) => [
-                form,
-                encodeURIComponent(form),
-                new URLSearchParams({ v: form }).toString().slice(2),
-                JSON.stringify(form).slice(1, -1),
-            ]),
+            (bare ? [value, bare] : [value]).flatMap((form) => {
+                const json = JSON.stringify(form).slice(1, -1);
+                return [form, ...percentEncoded(form), json, ...percentEncoded(json)];
+            }),
         ),
     ];
 }

package/src/contracts/json/index.ts

@@ -24,3 +24,24 @@ export function escapePointerToken(token: string) {
 export function unescapePointerToken(token: string) {
     return token.replaceAll("~1", "/").replaceAll("~0", "~");
 }
+
+/** An RFC 6901 array index: digits with no leading zero. */
+export const ARRAY_INDEX = /^(0|[1-9]\d*)$/;
+
+/** RFC 6901 lookup; `undefined` when the pointer names nothing (an inherited key, or an array index not written canonically, included). */
+export function readJsonPointer(value: unknown, pointer: string) {
+    if (pointer === "") return value;
+    let current: unknown = value;
+    for (const raw of pointer.slice(1).split("/")) {
+        const key = unescapePointerToken(raw);
+        if (Array.isArray(current)) {
+            if (!ARRAY_INDEX.test(key)) return undefined;
+            current = current[Number(key)];
+        } else if (isRecord(current) && Object.hasOwn(current, key)) {
+            current = current[key];
+        } else {
+            return undefined;
+        }
+    }
+    return current;
+}