@m6d/cortex-cli 0.17.0 → 0.18.0

package/README.md

@@ -225,6 +225,39 @@ or any name such a collision decided (`ambiguous_tool`), since a path the
 document adds or drops could move it to another endpoint. It also fails when an
 operation's name is already a listed tool's or another source's.
 
+A collection lists its documents. Each is inline `text`, a file `path`
+relative to the manifest file, or a `url` Cortex reads on every apply, with no
+credentials:
+
+```ts
+collections: [
+    {
+        name: "Policies",
+        audience: { agents: ["support"] },
+        documents: [
+            { name: "faq.md", text: "# FAQ\n\nReturns take 30 days." },
+            { path: "./docs/returns.pdf" },
+            { url: "https://shop.example.com/policies/shipping.html" },
+        ],
+        // overridable: ["documents"],
+    },
+],
+```
+
+A document's name sets its type: `.md`, `.txt`, `.html`, `.htm`, `.pdf` or
+`.docx`. A path or URL document is named after its file unless `name` says
+otherwise. Each is at most 10 MB. `cortex apply` reads paths and sends their
+bytes, so a request carries every inline and path document again; Cortex
+stores only what changed. A document with new bytes replaces the old one,
+which keeps answering until the new one is processed, and one the manifest
+drops is retired.
+
+Listing `documents` makes the manifest own them: an apply retires a document
+uploaded in the console, the first apply after upgrading included, unless
+the collection offers `documents` in `overridable`. Offered, a document
+uploaded or removed in Cortex stays so until the manifest changes that same
+document, which `cortex apply` then asks about.
+
 The manifest is checked against the contract before anything is sent, then
 sent as written, so Cortex fills its own defaults. It goes to the
 management API (`manifests/sync`, or `manifests/preview` with `--dry-run`,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@m6d/cortex-cli",
-    "version": "0.17.0",
+    "version": "0.18.0",
     "description": "Scaffold Cortex projects and manage a Cortex deployment",
     "license": "MIT",
     "type": "module",

package/src/commands/apply.ts

@@ -202,9 +202,14 @@ async function sendManifest(options: {
     }
 }
 
+/** The field both sides changed, and its part when the field is judged part by part. */
+function conflictField({ field, key }: Conflict) {
+    return key === undefined ? field : `${field} (${key})`;
+}
+
 /** How a conflict reads: its entry and the field both sides changed. */
-function conflictLabel({ kind, name, field }: Conflict) {
-    return `${kind} ${name}: ${field} changed in both places`;
+function conflictLabel(conflict: Conflict) {
+    return `${conflict.kind} ${conflict.name}: ${conflictField(conflict)} changed in both places`;
 }
 
 /** Ends a run that does not ask, naming every conflict and how to run `command` again to answer them. */
@@ -250,8 +255,9 @@ export function findManifest(path: string, command?: string) {
 
 /**
  * The file's default export, checked against the manifest contract here so
- * a mistake is named before anything is sent, then sent as written: Cortex
- * fills its own defaults, which may be newer than this CLI's.
+ * a mistake is named before anything is sent, then sent as written, each
+ * document `path` read from beside the file: Cortex fills its own defaults,
+ * which may be newer than this CLI's.
  */
 export async function loadManifest(path: string, verbose: boolean) {
     const { written, parsed } = await loadManifestFile(path).catch((error: unknown) =>
@@ -288,8 +294,12 @@ function printChanges(
             rows.set(kind, [...(rows.get(kind) ?? []), ...marked]);
         }
     }
-    for (const { kind, name, field } of conflicts) {
-        rows.set(kind, [...(rows.get(kind) ?? []), `! ${name}  ${field} changed in both places`]);
+    for (const conflict of conflicts) {
+        const { kind, name } = conflict;
+        rows.set(kind, [
+            ...(rows.get(kind) ?? []),
+            `! ${name}  ${conflictField(conflict)} changed in both places`,
+        ]);
     }
     for (const [kind, lines] of rows) {
         for (const [index, line] of lines.entries()) {

package/src/contracts/datasets/dataset-sql.ts

@@ -57,8 +57,13 @@ type ParsedDatasetSql = {
 /** A refusal the model can act on. */
 export class DatasetSqlError extends Error {}
 
+/**
+ * `upper` is a word's keyword spelling. A quoted identifier and a word after a
+ * dot (`plans.from`) have none (`""`), so no keyword check matches them and
+ * they always read as names.
+ */
 type Token =
-    | { kind: "word"; value: string; upper: string; quoted?: true }
+    | { kind: "word"; value: string; upper: string }
     | { kind: "string"; value: string }
     | { kind: "number"; value: number }
     | { kind: "punct"; value: string }
@@ -84,6 +89,14 @@ function stringLiteral(sql: string, start: number) {
     }
 }
 
+/** Whether `previous` ends a value, so a minus after it subtracts; after a keyword or an operator it signs the number. */
+function endsValue(previous: Token | undefined) {
+    if (previous === undefined) return false;
+    if (previous.kind === "punct") return previous.value === ")";
+    if (previous.kind !== "word") return true;
+    return ["END", "NULL", "TRUE", "FALSE"].includes(previous.upper) || isName(previous);
+}
+
 /** Reads the token at `i`: a quoted identifier, a number (negative only where a value is expected), a word or punctuation. */
 function lexeme(sql: string, i: number, previous: Token | undefined) {
     const char = sql[i]!;
@@ -92,27 +105,13 @@ function lexeme(sql: string, i: number, previous: Token | undefined) {
         if (close === -1) throw new DatasetSqlError("A quoted identifier is not closed.");
         const value = sql.slice(i + 1, close);
         return {
-            token: {
-                kind: "word",
-                value,
-                upper: value.toUpperCase(),
-                quoted: true,
-            } satisfies Token,
+            token: { kind: "word", value, upper: "" } satisfies Token,
             next: close + 1,
         };
     }
     const rest = sql.slice(i);
     const number = /^-?\d+(\.\d+)?/.exec(rest);
-    // After a value or a closing parenthesis a minus subtracts; after a keyword or an operator it signs the number.
-    const afterValue =
-        previous !== undefined &&
-        (previous.kind === "punct"
-            ? previous.value === ")"
-            : previous.kind === "word"
-              ? ["END", "NULL", "TRUE", "FALSE"].includes(previous.upper) || isName(previous)
-              : true);
-    const signIsOperator = char === "-" && afterValue;
-    if (number && !signIsOperator) {
+    if (number && !(char === "-" && endsValue(previous))) {
         return {
             token: { kind: "number", value: Number(number[0]) } satisfies Token,
             next: i + number[0].length,
@@ -120,8 +119,13 @@ function lexeme(sql: string, i: number, previous: Token | undefined) {
     }
     const word = /^[A-Za-z_][A-Za-z0-9_]*/.exec(rest);
     if (word) {
+        const qualified = previous?.kind === "punct" && previous.value === ".";
         return {
-            token: { kind: "word", value: word[0], upper: word[0].toUpperCase() } satisfies Token,
+            token: {
+                kind: "word",
+                value: word[0],
+                upper: qualified ? "" : word[0].toUpperCase(),
+            } satisfies Token,
             next: i + word[0].length,
         };
     }
@@ -338,12 +342,35 @@ const RESERVED = [
     "END",
 ];
 
+/** Whether a bare word reads as a keyword rather than a name. */
+function isKeyword(upper: string) {
+    return RESERVED.includes(upper) || upper in REFUSED_CLAUSES;
+}
+
 function isName(token: Token): token is Token & { kind: "word" } {
-    if (token.kind !== "word") return false;
-    return (
-        token.quoted === true ||
-        (!RESERVED.includes(token.upper) && !(token.upper in REFUSED_CLAUSES))
-    );
+    return token.kind === "word" && !isKeyword(token.upper);
+}
+
+/** Words that bare read as a value rather than a column. */
+const VALUE_WORDS = ["TRUE", "FALSE", "CURRENT_DATE", "CURRENT_TIMESTAMP"];
+
+/**
+ * A column name as a query writes it: double quoted when bare it would read
+ * as a keyword or a value (`"from"`), so what the model is shown is what it
+ * can send.
+ */
+export function sqlColumnName(name: string) {
+    const upper = name.toUpperCase();
+    return isKeyword(upper) || VALUE_WORDS.includes(upper) ? `"${name}"` : name;
+}
+
+/** `message`, and for a keyword where a column belongs, how to write a column spelled like it. */
+function keywordError(token: Token, message: string) {
+    const hint =
+        token.kind === "word" && isKeyword(token.upper)
+            ? ` ${token.value} is a SQL keyword: write a column of that name as "${token.value}" or dataset.${token.value}.`
+            : "";
+    return new DatasetSqlError(`${message}${hint}`);
 }
 
 /** A unit written as `'month'`, `month` or `MONTH`. */
@@ -971,11 +998,21 @@ class Parser {
             );
         }
         const op = (operator.value === "<>" ? "!=" : operator.value) as DatasetFilterOp;
+        const message = `"${shown} ${operator.value}" needs a value on the right.`;
         if (this.startsTerm()) {
-            return { kind: "cond", left, op, value: { term: this.term("") } } satisfies Predicate;
+            return {
+                kind: "cond",
+                left,
+                op,
+                value: { term: this.term(message) },
+            } satisfies Predicate;
+        }
+        // A keyword here may be a column spelled like one (`x > from`); NULL keeps its own refusal.
+        const next = this.peek();
+        if (next.kind === "word" && next.upper !== "NULL" && isKeyword(next.upper)) {
+            throw keywordError(next, message);
         }
-        const value = this.literal(`"${shown} ${operator.value}" needs a value on the right.`);
-        return { kind: "cond", left, op, value } satisfies Predicate;
+        return { kind: "cond", left, op, value: this.literal(message) } satisfies Predicate;
     }
 
     /** Whether the next tokens are a value expression rather than a literal: a name, `(`, CASE, or a number followed by an operator. */
@@ -1174,10 +1211,10 @@ class Parser {
         return count.value;
     }
 
-    /** `a.b` as one reference; a bare keyword in reference position gets the caller's message. */
+    /** `a.b` as one reference; a bare keyword in reference position gets the caller's message (`keywordError`). */
     private reference(message: string) {
         const token = this.peek();
-        if (!isName(token)) throw new DatasetSqlError(message);
+        if (!isName(token)) throw keywordError(token, message);
         this.take();
         let ref = token.value;
         while (this.optionalPunct(".")) {

package/src/contracts/manifest/index.ts

@@ -1243,22 +1243,163 @@ const manifestWebhookSchema = z.strictObject({
 });
 export type ManifestWebhook = z.input<typeof manifestWebhookSchema>;
 
-/** A collection's name and description in characters. The console caps its editor at the same. */
-export const COLLECTION_LIMITS = { nameLength: 80, descriptionLength: 2_000 } as const;
-
-/** Holds documents, which are uploaded to it apart from the manifest. */
-const manifestCollectionSchema = z.strictObject({
-    name: z.string().trim().min(1).max(COLLECTION_LIMITS.nameLength),
-    description: z.string().trim().max(COLLECTION_LIMITS.descriptionLength).default(""),
-    /** The key of the service every document in the collection belongs to. */
-    service: slugSchema.optional(),
-    /** Whether answers may name its documents; false keeps their names, sections and pages from the model. */
-    citable: z.boolean().default(true),
-    audience: audienceSchema,
-    overridable: overridable(["description"]),
+/**
+ * A collection's name and description in characters, and how many documents
+ * a manifest lists in one. The console caps its editor at the same.
+ */
+export const COLLECTION_LIMITS = {
+    nameLength: 80,
+    descriptionLength: 2_000,
+    documents: 100,
+} as const;
+
+/** A knowledge document's name in characters and its size in bytes, however it arrives. */
+export const DOCUMENT_LIMITS = { nameLength: 255, bytes: 10 * 1024 * 1024 } as const;
+
+export const DOCX_MIME_TYPE =
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+
+/** The content types Cortex parses a knowledge document as. */
+export const DOCUMENT_MIME_TYPES = [
+    "application/pdf",
+    DOCX_MIME_TYPE,
+    "text/markdown",
+    "text/plain",
+    "text/html",
+] as const;
+export type DocumentMimeType = (typeof DOCUMENT_MIME_TYPES)[number];
+
+const mimeTypeByExtension = new Map<string, DocumentMimeType>([
+    ["md", "text/markdown"],
+    ["txt", "text/plain"],
+    ["html", "text/html"],
+    ["htm", "text/html"],
+    ["pdf", "application/pdf"],
+    ["docx", DOCX_MIME_TYPE],
+]);
+
+/** The content type a document is parsed as, from the extension its name ends in; none for any other name. */
+export function documentType(name: string) {
+    const dot = name.lastIndexOf(".");
+    return dot === -1 ? undefined : mimeTypeByExtension.get(name.slice(dot + 1).toLowerCase());
+}
+
+/** How many bytes a base64 string decodes to. */
+function base64Bytes(data: string) {
+    const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0;
+    return (data.length / 4) * 3 - padding;
+}
+
+const documentNameSchema = z.string().trim().min(1).max(DOCUMENT_LIMITS.nameLength);
+
+/** A document written in the manifest itself; its name's extension says how it is parsed. */
+const textDocumentSchema = z.strictObject({
+    name: documentNameSchema,
+    text: z
+        .string()
+        .min(1)
+        .refine(
+            (text) => new TextEncoder().encode(text).byteLength <= DOCUMENT_LIMITS.bytes,
+            `larger than ${DOCUMENT_LIMITS.bytes} bytes`,
+        ),
 });
+
+/** A file `cortex apply` reads, its path relative to the manifest file; named after the file unless `name` says otherwise. */
+const pathDocumentSchema = z.strictObject({
+    path: z.string().trim().min(1),
+    name: documentNameSchema.optional(),
+});
+
+/** A file Cortex reads on every apply, with no credentials; named after the URL's last segment unless `name` says otherwise. */
+const urlDocumentSchema = z.strictObject({
+    url: upstreamUrlSchema,
+    name: documentNameSchema.optional(),
+});
+
+/** A file's bytes in base64, as `cortex apply` sends a `path`. */
+const dataDocumentSchema = z.strictObject({
+    name: documentNameSchema,
+    data: z
+        .base64()
+        .refine(
+            (data) => base64Bytes(data) <= DOCUMENT_LIMITS.bytes,
+            `larger than ${DOCUMENT_LIMITS.bytes} bytes`,
+        ),
+});
+
+/** The name a document goes by: its own, else its file's or its URL's last segment. */
+export function documentName(document: { name?: string; path?: string; url?: string }) {
+    if (document.name !== undefined) return document.name;
+    if (document.path !== undefined) return document.path.split(/[/\\]/).pop() ?? "";
+    const segment = new URL(document.url ?? "", "https://x").pathname.split("/").pop() ?? "";
+    try {
+        return decodeURIComponent(segment);
+    } catch {
+        return segment;
+    }
+}
+
+/** A collection's documents: each name once, and each a name whose extension Cortex parses. */
+function documentsSchema<TDocument extends z.ZodType<Parameters<typeof documentName>[0]>>(
+    document: TDocument,
+) {
+    return z
+        .array(document)
+        .max(COLLECTION_LIMITS.documents)
+        .superRefine((documents, ctx) => {
+            const names = documents.map(documentName);
+            names.forEach((name, index) => {
+                if (documentType(name) === undefined) {
+                    ctx.addIssue({
+                        code: "custom",
+                        path: [index, "name"],
+                        message: "a document's name ends in .md, .txt, .html, .htm, .pdf or .docx",
+                    });
+                } else if (names.indexOf(name) !== index) {
+                    ctx.addIssue({
+                        code: "custom",
+                        path: [index, "name"],
+                        message: "a name repeats",
+                    });
+                }
+            });
+        })
+        .default([]);
+}
+
+/**
+ * Holds documents. Its `documents` are the manifest's: an apply stores what
+ * is new or changed and retires what it no longer lists, unless
+ * `overridable` offers them, and then a document added or removed in
+ * Cortex stays so until the manifest changes that document too.
+ */
+function collectionSchema<TDocument extends z.ZodType<Parameters<typeof documentName>[0]>>(
+    document: TDocument,
+) {
+    return z.strictObject({
+        name: z.string().trim().min(1).max(COLLECTION_LIMITS.nameLength),
+        description: z.string().trim().max(COLLECTION_LIMITS.descriptionLength).default(""),
+        /** The key of the service every document in the collection belongs to. */
+        service: slugSchema.optional(),
+        /** Whether answers may name its documents; false keeps their names, sections and pages from the model. */
+        citable: z.boolean().default(true),
+        documents: documentsSchema(document),
+        audience: audienceSchema,
+        overridable: overridable(["description", "documents"]),
+    });
+}
+
+const manifestDocumentSchema = z.union([textDocumentSchema, pathDocumentSchema, urlDocumentSchema]);
+export type ManifestDocument = z.input<typeof manifestDocumentSchema>;
+
+const manifestCollectionSchema = collectionSchema(manifestDocumentSchema);
 export type ManifestCollection = z.input<typeof manifestCollectionSchema>;
 
+/** A collection as Cortex receives it: each `path` read into `data`. */
+const resolvedCollectionSchema = collectionSchema(
+    z.union([textDocumentSchema, dataDocumentSchema, urlDocumentSchema]),
+);
+
 export const AUTH_JWT_SECRET_ALGORITHMS = ["HS256", "HS384", "HS512"] as const;
 
 const issuerSchema = z.string().trim().min(1).max(2048);
@@ -1477,6 +1618,15 @@ export const manifestSchema = z.strictObject({
 /** A manifest as written: every field with a default may be left out. */
 export type Manifest = z.input<typeof manifestSchema>;
 
+/**
+ * A manifest as Cortex receives it: the one written, each document `path`
+ * read into `data` by `cortex apply`, since Cortex cannot read the files
+ * beside it.
+ */
+export const resolvedManifestSchema = manifestSchema.extend({
+    collections: listOf(MANIFEST_NAME_FIELDS.collections, resolvedCollectionSchema),
+});
+
 const manifestKindSchema = z.enum(MANIFEST_KINDS);
 
 /**
@@ -1492,11 +1642,17 @@ export const manifestChangesSchema = z.object({
     delete: namesByKindSchema,
 });
 
-/** A field of one entry that both the manifest and a live write changed since the last apply. */
+/**
+ * A field of one entry that both the manifest and a live write changed since
+ * the last apply. A field holding named parts (an agent's settings, its
+ * greeting and suggestions by locale, a collection's documents by name) is
+ * judged part by part, and `key` names the part.
+ */
 const manifestConflictSchema = z.strictObject({
     kind: manifestKindSchema,
     name: z.string(),
     field: z.string(),
+    key: z.string().optional(),
 });
 
 /** Which value a conflict keeps: the one changed live in Cortex, or the manifest's. */
@@ -1506,9 +1662,10 @@ const conflictSideSchema = z.enum(["live", "manifest"]);
  * What preview and sync take: the manifest, and how its conflicts resolve.
  * A conflict takes its own entry in `resolutions`, else `prefer`; a sync
  * left with one neither answers is refused as `unresolved_conflicts`,
- * naming them. A resolution for a field that no longer conflicts is ignored.
+ * naming them. A resolution answers the conflict with its kind, name, field
+ * and key; one for a field that no longer conflicts is ignored.
  */
-export const manifestApplyInputSchema = manifestSchema.extend({
+export const manifestApplyInputSchema = resolvedManifestSchema.extend({
     prefer: conflictSideSchema.optional(),
     resolutions: z.array(manifestConflictSchema.extend({ keep: conflictSideSchema })).default([]),
 });

package/src/contracts/manifest-file/index.ts

@@ -1,16 +1,68 @@
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
-import { manifestSchema } from "../manifest/index";
+import { isRecord } from "../json/index";
+import { DOCUMENT_LIMITS, documentName, manifestSchema } from "../manifest/index";
 
 /**
  * Imports a manifest file, TypeScript or JSON (Bun imports both), and checks
- * its default export against the contract: `written` is the export as
- * written, for a caller that lets Cortex fill its own defaults, and
- * `parsed` the check. Throws when the file does not import or exports
- * nothing. Kept out of the contract's module, which the browser imports too.
+ * its default export against the contract: `parsed` is the check, and
+ * `written` the export as written, for a caller that lets Cortex fill its
+ * own defaults, with each document `path` read, relative to the file, into
+ * the `data` Cortex takes (`resolvedManifestSchema`). Throws when the file
+ * does not import, exports nothing, or names a document file it cannot
+ * read, that is empty, or that is too large. Kept out of the contract's
+ * module, which the browser imports too.
  */
 export async function loadManifestFile(path: string) {
     const module = (await import(pathToFileURL(path).href)) as { default?: unknown };
     if (module.default === undefined) throw new Error("The module has no default export.");
-    return { written: module.default, parsed: manifestSchema.safeParse(module.default) };
+    const parsed = manifestSchema.safeParse(module.default);
+    return {
+        written: parsed.success
+            ? await readDocuments(module.default, dirname(path))
+            : module.default,
+        parsed,
+    };
+}
+
+/** `written` with every document `path` read from `directory` into `data`. */
+async function readDocuments(written: unknown, directory: string) {
+    if (!isRecord(written) || !Array.isArray(written.collections)) return written;
+    const collections = await Promise.all(
+        written.collections.map(async (collection: unknown) => {
+            if (!isRecord(collection) || !Array.isArray(collection.documents)) return collection;
+            const documents = await Promise.all(
+                collection.documents.map((document: unknown) =>
+                    isRecord(document) && typeof document.path === "string"
+                        ? readDocument(
+                              String(collection.name),
+                              document.path,
+                              document.name,
+                              directory,
+                          )
+                        : document,
+                ),
+            );
+            return { ...collection, documents };
+        }),
+    );
+    return { ...written, collections };
+}
+
+/** One document file as `{ name, data }`, named after the file unless `name` says otherwise. */
+async function readDocument(collection: string, path: string, name: unknown, directory: string) {
+    const where = `Collection ${collection}: ${path}`;
+    const bytes = await readFile(resolve(directory, path)).catch((cause: unknown) => {
+        throw new Error(`${where} could not be read.`, { cause });
+    });
+    if (bytes.byteLength === 0) throw new Error(`${where} is empty.`);
+    if (bytes.byteLength > DOCUMENT_LIMITS.bytes) {
+        throw new Error(`${where} is larger than ${DOCUMENT_LIMITS.bytes} bytes.`);
+    }
+    return {
+        name: documentName({ name: typeof name === "string" ? name : undefined, path }),
+        data: bytes.toString("base64"),
+    };
 }