@m6d/cortex-cli 0.17.0 → 0.19.0

package/README.md

@@ -225,6 +225,39 @@ or any name such a collision decided (`ambiguous_tool`), since a path the
 document adds or drops could move it to another endpoint. It also fails when an
 operation's name is already a listed tool's or another source's.
 
+A collection lists its documents. Each is inline `text`, a file `path`
+relative to the manifest file, or a `url` Cortex reads on every apply, with no
+credentials:
+
+```ts
+collections: [
+    {
+        name: "Policies",
+        audience: { agents: ["support"] },
+        documents: [
+            { name: "faq.md", text: "# FAQ\n\nReturns take 30 days." },
+            { path: "./docs/returns.pdf" },
+            { url: "https://shop.example.com/policies/shipping.html" },
+        ],
+        // overridable: ["documents"],
+    },
+],
+```
+
+A document's name sets its type: `.md`, `.txt`, `.html`, `.htm`, `.pdf` or
+`.docx`. A path or URL document is named after its file unless `name` says
+otherwise. Each is at most 10 MB. `cortex apply` reads paths and sends their
+bytes, so a request carries every inline and path document again; Cortex
+stores only what changed. A document with new bytes replaces the old one,
+which keeps answering until the new one is processed, and one the manifest
+drops is retired.
+
+Listing `documents` makes the manifest own them: an apply retires a document
+uploaded in the console, the first apply after upgrading included, unless
+the collection offers `documents` in `overridable`. Offered, a document
+uploaded or removed in Cortex stays so until the manifest changes that same
+document, which `cortex apply` then asks about.
+
 The manifest is checked against the contract before anything is sent, then
 sent as written, so Cortex fills its own defaults. It goes to the
 management API (`manifests/sync`, or `manifests/preview` with `--dry-run`,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@m6d/cortex-cli",
-    "version": "0.17.0",
+    "version": "0.19.0",
     "description": "Scaffold Cortex projects and manage a Cortex deployment",
     "license": "MIT",
     "type": "module",

package/src/commands/apply.ts

@@ -4,6 +4,11 @@ import { relative, resolve } from "node:path";
 import { Option } from "@commander-js/extra-typings";
 import type { Command } from "@commander-js/extra-typings";
 import { isRecord } from "@cortex/contracts/json";
+import {
+    callManagement,
+    ManagementClientError,
+    managementPath,
+} from "@cortex/contracts/management";
 import {
     appliedManifestSchema,
     manifestApplyInputSchema,
@@ -12,7 +17,7 @@ import {
 import { loadManifestFile } from "@cortex/contracts/manifest-file";
 import { z } from "zod";
 
-import { callManagement, ManagementError } from "@/management/client";
+import { managementTarget } from "@/management/client";
 import { callOrFail, confirmDestructive, requireProfile } from "@/commands/management";
 import { askChoice, canAsk } from "@/ui/prompts";
 import * as report from "@/ui/report";
@@ -137,7 +142,7 @@ export async function applyManifest(options: {
     const { profile, dryRun } = options;
     const answer = await sendManifest({
         ...options,
-        path: dryRun ? "/manifests/preview" : "/manifests/sync",
+        path: managementPath(dryRun ? "manifests.preview" : "manifests.sync"),
     });
     // The secrets are read apart from the changes: a sync has committed by
     // now, and a summary this CLI is too old to read must not lose them.
@@ -179,14 +184,14 @@ async function sendManifest(options: {
     const resolutions: Resolution = [];
     for (;;) {
         try {
-            return await callManagement(profile, "post", path, {
-                ...manifest,
-                prefer,
-                resolutions,
+            return await callManagement(managementTarget(profile), {
+                method: "post",
+                path,
+                input: { ...manifest, prefer, resolutions },
             });
         } catch (error) {
             const refused =
-                error instanceof ManagementError
+                error instanceof ManagementClientError
                     ? unresolvedSchema.safeParse(error.detail)
                     : undefined;
             if (!refused?.success) throw error;
@@ -202,9 +207,14 @@ async function sendManifest(options: {
     }
 }
 
+/** The field both sides changed, and its part when the field is judged part by part. */
+function conflictField({ field, key }: Conflict) {
+    return key === undefined ? field : `${field} (${key})`;
+}
+
 /** How a conflict reads: its entry and the field both sides changed. */
-function conflictLabel({ kind, name, field }: Conflict) {
-    return `${kind} ${name}: ${field} changed in both places`;
+function conflictLabel(conflict: Conflict) {
+    return `${conflict.kind} ${conflict.name}: ${conflictField(conflict)} changed in both places`;
 }
 
 /** Ends a run that does not ask, naming every conflict and how to run `command` again to answer them. */
@@ -250,8 +260,9 @@ export function findManifest(path: string, command?: string) {
 
 /**
  * The file's default export, checked against the manifest contract here so
- * a mistake is named before anything is sent, then sent as written: Cortex
- * fills its own defaults, which may be newer than this CLI's.
+ * a mistake is named before anything is sent, then sent as written, each
+ * document `path` read from beside the file: Cortex fills its own defaults,
+ * which may be newer than this CLI's.
  */
 export async function loadManifest(path: string, verbose: boolean) {
     const { written, parsed } = await loadManifestFile(path).catch((error: unknown) =>
@@ -288,8 +299,12 @@ function printChanges(
             rows.set(kind, [...(rows.get(kind) ?? []), ...marked]);
         }
     }
-    for (const { kind, name, field } of conflicts) {
-        rows.set(kind, [...(rows.get(kind) ?? []), `! ${name}  ${field} changed in both places`]);
+    for (const conflict of conflicts) {
+        const { kind, name } = conflict;
+        rows.set(kind, [
+            ...(rows.get(kind) ?? []),
+            `! ${name}  ${conflictField(conflict)} changed in both places`,
+        ]);
     }
     for (const [kind, lines] of rows) {
         for (const [index, line] of lines.entries()) {

package/src/commands/management.ts

@@ -3,12 +3,13 @@ import { readFileSync } from "node:fs";
 import { type Command, Option } from "@commander-js/extra-typings";
 import { canCarrySecret } from "@cortex/contracts/egress-policy";
 import { isRecord } from "@cortex/contracts/json";
+import { callManagement, ManagementClientError } from "@cortex/contracts/management";
 
 import {
-    callManagement,
     inputSchemaOf,
     loadOpenApi,
-    ManagementError,
+    managementClient,
+    managementTarget,
     UnreachableError,
 } from "@/management/client";
 import type { OpenApiDocument, OpenApiOperation } from "@/management/client";
@@ -134,14 +135,11 @@ async function login(given: string | undefined, name: string) {
  */
 async function whoami(profile: { url: string; key: string }) {
     try {
-        const me = (await callManagement(profile, "get", "/management-keys/mine", undefined)) as {
-            name: string;
-            fingerprint: string;
-        };
+        const me = await managementClient(profile).managementKeys.mine();
         return `key "${me.name}" (${me.fingerprint})`;
     } catch (error) {
-        if (error instanceof ManagementError && error.status === 401) return null;
-        if (error instanceof ManagementError) {
+        if (error instanceof ManagementClientError && error.status === 401) return null;
+        if (error instanceof ManagementClientError) {
             report.fail({
                 what: `${profile.url} answered ${error.status} ${error.code}: ${error.message}`,
             });
@@ -323,7 +321,7 @@ function registerOperation(scope: {
         const result = await callOrFail(profile, () =>
             all
                 ? walkPages(profile, method, path, body)
-                : callManagement(profile, method, path, body),
+                : callManagement(managementTarget(profile), { method, path, input: body }),
         );
         print(result, format);
         if (publisher) {
@@ -427,7 +425,11 @@ async function publishAfterWrite(
     }
     if (publish) {
         const published = await callOrFail(profile, () =>
-            callManagement(profile, publisher.method, publisher.path, { id }),
+            callManagement(managementTarget(profile), {
+                method: publisher.method,
+                path: publisher.path,
+                input: { id },
+            }),
         );
         options.note(`Published ${id}.`);
         for (const warning of warningsOf(published)) options.note(`Warning: ${warning}`);
@@ -454,7 +456,7 @@ export async function callOrFail<T>(profile: ResolvedProfile, call: () => Promis
  * rethrown.
  */
 export function failCall(profile: { url: string }, error: unknown): never {
-    if (error instanceof ManagementError) {
+    if (error instanceof ManagementClientError) {
         report.fail({
             what: `${error.status} ${error.code}: ${error.message}`,
             blocks: [
@@ -605,10 +607,10 @@ export async function walkPages(
     // Bounded: if Cortex reports more rows than it serves, the walk must not
     // spin, and a listing the bound cannot cover is an error, never a short result.
     for (let pageIndex = 0; pageIndex < MAX_PAGES; pageIndex += 1) {
-        const page = (await callManagement(profile, method, path, {
-            ...given,
-            pageIndex,
-            pageSize,
+        const page = (await callManagement(managementTarget(profile), {
+            method,
+            path,
+            input: { ...given, pageIndex, pageSize },
         })) as { items?: unknown[]; count?: number; filteredCount?: number };
         if (!Array.isArray(page.items)) return page;
         items.push(...page.items);

package/src/commands/setup.ts

@@ -12,6 +12,7 @@ import {
     normalizeHost,
     scrubCredentialStrings,
 } from "@cortex/contracts/egress-policy";
+import { ManagementClientError, managementPath } from "@cortex/contracts/management";
 import { MODEL_PROVIDER_LIMITS, NAME_MAX_LENGTH, SETTING_BOUNDS } from "@cortex/contracts/manifest";
 import { z } from "zod";
 
@@ -29,7 +30,7 @@ import {
     requireProfile,
     walkPages,
 } from "@/commands/management";
-import { callManagement, ManagementError, UnreachableError } from "@/management/client";
+import { managementClient, UnreachableError } from "@/management/client";
 import { DOCKER_HOST, EMBEDDING, withEnvValues } from "@/scaffold/files";
 import * as prompts from "@/ui/prompts";
 import * as report from "@/ui/report";
@@ -154,7 +155,7 @@ export function registerSetup(program: Command<[], { verbose?: true; profile?: s
             interactive,
             command: "bun cortex apply --yes",
         }).catch(function (error: unknown) {
-            if (error instanceof ManagementError && error.message === "chat_model_required") {
+            if (error instanceof ManagementClientError && error.message === "chat_model_required") {
                 return undefined;
             }
             return failCall(profile, error);
@@ -233,12 +234,12 @@ async function waitForCortex(profile: Profile, wait: boolean) {
     let announced = false;
     for (;;) {
         try {
-            await callManagement(profile, "get", "/secrets/list", { pageSize: 1 });
+            await managementClient(profile).secrets.list({ pageSize: 1 });
             return;
         } catch (error) {
             const starting =
                 error instanceof UnreachableError ||
-                (error instanceof ManagementError && error.status >= 500);
+                (error instanceof ManagementClientError && error.status >= 500);
             if (!starting || Date.now() >= deadline) failCall(profile, error);
             if (!announced) {
                 report.row("", `Waiting for ${profile.url}. A first start pulls the image.`);
@@ -249,24 +250,21 @@ async function waitForCortex(profile: Profile, wait: boolean) {
     }
 }
 
-function call(profile: Profile, method: "get" | "post", path: string, input: unknown) {
-    return callOrFail(profile, () => callManagement(profile, method, path, input));
-}
-
 /** The entry named exactly `name`, across every page; a listing's search matches any part of it. */
 async function findByName<TItem extends typeof namedSchema>(
     profile: Profile,
-    router: string,
+    router: "secrets" | "modelProviders" | "servers",
     name: string,
     item: TItem,
 ) {
-    const { items } = z
-        .object({ items: z.array(item) })
-        .parse(
-            await callOrFail(profile, () =>
-                walkPages(profile, "get", `/${router}/list`, { q: name, pageSize: 100 }),
-            ),
-        );
+    const { items } = z.object({ items: z.array(item) }).parse(
+        await callOrFail(profile, () =>
+            walkPages(profile, "get", managementPath(`${router}.list`), {
+                q: name,
+                pageSize: 100,
+            }),
+        ),
+    );
     return items.find((entry) => entry.name === name);
 }
 
@@ -284,7 +282,9 @@ function secretFor(owner: string, role: "api-key" | "server-key", value: string)
 
 async function ensureSecret(profile: Profile, name: string, value: string) {
     const found = await findByName(profile, "secrets", name, namedSchema);
-    if (!found) await call(profile, "post", "/secrets/set", { name, value });
+    if (!found) {
+        await callOrFail(profile, () => managementClient(profile).secrets.set({ name, value }));
+    }
 }
 
 /**
@@ -307,7 +307,7 @@ function failCallsElsewhere(what: string, calls: string | null, wanted: string,
  * write.
  */
 async function ensureProvider(profile: Profile, provider: Provider) {
-    const found = await findByName(profile, "model-providers", provider.name, providerRowSchema);
+    const found = await findByName(profile, "modelProviders", provider.name, providerRowSchema);
     if (found && found.baseUrl !== provider.baseUrl) {
         failCallsElsewhere(
             `Provider ${provider.name}`,
@@ -330,13 +330,15 @@ async function ensureProvider(profile: Profile, provider: Provider) {
     await ensureSecret(profile, secretName, provider.apiKey);
 
     if (!found) {
-        await call(profile, "post", "/model-providers/create", {
-            name: provider.name,
-            baseUrl: provider.baseUrl,
-            secretName,
-            models,
-            efforts: [],
-        });
+        await callOrFail(profile, () =>
+            managementClient(profile).modelProviders.create({
+                name: provider.name,
+                baseUrl: provider.baseUrl,
+                secretName,
+                models,
+                efforts: [],
+            }),
+        );
         report.input("Provider", `${provider.name} · created`);
         return;
     }
@@ -344,13 +346,15 @@ async function ensureProvider(profile: Profile, provider: Provider) {
         report.input("Provider", `${provider.name} · kept`);
         return;
     }
-    await call(profile, "post", "/model-providers/update", {
-        id: found.id,
-        baseUrl: found.baseUrl,
-        secretName,
-        models,
-        efforts: found.efforts,
-    });
+    await callOrFail(profile, () =>
+        managementClient(profile).modelProviders.update({
+            id: found.id,
+            baseUrl: found.baseUrl,
+            secretName,
+            models,
+            efforts: found.efforts,
+        }),
+    );
     report.input("Provider", `${provider.name} · updated`);
 }
 
@@ -369,7 +373,11 @@ async function ensureServer(
     const url = `${project.CORTEX_SERVER_URL}/cortex`;
     const found = await findByName(profile, "servers", name, namedSchema);
     const server = found
-        ? serverSchema.parse(await call(profile, "get", "/servers/get", { id: found.id }))
+        ? serverSchema.parse(
+              await callOrFail(profile, () =>
+                  managementClient(profile).servers.get({ id: found.id }),
+              ),
+          )
         : undefined;
     if (server && server.url !== url) {
         failCallsElsewhere(`Server ${name}`, server.url, url, "CORTEX_SERVER_NAME");
@@ -384,15 +392,17 @@ async function ensureServer(
     };
     if (!server) {
         const created = serverSchema.parse(
-            await call(profile, "post", "/servers/create", {
-                name,
-                description: "",
-                url,
-                keySecretName,
-                ...model,
-                signInChecksPerMinute: SETTING_BOUNDS.server.signInChecksPerMinute.default,
-                triggerChecksPerMinute: SETTING_BOUNDS.server.triggerChecksPerMinute.default,
-            }),
+            await callOrFail(profile, () =>
+                managementClient(profile).servers.create({
+                    name,
+                    description: "",
+                    url,
+                    keySecretName,
+                    ...model,
+                    signInChecksPerMinute: SETTING_BOUNDS.server.signInChecksPerMinute.default,
+                    triggerChecksPerMinute: SETTING_BOUNDS.server.triggerChecksPerMinute.default,
+                }),
+            ),
         );
         report.input("Server", `${name} · created`);
         return created;
@@ -404,22 +414,24 @@ async function ensureServer(
         return server;
     }
     const updated = serverSchema.parse(
-        await call(profile, "post", "/servers/update", {
-            id: server.id,
-            name,
-            description: server.description ?? "",
-            url,
-            keySecretName,
-            ...(takesModel
-                ? model
-                : {
-                      modelProvider: server.modelProvider,
-                      modelName: server.modelName,
-                      modelEffort: server.modelEffort,
-                  }),
-            signInChecksPerMinute: server.signInChecksPerMinute,
-            triggerChecksPerMinute: server.triggerChecksPerMinute,
-        }),
+        await callOrFail(profile, () =>
+            managementClient(profile).servers.update({
+                id: server.id,
+                name,
+                description: server.description ?? "",
+                url,
+                keySecretName,
+                ...(takesModel
+                    ? model
+                    : {
+                          modelProvider: server.modelProvider,
+                          modelName: server.modelName,
+                          modelEffort: server.modelEffort,
+                      }),
+                signInChecksPerMinute: server.signInChecksPerMinute,
+                triggerChecksPerMinute: server.triggerChecksPerMinute,
+            }),
+        ),
     );
     report.input("Server", `${name} · updated`);
     return updated;
@@ -434,14 +446,14 @@ async function syncServer(profile: Profile, id: string, url: string) {
     const deadline = Date.now() + SYNC_WAIT_MS;
     for (;;) {
         try {
-            await callManagement(profile, "post", "/servers/sync", { id });
+            await managementClient(profile).servers.sync({ id });
             report.input("Synced", `${url}/cortex`);
             return;
         } catch (error) {
             // Only a backend Cortex cannot reach yet is worth asking again; a
             // refused manifest or blocked host fails the same way next time.
             const starting =
-                error instanceof ManagementError && error.message === "upstream_failed";
+                error instanceof ManagementClientError && error.message === "upstream_failed";
             if (starting && Date.now() < deadline) {
                 await Bun.sleep(POLL_MS);
                 continue;

package/src/contracts/datasets/dataset-sql.ts

@@ -57,8 +57,13 @@ type ParsedDatasetSql = {
 /** A refusal the model can act on. */
 export class DatasetSqlError extends Error {}
 
+/**
+ * `upper` is a word's keyword spelling. A quoted identifier and a word after a
+ * dot (`plans.from`) have none (`""`), so no keyword check matches them and
+ * they always read as names.
+ */
 type Token =
-    | { kind: "word"; value: string; upper: string; quoted?: true }
+    | { kind: "word"; value: string; upper: string }
     | { kind: "string"; value: string }
     | { kind: "number"; value: number }
     | { kind: "punct"; value: string }
@@ -84,6 +89,14 @@ function stringLiteral(sql: string, start: number) {
     }
 }
 
+/** Whether `previous` ends a value, so a minus after it subtracts; after a keyword or an operator it signs the number. */
+function endsValue(previous: Token | undefined) {
+    if (previous === undefined) return false;
+    if (previous.kind === "punct") return previous.value === ")";
+    if (previous.kind !== "word") return true;
+    return ["END", "NULL", "TRUE", "FALSE"].includes(previous.upper) || isName(previous);
+}
+
 /** Reads the token at `i`: a quoted identifier, a number (negative only where a value is expected), a word or punctuation. */
 function lexeme(sql: string, i: number, previous: Token | undefined) {
     const char = sql[i]!;
@@ -92,27 +105,13 @@ function lexeme(sql: string, i: number, previous: Token | undefined) {
         if (close === -1) throw new DatasetSqlError("A quoted identifier is not closed.");
         const value = sql.slice(i + 1, close);
         return {
-            token: {
-                kind: "word",
-                value,
-                upper: value.toUpperCase(),
-                quoted: true,
-            } satisfies Token,
+            token: { kind: "word", value, upper: "" } satisfies Token,
             next: close + 1,
         };
     }
     const rest = sql.slice(i);
     const number = /^-?\d+(\.\d+)?/.exec(rest);
-    // After a value or a closing parenthesis a minus subtracts; after a keyword or an operator it signs the number.
-    const afterValue =
-        previous !== undefined &&
-        (previous.kind === "punct"
-            ? previous.value === ")"
-            : previous.kind === "word"
-              ? ["END", "NULL", "TRUE", "FALSE"].includes(previous.upper) || isName(previous)
-              : true);
-    const signIsOperator = char === "-" && afterValue;
-    if (number && !signIsOperator) {
+    if (number && !(char === "-" && endsValue(previous))) {
         return {
             token: { kind: "number", value: Number(number[0]) } satisfies Token,
             next: i + number[0].length,
@@ -120,8 +119,13 @@ function lexeme(sql: string, i: number, previous: Token | undefined) {
     }
     const word = /^[A-Za-z_][A-Za-z0-9_]*/.exec(rest);
     if (word) {
+        const qualified = previous?.kind === "punct" && previous.value === ".";
         return {
-            token: { kind: "word", value: word[0], upper: word[0].toUpperCase() } satisfies Token,
+            token: {
+                kind: "word",
+                value: word[0],
+                upper: qualified ? "" : word[0].toUpperCase(),
+            } satisfies Token,
             next: i + word[0].length,
         };
     }
@@ -338,12 +342,35 @@ const RESERVED = [
     "END",
 ];
 
+/** Whether a bare word reads as a keyword rather than a name. */
+function isKeyword(upper: string) {
+    return RESERVED.includes(upper) || upper in REFUSED_CLAUSES;
+}
+
 function isName(token: Token): token is Token & { kind: "word" } {
-    if (token.kind !== "word") return false;
-    return (
-        token.quoted === true ||
-        (!RESERVED.includes(token.upper) && !(token.upper in REFUSED_CLAUSES))
-    );
+    return token.kind === "word" && !isKeyword(token.upper);
+}
+
+/** Words that bare read as a value rather than a column. */
+const VALUE_WORDS = ["TRUE", "FALSE", "CURRENT_DATE", "CURRENT_TIMESTAMP"];
+
+/**
+ * A column name as a query writes it: double quoted when bare it would read
+ * as a keyword or a value (`"from"`), so what the model is shown is what it
+ * can send.
+ */
+export function sqlColumnName(name: string) {
+    const upper = name.toUpperCase();
+    return isKeyword(upper) || VALUE_WORDS.includes(upper) ? `"${name}"` : name;
+}
+
+/** `message`, and for a keyword where a column belongs, how to write a column spelled like it. */
+function keywordError(token: Token, message: string) {
+    const hint =
+        token.kind === "word" && isKeyword(token.upper)
+            ? ` ${token.value} is a SQL keyword: write a column of that name as "${token.value}" or dataset.${token.value}.`
+            : "";
+    return new DatasetSqlError(`${message}${hint}`);
 }
 
 /** A unit written as `'month'`, `month` or `MONTH`. */
@@ -971,11 +998,21 @@ class Parser {
             );
         }
         const op = (operator.value === "<>" ? "!=" : operator.value) as DatasetFilterOp;
+        const message = `"${shown} ${operator.value}" needs a value on the right.`;
         if (this.startsTerm()) {
-            return { kind: "cond", left, op, value: { term: this.term("") } } satisfies Predicate;
+            return {
+                kind: "cond",
+                left,
+                op,
+                value: { term: this.term(message) },
+            } satisfies Predicate;
+        }
+        // A keyword here may be a column spelled like one (`x > from`); NULL keeps its own refusal.
+        const next = this.peek();
+        if (next.kind === "word" && next.upper !== "NULL" && isKeyword(next.upper)) {
+            throw keywordError(next, message);
         }
-        const value = this.literal(`"${shown} ${operator.value}" needs a value on the right.`);
-        return { kind: "cond", left, op, value } satisfies Predicate;
+        return { kind: "cond", left, op, value: this.literal(message) } satisfies Predicate;
     }
 
     /** Whether the next tokens are a value expression rather than a literal: a name, `(`, CASE, or a number followed by an operator. */
@@ -1174,10 +1211,10 @@ class Parser {
         return count.value;
     }
 
-    /** `a.b` as one reference; a bare keyword in reference position gets the caller's message. */
+    /** `a.b` as one reference; a bare keyword in reference position gets the caller's message (`keywordError`). */
     private reference(message: string) {
         const token = this.peek();
-        if (!isName(token)) throw new DatasetSqlError(message);
+        if (!isName(token)) throw keywordError(token, message);
         this.take();
         let ref = token.value;
         while (this.optionalPunct(".")) {