@m6d/cortex-cli 0.17.0 → 0.19.0

package/src/contracts/manifest/index.ts

@@ -1243,22 +1243,163 @@ const manifestWebhookSchema = z.strictObject({
 });
 export type ManifestWebhook = z.input<typeof manifestWebhookSchema>;
 
-/** A collection's name and description in characters. The console caps its editor at the same. */
-export const COLLECTION_LIMITS = { nameLength: 80, descriptionLength: 2_000 } as const;
-
-/** Holds documents, which are uploaded to it apart from the manifest. */
-const manifestCollectionSchema = z.strictObject({
-    name: z.string().trim().min(1).max(COLLECTION_LIMITS.nameLength),
-    description: z.string().trim().max(COLLECTION_LIMITS.descriptionLength).default(""),
-    /** The key of the service every document in the collection belongs to. */
-    service: slugSchema.optional(),
-    /** Whether answers may name its documents; false keeps their names, sections and pages from the model. */
-    citable: z.boolean().default(true),
-    audience: audienceSchema,
-    overridable: overridable(["description"]),
+/**
+ * A collection's name and description in characters, and how many documents
+ * a manifest lists in one. The console caps its editor at the same.
+ */
+export const COLLECTION_LIMITS = {
+    nameLength: 80,
+    descriptionLength: 2_000,
+    documents: 100,
+} as const;
+
+/** A knowledge document's name in characters and its size in bytes, however it arrives. */
+export const DOCUMENT_LIMITS = { nameLength: 255, bytes: 10 * 1024 * 1024 } as const;
+
+export const DOCX_MIME_TYPE =
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+
+/** The content types Cortex parses a knowledge document as. */
+export const DOCUMENT_MIME_TYPES = [
+    "application/pdf",
+    DOCX_MIME_TYPE,
+    "text/markdown",
+    "text/plain",
+    "text/html",
+] as const;
+export type DocumentMimeType = (typeof DOCUMENT_MIME_TYPES)[number];
+
+const mimeTypeByExtension = new Map<string, DocumentMimeType>([
+    ["md", "text/markdown"],
+    ["txt", "text/plain"],
+    ["html", "text/html"],
+    ["htm", "text/html"],
+    ["pdf", "application/pdf"],
+    ["docx", DOCX_MIME_TYPE],
+]);
+
+/** The content type a document is parsed as, from the extension its name ends in; none for any other name. */
+export function documentType(name: string) {
+    const dot = name.lastIndexOf(".");
+    return dot === -1 ? undefined : mimeTypeByExtension.get(name.slice(dot + 1).toLowerCase());
+}
+
+/** How many bytes a base64 string decodes to. */
+function base64Bytes(data: string) {
+    const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0;
+    return (data.length / 4) * 3 - padding;
+}
+
+const documentNameSchema = z.string().trim().min(1).max(DOCUMENT_LIMITS.nameLength);
+
+/** A document written in the manifest itself; its name's extension says how it is parsed. */
+const textDocumentSchema = z.strictObject({
+    name: documentNameSchema,
+    text: z
+        .string()
+        .min(1)
+        .refine(
+            (text) => new TextEncoder().encode(text).byteLength <= DOCUMENT_LIMITS.bytes,
+            `larger than ${DOCUMENT_LIMITS.bytes} bytes`,
+        ),
+});
+
+/** A file `cortex apply` reads, its path relative to the manifest file; named after the file unless `name` says otherwise. */
+const pathDocumentSchema = z.strictObject({
+    path: z.string().trim().min(1),
+    name: documentNameSchema.optional(),
+});
+
+/** A file Cortex reads on every apply, with no credentials; named after the URL's last segment unless `name` says otherwise. */
+const urlDocumentSchema = z.strictObject({
+    url: upstreamUrlSchema,
+    name: documentNameSchema.optional(),
 });
+
+/** A file's bytes in base64, as `cortex apply` sends a `path`. */
+const dataDocumentSchema = z.strictObject({
+    name: documentNameSchema,
+    data: z
+        .base64()
+        .refine(
+            (data) => base64Bytes(data) <= DOCUMENT_LIMITS.bytes,
+            `larger than ${DOCUMENT_LIMITS.bytes} bytes`,
+        ),
+});
+
+/** The name a document goes by: its own, else its file's or its URL's last segment. */
+export function documentName(document: { name?: string; path?: string; url?: string }) {
+    if (document.name !== undefined) return document.name;
+    if (document.path !== undefined) return document.path.split(/[/\\]/).pop() ?? "";
+    const segment = new URL(document.url ?? "", "https://x").pathname.split("/").pop() ?? "";
+    try {
+        return decodeURIComponent(segment);
+    } catch {
+        return segment;
+    }
+}
+
+/** A collection's documents: each name once, and each a name whose extension Cortex parses. */
+function documentsSchema<TDocument extends z.ZodType<Parameters<typeof documentName>[0]>>(
+    document: TDocument,
+) {
+    return z
+        .array(document)
+        .max(COLLECTION_LIMITS.documents)
+        .superRefine((documents, ctx) => {
+            const names = documents.map(documentName);
+            names.forEach((name, index) => {
+                if (documentType(name) === undefined) {
+                    ctx.addIssue({
+                        code: "custom",
+                        path: [index, "name"],
+                        message: "a document's name ends in .md, .txt, .html, .htm, .pdf or .docx",
+                    });
+                } else if (names.indexOf(name) !== index) {
+                    ctx.addIssue({
+                        code: "custom",
+                        path: [index, "name"],
+                        message: "a name repeats",
+                    });
+                }
+            });
+        })
+        .default([]);
+}
+
+/**
+ * Holds documents. Its `documents` are the manifest's: an apply stores what
+ * is new or changed and retires what it no longer lists, unless
+ * `overridable` offers them, and then a document added or removed in
+ * Cortex stays so until the manifest changes that document too.
+ */
+function collectionSchema<TDocument extends z.ZodType<Parameters<typeof documentName>[0]>>(
+    document: TDocument,
+) {
+    return z.strictObject({
+        name: z.string().trim().min(1).max(COLLECTION_LIMITS.nameLength),
+        description: z.string().trim().max(COLLECTION_LIMITS.descriptionLength).default(""),
+        /** The key of the service every document in the collection belongs to. */
+        service: slugSchema.optional(),
+        /** Whether answers may name its documents; false keeps their names, sections and pages from the model. */
+        citable: z.boolean().default(true),
+        documents: documentsSchema(document),
+        audience: audienceSchema,
+        overridable: overridable(["description", "documents"]),
+    });
+}
+
+const manifestDocumentSchema = z.union([textDocumentSchema, pathDocumentSchema, urlDocumentSchema]);
+export type ManifestDocument = z.input<typeof manifestDocumentSchema>;
+
+const manifestCollectionSchema = collectionSchema(manifestDocumentSchema);
 export type ManifestCollection = z.input<typeof manifestCollectionSchema>;
 
+/** A collection as Cortex receives it: each `path` read into `data`. */
+const resolvedCollectionSchema = collectionSchema(
+    z.union([textDocumentSchema, dataDocumentSchema, urlDocumentSchema]),
+);
+
 export const AUTH_JWT_SECRET_ALGORITHMS = ["HS256", "HS384", "HS512"] as const;
 
 const issuerSchema = z.string().trim().min(1).max(2048);
@@ -1335,22 +1476,6 @@ export const agentSettingsSchema = z.strictObject({
     /** Off, end users cannot message the agent: only its triggers reach it. */
     messageTrigger: z.boolean().default(true),
     defaultLocale: localeSchema.default(BASE_LOCALE),
-    rateLimits: z
-        .strictObject({
-            allUsers: z
-                .strictObject({
-                    messagesPerMinute: rateLimit(agentRateLimits.messagesPerMinute, "allUsers"),
-                    tokensPerDay: rateLimit(agentRateLimits.tokensPerDay, "allUsers"),
-                })
-                .prefault({}),
-            perUser: z
-                .strictObject({
-                    messagesPerMinute: rateLimit(agentRateLimits.messagesPerMinute, "perUser"),
-                    tokensPerDay: rateLimit(agentRateLimits.tokensPerDay, "perUser"),
-                })
-                .prefault({}),
-        })
-        .prefault({}),
     context: z
         .strictObject({
             maxContextTokens: setting(agentContext.maxContextTokens),
@@ -1395,9 +1520,34 @@ const manifestAgentSchema = z.strictObject({
         })
         .default({}),
     settings: agentSettingsSchema.prefault({}),
+    /** Messages a minute and model tokens a UTC day, for all users together and for each; null, no limit. */
+    rateLimits: z
+        .strictObject({
+            allUsers: z
+                .strictObject({
+                    messagesPerMinute: rateLimit(agentRateLimits.messagesPerMinute, "allUsers"),
+                    tokensPerDay: rateLimit(agentRateLimits.tokensPerDay, "allUsers"),
+                })
+                .prefault({}),
+            perUser: z
+                .strictObject({
+                    messagesPerMinute: rateLimit(agentRateLimits.messagesPerMinute, "perUser"),
+                    tokensPerDay: rateLimit(agentRateLimits.tokensPerDay, "perUser"),
+                })
+                .prefault({}),
+        })
+        .prefault({}),
     suggestions: suggestionsSchema.default(() => byLocale(() => [])),
     greeting: greetingSchema.default(() => byLocale(() => "")),
-    overridable: overridable(["name", "prompt", "models", "settings", "suggestions", "greeting"]),
+    overridable: overridable([
+        "name",
+        "prompt",
+        "models",
+        "settings",
+        "rateLimits",
+        "suggestions",
+        "greeting",
+    ]),
 });
 export type ManifestAgent = z.input<typeof manifestAgentSchema>;
 
@@ -1477,6 +1627,15 @@ export const manifestSchema = z.strictObject({
 /** A manifest as written: every field with a default may be left out. */
 export type Manifest = z.input<typeof manifestSchema>;
 
+/**
+ * A manifest as Cortex receives it: the one written, each document `path`
+ * read into `data` by `cortex apply`, since Cortex cannot read the files
+ * beside it.
+ */
+export const resolvedManifestSchema = manifestSchema.extend({
+    collections: listOf(MANIFEST_NAME_FIELDS.collections, resolvedCollectionSchema),
+});
+
 const manifestKindSchema = z.enum(MANIFEST_KINDS);
 
 /**
@@ -1492,11 +1651,17 @@ export const manifestChangesSchema = z.object({
     delete: namesByKindSchema,
 });
 
-/** A field of one entry that both the manifest and a live write changed since the last apply. */
+/**
+ * A field of one entry that both the manifest and a live write changed since
+ * the last apply. A field holding named parts (an agent's settings, its
+ * greeting and suggestions by locale, a collection's documents by name) is
+ * judged part by part, and `key` names the part.
+ */
 const manifestConflictSchema = z.strictObject({
     kind: manifestKindSchema,
     name: z.string(),
     field: z.string(),
+    key: z.string().optional(),
 });
 
 /** Which value a conflict keeps: the one changed live in Cortex, or the manifest's. */
@@ -1506,9 +1671,10 @@ const conflictSideSchema = z.enum(["live", "manifest"]);
  * What preview and sync take: the manifest, and how its conflicts resolve.
  * A conflict takes its own entry in `resolutions`, else `prefer`; a sync
  * left with one neither answers is refused as `unresolved_conflicts`,
- * naming them. A resolution for a field that no longer conflicts is ignored.
+ * naming them. A resolution answers the conflict with its kind, name, field
+ * and key; one for a field that no longer conflicts is ignored.
  */
-export const manifestApplyInputSchema = manifestSchema.extend({
+export const manifestApplyInputSchema = resolvedManifestSchema.extend({
     prefer: conflictSideSchema.optional(),
     resolutions: z.array(manifestConflictSchema.extend({ keep: conflictSideSchema })).default([]),
 });

package/src/contracts/manifest-file/index.ts

@@ -1,16 +1,68 @@
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
-import { manifestSchema } from "../manifest/index";
+import { isRecord } from "../json/index";
+import { DOCUMENT_LIMITS, documentName, manifestSchema } from "../manifest/index";
 
 /**
  * Imports a manifest file, TypeScript or JSON (Bun imports both), and checks
- * its default export against the contract: `written` is the export as
- * written, for a caller that lets Cortex fill its own defaults, and
- * `parsed` the check. Throws when the file does not import or exports
- * nothing. Kept out of the contract's module, which the browser imports too.
+ * its default export against the contract: `parsed` is the check, and
+ * `written` the export as written, for a caller that lets Cortex fill its
+ * own defaults, with each document `path` read, relative to the file, into
+ * the `data` Cortex takes (`resolvedManifestSchema`). Throws when the file
+ * does not import, exports nothing, or names a document file it cannot
+ * read, that is empty, or that is too large. Kept out of the contract's
+ * module, which the browser imports too.
  */
 export async function loadManifestFile(path: string) {
     const module = (await import(pathToFileURL(path).href)) as { default?: unknown };
     if (module.default === undefined) throw new Error("The module has no default export.");
-    return { written: module.default, parsed: manifestSchema.safeParse(module.default) };
+    const parsed = manifestSchema.safeParse(module.default);
+    return {
+        written: parsed.success
+            ? await readDocuments(module.default, dirname(path))
+            : module.default,
+        parsed,
+    };
+}
+
+/** `written` with every document `path` read from `directory` into `data`. */
+async function readDocuments(written: unknown, directory: string) {
+    if (!isRecord(written) || !Array.isArray(written.collections)) return written;
+    const collections = await Promise.all(
+        written.collections.map(async (collection: unknown) => {
+            if (!isRecord(collection) || !Array.isArray(collection.documents)) return collection;
+            const documents = await Promise.all(
+                collection.documents.map((document: unknown) =>
+                    isRecord(document) && typeof document.path === "string"
+                        ? readDocument(
+                              String(collection.name),
+                              document.path,
+                              document.name,
+                              directory,
+                          )
+                        : document,
+                ),
+            );
+            return { ...collection, documents };
+        }),
+    );
+    return { ...written, collections };
+}
+
+/** One document file as `{ name, data }`, named after the file unless `name` says otherwise. */
+async function readDocument(collection: string, path: string, name: unknown, directory: string) {
+    const where = `Collection ${collection}: ${path}`;
+    const bytes = await readFile(resolve(directory, path)).catch((cause: unknown) => {
+        throw new Error(`${where} could not be read.`, { cause });
+    });
+    if (bytes.byteLength === 0) throw new Error(`${where} is empty.`);
+    if (bytes.byteLength > DOCUMENT_LIMITS.bytes) {
+        throw new Error(`${where} is larger than ${DOCUMENT_LIMITS.bytes} bytes.`);
+    }
+    return {
+        name: documentName({ name: typeof name === "string" ? name : undefined, path }),
+        data: bytes.toString("base64"),
+    };
 }

package/src/management/client.ts

@@ -2,7 +2,8 @@ import { createHash } from "node:crypto";
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 
-import { isRecord } from "@cortex/contracts/json";
+import { createManagementClient, MANAGEMENT_BASE_PATH } from "@cortex/contracts/management";
+import type { ManagementTarget } from "@cortex/contracts/management";
 import { z } from "zod";
 
 import { profilesPath } from "./profiles";
@@ -109,7 +110,7 @@ export async function loadOpenApi(url: string) {
     const path = cachePath(url);
     const cached = readCache(path);
     try {
-        const response = await fetch(`${url}/api/management/openapi.json`, {
+        const response = await fetch(`${url}${MANAGEMENT_BASE_PATH}/openapi.json`, {
             headers: cached ? { "If-None-Match": cached.etag } : {},
         });
         if (response.status === 304 && cached)
@@ -159,71 +160,28 @@ export function inputSchemaOf(operation: OpenApiOperation) {
 }
 
 /**
- * One management call. A query's input object is spread over the query
- * string, one parameter per field, with a scalar written plainly and
- * anything else JSON-encoded, which is how the document declares it. A
- * mutation posts its input as the JSON body. A refusal throws
- * `ManagementError` with the error body's code, message, detail and sentence.
+ * Where `profile`'s management calls go; a Cortex that does not answer at
+ * all rejects as `UnreachableError`, apart from one that refused.
  */
-export async function callManagement(
-    profile: { url: string; key: string },
-    method: "get" | "post",
-    path: string,
-    input: unknown,
-) {
-    const base = `${profile.url}/api/management${path}`;
-    const url = method === "get" && isRecord(input) ? `${base}${queryString(input)}` : base;
-    const response = await fetch(url, {
-        method: method.toUpperCase(),
-        headers: {
-            Authorization: `Bearer ${profile.key}`,
-            ...(method === "post" ? { "Content-Type": "application/json" } : {}),
-        },
-        body: method === "post" && input !== undefined ? JSON.stringify(input) : undefined,
-    }).catch((error: unknown) => {
-        throw new UnreachableError(error);
-    });
-    const body: unknown = await response.json().catch(() => null);
-    if (!response.ok) {
-        const { code, message, refusal, cause } = (body ?? {}) as {
-            code?: string;
-            message?: string;
-            refusal?: string;
-            cause?: unknown;
-        };
-        throw new ManagementError(
-            response.status,
-            code ?? "error",
-            message ?? response.statusText,
-            isRecord(cause) ? cause : undefined,
-            refusal,
-        );
-    }
-    return body;
-}
-
-function isScalar(value: unknown): value is string | number | boolean | null {
-    return value === null || ["string", "number", "boolean"].includes(typeof value);
+export function managementTarget(profile: { url: string; key: string }) {
+    return {
+        ...profile,
+        fetch: (url: string, init: RequestInit) =>
+            fetch(url, init).catch((error: unknown) => {
+                throw new UnreachableError(error);
+            }),
+    } satisfies ManagementTarget;
 }
 
-/** A scalar as itself, an array of scalars as a repeated parameter, anything else as JSON, matching what the document declares. */
-function queryString(input: Record<string, unknown>) {
-    const params = new URLSearchParams();
-    for (const [name, value] of Object.entries(input)) {
-        if (value === undefined) continue;
-        if (isScalar(value)) params.set(name, String(value));
-        else if (Array.isArray(value) && value.length > 0 && value.every(isScalar)) {
-            for (const item of value) params.append(name, String(item));
-        } else params.set(name, JSON.stringify(value));
-    }
-    const encoded = params.toString();
-    return encoded === "" ? "" : `?${encoded}`;
+/** The typed client for the calls the CLI makes by name, over `managementTarget`. */
+export function managementClient(profile: { url: string; key: string }) {
+    return createManagementClient(managementTarget(profile));
 }
 
 /**
  * Cortex did not answer at all: refused, reset, unresolvable. `fetch` rejects
  * only for these, with whatever its runtime throws (Bun's is a plain `Error`
- * with a `code`), so `callManagement` names them once here.
+ * with a `code`), so `managementTarget` names them once here.
  */
 export class UnreachableError extends Error {
     public constructor(cause: unknown) {
@@ -231,21 +189,3 @@ export class UnreachableError extends Error {
         this.name = "UnreachableError";
     }
 }
-
-/**
- * A refusal; `detail` is what the refusal names, when it names something (an
- * entry, the row holding a name), and `refusal` the sentence Cortex words it
- * in, when it has one.
- */
-export class ManagementError extends Error {
-    public constructor(
-        public readonly status: number,
-        public readonly code: string,
-        message: string,
-        public readonly detail?: Record<string, unknown>,
-        public readonly refusal?: string,
-    ) {
-        super(message);
-        this.name = "ManagementError";
-    }
-}

package/src/scaffold/files.ts

@@ -29,7 +29,7 @@ import { applyPorts, type Ports } from "@/scaffold/ports";
  * `@m6d/cortex` range below are stamped by scripts/sync-package-versions.ts
  * before the CLI publishes, so a scaffold pins what was current then.
  */
-const CORTEX_VERSION = "0.17.0";
+const CORTEX_VERSION = "0.18.0";
 
 /** This machine as Cortex's containers name it: the one private host the stack's egress policy admits. */
 export const DOCKER_HOST = "host.docker.internal";
@@ -150,7 +150,7 @@ function packageJson({ name, datasets, setup }: Project) {
                 verify: "bun run format:check && bun run lint && bun run spellcheck && bun run check && bun run audit",
             },
             dependencies: {
-                "@m6d/cortex": "~0.5.0",
+                "@m6d/cortex": "~0.6.0",
                 // drizzle 1.0 is in beta; this is the build `@m6d/cortex/drizzle` is tested against.
                 ...(datasets ? { "drizzle-orm": "1.0.0-beta.15-859cf75" } : {}),
                 hono: "^4.12.3",