@m6d/cortex-cli 0.12.1 → 0.13.0

package/src/commands/new.ts

@@ -2,13 +2,13 @@ import { spawnSync } from "node:child_process";
 import { mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 
-import { Option, type Command } from "@commander-js/extra-typings";
+import type { Command } from "@commander-js/extra-typings";
+import { CORTEX_MANIFEST_ID } from "@cortex/contracts/manifest";
 
 import pkg from "../../package.json";
-import { FEATURE_NAMES, FEATURES, type Feature } from "@/scaffold/features";
-import { scaffoldFiles, type Database } from "@/scaffold/files";
-import { allocatePorts } from "@/scaffold/ports";
-import { askDatabase, askFeatures, askName, intro, log, outro, runSteps } from "@/ui/prompts";
+import { manifestIdOf, scaffoldFiles } from "@/scaffold/files";
+import { allocatePorts, type Ports } from "@/scaffold/ports";
+import { askConfirm, askName, intro, log, outro, runSteps } from "@/ui/prompts";
 import * as report from "@/ui/report";
 
 /** npm's rule for an unscoped package name. The name is the directory too. */
@@ -18,55 +18,31 @@ const MAX_NAME = 214;
 /** The two names npm rejects outright, however well-formed they otherwise are. */
 const RESERVED_NAMES = ["node_modules", "favicon.ico"];
 
-export function registerNew(program: Command) {
+export function registerNew(program: Command<[], { verbose?: true }>) {
     const command = program
         .command("new")
         .helpGroup("Project:")
-        .description("Scaffold a new Cortex server")
+        .description("Scaffold a backend, its agent, and Cortex to run it on")
         .argument("[name]", "Directory to create, and the generated package name")
-        .addOption(
-            new Option("--database <type>", "Database the project is configured for")
-                .choices(["postgres", "mssql"] as const)
-                .default("postgres" as const),
-        )
-        // Both halves of every toggle, so a flag can answer its question either
+        // Both halves of the toggle, so the flag can answer its question either
         // way and `--yes` is not the only way to say "no". Declaring the positive
         // first leaves the value `undefined` when neither is passed, which is
         // what marks the question as still unanswered.
-        .option("--attachments", FEATURES.attachments.hint)
-        .option("--no-attachments", "Leave attachments out")
-        .option("--redis", FEATURES.redis.hint)
-        .option("--no-redis", "Leave Redis out")
-        .option("--auth", FEATURES.auth.hint)
-        .option("--no-auth", "Leave auth out")
-        .option("--knowledge", FEATURES.knowledge.hint)
-        .option("--no-knowledge", "Leave the knowledge graph out")
-        .option("--control-center", FEATURES.controlCenter.hint)
-        .option("--no-control-center", "Leave the Control Center out")
+        .option("--datasets", "Serve an example table as a dataset host")
+        .option("--no-datasets", "Leave the dataset host out")
         .option("--yes", "Accept the defaults instead of asking");
 
     command.action(async function (name, options) {
-        // The globals live on the root command, which reaches this file untyped —
-        // `Command` here is the parameter type, not cli.ts's inferred one.
-        const { verbose = false } = program.opts() as { verbose?: boolean };
-
-        await scaffold(name, {
-            ...options,
-            verbose,
-            databasePassed: command.getOptionValueSource("database") !== "default",
-        });
+        const { verbose = false } = program.opts();
+
+        await scaffold(name, { ...options, verbose });
     });
 }
 
-type Options = {
-    database: Database;
-    yes?: boolean;
-    verbose: boolean;
-    databasePassed: boolean;
-} & Partial<Record<Feature, boolean>>;
+type Options = { datasets?: boolean; yes?: boolean; verbose: boolean };
 
 async function scaffold(name: string | undefined, options: Options) {
-    const { verbose, databasePassed } = options;
+    const { verbose } = options;
 
     // No TTY ⇒ never prompt and behave as `--yes`. On Bun a piped stdin reports
     // `undefined` rather than `false`, which is why this compares against `true`.
@@ -77,6 +53,7 @@ async function scaffold(name: string | undefined, options: Options) {
     // A name that came from a flag is checked before anything prints; one that
     // is prompted for is checked by the prompt, which can ask again.
     if (name !== undefined && !isValidName(name)) failInvalidName(name);
+    if (name !== undefined && manifestIdOf(name) === CORTEX_MANIFEST_ID) failCortexId(name);
     if (name !== undefined && isTaken(resolve(process.cwd(), name))) failTaken(name);
 
     const started = Date.now();
@@ -84,15 +61,17 @@ async function scaffold(name: string | undefined, options: Options) {
 
     const project = name ?? (await askName(checkName)).trim();
 
-    const database = interactive && !databasePassed ? await askDatabase() : options.database;
-    const features = await chooseFeatures(options, interactive);
+    // A flag settles the question; otherwise it is asked, or under `--yes` left out.
+    const datasets =
+        options.datasets ??
+        (interactive && (await askConfirm("Serve an example table as a dataset host?")));
 
     const target = resolve(process.cwd(), project);
 
-    // Allocated here rather than inside the generators, which stay pure: picking a
+    // Drawn here rather than inside the generators, which stay pure: picking a
     // port means probing the machine for a free one.
     const ports = allocatePorts();
-    const files = scaffoldFiles(project, database, features, ports);
+    const files = scaffoldFiles({ name: project, datasets, ports });
     let failedInstall: string | undefined;
 
     await runSteps([
@@ -105,8 +84,13 @@ async function scaffold(name: string | undefined, options: Options) {
                     writeFileSync(file, contents);
                 }
 
-                const summary = [database, ...features, "model"].join(" · ");
-                return `Wrote ${Object.keys(files).length} files to ./${project}\n${summary}`;
+                const summary = [
+                    "server",
+                    ...(datasets ? ["dataset host"] : []),
+                    "manifest",
+                    "Cortex",
+                ];
+                return `Wrote ${Object.keys(files).length} files to ./${project}\n${summary.join(" · ")}`;
             },
         },
         {
@@ -152,7 +136,7 @@ async function scaffold(name: string | undefined, options: Options) {
     if (failedInstall !== undefined) warnInstall(project, failedInstall, verbose);
 
     outro(`Done in ${seconds(started)}`);
-    process.stdout.write(nextSteps(project, features, files));
+    process.stdout.write(nextSteps(project, ports));
 }
 
 /**
@@ -172,41 +156,19 @@ function warnInstall(project: string, output: string, verbose: boolean) {
     );
 }
 
-function nextSteps(project: string, features: Feature[], files: ReturnType<typeof scaffoldFiles>) {
-    const keys = [
-        "CORTEX_DATABASE_URL",
-        "CORTEX_MODEL_*",
-        ...features.map((feature) => FEATURES[feature].secrets).filter((key) => key !== undefined),
-    ];
-
+function nextSteps(project: string, ports: Ports) {
     return (
         `\nNext steps:\n\n` +
         step(`cd ${project}`) +
-        step("edit .env", `fill in ${keys.length === 2 ? keys.join(" and ") : keys.join(", ")}`) +
-        (files["docker-compose.yml"] === undefined ? "" : step("docker compose up -d")) +
-        // The `port` the generated cortex.config.ts sets, which is the server's
-        // own default. Service ports move at scaffold time; this one does not.
+        step("edit cortex.env", "fill in EMBEDDING_API_KEY") +
+        step("docker compose up -d", `→ http://localhost:${ports.cortex}`) +
+        // The port the generated index.ts listens on; only Cortex's ports move.
         step("bun run dev", "→ http://localhost:3331") +
+        step("README.md", "register the server, then cortex apply") +
         `\n`
     );
 }
 
-/**
- * One multi-select for the whole catalogue, minus whatever the flags already
- * settled: a passed flag suppresses its question, the rest are still asked.
- */
-async function chooseFeatures(options: Options, interactive: boolean) {
-    const chosen = new Set(FEATURE_NAMES.filter((feature) => options[feature] === true));
-    const undecided = FEATURE_NAMES.filter((feature) => options[feature] === undefined);
-
-    if (interactive && undecided.length > 0) {
-        for (const feature of await askFeatures(undecided)) chosen.add(feature);
-    }
-
-    // Catalogue order, not the order the flags or the checkboxes came in.
-    return FEATURE_NAMES.filter((feature) => chosen.has(feature));
-}
-
 async function install(cwd: string) {
     try {
         // Piped rather than inherited: the spinner owns the screen while this runs.
@@ -245,11 +207,14 @@ function isValidName(name: string) {
     return name.length <= MAX_NAME && VALID_NAME.test(name) && !RESERVED_NAMES.includes(name);
 }
 
-/** The prompt's rules, which are the flag path's two failures asked one at a time. */
+/** The prompt's rules, which are the flag path's failures asked one at a time. */
 function checkName(candidate: string) {
     // One line each: clack prints them inside the prompt, which will not wrap.
     if (RESERVED_NAMES.includes(candidate)) return `npm does not allow the name ${candidate}.`;
     if (!isValidName(candidate)) return "Lowercase letters, digits, and . _ - only.";
+    if (manifestIdOf(candidate) === CORTEX_MANIFEST_ID) {
+        return `The manifest would take Cortex's own id, ${CORTEX_MANIFEST_ID}.`;
+    }
     if (isTaken(resolve(process.cwd(), candidate))) {
         return `./${candidate} already exists and is not an empty directory.`;
     }
@@ -307,6 +272,17 @@ function missingName(): never {
     });
 }
 
+/** The manifest takes its id from the name, and Cortex refuses any manifest under its own. */
+function failCortexId(name: string): never {
+    report.fail({
+        what: `"${name}" would give the manifest Cortex's own id, ${CORTEX_MANIFEST_ID}`,
+        blocks: [
+            "cortex.manifest.ts takes its id from the project name, and Cortex keeps that\nid for its own manifest. Pick another name:",
+            `  cortex new ${CORTEX_MANIFEST_ID}-app`,
+        ],
+    });
+}
+
 function failTaken(name: string): never {
     report.fail({
         what: `./${name} already exists and is not an empty directory.`,

package/src/contracts/charts/chart-spec.ts

@@ -1,6 +1,6 @@
 /*
  * The boundary on a Vega-Lite spec the model wrote, shared by the widget
- * (which draws it live) and the server (which describes it to the model):
+ * (which draws it live) and Cortex (which describes it to the model):
  * inline data only, no fetches, no interactivity, bounded work.
  */
 

package/src/contracts/client-tools/index.ts

@@ -43,9 +43,9 @@ export type ClientToolHandshake = ClientToolOutcome & {
 };
 
 /*
- * Built-in input tools: static client tools the server package exports and
- * the widget answers with a component of its own, keyed by tool name. A
- * consumer's `toolComponents` entry under the same name replaces the built-in.
+ * Built-in input tools: client tools Cortex declares and the widget answers
+ * with a component of its own, keyed by tool name. A consumer's
+ * `toolComponents` entry under the same name replaces the built-in.
  */
 
 export const REQUEST_LOCATION_TOOL = "requestLocation";

package/src/contracts/{runtime → datasets}/datasets.ts

@@ -11,10 +11,15 @@ import { isRecord } from "../json";
  */
 
 export const DATASET_IDENTIFIER_PATTERN = /^[a-zA-Z][a-zA-Z0-9_]*$/;
+/** The longest dataset or host name. */
+export const DATASET_IDENTIFIER_MAX = 64;
 /** `column` on the dataset queried, or `relation.relation.column` along declared relations. */
 export const DATASET_COLUMN_REF_PATTERN = /^[a-zA-Z][a-zA-Z0-9_]*(\.[a-zA-Z][a-zA-Z0-9_]*)*$/;
 
-export const datasetIdentifierSchema = z.string().regex(DATASET_IDENTIFIER_PATTERN).max(64);
+export const datasetIdentifierSchema = z
+    .string()
+    .regex(DATASET_IDENTIFIER_PATTERN)
+    .max(DATASET_IDENTIFIER_MAX);
 export const datasetColumnRefSchema = z.string().regex(DATASET_COLUMN_REF_PATTERN).max(256);
 
 export const DATASET_COLUMN_TYPES = ["string", "number", "boolean", "date"] as const;
@@ -355,46 +360,6 @@ export const datasetResultSchema = z.object({
 });
 export type DatasetResult = z.infer<typeof datasetResultSchema>;
 
-export const DATASET_ERROR_KINDS = [
-    "invalid_query",
-    "unsupported",
-    "forbidden",
-    "timeout",
-    "internal",
-] as const;
-export type DatasetErrorKind = (typeof DATASET_ERROR_KINDS)[number];
-
-/** The HTTP status each error kind travels under, both from a host and from the server's own refusals. */
-export const DATASET_ERROR_STATUS = {
-    invalid_query: 400,
-    unsupported: 400,
-    forbidden: 403,
-    timeout: 504,
-    internal: 500,
-} as const satisfies Record<DatasetErrorKind, number>;
-
-export const datasetErrorSchema = z.object({
-    error: z.object({
-        kind: z.enum(DATASET_ERROR_KINDS),
-        detail: z.string().max(512),
-    }),
-});
-export type DatasetError = z.infer<typeof datasetErrorSchema>;
-
-/**
- * A dataset host as an agent's published configuration carries it: the
- * datasets the agent exposes, already the published manifest's definitions,
- * and which of them ride in every turn. The Control Center runs the queries
- * for a host it registered, so the server needs no URL or credential here.
- */
-export const datasetHostDeclarationSchema = z.object({
-    name: datasetIdentifierSchema,
-    manifestVersion: datasetManifestSchema.shape.version,
-    datasets: z.array(datasetDefinitionSchema).max(500),
-    pinned: z.array(datasetIdentifierSchema).max(50).default([]),
-});
-export type DatasetHostDeclaration = z.infer<typeof datasetHostDeclarationSchema>;
-
 /** The definitions a query may reach, by dataset name. */
 export type DatasetCatalog = ReadonlyMap<string, DatasetDefinition>;
 
@@ -659,7 +624,7 @@ export function boundDatasetResult(
 /**
  * Every way a query can be wrong given the catalog, as messages the model
  * can act on; an empty list means the host will understand it. Checked
- * before anything leaves the server, so a host can trust the shape and only
+ * before anything leaves Cortex, so a host can trust the shape and only
  * has to translate.
  */
 export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuery) {

package/src/contracts/datasets/index.ts

@@ -0,0 +1,3 @@
+export * from "./datasets";
+export * from "./dataset-sql";
+export * from "./dataset-sql-guide";

package/src/contracts/egress-policy/index.ts

@@ -0,0 +1,16 @@
+/*
+ * The egress policy as far as no socket is needed: which addresses are
+ * private, which hosts the deployment admits, which URLs a secret may travel
+ * to, and the redaction of the credentials an answer echoes back. Free of
+ * node modules, so a browser bundle judges a URL the same way Cortex does.
+ * Cortex's `guardedFetch` enforces the rest at the socket.
+ */
+
+export {
+    canCarrySecret,
+    egressAllowedHosts,
+    isEgressAllowedHost,
+    isPrivateAddress,
+    normalizeHost,
+} from "./url";
+export { credentialForms, redactedUrl, scrubAnswer, scrubCredentialStrings } from "./scrub";

package/src/contracts/{egress → egress-policy}/scrub.ts

@@ -25,11 +25,11 @@ export function credentialForms(value: string) {
 }
 
 /**
- * The one boundary every upstream body crosses before it is stored or sent
- * on, in cc and the server alike: U+0000 is dropped from keys and values
- * because Postgres refuses it in text and jsonb on both cc's execution log
- * and the server's transcript, then the credentials the request presented
- * are redacted. Dropping first, on the patterns as well, means a credential
+ * The one boundary every upstream body crosses before Cortex stores it or
+ * sends it on: U+0000 is dropped from keys and values because Postgres
+ * refuses it in text and jsonb, in the tool execution log and the thread's
+ * transcript alike, then the credentials the request presented are
+ * redacted. Dropping first, on the patterns as well, means a credential
  * an upstream spelled with U+0000 inside it still matches; redacting first
  * would let the strip reassemble it.
  */
@@ -59,3 +59,32 @@ export function scrubCredentialStrings(value: unknown, credentials: string[]): u
         ]),
     );
 }
+
+/**
+ * What a party answered (a manifest, a tool it lists, the values it fills),
+ * fit to store or show a model: the credentials it was sent redacted from
+ * every string in it, or null when one of its `names` carries one, since a
+ * redacted identifier would name something else. Functions in it, such as a
+ * tool's executor, stay as they are.
+ */
+export function scrubAnswer<T>(answer: T, credentials: string[], names: readonly string[] = []) {
+    const carries = names.some((name) => {
+        const plain = name.replaceAll("\u0000", "");
+        return scrubCredentialStrings(plain, credentials) !== plain;
+    });
+    return carries ? null : (scrubCredentialStrings(answer, credentials) as T);
+}
+
+/**
+ * A URL as it may be logged or shown: its origin and path, without the
+ * userinfo, query or fragment a secret can ride in. Text that does not
+ * parse as a URL keeps what comes before its query or fragment.
+ */
+export function redactedUrl(raw: string) {
+    try {
+        const url = new URL(raw);
+        return `${url.origin}${url.pathname}`;
+    } catch {
+        return raw.split(/[?#]/)[0] ?? raw;
+    }
+}

package/src/contracts/{egress → egress-policy}/url.ts

@@ -2,7 +2,7 @@
  * The egress policy as far as a URL alone can tell: which addresses are
  * private, which hosts the deployment admits, and where a secret may be
  * sent. Pure and free of node modules, so a browser bundle judges a URL the
- * same way the server does.
+ * same way Cortex does.
  */
 
 type Range = { first: number; last: number };
@@ -33,8 +33,8 @@ function isBlockedV4(address: string) {
 }
 
 /**
- * The IPv4 address inside an IPv4-mapped IPv6 one, which arrives dotted from a
- * DNS answer (`::ffff:8.8.8.8`) and as two hex groups from a parsed URL
+ * The IPv4 address in the last 32 bits of an IPv6 one, which arrives dotted
+ * from a DNS answer (`::ffff:8.8.8.8`) and as two hex groups from a parsed URL
  * (`::ffff:808:808`). `null` when the tail is neither.
  */
 function mappedV4(mapped: string) {
@@ -45,15 +45,28 @@ function mappedV4(mapped: string) {
     return [high >> 8, high & 255, low >> 8, low & 255].join(".");
 }
 
-function isBlockedV6(address: string) {
-    const normalized = address.toLowerCase();
-    if (normalized.startsWith("::ffff:")) {
-        const mapped = mappedV4(normalized.slice("::ffff:".length));
-        return mapped === null || isBlockedV4(mapped);
+/**
+ * The IPv4 address an IPv6 one reaches, for the prefixes where only the
+ * address may follow: IPv4-mapped (`::ffff:0:0/96`) and NAT64's well-known
+ * `64:ff9b::/96`. `undefined` for any other IPv6 address.
+ */
+function embeddedV4(address: string) {
+    for (const prefix of ["::ffff:", "64:ff9b::"]) {
+        if (address.startsWith(prefix)) return mappedV4(address.slice(prefix.length));
     }
+    return undefined;
+}
+
+function isBlockedV6(address: string) {
+    const normalized = normalizeHost(address);
+    const embedded = embeddedV4(normalized);
+    if (embedded !== undefined) return embedded === null || isBlockedV4(embedded);
     return (
         normalized === "::" ||
         normalized === "::1" ||
+        // NAT64's local-use /48: the operator may embed the IPv4 address at
+        // any RFC 6052 prefix length, so no position can be read reliably.
+        normalized.startsWith("64:ff9b:1:") ||
         normalized.startsWith("fc") ||
         normalized.startsWith("fd") ||
         /^fe[89ab]/.test(normalized)
@@ -115,7 +128,7 @@ export function isEgressAllowedHost(hostname: string, allowed = egressAllowedHos
  * http to loopback, or over plain http to a host the policy admits. Pass
  * `allowed` explicitly where the policy is known from elsewhere (the browser
  * form reads it from the session) or must stay strict regardless of it (a
- * URL the browser or the CLI dials, not the server): `[]` admits nothing.
+ * URL the browser or the CLI dials, not Cortex): `[]` admits nothing.
  */
 export function canCarrySecret(url: string, allowed = egressAllowedHosts()) {
     try {

package/src/contracts/errors/index.ts

@@ -0,0 +1,38 @@
+import { z } from "zod";
+
+/*
+ * The one error envelope a backend answers Cortex with, in the dataset
+ * protocol and the server protocol alike: `{ error: { kind, detail } }`
+ * under the kind's HTTP status, or a closer one HTTP defines (401 for a
+ * wrong server key, 404 for an unknown tool, 413 for a body too large). The
+ * detail reaches the model.
+ */
+
+export const BACKEND_ERROR_KINDS = [
+    "invalid_request",
+    "unsupported",
+    "forbidden",
+    "timeout",
+    "internal",
+] as const;
+export type BackendErrorKind = (typeof BACKEND_ERROR_KINDS)[number];
+
+/** The HTTP status each kind travels under. */
+export const BACKEND_ERROR_STATUS = {
+    invalid_request: 400,
+    unsupported: 400,
+    forbidden: 403,
+    timeout: 504,
+    internal: 500,
+} as const satisfies Record<BackendErrorKind, number>;
+
+/** The longest detail Cortex accepts; a backend cuts a longer one down to it. */
+export const BACKEND_ERROR_DETAIL_MAX = 512;
+
+export const backendErrorSchema = z.object({
+    error: z.object({
+        kind: z.enum(BACKEND_ERROR_KINDS),
+        detail: z.string().max(BACKEND_ERROR_DETAIL_MAX),
+    }),
+});
+export type BackendError = z.infer<typeof backendErrorSchema>;

package/src/contracts/json/index.ts

@@ -7,6 +7,11 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+/** `value` when it is a record, else undefined. */
+export function asRecord(value: unknown) {
+    return isRecord(value) ? value : undefined;
+}
+
 /**
  * One reference token of a JSON Pointer (RFC 6901): `~` is written `~0` and `/` is written
  * `~1`. Escaping replaces `~` first and unescaping `~1` first, so a literal `~1` survives the