@m6d/cortex-cli 0.13.0 → 0.14.0

package/README.md

@@ -10,8 +10,8 @@ bunx cortex apply                 # make it match ./cortex.manifest.ts
 ```
 
 Commands are noun-first, `cortex <resource> <procedure>`, except `new`, which acts on no existing
-resource, `apply`, which acts on a whole deployment, and `login`, `logout` and `status`, which act
-on your profiles. Run `cortex --help`, then `cortex <resource> --help`.
+resource, `setup` and `apply`, which act on a whole deployment, and `login`, `logout` and `status`,
+which act on your profiles. Run `cortex --help`, then `cortex <resource> --help`.
 
 ```sh
 bun add -d @m6d/cortex-cli
@@ -28,15 +28,16 @@ Scaffolds `./<name>`: a backend for Cortex agents, and Cortex itself to run them
   (`userName`) and one native tool (`rollDice`), mounted at `/cortex` by `index.ts`.
 - `cortex.manifest.ts` declares a `helper` agent on that server, for `cortex apply`.
 - `docker-compose.yml` runs Cortex's app, worker and engine on the one `syscodeia/cortex` image,
-  with Postgres, Redis and MinIO. Only the app and the engine publish host ports, picked at scaffold
-  time from what was free.
-- `.env` holds the backend's own key and the checkout's Compose project name, and `cortex.env`
-  Cortex's secrets, which only Compose reads. Neither is committed: `setup.ts` writes both from
-  their `.example` copies on every `bun install`, generating each value left blank, so a fresh
-  clone gets its own secrets and a stack that never shares the original's volumes. `.dockerignore`
-  keeps all four out of images.
-- `README.md` lists the steps: start the stack and the backend, register the server in the
-  console, then `cortex apply`.
+  with Postgres, Redis and MinIO. Only the app and the engine publish host ports. They and the
+  backend's port are picked at scaffold time from what was free.
+- `.env` holds every setting: Cortex's secrets, the backend's key and name, the model provider, and
+  the `CORTEX_URL` and `CORTEX_KEY` the CLI reaches this checkout's Cortex with. It is not
+  committed: `setup.ts` writes it from `.env.example` on every `bun install`, generating each
+  secret left blank, so a fresh clone gets its own secrets and a stack that never shares the
+  original's volumes. `.dockerignore` keeps both out of images.
+- `bun run dev` starts Cortex (`docker compose up`), the backend and `bun cortex setup --wait --yes` in
+  an mprocs dashboard. `bun run dev:agent` runs the same through `concurrently`, one labelled log for
+  coding agents and CI.
 - eslint, prettier, cspell, fallow and tsconfig are inlined, so `bun run verify` works standalone.
 
 It then runs `bun install`, and `git init` unless the target is already inside a work tree. An
@@ -47,20 +48,48 @@ manifest takes its id from the name and Cortex keeps the id `cortex` for its own
 bunx @m6d/cortex-cli new shop --datasets
 ```
 
-On a TTY it asks only what the flags left open: the name, and whether to add the dataset host. With
-no TTY it never prompts and behaves as `--yes`, so a missing name is a hard error rather than a
-hang.
+On a TTY it asks only what the flags left open: the name, whether to set up the `helper` agent on
+the first `bun run dev`, and if so the model provider (name, base URL, API key, model). It checks
+the provider with one chat message to the model, and reuses it for embeddings when it serves the
+stack's embedding model. With no TTY it never prompts and behaves as `--yes`, so a missing name is a
+hard error rather than a hang.
 
-| Flag                           | Default | Writes                                                                                      |
-| ------------------------------ | ------- | ------------------------------------------------------------------------------------------- |
-| `--datasets` / `--no-datasets` | off     | `datasets.ts`, an in-memory `orders` table served as a dataset host, and its manifest entry |
-| `--yes`                        |         | Accept the defaults instead of asking                                                       |
+| Flag                     | Default | Writes                                                                                      |
+| ------------------------ | ------- | ------------------------------------------------------------------------------------------- |
+| `--datasets`             | off     | `datasets.ts`, an in-memory `orders` table served as a dataset host, and its manifest entry |
+| `--setup` / `--no-setup` | on      | `cortex setup` among the processes `bun run dev` starts                                     |
+| `--yes`                  |         | Accept the defaults instead of asking; setup then runs with no model provider               |
 
 The image runs at the Cortex release that was current when this CLI was published
 (`CORTEX_VERSION` in the shell picks another); the backend depends on the `@m6d/cortex` release of
 that time. A server name stops at Cortex's 100 characters, so a longer project name is
 registered cut to that length.
 
+## `cortex setup`
+
+Registers a `cortex new` project's backend in its Cortex, then applies its manifest. It reads the
+backend's name, URL and key (`CORTEX_SERVER_*`) and the model provider (`MODEL_PROVIDER_*`) from
+`.env`, and goes in the one order Cortex accepts: the server key's secret, the provider's key and
+the provider, the server on the provider's model, a sync of the server, then the apply. Each step
+creates what is missing and keeps what exists, so a second run changes nothing. A key's secret is
+named for its value (`openai-api-key-1a2b3c4d`), so a changed key gets a new secret the provider or
+server moves to, and whatever else presents the old one keeps it. A provider or server under
+`.env`'s name that calls another URL belongs to something else: setup leaves it alone and stops. So
+does a `MODEL_PROVIDER_*` set only in part, or a provider URL on this machine or a private network
+that `EGRESS_ALLOWED_HOSTS` in `.env` does not admit. A server with no model yet gets the provider's.
+
+```sh
+bun cortex setup              # asks for a model provider when .env has none
+bun cortex setup --wait --yes # what `bun run dev` runs
+```
+
+`--wait` waits up to 10 minutes for Cortex to answer, long enough for a first start to pull the
+image. Without `-p`, setup sets up the Cortex `.env` names: a different `CORTEX_URL` exported in the
+shell, which Bun lets win over `.env`, ends the run. The retried sync, a minute while Cortex cannot
+reach the backend, gives up at once on any other refusal. With no model provider and `--yes`, it
+registers the server and applies nothing: Cortex publishes no agent without a model. The run that
+first creates something says how to sign in, with the password only on a terminal.
+
 ## Management commands
 
 Manage a Cortex deployment from the terminal. The commands come from
@@ -80,9 +109,9 @@ cortex logout
 
 Profiles live in `~/.cortex/profiles.json` (owner-only). `-p, --profile <name>`,
 a global option, picks one for any command, and names the one `login` saves
-and `logout` forgets (`default` otherwise); `CORTEX_URL` and
-`CORTEX_KEY`, both set, win over the file for CI. `CORTEX_HOME` relocates
-the directory.
+and `logout` forgets (`default` otherwise). `CORTEX_URL` and `CORTEX_KEY`, both
+set, win over the current profile, for CI and for a project's `.env`; a
+profile `-p` names wins over them. `CORTEX_HOME` relocates the directory.
 
 The grammar is `cortex <router> <procedure> [id] [flags]`, with routers as
 the API names them (`datasetHosts`, `modelProviders`). Each top-level input
@@ -144,6 +173,36 @@ export default defineManifest({
 });
 ```
 
+An `openapi` source makes every operation of an OpenAPI document a tool the
+manifest owns, so a backend with many endpoints needs no entry per tool:
+
+```ts
+openapi: [
+    {
+        name: "shop",
+        url: "https://shop.example.com/openapi.json",
+        credentials: [{ header: "Authorization", prefix: "Bearer ", credential: null }],
+        pinned: ["findOrder"],
+        // A GET runs at once; any other method waits for the end user's approval.
+        approval: { never: ["addOrderNote"], always: ["exportOrders"] },
+        audience: { agents: ["support"] },
+    },
+],
+```
+
+Every apply reads the document first, without the source's credentials. An
+operation it adds is created, a changed one updated, and a dropped one
+deleted. A source with credentials is read only over https, or from a host
+the deployment allows, since the document says where they go. A source needs the `tools` write permission even when nothing
+changes. Each tool is named after its `operationId`, made an identifier of at
+most 64 characters; an operation Cortex cannot import is left
+out. Two operations that come out with one name are told apart by a number
+(`getOrders`, `getOrders2`) in the order of their paths. The apply fails
+when `pinned` or `approval` names a tool no operation has (`unknown_tool`)
+or any name such a collision decided (`ambiguous_tool`), since a path the
+document adds or drops could move it to another endpoint. It also fails when an
+operation's name is already a listed tool's or another source's.
+
 The manifest is checked against the contract before anything is sent, then
 sent as written, so Cortex fills its own defaults. It goes to the
 management API (`manifests/sync`, or `manifests/preview` with `--dry-run`,
@@ -173,8 +232,9 @@ it; drop it first, then apply again.
 
 ## Exit codes
 
-| Command | 0                                            | 1                                   |
-| ------- | -------------------------------------------- | ----------------------------------- |
-| `new`   | Files written — even if `bun install` failed | Bad target, missing name, cancelled |
-| `apply` | Applied, or previewed with `--dry-run`       | Invalid manifest, or refused        |
-| others  | The procedure ran                            | It failed, or couldn't run          |
+| Command | 0                                                          | 1                                                   |
+| ------- | ---------------------------------------------------------- | --------------------------------------------------- |
+| `new`   | Files written — even if `bun install` failed               | Bad target, missing name, cancelled                 |
+| `setup` | Set up, or the server registered with no model to apply on | Not a project, or Cortex refused or was unreachable |
+| `apply` | Applied, or previewed with `--dry-run`                     | Invalid manifest, or refused                        |
+| others  | The procedure ran                                          | It failed, or couldn't run                          |

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@m6d/cortex-cli",
-    "version": "0.13.0",
+    "version": "0.14.0",
     "description": "Scaffold Cortex projects and manage a Cortex deployment",
     "license": "MIT",
     "type": "module",

package/src/cli.ts

@@ -4,6 +4,7 @@ import pkg from "../package.json";
 import { registerApply } from "@/commands/apply";
 import { registerManagement } from "@/commands/management";
 import { registerNew } from "@/commands/new";
+import { registerSetup } from "@/commands/setup";
 import * as report from "@/ui/report";
 
 const program = new Command()
@@ -34,6 +35,7 @@ const program = new Command()
 // Each command owns its help group, so the two headings survive these files being
 // rewritten. Registration order fixes the order the groups print in.
 registerNew(program);
+registerSetup(program);
 registerApply(program);
 // The management commands come from Cortex's API, so this one may fetch
 // before parsing; it looks at argv to decide whether it needs to, which it

package/src/commands/apply.ts

@@ -12,7 +12,7 @@ import * as report from "@/ui/report";
 
 // Discovery is this exact name in the cwd, with no upward walk, so the
 // argument's default is the discovery rule.
-const DEFAULT_MANIFEST = "./cortex.manifest.ts";
+export const DEFAULT_MANIFEST = "./cortex.manifest.ts";
 
 const CHANGES = ["create", "update", "delete"] as const;
 const MARKS = { create: "+", update: "~", delete: "-" } satisfies Record<
@@ -40,39 +40,13 @@ export function registerApply(program: Command<[], { verbose?: true; profile?: s
         const profile = requireProfile(profileName);
 
         report.header(dryRun ? "cortex apply --dry-run" : "cortex apply");
-        const manifest = await loadManifest(file, verbose);
         report.input("Manifest", label(file));
         report.input("Cortex", profile.url);
 
-        const answer = await callOrFail(profile, () =>
-            callManagement(
-                profile,
-                "post",
-                dryRun ? "/manifests/preview" : "/manifests/sync",
-                manifest,
-            ),
-        );
-        // The secrets are read apart from the changes: a sync has committed by
-        // now, and a summary this CLI is too old to read must not lose them.
-        const { triggerSecrets = {} } = appliedManifestSchema
-            .pick({ triggerSecrets: true })
-            .partial()
-            .parse(answer);
-        const applied = manifestChangesSchema.safeParse(answer);
-        if (!applied.success) {
-            printTriggerSecrets(triggerSecrets);
-            report.fail({
-                what: `${profile.url} answered with a summary this CLI cannot read`,
-                blocks: [
-                    `The ${dryRun ? "preview" : "apply"} went through. Update the CLI to read its summary:`,
-                    "  bun add -d @m6d/cortex-cli@latest",
-                ],
-            });
-        }
-        printChanges(applied.data);
-        const created = countNames(applied.data.create);
-        const updated = countNames(applied.data.update);
-        const deleted = countNames(applied.data.delete);
+        const { changes, triggerSecrets } = await applyManifest({ profile, file, dryRun, verbose });
+        const created = countNames(changes.create);
+        const updated = countNames(changes.update);
+        const deleted = countNames(changes.delete);
         report.footer(
             created + updated + deleted === 0
                 ? "no changes"
@@ -84,6 +58,49 @@ export function registerApply(program: Command<[], { verbose?: true; profile?: s
     });
 }
 
+/**
+ * Loads the manifest at `file` and sends it, a preview under `dryRun`, then
+ * prints its changes one row per entry. Ends the run on an invalid file, a
+ * refusal, or a summary this CLI cannot read. `cortex apply` and `cortex
+ * setup` both apply through it; each prints the trigger secrets it returns.
+ */
+export async function applyManifest(options: {
+    profile: ReturnType<typeof requireProfile>;
+    file: string;
+    dryRun: boolean;
+    verbose: boolean;
+}) {
+    const { profile, file, dryRun, verbose } = options;
+    const manifest = await loadManifest(file, verbose);
+    const answer = await callOrFail(profile, () =>
+        callManagement(
+            profile,
+            "post",
+            dryRun ? "/manifests/preview" : "/manifests/sync",
+            manifest,
+        ),
+    );
+    // The secrets are read apart from the changes: a sync has committed by
+    // now, and a summary this CLI is too old to read must not lose them.
+    const { triggerSecrets = {} } = appliedManifestSchema
+        .pick({ triggerSecrets: true })
+        .partial()
+        .parse(answer);
+    const applied = manifestChangesSchema.safeParse(answer);
+    if (!applied.success) {
+        printTriggerSecrets(triggerSecrets);
+        report.fail({
+            what: `${profile.url} answered with a summary this CLI cannot read`,
+            blocks: [
+                `The ${dryRun ? "preview" : "apply"} went through. Update the CLI to read its summary:`,
+                "  bun add -d @m6d/cortex-cli@latest",
+            ],
+        });
+    }
+    printChanges(applied.data);
+    return { changes: applied.data, triggerSecrets };
+}
+
 /** How a manifest path prints: relative to the cwd, `./`-prefixed. */
 function label(path: string) {
     const relativePath = relative(process.cwd(), path);
@@ -91,7 +108,7 @@ function label(path: string) {
 }
 
 /** The manifest's absolute path; a missing file ends the run before anything prints. */
-function findManifest(path: string) {
+export function findManifest(path: string) {
     const absolute = resolve(process.cwd(), path);
     if (existsSync(absolute)) return absolute;
     return report.fail({
@@ -156,7 +173,7 @@ function countNames(byKind: Record<string, string[]>) {
  * The fire secrets this apply minted, for each trigger it created or whose
  * `fireCredential` it dropped; nothing shows them again.
  */
-function printTriggerSecrets(secrets: Record<string, string>) {
+export function printTriggerSecrets(secrets: Record<string, string>) {
     const entries = Object.entries(secrets);
     if (entries.length === 0) return;
     report.note(

package/src/commands/management.ts

@@ -4,7 +4,13 @@ import { type Command, Option } from "@commander-js/extra-typings";
 import { canCarrySecret } from "@cortex/contracts/egress-policy";
 import { isRecord } from "@cortex/contracts/json";
 
-import { callManagement, inputSchemaOf, loadOpenApi, ManagementError } from "@/management/client";
+import {
+    callManagement,
+    inputSchemaOf,
+    loadOpenApi,
+    ManagementError,
+    UnreachableError,
+} from "@/management/client";
 import type { OpenApiDocument, OpenApiOperation } from "@/management/client";
 import { flagsFor, parseAnswer, toOption } from "@/management/flags";
 import type { Flag } from "@/management/flags";
@@ -383,30 +389,35 @@ async function publishAfterWrite(
     }
 }
 
-/**
- * A management refusal, or a Cortex that cannot be reached, is the
- * run's outcome, in the error grammar, with whatever the refusal names;
- * anything else is a bug.
- */
+/** `call`'s result, or the run ended by `failCall`. */
 export async function callOrFail<T>(profile: ResolvedProfile, call: () => Promise<T>) {
     try {
         return await call();
     } catch (error) {
-        if (error instanceof ManagementError) {
-            report.fail({
-                what: `${error.status} ${error.code}: ${error.message}`,
-                blocks: refusalBlocks(error.detail ?? {}),
-            });
-        }
-        if (error instanceof TypeError) {
-            report.fail({
-                what: `Could not reach ${profile.url}`,
-                error,
-                blocks: ["Check the URL and that Cortex is up."],
-            });
-        }
-        throw error;
+        return failCall(profile, error);
+    }
+}
+
+/**
+ * A management refusal, or a Cortex that cannot be reached, is the
+ * run's outcome, in the error grammar, with whatever the refusal names;
+ * anything else is a bug, and rethrown.
+ */
+export function failCall(profile: { url: string }, error: unknown): never {
+    if (error instanceof ManagementError) {
+        report.fail({
+            what: `${error.status} ${error.code}: ${error.message}`,
+            blocks: refusalBlocks(error.detail ?? {}),
+        });
+    }
+    if (error instanceof UnreachableError) {
+        report.fail({
+            what: `Could not reach ${profile.url}`,
+            error,
+            blocks: ["Check the URL and that Cortex is up."],
+        });
     }
+    throw error;
 }
 
 /**
@@ -527,7 +538,7 @@ function readJsonBody(source: string) {
     return parsed;
 }
 
-async function walkPages(
+export async function walkPages(
     profile: ResolvedProfile,
     method: "get" | "post",
     path: string,

package/src/commands/new.ts

@@ -6,7 +6,8 @@ import type { Command } from "@commander-js/extra-typings";
 import { CORTEX_MANIFEST_ID } from "@cortex/contracts/manifest";
 
 import pkg from "../../package.json";
-import { manifestIdOf, scaffoldFiles } from "@/scaffold/files";
+import { askProvider } from "@/commands/setup";
+import { AGENT_SLUG, manifestIdOf, scaffoldFiles } from "@/scaffold/files";
 import { allocatePorts, type Ports } from "@/scaffold/ports";
 import { askConfirm, askName, intro, log, outro, runSteps } from "@/ui/prompts";
 import * as report from "@/ui/report";
@@ -24,12 +25,13 @@ export function registerNew(program: Command<[], { verbose?: true }>) {
         .helpGroup("Project:")
         .description("Scaffold a backend, its agent, and Cortex to run it on")
         .argument("[name]", "Directory to create, and the generated package name")
-        // Both halves of the toggle, so the flag can answer its question either
-        // way and `--yes` is not the only way to say "no". Declaring the positive
-        // first leaves the value `undefined` when neither is passed, which is
-        // what marks the question as still unanswered.
         .option("--datasets", "Serve an example table as a dataset host")
-        .option("--no-datasets", "Leave the dataset host out")
+        // Both halves of the toggle, so the flag can answer its question either
+        // way and `--yes` is not the only way to say "yes". Declaring the
+        // positive first leaves the value `undefined` when neither is passed,
+        // which is what marks the question as still unanswered.
+        .option("--setup", `Set up the ${AGENT_SLUG} agent on the first bun run dev`)
+        .option("--no-setup", "Leave Cortex empty; you register the backend yourself")
         .option("--yes", "Accept the defaults instead of asking");
 
     command.action(async function (name, options) {
@@ -39,7 +41,7 @@ export function registerNew(program: Command<[], { verbose?: true }>) {
     });
 }
 
-type Options = { datasets?: boolean; yes?: boolean; verbose: boolean };
+type Options = { datasets?: true; setup?: boolean; yes?: true; verbose: boolean };
 
 async function scaffold(name: string | undefined, options: Options) {
     const { verbose } = options;
@@ -61,17 +63,15 @@ async function scaffold(name: string | undefined, options: Options) {
 
     const project = name ?? (await askName(checkName)).trim();
 
-    // A flag settles the question; otherwise it is asked, or under `--yes` left out.
-    const datasets =
-        options.datasets ??
-        (interactive && (await askConfirm("Serve an example table as a dataset host?")));
+    const datasets = options.datasets === true;
+    const { setup, provider } = await askSetup(options.setup, interactive);
 
     const target = resolve(process.cwd(), project);
 
     // Drawn here rather than inside the generators, which stay pure: picking a
     // port means probing the machine for a free one.
     const ports = allocatePorts();
-    const files = scaffoldFiles({ name: project, datasets, ports });
+    const files = scaffoldFiles({ name: project, datasets, setup, ports, env: provider?.env });
     let failedInstall: string | undefined;
 
     await runSteps([
@@ -81,7 +81,8 @@ async function scaffold(name: string | undefined, options: Options) {
                 for (const [path, contents] of Object.entries(files)) {
                     const file = join(target, path);
                     mkdirSync(dirname(file), { recursive: true });
-                    writeFileSync(file, contents);
+                    // .env holds the provider's key before setup.ts adds the rest.
+                    writeFileSync(file, contents, { mode: path === ".env" ? 0o600 : 0o644 });
                 }
 
                 const summary = [
@@ -89,6 +90,7 @@ async function scaffold(name: string | undefined, options: Options) {
                     ...(datasets ? ["dataset host"] : []),
                     "manifest",
                     "Cortex",
+                    ...(setup ? ["first-run setup"] : []),
                 ];
                 return `Wrote ${Object.keys(files).length} files to ./${project}\n${summary.join(" · ")}`;
             },
@@ -136,7 +138,25 @@ async function scaffold(name: string | undefined, options: Options) {
     if (failedInstall !== undefined) warnInstall(project, failedInstall, verbose);
 
     outro(`Done in ${seconds(started)}`);
-    process.stdout.write(nextSteps(project, ports));
+    process.stdout.write(nextSteps(project, ports, setup, provider !== undefined));
+}
+
+/**
+ * Whether the first `bun run dev` sets up the agent, and the model provider
+ * it runs on. A flag settles the first question; otherwise it is asked, or
+ * under `--yes` answered yes. The provider has no default, so only a terminal
+ * asks for one; without it, setup registers the server and stops.
+ */
+async function askSetup(flag: boolean | undefined, interactive: boolean) {
+    const setup =
+        flag ??
+        (!interactive ||
+            (await askConfirm(
+                `Set up the ${AGENT_SLUG} agent when you first run bun run dev?`,
+                true,
+            )));
+    const provider = setup && interactive ? await askProvider() : undefined;
+    return { setup, provider };
 }
 
 /**
@@ -156,17 +176,20 @@ function warnInstall(project: string, output: string, verbose: boolean) {
     );
 }
 
-function nextSteps(project: string, ports: Ports) {
-    return (
-        `\nNext steps:\n\n` +
-        step(`cd ${project}`) +
-        step("edit cortex.env", "fill in EMBEDDING_API_KEY") +
-        step("docker compose up -d", `→ http://localhost:${ports.cortex}`) +
-        // The port the generated index.ts listens on; only Cortex's ports move.
-        step("bun run dev", "→ http://localhost:3331") +
-        step("README.md", "register the server, then cortex apply") +
-        `\n`
-    );
+/** What the first `bun run dev` sets up: the agent, the server alone (no model yet), or nothing. */
+function nextSteps(project: string, ports: Ports, setup: boolean, provider: boolean) {
+    const runs = {
+        agent: step("bun run dev", `→ sets up ${AGENT_SLUG}, then prints where to chat`),
+        server:
+            step("bun run dev", "→ registers the server") +
+            step("bun cortex setup", "adds a model provider, then applies"),
+        none:
+            step("bun run dev", `→ Cortex at http://localhost:${ports.cortex}`) +
+            step("README.md", "register the server, then bun cortex apply"),
+    };
+    const run = setup ? (provider ? runs.agent : runs.server) : runs.none;
+
+    return `\nNext steps:\n\n${step(`cd ${project}`)}${run}\n`;
 }
 
 async function install(cwd: string) {