@m6d/cortex-cli 0.12.1 → 0.14.0

package/src/scaffold/files.ts

@@ -1,61 +1,128 @@
 /**
  * Every file `cortex new` writes, as one pure function. Files with nothing to
  * fill in are verbatim text under templates/; the ones that vary with the
- * selection are assembled here. The tooling configs are inlined rather than
+ * project are assembled here. The tooling configs are inlined rather than
  * pulled from `@m6d/eslint-config`, `@m6d/prettier-config` and `@m6d/tsconfig`:
  * those are `private: true` at `0.0.0` and resolvable only through this
  * workspace, and a standalone scaffold has no workspace.
  */
 
-import { FEATURE_NAMES, FEATURES, type Feature } from "@/scaffold/features";
-import indexTs from "@/scaffold/templates/index.ts.txt";
-import domainsReadme from "@/scaffold/templates/domains-README.md.txt";
+import pkg from "../../package.json";
+import serverTs from "@/scaffold/templates/server.ts.txt";
+import setupTs from "@/scaffold/templates/setup.ts.txt";
+import datasetsTs from "@/scaffold/templates/datasets.ts.txt";
 import tsconfigJson from "@/scaffold/templates/tsconfig.json.txt";
 import gitignoreText from "@/scaffold/templates/gitignore.txt";
+import dockerignoreText from "@/scaffold/templates/dockerignore.txt";
 import eslintConfig from "@/scaffold/templates/eslint.config.mjs.txt";
 import prettierConfig from "@/scaffold/templates/prettier.config.mjs.txt";
 import fallowrc from "@/scaffold/templates/fallowrc.jsonc.txt";
+// Project jargon is the project's to add. Only `mprocs`, which its own
+// package.json names, is seeded: a stranger's vocabulary is noise in a repo.
+import cspellJson from "@/scaffold/templates/cspell.json.txt";
 import { applyPorts, type Ports } from "@/scaffold/ports";
 
-export type Database = "postgres" | "mssql";
-
-// The generated project depends on these directly, not transitively: its own code
-// imports `zod` for tool schemas and `hono` for extra routes on `server.app`.
-// The ranges are packages/cortex-server's own peers, which is the compatibility
-// statement the server itself publishes, and a server range pinned to its
-// current minor, since a minor bump is where the server breaks; the release
-// sync (scripts/sync-package-versions.ts) restamps it and files.test.ts holds
-// them to that.
-export const DEPENDENCIES = {
-    "@m6d/cortex-server": "~0.12.1",
-    "@tanstack/ai": "^0.55.0",
-    "@tanstack/ai-openai": "^0.22.8",
-    hono: "^4.12.3",
-    jose: "^6.1.3",
-    openai: "^6",
-    zod: "^4.1.8",
-};
+/**
+ * The release of the Cortex image the generated stack runs. It and the
+ * `@m6d/cortex` range below are stamped by scripts/sync-package-versions.ts
+ * before the CLI publishes, so a scaffold pins what was current then.
+ */
+const CORTEX_VERSION = "0.14.1";
+
+/** This machine as Cortex's containers name it: the one private host the stack's egress policy admits. */
+export const DOCKER_HOST = "host.docker.internal";
+
+/** How Cortex's containers reach the backend on this machine, at the port drawn for it. */
+const SERVER_URL = `http://${DOCKER_HOST}:{{port.server}}`;
+
+/** The longest server name Cortex accepts. */
+const SERVER_NAME_MAX = 100;
+
+/**
+ * The agent the manifest declares and the README sends you to chat with.
+ * Not `assistant`: Cortex holds that slug for its console's own assistant.
+ */
+export const AGENT_SLUG = "helper";
+
+/**
+ * The embedding model the stack asks for, and the dimension the image's
+ * migrations give the vector columns. `cortex setup` probes a provider with
+ * the same pair before reusing it for embeddings.
+ */
+export const EMBEDDING = { model: "text-embedding-3-small", dimension: 768 } as const;
 
 const DEV_DEPENDENCIES = {
+    // `cortex apply`, `cortex setup` and the management commands, at the release that wrote the project.
+    "@m6d/cortex-cli": `~${pkg.version}`,
     "@types/bun": "latest",
+    // The two runners of `bun run dev` and `bun run dev:agent`.
+    concurrently: "^10.0.6",
     cspell: "^9.6.4",
     eslint: "^9",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.5",
     fallow: "^3.9.1",
+    mprocs: "^0.9.6",
     prettier: "^3.6.2",
     typescript: "^5",
     "typescript-eslint": "^8.65.0",
 };
 
-// Postgres is the one we run, so it gets a scaffolded port. MSSQL is normally an
-// existing instance nobody moved, which is why it keeps 1433 and has no service.
-const CONNECTION_STRING = {
-    postgres: "postgres://postgres:postgres@localhost:{{port.postgres}}/cortex",
-    mssql: "Server=localhost,1433;Database=cortex;User Id=sa;Password=your-password;Encrypt=false",
-} satisfies Record<Database, string>;
+/**
+ * `setup` puts `cortex setup` among the processes `bun run dev` starts; `env`
+ * holds values `cortex new` was given for `.env`, such as the model provider,
+ * which is then written beside `.env.example` with them filled in.
+ */
+type Input = {
+    name: string;
+    datasets: boolean;
+    setup: boolean;
+    ports: Ports;
+    env?: Record<string, string>;
+};
+
+/** The input, and the name the backend is registered under in Cortex. */
+type Project = Input & { server: string };
+
+/**
+ * `text` with each `KEY=` line named in `values` set to its value, and every
+ * other line left as written. `cortex new` fills `.env` with it, and
+ * `cortex setup` the `.env` it asks the model provider for.
+ */
+export function withEnvValues(text: string, values: Record<string, string>) {
+    return Object.entries(values).reduce(
+        (filled, [key, value]) =>
+            filled.replace(new RegExp(`^${key}=.*$`, "m"), () => `${key}=${value}`),
+        text,
+    );
+}
+
+/** What `bun run dev` starts side by side, named as the runners label them. */
+function devProcesses(setup: boolean) {
+    return {
+        cortex: "docker compose up",
+        server: "bun run dev:server",
+        ...(setup ? { setup: "bun cortex setup --wait --yes" } : {}),
+    };
+}
+
+/** One runner's command line: the process names, then each command quoted. */
+function runAll(runner: "mprocs" | "concurrently", processes: Record<string, string>) {
+    const commands = Object.values(processes).map((command) => `"${command}"`);
+    return `${runner} --names ${Object.keys(processes).join(",")} ${commands.join(" ")}`;
+}
+
+/** The name as the manifest `id` takes it: lowercase words and hyphens. */
+export function manifestIdOf(name: string) {
+    return name
+        .replace(/[^a-z0-9]+/g, "-")
+        .slice(0, 64)
+        .replace(/^-+|-+$/g, "");
+}
+
+function packageJson({ name, datasets, setup }: Project) {
+    const processes = devProcesses(setup);
 
-function packageJson(name: string) {
     return `${JSON.stringify(
         {
             name,
@@ -69,14 +136,26 @@ function packageJson(name: string) {
                 // analyses — dead code, duplication, complexity — over everything.
                 audit: "fallow",
                 check: "tsc --noEmit",
-                dev: "bun run --hot index.ts",
+                // The same processes twice: a dashboard for people, and one
+                // labelled log for coding agents and CI.
+                dev: runAll("mprocs", processes),
+                "dev:agent": runAll("concurrently", processes),
+                "dev:server": "bun run --hot index.ts",
+                // A no-op where setup.ts is not copied in, as in an image build.
+                postinstall: "bun run --if-present setup.ts",
                 format: "prettier --write .",
                 "format:check": "prettier --check .",
                 lint: "eslint",
                 spellcheck: 'cspell "**/*.{ts,mjs,json,md}"',
                 verify: "bun run format:check && bun run lint && bun run spellcheck && bun run check && bun run audit",
             },
-            dependencies: DEPENDENCIES,
+            dependencies: {
+                "@m6d/cortex": "~0.2.0",
+                // drizzle 1.0 is in beta; this is the build `@m6d/cortex/drizzle` is tested against.
+                ...(datasets ? { "drizzle-orm": "1.0.0-beta.15-859cf75" } : {}),
+                hono: "^4.12.3",
+                zod: "^4.6.5",
+            },
             devDependencies: DEV_DEPENDENCIES,
         },
         null,
@@ -84,275 +163,359 @@ function packageJson(name: string) {
     )}\n`;
 }
 
-/**
- * Never toggled, so they only ever appear commented out: `context` is fully
- * defaulted, and `reranker` is what you reach for once graph retrieval comes
- * back noisy, not on day one.
- */
-const NEVER_SCAFFOLDED = [
-    [
-        "Re-ranks knowledge-graph retrieval when it comes back noisy.",
-        `reranker: {
-    url: process.env["CORTEX_RERANKER_URL"]!,
-    apiKey: process.env["CORTEX_RERANKER_KEY"]!,
-},`,
-    ],
-    [
-        "Context-window tuning. Every field is defaulted, so override only what you\nhave measured a reason to change.",
-        `context: {
-    maxContextTokens: 120_000,
-    recentMessagesToKeep: 6,
-},`,
-    ],
-] satisfies [note: string, config: string][];
-
-/** Fragments are authored at zero indentation and placed by these two. */
-function indented(block: string) {
-    return block.replace(/^(?!$)/gm, "    ");
-}
+function indexTs({ datasets }: Project) {
+    const routes = datasets
+        ? "its server protocol under /cortex and its\n * dataset host under /datasets; your own routes go beside them"
+        : "its server protocol under /cortex; your own routes go beside it";
 
-function commented(block: string) {
-    return block.replace(/^(?!$)/gm, "    // ");
-}
-
-/** A note, then its fragment — written live, or written disabled. */
-function block(note: string, config: string, enabled: boolean) {
-    return `\n${commented(note)}\n${enabled ? indented(config) : commented(config)}\n`;
-}
+    return `import { Hono } from "hono";
 
-function cortexConfig(database: Database, features: Feature[]) {
-    const fragment = (feature: Feature) =>
-        block(FEATURES[feature].note, FEATURES[feature].config, features.includes(feature));
-
-    const on = features.map(fragment).join("");
-    const off = FEATURE_NAMES.filter((feature) => !features.includes(feature))
-        .map(fragment)
-        .join("");
-    const never = NEVER_SCAFFOLDED.map(([note, config]) => block(note, config, false)).join("");
-
-    // cspell.json ships an empty word list on purpose, so the Cortex jargon the
-    // config names teaches the checker from the file that uses it.
-    return `import { defineAgent } from "@m6d/cortex-server";
-import type { CortexConfig } from "@m6d/cortex-server";
-
-// cspell:words reranker
+${datasets ? `import { datasets } from "./datasets";\n` : ""}import { server } from "./server";
 
 /**
- * Inert on purpose: this file default-exports data and nothing else, so importing
- * it never opens a connection or binds a port. Construction lives in index.ts,
- * which is what lets tooling read this config without booting the server.
+ * This backend. Cortex reaches ${routes}.
  */
-export default {
-    port: 3331,
-    database: {
-        type: "${database}",
-        connectionString: process.env["CORTEX_DATABASE_URL"]!,
-    },
-    model: {
-        baseURL: process.env["CORTEX_MODEL_URL"]!,
-        apiKey: process.env["CORTEX_MODEL_KEY"]!,
-        modelName: process.env["CORTEX_MODEL_NAME"]!,
-    },
-${on === "" ? "" : `${on}\n`}    agents: {
-        assistant: defineAgent({
-            systemPrompt: "You are a helpful assistant.",
-            // Applications that serve datasets for grouped questions (see the
-            // README's "Dataset hosts"): the host applies its own access rules
-            // under the run's token the list below forwards, the agent only
-            // picks which datasets to expose and pin.
-            // datasetHosts: [{ name: "store", url: "https://store.example/api/datasets", credentials: [{ header: "X-End-User-Token", credential: { kind: "run-token" } }], pinned: ["orders"] }],
-        }),
-    },
-
-    // Everything below is off, written out so the file documents what else the
-    // server does. Uncomment a block and fill in its keys in .env to enable it.
-${off}${never}} satisfies CortexConfig;
+const app = new Hono();
+app.route("/cortex", server);
+${datasets ? `app.route("/datasets", datasets);\n` : ""}
+export default { port: {{port.server}}, fetch: app.fetch };
 `;
 }
 
-/**
- * Whether this selection runs anything of its own — which is the same question as
- * "did any port get chosen", since a chosen port is always some service's.
+function manifestTs({ name, datasets }: Project) {
+    const fromEnv = datasets
+        ? `/** The server this backend is registered as, and where Cortex reaches it, from .env. */
+const { CORTEX_SERVER_NAME: server, CORTEX_SERVER_URL: serverUrl } = process.env;
+if (!server || !serverUrl) {
+    throw new Error("Apply from this project: its .env names the server.");
+}`
+        : `/** The server this backend is registered as, from .env. */
+const server = process.env["CORTEX_SERVER_NAME"];
+if (!server) throw new Error("Apply from this project: its .env names the server.");`;
+    const datasetHosts = `
+    // The orders this backend serves (datasets.ts), for every agent on its server.
+    datasetHosts: [
+        {
+            name: "store",
+            url: \`\${serverUrl}/datasets\`,
+            credentials: [],
+            expose: ["orders"],
+            pinned: ["orders"],
+            audience: { server },
+        },
+    ],`;
+
+    return `import { defineManifest } from "@m6d/cortex";
+
+/*
+ * What this project declares in Cortex, which \`cortex apply\` makes Cortex
+ * match. The agent runs on this backend's server, whose variable its prompt
+ * reads and whose native tool it calls.
  */
-function hasServices(database: Database, features: Feature[]) {
-    return (
-        database === "postgres" ||
-        features.some((feature) => FEATURES[feature].compose !== undefined)
-    );
+
+${fromEnv}
+
+export default defineManifest({
+    id: ${JSON.stringify(manifestIdOf(name))},
+    agents: [
+        {
+            slug: "${AGENT_SLUG}",
+            name: "Helper",
+            server,
+            prompt: "You are a helpful assistant. Address the user as {{userName}}.",
+            // No identity provider yet: every caller is one shared identity.
+            auth: { kind: "anonymous" },
+        },
+    ],${datasets ? datasetHosts : ""}
+});
+`;
 }
 
-function env(database: Database, features: Feature[]) {
-    const blocks = features.map((feature) => `\n${FEATURES[feature].env}\n`);
-
-    // Only the URLs pointing at scaffolded services carry a chosen port, and those
-    // are exactly the ones docker-compose.yml publishes.
-    const note = hasServices(database, features)
-        ? `#
-# The service ports below are not the canonical ones. They were picked when this
-# project was scaffolded, from what was free on the machine, so it does not
-# collide with anything else you run. Change them freely — in step with
-# docker-compose.yml.
-`
-        : "";
-
-    return `# Environment for this Cortex server. Bun loads this file from the directory you
-# run in, so it has to sit next to cortex.config.ts.
+/** Every setting the project has, which setup.ts copies into .env, generating the secrets. */
+function envExample({ server }: Project) {
+    return `# Every setting this project has. Bun loads .env from the directory you run
+# in, into the backend and the \`cortex\` CLI; docker compose hands it to the
+# Cortex containers and reads COMPOSE_PROJECT_NAME from it.
 #
-# .env.example is the committed copy. Keep the two in step; keep secrets in .env.
-${note}
-# --- Model -------------------------------------------------------------------
-# Any OpenAI-compatible endpoint: base URL, key, and model name.
-CORTEX_MODEL_URL=https://api.openai.com/v1
-CORTEX_MODEL_KEY=your-model-api-key
-CORTEX_MODEL_NAME=your-model-name
-
-# --- Database ----------------------------------------------------------------
-# The dialect is ${database}, set in cortex.config.ts. Change it there, not here.
-CORTEX_DATABASE_URL=${CONNECTION_STRING[database]}
-${blocks.join("")}`;
+# This is the committed copy. \`bun install\` writes .env from it, generating
+# the secrets it leaves blank, and never changes a value .env already holds.
+
+# The Compose project this checkout's Cortex stack runs as, drawn for each
+# checkout, so a clone never shares another checkout's containers or volumes.
+COMPOSE_PROJECT_NAME=
+
+# Cortex's first admin, created when the stack first starts, and its
+# management key.
+DEFAULT_ADMIN_EMAIL=admin@example.com
+DEFAULT_ADMIN_PASSWORD=
+BOOTSTRAP_MANAGEMENT_KEY=
+# Session signing, and the key the secrets store is encrypted with.
+BETTER_AUTH_SECRET=
+SECRETS_MASTER_KEY=
+# Any OpenAI-compatible embeddings endpoint and model. Tool search and
+# knowledge collections embed with it, and stay off while the key is blank.
+EMBEDDING_BASE_URL=https://api.openai.com/v1
+EMBEDDING_API_KEY=
+EMBEDDING_MODEL=${EMBEDDING.model}
+# The private hosts Cortex's containers may call, and send a secret to over
+# plain http, comma-separated: this machine, for the backend. A model provider
+# on a private network goes here too. The policy admits a host by name,
+# whatever the port: see README.md.
+EGRESS_ALLOWED_HOSTS=${DOCKER_HOST}
+
+# The Cortex \`cortex apply\` and \`cortex setup\` talk to: this checkout's,
+# under its management key, so neither needs \`cortex login\`. \`-p <profile>\`
+# still picks a saved one.
+CORTEX_URL=http://localhost:{{port.cortex}}
+CORTEX_KEY=\${BOOTSTRAP_MANAGEMENT_KEY}
+
+# This backend as Cortex registers it: its name, where Cortex's containers
+# reach it, and the key Cortex presents on every call to it.
+CORTEX_SERVER_NAME=${server}
+CORTEX_SERVER_URL=${SERVER_URL}
+CORTEX_SERVER_KEY=
+
+# The model provider \`cortex setup\` registers and the server's agents run on.
+# While these are blank, setup registers the server and applies nothing.
+MODEL_PROVIDER_NAME=
+MODEL_PROVIDER_URL=
+MODEL_PROVIDER_API_KEY=
+MODEL_PROVIDER_MODEL=
+`;
 }
 
-/**
- * Services the selections actually need, and nothing else. `auth` and the
- * Control Center are external, the embedding endpoint is a URL like the main
- * model, and MSSQL is normally an existing instance — so with `--yes --database
- * mssql` there is no service at all, and no file is written.
- */
-function dockerCompose(name: string, database: Database, features: Feature[]) {
-    const services: string[] = [];
-    const volumes: string[] = [];
+/** Cortex itself: its app, worker and engine, all on the one Cortex image, and their stores. */
+function dockerCompose() {
+    return `# Cortex, for this project's agents to run on: the console at
+# http://localhost:{{port.cortex}}, the engine widgets and trigger callers reach at
+# http://localhost:{{port.engine}}, and the stores they keep state in. \`bun run dev\`
+# starts it; .env holds its secrets.
+#
+# The host ports were picked at scaffold time from what was free on this
+# machine. Move one and the URLs that name it here, in .env and in README.md
+# follow.
+# CORTEX_VERSION, set in the shell, runs another release of the image.
+#
+# The console's browser uploads (knowledge documents) go straight to
+# S3_ENDPOINT, which only this stack resolves. Point it at storage the
+# browser reaches too before relying on them.
+
+# The stack runs as .env's COMPOSE_PROJECT_NAME, drawn for each checkout by
+# setup.ts, so no two checkouts share containers or volumes, whatever their
+# directories are called.
+
+x-cortex: &cortex
+    image: syscodeia/cortex:\${CORTEX_VERSION:-${CORTEX_VERSION}}
+    working_dir: /repo/apps/cortex
+    # The whole .env, the backend's settings too. What \`environment\` sets
+    # wins over it, so nothing there can move these containers' ports or mode.
+    env_file: .env
+    environment: &cortex-environment
+        NODE_ENV: production
+        PORT: "3000"
+        DATABASE_URL: postgres://postgres:postgres@postgres:5432/cortex
+        REDIS_URL: redis://redis:6379
+        S3_ENDPOINT: http://minio:9000
+        S3_REGION: us-east-1
+        S3_ACCESS_KEY_ID: minio
+        S3_SECRET_ACCESS_KEY: minio-secret-key
+        S3_BUCKET: cortex
+        BETTER_AUTH_ALLOWED_HOSTS: localhost:{{port.cortex}},127.0.0.1:{{port.cortex}}
+        # The dimension the image's migrations give the vector columns.
+        EMBEDDING_DIMENSION: "${EMBEDDING.dimension}"
+        # The engine's public origin (browsers, trigger callers) and the
+        # address the app reaches it at inside the stack.
+        ENGINE_PUBLIC_URL: http://localhost:{{port.engine}}
+        ENGINE_INTERNAL_URL: http://engine:3001
+    # Docker Desktop resolves host.docker.internal on its own; Linux needs this.
+    extra_hosts:
+        - host.docker.internal:host-gateway
 
-    if (database === "postgres") {
-        services.push(`    postgres:
-        image: postgres:17-alpine
+services:
+    postgres:
+        image: pgvector/pgvector:pg17
+        restart: unless-stopped
         environment:
             POSTGRES_USER: postgres
             POSTGRES_PASSWORD: postgres
             POSTGRES_DB: cortex
-        ports:
-            - "{{port.postgres}}:5432"
         volumes:
-            - postgres-data:/var/lib/postgresql/data`);
-        volumes.push("postgres-data");
-    }
-
-    for (const feature of features) {
-        const { compose, volume } = FEATURES[feature];
-        if (compose === undefined) continue;
-        services.push(compose);
-        if (volume !== undefined) volumes.push(volume);
-    }
-
-    if (services.length === 0) return undefined;
-
-    const volumeBlock =
-        volumes.length === 0
-            ? ""
-            : `\nvolumes:\n${volumes.map((name) => `    ${name}:\n`).join("")}`;
-
-    return `# Local infrastructure for this project. The .env this was scaffolded with
-# already points at these, so \`docker compose up -d\` is enough.
-#
-# The host ports were picked at scaffold time from what was free on the machine —
-# the container ports next to them are the canonical ones and should stay put.
-# Move a host port and .env has to follow.
-
-# Explicit so the project never collides with another checkout whose directory
-# happens to share this one's name (Compose defaults to the directory name, and
-# a name clash silently replaces the other project's containers and volumes).
-name: ${name}-cortex-server
-
-services:
-${services.join("\n\n")}
-${volumeBlock}`;
+            - postgres-data:/var/lib/postgresql/data
+        healthcheck:
+            test: ["CMD-SHELL", "pg_isready -U postgres -d cortex"]
+            interval: 5s
+            timeout: 5s
+            retries: 10
+
+    redis:
+        image: redis:7-alpine
+        restart: unless-stopped
+        healthcheck:
+            test: ["CMD", "redis-cli", "ping"]
+            interval: 5s
+            timeout: 5s
+            retries: 10
+
+    minio:
+        # A community rebuild of MinIO's last free release, pinned by digest:
+        # MinIO's own images left Docker Hub. It runs as nobody, so it takes
+        # root to reuse a volume another image wrote.
+        image: sourcemation/minio@sha256:538065d2dd0ef360fe3f22f25d2384e6bae9075b6d2603cb5587193bd73e6276
+        user: "0:0"
+        restart: unless-stopped
+        command: server /data
+        environment:
+            MINIO_ROOT_USER: minio
+            MINIO_ROOT_PASSWORD: minio-secret-key
+        volumes:
+            - minio-data:/data
+        healthcheck:
+            test: ["CMD", "bash", "-c", "</dev/tcp/127.0.0.1/9000"]
+            interval: 5s
+            timeout: 5s
+            retries: 10
+
+    minio-init:
+        image: chainguard/minio-client@sha256:b2bd7824d23d3e3b15bedd7e87fbc3be29d2e213307b4f901e4a1d92356dc20f
+        depends_on:
+            minio:
+                condition: service_healthy
+        environment:
+            MC_HOST_local: http://minio:minio-secret-key@minio:9000
+        command: ["mb", "--ignore-existing", "local/cortex"]
+
+    # One-shot: migrate the database, then create the first admin and its key.
+    migrator:
+        <<: *cortex
+        command: ["bun", "run", "db:migrate"]
+        depends_on:
+            postgres:
+                condition: service_healthy
+
+    seeder:
+        <<: *cortex
+        command: ["bun", "run", "db:seed:admin"]
+        depends_on:
+            migrator:
+                condition: service_completed_successfully
+
+    # The console and the management API.
+    app:
+        <<: *cortex
+        restart: unless-stopped
+        ports:
+            - "127.0.0.1:{{port.cortex}}:3000"
+        depends_on:
+            seeder:
+                condition: service_completed_successfully
+            minio-init:
+                condition: service_completed_successfully
+
+    worker:
+        <<: *cortex
+        restart: unless-stopped
+        command: ["bun", "src/jobs/worker.ts"]
+        depends_on:
+            seeder:
+                condition: service_completed_successfully
+            minio-init:
+                condition: service_completed_successfully
+            redis:
+                condition: service_healthy
+
+    # Runs the agents: widgets, trigger callers and the console's playground
+    # and assistant talk to it.
+    engine:
+        <<: *cortex
+        restart: unless-stopped
+        command: ["bun", "src/engine/server.ts"]
+        environment:
+            <<: *cortex-environment
+            PORT: "3001"
+            # Browser origins allowed to call the engine: the console's, for its playground and assistant.
+            ENGINE_CORS_ORIGINS: http://localhost:{{port.cortex}},http://127.0.0.1:{{port.cortex}}
+        ports:
+            - "127.0.0.1:{{port.engine}}:3001"
+        depends_on:
+            seeder:
+                condition: service_completed_successfully
+            redis:
+                condition: service_healthy
+
+volumes:
+    postgres-data:
+    minio-data:
+`;
 }
 
-function cspellConfig() {
-    return `${JSON.stringify(
-        {
-            version: "0.2",
-            language: "en",
-            // Yours to fill in. Project jargon belongs here; nothing is seeded,
-            // because a stranger's vocabulary is noise in your repo.
-            words: [],
-            ignorePaths: [
-                "node_modules/**",
-                "out/**",
-                "build/**",
-                "**/dist/**",
-                "coverage/**",
-                "*.tsbuildinfo",
-                "bun.lock",
-                "package-lock.json",
-                "pnpm-lock.yaml",
-                "yarn.lock",
-                "*.db",
-                "*.sqlite",
-                "scratch.*",
-                ".env*",
-                "**/*.min.js",
-                "**/*.min.css",
-                "**/db/migrations/**",
-                "**/i18n/**",
-                "docs",
-                ".claude",
-                ".angular",
-                ".vercel/**",
-                "vendor/**",
-                ".gitignore",
-                "**/domains/**/*.concept.ts",
-            ],
-            dictionaries: [
-                "typescript",
-                "node",
-                "npm",
-                "html",
-                "css",
-                "bash",
-                "companies",
-                "softwareTerms",
-            ],
-        },
-        null,
-        4,
-    )}\n`;
-}
+function readme({ name, server, datasets, setup }: Project) {
+    const gettingStarted = setup
+        ? `That starts Cortex (\`docker compose up\`), this backend, and \`cortex setup\` in one dashboard.
+Setup waits for Cortex, registers this backend as the server \`${server}\`, adds the model provider
+in \`.env\`, and applies the manifest. Then it prints where to chat with \`${AGENT_SLUG}\`, and the first
+time, how to sign in. It adds only what is missing, so every \`bun run dev\` runs it again.
+
+While \`MODEL_PROVIDER_*\` in \`.env\` is blank, setup registers the server and applies nothing:
+Cortex publishes no agent without a model. Run \`bun cortex setup\` in a terminal to be asked for
+one.`
+        : `That starts Cortex (\`docker compose up\`) and this backend in one dashboard. Then register the
+backend and apply the manifest, either way:
+
+- \`bun cortex setup\` in a terminal asks for a model provider, registers the server
+  \`${server}\`, and applies the manifest.
+- By hand: open the console at http://localhost:{{port.cortex}} and sign in with
+  \`DEFAULT_ADMIN_EMAIL\` and \`DEFAULT_ADMIN_PASSWORD\` from \`.env\`. Add a model provider on the
+  **Model providers** page, and store \`CORTEX_SERVER_KEY\` from \`.env\` on the **Secrets** page.
+  On the **Servers** page, add a server named \`${server}\` at \`${SERVER_URL}/cortex\` with that
+  secret as its key, and pick the model its agents run on. Then run \`bun cortex apply\`.`;
 
-function readme(name: string, database: Database, features: Feature[], compose: boolean) {
     return `# ${name}
 
-A Cortex server: one agent, a ${database} database, and an OpenAI-compatible model.${
-        features.length === 0 ? "" : ` Also on: ${features.join(", ")}.`
-    }
+A backend for Cortex agents, and Cortex itself to run them on.
+
+- \`server.ts\` is this backend's Cortex server: the \`userName\` variable the agents' prompts read,
+  and the \`rollDice\` tool it runs for them.${
+      datasets
+          ? "\n- `datasets.ts` serves an `orders` table as a dataset host, for agents to ask grouped\n  questions of."
+          : ""
+  }
+- \`cortex.manifest.ts\` declares the \`${AGENT_SLUG}\` agent on this server${
+        datasets ? " and the dataset host" : ""
+    }. \`cortex apply\` makes Cortex match it.
+- \`docker-compose.yml\` runs Cortex: its app, worker and engine, and the stores they use.
+- \`.env\` holds every setting: Cortex's secrets, this backend's key, and the model provider.
+  \`setup.ts\` runs on every \`bun install\`: it writes \`.env\` from the committed \`.env.example\` and
+  generates each secret left blank, so a fresh clone gets its own secrets and its own Compose stack.
+  It never changes a value already set. \`.dockerignore\` keeps both files out of any image built
+  here.
 
 ## Getting started
 
+On a fresh clone, run \`bun install\` first (\`cortex new\` already did): it writes \`.env\`.
+
 \`\`\`sh
-bun install
-${compose ? "docker compose up -d\n" : ""}# fill in CORTEX_MODEL_* in .env
 bun run dev
 \`\`\`
 
-The server listens on http://localhost:3331. Change \`port\` in \`cortex.config.ts\` to move it.${
-        compose
-            ? `
+${gettingStarted}
 
-The service ports in \`docker-compose.yml\` are not the canonical ones: they were picked at scaffold
-time from what was free on this machine, so this project does not fight whatever else you run.
-\`.env\` already points at them.`
-            : ""
-    }
+Then chat with \`${AGENT_SLUG}\` on the console's **Playground** page. Edit \`cortex.manifest.ts\` and run
+\`bun cortex apply\` to change it; an apply changes only what the edit changed. The CLI reaches this
+checkout's Cortex through \`CORTEX_URL\` and \`CORTEX_KEY\` in \`.env\`, so it needs no login, and
+\`-p <profile>\` picks a saved one instead.
 
-## Layout
+\`bun run dev:agent\` runs the same processes without the dashboard, each log line labelled with
+its process, for coding agents and CI.
 
-- \`cortex.config.ts\` — inert data. It default-exports a \`CortexConfig\` and never constructs
-  anything, so tooling can import it without booting a server.
-- \`index.ts\` — construction. It reads the config, calls \`createCortex\`, serves, and is where your
-  own routes go.
-- \`.env\` — local secrets, loaded by Bun from the directory you run in. \`.env.example\` is the
-  committed copy.
+The backend listens on http://localhost:{{port.server}}. Move it in \`index.ts\`, and
+\`CORTEX_SERVER_URL\` and the server's URL in Cortex with it.
 
-Commented-out sections in \`cortex.config.ts\` document what else the server can do.
+## Reaching this machine
+
+Cortex's containers reach the backend at \`host.docker.internal\`, which \`EGRESS_ALLOWED_HOSTS\` in
+\`.env\` admits. The policy admits a host by name, not by port, so every HTTP tool, MCP
+server and credential you define in Cortex can reach any port this machine listens
+on, over plain http. Keep the stack on your own machine, and define only tools you trust.
 
 ## Checks
 
@@ -363,36 +526,37 @@ pre-commit hook or CI job should run. Each is also available on its own: \`forma
 }
 
 /**
- * Relative path → contents. Nothing here touches the filesystem. Every
- * generator lists features in catalogue order, whatever order they came in.
+ * Relative path → contents. Nothing here touches the filesystem; the ports
+ * are drawn by the caller.
  */
-export function scaffoldFiles(name: string, database: Database, selected: Feature[], ports: Ports) {
-    const features = FEATURE_NAMES.filter((feature) => selected.includes(feature));
-    const envFile = env(database, features);
-    const compose = dockerCompose(name, database, features);
-
+export function scaffoldFiles(input: Input) {
+    const project = { ...input, server: input.name.slice(0, SERVER_NAME_MAX) };
     const files = {
-        "cortex.config.ts": cortexConfig(database, features),
-        "index.ts": indexTs,
-        "package.json": packageJson(name),
+        "server.ts": serverTs,
+        "setup.ts": setupTs,
+        ...(project.datasets ? { "datasets.ts": datasetsTs } : {}),
+        "index.ts": indexTs(project),
+        "cortex.manifest.ts": manifestTs(project),
+        "docker-compose.yml": dockerCompose(),
+        "package.json": packageJson(project),
         "tsconfig.json": tsconfigJson,
-        ".env": envFile,
-        ".env.example": envFile,
+        ".env.example": envExample(project),
+        ...(project.env ? { ".env": withEnvValues(envExample(project), project.env) } : {}),
         ".gitignore": gitignoreText,
-        "README.md": readme(name, database, features, compose !== undefined),
+        ".dockerignore": dockerignoreText,
+        "README.md": readme(project),
         "eslint.config.mjs": eslintConfig,
         "prettier.config.mjs": prettierConfig,
-        "cspell.json": cspellConfig(),
+        "cspell.json": cspellJson,
         ".fallowrc.jsonc": fallowrc,
-        ...(compose === undefined ? {} : { "docker-compose.yml": compose }),
-        // The directory `domainsDir` defaults to, which makes the convention true
-        // by construction: `cortex swagger sync` finds it with no config key.
-        ...(features.includes("knowledge") ? { "src/domains/README.md": domainsReadme } : {}),
     } satisfies Record<string, string>;
 
     // One pass at the end, over everything: no generator holds a port number of
-    // its own, so compose, .env and the README cannot disagree about one.
+    // its own, so compose and the README cannot disagree about one.
     return Object.fromEntries(
-        Object.entries(files).map(([path, contents]) => [path, applyPorts(contents, ports)]),
+        Object.entries(files).map(([path, contents]) => [
+            path,
+            applyPorts(contents, project.ports),
+        ]),
     );
 }