@m6d/cortex-cli 0.1.0

package/src/commands/graph.ts

@@ -0,0 +1,229 @@
+import type { Command } from "@commander-js/extra-typings";
+import {
+    GRAPH_SCHEMA,
+    GRAPH_SCHEMA_VERSION,
+    createEmbedder,
+    createNeo4jClient,
+    type DomainDef,
+    type EmbeddingProviderConfig,
+    type Neo4jClient,
+    type Neo4jConfig,
+} from "@cortex/contracts/graph";
+import pc from "picocolors";
+
+import { configLabel, configOption, openProject } from "@/config/load";
+import { seedGraph } from "@/graph/seed";
+import * as report from "@/ui/report";
+
+const TITLE = "cortex graph seed";
+
+/** Domain names are longer than swagger's finding labels, so the body is wider. */
+const DOMAIN_COLUMN = 14;
+
+/** A failing statement is worth one line: enough to identify, not enough to drown. */
+const DETAIL_WIDTH = 60;
+
+/** Only the parts of a loaded config `graph seed` reads. */
+type ProjectConfig = {
+    neo4j?: Neo4jConfig;
+    embedding?: EmbeddingProviderConfig & { dimension: number };
+    knowledge?: { domains?: Record<string, DomainDef> };
+    agents?: Record<string, { knowledge?: { domains?: Record<string, DomainDef> } | null }>;
+};
+
+// Connection failures reach us as whatever `fetch` threw, so match on shape
+// rather than on a driver's error class.
+const UNREACHABLE =
+    /unable to connect|connection refused|econnrefused|failed to fetch|fetch failed|enotfound|getaddrinfo|timed out/i;
+
+export function registerGraph(program: Command) {
+    const graph = program
+        .command("graph")
+        .helpGroup("Resources:")
+        .summary("The Neo4j knowledge graph")
+        .description(
+            "The Neo4j knowledge graph.\n\nReads `knowledge.domains` and `neo4j` from your cortex.config.ts.",
+        )
+        .addOption(configOption())
+        .helpCommand(false);
+
+    // No flags. `--domain <name>` is flag-shaped design work, not a flag:
+    // `expandDomains` resolves cross-domain references and the node pass creates every
+    // node before any edge, so a subset seed risks dangling edges.
+    graph
+        .command("seed")
+        .description("Load concepts, endpoints, services, and rules into Neo4j")
+        .action(async function () {
+            await seed(graph);
+        });
+}
+
+async function seed(group: Command) {
+    const { config: loaded, configPath, verbose } = await openProject(group, TITLE);
+    const config = loaded as ProjectConfig;
+
+    // Inputs are reported as they resolve, and only when the config has them:
+    // an absent section is `runSeed`'s error to throw, not ours to predict.
+    if (config.neo4j?.url) report.input("Neo4j", config.neo4j.url);
+    const domains = collectDomains(config);
+    const names = Object.keys(domains);
+    if (names.length) report.input("Domains", `${names.length} — ${names.join(", ")}`);
+
+    const startedAt = Date.now();
+    const result = await runSeed(config, domains).catch(function (error: unknown): never {
+        return failSeed({ error, config, configPath, verbose });
+    });
+
+    // A domain is the unit the user edits, so it is the unit the report counts
+    // against: a wrong number next to `attendance` points at src/domains/attendance/.
+    report.blank();
+    for (const [name, domain] of Object.entries(result.domains)) {
+        const counts = [report.plural(domain.nodes, "node"), report.plural(domain.edges, "edge")];
+        if (domain.failed) counts.push(`${domain.failed} failed`);
+        report.row(name, counts.join(" · "), DOMAIN_COLUMN);
+
+        // The first failure only: enough to identify what broke, not enough to
+        // drown the run — the count above says how many followed it.
+        const failure = domain.failures[0];
+        if (failure) {
+            report.row("", `${pc.red("✗")} ${truncate(failure.error)}`, DOMAIN_COLUMN);
+            report.row("", `  ${truncate(oneLine(failure.statement))}`, DOMAIN_COLUMN);
+        }
+    }
+
+    // Embeddings are a second pass over the whole graph, so they get their own
+    // step rather than a row among the counts.
+    report.blank();
+    report.input(
+        "Embeddings",
+        `${report.plural(result.embeddings.concepts, "concept")}, dimension ${result.embeddings.dimension}`,
+    );
+    report.blank();
+
+    const counts = [report.plural(result.nodes, "node"), report.plural(result.edges, "edge")];
+    if (result.failed) counts.push(`${result.failed} failed`);
+    report.footer(`${counts.join(" · ")}  (${((Date.now() - startedAt) / 1000).toFixed(1)}s)`);
+
+    if (result.failed) {
+        report.note("The graph is partially seeded. Re-running is safe — seeding is MERGE-based.");
+        process.exit(1);
+    }
+}
+
+/** Cypher is generated multi-line and indented; a report row is neither. */
+function oneLine(statement: string) {
+    return statement.replace(/\s+/g, " ").trim();
+}
+
+function truncate(text: string) {
+    return text.length > DETAIL_WIDTH ? `${text.slice(0, DETAIL_WIDTH - 2)} …` : text;
+}
+
+/** Server level first, then per agent — an agent's domain of the same name wins. */
+function collectDomains(config: ProjectConfig) {
+    const all: Record<string, DomainDef> = { ...config.knowledge?.domains };
+
+    for (const agent of Object.values(config.agents ?? {})) {
+        Object.assign(all, agent.knowledge?.domains);
+    }
+
+    return all;
+}
+
+/**
+ * The whole pipeline is the CLI's, built on the graph contract it depends on:
+ * `@cortex/contracts/graph` is where the vocabulary, the Neo4j client and the embedder
+ * come from, and the version it seeds at is the version of that dependency.
+ */
+async function runSeed(config: ProjectConfig, domains: Record<string, DomainDef>) {
+    if (!Object.keys(domains).length) throw new Error("No domains found in knowledge config");
+    if (!config.neo4j) throw new Error("neo4j config is required for graph seed");
+    if (!config.embedding) throw new Error("embedding config is required for graph seed");
+
+    const embed = createEmbedder(config.embedding);
+    const neo4j = createNeo4jClient(config.neo4j, embed);
+
+    await warnOnRestamp(neo4j);
+
+    return await seedGraph(
+        {
+            neo4j,
+            embedding: { embed, dimension: config.embedding.dimension },
+            schema: GRAPH_SCHEMA,
+            schemaVersion: GRAPH_SCHEMA_VERSION,
+        },
+        domains,
+    );
+}
+
+/**
+ * Seeding is MERGE-based, so a graph written under an older schema is not cleaned
+ * up — its nodes stay, under labels nothing reads. Worth saying out loud; not
+ * worth blocking on, since re-seeding is exactly how that graph gets fixed.
+ */
+async function warnOnRestamp(neo4j: Neo4jClient) {
+    // An unreachable graph is the seed's to report, far better than a warning could.
+    const rows = await neo4j
+        .query(
+            `MATCH (g:${GRAPH_SCHEMA.label.marker}) RETURN g.${GRAPH_SCHEMA.prop.version} AS version`,
+        )
+        .catch(() => undefined);
+    if (!rows) return;
+
+    const stamped = (rows as { version?: number }[])[0]?.version;
+    if (stamped === undefined || stamped === GRAPH_SCHEMA_VERSION) return;
+
+    report.blank();
+    report.row(
+        "",
+        `${pc.yellow("!")} Graph was written with schema v${stamped}; restamping as v${GRAPH_SCHEMA_VERSION}.`,
+        DOMAIN_COLUMN,
+    );
+    report.row("", "  Nodes from the old schema are not removed.", DOMAIN_COLUMN);
+}
+
+/**
+ * Section requirements are `runSeed`'s to enforce — it already throws
+ * `"neo4j config is required for graph seed"`. This renders what it threw.
+ */
+function failSeed(options: {
+    error: unknown;
+    config: ProjectConfig;
+    configPath: string;
+    verbose: boolean;
+}): never {
+    const { error, config, configPath, verbose } = options;
+    const label = configLabel(configPath);
+    const message = error instanceof Error ? error.message : String(error);
+
+    if (message === "No domains found in knowledge config") {
+        report.fail({
+            what: `${label} has no knowledge.domains`,
+            blocks: [
+                "`graph seed` loads domain definitions into Neo4j. Add them:",
+                "  knowledge: { domains: { leaves: leavesDomain } }",
+                "See https://github.com/boring91/cortex/blob/main/packages/cortex-server/README.md#file-structure-convention",
+            ],
+        });
+    }
+
+    const url = config.neo4j?.url;
+    if (url !== undefined && UNREACHABLE.test(message)) {
+        report.fail({
+            what: `Cannot reach Neo4j at ${url}`,
+            error,
+            blocks: [
+                "Is it running?  docker compose up -d neo4j",
+                `Otherwise check neo4j.url / neo4j.user / neo4j.password in\n${label}.`,
+            ],
+            verbose,
+        });
+    }
+
+    report.fail({
+        what: "Could not seed the graph",
+        error,
+        blocks: [`Check neo4j, embedding, and knowledge.domains in ${label}.`],
+        verbose,
+    });
+}

package/src/commands/new.ts

@@ -0,0 +1,333 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, readdirSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+import { Option, type Command } from "@commander-js/extra-typings";
+
+import pkg from "../../package.json";
+import { FEATURE_NAMES, FEATURES, type Feature } from "@/scaffold/features";
+import { scaffoldFiles, type Database } from "@/scaffold/files";
+import { allocatePorts } from "@/scaffold/ports";
+import { askDatabase, askFeatures, askName, intro, log, outro, runSteps } from "@/ui/prompts";
+import * as report from "@/ui/report";
+
+/** npm's rule for an unscoped package name. The name is the directory too. */
+const VALID_NAME = /^[a-z0-9][a-z0-9._-]*$/;
+const MAX_NAME = 214;
+
+/** The two names npm rejects outright, however well-formed they otherwise are. */
+const RESERVED_NAMES = ["node_modules", "favicon.ico"];
+
+export function registerNew(program: Command) {
+    const command = program
+        .command("new")
+        .helpGroup("Project:")
+        .description("Scaffold a new Cortex server")
+        .argument("[name]", "Directory to create, and the generated package name")
+        .addOption(
+            new Option("--database <type>", "Database the project is configured for")
+                .choices(["postgres", "mssql"] as const)
+                .default("postgres" as const),
+        )
+        // Both halves of every toggle, so a flag can answer its question either
+        // way and `--yes` is not the only way to say "no". Declaring the positive
+        // first leaves the value `undefined` when neither is passed, which is
+        // what marks the question as still unanswered.
+        .option("--attachments", FEATURES.attachments.hint)
+        .option("--no-attachments", "Leave attachments out")
+        .option("--redis", FEATURES.redis.hint)
+        .option("--no-redis", "Leave Redis out")
+        .option("--auth", FEATURES.auth.hint)
+        .option("--no-auth", "Leave auth out")
+        .option("--knowledge", FEATURES.knowledge.hint)
+        .option("--no-knowledge", "Leave the knowledge graph out")
+        .option("--control-center", FEATURES.controlCenter.hint)
+        .option("--no-control-center", "Leave the Control Center out")
+        .option("--yes", "Accept the defaults instead of asking");
+
+    command.action(async function (name, options) {
+        // The globals live on the root command, which reaches this file untyped —
+        // `Command` here is the parameter type, not cli.ts's inferred one.
+        const { verbose = false } = program.opts() as { verbose?: boolean };
+
+        await scaffold(name, {
+            ...options,
+            verbose,
+            databasePassed: command.getOptionValueSource("database") !== "default",
+        });
+    });
+}
+
+type Options = {
+    database: Database;
+    yes?: boolean;
+    verbose: boolean;
+    databasePassed: boolean;
+} & Partial<Record<Feature, boolean>>;
+
+async function scaffold(name: string | undefined, options: Options) {
+    const { verbose, databasePassed } = options;
+
+    // No TTY ⇒ never prompt and behave as `--yes`. On Bun a
+    // piped stdin reports `undefined` rather than `false`, which is why this
+    // compares against `true`.
+    const interactive = process.stdin.isTTY === true && options.yes !== true;
+
+    if (name === undefined && !interactive) missingName();
+
+    // A name that came from a flag is checked before anything prints; one that
+    // is prompted for is checked by the prompt, which can ask again.
+    if (name !== undefined && !isValidName(name)) failInvalidName(name);
+    if (name !== undefined && isTaken(resolve(process.cwd(), name))) failTaken(name);
+
+    const started = Date.now();
+    intro(`cortex new  ·  ${pkg.name} ${pkg.version}`);
+
+    const project = name ?? (await askName(checkName)).trim();
+
+    const database = interactive && !databasePassed ? await askDatabase() : options.database;
+    const features = await chooseFeatures(options, interactive);
+
+    const target = resolve(process.cwd(), project);
+
+    // Allocated here rather than inside the generators, which stay pure: picking a
+    // port means probing the machine for a free one.
+    const ports = allocatePorts();
+    const files = scaffoldFiles(project, database, features, ports);
+    let failedInstall: string | undefined;
+
+    await runSteps([
+        {
+            title: `Writing ${Object.keys(files).length} files`,
+            run: function () {
+                for (const [path, contents] of Object.entries(files)) {
+                    const file = join(target, path);
+                    mkdirSync(dirname(file), { recursive: true });
+                    writeFileSync(file, contents);
+                }
+
+                const summary = [database, ...features, "model"].join(" · ");
+                return `Wrote ${Object.keys(files).length} files to ./${project}\n${summary}`;
+            },
+        },
+        {
+            // Always, no flag: running it is what separates a scaffold from a
+            // template dump.
+            title: "Installing dependencies",
+            run: async function () {
+                // Timed from here, not from the run: the prompts above are the
+                // user's time, and "in 41.0s" of which 38 were spent answering
+                // questions is a number nobody can act on.
+                const startedInstall = Date.now();
+                const result = await install(target);
+
+                if (!result.ok) {
+                    failedInstall = result.output;
+                    // What went wrong is reported below, once the run is over;
+                    // --verbose still owes the reader the command that failed.
+                    return verbose ? "$ bun install" : undefined;
+                }
+
+                const packages = /(\d+) packages? installed/.exec(result.output)?.[1];
+                const what = packages === undefined ? "dependencies" : `${packages} packages`;
+                const done = `Installed ${what} in ${seconds(startedInstall)}`;
+                return verbose ? `$ bun install\n${done}` : done;
+            },
+        },
+        {
+            // A repo nested inside another is a mess to discover later, and the
+            // .gitignore just written is inert without one. Silent either way,
+            // which is why this step reports nothing.
+            title: "Initializing git",
+            run: function () {
+                if (isInsideWorkTree(target)) {
+                    return verbose ? "git init skipped — already inside a work tree" : undefined;
+                }
+
+                spawnSync("git", ["init"], { cwd: target, stdio: "ignore" });
+                return verbose ? "$ git init" : undefined;
+            },
+        },
+    ]);
+
+    if (failedInstall !== undefined) warnInstall(project, failedInstall, verbose);
+
+    outro(`Done in ${seconds(started)}`);
+    process.stdout.write(nextSteps(project, features, files));
+}
+
+/**
+ * The files are correct; only the install did not happen, so this warns and the
+ * run still exits 0. What bun said is the whole diagnosis, so it is kept: the
+ * last line by default, the captured log under --verbose.
+ */
+function warnInstall(project: string, output: string, verbose: boolean) {
+    const detail = verbose ? output.trim() : lastLine(output);
+
+    log.warn(
+        // clack indents continuation lines by 3, so 2 more puts the command in
+        // the same column the error grammar uses.
+        `bun install did not finish. The files are written — run it yourself:\n\n` +
+            `  cd ${project} && bun install` +
+            (detail === "" ? "" : `\n\n${detail}`),
+    );
+}
+
+function nextSteps(project: string, features: Feature[], files: ReturnType<typeof scaffoldFiles>) {
+    const keys = [
+        "CORTEX_DATABASE_URL",
+        "CORTEX_MODEL_*",
+        ...features.map((feature) => FEATURES[feature].secrets).filter((key) => key !== undefined),
+    ];
+
+    return (
+        `\nNext steps:\n\n` +
+        step(`cd ${project}`) +
+        step("edit .env", `fill in ${keys.length === 2 ? keys.join(" and ") : keys.join(", ")}`) +
+        (files["docker-compose.yml"] === undefined ? "" : step("docker compose up -d")) +
+        // The `port` the generated cortex.config.ts sets, which is the server's
+        // own default. Service ports move at scaffold time; this one does not.
+        step("bun run dev", "→ http://localhost:3331") +
+        `\n`
+    );
+}
+
+/**
+ * One multi-select for the whole catalogue, minus whatever the flags already
+ * settled: a passed flag suppresses its question, the rest are still asked.
+ */
+async function chooseFeatures(options: Options, interactive: boolean) {
+    const chosen = new Set(FEATURE_NAMES.filter((feature) => options[feature] === true));
+    const undecided = FEATURE_NAMES.filter((feature) => options[feature] === undefined);
+
+    if (interactive && undecided.length > 0) {
+        for (const feature of await askFeatures(undecided)) chosen.add(feature);
+    }
+
+    // Catalogue order, not the order the flags or the checkboxes came in.
+    return FEATURE_NAMES.filter((feature) => chosen.has(feature));
+}
+
+async function install(cwd: string) {
+    try {
+        // Piped rather than inherited: the spinner owns the screen while this runs.
+        // bun prints its summary to stderr, so both streams are read.
+        const child = Bun.spawn(["bun", "install"], { cwd, stdout: "pipe", stderr: "pipe" });
+        const output = await Promise.all([
+            new Response(child.stdout).text(),
+            new Response(child.stderr).text(),
+        ]);
+
+        return { ok: (await child.exited) === 0, output: output.join("") };
+    } catch (error) {
+        // A spawn that never starts — EACCES, EMFILE — is still just an install
+        // that did not happen, and the files are already written. It takes the
+        // same warn-and-exit-0 path as a bun install that ran and failed, rather
+        // than surfacing as "cortex hit an unexpected error".
+        return { ok: false, output: error instanceof Error ? error.message : String(error) };
+    }
+}
+
+function step(command: string, annotation?: string) {
+    // 27 aligns the annotation column across all report rows.
+    return annotation ? `  ${command.padEnd(27)}${annotation}\n` : `  ${command}\n`;
+}
+
+function seconds(started: number) {
+    return `${((Date.now() - started) / 1000).toFixed(1)}s`;
+}
+
+/** bun puts the reason last, and that one line is what gets pasted into a search. */
+function lastLine(output: string) {
+    return output.trim().split("\n").at(-1)?.trim() ?? "";
+}
+
+function isValidName(name: string) {
+    return name.length <= MAX_NAME && VALID_NAME.test(name) && !RESERVED_NAMES.includes(name);
+}
+
+/** The prompt's rules, which are the flag path's two failures asked one at a time. */
+function checkName(candidate: string) {
+    // One line each: clack prints them inside the prompt, which will not wrap.
+    if (RESERVED_NAMES.includes(candidate)) return `npm does not allow the name ${candidate}.`;
+    if (!isValidName(candidate)) return "Lowercase letters, digits, and . _ - only.";
+    if (isTaken(resolve(process.cwd(), candidate))) {
+        return `./${candidate} already exists and is not an empty directory.`;
+    }
+    return undefined;
+}
+
+/** `My App` → `my-app`, so the suggested command is one the user can paste. */
+function slugify(name: string) {
+    const slug = name
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, "-")
+        .replace(/^[^a-z0-9]+/, "")
+        .slice(0, MAX_NAME);
+
+    return slug === "" || RESERVED_NAMES.includes(slug) ? "my-app" : slug;
+}
+
+/**
+ * A missing path is the only clean go-ahead, and an empty directory the one
+ * tolerated exception. Anything else — a regular file of that name, a directory
+ * with contents, one we lack permission to read — is a conflict: swallowing it
+ * scaffolds halfway and then dies on the first write.
+ */
+function isTaken(target: string) {
+    try {
+        return readdirSync(target).length > 0;
+    } catch (error) {
+        return !(error instanceof Error && "code" in error && error.code === "ENOENT");
+    }
+}
+
+function isInsideWorkTree(target: string) {
+    return (
+        spawnSync("git", ["rev-parse", "--is-inside-work-tree"], {
+            cwd: target,
+            stdio: "ignore",
+        }).status === 0
+    );
+}
+
+function missingName(): never {
+    // No TTY ⇒ never prompt, so a missing required argument is a hard error
+    // With a TTY this branch is only reached under `--yes`,
+    // which answers every question including the one that has no default.
+    report.fail({
+        what:
+            process.stdin.isTTY === true
+                ? "cortex new needs a project name."
+                : "cortex new needs a project name when stdin is not a TTY.",
+        blocks: [
+            "Pass it as an argument, and --yes to accept defaults for the rest:",
+            "  cortex new my-app --yes",
+        ],
+    });
+}
+
+function failTaken(name: string): never {
+    report.fail({
+        what: `./${name} already exists and is not an empty directory.`,
+        blocks: [
+            "cortex new never merges into an existing project. Pick another name,\nor remove it first:",
+            `  rm -rf ${name}`,
+        ],
+    });
+}
+
+/**
+ * The name is the directory *and* the generated `package.json` name, so an
+ * illegal one produces a project npm refuses and a `cd` that does not parse.
+ */
+function failInvalidName(name: string): never {
+    report.fail({
+        what: `"${name}" is not a usable project name`,
+        blocks: [
+            "It becomes the directory and the generated package.json `name`, so it has\nto be one npm accepts: lowercase, no spaces, not a name npm reserves. Try:",
+            `  cortex new ${slugify(name)}`,
+        ],
+    });
+}