@m6d/cortex-cli 0.15.0 → 0.17.0

package/src/commands/setup.ts

@@ -12,15 +12,23 @@ import {
     normalizeHost,
     scrubCredentialStrings,
 } from "@cortex/contracts/egress-policy";
+import { MODEL_PROVIDER_LIMITS, NAME_MAX_LENGTH, SETTING_BOUNDS } from "@cortex/contracts/manifest";
 import { z } from "zod";
 
 import {
     applyManifest,
     DEFAULT_MANIFEST,
     findManifest,
+    loadManifest,
     printTriggerSecrets,
 } from "@/commands/apply";
-import { callOrFail, failCall, requireProfile, walkPages } from "@/commands/management";
+import {
+    callOrFail,
+    confirmDestructive,
+    failCall,
+    requireProfile,
+    walkPages,
+} from "@/commands/management";
 import { callManagement, ManagementError, UnreachableError } from "@/management/client";
 import { DOCKER_HOST, EMBEDDING, withEnvValues } from "@/scaffold/files";
 import * as prompts from "@/ui/prompts";
@@ -34,7 +42,7 @@ const POLL_MS = 2_000;
 
 /** The backend as `cortex new` writes it into .env, which Bun loads from the directory it runs in. */
 const projectSchema = z.object({
-    CORTEX_SERVER_NAME: z.string().min(1),
+    CORTEX_SERVER_NAME: z.string().min(1).max(NAME_MAX_LENGTH),
     CORTEX_SERVER_URL: z.url(),
     CORTEX_SERVER_KEY: z.string().min(1),
 });
@@ -48,14 +56,16 @@ const ENV_SAFE = /^[^\s$#'"`\\]+$/;
 
 /** The model provider in .env: all four values, or none. */
 const providerFields = z.object({
-    // Its key's secret is `<name>-api-key-<digest>`, within Cortex's 120 characters.
-    MODEL_PROVIDER_NAME: z.string().min(1).max(103),
-    MODEL_PROVIDER_URL: z.string().superRefine(function (value, context) {
-        const problem = checkBaseUrl(value);
-        if (problem) context.addIssue({ code: "custom", message: problem });
-    }),
+    MODEL_PROVIDER_NAME: z.string().min(1).max(NAME_MAX_LENGTH),
+    MODEL_PROVIDER_URL: z
+        .string()
+        .max(MODEL_PROVIDER_LIMITS.urlLength)
+        .superRefine(function (value, context) {
+            const problem = checkBaseUrl(value);
+            if (problem) context.addIssue({ code: "custom", message: problem });
+        }),
     MODEL_PROVIDER_API_KEY: z.string().min(1),
-    MODEL_PROVIDER_MODEL: z.string().min(1).max(200),
+    MODEL_PROVIDER_MODEL: z.string().min(1).max(MODEL_PROVIDER_LIMITS.modelLength),
 });
 const providerEnvSchema = providerFields.transform((env) => ({
     name: env.MODEL_PROVIDER_NAME,
@@ -81,6 +91,8 @@ const serverSchema = namedSchema.extend({
     modelProvider: z.string().nullable(),
     modelName: z.string().nullable(),
     modelEffort: z.string().nullable(),
+    signInChecksPerMinute: z.number(),
+    triggerChecksPerMinute: z.number(),
 });
 const chatSchema = z.object({ choices: z.array(z.object({ message: z.object({}) })).min(1) });
 const embeddingsSchema = z.object({
@@ -94,8 +106,12 @@ const embeddingsSchema = z.object({
  * creates what is missing and keeps what exists, so it runs again on every
  * `bun run dev`. A provider or server under .env's name must call .env's URL,
  * or setup leaves it alone and stops, so no key reaches a URL .env did not name. With no model provider in .env it asks for
- * one on a terminal, and under `--yes` registers the server and applies
- * nothing, since Cortex publishes no agent without a model.
+ * one on a terminal, and under `--yes` registers the server without a model
+ * and applies all the same, since an agent may name its own; when Cortex
+ * refuses an agent for want of one, setup says so and applies nothing. The
+ * apply deletes what the manifest no longer lists, so setup checks the
+ * manifest, then waits for a yes on a terminal or `--yes`, before it asks or
+ * writes anything else, as `cortex apply` does.
  */
 export function registerSetup(program: Command<[], { verbose?: true; profile?: string }>) {
     const command = program
@@ -103,19 +119,26 @@ export function registerSetup(program: Command<[], { verbose?: true; profile?: s
         .helpGroup("Project:")
         .description("Register this project's backend and model in Cortex, then apply its manifest")
         .option("--wait", "Wait for Cortex to come up first")
-        .option("--yes", "Never ask; leave out what .env does not hold");
+        .option(
+            "--yes",
+            "Never ask: leave out what .env does not hold, and apply without confirming",
+        );
 
     command.action(async function (options) {
         const { verbose = false, profile: profileName } = program.opts();
         const project = readProject();
         const profile = requireProfile(profileName);
-        const file = findManifest(DEFAULT_MANIFEST);
-        const interactive = process.stdin.isTTY === true && options.yes !== true;
+        const manifest = await loadManifest(findManifest(DEFAULT_MANIFEST), verbose);
+        const interactive = prompts.canAsk() && options.yes !== true;
         if (profileName === undefined) requireEnvCortex(profile.url);
 
         report.header("cortex setup");
         await waitForCortex(profile, options.wait === true);
         report.input("Cortex", profile.url);
+        await confirmDestructive(
+            `Applying deletes what the manifest created on ${profile.url} and no longer lists.`,
+            options.yes,
+        );
 
         const provider = readProvider() ?? (interactive ? await askAndSave() : undefined);
         if (provider) await requireReachable(provider.baseUrl);
@@ -124,22 +147,26 @@ export function registerSetup(program: Command<[], { verbose?: true; profile?: s
         const server = await ensureServer(profile, project, provider);
         await syncServer(profile, server.id, project.CORTEX_SERVER_URL);
 
-        if (server.modelProvider === null) {
-            report.footer("no model yet, so nothing applied");
-            report.note("Run `bun cortex setup` in a terminal to add a model provider.");
-            return;
-        }
-
-        const { changes, triggerSecrets } = await applyManifest({
+        const applied = await applyManifest({
             profile,
-            file,
+            manifest,
             dryRun: false,
-            verbose,
             interactive,
+            command: "bun cortex apply --yes",
+        }).catch(function (error: unknown) {
+            if (error instanceof ManagementError && error.message === "chat_model_required") {
+                return undefined;
+            }
+            return failCall(profile, error);
         });
+        if (!applied) {
+            report.footer("no model yet, so nothing applied");
+            report.note("Run `bun cortex setup` in a terminal to add a model provider.");
+            return;
+        }
         report.footer(`live · chat at ${profile.url}/playground`);
-        printTriggerSecrets(triggerSecrets);
-        if (Object.values(changes.create).flat().length > 0) printSignIn(profile.url);
+        printTriggerSecrets(applied.triggerSecrets);
+        if (Object.values(applied.changes.create).flat().length > 0) printSignIn(profile.url);
     });
 }
 
@@ -147,8 +174,9 @@ function readProject() {
     const parsed = projectSchema.safeParse(process.env);
     if (parsed.success) return parsed.data;
     return report.fail({
-        what: "No project here for cortex setup",
+        what: "No project here cortex setup can register",
         blocks: [
+            z.prettifyError(parsed.error),
             "It reads CORTEX_SERVER_NAME, CORTEX_SERVER_URL and CORTEX_SERVER_KEY from\n.env, which `cortex new` writes. Run it from that project's root.",
         ],
     });
@@ -243,13 +271,15 @@ async function findByName<TItem extends typeof namedSchema>(
 }
 
 /**
- * A secret's name for one value: its base and a short digest of the value.
- * A name then holds one value only, so a secret that exists is the value
- * .env holds, and a changed key gets a secret of its own while every row
- * presenting the old one keeps it.
+ * A secret's name for one value: its owner's name, what it is to the owner,
+ * and a short digest of the value, the owner's name cut so the whole is a
+ * name Cortex holds. A name then holds one value only, so a secret that
+ * exists is the value .env holds, and a changed key gets a secret of its own
+ * while every row presenting the old one keeps it.
  */
-function secretFor(base: string, value: string) {
-    return `${base}-${createHash("sha256").update(value).digest("hex").slice(0, 8)}`;
+function secretFor(owner: string, role: "api-key" | "server-key", value: string) {
+    const suffix = `-${role}-${createHash("sha256").update(value).digest("hex").slice(0, 8)}`;
+    return `${owner.slice(0, NAME_MAX_LENGTH - suffix.length)}${suffix}`;
 }
 
 async function ensureSecret(profile: Profile, name: string, value: string) {
@@ -272,7 +302,9 @@ function failCallsElsewhere(what: string, calls: string | null, wanted: string,
 
 /**
  * The provider, on its key's secret. One that exists takes a changed key and
- * the .env model as well; one that calls another URL ends the run.
+ * the .env model as well; one that calls another URL, or lists as many
+ * models as Cortex allows without the .env one, ends the run before any
+ * write.
  */
 async function ensureProvider(profile: Profile, provider: Provider) {
     const found = await findByName(profile, "model-providers", provider.name, providerRowSchema);
@@ -284,7 +316,17 @@ async function ensureProvider(profile: Profile, provider: Provider) {
             "MODEL_PROVIDER_NAME",
         );
     }
-    const secretName = secretFor(`${provider.name}-api-key`, provider.apiKey);
+    // Added, never replaced: a model a server still runs on stays offered.
+    const models = [...new Set([...(found?.models ?? []), provider.model])];
+    if (models.length > MODEL_PROVIDER_LIMITS.models) {
+        report.fail({
+            what: `Provider ${provider.name} in Cortex lists ${MODEL_PROVIDER_LIMITS.models} models, not ${provider.model}`,
+            blocks: [
+                "Cortex lists no more on one provider. Remove one in the console, or set\nMODEL_PROVIDER_MODEL in .env to a model it lists.",
+            ],
+        });
+    }
+    const secretName = secretFor(provider.name, "api-key", provider.apiKey);
     await ensureSecret(profile, secretName, provider.apiKey);
 
     if (!found) {
@@ -292,7 +334,7 @@ async function ensureProvider(profile: Profile, provider: Provider) {
             name: provider.name,
             baseUrl: provider.baseUrl,
             secretName,
-            models: [provider.model],
+            models,
             efforts: [],
         });
         report.input("Provider", `${provider.name} · created`);
@@ -306,8 +348,7 @@ async function ensureProvider(profile: Profile, provider: Provider) {
         id: found.id,
         baseUrl: found.baseUrl,
         secretName,
-        // Added, never replaced: a model a server still runs on stays offered.
-        models: [...new Set([...found.models, provider.model])],
+        models,
         efforts: found.efforts,
     });
     report.input("Provider", `${provider.name} · updated`);
@@ -333,7 +374,7 @@ async function ensureServer(
     if (server && server.url !== url) {
         failCallsElsewhere(`Server ${name}`, server.url, url, "CORTEX_SERVER_NAME");
     }
-    const keySecretName = secretFor(`${name}-server-key`, project.CORTEX_SERVER_KEY);
+    const keySecretName = secretFor(name, "server-key", project.CORTEX_SERVER_KEY);
     await ensureSecret(profile, keySecretName, project.CORTEX_SERVER_KEY);
 
     const model = {
@@ -349,6 +390,8 @@ async function ensureServer(
                 url,
                 keySecretName,
                 ...model,
+                signInChecksPerMinute: SETTING_BOUNDS.server.signInChecksPerMinute.default,
+                triggerChecksPerMinute: SETTING_BOUNDS.server.triggerChecksPerMinute.default,
             }),
         );
         report.input("Server", `${name} · created`);
@@ -374,6 +417,8 @@ async function ensureServer(
                       modelName: server.modelName,
                       modelEffort: server.modelEffort,
                   }),
+            signInChecksPerMinute: server.signInChecksPerMinute,
+            triggerChecksPerMinute: server.triggerChecksPerMinute,
         }),
     );
     report.input("Server", `${name} · updated`);

package/src/contracts/datasets/dataset-sql.ts

@@ -18,6 +18,7 @@ import type {
 import {
     DATASET_BUCKETS,
     DATASET_DATE_PARTS,
+    DATASET_REQUEST_MAX_BYTES,
     describeOperand,
     groupKeyName,
     unknownColumnError,
@@ -278,6 +279,15 @@ const WINDOW_PLACE =
     "RANK() goes in QUALIFY (QUALIFY RANK() OVER (PARTITION BY a ORDER BY b DESC) <= n) or in a derived table filtered on its alias (FROM (SELECT ..., RANK() OVER (...) AS rn FROM ...) WHERE rn <= n).";
 const INTERVAL_PLACE =
     "INTERVAL only shifts CURRENT_DATE (CURRENT_DATE - INTERVAL '7 days'); compare the column against a date instead.";
+const TOO_LARGE =
+    "The statement grows too large to send: a simple CASE repeats its subject in every WHEN, so a CASE nested in that subject multiplies. Compare a column instead, or ask in smaller queries.";
+
+/**
+ * The most tokens a statement may read, a simple CASE's subject counted again
+ * for every WHEN that repeats it. A token writes about a byte or more of the
+ * wire query, so a statement past this could never be sent.
+ */
+const EXPANDED_TOKENS_MAX = DATASET_REQUEST_MAX_BYTES;
 
 const REFUSED_CLAUSES = {
     UNION: "UNION is not supported: ask each dataset in its own query.",
@@ -402,6 +412,8 @@ function datePartTerm(unit: DatasetDatePart, of: Term) {
 
 class Parser {
     private position = 0;
+    /** Tokens read so far, a simple CASE's subject counted once per WHEN: how large the statement grows once written out. */
+    private expanded = 0;
 
     public constructor(
         private readonly tokens: Token[],
@@ -769,11 +781,15 @@ class Parser {
      */
     private caseTerm() {
         this.take();
+        const start = this.expanded;
         const subject = this.peekWord("WHEN")
             ? undefined
             : this.term("CASE is followed by WHEN, or by a value to compare.");
+        const subjectSize = this.expanded - start;
         const branches: { when: Predicate[]; then: Term }[] = [];
         while (this.optionalWord("WHEN")) {
+            // The wire has no simple CASE: every WHEN after the first writes the subject out again.
+            if (branches.length > 0) this.expand(subjectSize);
             const when = subject ? [this.simpleCaseMatch(subject)] : this.predicates();
             this.expectWord("THEN", "CASE WHEN condition needs THEN value.");
             if (this.peekWord("NULL")) {
@@ -1230,8 +1246,14 @@ class Parser {
     private take() {
         const token = this.peek();
         this.position += 1;
+        this.expand(1);
         return token;
     }
+    /** Counts `tokens` toward the statement's written-out size, refused the moment it passes what a request carries. */
+    private expand(tokens: number) {
+        this.expanded += tokens;
+        if (this.expanded > EXPANDED_TOKENS_MAX) throw new DatasetSqlError(TOO_LARGE);
+    }
     private peekWord(upper: string) {
         const token = this.peek();
         return token.kind === "word" && token.upper === upper;
@@ -1334,11 +1356,16 @@ function unqualified(ref: string, table: string | undefined) {
     return table !== undefined && ref.startsWith(`${table}.`) ? ref.slice(table.length + 1) : ref;
 }
 
-/** The outer list of a derived table: `*` keeps every inner value, names keep those; the RANK() itself is never answered. */
+/**
+ * The outer list of a derived table: `*` keeps every inner value, names keep
+ * those, each once since every listing writes the value out again; the
+ * RANK() itself is never answered.
+ */
 function narrowed(inner: SelectItem[], outer: SelectItem[], table: string | undefined) {
     const values = inner.filter((item) => item.kind !== "rankWindow");
     if (outer.some((item) => item.kind === "star")) return values;
     const windowNames = inner.flatMap((item) => (item.kind === "rankWindow" ? [item.alias] : []));
+    const listed = new Set<SelectItem>();
     const kept = outer.flatMap((item) => {
         const name =
             item.kind === "column" && item.alias === undefined
@@ -1352,6 +1379,12 @@ function narrowed(inner: SelectItem[], outer: SelectItem[], table: string | unde
                 "The outer SELECT of a derived table lists names the inner SELECT answers, or *.",
             );
         }
+        if (listed.has(found)) {
+            throw new DatasetSqlError(
+                `The outer SELECT lists "${name}" twice; list each value of the derived table once.`,
+            );
+        }
+        listed.add(found);
         return [found];
     });
     const unnamed = values.flatMap((value) =>
@@ -2082,7 +2115,8 @@ function resolveGroupKeys(
             key = twin ?? { expr, as: lastSegment(column ?? "") };
         }
         const taken = keys.get(groupKeyName(key));
-        if (taken && !same(operandOf(taken), operandOf(key))) {
+        // A name grouped again is the very same key; only a different one is written out to compare.
+        if (taken && taken !== key && !same(operandOf(taken), operandOf(key))) {
             throw new DatasetSqlError(
                 `Two group keys would answer as "${groupKeyName(key)}"; name each value: ${describeTerm(group)} AS name.`,
             );