@zhihaohong52/sonata 0.4.0

package/dist/commands/serve.js

@@ -0,0 +1,818 @@
+import { spawn, execFileSync } from 'node:child_process';
+import { randomBytes, randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { loadAiPricing } from '../aipricing.js';
+import { configPath as resolveSonataConfigPath, loadConfig, resolveTierAlias } from '../config.js';
+import { appendRow, LEDGER_RETENTION_DAYS, pruneLedger } from '../ledger.js';
+import { pruneSessions } from '../sessions.js';
+import { resolveKeyFromSource, resolveKeys } from '../native/credentials.js';
+import { codexAuthPath, opencodeAuthPath, readChatGptOAuth } from '../native/codex-auth.js';
+import { credentialDir } from '../native/oauth-login.js';
+import { readCopilotToken } from '../native/copilot-auth.js';
+import { envVarForGateway, litellmConfigYaml } from '../native/litellm.js';
+import { createRouterServer } from '../native/router.js';
+import { resolvePrice } from '../pricing.js';
+import { timestampedLogPath } from './init-log.js';
+/**
+ * Where a serve instance records its own pid and its litellm child's pid.
+ *
+ * The router dies with the process that started serve, but the spawned litellm
+ * child is reparented and survives. The next
+ * serve then cannot bind the litellm port: its own child dies silently, and
+ * the new router forwards a new master key to the ORPHANED litellm, whose
+ * virtual-key lookup fails as "No connected db". Measured 2026-08-20, twice.
+ * Recording the pid lets the next serve kill its predecessor's orphan —
+ * only a pid sonata itself recorded is ever killed.
+ *
+ * `routerPid` is `process.pid` at the point the router successfully binds,
+ * allowing `sonata restart` to stop only a process Sonata recorded itself.
+ * `sonata restart` reads it to kill a stale router without guessing a pid by
+ * scanning the OS.
+ */
+export function serveStatePath(home) {
+    return join(home, '.config', 'sonata', 'serve-state.json');
+}
+function readServeState(home) {
+    const path = serveStatePath(home);
+    if (!existsSync(path))
+        return undefined;
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+function writeServeState(home, state) {
+    const path = serveStatePath(home);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, JSON.stringify({ ...state, recordedAt: new Date().toISOString() }));
+}
+function killPid(pid) {
+    if (typeof pid !== 'number' || pid <= 0)
+        return;
+    try {
+        process.kill(pid);
+    }
+    catch { /* already dead */ }
+}
+function killRecordedOrphan(home) {
+    const state = readServeState(home);
+    killPid(state?.litellmPid);
+    try {
+        unlinkSync(serveStatePath(home));
+    }
+    catch { /* gone is the goal */ }
+}
+function recordLitellmPid(home, pid) {
+    writeServeState(home, { ...readServeState(home), litellmPid: pid });
+}
+function recordRouterPid(home, pid) {
+    writeServeState(home, { ...readServeState(home), routerPid: pid });
+}
+/** Polls litellm until it answers, so a silent bind failure surfaces here. */
+async function defaultWaitForLitellm(port) {
+    const deadline = Date.now() + 30_000;
+    for (;;) {
+        try {
+            const res = await fetch(`http://localhost:${port}/health/liveliness`);
+            if (res.ok)
+                return;
+        }
+        catch { /* not up yet */ }
+        if (Date.now() > deadline) {
+            throw new Error(`sonata serve: litellm did not come up on port ${port} within 30s — ` +
+                'it may have failed to bind (another litellm running?) or failed to start. ' +
+                'Check `litellm --config` by hand.');
+        }
+        await new Promise((r) => setTimeout(r, 500));
+    }
+}
+export function serveHealthUrl(routerPort) {
+    return `http://localhost:${routerPort}/__sonata_health`;
+}
+/** Whether whatever holds a port is a sonata router. */
+export async function isSonataRouter(port, doFetch = fetch) {
+    try {
+        const response = await doFetch(serveHealthUrl(port), {
+            signal: AbortSignal.timeout(2000),
+        });
+        if (!response.ok)
+            return false;
+        const body = await response.json();
+        return body?.sonata === true;
+    }
+    catch {
+        return false;
+    }
+}
+/** The resolved sonata.toml path a running sonata router reports, or null if the port isn't a sonata router (or reports no configPath). */
+export async function sonataRouterConfigPath(port, doFetch = fetch) {
+    try {
+        const response = await doFetch(serveHealthUrl(port), {
+            signal: AbortSignal.timeout(2000),
+        });
+        if (!response.ok)
+            return null;
+        const body = await response.json();
+        if (body?.sonata !== true)
+            return null;
+        return typeof body.configPath === 'string' ? body.configPath : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** The instance id a running sonata router reports on /__sonata_health, or null if the port isn't a sonata router (or reports none). */
+export async function sonataRouterInstanceId(port, doFetch = fetch) {
+    try {
+        const response = await doFetch(serveHealthUrl(port), {
+            signal: AbortSignal.timeout(2000),
+        });
+        if (!response.ok)
+            return null;
+        const body = await response.json();
+        if (body?.sonata !== true)
+            return null;
+        return typeof body.instanceId === 'string' ? body.instanceId : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * What to say when the router port is taken.
+ *
+ * The occupant may be another Sonata process. Probe the health endpoint
+ * before describing the listener so the error tells the user what actually
+ * holds the port.
+ */
+export async function occupiedPortMessage(port, doFetch = fetch) {
+    if (await isSonataRouter(port, doFetch)) {
+        return `sonata serve: router port ${port} is already served by another sonata router — ` +
+            'usually an earlier native router. Use that one, restart it to retire it, ' +
+            `or give this instance a different [native.ports] router port.`;
+    }
+    return `sonata serve: router port ${port} is occupied by a non-sonata listener`;
+}
+/**
+ * LiteLLM's own output is the only place a per-model startup failure appears.
+ *
+ * It drops a deployment it cannot authenticate and carries on, so the model
+ * simply vanishes from the catalogue and the next request answers "no healthy
+ * deployments for this model" — with the actual cause (for one real case, a 403
+ * from GitHub's Copilot token exchange) written only to a stream nobody read.
+ */
+function defaultSpawnLitellm(configPath, env, port) {
+    const child = spawn('litellm', ['--config', configPath, '--port', String(port)], {
+        env,
+        stdio: ['ignore', 'inherit', 'inherit'],
+    });
+    return {
+        pid: child.pid ?? 0,
+        kill: () => child.kill(),
+        forceKill: () => child.kill('SIGKILL'),
+        onExit: (cb) => child.on('exit', cb),
+    };
+}
+function buildChildEnv(native, home, tempDir) {
+    // LiteLLM still needs PATH for executable lookup; no other parent values are forwarded.
+    const childEnv = process.env.PATH ? { PATH: process.env.PATH } : {};
+    const automaticallyResolved = Object.entries(native.gateways)
+        .filter(([, gateway]) => gateway.auth !== 'api-key' || gateway.credentialSource === undefined)
+        .map(([name]) => name);
+    for (const { gateway, key } of resolveKeys(automaticallyResolved, home)) {
+        childEnv[envVarForGateway(gateway)] = key;
+    }
+    for (const [name, gateway] of Object.entries(native.gateways)) {
+        const source = gateway.credentialSource;
+        if (gateway.auth !== 'api-key' || (source !== 'sonata' && source !== 'opencode'))
+            continue;
+        const key = resolveKeyFromSource(name, home, source);
+        if (key === undefined) {
+            throw new Error(`sonata serve: gateway "${name}" takes its credential from ${source} but none was found — ` +
+                `run \`sonata auth add ${name}\` (for sonata) or check opencode's own credential store.`);
+        }
+        childEnv[envVarForGateway(name)] = key;
+    }
+    // A sonata-owned credential is already in LiteLLM's native format. Point
+    // directly at its persistent directory so LiteLLM's refresh survives serve.
+    const chatgptGateway = Object.entries(native.gateways)
+        .find(([, gateway]) => gateway.auth === 'codex-oauth');
+    if (chatgptGateway) {
+        const [name, gateway] = chatgptGateway;
+        if (gateway.credentialSource === 'sonata') {
+            const dir = credentialDir(home, name);
+            if (!existsSync(join(dir, 'auth.json'))) {
+                throw new Error(`sonata serve: gateway "${name}" takes its credential from sonata but none is stored — ` +
+                    `run \`sonata auth login ${name}\`.`);
+            }
+            childEnv.CHATGPT_TOKEN_DIR = dir;
+        }
+        else {
+            const record = readChatGptOAuth(home, gateway.credentialSource);
+            if (record === null) {
+                throw new Error('sonata serve: a native gateway uses codex-oauth but no ChatGPT credential was found ' +
+                    `in ${codexAuthPath(home)} or ${opencodeAuthPath(home)} — ` +
+                    `run \`sonata auth login ${name}\`, or \`codex login\`.`);
+            }
+            const tokenDir = join(tempDir, 'chatgpt');
+            mkdirSync(tokenDir, { recursive: true, mode: 0o700 });
+            writeFileSync(join(tokenDir, 'auth.json'), JSON.stringify(record), { mode: 0o600 });
+            childEnv.CHATGPT_TOKEN_DIR = tokenDir;
+        }
+    }
+    // Copilot's api-key.json is refreshed and re-exchanged in place too, so a
+    // sonata-owned credential must likewise avoid the temporary directory.
+    const copilotGateway = Object.entries(native.gateways)
+        .find(([, gateway]) => gateway.auth === 'copilot-oauth');
+    if (copilotGateway) {
+        const [name, gateway] = copilotGateway;
+        if (gateway.credentialSource === 'sonata') {
+            const dir = credentialDir(home, name);
+            if (!existsSync(join(dir, 'api-key.json'))) {
+                throw new Error(`sonata serve: gateway "${name}" takes its credential from sonata but none is stored — ` +
+                    `run \`sonata auth login ${name}\`.`);
+            }
+            childEnv.GITHUB_COPILOT_TOKEN_DIR = dir;
+        }
+        else {
+            const token = readCopilotToken(home);
+            if (token === null) {
+                throw new Error('sonata serve: a native gateway uses copilot-oauth but no Copilot login was found ' +
+                    `in ${opencodeAuthPath(home)} — run \`sonata auth login ${name}\`, ` +
+                    'or `opencode auth login` and choose github-copilot.');
+            }
+            const tokenDir = join(tempDir, 'copilot');
+            mkdirSync(tokenDir, { recursive: true, mode: 0o700 });
+            writeFileSync(join(tokenDir, 'access-token'), token, { mode: 0o600 });
+            childEnv.GITHUB_COPILOT_TOKEN_DIR = tokenDir;
+        }
+    }
+    return childEnv;
+}
+/** Default bound on how long a model-registry restart waits for the old litellm child to exit (see `litellmExitTimeoutMs`). */
+const LITELLM_EXIT_TIMEOUT_MS = 5000;
+/** Resolves `true` if `promise` had not settled after `timeoutMs`, `false` if it settled first. Never rejects. */
+async function raceTimeout(promise, timeoutMs, sleepFn) {
+    let timedOut = false;
+    await Promise.race([
+        promise,
+        sleepFn(timeoutMs).then(() => { timedOut = true; }),
+    ]);
+    return timedOut;
+}
+function listen(server, port) {
+    return new Promise((resolve, reject) => {
+        const onError = (error) => {
+            server.off('listening', onListening);
+            reject(error);
+        };
+        const onListening = () => {
+            server.off('error', onError);
+            resolve();
+        };
+        server.once('error', onError);
+        server.once('listening', onListening);
+        server.listen(port, 'localhost');
+    });
+}
+/**
+ * A shutdown, not a graceful drain: `server.close()` alone stops accepting
+ * new connections but waits for every existing one to end on its own —
+ * including idle keep-alive sockets, which under an active session can sit
+ * open well past any reasonable restart timeout. That's what made a live
+ * `sonata restart` report a killed router pid as "still running" long after
+ * the process should have exited (observed 2026-08-24, `stopServe`'s 10s
+ * wait). Idle connections are closed immediately, since nothing is lost;
+ * anything genuinely in-flight gets a short grace window before every
+ * remaining connection is forced closed, so this can never hang forever.
+ */
+function close(server) {
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        server.close((error) => {
+            if (settled)
+                return;
+            settled = true;
+            error ? reject(error) : resolve();
+        });
+        server.closeIdleConnections();
+        // `closeIdleConnections()` only reaches keep-alive sockets that already
+        // completed a request — measured directly against a connection that
+        // never sent one (e.g. a lingering TCP probe), it does nothing. A
+        // restart wants to be fast, not gentle: losing an in-flight response is
+        // an acceptable cost of the user explicitly asking to restart, so this
+        // window is short.
+        setTimeout(() => {
+            if (settled)
+                return;
+            server.closeAllConnections();
+        }, 500).unref();
+    });
+}
+/**
+ * Attaches a price to a row the router produced unpriced.
+ *
+ * Pricing lives here rather than in the router because it needs the config and
+ * the price cache, and because a token count must never be lost to a pricing
+ * failure — the row is written either way, with `source: 'none'` when no rate
+ * applies.
+ *
+ * Priced at the row's own timestamp, not at now: a row is priced by when the
+ * request ran, which is what makes a time-windowed rate mean anything.
+ */
+export function priceRow(config, home, row) {
+    try {
+        const price = resolvePrice(config, row.key, row.tokens, new Date(row.ts), loadAiPricing(home));
+        return { ...row, price };
+    }
+    catch {
+        return row; // an unpriceable row is still a row
+    }
+}
+export async function cmdServe(opts) {
+    const config = loadConfig(opts.cwd, opts.home);
+    if (!config.native)
+        throw new Error('sonata serve: no [native] table');
+    const native = config.native;
+    const masterKey = `sk-sonata-${randomBytes(32).toString('hex')}`;
+    const instanceId = opts.instanceId ?? process.env.SONATA_SERVE_INSTANCE_ID ?? randomUUID();
+    const tempDir = opts.tempDir ?? mkdtempSync(join(tmpdir(), 'sonata-litellm-'));
+    mkdirSync(tempDir, { recursive: true });
+    // Everything from here to a listening router owns `tempDir`. Cleanup used to
+    // be duplicated on two failure branches and absent from every other throw, so
+    // a run that died in between left its config behind.
+    let child;
+    let router;
+    let stopping = false;
+    const now = opts.now ?? Date.now;
+    const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const respawnDelayMs = opts.respawnDelayMs ?? 1000;
+    const maxRespawns = opts.maxRespawns ?? 5;
+    const respawnWindowMs = opts.respawnWindowMs ?? 60_000;
+    const respawnTimestamps = [];
+    try {
+        const configPath = join(tempDir, 'config.json');
+        writeFileSync(configPath, litellmConfigYaml(native, masterKey, config.unifiedModels), { mode: 0o600 });
+        let childEnv = buildChildEnv(native, opts.home, tempDir);
+        // A predecessor's orphaned litellm would hold the port and answer with the
+        // wrong master key; kill it (recorded pid only) before spawning our own.
+        killRecordedOrphan(opts.home);
+        // The litellm child dying on its own (not via `stop()`) used to go
+        // unnoticed until the next request 502'd and someone ran `sonata restart`
+        // by hand — measured directly: the child exits, nothing is watching, the
+        // router stays up and answers requests with a dead upstream. This watches
+        // the exact child this process spawned and respawns it in place, which is
+        // why it's safe where `ensure-serve.mjs`'s external health-probe respawn
+        // (bug D in the ledger) was not: there is only ever one spawn racing here,
+        // never a second `serve` guessing whether an existing one is healthy.
+        // Resolves once the current child is confirmed healthy, or once serve has
+        // given up waiting on it — awaited by the router before every litellm-bound
+        // request so a request landing in a respawn's brief startup gap waits for
+        // it, rather than getting a connection-refused failure that would cool the
+        // candidate down for a crash it already recovered from. The `.catch` means
+        // a hard failure resolves this too: the router's own fetch then fails for
+        // real, and a genuine outage cools down exactly as before.
+        // Tracks the native config LiteLLM's own config file currently reflects
+        // — legacy [native.models], unified [models], AND gateways, not just the
+        // unified table — so a config change (a new model added, an existing
+        // model's id/gateway edited under EITHER table, or a gateway's
+        // base_url/wire_format/auth/credential_source edited) while the daemon
+        // is already running can be told apart from "nothing changed" without
+        // restarting litellm on every single tier-resolution call. Gateways
+        // matter here even when the model list itself is unchanged: rerunning
+        // `sonata init` without touching model selection can still rewrite a
+        // gateway's endpoint or credential source, and litellm's own config
+        // would otherwise keep the stale one indefinitely. Legacy
+        // `native.models` matters too — `litellmConfig` (native/litellm.ts)
+        // builds the model list from `native.models` first, unconditionally, so
+        // a transitional config editing a legacy entry's id/gateway needs the
+        // same restart a unified edit gets.
+        const activeNativeSnapshot = (cfg) => JSON.stringify({ legacyModels: cfg.native?.models, models: cfg.unifiedModels, gateways: cfg.native?.gateways });
+        let activeModelsJson = activeNativeSnapshot(config);
+        // Set right before a deliberate kill-for-config-change so the crash-exit
+        // handler below (which fires for ANY exit, deliberate or not) does not
+        // also schedule its own duplicate respawn on top of the one already in
+        // flight from that deliberate restart.
+        let expectingConfigRestart = false;
+        let litellmReady = Promise.resolve();
+        const spawnLitellmChild = () => {
+            const spawned = (opts.spawnLitellm ?? defaultSpawnLitellm)(configPath, childEnv, native.ports.litellm);
+            recordLitellmPid(opts.home, spawned.pid);
+            spawned.onExit?.((code, signal) => {
+                if (stopping)
+                    return;
+                if (expectingConfigRestart) {
+                    expectingConfigRestart = false;
+                    return;
+                }
+                const nowMs = now();
+                respawnTimestamps.push(nowMs);
+                while (respawnTimestamps.length > 0 && nowMs - respawnTimestamps[0] > respawnWindowMs) {
+                    respawnTimestamps.shift();
+                }
+                console.error(`sonata serve: litellm exited unexpectedly (code=${code}, signal=${signal})`);
+                if (respawnTimestamps.length > maxRespawns) {
+                    console.error(`sonata serve: litellm crashed ${respawnTimestamps.length} times within ` +
+                        `${Math.round(respawnWindowMs / 1000)}s — giving up on automatic respawn. ` +
+                        'Fix the underlying problem, then run `sonata restart`.');
+                    return;
+                }
+                litellmReady = (async () => {
+                    await sleep(respawnDelayMs);
+                    if (stopping)
+                        return;
+                    console.error('sonata serve: respawning litellm...');
+                    child = spawnLitellmChild();
+                    await (opts.waitForLitellm ?? defaultWaitForLitellm)(native.ports.litellm);
+                })().catch((error) => {
+                    console.error(`sonata serve: respawned litellm never came up: ${String(error)}`);
+                });
+            });
+            return spawned;
+        };
+        // Detects a model-registry change (e.g. `sonata init` adding a new
+        // model while this daemon is already running) and restarts litellm with
+        // a freshly generated config so it actually knows about the new model —
+        // hot-reloading only the tier half (as `resolveTier` below already does)
+        // is not enough, since litellm's own model list is otherwise frozen at
+        // whatever it was given at startup. Reuses the same respawn machinery
+        // already proven for crash recovery, including the `litellmReady` gate
+        // every request already awaits before reaching litellm.
+        const maybeRestartForModelChange = async (freshConfig) => {
+            if (stopping)
+                return;
+            const freshModelsJson = activeNativeSnapshot(freshConfig);
+            if (freshModelsJson === activeModelsJson)
+                return;
+            if (!freshConfig.native) {
+                activeModelsJson = freshModelsJson;
+                return;
+            }
+            // Only committed once the replacement config and credentials are
+            // successfully prepared below — not up front. A gateway added without
+            // its credential yet available makes `buildChildEnv` throw; if this
+            // were set before that point, a later request (after the credential is
+            // fixed) would see `freshModelsJson === activeModelsJson` and never
+            // retry, leaving the new model unreachable until a manual `sonata
+            // restart` or another edit. Left unset here, the next request's
+            // comparison still differs and tries the restart again.
+            try {
+                writeFileSync(configPath, litellmConfigYaml(freshConfig.native, masterKey, freshConfig.unifiedModels), { mode: 0o600 });
+                childEnv = buildChildEnv(freshConfig.native, opts.home, tempDir);
+                activeModelsJson = freshModelsJson;
+                console.error('sonata serve: model registry changed — restarting litellm to pick it up...');
+                const oldChild = child;
+                expectingConfigRestart = true;
+                litellmReady = (async () => {
+                    // Wait for the old child's actual exit before spawning its
+                    // replacement: kill() only requests termination, and racing a new
+                    // spawn/probe against a still-alive old process can either fail to
+                    // bind the port or let the health probe see the stale
+                    // (old-model-list) process and declare the restart done before it
+                    // actually happened. `onExit` supports multiple independent
+                    // listeners (it's backed by `child.on('exit', cb)`), so this does
+                    // not disturb the crash-respawn handler's own listener on the
+                    // same child.
+                    //
+                    // The wait is bounded: a litellm that ignores SIGTERM (hung, or
+                    // wedged on a slow shutdown) would otherwise leave this promise
+                    // unresolved forever, and since it's installed as `litellmReady`,
+                    // every subsequent foreign-model request would then wait forever
+                    // too. Escalate to `forceKill` (SIGKILL) once, wait once more
+                    // bounded by the same timeout, then proceed regardless — a stray
+                    // process holding the port fails the following bind/probe loudly,
+                    // which is recoverable; a hung `litellmReady` is not.
+                    const exited = new Promise((resolve) => {
+                        if (oldChild?.onExit)
+                            oldChild.onExit(() => resolve());
+                        else
+                            resolve();
+                    });
+                    const exitTimeoutMs = opts.litellmExitTimeoutMs ?? LITELLM_EXIT_TIMEOUT_MS;
+                    if (await raceTimeout(exited, exitTimeoutMs, sleep)) {
+                        console.error(`sonata serve: old litellm child did not exit within ${exitTimeoutMs}ms — sending SIGKILL`);
+                        (oldChild?.forceKill ?? oldChild?.kill)?.call(oldChild);
+                        if (await raceTimeout(exited, exitTimeoutMs, sleep)) {
+                            console.error('sonata serve: old litellm child still has not exited after SIGKILL — proceeding anyway; ' +
+                                'a stray process may be holding the litellm port');
+                        }
+                    }
+                    if (stopping)
+                        return;
+                    child = spawnLitellmChild();
+                    await (opts.waitForLitellm ?? defaultWaitForLitellm)(native.ports.litellm);
+                })().catch((error) => {
+                    console.error(`sonata serve: restarted litellm never came up: ${String(error)}`);
+                });
+                oldChild?.kill();
+                await litellmReady;
+            }
+            catch (error) {
+                console.error(`sonata serve: failed to restart litellm for a model registry change: ${String(error)}`);
+            }
+        };
+        child = spawnLitellmChild();
+        await (opts.waitForLitellm ?? defaultWaitForLitellm)(native.ports.litellm);
+        // Retention is enforced where the writer starts, so a long-lived daemon
+        // cannot accumulate day-files indefinitely the way opencode's event table
+        // did (6.5 GB, and not something sonata gets to repeat in its own store).
+        try {
+            const removed = pruneLedger(opts.home, LEDGER_RETENTION_DAYS);
+            if (removed > 0)
+                console.log(`ledger: pruned ${removed} day file(s) older than ${LEDGER_RETENTION_DAYS}d`);
+        }
+        catch { /* pruning is housekeeping; it never blocks serving */ }
+        // Same retention window, same defensive posture: a long-lived daemon must
+        // also expire the session->project map, not just the ledger it relies on.
+        // Awaited so `serve` does not race its own lock against a concurrent hook;
+        // its own failure is swallowed identically to the ledger prune above.
+        try {
+            const removedSessions = await pruneSessions(opts.home, LEDGER_RETENTION_DAYS);
+            if (removedSessions > 0)
+                console.log(`sessions: pruned ${removedSessions} record(s) older than ${LEDGER_RETENTION_DAYS}d`);
+        }
+        catch { /* pruning is housekeeping; it never blocks serving */ }
+        router = createRouterServer({
+            fetch,
+            litellmBase: `http://localhost:${native.ports.litellm}`,
+            litellmKey: masterKey,
+            health: true,
+            // Which sonata.toml actually started this router, so a caller sharing the
+            // default port can tell this project's router apart from another
+            // project's on the same port.
+            configPath: resolveSonataConfigPath(opts.cwd, opts.home) ?? undefined,
+            instanceId,
+            // Goes to serve's stdout, which --daemon captures to its log file. This
+            // is the only record of which upstream served a request: litellm's access
+            // log has the path and status but not the model, so without it "did that
+            // agent really run on the foreign model?" cannot be answered from
+            // evidence.
+            log: (line) => console.log(line),
+            // Config is re-read per call (not the `config`/`native` closed over
+            // above) so a tier edit in sonata.toml takes effect without a restart.
+            resolveTier: (alias) => resolveTierAlias(loadConfig(opts.cwd, opts.home), alias),
+            // A direct `--model <key>` request's key maps to its gateway through the
+            // same unified-model table tier resolution uses, so such a row carries
+            // `gateway` and can reach pricing's gateway step. Config is re-read here
+            // too, for the same reason as `resolveTier` above.
+            resolveGateway: (key) => loadConfig(opts.cwd, opts.home).unifiedModels[key]?.gateway,
+            // Fire-and-forget, called on every litellm-bound request (direct model
+            // calls and each tier candidate alike) — not just tier resolution,
+            // since a direct `--model <key>` request for a newly added native-only
+            // model never calls `resolveTier` at all. The request that triggers
+            // this still awaits the current readiness gate below before reaching
+            // litellm, so a config-triggered restart is awaited without making
+            // this check itself asynchronous.
+            checkModelChange: () => {
+                const freshConfig = loadConfig(opts.cwd, opts.home);
+                void maybeRestartForModelChange(freshConfig).catch((error) => {
+                    console.error(`sonata serve: model-registry restart check failed: ${String(error)}`);
+                });
+            },
+            litellmReady: () => litellmReady,
+            // The injected seam (when present) receives the router's raw, unpriced
+            // row unchanged. The default below prices it against the config and
+            // writes it. Either way a ledger write is fire-and-forget.
+            recordUsage: opts.recordUsage ?? ((row) => {
+                // Deferred past the current I/O cycle so the synchronous loadConfig,
+                // pricing, and appendFileSync below never sit on the request-response
+                // path: the response is handed back to the client first. `appendRow`
+                // stays synchronous by design (Task 2); this only moves WHEN it runs.
+                setImmediate(() => {
+                    let priced = row;
+                    try {
+                        priced = priceRow(loadConfig(opts.cwd, opts.home), opts.home, row);
+                    }
+                    catch {
+                        // A config that will not load is still no reason to drop the row —
+                        // priceRow's guarantee ("written either way, with source 'none'")
+                        // must hold here too. `priced` stays the raw router row.
+                    }
+                    try {
+                        appendRow(opts.home, priced);
+                    }
+                    catch { /* a ledger write never breaks a request */ }
+                });
+            }),
+        });
+        try {
+            await listen(router, native.ports.router);
+        }
+        catch (error) {
+            if (error.code === 'EADDRINUSE') {
+                throw new Error(await occupiedPortMessage(native.ports.router, opts.probeHealth));
+            }
+            throw error;
+        }
+        recordRouterPid(opts.home, process.pid);
+    }
+    catch (error) {
+        // Suppress the respawn watcher before killing the child — otherwise its
+        // `exit` handler schedules a respawn against `configPath`, which the
+        // `rmSync` below is about to delete, producing a doomed child spawned
+        // after this whole call has already thrown.
+        stopping = true;
+        child?.kill();
+        rmSync(tempDir, { force: true, recursive: true });
+        throw error;
+    }
+    // Router is assigned by the time the try block completes; the catch rethrows.
+    // `child` is read fresh in `stop()` below (not frozen here) because a respawn
+    // can replace it after this point.
+    const startedRouter = router;
+    const address = startedRouter.address();
+    const routerPort = typeof address === 'object' && address !== null ? address.port : native.ports.router;
+    let stopped = false;
+    return {
+        routerPort,
+        litellmPort: native.ports.litellm,
+        async stop() {
+            if (stopped)
+                return;
+            stopped = true;
+            stopping = true;
+            child?.kill();
+            try {
+                unlinkSync(serveStatePath(opts.home));
+            }
+            catch { /* already gone */ }
+            try {
+                await close(startedRouter);
+            }
+            finally {
+                rmSync(tempDir, { force: true, recursive: true });
+            }
+        },
+    };
+}
+/**
+ * Starts `sonata serve` in a detached child and waits until it answers.
+ *
+ * `--daemon` used to be parsed, passed to `cmdServe`, and then ignored — the
+ * command blocked forever like the foreground one, which is what "the flag does
+ * nothing" looked like from a shell.
+ *
+ * The wait is the part worth keeping: a detached child that fails (an occupied
+ * port, a gateway LiteLLM drops) would otherwise exit silently, leaving the
+ * user with a success message and no server. Its output goes to a log file for
+ * the same reason — a detached process has nowhere else to say why it stopped.
+ */
+export async function startServeDaemon(home, argv, deps = {}, cwd = process.cwd()) {
+    const spawnFn = deps.spawn ?? spawn;
+    const probe = deps.probe ?? (async (port, id) => (await sonataRouterInstanceId(port)) === id);
+    const now = deps.now ?? Date.now;
+    const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const timeoutMs = deps.timeoutMs ?? 60_000;
+    const config = loadConfig(cwd, home);
+    if (!config.native)
+        throw new Error('sonata serve: no [native] table');
+    const port = config.native.ports.router;
+    const logPath = timestampedLogPath(home, 'serve');
+    mkdirSync(dirname(logPath), { recursive: true });
+    const log = openSync(logPath, 'a');
+    // Generated here, before spawning, and handed to the child via its own
+    // environment — so the polling loop below can tell its own freshly-spawned
+    // process apart from a stale router that happens to still be answering the
+    // same port, which is what let `sonata restart` false-report success
+    // against a leftover daemon (see the design doc for the reproduction).
+    const instanceId = randomUUID();
+    // Explicit, not inherited: a daemon started to serve *every* project
+    // (`route on/auto --global`) must not bind itself to whichever project's
+    // session happened to trigger it first — the router is a single process,
+    // so its config has to be the one every routed project actually shares.
+    // The caller passes `home` here for that case (see route.ts); a plain
+    // `sonata serve --daemon` keeps inheriting the shell's own cwd.
+    const child = spawnFn(argv[0], argv.slice(1), {
+        detached: true,
+        stdio: ['ignore', log, log],
+        cwd,
+        env: { ...process.env, SONATA_SERVE_INSTANCE_ID: instanceId },
+    });
+    child.unref();
+    const deadline = now() + timeoutMs;
+    for (;;) {
+        if (await probe(port, instanceId))
+            return { pid: child.pid ?? 0, port, logPath };
+        if (now() > deadline) {
+            throw new Error(`sonata serve: the daemon did not answer on port ${port} within ${Math.round(timeoutMs / 1000)}s. ` +
+                `See ${logPath}`);
+        }
+        await sleep(500);
+    }
+}
+/**
+ * Finds the OS pid bound to a TCP port, purely so `stopServe` can print it —
+ * sonata never acts on what this returns. `lsof -ti` prints one pid per line;
+ * an empty or ambiguous (more than one) result means "don't know", which the
+ * caller treats the same as a lookup failure.
+ */
+function defaultFindPortPid(port) {
+    try {
+        const out = execFileSync('lsof', ['-ti', `:${port}`, '-sTCP:LISTEN'], { encoding: 'utf8' }).trim();
+        if (out === '')
+            return undefined;
+        const pids = out.split('\n').filter((line) => line !== '');
+        return pids.length === 1 ? pids[0] : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Whether a pid still exists. `process.kill(pid, 0)` sends no signal, only
+ * probes: `ESRCH` means the process is gone, anything else (including
+ * `EPERM` — exists, just not owned by us) means it is still alive. An
+ * unrecognized error is treated as alive too, so a probe failure never makes
+ * `stopServe` declare victory early.
+ */
+function defaultIsAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code !== 'ESRCH';
+    }
+}
+/**
+ * Kills whatever sonata router currently holds the configured port, using
+ * only the pids `cmdServe` itself recorded — never a pid found by scanning
+ * the OS, which could belong to an unrelated process reusing the port after
+ * a previous sonata instance already exited.
+ *
+ * The recorded router pid is `process.pid` of the process that called
+ * `cmdServe` and won the bind. Killing it is intentional: `sonata restart`
+ * makes the lifecycle trade explicit instead of leaving a stale router
+ * unreachable forever.
+ */
+export async function stopServe(opts) {
+    const config = loadConfig(opts.cwd, opts.home);
+    if (!config.native)
+        throw new Error('sonata restart: no [native] table');
+    const port = config.native.ports.router;
+    const probeHealth = opts.probeHealth;
+    const now = opts.now ?? Date.now;
+    const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const timeoutMs = opts.timeoutMs ?? 10_000;
+    if (!(await isSonataRouter(port, probeHealth)))
+        return { killed: false };
+    const state = readServeState(opts.home);
+    if (state?.routerPid === undefined) {
+        const findPortPid = opts.findPortPid ?? defaultFindPortPid;
+        const foundPid = findPortPid(port);
+        const nextStep = foundPid !== undefined
+            ? ` Kill it yourself, then run \`sonata serve --daemon\`:\n  kill ${foundPid}`
+            : ' Kill it by hand, then run `sonata serve --daemon`.';
+        throw new Error(`sonata restart: router port ${port} answers as a sonata router, but no recorded pid for it ` +
+            `was found in ${serveStatePath(opts.home)} — it may have been started by a different sonata ` +
+            `install or an older version.${nextStep}`);
+    }
+    const kill = opts.kill ?? killPid;
+    const isAlive = opts.isAlive ?? defaultIsAlive;
+    const pids = [state.routerPid, state.litellmPid].filter((pid) => pid !== undefined);
+    for (const pid of pids)
+        kill(pid);
+    try {
+        unlinkSync(serveStatePath(opts.home));
+    }
+    catch { /* already gone */ }
+    // Wait for the pids we killed to actually exit — not for the port to go
+    // quiet. Polling the port instead confused "still dying" with "already
+    // replaced": an external supervisor that respawns `sonata serve` the
+    // instant the port frees can put a brand-new, legitimate router on the
+    // same port before our old process has finished exiting, so the port
+    // never stops answering and this used to time out reporting failure even
+    // though the kill had already succeeded. Checking the specific pids
+    // sidesteps that race entirely.
+    const deadline = now() + timeoutMs;
+    while (pids.some((pid) => isAlive(pid))) {
+        if (now() > deadline) {
+            const stillAlive = pids.filter((pid) => isAlive(pid));
+            throw new Error(`sonata restart: killed the recorded process(es) but pid(s) ${stillAlive.join(', ')} ` +
+                `are still running after ${Math.round(timeoutMs / 1000)}s.`);
+        }
+        await sleep(300);
+    }
+    return { killed: true };
+}
+/**
+ * Stops whatever router currently holds the configured port, then starts a
+ * fresh daemon in its place. The two-step split — rather than one call that
+ * always wins the bind — exists so a stale in-process router or a daemon left
+ * over from a previous build gets cleared out first: `startServeDaemon` alone
+ * just times out with "the daemon did not answer" against `EADDRINUSE`,
+ * which reads as a startup failure rather than the actual cause.
+ */
+export async function cmdRestart(home, argv, opts = { cwd: process.cwd() }) {
+    await stopServe({
+        cwd: opts.cwd, home,
+        probeHealth: opts.probeHealth, now: opts.now, sleep: opts.sleep, timeoutMs: opts.timeoutMs, kill: opts.kill,
+        findPortPid: opts.findPortPid,
+    });
+    return startServeDaemon(home, argv, opts);
+}
+//# sourceMappingURL=serve.js.map