@north-light/api 0.3.580 → 0.3.587

package/dist/core/runtime/warm-pool.js

@@ -0,0 +1,473 @@
+// warm-pool.ts — spare pre-booted roots, so `POST /v1/nodes` can answer in
+// milliseconds instead of waiting out a full pi engine boot (~3.6s in a guest
+// container, all of it `waitForBrokerViewSocket`).
+//
+// A spare is an ordinary canvas node: root, no kickoff prompt (so it never runs
+// a turn before anyone owns it), booted and holding a live broker, carrying a
+// `warm_pool` marker that hides it from every listing until it is claimed.
+// Claiming is the atomic DELETE of that marker (canvas.ts `claimWarmSpare`)
+// plus a `created` restamp and a RESHAPE onto the claimant's persona, after
+// which the node is indistinguishable from one that was just cold-spawned.
+//
+// What a spare must freeze is ONLY what a booted pi process genuinely cannot
+// change: its cwd, its profile, and the launch spec minus the two properties a
+// live session can still be told (tools, extensions, system prompt, env).
+// Everything else is claim-time reshape:
+//   • kind / mode / name / launch spec — rewritten on the row exactly as the
+//     `promote`/`yield --kind` polymorph rewrites them, so the next revive comes
+//     back as the claimed persona and `nodeEnv`'s CRTR_KIND/CRTR_MODE flip for
+//     anything the live process spawns.
+//   • persona_ack — stamped to the claimed persona so a newborn has no pending
+//     transition on its first turn.
+//   • model — swapped over the live broker socket (`setModelLive`, the same
+//     seam provider rotation uses). A model swap that cannot reach the broker
+//     FAILS the claim: the spare is closed and the caller cold-spawns, because
+//     a wrong-model node is worse than a slow one.
+//   • bearings — a spare mints with NONE (the context-intro extension withholds
+//     them from an unclaimed spare), because a block built at mint would assert
+//     the spare's placeholder `general`/`base` persona and snapshot the repo as
+//     it looked at mint time. Built and delivered here instead, over
+//     `deliver-live.ts`, once the row already wears the claimant's persona — and
+//     still landing first, since nobody has spoken in the session yet.
+//   • situational context — the sidecar is written and the block handed to the
+//     running session over `set_situational_context` (`situational-live.ts`),
+//     which queues it for the claimant's FIRST turn rather than starting one.
+//     Fails the claim on the same terms as the model: a node holding the wrong
+//     situation is worse than a slow one.
+//
+// The whole reshape is sound because every installed kind×mode builds an
+// IDENTICAL launch spec apart from `model` — verified across all 34 of them.
+// The key is derived from the actual built spec, so if that ever stops being
+// true the affected kinds simply key into their own pool instead of silently
+// being handed a mismatched node.
+//
+// Pool depth is 1 per key. It is populated lazily (the first `prefer_warm`
+// create for a key falls back to a cold spawn and mints the spare behind it)
+// and at daemon start for the most recently used profiles, so the FIRST node
+// after sitting down is warm too.
+//
+// A spare freezes the directory it was booted in, so a create launched from a
+// different directory than the last one simply misses and cold-spawns — the
+// pool serves the common case of returning to the same working directory.
+//
+// Nothing ever resumes a spare: the daemon's row query hides the pool, so a
+// spare whose broker dies (every daemon handover tears down every broker of the
+// epoch) is garbage, not a parked node. It is also invisible to the count-based
+// prune for the same reason. So the pool owns
+// its own garbage collection — `reapStaleSpares`, below, run at daemon start
+// and on every supervision tick.
+import { createHash, randomUUID } from 'node:crypto';
+import { openDb } from '../canvas/db.js';
+import { claimReconcilerLease, releaseReconcilerLease, renewReconcilerLease } from '../canvas/reconciler-leases.js';
+import { claimWarmSpare, deleteNode, getNode, listNodes, listWarmSpares, registerWarmSpare, unregisterWarmSpare, updateNode, warmSpareCount, } from '../canvas/index.js';
+import { recordedPidLiveness } from '../canvas/pid.js';
+import { nowIso } from '../fs-utils.js';
+import { admitNodeCreate } from '../command-hooks/lifecycle-create.js';
+import { isDrained } from '../../daemon/drain.js';
+import { spawnChildPrepared } from './spawn.js';
+import { blaxelEnvironmentOf } from './blaxel-environment.js';
+import { newNodeId } from './nodes.js';
+import { checkStart } from './start-check.js';
+import { buildLaunchSpecAsync } from './launch.js';
+import { setModelLive } from './model-swap.js';
+import { setSituationalContextLive } from './situational-live.js';
+import { deliverBearingsLive } from './deliver-live.js';
+import { closeNode } from './close.js';
+/** Kill switch. `CRTR_WARM_POOL=0` makes every create cold again: claims stop,
+ *  refills stop, and daemon-start pre-warm no-ops. Read live (not cached at
+ *  module load) so flipping it takes effect on the next `crtr sys daemon
+ *  restart` without a rebuild. Env rather than config because the operators who
+ *  need it are guest containers — a Blaxel sandbox spec and the local-docker
+ *  compose both already carry per-home env, and neither has a config file the
+ *  host controls.
+ *
+ *  Turn it off when a spare's ~150-300 MB resident broker is the scarce
+ *  resource (a memory-capped guest), or to isolate the pool while debugging a
+ *  node that came back wearing the wrong persona. */
+export function warmPoolEnabled() {
+    return process.env['CRTR_WARM_POOL'] !== '0';
+}
+/** Stable digest of everything a booted broker bakes in. Deliberately excludes
+ *  `model`/`modelIntent`/`modelExact` AND the situational context: both
+ *  are a socket round-trip on a live session, so both are claim-time, not
+ *  mint-time. Keeping the situation out is what stops the pool fanning into a
+ *  key (and a resident broker) per distinct ambient context. */
+function warmRecipeKey(cwd, profileId, launch) {
+    const env = Object.entries(launch.env ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    const canonical = JSON.stringify([
+        cwd,
+        profileId,
+        launch.tools ?? [],
+        launch.extensions ?? [],
+        launch.systemPrompt ?? null,
+        env,
+    ]);
+    return createHash('sha256').update(canonical).digest('hex').slice(0, 32);
+}
+/** Resolve a request's frozen half. Both cwd and profile arrive already
+ *  resolved from the create handler, so this is pure launch-spec construction. */
+async function resolveRecipe(request) {
+    const { launch } = await buildLaunchSpecAsync(request.kind, request.mode, {
+        lifecycle: 'resident',
+        hasManager: false,
+        ...(request.model !== null ? { model: request.model } : {}),
+        cwd: request.cwd,
+        profileId: request.profileId,
+    });
+    return { key: warmRecipeKey(request.cwd, request.profileId, launch), launch, request };
+}
+/** The start check on the claimant's payer, before a spare is claimed or a claim resumed: a refusal throws with the spare unclaimed
+ *  and the create request uncommitted. A claimed spare is a started node like any birth. */
+async function checkClaimantStart(request) {
+    const creator = request.creator === null ? null : await getNode(request.creator);
+    await checkStart({ grantee: creator?.grantee ?? 'app:terminal', sub: creator?.sub, grant_id: creator?.grant_id });
+}
+/** Claim a spare for `request`, reshaped onto its persona — or null when the
+ *  pool cannot serve it. The caller falls back to a cold spawn; a refused
+ *  create hook leaves the spare untouched and rejects the create. `beforeClaimDelivery`
+ *  persists birth state before the claimant's first session message can arrive. */
+export async function claimWarmNode(request, birth = {}, beforeClaimDelivery) {
+    await checkClaimantStart(request);
+    const recipe = await resolveRecipe(request);
+    if (await warmSpareCount(recipe.key) === 0)
+        return null;
+    const created = nowIso();
+    // The claim's requested deadline and delivery registration move with its
+    // marker deletion, so the spare becomes caller-owned only as a fully armed
+    // birth. meta.json follows with the same created stamp.
+    const releaseCreateGate = await admitNodeCreate({
+        kind: request.kind,
+        mode: request.mode,
+        cwd: request.cwd,
+        profile: request.profileId,
+        root: true,
+        parentId: null,
+        requestingNodeId: request.creator,
+        lifecycle: 'resident',
+        replaces: null,
+    }, request.creator === null ? 'app:terminal' : await (async () => {
+        const creator = await getNode(request.creator);
+        if (!creator?.grantee)
+            throw new Error('warm node creator has no app');
+        return creator.grantee;
+    })());
+    let nodeId;
+    try {
+        nodeId = await claimWarmSpare(recipe.key, created, birth);
+    }
+    finally {
+        releaseCreateGate();
+    }
+    if (nodeId === null)
+        return null;
+    return completeWarmClaim(nodeId, recipe, created, beforeClaimDelivery);
+}
+/** Finish a warm claim that was durably assigned to an idempotency key before a
+ *  daemon handover. The marker is already gone, so this only restores the
+ *  claimant's node shape and live-session deliveries. */
+export async function resumeWarmClaim(request, nodeId, beforeClaimDelivery) {
+    await checkClaimantStart(request);
+    return completeWarmClaim(nodeId, await resolveRecipe(request), nowIso(), beforeClaimDelivery);
+}
+async function completeWarmClaim(nodeId, recipe, created, beforeClaimDelivery) {
+    const { request } = recipe;
+    const spare = await getNode(nodeId);
+    const bootedModel = spare?.launch?.model;
+    await updateNode(nodeId, {
+        created,
+        kind: request.kind,
+        mode: request.mode,
+        name: request.name ?? request.kind,
+        creator: request.creator,
+        // The recipe a later revive replays — skip it and the node silently comes
+        // back as the spare's persona.
+        launch: recipe.launch,
+        // A newborn must not be told its persona just changed.
+        persona_ack: { kind: request.kind, mode: request.mode, lifecycle: 'resident' },
+        ...(request.model !== null ? { model_override: request.model } : {}),
+    });
+    // Persist any caller-owned birth state before the first claimant delivery.
+    // A spare has no claimant turn yet, but this keeps claimed and cold roots on
+    // the same durable-before-observable boundary.
+    try {
+        beforeClaimDelivery?.(nodeId);
+    }
+    catch (err) {
+        return await abandonSpare(nodeId, 'birth preparation', err);
+    }
+    // Bearings FIRST, before any other delivery: a spare mints without them (see
+    // canvas-context-intro.ts), so this is the node's orienting frame and it must
+    // land as the session's first entry. Built from the row we just reshaped, so
+    // the identity assertion, knowledge catalog, and `<project_context>` snapshot
+    // all describe the node actually being handed out rather than the spare as it
+    // looked whenever it happened to boot.
+    try {
+        await deliverBearingsLive(nodeId);
+    }
+    catch (err) {
+        return await abandonSpare(nodeId, 'bearings delivery', err);
+    }
+    // Spares are always minted at the kind's DEFAULT model (never an explicit
+    // pin), so this fires only when the claimant asked for a different tier.
+    const target = recipe.launch.model;
+    if (target !== undefined && target !== bootedModel) {
+        try {
+            await setModelLive(nodeId, target, recipe.launch.modelIntent);
+        }
+        catch (err) {
+            return await abandonSpare(nodeId, 'model swap', err);
+        }
+    }
+    // Spares mint with NO situational context, so this is the claimant's, handed
+    // to a session that is already past session_start. `nextTurn` delivery means
+    // it rides the first real turn instead of waking a node nobody has spoken to.
+    if (request.situationalContext !== null) {
+        try {
+            await setSituationalContextLive(nodeId, request.situationalContext);
+        }
+        catch (err) {
+            return await abandonSpare(nodeId, 'situational-context delivery', err);
+        }
+    }
+    return nodeId;
+}
+/** A claim-time reshape that could not be applied leaves a node wearing the
+ *  WRONG persona/model/situation — worse than a slow one. Close it and answer
+ *  null so the caller cold-spawns. */
+async function abandonSpare(nodeId, what, err) {
+    console.error(`[warm-pool] ${what} failed on claim; closing the spare: ${err.message}`);
+    try {
+        await closeNode(nodeId);
+    }
+    catch { /* best-effort */ }
+    return null;
+}
+// Reaping
+/** How long an UNCLAIMED spare is worth its resident broker (~150-300 MB).
+ *  A spare exists to make the NEXT create instant; one nobody has claimed in
+ *  this long is memory rented for nothing, so it is returned. Over-reaping
+ *  costs exactly one cold create (which then refills the pool), while
+ *  under-reaping accumulates hidden brokers — nothing else reaps them.
+ *
+ *  It is also what bounds the number of KEYS. Depth is 1 per key, but the key
+ *  digests the launch spec, whose extension paths carry the runtime GENERATION
+ *  hash — so every `install-runtime` re-keys the pool and strands the previous
+ *  key's spare, permanently claimable by nobody. The TTL is what collects them.
+ */
+const SPARE_TTL_MS = 60 * 60 * 1000;
+/** Return every spare that is no longer worth its broker: one whose engine is
+ *  confirmed GONE (the common case — a daemon handover tears down every broker
+ *  of its epoch and nothing resumes a spare), one whose row left `active`/`idle`,
+ *  and one nobody claimed inside `SPARE_TTL_MS`.
+ *
+ *  Order matters: the marker is deleted FIRST, atomically, which is the same
+ *  move a claim makes — so a spare handed out a microsecond earlier is left
+ *  alone (`unregisterWarmSpare` returns false) and never torn down under its
+ *  new owner. Only after winning the marker is the broker killed and the row +
+ *  dir deleted. Never throws: a failed reap is retried on the next sweep. */
+export async function reapStaleSpares() {
+    const reaped = [];
+    let spares;
+    try {
+        spares = await listWarmSpares();
+    }
+    catch (err) {
+        console.error(`[warm-pool] spare scan failed: ${err.message}`);
+        return reaped;
+    }
+    const cutoff = Date.now() - SPARE_TTL_MS;
+    for (const spare of spares) {
+        // Registration precedes the awaited birth hooks and broker launch. Until
+        // spawnChildPrepared settles, a null PID is an in-progress mint, not a
+        // dead broker. Never delete its node out from under beginExecution.
+        if (await mintLeaseHeld(spare.node_id))
+            continue;
+        const mintedAt = Date.parse(spare.created);
+        const expired = Number.isNaN(mintedAt) || mintedAt < cutoff;
+        const liveness = recordedPidLiveness(spare.pi_pid, spare.pi_pid_identity);
+        const live = (spare.status === 'active' || spare.status === 'idle') && liveness === 'alive';
+        if (live && !expired)
+            continue;
+        // A failed probe is not evidence of death — leave it for the next sweep
+        // unless it is past its TTL anyway, where the verdict is the same.
+        if (liveness === 'indeterminate' && !expired)
+            continue;
+        try {
+            if (!await unregisterWarmSpare(spare.node_id))
+                continue; // claimed out from under us
+            if (live) {
+                try {
+                    await closeNode(spare.node_id);
+                }
+                catch { /* best-effort broker teardown */ }
+            }
+            await deleteNode(spare.node_id);
+            reaped.push(spare.node_id);
+        }
+        catch (err) {
+            console.error(`[warm-pool] reap failed for ${spare.node_id}: ${err.message}`);
+        }
+    }
+    return reaped;
+}
+/** Remove every spare, live or not, with its broker. A drained daemon mints none, so the pool stays empty until the drain ends. */
+export async function emptyWarmPool() {
+    for (const spare of await listWarmSpares()) {
+        if (!await unregisterWarmSpare(spare.node_id))
+            continue; // claimed out from under us
+        try {
+            await closeNode(spare.node_id);
+        }
+        catch (err) {
+            console.error(`[warm-pool] close failed for ${spare.node_id}: ${err.message}`);
+        }
+        await deleteNode(spare.node_id);
+    }
+}
+/** A mint in flight holds lease `warm-pool`/<key>, so two processes never mint one key at once; the spare it births holds
+ *  `warm-pool-mint`/<node id>, so no process's reaper deletes a node whose broker is still starting. A holder that dies
+ *  stops renewing, and the lease lapses. */
+const MINT_LEASE_MS = 60_000;
+/** Claims the lease and renews it until the returned release runs; undefined when another holder has it. */
+export async function holdMintLease(lane, key) {
+    const owner = randomUUID();
+    if (await claimReconcilerLease(lane, key, owner, MINT_LEASE_MS) === undefined)
+        return undefined;
+    const timer = setInterval(() => { void renewReconcilerLease(lane, key, owner, MINT_LEASE_MS).catch(() => { }); }, MINT_LEASE_MS / 3);
+    timer.unref();
+    return async () => { clearInterval(timer); await releaseReconcilerLease(lane, key, owner); };
+}
+async function mintLeaseHeld(nodeId) {
+    return await (await openDb()).prepare('SELECT 1 FROM reconciler_leases WHERE lane = ? AND key = ? AND expires_at > ?')
+        .get('warm-pool-mint', nodeId, new Date().toISOString()) !== undefined;
+}
+/** Bring the pool for `request` back to depth 1, in the background. Never
+ *  throws and never blocks a caller: a failed mint just means the next create
+ *  for this key is cold again. */
+export function refillWarmPool(request) {
+    if (!warmPoolEnabled())
+        return;
+    try {
+        // Mint at the kind's DEFAULT model and with NO situational context: both
+        // are claim-time properties, so baking either in would only narrow who the
+        // spare can serve.
+        void resolveRecipe({ ...request, model: null, situationalContext: null })
+            .then(mint)
+            .catch((err) => console.error(`[warm-pool] recipe resolution failed: ${err.message}`));
+    }
+    catch (err) {
+        console.error(`[warm-pool] recipe resolution failed: ${err.message}`);
+    }
+}
+/** Mint one spare for an already-resolved recipe, if the pool is short. */
+async function mint(recipe) {
+    if (await isDrained())
+        return;
+    const { key } = recipe;
+    // A node in a Blaxel environment is born frozen and launched by the lifecycle lane, so a spare for it would never hold a broker to claim.
+    if (await blaxelEnvironmentOf(recipe.request.profileId) !== null)
+        return;
+    const releaseKey = await holdMintLease('warm-pool', key);
+    if (releaseKey === undefined)
+        return;
+    if (await warmSpareCount(key) > 0) {
+        await releaseKey();
+        return;
+    }
+    void birthSpare(recipe, await newNodeId()).finally(() => releaseKey().catch(() => { }));
+}
+/** Births one spare as node `nodeId`. The node's mint lease is held before the node exists and until its birth settles, so no
+ *  process's reaper can take the spare for a dead broker; when the lease is not obtained nothing is born. */
+export async function birthSpare({ key, request }, nodeId) {
+    const releaseNode = await holdMintLease('warm-pool-mint', nodeId);
+    if (releaseNode === undefined) {
+        console.error(`[warm-pool] mint refused for ${request.kind}: the mint lease of ${nodeId} is held elsewhere`);
+        return;
+    }
+    try {
+        await spawnChildPrepared({
+            nodeId,
+            kind: request.kind,
+            mode: request.mode,
+            cwd: request.cwd,
+            // No kickoff: an empty prompt means no launch turn, so a spare burns no
+            // tokens and holds no history before somebody owns it.
+            prompt: '',
+            root: true,
+            // The spare is only useful once its broker accepts viewers — that wait
+            // is precisely the cost being moved off the create path.
+            readiness: 'await',
+            // Mint at the kind's DEFAULT model: `model_override` then stays unset on
+            // the spare, so a claim only ever SETS it (updateNode cannot clear a
+            // field) and a no-model claim gets a clean row.
+            profile: request.profileId,
+            // No situational context either: the claimant's is delivered live at
+            // claim time, so a spare that baked one in would serve nobody else.
+            // Mark it unclaimed the instant its row exists and BEFORE the broker
+            // launch this call then waits out — otherwise the spare would sit in
+            // every listing for the whole boot as a phantom empty conversation.
+        }, async (id) => { await registerWarmSpare(id, key); });
+    }
+    catch (err) {
+        console.error(`[warm-pool] mint failed for ${request.kind}: ${err.message}`);
+    }
+    finally {
+        await releaseNode().catch(() => { });
+    }
+}
+// Daemon-start pre-warm
+/** How many profiles get a spare at daemon start. Each spare is a resident
+ *  broker at ~150-300 MB, so this is deliberately tiny. */
+const PREWARM_PROFILES = 2;
+/** Mint a spare for the most recently used profiles so the FIRST create after
+ *  crtrd boots is warm too, not only the second. The profile is the whole
+ *  recipe: it supplies the cwd, and everything else a claim reshapes live.
+ *  Fire-and-forget — returns as soon as the mints are scheduled, never blocking
+ *  the daemon's work lane. */
+export async function prewarmRecentRecipes() {
+    if (!warmPoolEnabled())
+        return;
+    let candidates;
+    try {
+        const seen = new Set();
+        candidates = [];
+        // Newest first; `listNodes` already hides unclaimed spares.
+        for (const row of (await listNodes({ status: ['active', 'idle'] })).reverse()) {
+            if (row.parent !== null || row.kind === 'human')
+                continue;
+            const profileId = row.profile_id ?? null;
+            if (seen.has(profileId ?? ''))
+                continue;
+            seen.add(profileId ?? '');
+            // A node lands in the directory its launch names, so the profile's last
+            // root predicts the next one better than the profile's home does.
+            candidates.push({ cwd: row.cwd, profileId });
+            if (candidates.length >= PREWARM_PROFILES)
+                break;
+        }
+    }
+    catch (err) {
+        console.error(`[warm-pool] prewarm scan failed: ${err.message}`);
+        return;
+    }
+    for (const candidate of candidates) {
+        try {
+            // `general`/`base` is only the shape the spare BOOTS as — the key it is
+            // filed under excludes everything a claim can reshape, so any kind can
+            // claim it.
+            void resolveRecipe({
+                kind: 'general',
+                mode: 'base',
+                cwd: candidate.cwd,
+                profileId: candidate.profileId,
+                model: null,
+                situationalContext: null,
+                name: null,
+                creator: null,
+            }).then(mint).catch((err) => console.error(`[warm-pool] prewarm failed for ${candidate.cwd}: ${err.message}`));
+        }
+        catch (err) {
+            console.error(`[warm-pool] prewarm failed for ${candidate.cwd}: ${err.message}`);
+        }
+    }
+}

package/dist/core/scoped-state/db.d.ts

@@ -0,0 +1,17 @@
+import { DatabaseSync } from 'node:sqlite';
+import { type ScopedStore } from './paths.js';
+export declare const PROFILE_ID_SHAPE: RegExp;
+export declare function writeOwnerRoot(db: DatabaseSync, key: string, root: string): void;
+export declare function assertScopedStoreReady(store: ScopedStore): void;
+export declare function discardIncompleteScopedStore(store: ScopedStore): void;
+export declare function withProjectStateLock<T>(root: string, run: () => T): T;
+export declare function scopedStoreExists(store: ScopedStore): boolean;
+export declare function legacyScopedStorePaths(store: ScopedStore): string[];
+export declare function initializeFreshScopedStore(input: {
+    kind: 'user' | 'project' | 'provider';
+    root: string;
+}): void;
+export declare function openScopedStore(store: ScopedStore): DatabaseSync;
+export declare function createScopedStore(store: ScopedStore, create: (db: DatabaseSync) => void): void;
+export declare function withScopedWrite<T>(store: ScopedStore, run: (db: DatabaseSync) => T): T;
+export declare function closeScopedStores(): void;