@north-light/cli 0.3.560 → 0.3.562

package/dist/daemon/reconcilers/recipe-image.js

@@ -0,0 +1,7 @@
+var E=Object.defineProperty;var _=(n,e)=>E(n,"name",{value:e,configurable:!0});import{mkdtempSync as w}from"node:fs";import{tmpdir as b}from"node:os";import{join as v}from"node:path";import{hasUnseenWakeMail as S}from"../../core/canvas/deliveries.js";import{openDb as s,withCanvasWrite as N}from"../../core/canvas/db.js";import{emitEvent as d}from"../../core/events/emit.js";import{appendRunEvent as D}from"../../core/runs/events.js";import{blaxelConfigFromEnv as A}from"../../core/runtime/runner/blaxel/client.js";import{buildImage as R,environmentImageName as h,prepareBuildDir as T,removeBuildDir as L}from"../../core/runtime/runner/blaxel/image-build.js";import{mintInstallationToken as O}from"../api/handlers/git-credential.js";import{RECIPE_HAS_CONTENT_SQL as p,recipeOf as y}from"../api/environment-of.js";import{brokerSignalBus as C}from"../broker-signals/bus.js";const g=3*6e4,I=10*6e4;async function X(n){const e=await(await s()).prepare(`SELECT ${p} AS has_content, image_revision, image_ref, image_error FROM environments WHERE object = ?`).get(n);if(e===void 0)throw new Error(`no environment ${n}`);return e.has_content?e.image_ref===null||e.image_revision===null?{state:"pending",error:e.image_error}:{state:"image",ref:e.image_ref,revision:Number(e.image_revision)}:{state:"base"}}_(X,"resolveEnvironmentImage");async function q(n){return await C.subscribe(e=>{n(e)},{kinds:["environment_image"]})}_(q,"subscribeImageBuilt");async function U(n,e){const i=n.recipe.repositories.map(a=>a.github_full_name),r=i.length===0?null:await O(n.person,i),t=w(v(b(),"crtr-recipe-image-"));try{return await T({recipe:n.recipe,revision:n.revision,token:r?.token??null,dir:t}),await R(e,h(n.environmentId),t)}finally{L(t)}}_(U,"defaultBuild");class z{static{_(this,"RecipeImageReconciler")}deps;constructor(e={}){this.deps=e}async run(e,i){if(i.lifecycle.acceptsDetachedWork())try{const r=new Date(e).toISOString(),t=await(await s()).prepare(`SELECT object FROM environments WHERE kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${p}
+        AND (image_building_at IS NULL OR image_building_at <= ?)`).all(r);for(const{object:a}of t){const c=await this.claim(a,e);c!==null&&i.lifecycle.registerDetached(this.buildClaimed(a,c.revision,c.lease).catch(o=>{d({level:"error",event:"recipe_image.failed",error:o})}))}}catch(r){d({level:"error",event:"recipe_image.reconcile_failed",error:r})}}async claim(e,i){const r=new Date(i+g).toISOString(),t=await(await s()).prepare(`UPDATE environments SET image_building_at = ?
+      WHERE object = ? AND kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${p} AND (image_building_at IS NULL OR image_building_at <= ?)
+      RETURNING recipe_revision`).get(r,e,new Date(i).toISOString());return t===void 0?null:{revision:Number(t.recipe_revision),lease:r}}async buildClaimed(e,i,r){const t=this.deps.now??Date.now;let a=r;const c=setInterval(()=>{const o=new Date(t()+g).toISOString();(async()=>{(await(await s()).prepare("UPDATE environments SET image_building_at = ? WHERE object = ? AND image_building_at = ?").run(o,e,a)).changes===1&&(a=o)})().catch(()=>{})},g/3);c.unref();try{const o=await y(e);if(o===null||o.revision!==i){await(await s()).prepare("UPDATE environments SET image_building_at = NULL WHERE object = ? AND image_building_at = ?").run(e,a);return}let l;try{const m=await(await s()).prepare("SELECT person FROM profiles WHERE environment_id = ? ORDER BY created_at LIMIT 1").get(e);if(m===void 0)throw new Error(`no profile runs in environment ${e}, so there is no person to take a GitHub token for`);const u={environmentId:e,revision:i,recipe:o,person:m.person};l=await(this.deps.build??(f=>U(f,A())))(u)}catch(m){await this.recordFailure(e,a,m instanceof Error?m.message:String(m),t());return}await(await s()).prepare(`UPDATE environments SET image_revision = ?, image_ref = ?, image_error = NULL, image_building_at = NULL
+        WHERE object = ? AND image_building_at = ? AND COALESCE(image_revision, 0) < ?`).run(i,l.ref,e,a,i)}finally{clearInterval(c)}}async recordFailure(e,i,r,t){const a=new Date(t+I).toISOString();if((await(await s()).prepare("UPDATE environments SET image_error = ?, image_building_at = ? WHERE object = ? AND image_building_at = ?").run(r,a,e,i)).changes===1){d({level:"error",event:"recipe_image.build_failed",fields:{environment_id:e,error:r}});for(const o of await W(e))try{await N(async l=>{await D(l,o.run_id,o.node_id,"environment.image_failed",{environment_id:e,error:r})})}catch(l){d({level:"error",event:"recipe_image.failure_event_failed",node_id:o.node_id,error:l})}}}}async function W(n){const e=await(await s()).prepare(`SELECT n.node_id, n.run_id FROM nodes n JOIN profiles p ON p.id = n.profile_id
+    WHERE p.environment_id = ? AND n.status IN ('active', 'idle') AND n.run_id IS NOT NULL
+      AND NOT EXISTS (SELECT 1 FROM executions e WHERE e.node_id = n.node_id AND e.state != 'exited')`).all(n),i=[];for(const r of e)await S(r.node_id)&&i.push(r);return i}_(W,"nodesWaitingOnImage");export{g as LEASE_MS,I as RETRY_AFTER_FAILURE_MS,z as RecipeImageReconciler,W as nodesWaitingOnImage,X as resolveEnvironmentImage,q as subscribeImageBuilt};

package/dist/daemon/reconcilers/row-claim.d.ts

@@ -0,0 +1,19 @@
+/** Tables whose rows carry `claimed_by` and `lease_expires_at` (002-fleet.sql), with their id column. */
+declare const CLAIMABLE: {
+    readonly nodes: "node_id";
+    readonly structured_requests: "request_id";
+    readonly human_requests: "request_id";
+};
+export type ClaimableTable = keyof typeof CLAIMABLE;
+export declare class ClaimLostError extends Error {
+    constructor(table: string, id: string);
+}
+export interface RowClaim {
+    /** Throws `ClaimLostError` unless this claim still owns the row (not taken over, not expired). Called inside a `withCanvasWrite`
+     *  body it locks the row (`FOR UPDATE`) until commit, so a takeover cannot slip in between the check and the writes that follow. */
+    assertOwned(): Promise<void>;
+}
+/** Runs `work` when this process claims the row; returns false without running it when another process holds the claim. A failed
+ *  `work` releases the claim and rethrows, so the next pass retries it. */
+export declare function withRowClaim(table: ClaimableTable, id: string, ttlMs: number, work: (claim: RowClaim) => Promise<void>): Promise<boolean>;
+export {};

package/dist/daemon/reconcilers/row-claim.js

@@ -0,0 +1,2 @@
+var E=Object.defineProperty;var i=(e,r)=>E(e,"name",{value:r,configurable:!0});import{randomUUID as D}from"node:crypto";import{openDb as w}from"../../core/canvas/db.js";const d={nodes:"node_id",structured_requests:"request_id",human_requests:"request_id"};class _ extends Error{static{i(this,"ClaimLostError")}constructor(r,t){super(`row claim lost: ${r} ${t} is no longer held by this process`)}}async function L(e,r,t,p){const a=d[e],n=D(),o=await w(),u=new Date().toISOString();if((await o.prepare(`UPDATE ${e} SET claimed_by = ?, lease_expires_at = ?
+    WHERE ${a} = ? AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?)`).run(n,new Date(Date.now()+t).toISOString(),r,u)).changes!==1)return!1;let s=!1;const c=setInterval(()=>{o.prepare(`UPDATE ${e} SET lease_expires_at = ? WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ?`).run(new Date(Date.now()+t).toISOString(),r,n,new Date().toISOString()).then(l=>{l.changes!==1&&(s=!0)},()=>{s=!0})},Math.max(1,Math.floor(t/3)));c.unref();const m={async assertOwned(){if(s)throw new _(e,r);if(await(await w()).prepare(`SELECT 1 AS held FROM ${e} WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ? FOR UPDATE`).get(r,n,new Date().toISOString())===void 0)throw s=!0,new _(e,r)}};try{await p(m)}finally{clearInterval(c),await o.prepare(`UPDATE ${e} SET claimed_by = NULL, lease_expires_at = NULL WHERE ${a} = ? AND claimed_by = ?`).run(r,n)}return!0}i(L,"withRowClaim");export{_ as ClaimLostError,L as withRowClaim};

package/dist/daemon/reconcilers/storage-due-work.d.ts

@@ -1,10 +1,16 @@
+import { type BlobStore } from '../../core/blobs/index.js';
 import { type StorageClient, type StorageConfig } from '../../core/storage/client.js';
 import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
 export declare const STORAGE_DUE_WORK_INTERVAL_MS = 60000;
 /** An upload no run has used this long after creation is released. */
 export declare const UNUSED_UPLOAD_TTL_MS: number;
 /** Storage deletes attempted per kind per pass; the rest wait for the next pass. */
 export declare const STORAGE_DUE_WORK_BUDGET = 200;
+/** A deleted node's blobs and `objects` row are removed this long after `deleted_at`; until then the owner can undelete it. */
+export declare const NODE_BLOB_RETENTION_MS: number;
+/** Nodes purged per pass; the rest wait for the next pass. */
+export declare const NODE_PURGE_BUDGET = 25;
 /** Consecutive `storage_unavailable` failures after which a pass stops early. */
 export declare const STORAGE_DOWN_AFTER = 3;
 export interface StorageDueWorkContext {
@@ -13,20 +19,29 @@ export interface StorageDueWorkContext {
 export interface StorageDueWorkDeps {
     config: () => StorageConfig | null;
     client: () => StorageClient;
+    blobs?: () => Promise<BlobStore>;
 }
 export interface StorageDueWorkResult {
     marked_uploads: number;
     expired_shares: number;
     deleted_uploads: number;
     deleted_share_objects: number;
+    purged_nodes: number;
     failed: number;
 }
 export declare class StorageDueWorkReconciler {
     private readonly deps;
-    private lastPassAt;
-    private inFlight;
     constructor(deps?: StorageDueWorkDeps);
     run(now: number, ctx: StorageDueWorkContext): void;
     /** One pass over the due-work rows. Exposed for tests. */
-    pass(now: number): Promise<StorageDueWorkResult>;
+    pass(now: number, lane?: LanePass): Promise<StorageDueWorkResult>;
+    /** Delete the blobs of nodes deleted over 90 days ago, then the node's `objects` row. Every such node loses its blobs;
+     *  its row goes only if no foreign key refers to it (`objects.owner`, `revisions.author`, `object_kinds.owner` have no
+     *  cascade), which Postgres decides at the DELETE (23503), so a reference added meanwhile cannot slip through. Blobs go
+     *  first, outside any transaction: a crash between the two is retried (the retry finds no blobs), and two passes at once
+     *  both succeed because deleting an absent blob and an absent row are no-ops. The sweep walks `(deleted_at, id)` from a
+     *  cursor kept in the lane's state, so a row that stays referenced is passed over once per sweep, not re-selected on every
+     *  pass; the cursor restarts at the end of the sweep. */
+    private purgeDeletedNodes;
+    private settleFileTransfers;
 }

package/dist/daemon/reconcilers/storage-due-work.js

@@ -1 +1,2 @@
-var D=Object.defineProperty;var d=(i,t)=>D(i,"name",{value:t,configurable:!0});import{withCanvasWrite as v}from"../../core/canvas/db.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{RunError as E}from"../../core/runs/operations.js";import{storageClient as S,storageConfig as k}from"../../core/storage/client.js";import{deleteUploadRow as O,expireShares as U,markShareObjectDeleted as R,markUploadsDeletePending as b,pendingUploadDeletes as A,sharesOwingObjectDelete as T,staleUnusedUploads as x}from"../../core/storage/tables.js";const y=6e4,I=1440*6e4,w=200,N=3,F=d(i=>i instanceof E&&i.code==="storage_unavailable","unavailable");class M{static{d(this,"StorageDueWorkReconciler")}deps;lastPassAt=Number.NEGATIVE_INFINITY;inFlight=!1;constructor(t={config:k,client:S}){this.deps=t}run(t,l){if(this.inFlight||t-this.lastPassAt<y||!l.lifecycle.acceptsDetachedWork())return;this.lastPassAt=t;let o;try{o=this.deps.config()}catch(e){p.fresh(()=>c({level:"error",event:"storage.due_work.failed",error:e}));return}if(!o)return;this.inFlight=!0;const f=p.fresh(()=>this.pass(t)).then(()=>{}).catch(e=>{p.fresh(()=>c({level:"error",event:"storage.due_work.failed",error:e}))}).finally(()=>{this.inFlight=!1});l.lifecycle.registerDetached(f)}async pass(t){const l=new Date(t).toISOString(),{marked:o,expired:f}=await v(async r=>{const s=await x(new Date(t-I).toISOString(),r);return await b(s,r,l),{marked:s.length,expired:(await U(l,r)).length}}),e={marked_uploads:o,expired_shares:f,deleted_uploads:0,deleted_share_objects:0,failed:0},h=(await A()).slice(0,w),_=(await T()).slice(0,w);if(h.length||_.length){const r=this.deps.client();let s,n=0,u=!1;const g=d(async a=>{if(u)return!1;try{return await a(),n=0,!0}catch(m){return e.failed++,s??=m,n=F(m)?n+1:0,n>=N&&(u=!0),!1}},"attempt");for(const a of h)await g(()=>r.deleteUpload(a.upload_id))&&(await O(a.upload_id),e.deleted_uploads++);for(const a of _)await g(()=>r.deleteShare(a.share_id))&&(await R(a.share_id),e.deleted_share_objects++);s!==void 0&&c({level:"warn",event:"storage.due_work.deletes_failed",error:s,fields:{failed:e.failed,storage_unavailable:u}})}return(e.marked_uploads||e.expired_shares||e.deleted_uploads||e.deleted_share_objects)&&c({level:"info",event:"storage.due_work.settled",outcome:"succeeded",fields:{...e}}),e}}export{N as STORAGE_DOWN_AFTER,w as STORAGE_DUE_WORK_BUDGET,y as STORAGE_DUE_WORK_INTERVAL_MS,M as StorageDueWorkReconciler,I as UNUSED_UPLOAD_TTL_MS};
+var S=Object.defineProperty;var u=(_,a)=>S(_,"name",{value:a,configurable:!0});import{blobStore as O}from"../../core/blobs/index.js";import{openDb as v,withCanvasWrite as w}from"../../core/canvas/db.js";import{emitEvent as h}from"../../core/events/emit.js";import{operationIdContext as E}from"../../core/events/operation-id.js";import{RunError as R}from"../../core/runs/operations.js";import{storageClient as b,storageConfig as N}from"../../core/storage/client.js";import{deleteUploadRow as T,expireShares as y,markShareObjectDeleted as k,markUploadsDeletePending as U,pendingUploadDeletes as A,sharesOwingObjectDelete as x,staleUnusedUploads as L}from"../../core/storage/tables.js";import{runThrottledLane as I}from"./throttled-lane.js";const j=6e4,W=1440*6e4,D=200,B=5*6e4,M=2160*60*6e4,m=25,F=["conversations","events","jobs","attachments"],G=3,C=u(_=>_ instanceof R&&_.code==="storage_unavailable","unavailable");class z{static{u(this,"StorageDueWorkReconciler")}deps;constructor(a={config:N,client:b}){this.deps=a}run(a,e){if(!e.lifecycle.acceptsDetachedWork())return;const t=E.fresh(()=>I("storage-due-work","",a,j,B,async l=>{await this.pass(a,l)})).then(()=>{}).catch(l=>{E.fresh(()=>h({level:"error",event:"storage.due_work.failed",error:l}))});e.lifecycle.registerDetached(t)}async pass(a,e){const t={marked_uploads:0,expired_shares:0,deleted_uploads:0,deleted_share_objects:0,purged_nodes:0,failed:0};return t.purged_nodes=await this.purgeDeletedNodes(a,e),this.deps.config()&&await this.settleFileTransfers(a,t),(t.purged_nodes||t.marked_uploads||t.expired_shares||t.deleted_uploads||t.deleted_share_objects)&&h({level:"info",event:"storage.due_work.settled",outcome:"succeeded",fields:{...t}}),t}async purgeDeletedNodes(a,e){const t=new Date(a-M).toISOString(),l=await v(),f=e?.state?JSON.parse(e.state):null,o=await l.prepare(`SELECT id, deleted_at FROM objects WHERE type = 'node' AND deleted_at < ?
+      AND (deleted_at, id) > (?, ?) ORDER BY deleted_at, id LIMIT ?`).all(t,f?.deleted_at??"",f?.id??"",m);if(o.length===0)return f&&await e?.save(null),0;const p=await(this.deps.blobs??O)();let d=0,s=-1;for(const[r,{id:c}]of o.entries())try{for(const i of F)for(const n of await p.list(`${i}/${c}/`))await p.delete(n);await w(async i=>{await i.prepare("DELETE FROM objects WHERE id = ? AND type = 'node' AND deleted_at < ?").run(c,t)},{savepoint:!0}),d++}catch(i){i.code!=="23503"&&(s<0&&(s=r),h({level:"warn",event:"storage.due_work.node_purge_failed",error:i,fields:{node_id:c}}))}if(s>=0){const r=o[s-1];await e?.save(r?JSON.stringify({deleted_at:r.deleted_at,id:r.id}):e.state)}else{const r=o[o.length-1];await e?.save(o.length<m?null:JSON.stringify({deleted_at:r.deleted_at,id:r.id}))}return d}async settleFileTransfers(a,e){const t=new Date(a).toISOString(),{marked:l,expired:f}=await w(async d=>{const s=await L(new Date(a-W).toISOString(),d);return await U(s,d,t),{marked:s.length,expired:(await y(t,d)).length}});e.marked_uploads=l,e.expired_shares=f;const o=(await A()).slice(0,D),p=(await x()).slice(0,D);if(o.length||p.length){const d=this.deps.client();let s,r=0,c=!1;const i=u(async n=>{if(c)return!1;try{return await n(),r=0,!0}catch(g){return e.failed++,s??=g,r=C(g)?r+1:0,r>=G&&(c=!0),!1}},"attempt");for(const n of o)await i(()=>d.deleteUpload(n.upload_id))&&(await T(n.upload_id),e.deleted_uploads++);for(const n of p)await i(()=>d.deleteShare(n.share_id))&&(await k(n.share_id),e.deleted_share_objects++);s!==void 0&&h({level:"warn",event:"storage.due_work.deletes_failed",error:s,fields:{failed:e.failed,storage_unavailable:c}})}}}export{M as NODE_BLOB_RETENTION_MS,m as NODE_PURGE_BUDGET,G as STORAGE_DOWN_AFTER,D as STORAGE_DUE_WORK_BUDGET,j as STORAGE_DUE_WORK_INTERVAL_MS,z as StorageDueWorkReconciler,W as UNUSED_UPLOAD_TTL_MS};

package/dist/daemon/reconcilers/storage-maintenance.d.ts

@@ -10,20 +10,9 @@ export interface StorageMaintenanceContext {
     /** A live engine handle is the sweep's proof that a node is NOT disposable. */
     fleet: FleetRegistry;
 }
-/** Recurring storage cleanup. The in-memory throttle clocks intentionally reset
- * on daemon boot, so the first tick runs every full sweep — which is also how
- * managed-worktree reconciliation gets its start-up pass. */
+/** Recurring storage cleanup. Each sweep's throttle and overlap guard is a `reconciler_leases` row (`runThrottledLane`),
+ * so two reconciler processes never run one sweep at once and a restart does not repeat a sweep early. */
 export declare class StorageMaintenanceReconciler {
-    private lastSpareSweepAt;
-    private lastCreateRequestSweepAt;
-    private lastEnvironmentTouchAt;
-    private lastRunExpiryAt;
-    private lastRunEventPassAt;
-    private runEventCursor;
-    private lastAnalyzeAt;
-    private lastRetentionSweepAt;
-    private lastContextStripAt;
-    private contextStripInFlight;
     private focusGcInFlight;
     /** Wall time of the last cache pass; null until the first one, which runs at daemon boot. */
     private lastCachePassAt;
@@ -59,6 +48,9 @@ export declare class StorageMaintenanceReconciler {
     private gcStaleFocuses;
     /** Delete create idempotency keys once their bounded recovery window ends. */
     private gcCreateRequests;
+    /** Delete expired `viewer_attachments` rows. A viewer that never detached cleanly (its API process died) leaves its
+     * row behind; readers already ignore it by `expires_at`, so this only bounds the table. */
+    private gcViewerAttachments;
     /** Stamp the machine environment as seen. A failed stamp is an event, never an error out of the tick. */
     private touchEnvironment;
     /** Collect dead or expired hidden warm-pool spares. */

package/dist/daemon/reconcilers/storage-maintenance.js

@@ -1 +1 @@
-var I=Object.defineProperty;var l=(f,t)=>I(f,"name",{value:t,configurable:!0});import{rm as y}from"node:fs/promises";import{closeFocusRow as A,getFocusById as S,listFocuses as N,listNodes as w}from"../../core/canvas/index.js";import{pruneExpiredIdempotencyKeys as R}from"../../core/runs/operations.js";import{getNode as g,lastActivityMs as u,listTerminalPlacementResidue as T,pruneExpiredCreateRequests as C,pruneNodes as F}from"../../core/canvas/canvas.js";import{lastContextStripMs as x,stripContextDir as P}from"../../core/canvas/context-strip.js";import{isAppRun as D}from"../../core/canvas/app-run.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{emitEvent as i}from"../../core/events/emit.js";import{operationIdContext as s}from"../../core/events/operation-id.js";import{listLivePanesAsync as G,tearDownNode as V}from"../../core/runtime/placement.js";import{reapStaleSpares as b}from"../../core/runtime/warm-pool.js";import{GroveInstanceSweepReconciler as M}from"./grove-instance-sweep.js";import{ManagedWorktreeSweepReconciler as Y}from"./managed-worktree-sweep.js";import{compactRunEvents as W,expireRunEvents as k}from"../../core/runs/events.js";import{openDb as q}from"../../core/canvas/db.js";import{touchMachineEnvironment as L}from"../../core/environments.js";import{retentionHold as _}from"./live-obligation.js";import{pruneMemoryReads as O}from"../../core/canvas/memory-reads.js";import{existsSync as z}from"node:fs";import{launcherRequest as U}from"../../launcher/client.js";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{cachePassDue as K}from"../../launcher/limits/policy.js";const $=10*6e4,m=60*1e3,B=60*1e3,v=5*6e4,Q=2,X=1440*6e4,Z=2e3;class Ae{static{l(this,"StorageMaintenanceReconciler")}lastSpareSweepAt=Number.NEGATIVE_INFINITY;lastCreateRequestSweepAt=Number.NEGATIVE_INFINITY;lastEnvironmentTouchAt=Number.NEGATIVE_INFINITY;lastRunExpiryAt=Number.NEGATIVE_INFINITY;lastRunEventPassAt=Number.NEGATIVE_INFINITY;runEventCursor=0;lastAnalyzeAt=Number.NEGATIVE_INFINITY;lastRetentionSweepAt=Number.NEGATIVE_INFINITY;lastContextStripAt=Number.NEGATIVE_INFINITY;contextStripInFlight=!1;focusGcInFlight=!1;lastCachePassAt=null;cachePassInFlight=!1;worktreeSweep=new Y;groveInstanceSweep=new M;async run(t,r){if(await this.reapDeadResidue(t),this.scheduleFocusGc(r),await this.gcWarmPool(t),await this.touchEnvironment(t),await this.gcCreateRequests(t),t-this.lastRunEventPassAt>=Z){this.lastRunEventPassAt=t;try{await k(t);const e=await W(t,this.runEventCursor);this.runEventCursor=e.cursor}catch(e){console.warn(`run event maintenance failed: ${String(e)}`)}}if(t-this.lastRunExpiryAt>=60*6e4){try{await O(t)}catch(e){console.warn(`memory-read log prune failed: ${String(e)}`)}this.lastRunExpiryAt=t}t-this.lastAnalyzeAt>=1440*6e4&&(await(await q()).exec("ANALYZE"),this.lastAnalyzeAt=t),await this.sweepRetention(t,r),this.scheduleCachePass(t,r),this.scheduleContextStrip(t,r),this.worktreeSweep.run(t,r),this.groveInstanceSweep.run(t,r)}scheduleCachePass(t,r){this.cachePassInFlight||!K(this.lastCachePassAt,t)||process.platform!=="linux"||!z(H)||r.lifecycle.acceptsDetachedWork()&&(this.lastCachePassAt=t,this.cachePassInFlight=!0,r.lifecycle.registerDetached(U({operation:"cache.purge"}).then(()=>{}).catch(e=>{s.fresh(()=>{i({level:"warn",event:"app.cache.pass_failed",error:e})})}).then(()=>{this.cachePassInFlight=!1})))}msSinceTerminalActivity(t,r){const e=u(t);return e===0?Number.POSITIVE_INFINITY:r-e}async sweepRetention(t,r){t-this.lastRetentionSweepAt<v||r.lifecycle.acceptsDetachedWork()&&(this.lastRetentionSweepAt=t,await s.fresh(async()=>{try{const e=await F({ttlDays:Q,held:await _(r.fleet),removeDirs:l(a=>{r.lifecycle.registerDetached(Promise.all(a.map(o=>y(o,{recursive:!0,force:!0}))).then(()=>{}).catch(o=>{s.fresh(()=>{i({level:"error",event:"canvas.retention.failed",error:o})})}))},"removeDirs")});e.pruned.length>0&&i({level:"info",event:"canvas.retention.pruned",outcome:"succeeded",fields:{pruned_count:e.pruned.length}})}catch(e){i({level:"error",event:"canvas.retention.failed",error:e})}}))}scheduleContextStrip(t,r){if(this.contextStripInFlight||t-this.lastContextStripAt<v||!r.lifecycle.acceptsDetachedWork())return;this.lastContextStripAt=t,this.contextStripInFlight=!0;const e=this.stripFinishedContexts(t,r).finally(()=>{this.contextStripInFlight=!1});r.lifecycle.registerDetached(e)}async stripFinishedContexts(t,r){const e=await _(r.fleet),a=l(async n=>n!==null&&["done","canceled","dead"].includes(n.status)&&(n.pi_pid??null)===null&&(n.frozen_at??null)===null&&!D(n)&&u(n)<t-X&&!await e(n.node_id),"eligible");let o=0,p=0;try{for(const n of await w({status:["done","canceled","dead"]})){if(!c(n.node_id)||!await a(n))continue;const h=await x(n.node_id);if(h!==null&&h>=u(n)||!await a(await g(n.node_id)))continue;let d;try{d=await P(n.node_id)}catch(E){s.fresh(()=>{i({level:"warn",event:"canvas.context_strip.node_failed",error:E,fields:{node_id:n.node_id}})});continue}d.removed.length>0&&(o+=1,p+=d.bytes)}}catch(n){s.fresh(()=>{i({level:"error",event:"canvas.context_strip.failed",error:n})})}o>0&&s.fresh(()=>{i({level:"info",event:"canvas.context_strip.completed",outcome:"succeeded",fields:{node_count:o,bytes:p}})})}async reapDeadResidue(t){let r;try{r=await T()}catch(e){s.fresh(()=>{i({level:"error",event:"residue.reap_scan.failed",error:e})});return}for(const e of r){try{if(this.msSinceTerminalActivity(e,t)<$)continue}catch(a){s.fresh(()=>{i({level:"error",event:"residue.reap.failed",...c(e.node_id)?{node_id:e.node_id}:{},error:a,fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})});continue}await s.fresh(async()=>{try{i({level:"info",event:"residue.reaped",...c(e.node_id)?{node_id:e.node_id}:{},fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}}),await V(e.node_id)}catch(a){i({level:"error",event:"residue.reap.failed",...c(e.node_id)?{node_id:e.node_id}:{},error:a,fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})}})}}scheduleFocusGc(t){if(this.focusGcInFlight||!t.lifecycle.acceptsDetachedWork())return;this.focusGcInFlight=!0;const r=this.gcStaleFocuses().finally(()=>{this.focusGcInFlight=!1});t.lifecycle.registerDetached(r)}async gcStaleFocuses(){let t;try{t=await N()}catch(e){s.fresh(()=>{i({level:"error",event:"focus.gc.failed",error:e})});return}if(t.length===0)return;let r;try{r=await G()}catch(e){s.fresh(()=>{i({level:"error",event:"focus.gc.failed",error:e})});return}if(r!==null)for(const e of t){if(e.pane===null||r.has(e.pane))continue;if(!await s.fresh(async()=>{try{return(await S(e.focus_id))?.pane!==e.pane||(i({level:"info",event:"focus.gc.removed",...c(e.node_id)?{node_id:e.node_id}:{},fields:{focus_id:e.focus_id,focus_node_id:e.node_id,pane:e.pane}}),await A(e.focus_id)),!0}catch(o){return i({level:"error",event:"focus.gc.failed",error:o}),!1}}))return}}async gcCreateRequests(t){t-this.lastCreateRequestSweepAt<B||(this.lastCreateRequestSweepAt=t,await s.fresh(async()=>{const r=await C(t),e=await R(t);e>0&&i({level:"info",event:"canvas.idempotency_keys.reaped",outcome:"succeeded",fields:{reaped_count:e}}),r>0&&i({level:"info",event:"canvas.create_requests.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async touchEnvironment(t){t-this.lastEnvironmentTouchAt<m||(this.lastEnvironmentTouchAt=t,await s.fresh(async()=>{try{await L(new Date(t).toISOString())}catch(r){i({level:"warn",event:"environment.touch.failed",error:r})}}))}async gcWarmPool(t){t-this.lastSpareSweepAt<m||(this.lastSpareSweepAt=t,await s.fresh(async()=>{const r=await b();r.length>0&&i({level:"info",event:"warm_pool.spares_reaped",outcome:"succeeded",fields:{reaped_count:r.length}})}))}}export{X as CONTEXT_STRIP_GRACE_MS,$ as DEAD_REAP_GRACE_MS,v as RETENTION_SWEEP_INTERVAL_MS,Q as RETENTION_TTL_DAYS,Ae as StorageMaintenanceReconciler};
+var S=Object.defineProperty;var f=(m,t)=>S(m,"name",{value:t,configurable:!0});import{rm as R}from"node:fs/promises";import{closeFocusRow as I,getFocusById as A,listFocuses as N,listNodes as T}from"../../core/canvas/index.js";import{pruneExpiredIdempotencyKeys as P}from"../../core/runs/operations.js";import{getNode as C,lastActivityMs as p,listTerminalPlacementResidue as D,pruneExpiredCreateRequests as x,pruneNodes as F}from"../../core/canvas/canvas.js";import{lastContextStripMs as M,stripContextDir as L}from"../../core/canvas/context-strip.js";import{isAppRun as W}from"../../core/canvas/app-run.js";import{isSafeNodeId as d}from"../../core/canvas/paths.js";import{emitEvent as n}from"../../core/events/emit.js";import{operationIdContext as i}from"../../core/events/operation-id.js";import{listLivePanesAsync as b,tearDownNode as G}from"../../core/runtime/placement.js";import{reapStaleSpares as V}from"../../core/runtime/warm-pool.js";import{GroveInstanceSweepReconciler as k}from"./grove-instance-sweep.js";import{ManagedWorktreeSweepReconciler as O}from"./managed-worktree-sweep.js";import{compactRunEvents as q}from"../../core/runs/events.js";import{archiveExpiredRunEvents as U}from"../../core/events/run-event-chunks.js";import{blobStore as H}from"../../core/blobs/index.js";import{runThrottledLane as c}from"./throttled-lane.js";import{openDb as w}from"../../core/canvas/db.js";import{touchMachineEnvironment as Y}from"../../core/environments.js";import{retentionHold as y}from"./live-obligation.js";import{pruneMemoryReads as z}from"../../core/canvas/memory-reads.js";import{existsSync as K}from"node:fs";import{launcherRequest as $}from"../../launcher/client.js";import{LAUNCHER_SOCKET as B}from"../../launcher/spec.js";import{cachePassDue as Q}from"../../launcher/limits/policy.js";const X=10*6e4,l=60*1e3,g=60*1e3,h=5*6e4,Z=2,j=1440*6e4,J=2e3,ee=6e4;class Te{static{f(this,"StorageMaintenanceReconciler")}focusGcInFlight=!1;lastCachePassAt=null;cachePassInFlight=!1;worktreeSweep=new O;groveInstanceSweep=new k;async run(t,r){await this.reapDeadResidue(t),this.scheduleFocusGc(r),await this.gcWarmPool(t),await this.touchEnvironment(t),await this.gcCreateRequests(t),await this.gcViewerAttachments(t);try{await c("run-event-maintenance","",t,J,ee,async e=>{await U(await H(),t);const s=await q(t,e.state===null?0:Number(e.state));await e.save(String(s.cursor))})}catch(e){console.warn(`run event maintenance failed: ${String(e)}`)}await c("memory-read-prune","",t,60*6e4,10*6e4,async()=>{try{await z(t)}catch(e){console.warn(`memory-read log prune failed: ${String(e)}`)}}),await c("storage-analyze","",t,1440*6e4,10*6e4,async()=>{await(await w()).exec("ANALYZE")}),await this.sweepRetention(t,r),this.scheduleCachePass(t,r),this.scheduleContextStrip(t,r),this.worktreeSweep.run(t,r),this.groveInstanceSweep.run(t,r)}scheduleCachePass(t,r){this.cachePassInFlight||!Q(this.lastCachePassAt,t)||process.platform!=="linux"||!K(B)||r.lifecycle.acceptsDetachedWork()&&(this.lastCachePassAt=t,this.cachePassInFlight=!0,r.lifecycle.registerDetached($({operation:"cache.purge"}).then(()=>{}).catch(e=>{i.fresh(()=>{n({level:"warn",event:"app.cache.pass_failed",error:e})})}).then(()=>{this.cachePassInFlight=!1})))}msSinceTerminalActivity(t,r){const e=p(t);return e===0?Number.POSITIVE_INFINITY:r-e}async sweepRetention(t,r){r.lifecycle.acceptsDetachedWork()&&await c("retention-sweep","",t,h,h,()=>i.fresh(async()=>{try{const e=await F({ttlDays:Z,held:await y(r.fleet),removeDirs:f(s=>{r.lifecycle.registerDetached(Promise.all(s.map(o=>R(o,{recursive:!0,force:!0}))).then(()=>{}).catch(o=>{i.fresh(()=>{n({level:"error",event:"canvas.retention.failed",error:o})})}))},"removeDirs")});e.pruned.length>0&&n({level:"info",event:"canvas.retention.pruned",outcome:"succeeded",fields:{pruned_count:e.pruned.length}})}catch(e){n({level:"error",event:"canvas.retention.failed",error:e})}}))}scheduleContextStrip(t,r){r.lifecycle.acceptsDetachedWork()&&r.lifecycle.registerDetached(c("context-strip","",t,h,10*6e4,()=>this.stripFinishedContexts(t,r)).then(()=>{},e=>{n({level:"error",event:"canvas.context_strip.failed",error:e})}))}async stripFinishedContexts(t,r){const e=await y(r.fleet),s=f(async a=>a!==null&&["done","canceled","dead"].includes(a.status)&&(a.pi_pid??null)===null&&(a.frozen_at??null)===null&&!W(a)&&p(a)<t-j&&!await e(a.node_id),"eligible");let o=0,_=0;try{for(const a of await T({status:["done","canceled","dead"]})){if(!d(a.node_id)||!await s(a))continue;const v=await M(a.node_id);if(v!==null&&v>=p(a)||!await s(await C(a.node_id)))continue;let u;try{u=await L(a.node_id)}catch(E){i.fresh(()=>{n({level:"warn",event:"canvas.context_strip.node_failed",error:E,fields:{node_id:a.node_id}})});continue}u.removed.length>0&&(o+=1,_+=u.bytes)}}catch(a){i.fresh(()=>{n({level:"error",event:"canvas.context_strip.failed",error:a})})}o>0&&i.fresh(()=>{n({level:"info",event:"canvas.context_strip.completed",outcome:"succeeded",fields:{node_count:o,bytes:_}})})}async reapDeadResidue(t){let r;try{r=await D()}catch(e){i.fresh(()=>{n({level:"error",event:"residue.reap_scan.failed",error:e})});return}for(const e of r){try{if(this.msSinceTerminalActivity(e,t)<X)continue}catch(s){i.fresh(()=>{n({level:"error",event:"residue.reap.failed",...d(e.node_id)?{node_id:e.node_id}:{},error:s,fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})});continue}await i.fresh(async()=>{try{n({level:"info",event:"residue.reaped",...d(e.node_id)?{node_id:e.node_id}:{},fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}}),await G(e.node_id)}catch(s){n({level:"error",event:"residue.reap.failed",...d(e.node_id)?{node_id:e.node_id}:{},error:s,fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})}})}}scheduleFocusGc(t){if(this.focusGcInFlight||!t.lifecycle.acceptsDetachedWork())return;this.focusGcInFlight=!0;const r=this.gcStaleFocuses().finally(()=>{this.focusGcInFlight=!1});t.lifecycle.registerDetached(r)}async gcStaleFocuses(){let t;try{t=await N()}catch(e){i.fresh(()=>{n({level:"error",event:"focus.gc.failed",error:e})});return}if(t.length===0)return;let r;try{r=await b()}catch(e){i.fresh(()=>{n({level:"error",event:"focus.gc.failed",error:e})});return}if(r!==null)for(const e of t){if(e.pane===null||r.has(e.pane))continue;if(!await i.fresh(async()=>{try{return(await A(e.focus_id))?.pane!==e.pane||(n({level:"info",event:"focus.gc.removed",...d(e.node_id)?{node_id:e.node_id}:{},fields:{focus_id:e.focus_id,focus_node_id:e.node_id,pane:e.pane}}),await I(e.focus_id)),!0}catch(o){return n({level:"error",event:"focus.gc.failed",error:o}),!1}}))return}}async gcCreateRequests(t){await c("create-request-gc","",t,g,g,()=>i.fresh(async()=>{const r=await x(t),e=await P(t);e>0&&n({level:"info",event:"canvas.idempotency_keys.reaped",outcome:"succeeded",fields:{reaped_count:e}}),r>0&&n({level:"info",event:"canvas.create_requests.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async gcViewerAttachments(t){await c("viewer-attachment-gc","",t,l,l,()=>i.fresh(async()=>{const r=Number((await(await w()).prepare("DELETE FROM viewer_attachments WHERE expires_at <= ?").run(new Date(t).toISOString())).changes);r>0&&n({level:"info",event:"viewer_attachments.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async touchEnvironment(t){await c("environment-touch","",t,l,l,()=>i.fresh(async()=>{try{await Y(new Date(t).toISOString())}catch(r){n({level:"warn",event:"environment.touch.failed",error:r})}}))}async gcWarmPool(t){await c("warm-pool-gc","",t,l,l,()=>i.fresh(async()=>{const r=await V();r.length>0&&n({level:"info",event:"warm_pool.spares_reaped",outcome:"succeeded",fields:{reaped_count:r.length}})}))}}export{j as CONTEXT_STRIP_GRACE_MS,X as DEAD_REAP_GRACE_MS,h as RETENTION_SWEEP_INTERVAL_MS,Z as RETENTION_TTL_DAYS,Te as StorageMaintenanceReconciler};

package/dist/daemon/reconcilers/structured-request-lane.js

@@ -1 +1 @@
-var n=Object.defineProperty;var i=(o,t)=>n(o,"name",{value:t,configurable:!0});import{withCanvasWrite as a}from"../../core/canvas/db.js";import{incompleteStructuredRequestEffects as u,pendingStructuredRequestsPast as d,settleStructuredRequestIn as f}from"../../core/canvas/structured-requests.js";import{emitEvent as l}from"../../core/events/emit.js";import{applySettlementEffects as c}from"../structured-requests/effects.js";class R{static{i(this,"StructuredRequestReconciler")}async run(t){try{for(const e of await d(t)){const r=await a(async s=>await f(s,e.request_id,{intended:"expired",now:t}));r.changed&&r.row!==null&&c(r.row)}for(const e of await u())c(e)}catch(e){l({level:"error",event:"structured_request.reconcile_failed",error:e})}}}export{R as StructuredRequestReconciler};
+var f=Object.defineProperty;var c=(t,r)=>f(t,"name",{value:r,configurable:!0});import{withCanvasWrite as w}from"../../core/canvas/db.js";import{incompleteStructuredRequestEffects as m,pendingStructuredRequestsPast as p,settleStructuredRequestIn as q}from"../../core/canvas/structured-requests.js";import{emitEvent as i}from"../../core/events/emit.js";import{applySettlementEffects as a}from"../structured-requests/effects.js";import{withRowClaim as u}from"./row-claim.js";const n=6e4,d=c(t=>{i({level:"error",event:"structured_request.reconcile_failed",error:t})},"reportFailure");class C{static{c(this,"StructuredRequestReconciler")}async run(r){try{for(const e of await p(r))u("structured_requests",e.request_id,n,async s=>{const o=await w(async l=>(await s.assertOwned(),await q(l,e.request_id,{intended:"expired",now:r})));o.changed&&o.row!==null&&await a(o.row,s)}).catch(d);for(const e of await m())u("structured_requests",e.request_id,n,async s=>{await a(e,s)}).catch(d)}catch(e){i({level:"error",event:"structured_request.reconcile_failed",error:e})}}}export{C as StructuredRequestReconciler};

package/dist/daemon/reconcilers/throttled-lane.d.ts

@@ -0,0 +1,11 @@
+export interface LanePass {
+    /** What the previous pass saved, or null. */
+    state: string | null;
+    /** Fenced: false when the lane was lost to another process, in which case the pass should stop. */
+    save(state: string | null): Promise<boolean>;
+    /** Renews the lane lease and says whether this pass still holds it. Call it before each external effect: false means another
+     *  process owns the lane (this one stalled or could not renew past the lease), and the pass must stop without enacting. */
+    holds(): Promise<boolean>;
+}
+/** Runs `pass` when this process wins the lane's claim and the last pass began at least `intervalMs` before `now`. Returns whether it ran. */
+export declare function runThrottledLane(lane: string, key: string, now: number, intervalMs: number, leaseMs: number, pass: (lane: LanePass) => Promise<void>): Promise<boolean>;

package/dist/daemon/reconcilers/throttled-lane.js

@@ -0,0 +1 @@
+var v=Object.defineProperty;var a=(r,e)=>v(r,"name",{value:e,configurable:!0});import{randomUUID as E}from"node:crypto";import{openDb as L}from"../../core/canvas/db.js";import{advanceReconcilerCursor as p,claimReconcilerLease as N,releaseReconcilerLease as O,renewReconcilerLease as S}from"../../core/canvas/reconciler-leases.js";function d(r){if(r===null)return null;const e=JSON.parse(r);if(typeof e.at!="number")throw new Error(`reconciler lease cursor is not a lane cursor: ${r}`);return e}a(d,"parse");async function b(r,e,n,i,s,h){const y=await(await L()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? AND key = ?").get(r,e),c=d(y?.cursor??null);if(c!==null&&n-c.at<i)return!1;const t=E(),u=await N(r,e,t,s);if(u===void 0)return!1;try{const l=d(u.cursor);if(l!==null&&n-l.at<i)return!1;const f=l?.state??null;if(!await p(r,e,t,JSON.stringify({at:n,state:f})))return!1;let o=!1;const w=a(async()=>{if(o)return!1;try{o=!await S(r,e,t,s)}catch{o=!0}return!o},"renewNow"),m=setInterval(()=>{w()},Math.max(1,Math.floor(s/3)));m.unref();try{await h({state:f,save:a(R=>p(r,e,t,JSON.stringify({at:n,state:R})),"save"),holds:w})}finally{clearInterval(m)}return!0}finally{await O(r,e,t)}}a(b,"runThrottledLane");export{b as runThrottledLane};

package/dist/daemon/review/finish.js

@@ -1 +1 @@
-var g=Object.defineProperty;var n=(e,t)=>g(e,"name",{value:t,configurable:!0});import{withCanvasWrite as j}from"../../core/canvas/db.js";import{get as S}from"../../core/graph/bodies.js";import{documentById as q,editDocument as C,heads as $}from"../../core/graph/documents.js";import{readerFor as k}from"../../core/graph/reader.js";import{checkoutsForCwd as J}from"../../core/graph/repo-sync/repos.js";import{snapshotCommentsLocked as x}from"../../core/review/comments.js";import{readReviewSource as N}from"../../core/review/document.js";import{projectReviewCancellation as O,projectReviewResult as T,buildReviewResult as A}from"../../core/review/project.js";import{signalReviewActivity as p}from"../../core/review/signal.js";import{clearReviewSubmit as D,getReview as E,getReviewByCompanion as B,requestReviewSubmit as F,requireVisibleReview as u}from"../../core/review/store.js";import{ReviewOperationError as w}from"../../core/review/types.js";import{settleHumanRequest as v}from"../human/settle.js";import{retireCompanion as h}from"../companion-retire.js";import{companionHasQueuedWork as W,companionMidTurn as H,companionStillWorking as M}from"./companion.js";import{renderApprovalMessage as L}from"./deliver.js";function f(){return new Date().toISOString()}n(f,"now");function l(e,t){throw new w("review_terminal","review already has a terminal outcome",{state:e.state,requested:t,review:e})}n(l,"terminalConflict");function _(e){if(e.result_json===null)throw new Error(`approved review lacks its immutable result: ${e.review_id}`);return JSON.parse(e.result_json)}n(_,"approvedResult");function y(e){return`${e.review_dir}/result.json`}n(y,"resultPath");function P(e){return{project:n(async t=>{T(await u(t.request_id),e)},"project"),retire:n(async(t,o)=>{await h(t,o)},"retire")}}n(P,"effects");function Q(e){return{label:"review approved by the user",disposition:null,body_json:{review_id:e.review_id,result_path:y(e),body:L(e,y(e))}}}n(Q,"approvalNotification");function V(e,t,o){const a=o==="dismissed";return{label:a?"review dismissed":"review canceled",disposition:"human-canceled",body_json:{review_id:e.review_id,disposition:o,body:a?`The review of \`${e.document_name??e.file}\` was dismissed without approval.`:`The review of \`${e.document_name??e.file}\` was canceled${t===""?"":` \u2014 ${t}`}.`}}}n(V,"cancellationNotification");async function z(e,t,o){if(t.document_id===null||t.document_base===null)return;const a=n(async d=>await J(e,d),"reposFn"),i=await q(e,t.document_id),s=await k(e,{class:"person"},a),c=await C(e,s,i,{body:o},`edited in review ${t.review_id}`,t.document_base,{},a);if(!("stale"in c))return;const r=await $(e,i);if(!(r.length===1&&S(r[0].body_hash)===o))throw new w("stale_revision",`${t.document_name??i.name} changed since the review began; the reviewed text was not saved`,{review_id:t.review_id,document:t.document_name??i.name,base:t.document_base,heads_key:c.stale.heads_key,revision:c.stale.revision,next:`Read the current revision with \`crtr canvas read ${i.id}\`, then start a new review of it.`})}n(z,"saveReviewedDocument");async function b(e){if(e.state==="canceled")return l(e,"submit");if(e.state==="approved")return{outcome:"already_settled",record:e,result:_(e)};const t=N(e.file,"submit"),{result:o,resultJson:a}=await j(async c=>{const r=await u(e.review_id);if(r.state==="canceled")return l(r,"submit");if(r.state==="approved")return{result:_(r),resultJson:r.result_json};const d=await x(c,r.review_id),m=A(r,d,t.sha256,f());return m.changed&&await z(c,r,new TextDecoder("utf-8").decode(t.bytes)),{result:m,resultJson:JSON.stringify(m)}}),i=await v({request_id:e.review_id,outcome:"approved",result_json:a,notification:Q({...e,changed:o.changed}),post_commit:P(o)});i.outcome==="settled"&&p();const s=await u(e.review_id);return i.outcome==="already_settled"&&s.state!=="approved"?l(s,"submit"):{outcome:i.outcome,record:s,result:i.outcome==="settled"?o:_(s)}}n(b,"approveReview");async function de(e){const t=await u(e);return t.state==="canceled"?l(t,"submit"):t.state==="open"&&await M(t.companion_node_id)?(await F(e,f())&&p(),{outcome:"awaiting_companion",record:await u(e)}):b(t)}n(de,"submitReviewApproval");async function G(e,t){const o=await E(e);if(o===null||o.state!=="open"||o.submit_requested_at===null||await W(o.companion_node_id)||!t.atCompanionTurnEnd&&await H(o.companion_node_id))return null;try{return await b(o)}catch(a){throw a instanceof w&&a.code==="stale_revision"&&(await D(e),p()),a}}n(G,"completeRequestedSubmit");async function me(e){const t=await B(e);return t===null?null:G(t.review_id,{atCompanionTurnEnd:!0})}n(me,"completeCompanionRequestedSubmit");async function le(e,t={}){const o=await u(e);if(o.state==="approved")return l(o,"cancel");const a=f(),i=t.reason??"",s=t.disposition??"canceled",c=JSON.stringify({schema:"humanloop.cancel/v1",kind:"canceled",canceledAt:a,...i===""?{}:{reason:i},...t.actor===void 0?{}:{actor:t.actor},...s==="dismissed"?{disposition:s}:{}}),r=await v({request_id:e,outcome:s==="dismissed"?"dismissed":"canceled",result_json:c,notification:V(o,i,s),post_commit:{project:n(async m=>{O(await u(m.request_id),JSON.parse(c))},"project"),retire:n(async(m,R)=>{await h(m,R)},"retire")}});r.outcome==="settled"&&p();const d=await u(e);return r.outcome==="already_settled"&&d.state==="approved"?l(d,"cancel"):{outcome:r.outcome,record:d}}n(le,"cancelReview");export{le as cancelReview,me as completeCompanionRequestedSubmit,G as completeRequestedSubmit,de as submitReviewApproval};
+var g=Object.defineProperty;var n=(e,t)=>g(e,"name",{value:t,configurable:!0});import{withCanvasWrite as j}from"../../core/canvas/db.js";import{get as S}from"../../core/graph/bodies.js";import{documentById as q,editDocument as C,heads as $}from"../../core/graph/documents.js";import{readerFor as k}from"../../core/graph/reader.js";import{warmDocumentWatchers as J}from"../api/reader.js";import{checkoutsForCwd as x}from"../../core/graph/repo-sync/repos.js";import{snapshotCommentsLocked as N}from"../../core/review/comments.js";import{readReviewSource as O}from"../../core/review/document.js";import{projectReviewCancellation as T,projectReviewResult as D,buildReviewResult as A}from"../../core/review/project.js";import{signalReviewActivity as p}from"../../core/review/signal.js";import{clearReviewSubmit as E,getReview as W,getReviewByCompanion as B,requestReviewSubmit as F,requireVisibleReview as u}from"../../core/review/store.js";import{ReviewOperationError as w}from"../../core/review/types.js";import{settleHumanRequest as v}from"../human/settle.js";import{retireCompanion as h}from"../companion-retire.js";import{companionHasQueuedWork as H,companionMidTurn as M,companionStillWorking as L}from"./companion.js";import{renderApprovalMessage as P}from"./deliver.js";function f(){return new Date().toISOString()}n(f,"now");function l(e,t){throw new w("review_terminal","review already has a terminal outcome",{state:e.state,requested:t,review:e})}n(l,"terminalConflict");function _(e){if(e.result_json===null)throw new Error(`approved review lacks its immutable result: ${e.review_id}`);return JSON.parse(e.result_json)}n(_,"approvedResult");function y(e){return`${e.review_dir}/result.json`}n(y,"resultPath");function Q(e){return{project:n(async t=>{D(await u(t.request_id),e)},"project"),retire:n(async(t,o)=>{await h(t,o)},"retire")}}n(Q,"effects");function V(e){return{label:"review approved by the user",disposition:null,body_json:{review_id:e.review_id,result_path:y(e),body:P(e,y(e))}}}n(V,"approvalNotification");function z(e,t,o){const a=o==="dismissed";return{label:a?"review dismissed":"review canceled",disposition:"human-canceled",body_json:{review_id:e.review_id,disposition:o,body:a?`The review of \`${e.document_name??e.file}\` was dismissed without approval.`:`The review of \`${e.document_name??e.file}\` was canceled${t===""?"":` \u2014 ${t}`}.`}}}n(z,"cancellationNotification");async function G(e,t,o){if(t.document_id===null||t.document_base===null)return;const a=n(async d=>await x(e,d),"reposFn"),i=await q(e,t.document_id),s=await k(e,{class:"person"},a),c=await C(e,s,i,{body:o},`edited in review ${t.review_id}`,t.document_base,{},a);if(!("stale"in c))return;const r=await $(e,i);if(!(r.length===1&&S(r[0].body_hash)===o))throw new w("stale_revision",`${t.document_name??i.name} changed since the review began; the reviewed text was not saved`,{review_id:t.review_id,document:t.document_name??i.name,base:t.document_base,heads_key:c.stale.heads_key,revision:c.stale.revision,next:`Read the current revision with \`crtr canvas read ${i.id}\`, then start a new review of it.`})}n(G,"saveReviewedDocument");async function b(e){if(e.state==="canceled")return l(e,"submit");if(e.state==="approved")return{outcome:"already_settled",record:e,result:_(e)};const t=O(e.file,"submit");e.document_id!==null&&await J(e.document_id);const{result:o,resultJson:a}=await j(async c=>{const r=await u(e.review_id);if(r.state==="canceled")return l(r,"submit");if(r.state==="approved")return{result:_(r),resultJson:r.result_json};const d=await N(c,r.review_id),m=A(r,d,t.sha256,f());return m.changed&&await G(c,r,new TextDecoder("utf-8").decode(t.bytes)),{result:m,resultJson:JSON.stringify(m)}}),i=await v({request_id:e.review_id,outcome:"approved",result_json:a,notification:V({...e,changed:o.changed}),post_commit:Q(o)});i.outcome==="settled"&&p();const s=await u(e.review_id);return i.outcome==="already_settled"&&s.state!=="approved"?l(s,"submit"):{outcome:i.outcome,record:s,result:i.outcome==="settled"?o:_(s)}}n(b,"approveReview");async function le(e){const t=await u(e);return t.state==="canceled"?l(t,"submit"):t.state==="open"&&await L(t.companion_node_id)?(await F(e,f())&&p(),{outcome:"awaiting_companion",record:await u(e)}):b(t)}n(le,"submitReviewApproval");async function K(e,t){const o=await W(e);if(o===null||o.state!=="open"||o.submit_requested_at===null||await H(o.companion_node_id)||!t.atCompanionTurnEnd&&await M(o.companion_node_id))return null;try{return await b(o)}catch(a){throw a instanceof w&&a.code==="stale_revision"&&(await E(e),p()),a}}n(K,"completeRequestedSubmit");async function pe(e){const t=await B(e);return t===null?null:K(t.review_id,{atCompanionTurnEnd:!0})}n(pe,"completeCompanionRequestedSubmit");async function we(e,t={}){const o=await u(e);if(o.state==="approved")return l(o,"cancel");const a=f(),i=t.reason??"",s=t.disposition??"canceled",c=JSON.stringify({schema:"humanloop.cancel/v1",kind:"canceled",canceledAt:a,...i===""?{}:{reason:i},...t.actor===void 0?{}:{actor:t.actor},...s==="dismissed"?{disposition:s}:{}}),r=await v({request_id:e,outcome:s==="dismissed"?"dismissed":"canceled",result_json:c,notification:z(o,i,s),post_commit:{project:n(async m=>{T(await u(m.request_id),JSON.parse(c))},"project"),retire:n(async(m,R)=>{await h(m,R)},"retire")}});r.outcome==="settled"&&p();const d=await u(e);return r.outcome==="already_settled"&&d.state==="approved"?l(d,"cancel"):{outcome:r.outcome,record:d}}n(we,"cancelReview");export{we as cancelReview,pe as completeCompanionRequestedSubmit,K as completeRequestedSubmit,le as submitReviewApproval};

package/dist/daemon/run-notices.d.ts

@@ -47,3 +47,6 @@ export declare class RunNoticeOutbox {
 export declare function databaseDeps(): RunNoticeDeps;
 /** The daemon's outbox. */
 export declare function runNotices(): RunNoticeOutbox;
+/** Starts the daemon's run notices: the local append hook, plus the `run_event` signal so an append made by another process
+ *  also owes a notice. `appended` coalesces the two calls an append makes in its own process. Returns the stop function. */
+export declare function startRunNotices(outbox?: RunNoticeOutbox): Promise<() => void>;

package/dist/daemon/run-notices.js

@@ -1 +1 @@
-var h=Object.defineProperty;var u=(n,t)=>h(n,"name",{value:t,configurable:!0});import{openDb as d}from"../core/canvas/db.js";import{runNoticeTarget as p}from"../core/grants/mirror.js";import{emitEvent as w}from"../core/events/emit.js";import{describeOutboundFailure as m}from"../core/events/outbound.js";import{discardFile as _}from"../core/providers/token-cache.js";import{outboundToken as g}from"../core/providers/mint.js";const y="crtr:run_notice",A=[3e3,5e3],E=5e3,T=6e4,k=1e4;class l extends Error{static{u(this,"Fault")}retryable;status;where;constructor(t,e,s,o,a={step:"run_notice"}){super(t,o===void 0?void 0:{cause:o}),this.retryable=e,this.status=s,this.where=a}}class b{static{u(this,"RunNoticeOutbox")}deps;runs=new Map;constructor(t){this.deps=t}state(t){let e=this.runs.get(t);return e||(e={streams:0,owed:!1,inFlight:!1,timer:void 0,nextAllowedAt:0},this.runs.set(t,e)),e}forget(t,e){e.streams===0&&!e.owed&&!e.inFlight&&e.timer===void 0&&e.nextAllowedAt<=this.deps.now()&&this.runs.delete(t)}async streamOpened(t){const e=this.state(t);e.streams++,e.owed=!1,e.nextAllowedAt=0,e.timer&&(clearTimeout(e.timer),e.timer=void 0);let s=!1;return async o=>{if(s)return;s=!0,e.streams--;let a=null;try{a=await this.deps.latest(t)}catch{}e.streams===0&&a!==null&&a>o?await this.owe(t):this.forget(t,e)}}async appended(t){const e=this.runs.get(t);e&&e.streams>0||await this.owe(t)}async owe(t){let e;try{e=await this.deps.target(t)}catch{return}if(e===null)return;const s=this.state(t);s.streams>0||(s.owed=!0,this.schedule(t,s))}schedule(t,e){if(!e.owed||e.inFlight||e.timer!==void 0||e.streams>0)return;const s=Math.max(0,e.nextAllowedAt-this.deps.now());e.timer=setTimeout(()=>{e.timer=void 0,this.deliver(t,e)},s),e.timer.unref()}async deliver(t,e){if(!e.owed||e.streams>0){this.forget(t,e);return}const s=await this.deps.target(t),o=await this.deps.latest(t);if(s===null||o===null){e.owed=!1,this.forget(t,e);return}e.owed=!1,e.inFlight=!0;let a=0,r;try{for(;;){a++;try{await this.attempt(s,t,o),r=void 0;break}catch(i){r=i instanceof l?i:new l(i instanceof Error?i.message:String(i),!0);const c=this.deps.retryWaitsMs[a-1];if(!r.retryable||c===void 0)break;if(await new Promise(f=>{setTimeout(f,c).unref()}),e.streams>0){r=void 0;break}}}}finally{e.inFlight=!1}r?(e.nextAllowedAt=this.deps.now()+this.deps.failGapMs,w({level:"error",event:"run_notice.failed",fields:{run_id:t,grantee:s.grantee,attempts:a,latest_sequence:o,...r.status===void 0?{}:{status:r.status},error:r.message,...m(r.cause,r.where.url===void 0&&r.where.step==="run_notice"?{step:"run_notice",url:s.url}:r.where)}})):e.streams===0&&(e.nextAllowedAt=this.deps.now()+this.deps.ackGapMs),this.schedule(t,e),this.forget(t,e)}async attempt(t,e,s){for(let o=0;;o++){let a;try{a=await this.deps.token(t)}catch(i){throw new l(`token mint failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"notice_token"})}let r;try{r=await this.deps.fetch(t.url,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(k),headers:{Authorization:`Bearer ${a.access_token}`,"Content-Type":"application/json"},body:JSON.stringify({run_id:e,latest_sequence:s})})}catch(i){throw new l(`request failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"run_notice",url:t.url})}if(r.body?.cancel().catch(()=>{}),r.status>=200&&r.status<300)return;if(r.status===401){if(this.deps.discard(a.key),o===0)continue;throw new l("the app refused the notice token twice (401)",!0,401)}throw r.status>=500?new l(`the app answered ${r.status}`,!0,r.status):new l(`contract_fault: the app answered ${r.status}`,!1,r.status)}}}function x(){return{target:u(async n=>{const t=await(await d()).prepare("SELECT grant_id FROM nodes WHERE node_id = ? AND run_id = ?").get(n,n);return t?.grant_id?await p(t.grant_id):null},"target"),latest:u(async n=>{const t=await(await d()).prepare("SELECT event_cursor FROM nodes WHERE node_id = ? AND run_id = ?").get(n,n);return t?.event_cursor==null?null:t.event_cursor-1},"latest"),token:u(async n=>{const t=await g({audience:n.url,sub:n.sub,grantee:n.grantee,initiating_app:n.grantee,scope:[y]});return{access_token:t.access_token,key:t.key}},"token"),discard:_,fetch:u((n,t)=>fetch(n,t),"fetch"),now:Date.now,retryWaitsMs:A,ackGapMs:E,failGapMs:T}}u(x,"databaseDeps");let M;function P(){return M??=new b(x())}u(P,"runNotices");export{E as RUN_NOTICE_ACK_GAP_MS,k as RUN_NOTICE_ATTEMPT_TIMEOUT_MS,T as RUN_NOTICE_FAIL_GAP_MS,A as RUN_NOTICE_RETRY_WAITS_MS,y as RUN_NOTICE_SCOPE,b as RunNoticeOutbox,x as databaseDeps,P as runNotices};
+var w=Object.defineProperty;var u=(s,t)=>w(s,"name",{value:t,configurable:!0});import{openDb as d}from"../core/canvas/db.js";import{onRunEventAppended as f}from"../core/runs/events.js";import{brokerSignalBus as m}from"./broker-signals/bus.js";import{runNoticeTarget as _}from"../core/grants/mirror.js";import{emitEvent as p}from"../core/events/emit.js";import{describeOutboundFailure as g}from"../core/events/outbound.js";import{discardFile as y}from"../core/providers/token-cache.js";import{outboundToken as A}from"../core/providers/mint.js";const E="crtr:run_notice",k=[3e3,5e3],b=5e3,T=6e4,x=1e4;class c extends Error{static{u(this,"Fault")}retryable;status;where;constructor(t,e,n,o,a={step:"run_notice"}){super(t,o===void 0?void 0:{cause:o}),this.retryable=e,this.status=n,this.where=a}}class N{static{u(this,"RunNoticeOutbox")}deps;runs=new Map;constructor(t){this.deps=t}state(t){let e=this.runs.get(t);return e||(e={streams:0,owed:!1,inFlight:!1,timer:void 0,nextAllowedAt:0},this.runs.set(t,e)),e}forget(t,e){e.streams===0&&!e.owed&&!e.inFlight&&e.timer===void 0&&e.nextAllowedAt<=this.deps.now()&&this.runs.delete(t)}async streamOpened(t){const e=this.state(t);e.streams++,e.owed=!1,e.nextAllowedAt=0,e.timer&&(clearTimeout(e.timer),e.timer=void 0);let n=!1;return async o=>{if(n)return;n=!0,e.streams--;let a=null;try{a=await this.deps.latest(t)}catch{}e.streams===0&&a!==null&&a>o?await this.owe(t):this.forget(t,e)}}async appended(t){const e=this.runs.get(t);e&&e.streams>0||await this.owe(t)}async owe(t){let e;try{e=await this.deps.target(t)}catch{return}if(e===null)return;const n=this.state(t);n.streams>0||(n.owed=!0,this.schedule(t,n))}schedule(t,e){if(!e.owed||e.inFlight||e.timer!==void 0||e.streams>0)return;const n=Math.max(0,e.nextAllowedAt-this.deps.now());e.timer=setTimeout(()=>{e.timer=void 0,this.deliver(t,e)},n),e.timer.unref()}async deliver(t,e){if(!e.owed||e.streams>0){this.forget(t,e);return}const n=await this.deps.target(t),o=await this.deps.latest(t);if(n===null||o===null){e.owed=!1,this.forget(t,e);return}e.owed=!1,e.inFlight=!0;let a=0,r;try{for(;;){a++;try{await this.attempt(n,t,o),r=void 0;break}catch(i){r=i instanceof c?i:new c(i instanceof Error?i.message:String(i),!0);const l=this.deps.retryWaitsMs[a-1];if(!r.retryable||l===void 0)break;if(await new Promise(h=>{setTimeout(h,l).unref()}),e.streams>0){r=void 0;break}}}}finally{e.inFlight=!1}r?(e.nextAllowedAt=this.deps.now()+this.deps.failGapMs,p({level:"error",event:"run_notice.failed",fields:{run_id:t,grantee:n.grantee,attempts:a,latest_sequence:o,...r.status===void 0?{}:{status:r.status},error:r.message,...g(r.cause,r.where.url===void 0&&r.where.step==="run_notice"?{step:"run_notice",url:n.url}:r.where)}})):e.streams===0&&(e.nextAllowedAt=this.deps.now()+this.deps.ackGapMs),this.schedule(t,e),this.forget(t,e)}async attempt(t,e,n){for(let o=0;;o++){let a;try{a=await this.deps.token(t)}catch(i){throw new c(`token mint failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"notice_token"})}let r;try{r=await this.deps.fetch(t.url,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(x),headers:{Authorization:`Bearer ${a.access_token}`,"Content-Type":"application/json"},body:JSON.stringify({run_id:e,latest_sequence:n})})}catch(i){throw new c(`request failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"run_notice",url:t.url})}if(r.body?.cancel().catch(()=>{}),r.status>=200&&r.status<300)return;if(r.status===401){if(this.deps.discard(a.key),o===0)continue;throw new c("the app refused the notice token twice (401)",!0,401)}throw r.status>=500?new c(`the app answered ${r.status}`,!0,r.status):new c(`contract_fault: the app answered ${r.status}`,!1,r.status)}}}function M(){return{target:u(async s=>{const t=await(await d()).prepare("SELECT grant_id FROM nodes WHERE node_id = ? AND run_id = ?").get(s,s);return t?.grant_id?await _(t.grant_id):null},"target"),latest:u(async s=>{const t=await(await d()).prepare("SELECT event_cursor FROM nodes WHERE node_id = ? AND run_id = ?").get(s,s);return t?.event_cursor==null?null:t.event_cursor-1},"latest"),token:u(async s=>{const t=await A({audience:s.url,sub:s.sub,grantee:s.grantee,initiating_app:s.grantee,scope:[E]});return{access_token:t.access_token,key:t.key}},"token"),discard:y,fetch:u((s,t)=>fetch(s,t),"fetch"),now:Date.now,retryWaitsMs:k,ackGapMs:b,failGapMs:T}}u(M,"databaseDeps");let S;function v(){return S??=new N(M())}u(v,"runNotices");async function q(s=v()){f(e=>s.appended(e));const t=await m.subscribe(e=>{s.appended(e).catch(n=>p({level:"error",event:"runs.notice_hook_failed",error:n}))},{kinds:["run_event"]});return()=>{f(void 0),t()}}u(q,"startRunNotices");export{b as RUN_NOTICE_ACK_GAP_MS,x as RUN_NOTICE_ATTEMPT_TIMEOUT_MS,T as RUN_NOTICE_FAIL_GAP_MS,k as RUN_NOTICE_RETRY_WAITS_MS,E as RUN_NOTICE_SCOPE,N as RunNoticeOutbox,M as databaseDeps,v as runNotices,q as startRunNotices};

package/dist/daemon/runs/append-section.d.ts

@@ -3,4 +3,7 @@ import type { CanvasDb } from '../../core/canvas/db.js';
 export declare const TURN_INPUT_TYPE = "crtr-turn-input";
 export declare const SYSTEM_PROMPT_TYPE = "crtr-system-prompt";
 export declare const TURN_END_TYPE = "crtr-turn-end";
-export declare function appendConversationEvents(db: CanvasDb, runId: string, nodeId: string, entries: SessionEntry[], pendingInputIds: string[]): Promise<string[]>;
+/** The one run-status input that reads files (the armed structured-output request). A caller inside a write transaction
+ *  computes it first, outside the transaction, and passes it to `appendConversationEvents`. */
+export declare function rootRepromptPending(runId: string, nodeId: string, entries: SessionEntry[]): Promise<boolean>;
+export declare function appendConversationEvents(db: CanvasDb, runId: string, nodeId: string, entries: SessionEntry[], pendingInputIds: string[], repromptPending?: boolean): Promise<string[]>;

package/dist/daemon/runs/append-section.js

@@ -1,2 +1,2 @@
-var w=Object.defineProperty;var u=(t,e)=>w(t,"name",{value:e,configurable:!0});import{appendRunEvent as _}from"../../core/runs/events.js";import{openQuestionIds as R}from"../../core/runs/questions.js";import{getNode as T}from"../../core/canvas/canvas.js";import{payerOfGrant as x}from"../../core/grants/mirror.js";import{runSchemaRepromptPending as N}from"../../core/runtime/stop-guard.js";const h="crtr-turn-input",S="crtr-system-prompt",k="crtr-turn-end",f=u(()=>({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),"zero");function D(t){return typeof t=="string"?t:Array.isArray(t)?t.filter(e=>typeof e?.text=="string").map(e=>e.text).join(`
-`):""}u(D,"text");function O(t){if(t==null)return;const{code:e,message:r}=t;if(e!=="content_refused"&&e!=="model_error"||typeof r!="string")throw new Error("invalid crtr-turn-end error");return{code:e,message:r}}u(O,"turnEndError");function j(t){return{input_tokens:t.input??0,output_tokens:t.output??0,cache_read_tokens:t.cacheRead??0,cache_write_tokens:t.cacheWrite??0,cost_usd:t.cost?.total??0}}u(j,"usageParts");async function H(t,e,r,E,g){let d=g;for(const o of E){if(o.type==="custom"&&o.customType===h){const n=o.data?.message_ids;if(!Array.isArray(n)||!n.every(i=>typeof i=="string"))throw new Error("invalid crtr-turn-input message_ids");d=[...new Set([...d,...n])]}if(o.type==="custom"&&o.customType===S){const n=o.data?.text;if(typeof n!="string")throw new Error("invalid crtr-system-prompt text");await _(t,e,r,"node.system_prompt.set",{entry_id:o.id,text:n})}if(o.type==="custom"&&o.customType===k){const n=o.data,i=n?.stop_reason;if(typeof i!="string")throw new Error("invalid crtr-turn-end stop_reason");const a=O(n?.error),s=await t.prepare("SELECT body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'node.turn.started' ORDER BY sequence_number DESC LIMIT 1").get(e,r);if(!s)throw new Error("crtr-turn-end without node.turn.started");const{input:p}=JSON.parse(s.body_json),m=await t.prepare("SELECT status, pi_pid FROM node_rows WHERE run_id = ? AND node_id != ?").all(e,r),c=r===e?await T(e):null,l=m.some(y=>y.pi_pid!==null&&y.status==="active")||c!==null&&await N(c)?"running":(await R(e,t)).length?"waiting_on_user":"idle";await _(t,e,r,"node.turn.completed",{entry_id:o.id,message_ids:p.message_ids,run_status:l,stop_reason:i,...a?{error:a}:{}}),r===e&&await t.prepare("UPDATE nodes SET last_turn_error_json = ? WHERE node_id = ?").run(a?JSON.stringify(a):null,e)}if(o.type==="message"){const n=o.message;if(n.role==="user"){const i=await t.prepare("SELECT type, body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.turn.started', 'node.turn.completed') ORDER BY sequence_number DESC LIMIT 1").get(e,r),a=i?.type==="node.turn.started"?JSON.parse(i.body_json).input.message_ids:[];await _(t,e,r,"node.turn.started",{entry_id:o.id,input:{text:D(n.content),message_ids:[...new Set([...a,...d])]}}),d=[]}if(n.role==="assistant"){let i=!1;const a=u(async()=>{if(i)return;i=!0;const s=n.content.filter(p=>p.type==="text").map(p=>p.text).join("");await _(t,e,r,"node.output_text.done",{entry_id:o.id,text:s,stop_reason:n.stopReason})},"doneText");for(const s of n.content)s.type==="thinking"&&await _(t,e,r,"node.thinking.done",{entry_id:o.id,text:s.thinking}),s.type==="text"&&await a(),s.type==="toolCall"&&(await a(),await _(t,e,r,"node.tool_call.started",{entry_id:o.id,tool_call_id:s.id,name:s.name,arguments:s.arguments}));if(await a(),n.usage){const s=j(n.usage),p=await t.prepare("SELECT usage_json, grant_id FROM nodes WHERE node_id = ?").get(e),m=p.usage_json?JSON.parse(p.usage_json):f(),c=f();for(const l of Object.keys(c))c[l]=m[l]+s[l];await t.prepare("UPDATE nodes SET usage_json = ? WHERE node_id = ?").run(JSON.stringify(c),e),await _(t,e,r,"usage",{entry_id:o.id,served_model:n.model,payer:await x(p.grant_id),...s,usage:c})}await t.prepare("DELETE FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.thinking.delta', 'node.output_text.delta')").run(e,r)}n.role==="toolResult"&&await _(t,e,r,"node.tool_call.completed",{entry_id:o.id,tool_call_id:n.toolCallId,name:n.toolName,result:n.content,status:n.isError?"error":"ok"})}}return d}u(H,"appendConversationEvents");export{S as SYSTEM_PROMPT_TYPE,k as TURN_END_TYPE,h as TURN_INPUT_TYPE,H as appendConversationEvents};
+var T=Object.defineProperty;var _=(t,e)=>T(t,"name",{value:e,configurable:!0});import{appendRunEvent as c}from"../../core/runs/events.js";import{openQuestionIds as x}from"../../core/runs/questions.js";import{getNode as N}from"../../core/canvas/canvas.js";import{payerOfGrant as h}from"../../core/grants/mirror.js";import{runSchemaRepromptPending as S}from"../../core/runtime/stop-guard.js";const k="crtr-turn-input",D="crtr-system-prompt",g="crtr-turn-end",w=_(()=>({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),"zero");function O(t){return typeof t=="string"?t:Array.isArray(t)?t.filter(e=>typeof e?.text=="string").map(e=>e.text).join(`
+`):""}_(O,"text");function j(t){if(t==null)return;const{code:e,message:n}=t;if(e!=="content_refused"&&e!=="model_error"||typeof n!="string")throw new Error("invalid crtr-turn-end error");return{code:e,message:n}}_(j,"turnEndError");function A(t){return{input_tokens:t.input??0,output_tokens:t.output??0,cache_read_tokens:t.cacheRead??0,cache_write_tokens:t.cacheWrite??0,cost_usd:t.cost?.total??0}}_(A,"usageParts");async function v(t,e,n){if(e!==t||!n.some(f=>f.type==="custom"&&f.customType===g))return!1;const m=await N(t);return m!==null&&await S(m)}_(v,"rootRepromptPending");async function Y(t,e,n,m,f,R){let d=f;for(const r of m){if(r.type==="custom"&&r.customType===k){const o=r.data?.message_ids;if(!Array.isArray(o)||!o.every(i=>typeof i=="string"))throw new Error("invalid crtr-turn-input message_ids");d=[...new Set([...d,...o])]}if(r.type==="custom"&&r.customType===D){const o=r.data?.text;if(typeof o!="string")throw new Error("invalid crtr-system-prompt text");await c(t,e,n,"node.system_prompt.set",{entry_id:r.id,text:o})}if(r.type==="custom"&&r.customType===g){const o=r.data,i=o?.stop_reason;if(typeof i!="string")throw new Error("invalid crtr-turn-end stop_reason");const a=j(o?.error),s=await t.prepare("SELECT body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'node.turn.started' ORDER BY sequence_number DESC LIMIT 1").get(e,n);if(!s)throw new Error("crtr-turn-end without node.turn.started");const{input:p}=JSON.parse(s.body_json),y=await t.prepare("SELECT status, pi_pid FROM node_rows WHERE run_id = ? AND node_id != ?").all(e,n),u=R??await v(e,n,[r]),l=y.some(E=>E.pi_pid!==null&&E.status==="active")||u?"running":(await x(e,t)).length?"waiting_on_user":"idle";await c(t,e,n,"node.turn.completed",{entry_id:r.id,message_ids:p.message_ids,run_status:l,stop_reason:i,...a?{error:a}:{}}),n===e&&await t.prepare("UPDATE nodes SET last_turn_error_json = ? WHERE node_id = ?").run(a?JSON.stringify(a):null,e)}if(r.type==="message"){const o=r.message;if(o.role==="user"){const i=await t.prepare("SELECT type, body_json FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.turn.started', 'node.turn.completed') ORDER BY sequence_number DESC LIMIT 1").get(e,n),a=i?.type==="node.turn.started"?JSON.parse(i.body_json).input.message_ids:[];await c(t,e,n,"node.turn.started",{entry_id:r.id,input:{text:O(o.content),message_ids:[...new Set([...a,...d])]}}),d=[]}if(o.role==="assistant"){let i=!1;const a=_(async()=>{if(i)return;i=!0;const s=o.content.filter(p=>p.type==="text").map(p=>p.text).join("");await c(t,e,n,"node.output_text.done",{entry_id:r.id,text:s,stop_reason:o.stopReason})},"doneText");for(const s of o.content)s.type==="thinking"&&await c(t,e,n,"node.thinking.done",{entry_id:r.id,text:s.thinking}),s.type==="text"&&await a(),s.type==="toolCall"&&(await a(),await c(t,e,n,"node.tool_call.started",{entry_id:r.id,tool_call_id:s.id,name:s.name,arguments:s.arguments}));if(await a(),o.usage){const s=A(o.usage),p=await t.prepare("SELECT usage_json, grant_id FROM nodes WHERE node_id = ?").get(e),y=p.usage_json?JSON.parse(p.usage_json):w(),u=w();for(const l of Object.keys(u))u[l]=y[l]+s[l];await t.prepare("UPDATE nodes SET usage_json = ? WHERE node_id = ?").run(JSON.stringify(u),e),await c(t,e,n,"usage",{entry_id:r.id,served_model:o.model,payer:await h(p.grant_id),...s,usage:u})}await t.prepare("DELETE FROM run_events WHERE run_id = ? AND node_id = ? AND type IN ('node.thinking.delta', 'node.output_text.delta')").run(e,n)}o.role==="toolResult"&&await c(t,e,n,"node.tool_call.completed",{entry_id:r.id,tool_call_id:o.toolCallId,name:o.toolName,result:o.content,status:o.isError?"error":"ok"})}}return d}_(Y,"appendConversationEvents");export{D as SYSTEM_PROMPT_TYPE,g as TURN_END_TYPE,k as TURN_INPUT_TYPE,Y as appendConversationEvents,v as rootRepromptPending};

package/dist/daemon/runs/observer.d.ts

@@ -1,4 +1,3 @@
-export declare function openAssistantMessageStartSequence(nodeId: string): number | null;
 /** What the model is streaming now in a node's open assistant message. */
 export type OpenAssistantBlock = {
     kind: 'thinking' | 'text';
@@ -6,8 +5,10 @@ export type OpenAssistantBlock = {
     kind: 'tool_call';
     name: string;
 };
+/** The start sequence of the node's open assistant message, from the observer's columns; null when none is open or the node has no live broker. */
+export declare function openAssistantMessageStartSequence(nodeId: string): Promise<number | null>;
 /** The newest block of the node's open assistant message; null when no message is open or nothing has streamed yet. */
-export declare function openAssistantBlock(nodeId: string): OpenAssistantBlock | null;
+export declare function openAssistantBlock(nodeId: string): Promise<OpenAssistantBlock | null>;
 /** Test-only: stop every observer and wait out the polls in flight, so none touches a home the test is about to delete. */
 export declare function closeRunObserversForTests(): Promise<void>;
 export declare function ensureRunObserver(nodeId: string, runId: string): Promise<void>;

package/dist/daemon/runs/observer.js

@@ -1,5 +1,6 @@
-var y=Object.defineProperty;var r=(n,t)=>y(n,"name",{value:t,configurable:!0});import{randomUUID as k}from"node:crypto";import{openDb as l}from"../../core/canvas/db.js";import{getNode as g}from"../../core/canvas/canvas.js";import{BrokerClient as D,SocketTransport as T}from"../../core/broker-client/client.js";import{emitRunEvent as p,emitRunSettlement as w}from"../../core/runs/events.js";import{cancelOpenQuestions as I}from"../../core/runs/questions.js";import{emitEvent as c}from"../../core/events/emit.js";import{toRunOutcomeDTO as N}from"../api/map.js";import{readOutputResult as R}from"../../core/runtime/structured-output.js";const b=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),u=new Map,A=250;class m{static{r(this,"RunObserver")}nodeId;runId;client;timer;status;piPid=null;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;polling;frames=Promise.resolve();constructor(t,i,s,e){this.nodeId=t,this.runId=i,this.status=s,this.outcomeRevision=e,this.timer=setInterval(()=>{this.pollOrClose()},250),this.timer.unref(),u.set(t,this)}static async start(t,i){const s=await l(),e=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(t,t===i?"node.settled":"child.settled"),o=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(t),d=new m(t,i,o?JSON.parse(o.body_json).status:void 0,e?JSON.parse(e.body_json).outcome.revision:void 0);return await d.poll(),d}async pollOrClose(){try{await this.poll()}catch(t){this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:t})}}poll(){return this.polling===void 0&&(this.polling=this.pollOnce().finally(()=>{this.polling=void 0})),this.polling}async pollOnce(){const t=await g(this.nodeId);if(!t||t.run_id!==this.runId||!await E(this.runId)){this.close();return}this.piPid=t.pi_pid??null;const i=t.pi_pid===null?"dormant":t.status;this.status!==i&&await p(this.runId,this.nodeId,"node.status.changed",{status:i,reason:t.intent??null}),this.status=i;const s=N(t);if(s&&s.revision!==this.outcomeRevision&&Date.now()-Date.parse(s.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const e=await(await l()).prepare("SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const o of e)await v(o.node_id,this.runId),await u.get(o.node_id)?.poll()}if(this.outcomeRevision=s.revision,this.nodeId===this.runId){const e=s.kind==="result"?R(this.nodeId,t.grantee):void 0;await w(this.runId,this.nodeId,"node.settled",{outcome:s,usage:JSON.parse(t.usage_json??b),...e===void 0?{}:{result:e}})}else await w(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:s});if(t.grantee&&t.grantee!=="app:terminal"){const e=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};I(e,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(o=>c({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:o}))}}t.pi_pid===null?this.close():await this.connect()}async settlePoll(){await this.polling?.catch(()=>{}),await this.frames}get assistantMessageStartSequence(){return this.openAssistant&&this.client&&this.piPid!==null?this.assistantStartSequence:null}get openBlock(){return this.assistantMessageStartSequence===null?null:this.block}async connect(){if(this.client||this.closed)return;const t=new D(this.nodeId,{transport:new T(this.nodeId,{recordsViewerFaults:!1})});this.client=t,t.on("connect",()=>t.send({type:"hello",role:"observer",client_id:k(),attends:!1}));const i=r(async s=>{if(!(this.closed||this.client!==t)){if(s.type==="message_start"&&s.message.role==="assistant"&&(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",this.assistantStartSequence=(await(await l()).prepare("SELECT event_cursor FROM nodes WHERE node_id = ?").get(this.runId)).event_cursor),s.type==="message_update"&&s.message.role==="assistant"&&this.openAssistant){const e=s.message.content;if(!Array.isArray(e))return;const o=e.at(-1);o&&typeof o=="object"&&"type"in o&&(o.type==="toolCall"?this.block={kind:"tool_call",name:"name"in o&&typeof o.name=="string"?o.name:""}:(o.type==="text"||o.type==="thinking")&&(this.block={kind:o.type}));const d=e.filter(a=>a?.type==="thinking").map(a=>a.thinking).join(""),h=e.filter(a=>a?.type==="text").map(a=>a.text).join(""),S=d.startsWith(this.thinking)?d.slice(this.thinking.length):d,O=h.startsWith(this.output)?h.slice(this.output.length):h;this.thinking=d,this.output=h,this.pendingThinking+=S,this.pendingOutput+=O,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>{this.flushDeltas().catch(a=>c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:a}))},A),this.flushTimer.unref())}s.type==="message_end"&&s.message.role==="assistant"&&(await this.flushDeltas(),this.openAssistant=!1,this.block=null)}},"handleFrame");t.on("frame",s=>{this.frames=this.frames.then(()=>i(s)).catch(e=>{this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:e})})}),t.on("error",()=>{}),t.on("close",()=>{this.client===t&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null)}),t.connect()}async flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const t=this.pendingThinking,i=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!t&&!i)return;if(await(await l()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}t&&await p(this.runId,this.nodeId,"node.thinking.delta",{delta:t}),i&&await p(this.runId,this.nodeId,"node.output_text.delta",{delta:i})}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){this.closed=!0,this.discardDeltas(),clearInterval(this.timer),u.delete(this.nodeId);const t=this.client;this.client=void 0,t?.close()}}function J(n){return u.get(n)?.assistantMessageStartSequence??null}r(J,"openAssistantMessageStartSequence");function Y(n){return u.get(n)?.openBlock??null}r(Y,"openAssistantBlock");async function G(){const n=[...u.values()];for(const t of n)t.close();await Promise.all(n.map(t=>t.settlePoll())),await Promise.all([...f.values()].map(t=>t.catch(()=>{})))}r(G,"closeRunObserversForTests");async function E(n){return await(await l()).prepare("SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(n,n)!==void 0}r(E,"runRootExists");const f=new Map;async function v(n,t){const i=f.get(n);if(i)return i;if(u.has(n))return;const s=L(n,t).finally(()=>f.delete(n));return f.set(n,s),s}r(v,"ensureRunObserver");async function L(n,t){if(await E(t)){if(n!==t){const i=await g(n);!await(await l()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(t,n)&&i&&Date.now()-Date.parse(i.created)<720*60*6e4&&await p(t,n,"child.started",{child_node_id:n,kind:i.kind})}await m.start(n,t)}}r(L,"startRunObserver");async function Q(){const n=new Date(Date.now()-2592e6).toISOString(),t=await(await l()).prepare(`SELECT n.node_id, n.run_id, n.pi_pid, COALESCE(n.outcome_revision, 1) AS revision, e.body_json
+var k=Object.defineProperty;var l=(n,e)=>k(n,"name",{value:e,configurable:!0});import{randomUUID as v}from"node:crypto";import{openDb as d,withCanvasWrite as L}from"../../core/canvas/db.js";import{claimReconcilerLease as b,releaseReconcilerLease as O,renewReconcilerLease as C}from"../../core/canvas/reconciler-leases.js";import{getNode as R}from"../../core/canvas/canvas.js";import{BrokerClient as F,SocketTransport as y}from"../../core/broker-client/client.js";import{liveExecutionForNode as M}from"../../core/canvas/executions.js";import{runnerForExecution as x}from"../../core/runtime/runner/index.js";import{connectWssView as H}from"../../core/runtime/view-socket.js";import{ViewLinkTransport as W}from"./view-transport.js";import{emitRunEvent as w,emitRunSettlement as N,emitRunStatus as q}from"../../core/runs/events.js";import{cancelOpenQuestions as U}from"../../core/runs/questions.js";import{emitEvent as c}from"../../core/events/emit.js";import{toRunOutcomeDTO as j}from"../api/map.js";import{readOutputResult as J}from"../../core/runtime/output-result.js";const P=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),p=new Map,h="run-observer",g=1e4,B=250;async function f(n,e,t){return await L(async i=>await i.prepare("SELECT 1 AS held FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ? FOR SHARE").get(h,n,e,new Date().toISOString())===void 0?{held:!1}:{held:!0,value:await t()})}l(f,"whileHoldingLease");class S{static{l(this,"RunObserver")}nodeId;runId;owner;client;timer;status;piPid=null;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;lastRenewAt=Date.now();released=Promise.resolve();polling;frames=Promise.resolve();constructor(e,t,i,s,r){this.nodeId=e,this.runId=t,this.owner=i,this.status=s,this.outcomeRevision=r,this.timer=setInterval(()=>{this.pollOrClose()},250),this.timer.unref(),p.set(e,this)}static async start(e,t,i){const s=await d(),r=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(e,e===t?"node.settled":"child.settled"),o=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(e),a=new S(e,t,i,o?JSON.parse(o.body_json).status:void 0,r?JSON.parse(r.body_json).outcome.revision:void 0);return await a.writeStream(null,null),await a.poll(),a}async pollOrClose(){try{await this.poll()}catch(e){this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:e})}}poll(){return this.polling===void 0&&(this.polling=this.pollOnce().finally(()=>{this.polling=void 0})),this.polling}async holdLease(){return Date.now()-this.lastRenewAt<g/3?!0:await C(h,this.nodeId,this.owner,g)?(this.lastRenewAt=Date.now(),!0):!1}async writeStream(e,t,i=!1){return(await(await d()).prepare(`UPDATE nodes SET ${i?"":`stream_message_seq = ${e===null?"NULL":"(SELECT event_cursor FROM nodes WHERE node_id = ?)"},`} stream_block = ?
+      WHERE node_id = ? AND EXISTS (SELECT 1 FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?)`).run(...e===null||i?[]:[this.runId],t===null?null:JSON.stringify(t),this.nodeId,h,this.nodeId,this.owner,new Date().toISOString())).changes===1}async pollOnce(){if(this.closed)return;if(!await this.holdLease()){this.close();return}const e=await R(this.nodeId);if(!e||e.run_id!==this.runId||!await D(this.runId)){this.close();return}this.piPid=e.pi_pid??null;const t=e.pi_pid===null?"dormant":e.status;if(this.status!==t){if(!(await f(this.nodeId,this.owner,()=>q(this.runId,this.nodeId,t,e.intent??null))).held){this.close();return}this.status=t}const i=await j(e);if(i&&i.revision!==this.outcomeRevision&&Date.now()-Date.parse(i.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const o=await(await d()).prepare("SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const a of o)await T(a.node_id,this.runId),await p.get(a.node_id)?.poll()}this.outcomeRevision=i.revision;const s=this.nodeId===this.runId&&i.kind==="result"?await J(this.nodeId):void 0;if(!(await f(this.nodeId,this.owner,()=>this.nodeId===this.runId?N(this.runId,this.nodeId,"node.settled",{outcome:i,usage:JSON.parse(e.usage_json??P),...s===void 0?{}:{result:s}}):N(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:i}))).held){this.close();return}if(e.grantee&&e.grantee!=="app:terminal"){const o=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};U(o,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(a=>c({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:a}))}}e.pi_pid===null?this.close():await this.connect()}async settlePoll(){await this.polling?.catch(()=>{}),await this.frames,await this.released}async connect(){if(this.client||this.closed)return;const e=await V(this.nodeId);if(this.client||this.closed){e.destroy();return}const t=new F(this.nodeId,{transport:e});this.client=t,t.on("connect",()=>t.send({type:"hello",role:"observer",client_id:v(),attends:!1}));const i=l(async s=>{if(!(this.closed||this.client!==t)){if(s.type==="message_start"&&s.message.role==="assistant"){if(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",!await this.writeStream("run-cursor",null)){this.close();return}this.assistantStartSequence=Number((await(await d()).prepare("SELECT stream_message_seq FROM nodes WHERE node_id = ?").get(this.nodeId)).stream_message_seq)}if(s.type==="message_update"&&s.message.role==="assistant"&&this.openAssistant){const r=s.message.content;if(!Array.isArray(r))return;const o=r.at(-1);if(o&&typeof o=="object"&&"type"in o&&(o.type==="toolCall"?this.block={kind:"tool_call",name:"name"in o&&typeof o.name=="string"?o.name:""}:(o.type==="text"||o.type==="thinking")&&(this.block={kind:o.type})),!await this.writeStream(null,this.block,!0)){this.close();return}const a=r.filter(u=>u?.type==="thinking").map(u=>u.thinking).join(""),_=r.filter(u=>u?.type==="text").map(u=>u.text).join(""),A=a.startsWith(this.thinking)?a.slice(this.thinking.length):a,I=_.startsWith(this.output)?_.slice(this.output.length):_;this.thinking=a,this.output=_,this.pendingThinking+=A,this.pendingOutput+=I,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>{this.flushDeltas().catch(u=>c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:u}))},B),this.flushTimer.unref())}s.type==="message_end"&&s.message.role==="assistant"&&(await this.flushDeltas(),this.openAssistant=!1,this.block=null,await this.writeStream(null,null))}},"handleFrame");t.on("frame",s=>{this.frames=this.frames.then(()=>i(s)).catch(r=>{this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:r})})}),t.on("error",()=>{}),t.on("close",()=>{this.client===t&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null,this.released=this.released.then(async()=>{await this.writeStream(null,null)}).catch(()=>{}))}),t.connect()}async flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const e=this.pendingThinking,t=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!e&&!t)return;(await f(this.nodeId,this.owner,async()=>{if(await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}e&&await w(this.runId,this.nodeId,"node.thinking.delta",{delta:e}),t&&await w(this.runId,this.nodeId,"node.output_text.delta",{delta:t})})).held||this.close()}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){if(this.closed)return;this.closed=!0,this.discardDeltas(),clearInterval(this.timer),p.delete(this.nodeId),this.released=this.released.then(async()=>{await this.writeStream(null,null),await O(h,this.nodeId,this.owner)}).then(()=>{}).catch(t=>{c({level:"error",event:"daemon.runs.observer_release_failed",node_id:this.nodeId,error:t})});const e=this.client;this.client=void 0,e?.close()}}async function V(n){const e=await M(n);if(e===null)return new y(n,{recordsViewerFaults:!1});const t=await x(e.runner).viewerAddress(e);return t.kind==="unix"?new y(n,{recordsViewerFaults:!1}):new W(n,()=>H(t.url,t.headers))}l(V,"viewTransport");async function de(n){const e=await(await d()).prepare("SELECT stream_message_seq FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(n);return e?.stream_message_seq==null?null:Number(e.stream_message_seq)}l(de,"openAssistantMessageStartSequence");async function ue(n){const e=await(await d()).prepare("SELECT stream_message_seq, stream_block FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(n);return e?.stream_message_seq==null||e.stream_block===null?null:JSON.parse(e.stream_block)}l(ue,"openAssistantBlock");async function ce(){const n=[...p.values()];for(const e of n)e.close();await Promise.all(n.map(e=>e.settlePoll())),await Promise.all([...m.values()].map(e=>e.catch(()=>{})))}l(ce,"closeRunObserversForTests");async function D(n){return await(await d()).prepare("SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(n,n)!==void 0}l(D,"runRootExists");const m=new Map;async function T(n,e){const t=m.get(n);if(t)return t;if(p.has(n))return;const i=Y(n,e).finally(()=>m.delete(n));return m.set(n,i),i}l(T,"ensureRunObserver");async function Y(n,e){if(!await D(e))return;const t=v();if(await b(h,n,t,g)!==void 0)try{if(n!==e){const i=await R(n);if(!await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(e,n)&&i&&Date.now()-Date.parse(i.created)<720*60*6e4&&!(await f(n,t,()=>w(e,n,"child.started",{child_node_id:n,kind:i.kind}))).held)return}await S.start(n,e,t)}catch(i){throw await O(h,n,t).catch(()=>{}),i}}l(Y,"startRunObserver");async function he(){const n=new Date(Date.now()-2592e6).toISOString(),e=await(await d()).prepare(`SELECT n.node_id, n.run_id, n.pi_pid, COALESCE(n.outcome_revision, 1) AS revision, e.body_json
     FROM node_rows n
     LEFT JOIN run_events e ON n.pi_pid IS NULL AND e.run_id = n.run_id AND e.node_id = n.node_id
       AND e.type = CASE WHEN n.node_id = n.run_id THEN 'node.settled' ELSE 'child.settled' END
-    WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ? AND n.outcome_reason != 'parked'))`).all(n),i=new Map;for(const e of t){const o=i.get(e.node_id)??{node_id:e.node_id,run_id:e.run_id,reported:!1};e.body_json!==null&&JSON.parse(e.body_json).outcome?.revision===e.revision&&(o.reported=!0),i.set(e.node_id,o)}const s=[...i.values()].filter(e=>!e.reported);for(const e of s)try{await v(e.node_id,e.run_id),_.delete(e.node_id)}catch(o){M(e.node_id,o)}}r(Q,"pollRunObservers");const _=new Map;function M(n,t){const i=t instanceof Error?t.message:String(t);_.get(n)!==i&&(_.set(n,i),c({level:"error",event:"daemon.runs.observer_failed",node_id:n,error:t}))}r(M,"reportObserverFailure");export{G as closeRunObserversForTests,v as ensureRunObserver,Y as openAssistantBlock,J as openAssistantMessageStartSequence,Q as pollRunObservers};
+    WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ? AND n.outcome_reason != 'parked'))`).all(n),t=new Map;for(const s of e){const r=t.get(s.node_id)??{node_id:s.node_id,run_id:s.run_id,reported:!1};s.body_json!==null&&JSON.parse(s.body_json).outcome?.revision===s.revision&&(r.reported=!0),t.set(s.node_id,r)}const i=[...t.values()].filter(s=>!s.reported);for(const s of i)try{await T(s.node_id,s.run_id),E.delete(s.node_id)}catch(r){$(s.node_id,r)}}l(he,"pollRunObservers");const E=new Map;function $(n,e){const t=e instanceof Error?e.message:String(e);E.get(n)!==t&&(E.set(n,t),c({level:"error",event:"daemon.runs.observer_failed",node_id:n,error:e}))}l($,"reportObserverFailure");export{ce as closeRunObserversForTests,T as ensureRunObserver,ue as openAssistantBlock,de as openAssistantMessageStartSequence,he as pollRunObservers};

package/dist/daemon/runs/view-transport.d.ts

@@ -0,0 +1,19 @@
+import { EventEmitter } from 'node:events';
+import { type RedialOptions, type Transport } from '../../core/broker-client/transport.js';
+import { type ClientToBroker } from '../../core/runtime/broker-protocol.js';
+import type { BrokerViewLink } from '../../core/runtime/view-socket.js';
+export declare class ViewLinkTransport extends EventEmitter implements Transport {
+    private readonly nodeId;
+    private readonly open;
+    private link;
+    private decoder;
+    private closeEmitted;
+    constructor(nodeId: string, open: () => BrokerViewLink);
+    connect(): void;
+    redial(_options?: RedialOptions): Promise<void>;
+    send(frame: ClientToBroker): void;
+    destroy(): void;
+    private dial;
+    /** `error` is emitted only when a listener exists: a bare EventEmitter `error` throws. */
+    private emitError;
+}

package/dist/daemon/runs/view-transport.js

@@ -0,0 +1 @@
+var h=Object.defineProperty;var n=(s,e)=>h(s,"name",{value:e,configurable:!0});import{EventEmitter as a}from"node:events";import{BrokerUnavailableError as m}from"../../core/broker-client/transport.js";import{CLIENT_READ_CAPS as d,encodeFrame as f,FrameDecoder as c}from"../../core/runtime/broker-protocol.js";class w extends a{static{n(this,"ViewLinkTransport")}nodeId;open;link;decoder=new c(d);closeEmitted=!1;constructor(e,t){super(),this.nodeId=e,this.open=t}connect(){this.dial(()=>this.emit("connect"))}redial(e){return this.destroy(),this.decoder=new c(d),this.closeEmitted=!1,new Promise((t,i)=>{let r=!1;const o=this.dial(()=>{r||(r=!0,this.emit("connect"),t())});o.on("error",l=>{r||(r=!0,o.destroy(),i(l))})})}send(e){if(!(this.link===void 0||this.link.destroyed))try{this.link.write(f(e))}catch{}}destroy(){this.link!==void 0&&!this.link.destroyed&&this.link.destroy()}dial(e){const t=this.open();return this.link=t,t.on("connect",e),t.on("data",i=>{let r;try{r=this.decoder.push(i)}catch(o){this.emitError(new Error(`failed to decode a broker frame: ${String(o)}`)),this.destroy();return}for(const o of r)this.emit("frame",o)}),t.on("error",i=>{const r=i.code==="ECONNREFUSED"||i.code==="ENOTFOUND";this.emitError(r?new m(this.nodeId):new Error(`broker view error: ${i.message}`))}),t.on("close",()=>{this.closeEmitted||(this.closeEmitted=!0,this.emit("close"))}),t}emitError(e){this.listenerCount("error")>0&&this.emit("error",e)}}export{w as ViewLinkTransport};

package/dist/daemon/structured-requests/boot.d.ts

@@ -1,2 +1,3 @@
-/** Rebuild pending sidecars, settle pending requests on terminal nodes, and adopt pre-request-id sidecars. */
+/** At boot, settle the pending request of every node that already ended. The request row is the only
+ *  copy of a request, so nothing else needs rebuilding. */
 export declare function reconcileStructuredRequestSidecars(): Promise<void>;

package/dist/daemon/structured-requests/boot.js

@@ -1 +1 @@
-var o=Object.defineProperty;var n=(e,t)=>o(e,"name",{value:t,configurable:!0});import{listNodes as a}from"../../core/canvas/index.js";import{getPendingStructuredRequestForNode as u,registerLegacyStructuredRequest as s,settlePendingStructuredRequestForNode as c}from"../../core/canvas/structured-requests.js";import{readLegacyOutputRequest as l,readOutputRequest as f,writeStructuredRequestSidecar as d}from"../../core/runtime/structured-output.js";import{emitEvent as m}from"../../core/events/emit.js";function _(e){return e.status!=="done"&&e.status!=="dead"&&e.status!=="canceled"?null:e.terminal_reason==="deadline_exceeded"?{intended:"expired",failureReason:"node_deadline_exceeded"}:{intended:"failed",failureReason:e.terminal_reason??"finished"}}n(_,"terminalSettlement");async function y(){for(const e of await a())try{const t=_(e);t!==null&&await c(e.node_id,t);const i=await u(e.node_id);if(i!==null){d(i);continue}if(f(e.node_id).state==="valid"||t!==null)continue;const r=l(e.node_id);r!==null&&d(await s(e.node_id,r.mode,r.schema))}catch(t){m({level:"error",event:"structured_request.boot_reconcile_failed",node_id:e.node_id,error:t})}}n(y,"reconcileStructuredRequestSidecars");export{y as reconcileStructuredRequestSidecars};
+var n=Object.defineProperty;var r=(e,t)=>n(e,"name",{value:t,configurable:!0});import{listNodes as i}from"../../core/canvas/index.js";import{settlePendingStructuredRequestForNode as d}from"../../core/canvas/structured-requests.js";import{emitEvent as o}from"../../core/events/emit.js";function a(e){return e.status!=="done"&&e.status!=="dead"&&e.status!=="canceled"?null:e.terminal_reason==="deadline_exceeded"?{intended:"expired",failureReason:"node_deadline_exceeded"}:{intended:"failed",failureReason:e.terminal_reason??"finished"}}r(a,"terminalSettlement");async function f(){for(const e of await i())try{const t=a(e);t!==null&&await d(e.node_id,t)}catch(t){o({level:"error",event:"structured_request.boot_reconcile_failed",node_id:e.node_id,error:t})}}r(f,"reconcileStructuredRequestSidecars");export{f as reconcileStructuredRequestSidecars};

package/dist/daemon/structured-requests/effects.d.ts

@@ -1,4 +1,5 @@
 import { type StructuredRequestRow } from '../../core/canvas/structured-requests.js';
-import { type StagedStructuredProjection } from '../../core/runtime/structured-output.js';
-/** Replays only durable request-owned effects. A successful final never enters teardown. */
-export declare function applySettlementEffects(row: StructuredRequestRow, staged?: StagedStructuredProjection): Promise<void>;
+import type { RowClaim } from '../reconcilers/row-claim.js';
+/** Replays only durable request-owned effects. A successful final never enters teardown. A reconciler passes its row `claim`; each
+ *  effect then first confirms the claim is still held, so a holder whose claim was taken over stops before enacting. */
+export declare function applySettlementEffects(row: StructuredRequestRow, claim?: RowClaim): Promise<void>;

package/dist/daemon/structured-requests/effects.js

@@ -1 +1 @@
-var o=Object.defineProperty;var n=(e,d)=>o(e,"name",{value:d,configurable:!0});import{markStructuredRequestEffectsDone as l,ownsStructuredRequestDeclineProjection as _,ownsStructuredRequestResultProjection as f,registerStructuredRequestEffectRunner as m}from"../../core/canvas/structured-requests.js";import{getNode as a}from"../../core/canvas/index.js";import{emitEvent as p}from"../../core/events/emit.js";import{requestBrokerTeardown as q}from"../../core/runtime/host.js";import{transition as S}from"../../core/runtime/lifecycle.js";import{expireNodeDeadline as y}from"../reconcilers/node-deadline.js";import{installStagedStructuredProjection as c,readOutputRequest as g,removeOutputSchema as h,stageStructuredResult as x,writeOutputDeclined as D}from"../../core/runtime/structured-output.js";import{enqueueNodeMail as R}from"../../core/feed/messages.js";import{notifyStructuredRequest as v}from"./notifier.js";function b(e){if(e.node_id===null)return!1;const d=g(e.node_id);return d.state==="valid"&&d.request.request_id===e.request_id}n(b,"ownsCurrentSidecar");async function j(e,d){v(e.request_id);let s=!1,u=null;const r=n((t,i)=>{s=!0,p({level:"error",event:t,node_id:e.node_id??void 0,error:i,fields:{request_id:e.request_id}})},"report");if(e.node_id!==null&&e.state==="completed")try{const t=e.submission,i=(await a(e.node_id))?.grantee;await f(e)&&(u=d===void 0?c(x(e.node_id,e.request_id,t,e.result,i)):c(d))}catch(t){r("structured_request.projection_failed",t)}if(e.node_id!==null&&e.state==="declined"&&await _(e)&&e.declined!==null)try{u=D(e.node_id,{mode:e.submission,...e.declined,at:new Date(e.settled_at??Date.now()).toISOString()},(await a(e.node_id))?.grantee)}catch(t){r("structured_request.decline_projection_failed",t)}if(e.node_id!==null&&b(e))try{h(e.node_id,e.request_id)}catch(t){r("structured_request.sidecar_remove_failed",t)}if(e.node_id!==null&&!e.dedicated&&e.state==="expired")try{await R(e.node_id,{kind:"urgent",label:`Structured request ${e.request_id} expired`,bodyJson:{body:`Structured request ${e.request_id} expired. Stop trying to answer it; this request can no longer accept a result.`},urgency:"urgent",wake:1,idempotencyKey:`structured-request-expiry:${e.request_id}`})}catch(t){r("structured_request.expiry_notice_failed",t)}if(e.node_id!==null&&e.dedicated&&(e.state==="expired"||e.state==="canceled"||e.state==="failed"))try{if(e.state==="expired")await y(e.node_id,new Date(e.deadline_at??Date.now()).toISOString(),Date.now());else{const t=await a(e.node_id);t!==null&&(t.status==="active"||t.status==="idle")&&await S(e.node_id,"cancel",{reason:"stranded"});const i=await q(e.node_id);if(!i.confirmed)throw i.error??new Error("broker teardown was not confirmed")}}catch(t){r("structured_request.teardown_failed",t)}s||await l(e.request_id,u)}n(j,"applySettlementEffects"),m(e=>{j(e)});export{j as applySettlementEffects};
+var s=Object.defineProperty;var i=(e,r)=>s(e,"name",{value:r,configurable:!0});import{markStructuredRequestEffectsDone as u,registerStructuredRequestEffectRunner as o}from"../../core/canvas/structured-requests.js";import{getNode as c}from"../../core/canvas/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{requestBrokerTeardown as l}from"../../core/runtime/host.js";import{transition as p}from"../../core/runtime/lifecycle.js";import{expireNodeDeadline as _}from"../reconcilers/node-deadline.js";import{enqueueNodeMail as m}from"../../core/feed/messages.js";import{notifyStructuredRequest as q}from"./notifier.js";async function y(e,r){q(e.request_id);let n=!1;const a=i((t,d)=>{n=!0,f({level:"error",event:t,node_id:e.node_id??void 0,error:d,fields:{request_id:e.request_id}})},"report");if(e.node_id!==null&&!e.dedicated&&e.state==="expired"){await r?.assertOwned();try{await m(e.node_id,{kind:"urgent",label:`Structured request ${e.request_id} expired`,bodyJson:{body:`Structured request ${e.request_id} expired. Stop trying to answer it; this request can no longer accept a result.`},urgency:"urgent",wake:1,idempotencyKey:`structured-request-expiry:${e.request_id}`})}catch(t){a("structured_request.expiry_notice_failed",t)}}if(e.node_id!==null&&e.dedicated&&(e.state==="expired"||e.state==="canceled"||e.state==="failed")){await r?.assertOwned();try{if(e.state==="expired")await _(e.node_id,new Date(e.deadline_at??Date.now()).toISOString(),Date.now(),r);else{const t=await c(e.node_id);t!==null&&(t.status==="active"||t.status==="idle")&&await p(e.node_id,"cancel",{reason:"stranded"});const d=await l(e.node_id);if(!d.confirmed)throw d.error??new Error("broker teardown was not confirmed")}}catch(t){a("structured_request.teardown_failed",t)}}n||await u(e.request_id,null)}i(y,"applySettlementEffects"),o(e=>{y(e)});export{y as applySettlementEffects};

package/dist/daemon/structured-requests/notifier.d.ts

@@ -1,2 +1,4 @@
+/** Each waiter re-reads the request row when woken; a restarted LISTEN connection wakes all of them, since signals sent while it was down are lost. */
 export declare function subscribeStructuredRequest(requestId: string, listener: () => void): () => void;
+/** Wakes the request's waiters in this process now and in every other process through `crtr_signal`. */
 export declare function notifyStructuredRequest(requestId: string): void;

package/dist/daemon/structured-requests/notifier.js

@@ -1 +1 @@
-var r=Object.defineProperty;var s=(e,t)=>r(e,"name",{value:t,configurable:!0});const o=new Map;function u(e,t){const n=o.get(e)??new Set;return n.add(t),o.set(e,n),()=>{n.delete(t),n.size===0&&o.delete(e)}}s(u,"subscribeStructuredRequest");function i(e){for(const t of o.get(e)??[])t()}s(i,"notifyStructuredRequest");export{i as notifyStructuredRequest,u as subscribeStructuredRequest};
+var c=Object.defineProperty;var n=(e,t)=>c(e,"name",{value:t,configurable:!0});import{brokerSignalBus as d}from"../broker-signals/bus.js";import{openDb as l}from"../../core/canvas/db.js";import{emitEvent as p}from"../../core/events/emit.js";const r=new Map;let s;function u(e){for(const t of r.get(e)??[])t()}n(u,"wakeAll");function g(e,t){const o=r.get(e)??new Set;return o.add(t),r.set(e,o),s??=d.subscribe(u,{kinds:["structured_request"],onListenStart:n(()=>{for(const i of[...r.keys()])u(i)},"onListenStart")}),()=>{if(o.delete(t),!(o.size>0)&&(r.delete(e),r.size===0)){const i=s;s=void 0,i?.then(f=>f())}}}n(g,"subscribeStructuredRequest");function S(e){u(e),l().then(t=>t.prepare("SELECT pg_notify(?, ?)").get("crtr_signal",`structured_request:${e}`)).catch(t=>p({level:"error",event:"structured_request.notify_failed",error:t,fields:{request_id:e}}))}n(S,"notifyStructuredRequest");export{S as notifyStructuredRequest,g as subscribeStructuredRequest};

package/dist/daemon/structured-requests/sidecar.d.ts

@@ -1,3 +1,4 @@
-import { type StructuredOutputMode } from '../../core/runtime/structured-output.js';
-/** Admit legacy node/message schema requests before publishing their sidecar. */
-export declare function admitLegacyStructuredOutput(nodeId: string, submission: StructuredOutputMode, schema: Record<string, unknown>, requestMessageId?: string): Promise<string>;
+import { type StructuredRequestRow } from '../../core/canvas/structured-requests.js';
+import type { StructuredOutputMode } from '../../core/runtime/structured-output.js';
+/** Admit a node/message schema request: the pending `structured_requests` row is the whole request. */
+export declare function admitLegacyStructuredOutput(nodeId: string, submission: StructuredOutputMode, schema: Record<string, unknown>, requestMessageId?: string): Promise<StructuredRequestRow>;

package/dist/daemon/structured-requests/sidecar.js

@@ -1 +1 @@
-var i=Object.defineProperty;var e=(t,r)=>i(t,"name",{value:r,configurable:!0});import{registerLegacyStructuredRequest as a}from"../../core/canvas/structured-requests.js";import{writeStructuredRequestSidecar as o}from"../../core/runtime/structured-output.js";async function p(t,r,u,c){return o(await a(t,r,u,c))}e(p,"admitLegacyStructuredOutput");export{p as admitLegacyStructuredOutput};
+var c=Object.defineProperty;var e=(t,r)=>c(t,"name",{value:r,configurable:!0});import{registerLegacyStructuredRequest as i}from"../../core/canvas/structured-requests.js";async function d(t,r,u,a){return await i(t,r,u,a)}e(d,"admitLegacyStructuredOutput");export{d as admitLegacyStructuredOutput};