@north-light/cli 0.3.561 → 0.3.563

package/dist/daemon/reconcilers/managed-worktree-sweep.js

@@ -1 +1 @@
-var w=Object.defineProperty;var o=(t,e)=>w(t,"name",{value:e,configurable:!0});import{getNode as p,getRow as h}from"../../core/canvas/index.js";import{managedWorktreeSweepCandidates as y}from"../../core/canvas/managed-worktree-sweep-candidates.js";import{isSafeNodeId as _}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as E,recordedPidLiveness as k}from"../../core/canvas/pid.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as u}from"../../core/events/operation-id.js";import{reconcileManagedWorktree as S}from"../../core/worktree-sweep.js";const I=60*1e3,v=3,T=["done","dead","canceled"];function m(t){return T.includes(t.status)}o(m,"isTerminal");function g(t,e){return e?.state==="closed"||e?.state==="open"&&t.status==="done"&&t.final_report!==null}o(g,"hasSweepDisposition");async function A(t,e,n){return e.isClaimed(t.node_id)||t.frozen_at!==null||!m(t)||!g(t,(await p(t.node_id))?.managed_worktree)?!1:k(t.pi_pid,t.pi_pid_identity,n)==="dead"}o(A,"engineIsGone");function N(t,e){const n=t.node_id,r=t.pi_pid,a=t.pi_pid_identity;return{stillEligible:o(async()=>{if(e.isClaimed(n))return!1;const s=await h(n);return s===null||!m(s)||s.frozen_at!==null||!g(s,(await p(n))?.managed_worktree)?!1:s.pi_pid===r&&s.pi_pid_identity===a},"stillEligible")}}o(N,"eligibilityGuard");class L{static{o(this,"ManagedWorktreeSweepReconciler")}lastSweepAt=Number.NEGATIVE_INFINITY;inFlight=!1;cursor=0;run(e,n){if(this.inFlight||e-this.lastSweepAt<I||!n.lifecycle.acceptsDetachedWork())return;this.lastSweepAt=e,this.inFlight=!0;const r=this.sweep(e,n.fleet).finally(()=>{this.inFlight=!1});n.lifecycle.registerDetached(r)}async candidates(e){try{return await y(e)}catch(n){return u.fresh(()=>{c({level:"error",event:"worktree.sweep.scan_failed",error:n})}),null}}async sweep(e,n){const r=await this.candidates(e);if(r===null||r.length===0)return;this.cursor>=r.length&&(this.cursor=0);const a=this.cursor,s=Math.min(v,r.length),d=[];for(let l=0;l<s;l+=1)d.push(r[(a+l)%r.length]);this.cursor=(a+s)%r.length;const i=await E();if(i!==null)for(const l of d){const f=await h(l);f===null||!await A(f,n,i)||await this.reconcileOne(f,e,n)}}async reconcileOne(e,n,r){const a=e.node_id,s=_(a)?{node_id:a}:{},d=_(a)?{}:{affected_node_id:a};try{const i=await S(a,N(e,r),n);u.fresh(()=>{if(i.status==="complete"){c({level:"info",event:"worktree.sweep.reconciled",outcome:"succeeded",...s,fields:{...d,...i.stash===void 0?{}:{base_checkout_stash:i.stash}}});return}i.status!=="skipped"&&(i.status==="refused"&&i.unchanged===!0||c({level:i.status==="failed"?"error":"warn",event:i.status==="failed"?"worktree.sweep.failed":"worktree.sweep.refused",outcome:i.status==="failed"?"failed":"skipped",...s,fields:{...d,reason:i.reason,...i.detail===void 0?{}:{detail:i.detail}}}))})}catch(i){u.fresh(()=>{c({level:"error",event:"worktree.sweep.failed",...s,error:i,fields:d})})}}}export{L as ManagedWorktreeSweepReconciler,v as WORKTREE_SWEEP_BUDGET,I as WORKTREE_SWEEP_INTERVAL_MS};
+var E=Object.defineProperty;var o=(e,t)=>E(e,"name",{value:t,configurable:!0});import{getNode as _,getRow as m}from"../../core/canvas/index.js";import{managedWorktreeSweepCandidates as k}from"../../core/canvas/managed-worktree-sweep-candidates.js";import{isSafeNodeId as h}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as y,recordedPidLiveness as S}from"../../core/canvas/pid.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as f}from"../../core/events/operation-id.js";import{reconcileManagedWorktree as v}from"../../core/worktree-sweep.js";import{runThrottledLane as R}from"./throttled-lane.js";const T=60*1e3,W=300*1e3,I=3,b=["done","dead","canceled"];function w(e){return b.includes(e.status)}o(w,"isTerminal");function g(e,t){return t?.state==="closed"||t?.state==="open"&&e.status==="done"&&e.final_report!==null}o(g,"hasSweepDisposition");async function L(e,t,i){return await t.isClaimed(e.node_id)||e.frozen_at!==null||!w(e)||!g(e,(await _(e.node_id))?.managed_worktree)?!1:S(e.pi_pid,e.pi_pid_identity,i)==="dead"}o(L,"engineIsGone");function M(e,t){const i=e.node_id,s=e.pi_pid,r=e.pi_pid_identity;return{stillEligible:o(async()=>{if(await t.isClaimed(i))return!1;const a=await m(i);return a===null||!w(a)||a.frozen_at!==null||!g(a,(await _(i))?.managed_worktree)?!1:a.pi_pid===s&&a.pi_pid_identity===r},"stillEligible")}}o(M,"eligibilityGuard");class z{static{o(this,"ManagedWorktreeSweepReconciler")}run(t,i){if(!i.lifecycle.acceptsDetachedWork())return;const s=R("managed-worktree-sweep","",t,T,W,r=>this.sweep(t,i.fleet,r)).then(()=>{}).catch(r=>{f.fresh(()=>c({level:"error",event:"worktree.sweep.failed",error:r}))});i.lifecycle.registerDetached(s)}async candidates(t){try{return await k(t)}catch(i){return f.fresh(()=>{c({level:"error",event:"worktree.sweep.scan_failed",error:i})}),null}}async sweep(t,i,s){const r=await this.candidates(t);if(r===null||r.length===0)return;let a=Number(s.state??0);(!Number.isInteger(a)||a>=r.length)&&(a=0);const d=Math.min(I,r.length),n=[];for(let l=0;l<d;l+=1)n.push(r[(a+l)%r.length]);if(!await s.save(String((a+d)%r.length)))return;const p=await y();if(p!==null)for(const l of n){const u=await m(l);if(!(u===null||!await L(u,i,p))){if(!await s.holds())return;await this.reconcileOne(u,t,i)}}}async reconcileOne(t,i,s){const r=t.node_id,a=h(r)?{node_id:r}:{},d=h(r)?{}:{affected_node_id:r};try{const n=await v(r,M(t,s),i);f.fresh(()=>{if(n.status==="complete"){c({level:"info",event:"worktree.sweep.reconciled",outcome:"succeeded",...a,fields:{...d,...n.stash===void 0?{}:{base_checkout_stash:n.stash}}});return}n.status!=="skipped"&&(n.status==="refused"&&n.unchanged===!0||c({level:n.status==="failed"?"error":"warn",event:n.status==="failed"?"worktree.sweep.failed":"worktree.sweep.refused",outcome:n.status==="failed"?"failed":"skipped",...a,fields:{...d,reason:n.reason,...n.detail===void 0?{}:{detail:n.detail}}}))})}catch(n){f.fresh(()=>{c({level:"error",event:"worktree.sweep.failed",...a,error:n,fields:d})})}}}export{z as ManagedWorktreeSweepReconciler,I as WORKTREE_SWEEP_BUDGET,T as WORKTREE_SWEEP_INTERVAL_MS};

package/dist/daemon/reconcilers/node-deadline.d.ts

@@ -1,7 +1,8 @@
+import { type RowClaim } from './row-claim.js';
 /** Enforce absolute spawn deadlines independently of the tick's live-row scan. */
 export declare class NodeDeadlineReconciler {
     run(now: number): Promise<void>;
 }
 /** The one deadline terminalization path. Structured-request expiry reuses it
  * even when a profile is paused, because its dedicated broker must not resume. */
-export declare function expireNodeDeadline(nodeId: string, deadlineAt: string, now: number): Promise<void>;
+export declare function expireNodeDeadline(nodeId: string, deadlineAt: string, now: number, claim?: RowClaim): Promise<void>;

package/dist/daemon/reconcilers/node-deadline.js

@@ -1 +1 @@
-var m=Object.defineProperty;var s=(r,a)=>m(r,"name",{value:a,configurable:!0});import{dueNodeDeadlines as _,getNode as u}from"../../core/canvas/index.js";import{emitEvent as i}from"../../core/events/emit.js";import{isProfilePaused as p}from"../../core/profiles/manifest.js";import{operationIdContext as w}from"../../core/events/operation-id.js";import{errorClassFromFault as h}from"../../core/fault-classifier.js";import{publishNotice as v}from"../../core/runtime/close.js";import{readFault as x}from"../../core/canvas/recovery.js";import{requestBrokerTeardown as f}from"../../core/runtime/host.js";import{transition as y}from"../../core/runtime/lifecycle.js";class z{static{s(this,"NodeDeadlineReconciler")}async run(a){let o;try{o=await _(new Date(a).toISOString())}catch(e){i({level:"error",event:"node_deadline.scan_failed",error:e});return}for(const e of o)p(e.profile_id)||e.deadline_at===null||e.deadline_at===void 0||w.fresh(()=>{D(e.node_id,e.deadline_at,a).catch(d=>{i({level:"error",event:"node_deadline.expire_failed",node_id:e.node_id,error:d,fields:{deadline_at:e.deadline_at}})})})}}async function D(r,a,o){const e=await u(r);if(e===null||e.status==="done")return;if(e.status==="canceled"||e.status==="dead"){const t=await f(r);if(!t.confirmed)throw t.error??new Error("broker teardown was not confirmed");return}const d=await x(r),n=d?.retry.disposition==="fatal"?d:null;let l;try{l=await y(r,"cancel",{reason:"deadline_exceeded",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",deadline:{deadline_at:a,elapsed_ms:o-Date.parse(a)},...n===null?{}:{message:n.message,fault_kind:n.kind,error_class:h(n).class}}}})}catch(t){throw i({level:"error",event:"node_deadline.terminalize_failed",node_id:r,error:t,fields:{deadline_at:a}}),t}if(l.status!=="canceled"||l.outcome_reason!=="deadline_exceeded")return;const c=await f(r);if(!c.confirmed)throw c.error??new Error("broker teardown was not confirmed");try{await v(r,"notice.deadline","child-deadline-exceeded",{reason:"child-deadline-exceeded",child:r})}catch(t){i({level:"error",event:"node_deadline.notify_failed",node_id:r,error:t,fields:{deadline_at:a}})}try{i({level:"error",event:"node_deadline.expired",node_id:r,fields:{deadline_at:a,elapsed_ms:o-Date.parse(a)}})}catch{}}s(D,"expireNodeDeadline");export{z as NodeDeadlineReconciler,D as expireNodeDeadline};
+var m=Object.defineProperty;var s=(r,t)=>m(r,"name",{value:t,configurable:!0});import{dueNodeDeadlines as w,getNode as u}from"../../core/canvas/index.js";import{emitEvent as d}from"../../core/events/emit.js";import{isProfilePaused as p}from"../../core/profiles/manifest.js";import{operationIdContext as h}from"../../core/events/operation-id.js";import{errorClassFromFault as v}from"../../core/fault-classifier.js";import{publishNotice as x}from"../../core/runtime/close.js";import{readFault as y}from"../../core/canvas/recovery.js";import{requestBrokerTeardown as _}from"../../core/runtime/host.js";import{transition as D}from"../../core/runtime/lifecycle.js";import{withRowClaim as N}from"./row-claim.js";const O=6e4;class z{static{s(this,"NodeDeadlineReconciler")}async run(t){let i;try{i=await w(new Date(t).toISOString())}catch(e){d({level:"error",event:"node_deadline.scan_failed",error:e});return}for(const e of i)p(e.profile_id)||e.deadline_at===null||e.deadline_at===void 0||h.fresh(()=>{N("nodes",e.node_id,O,a=>k(e.node_id,e.deadline_at,t,a)).catch(a=>{d({level:"error",event:"node_deadline.expire_failed",node_id:e.node_id,error:a,fields:{deadline_at:e.deadline_at}})})})}}async function k(r,t,i,e){await e?.assertOwned();const a=await u(r);if(a===null||a.status==="done")return;if(a.status==="canceled"||a.status==="dead"){await e?.assertOwned();const o=await _(r);if(!o.confirmed)throw o.error??new Error("broker teardown was not confirmed");return}const c=await y(r),n=c?.retry.disposition==="fatal"?c:null;let l;await e?.assertOwned();try{l=await D(r,"cancel",{reason:"deadline_exceeded",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",deadline:{deadline_at:t,elapsed_ms:i-Date.parse(t)},...n===null?{}:{message:n.message,fault_kind:n.kind,error_class:v(n).class}}}})}catch(o){throw d({level:"error",event:"node_deadline.terminalize_failed",node_id:r,error:o,fields:{deadline_at:t}}),o}if(l.status!=="canceled"||l.outcome_reason!=="deadline_exceeded")return;await e?.assertOwned();const f=await _(r);if(!f.confirmed)throw f.error??new Error("broker teardown was not confirmed");await e?.assertOwned();try{await x(r,"notice.deadline","child-deadline-exceeded",{reason:"child-deadline-exceeded",child:r})}catch(o){d({level:"error",event:"node_deadline.notify_failed",node_id:r,error:o,fields:{deadline_at:t}})}try{d({level:"error",event:"node_deadline.expired",node_id:r,fields:{deadline_at:t,elapsed_ms:i-Date.parse(t)}})}catch{}}s(k,"expireNodeDeadline");export{z as NodeDeadlineReconciler,k as expireNodeDeadline};

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts

@@ -37,10 +37,9 @@ export declare function thawTier(row: NodeRow, pendingWake: boolean): ThawTier;
  *  splits the saturation counts). A candidate with no `frozen_at` has not waited at all and sorts last. */
 export declare function compareThaw(a: ThawEntry, b: ThawEntry): number;
 export declare class FreezeLaneReconciler {
-    private readonly relaunchState;
     /** End a stuck-delivery episode the tick decided is over — the node's profile
      * paused, or it has no wake-worthy mail left. */
-    endEpisode(nodeId: string): void;
+    endEpisode(nodeId: string): Promise<void>;
     run(now: number, ctx: FreezeLaneContext): Promise<FrozenCounts>;
     private enact;
     /** A parked resident owed wake mail: the message-delivery reopen (resident, latch check, `revive`

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js

@@ -1 +1 @@
-var E=Object.defineProperty;var d=(o,t)=>E(o,"name",{value:t,configurable:!0});import{clearFrozen as f,getNode as h,getRow as R}from"../../../core/canvas/index.js";import{fullName as S}from"../../../core/canvas/labels.js";import{isSafeNodeId as s}from"../../../core/canvas/paths.js";import{emitEvent as c}from"../../../core/events/emit.js";import{operationIdContext as g}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as z}from"../../../core/canvas/deliveries.js";import{publishNotice as A}from"../../../core/runtime/close.js";import{transition as l}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as w}from"../../../core/runtime/start-check.js";import{commitReopenResident as k}from"../../../core/runtime/reopen.js";import{reviveNode as p}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{mapAsync as M}from"../../../core/async-array.js";const m=6,C=10*6e4,N=2e4;function D(o,t){if(t)return 2;const a=o.launched_at;if(a==null)return 1;const e=o.last_settled_at;return e==null?2:Date.parse(e)>Date.parse(a)?1:2}d(D,"thawTier");function T(o,t){const a=o.row.frozen_at,e=t.row.frozen_at;return a==null?e==null?0:1:e==null?-1:Date.parse(a)-Date.parse(e)}d(T,"compareThaw");class G{static{d(this,"FreezeLaneReconciler")}relaunchState=new Map;endEpisode(t){this.relaunchState.delete(t)}async run(t,a){const e=await M(a.candidates,async n=>({...n,tier:D(n.row,await v(n.row.node_id))}));e.sort(T);const r={stalled:0,waitingOnMail:0};for(const n of e)await g.fresh(()=>this.enact(t,n,a.fleet))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(t,a,e){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(t,a.row):(await e.applyDeadRowPolicy(r),e.has(r)?"launched":(await R(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return c({level:"error",event:"node.lifecycle.enact_failed",...s(r)?{node_id:r}:{},error:n,fields:{...s(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(t){await k(t.node_id);try{return(await p(t.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(w(a))return await this.settleRefusedStart(t.node_id,a,"reopen");throw a}}async settleRefusedStart(t,a,e){return await l(t,"release"),await f(t),c({level:"warn",event:"node.start.refused",node_id:t,error:a,fields:{enact:e}}),"skipped"}async enactThaw(t){const a=(await L(t.node_id))?.interrupted===!0;if(!a&&t.lifecycle==="resident"&&(await h(t.node_id))?.pi_session_id!=null&&!await v(t.node_id))return await l(t.node_id,"release"),await f(t.node_id),"skipped";try{return(await p(t.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(e){if(w(e))return await this.settleRefusedStart(t.node_id,e,"thaw");const r=await h(t.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await l(t.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}),await f(t.node_id)),e}}async enactIdleRelease(t,a){const e=a.node_id,r=await z(e),n=this.relaunchState.get(e),u=n===void 0||n.acknowledgedCount!==r,i=u?0:n.attempts;if(i>=m)return this.relaunchState.delete(e),c({level:"warn",event:"broker.idle_release.exhausted",...s(e)?{node_id:e}:{},fields:{...s(e)?{}:{affected_node_id:e},attempts:i,max_attempts:m}}),await l(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:i}}}),await A(e,"notice.crash",`Child crashed \u2014 ${S(a)} (${e}) failed to relaunch after ${i} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,{reason:"child-crashed",child:e}),"skipped";const _=u?void 0:n?.lastAttempt,y=Math.min(N*2**i,C);return _!==void 0&&t-_<y?"skipped":(await p(e,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":(this.relaunchState.set(e,{acknowledgedCount:r,attempts:i+1,lastAttempt:t}),c({level:"info",event:"broker.revive.idle_release",...s(e)?{node_id:e}:{},attempt:i+1,fields:{...s(e)?{}:{affected_node_id:e},attempts:i+1,max_attempts:m}}),"launched")}}export{G as FreezeLaneReconciler,T as compareThaw,D as thawTier};
+var g=Object.defineProperty;var s=(i,e)=>g(i,"name",{value:e,configurable:!0});import{clearFrozen as p,getNode as h,getRow as y}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as k}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as S}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as D}from"../../../core/canvas/deliveries.js";import{publishNotice as N}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as E}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as R}from"./remote-wake.js";import{commitReopenResident as z}from"../../../core/runtime/reopen.js";import{reviveNode as m}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as T}from"../../../core/canvas/recovery.js";import{mapAsync as L}from"../../../core/async-array.js";const f=6,C=10*6e4,b=2e4;async function I(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(I,"readEpisode");async function M(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(M,"claimAttempt");async function O(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(O,"writeEpisode");function P(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s(P,"thawTier");function U(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(U,"compareThaw");class te{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await L(a.candidates,async n=>({...n,tier:P(n.row,await v(n.row.node_id))}));t.sort(U);const r={stalled:0,waitingOnMail:0};for(const n of t)await S.fresh(()=>this.enact(e,n,a.fleet))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await y(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await z(e.node_id);try{return(await m(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(E(a)||a instanceof R)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return await u(e,"release"),await p(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await T(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await h(e.node_id))?.pi_session_id!=null&&!await v(e.node_id))return await u(e.node_id,"release"),await p(e.node_id),"skipped";try{return(await m(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(E(t)||t instanceof R)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await h(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await p(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await D(t),n=await I(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=f)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:f}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await N(t,"notice.crash",`Child crashed \u2014 ${k(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const w=l?void 0:n.lastAttempt,A=Math.min(b*2**o,C);return w!==void 0&&e-w<A||!await M(t,n.attempts,o+1,r,e)?"skipped":(await m(t,{resume:!0,capacity:"freeze"})).outcome==="frozen"?(await O(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:f}}),"launched")}}export{te as FreezeLaneReconciler,U as compareThaw,P as thawTier};

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.d.ts

@@ -0,0 +1,31 @@
+import { type Execution } from '../../../core/canvas/executions.js';
+import type { NodeMeta } from '../../../core/canvas/index.js';
+import { type EnvironmentImage } from '../recipe-image.js';
+/** A `launching` execution running past this since `started_at` is stopped: the 30-minute recipe setup limit plus boot. */
+export declare const LAUNCH_DEADLINE_MS: number;
+/** A `launching` execution with no process is given this long before it is called lost, so the starter's own call is not raced. */
+export declare const LAUNCH_LOST_GRACE_MS = 120000;
+/** The release the control plane runs, which is the release a wake names. */
+export declare const acceptedRelease: () => string;
+/** The Blaxel environment the node's profile points at, or null when it runs on a machine. Reads rows only, so it is safe before any environment is registered. */
+export declare function blaxelEnvironmentOf(profileId: string | null): Promise<string | null>;
+/** Step 2: the `funding bill` check. A refusal writes `nodes.spend_refusal` and one refused `sandbox` row per node, limit and month,
+ *  then throws `402 cap_exhausted`; no execution row is written and no runner is called. A pass clears `spend_refusal`. */
+export declare function checkBilledWake(meta: NodeMeta): Promise<void>;
+/** A recipe environment with no built image: the wake is not started, the mail stays pending, and the image-built tick wakes it. */
+export declare class EnvironmentImagePending extends Error {
+    readonly environmentId: string;
+    readonly imageError: string | null;
+    constructor(environmentId: string, imageError: string | null);
+}
+/** D95: the image a new sandbox is created from. `pending` means the environment has a recipe and no image has built. */
+export declare function imageForWake(environmentId: string): Promise<EnvironmentImage>;
+/** Step 3: a `running` Blaxel execution is a resident broker, possibly frozen. On the accepted release it is woken and nothing else
+ *  is written. On another release it is asked to stop for the release first, so the thawed broker exits and the next claim starts
+ *  a new attempt on the accepted one. `lost` means no broker answered: the execution is ended and the caller starts a new attempt. */
+export declare function wakeResident(execution: Execution): Promise<'woken' | 'lost'>;
+/** A `running` remote execution is live while its runner reports the broker process running, frozen or not. When the runner
+ *  reports it gone (or ended non-zero), that exact execution is ended; a runner that cannot tell leaves it as it is. */
+export declare function endRemoteExecutionIfGone(execution: Execution): Promise<boolean>;
+/** The recovery table for a `launching` Blaxel execution found by a claim after its starter is gone. */
+export declare function recoverLaunching(nodeId: string, now: number): Promise<void>;

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.js

@@ -0,0 +1 @@
+var p=Object.defineProperty;var a=(e,t)=>p(e,"name",{value:t,configurable:!0});import{liveExecutionForNode as m,endExecution as d,recordExecutionAwake as w,endLaunchingExecution as s,requestExecutionStop as h}from"../../../core/canvas/executions.js";import{openDb as c}from"../../../core/canvas/db.js";import{emitEvent as u}from"../../../core/events/emit.js";import{findRefusal as _,refusalError as x,refusedMark as E,writeCharge as g}from"../../../core/ledger/index.js";import{transition as f}from"../../../core/runtime/lifecycle.js";import{runnerForExecution as l}from"../../../core/runtime/runner/index.js";import{startCandidateOf as b}from"../../../core/runtime/start-check.js";import{settleSandboxBeforeWake as v}from"../ledger-sandbox.js";import{crtrVersion as k}from"../../../shared/crtr-version.js";import{resolveEnvironmentImage as I}from"../recipe-image.js";const S=45*6e4,y=12e4,N=a(()=>k(),"acceptedRelease");async function j(e){return e===null?null:(await(await c()).prepare("SELECT e.object AS object FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ? AND e.kind = 'blaxel'").get(e))?.object??null}a(j,"blaxelEnvironmentOf");async function F(e){const t=await b(e);if(t===null)return;const n={scope:"call",payer:t.payer,funding:"bill",sub:t.sub,grantee:t.grantee,provider:"blaxel"},r=await _(n),o=await c();if(r===null){await o.prepare("UPDATE nodes SET spend_refusal = NULL WHERE node_id = ? AND spend_refusal IS NOT NULL").run(e.node_id);return}throw await o.prepare("UPDATE nodes SET spend_refusal = ? WHERE node_id = ?").run(JSON.stringify({limit:r.limit.id,usd:r.usd,reset_at:r.reset_at,at:new Date().toISOString()}),e.node_id),await g({id:`refused:sandbox:${e.node_id}:${r.limit.id}:${r.month}`,kind:"sandbox",what:"wake",usd:null,payer:n.payer,funding:"bill",sub:n.sub,grantee:n.grantee,node:e.node_id,run:e.run_id??null,provider:"blaxel",refused:E(r)}),x(r)}a(F,"checkBilledWake");class M extends Error{static{a(this,"EnvironmentImagePending")}environmentId;imageError;constructor(t,n){super(`environment ${t} has no built image yet${n===null?"":` (${n})`}; the wake waits for the build.`),this.environmentId=t,this.imageError=n,this.name="EnvironmentImagePending"}}async function P(e){return await I(e)}a(P,"imageForWake");async function B(e){return e.release!==N()&&await h(e.nodeId,{afterTurn:!1,reason:"release"}),await l(e.runner).wake(e)==="no_broker"?(await d(e.executionId,{reason:"lost"}),"lost"):(await v(e.executionId,Date.now()),await w(e.executionId),"woken")}a(B,"wakeResident");async function G(e){const t=await l(e.runner).status(e);if(t.state!=="gone"&&t.state!=="failed")return!1;const n=await d(e.executionId,{reason:"lost"});return n&&u({level:"warn",event:"execution.lost",node_id:e.nodeId}),n}a(G,"endRemoteExecutionIfGone");async function J(e,t){const n=await m(e);if(n===null||n.state!=="launching"||n.runner==="process")return;const r=l(n.runner),o=t-Date.parse(n.startedAt),i=await r.status(n);if(i.state!=="unknown"){if(i.state==="running"){if(o<=S||!await s(n.executionId,{reason:"launch_timeout"}))return;await r.stop(n,{reason:"launch_timeout"}),await f(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:"the sandbox launch did not report within the launch deadline"}}}),u({level:"warn",event:"execution.launch_timeout",node_id:e});return}if(i.state==="failed"){if(!await s(n.executionId,{reason:"launch_failed"}))return;await f(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:`the sandbox start script failed: ${i.reason}`}}});return}if(o>y){if(!await s(n.executionId,{reason:"launch_lost"}))return;u({level:"warn",event:"execution.launch_lost",node_id:e})}}}a(J,"recoverLaunching");export{M as EnvironmentImagePending,S as LAUNCH_DEADLINE_MS,y as LAUNCH_LOST_GRACE_MS,N as acceptedRelease,j as blaxelEnvironmentOf,F as checkBilledWake,G as endRemoteExecutionIfGone,P as imageForWake,J as recoverLaunching,B as wakeResident};

package/dist/daemon/reconcilers/node-lifecycle/requested-stops.d.ts

@@ -0,0 +1,5 @@
+/** A stop request this old was queued by a process that is no longer finishing it: the requesting process runs its own
+ *  teardown within seconds (after its response settles). */
+export declare const ORPHANED_STOP_AGE_MS = 30000;
+/** Enact every stop whose requesting process is gone, from the execution row's `stop_*` columns. */
+export declare function enactRequestedStops(now: number): Promise<void>;

package/dist/daemon/reconcilers/node-lifecycle/requested-stops.js

@@ -0,0 +1,2 @@
+var d=Object.defineProperty;var t=(e,r)=>d(e,"name",{value:r,configurable:!0});import{endLiveExecution as s,getNode as p}from"../../../core/canvas/index.js";import{executionsWithStopRequestedBefore as i}from"../../../core/canvas/executions.js";import{openDb as f}from"../../../core/canvas/db.js";import{emitEvent as o}from"../../../core/events/emit.js";import{runnerForExecution as u}from"../../../core/runtime/runner/index.js";const l=3e4;async function m(e,r){return r==null?!1:(await(await f()).prepare(`SELECT type FROM run_events WHERE node_id = ? AND type IN ('node.turn.started', 'node.turn.completed')
+    ORDER BY sequence_number DESC LIMIT 1`).get(e))?.type==="node.turn.started"}t(m,"turnIsOpen");async function c(e){const r=await p(e.nodeId);if(r===null||e.stopAfterTurn&&await m(e.nodeId,r.run_id))return;const a=await u(e.runner).stop(e,{reason:e.stopReason??"stopped"});if(!a.confirmed){o({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:a.error});return}await s(e.nodeId,{reason:e.stopReason??"stopped"}),o({level:"info",event:"broker.stop.enacted_from_row",node_id:e.nodeId,fields:{reason:e.stopReason,after_turn:e.stopAfterTurn}})}t(c,"enact");async function S(e){for(const r of await i(new Date(e-l).toISOString()))try{await c(r)}catch(n){o({level:"error",event:"broker.stop.enact_failed",node_id:r.nodeId,error:n})}}t(S,"enactRequestedStops");export{l as ORPHANED_STOP_AGE_MS,S as enactRequestedStops};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var y=Object.defineProperty;var f=(r,t)=>y(r,"name",{value:t,configurable:!0});import{clearFrozen as p,endLiveExecution as o,getNode as P}from"../../../core/canvas/index.js";import{isSafeNodeId as k}from"../../../core/canvas/paths.js";import{recordedPidLiveness as u}from"../../../core/canvas/pid.js";import{emitEvent as c}from"../../../core/events/emit.js";import{operationIdContext as v}from"../../../core/events/operation-id.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{isProfilePaused as l}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as m}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as h}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as R,isParkedResident as z}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as I}from"../../fleet.js";import{applyParkEvent as E,hasPendingWake as M,parkEvent as g}from"../live-obligation.js";import{FreezeLaneReconciler as N}from"./freeze-lane.js";import{isInBackoff as _}from"./respawn-policy.js";import{handleTerminatingRow as b}from"./terminating.js";import{wantsExecution as C}from"./wants-execution.js";function D(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}f(D,"isTerminal");const F=12e4;function T(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<F}f(T,"isLaunchInFlight");class ${static{f(this,"NodeLifecycleReconciler")}freezeLane=new N;async run(t,e){const a=[];for(const n of e.rows)await v.fresh(async()=>{try{await this.classifyRow(t,n,e,a)}catch(i){const s=n.node_id;c({level:"error",event:"node.lifecycle.classify_failed",...k(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(D(e)){if(b(e,a.fleet.has(i)))return;if(z(e)&&!a.fleet.has(i)&&!l(e.profile_id)&&!m(i)&&!_(e,t)&&await h(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&u(e.pi_pid,e.pi_pid_identity)==="dead"&&await o(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null?await p(i):n.push({row:e,enact:"thaw"}));return}if(a.fleet.has(i)){l(e.profile_id)?a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}if(l(e.profile_id)){this.freezeLane.endEpisode(i);return}if(m(i))return;const s=e.pi_pid!=null&&u(e.pi_pid,e.pi_pid_identity)==="dead";if(s&&!a.fleet.hasProfilePauseInterruption(i)&&await o(i,{reason:"observed-dead"}),!await C(e)){await this.thawMark(e),await this.settleParked(e);return}if(_(e,t))return;if(e.status==="idle"&&R(e)&&await h(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(a.fleet.isClaimed(i)||T(e,t))return;c({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!s)return;const d=I(await P(i),(await L(i))?.interrupted===!0,await M(i));if(d==="respawn-fresh"||d==="respawn-cycle"||d==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await p(t.node_id)}async settleParked(t){if(t.status!=="idle")return;this.freezeLane.endEpisode(t.node_id);const e=await g(t.node_id,t.lifecycle);e!=="release"&&await E(t.node_id,e)}}export{F as LAUNCH_GRACE_MS,$ as NodeLifecycleReconciler,T as isLaunchInFlight};
+var k=Object.defineProperty;var l=(r,t)=>k(r,"name",{value:t,configurable:!0});import{clearFrozen as m,endLiveExecution as h,getNode as R}from"../../../core/canvas/index.js";import{isSafeNodeId as b}from"../../../core/canvas/paths.js";import{recordedPidLiveness as _}from"../../../core/canvas/pid.js";import{hasObservedExit as E,hasAbsorbedProfilePauseStop as L}from"../../../core/canvas/executions.js";import{emitEvent as y}from"../../../core/events/emit.js";import{operationIdContext as z}from"../../../core/events/operation-id.js";import{readInterruptedTurn as I}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as g,isParkedResident as w}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as M}from"../../fleet.js";import{applyParkEvent as N,hasPendingWake as F,parkEvent as x}from"../live-obligation.js";import{FreezeLaneReconciler as A}from"./freeze-lane.js";import{isInBackoff as P}from"./respawn-policy.js";import{handleTerminatingRow as C}from"./terminating.js";import{wantsExecution as D}from"./wants-execution.js";import{enactRequestedStops as S}from"./requested-stops.js";import{endRemoteExecutionIfGone as T,recoverLaunching as G,wakeResident as H}from"./remote-wake.js";import{liveExecutionForNode as U}from"../../../core/canvas/executions.js";function W(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}l(W,"isTerminal");const q=12e4;function B(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<q}l(B,"isLaunchInFlight");class pe{static{l(this,"NodeLifecycleReconciler")}freezeLane=new A;async run(t,e){const a=[];await S(t);for(const n of e.rows)await z.fresh(async()=>{try{await this.classifyRow(t,n,e,a)}catch(i){const s=n.node_id;y({level:"error",event:"node.lifecycle.classify_failed",...b(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(W(e)){if(C(e,await a.fleet.has(i)))return;if(w(e)&&!await a.fleet.has(i)&&!d(e.profile_id)&&!await v(i)&&!P(e,t)&&await p(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead"&&await h(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await m(i):n.push({row:e,enact:"thaw"}));return}if(await a.fleet.has(i)){const f=await U(i);if(f?.runner==="blaxel"&&f.state==="running"&&!d(e.profile_id)&&await p(i)){if(await H(f)==="lost")return}else f?.runner==="blaxel"&&f.state==="running"&&await T(f);if(await a.fleet.has(i)){d(e.profile_id)?await a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}}if(await G(i,t),d(e.profile_id)){await this.freezeLane.endEpisode(i);return}if(await v(i))return;const s=e.pi_pid==null&&await E(i),u=e.pi_pid==null&&!s&&await L(i),c=s||u||e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead";if(c&&!await a.fleet.hasProfilePauseInterruption(i)&&await h(i,{reason:"observed-dead"}),!await D(e)){await this.thawMark(e),await this.settleParked(e);return}if(P(e,t))return;if(e.status==="idle"&&g(e)&&await p(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!s||e.frozen_at!=null)&&!u){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(await a.fleet.isClaimed(i)||B(e,t))return;y({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!c)return;const o=M(await R(i),(await I(i))?.interrupted===!0,await F(i));if(o==="respawn-fresh"||o==="respawn-cycle"||o==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await m(t.node_id)}async settleParked(t){if(t.status!=="idle")return;await this.freezeLane.endEpisode(t.node_id);const e=await x(t.node_id,t.lifecycle);e!=="release"&&await N(t.node_id,e)}}export{q as LAUNCH_GRACE_MS,pe as NodeLifecycleReconciler,B as isLaunchInFlight};

package/dist/daemon/reconcilers/pending-review-submit.js

@@ -1 +1 @@
-var f=Object.defineProperty;var n=(t,e)=>f(t,"name",{value:e,configurable:!0});import{emitEvent as s}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{listSubmitRequestedRequests as l}from"../../core/human/requests.js";import{completeRequestedSubmit as m}from"../review/finish.js";class b{static{n(this,"PendingReviewSubmitReconciler")}async run(e,i){if(!i.lifecycle.acceptsDetachedWork())return;let o;try{o=await l()}catch(r){c.fresh(()=>{s({level:"error",event:"review.deferred_submit.scan_failed",error:r})});return}for(const r of o)i.lifecycle.registerDetached(m(r.request_id,{atCompanionTurnEnd:!1}).then(()=>{},d=>{c.fresh(()=>{s({level:"error",event:"review.deferred_submit.failed",error:d})})}))}}export{b as PendingReviewSubmitReconciler};
+var c=Object.defineProperty;var s=(i,r)=>c(i,"name",{value:r,configurable:!0});import{emitEvent as a}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{listSubmitRequestedRequests as d}from"../../core/human/requests.js";import{completeRequestedSubmit as f}from"../review/finish.js";import{withRowClaim as l}from"./row-claim.js";const u=6e4;class R{static{s(this,"PendingReviewSubmitReconciler")}async run(r,o){if(!o.lifecycle.acceptsDetachedWork())return;let n;try{n=await d()}catch(e){m.fresh(()=>{a({level:"error",event:"review.deferred_submit.scan_failed",error:e})});return}for(const e of n)o.lifecycle.registerDetached(l("human_requests",e.request_id,u,async t=>{await t.assertOwned(),await f(e.request_id,{atCompanionTurnEnd:!1})}).then(()=>{},t=>{m.fresh(()=>{a({level:"error",event:"review.deferred_submit.failed",error:t})})}))}}export{R as PendingReviewSubmitReconciler};

package/dist/daemon/reconcilers/recipe-image.d.ts

@@ -0,0 +1,49 @@
+import { type Recipe } from '../../core/runtime/runner/blaxel/image-build.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+/** How long a claim holds without renewal. The holder renews at a third of it. */
+export declare const LEASE_MS: number;
+/** After a failed build the same revision is not tried again for this long. */
+export declare const RETRY_AFTER_FAILURE_MS: number;
+export type EnvironmentImage = {
+    state: 'base';
+} | {
+    state: 'image';
+    ref: string;
+    revision: number;
+} | {
+    state: 'pending';
+    error: string | null;
+};
+/** What a wake of a node in `environmentId` creates its sandbox from (D25, D95). `base`: the environment has no recipe content (never set, or cleared). `pending`: it has one and no image has ever built, so the wake must not start a runner call. */
+export declare function resolveEnvironmentImage(environmentId: string): Promise<EnvironmentImage>;
+/** Starts a supervision tick when an environment's image is built, so the nodes waiting on it wake now instead of at the next beat. Returns the unsubscribe. */
+export declare function subscribeImageBuilt(requestTick: (environmentId: string) => void): Promise<() => void>;
+export interface RecipeBuildInput {
+    environmentId: string;
+    revision: number;
+    recipe: Recipe;
+    person: string;
+}
+export interface RecipeImageDeps {
+    /** Builds and pushes the image; returns its reference. The default clones with a G-1 token and uses Blaxel's build API. */
+    build?: (input: RecipeBuildInput) => Promise<{
+        ref: string;
+    }>;
+    now?: () => number;
+}
+export declare class RecipeImageReconciler {
+    private readonly deps;
+    constructor(deps?: RecipeImageDeps);
+    /** One pass: claim every environment with an unbuilt revision and build it detached from the tick. */
+    run(nowMs: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): Promise<void>;
+    private claim;
+    private buildClaimed;
+    private recordFailure;
+}
+/** Nodes of profiles that run in `environmentId` with wake mail unseen and no execution: the ones a missing image keeps dormant. */
+export declare function nodesWaitingOnImage(environmentId: string): Promise<Array<{
+    node_id: string;
+    run_id: string;
+}>>;

package/dist/daemon/reconcilers/recipe-image.js

@@ -0,0 +1,7 @@
+var E=Object.defineProperty;var _=(n,e)=>E(n,"name",{value:e,configurable:!0});import{mkdtempSync as w}from"node:fs";import{tmpdir as b}from"node:os";import{join as v}from"node:path";import{hasUnseenWakeMail as S}from"../../core/canvas/deliveries.js";import{openDb as s,withCanvasWrite as N}from"../../core/canvas/db.js";import{emitEvent as d}from"../../core/events/emit.js";import{appendRunEvent as D}from"../../core/runs/events.js";import{blaxelConfigFromEnv as A}from"../../core/runtime/runner/blaxel/client.js";import{buildImage as R,environmentImageName as h,prepareBuildDir as T,removeBuildDir as L}from"../../core/runtime/runner/blaxel/image-build.js";import{mintInstallationToken as O}from"../api/handlers/git-credential.js";import{RECIPE_HAS_CONTENT_SQL as p,recipeOf as y}from"../api/environment-of.js";import{brokerSignalBus as C}from"../broker-signals/bus.js";const g=3*6e4,I=10*6e4;async function X(n){const e=await(await s()).prepare(`SELECT ${p} AS has_content, image_revision, image_ref, image_error FROM environments WHERE object = ?`).get(n);if(e===void 0)throw new Error(`no environment ${n}`);return e.has_content?e.image_ref===null||e.image_revision===null?{state:"pending",error:e.image_error}:{state:"image",ref:e.image_ref,revision:Number(e.image_revision)}:{state:"base"}}_(X,"resolveEnvironmentImage");async function q(n){return await C.subscribe(e=>{n(e)},{kinds:["environment_image"]})}_(q,"subscribeImageBuilt");async function U(n,e){const i=n.recipe.repositories.map(a=>a.github_full_name),r=i.length===0?null:await O(n.person,i),t=w(v(b(),"crtr-recipe-image-"));try{return await T({recipe:n.recipe,revision:n.revision,token:r?.token??null,dir:t}),await R(e,h(n.environmentId),t)}finally{L(t)}}_(U,"defaultBuild");class z{static{_(this,"RecipeImageReconciler")}deps;constructor(e={}){this.deps=e}async run(e,i){if(i.lifecycle.acceptsDetachedWork())try{const r=new Date(e).toISOString(),t=await(await s()).prepare(`SELECT object FROM environments WHERE kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${p}
+        AND (image_building_at IS NULL OR image_building_at <= ?)`).all(r);for(const{object:a}of t){const c=await this.claim(a,e);c!==null&&i.lifecycle.registerDetached(this.buildClaimed(a,c.revision,c.lease).catch(o=>{d({level:"error",event:"recipe_image.failed",error:o})}))}}catch(r){d({level:"error",event:"recipe_image.reconcile_failed",error:r})}}async claim(e,i){const r=new Date(i+g).toISOString(),t=await(await s()).prepare(`UPDATE environments SET image_building_at = ?
+      WHERE object = ? AND kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${p} AND (image_building_at IS NULL OR image_building_at <= ?)
+      RETURNING recipe_revision`).get(r,e,new Date(i).toISOString());return t===void 0?null:{revision:Number(t.recipe_revision),lease:r}}async buildClaimed(e,i,r){const t=this.deps.now??Date.now;let a=r;const c=setInterval(()=>{const o=new Date(t()+g).toISOString();(async()=>{(await(await s()).prepare("UPDATE environments SET image_building_at = ? WHERE object = ? AND image_building_at = ?").run(o,e,a)).changes===1&&(a=o)})().catch(()=>{})},g/3);c.unref();try{const o=await y(e);if(o===null||o.revision!==i){await(await s()).prepare("UPDATE environments SET image_building_at = NULL WHERE object = ? AND image_building_at = ?").run(e,a);return}let l;try{const m=await(await s()).prepare("SELECT person FROM profiles WHERE environment_id = ? ORDER BY created_at LIMIT 1").get(e);if(m===void 0)throw new Error(`no profile runs in environment ${e}, so there is no person to take a GitHub token for`);const u={environmentId:e,revision:i,recipe:o,person:m.person};l=await(this.deps.build??(f=>U(f,A())))(u)}catch(m){await this.recordFailure(e,a,m instanceof Error?m.message:String(m),t());return}await(await s()).prepare(`UPDATE environments SET image_revision = ?, image_ref = ?, image_error = NULL, image_building_at = NULL
+        WHERE object = ? AND image_building_at = ? AND COALESCE(image_revision, 0) < ?`).run(i,l.ref,e,a,i)}finally{clearInterval(c)}}async recordFailure(e,i,r,t){const a=new Date(t+I).toISOString();if((await(await s()).prepare("UPDATE environments SET image_error = ?, image_building_at = ? WHERE object = ? AND image_building_at = ?").run(r,a,e,i)).changes===1){d({level:"error",event:"recipe_image.build_failed",fields:{environment_id:e,error:r}});for(const o of await W(e))try{await N(async l=>{await D(l,o.run_id,o.node_id,"environment.image_failed",{environment_id:e,error:r})})}catch(l){d({level:"error",event:"recipe_image.failure_event_failed",node_id:o.node_id,error:l})}}}}async function W(n){const e=await(await s()).prepare(`SELECT n.node_id, n.run_id FROM nodes n JOIN profiles p ON p.id = n.profile_id
+    WHERE p.environment_id = ? AND n.status IN ('active', 'idle') AND n.run_id IS NOT NULL
+      AND NOT EXISTS (SELECT 1 FROM executions e WHERE e.node_id = n.node_id AND e.state != 'exited')`).all(n),i=[];for(const r of e)await S(r.node_id)&&i.push(r);return i}_(W,"nodesWaitingOnImage");export{g as LEASE_MS,I as RETRY_AFTER_FAILURE_MS,z as RecipeImageReconciler,W as nodesWaitingOnImage,X as resolveEnvironmentImage,q as subscribeImageBuilt};

package/dist/daemon/reconcilers/row-claim.d.ts

@@ -0,0 +1,19 @@
+/** Tables whose rows carry `claimed_by` and `lease_expires_at` (002-fleet.sql), with their id column. */
+declare const CLAIMABLE: {
+    readonly nodes: "node_id";
+    readonly structured_requests: "request_id";
+    readonly human_requests: "request_id";
+};
+export type ClaimableTable = keyof typeof CLAIMABLE;
+export declare class ClaimLostError extends Error {
+    constructor(table: string, id: string);
+}
+export interface RowClaim {
+    /** Throws `ClaimLostError` unless this claim still owns the row (not taken over, not expired). Called inside a `withCanvasWrite`
+     *  body it locks the row (`FOR UPDATE`) until commit, so a takeover cannot slip in between the check and the writes that follow. */
+    assertOwned(): Promise<void>;
+}
+/** Runs `work` when this process claims the row; returns false without running it when another process holds the claim. A failed
+ *  `work` releases the claim and rethrows, so the next pass retries it. */
+export declare function withRowClaim(table: ClaimableTable, id: string, ttlMs: number, work: (claim: RowClaim) => Promise<void>): Promise<boolean>;
+export {};

package/dist/daemon/reconcilers/row-claim.js

@@ -0,0 +1,2 @@
+var E=Object.defineProperty;var i=(e,r)=>E(e,"name",{value:r,configurable:!0});import{randomUUID as D}from"node:crypto";import{openDb as w}from"../../core/canvas/db.js";const d={nodes:"node_id",structured_requests:"request_id",human_requests:"request_id"};class _ extends Error{static{i(this,"ClaimLostError")}constructor(r,t){super(`row claim lost: ${r} ${t} is no longer held by this process`)}}async function L(e,r,t,p){const a=d[e],n=D(),o=await w(),u=new Date().toISOString();if((await o.prepare(`UPDATE ${e} SET claimed_by = ?, lease_expires_at = ?
+    WHERE ${a} = ? AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?)`).run(n,new Date(Date.now()+t).toISOString(),r,u)).changes!==1)return!1;let s=!1;const c=setInterval(()=>{o.prepare(`UPDATE ${e} SET lease_expires_at = ? WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ?`).run(new Date(Date.now()+t).toISOString(),r,n,new Date().toISOString()).then(l=>{l.changes!==1&&(s=!0)},()=>{s=!0})},Math.max(1,Math.floor(t/3)));c.unref();const m={async assertOwned(){if(s)throw new _(e,r);if(await(await w()).prepare(`SELECT 1 AS held FROM ${e} WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ? FOR UPDATE`).get(r,n,new Date().toISOString())===void 0)throw s=!0,new _(e,r)}};try{await p(m)}finally{clearInterval(c),await o.prepare(`UPDATE ${e} SET claimed_by = NULL, lease_expires_at = NULL WHERE ${a} = ? AND claimed_by = ?`).run(r,n)}return!0}i(L,"withRowClaim");export{_ as ClaimLostError,L as withRowClaim};

package/dist/daemon/reconcilers/storage-due-work.d.ts

@@ -1,10 +1,16 @@
+import { type BlobStore } from '../../core/blobs/index.js';
 import { type StorageClient, type StorageConfig } from '../../core/storage/client.js';
 import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
 export declare const STORAGE_DUE_WORK_INTERVAL_MS = 60000;
 /** An upload no run has used this long after creation is released. */
 export declare const UNUSED_UPLOAD_TTL_MS: number;
 /** Storage deletes attempted per kind per pass; the rest wait for the next pass. */
 export declare const STORAGE_DUE_WORK_BUDGET = 200;
+/** A deleted node's blobs and `objects` row are removed this long after `deleted_at`; until then the owner can undelete it. */
+export declare const NODE_BLOB_RETENTION_MS: number;
+/** Nodes purged per pass; the rest wait for the next pass. */
+export declare const NODE_PURGE_BUDGET = 25;
 /** Consecutive `storage_unavailable` failures after which a pass stops early. */
 export declare const STORAGE_DOWN_AFTER = 3;
 export interface StorageDueWorkContext {
@@ -13,20 +19,29 @@ export interface StorageDueWorkContext {
 export interface StorageDueWorkDeps {
     config: () => StorageConfig | null;
     client: () => StorageClient;
+    blobs?: () => Promise<BlobStore>;
 }
 export interface StorageDueWorkResult {
     marked_uploads: number;
     expired_shares: number;
     deleted_uploads: number;
     deleted_share_objects: number;
+    purged_nodes: number;
     failed: number;
 }
 export declare class StorageDueWorkReconciler {
     private readonly deps;
-    private lastPassAt;
-    private inFlight;
     constructor(deps?: StorageDueWorkDeps);
     run(now: number, ctx: StorageDueWorkContext): void;
     /** One pass over the due-work rows. Exposed for tests. */
-    pass(now: number): Promise<StorageDueWorkResult>;
+    pass(now: number, lane?: LanePass): Promise<StorageDueWorkResult>;
+    /** Delete the blobs of nodes deleted over 90 days ago, then the node's `objects` row. Every such node loses its blobs;
+     *  its row goes only if no foreign key refers to it (`objects.owner`, `revisions.author`, `object_kinds.owner` have no
+     *  cascade), which Postgres decides at the DELETE (23503), so a reference added meanwhile cannot slip through. Blobs go
+     *  first, outside any transaction: a crash between the two is retried (the retry finds no blobs), and two passes at once
+     *  both succeed because deleting an absent blob and an absent row are no-ops. The sweep walks `(deleted_at, id)` from a
+     *  cursor kept in the lane's state, so a row that stays referenced is passed over once per sweep, not re-selected on every
+     *  pass; the cursor restarts at the end of the sweep. */
+    private purgeDeletedNodes;
+    private settleFileTransfers;
 }

package/dist/daemon/reconcilers/storage-due-work.js

@@ -1 +1,2 @@
-var D=Object.defineProperty;var d=(i,t)=>D(i,"name",{value:t,configurable:!0});import{withCanvasWrite as v}from"../../core/canvas/db.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{RunError as E}from"../../core/runs/operations.js";import{storageClient as S,storageConfig as k}from"../../core/storage/client.js";import{deleteUploadRow as O,expireShares as U,markShareObjectDeleted as R,markUploadsDeletePending as b,pendingUploadDeletes as A,sharesOwingObjectDelete as T,staleUnusedUploads as x}from"../../core/storage/tables.js";const y=6e4,I=1440*6e4,w=200,N=3,F=d(i=>i instanceof E&&i.code==="storage_unavailable","unavailable");class M{static{d(this,"StorageDueWorkReconciler")}deps;lastPassAt=Number.NEGATIVE_INFINITY;inFlight=!1;constructor(t={config:k,client:S}){this.deps=t}run(t,l){if(this.inFlight||t-this.lastPassAt<y||!l.lifecycle.acceptsDetachedWork())return;this.lastPassAt=t;let o;try{o=this.deps.config()}catch(e){p.fresh(()=>c({level:"error",event:"storage.due_work.failed",error:e}));return}if(!o)return;this.inFlight=!0;const f=p.fresh(()=>this.pass(t)).then(()=>{}).catch(e=>{p.fresh(()=>c({level:"error",event:"storage.due_work.failed",error:e}))}).finally(()=>{this.inFlight=!1});l.lifecycle.registerDetached(f)}async pass(t){const l=new Date(t).toISOString(),{marked:o,expired:f}=await v(async r=>{const s=await x(new Date(t-I).toISOString(),r);return await b(s,r,l),{marked:s.length,expired:(await U(l,r)).length}}),e={marked_uploads:o,expired_shares:f,deleted_uploads:0,deleted_share_objects:0,failed:0},h=(await A()).slice(0,w),_=(await T()).slice(0,w);if(h.length||_.length){const r=this.deps.client();let s,n=0,u=!1;const g=d(async a=>{if(u)return!1;try{return await a(),n=0,!0}catch(m){return e.failed++,s??=m,n=F(m)?n+1:0,n>=N&&(u=!0),!1}},"attempt");for(const a of h)await g(()=>r.deleteUpload(a.upload_id))&&(await O(a.upload_id),e.deleted_uploads++);for(const a of _)await g(()=>r.deleteShare(a.share_id))&&(await R(a.share_id),e.deleted_share_objects++);s!==void 0&&c({level:"warn",event:"storage.due_work.deletes_failed",error:s,fields:{failed:e.failed,storage_unavailable:u}})}return(e.marked_uploads||e.expired_shares||e.deleted_uploads||e.deleted_share_objects)&&c({level:"info",event:"storage.due_work.settled",outcome:"succeeded",fields:{...e}}),e}}export{N as STORAGE_DOWN_AFTER,w as STORAGE_DUE_WORK_BUDGET,y as STORAGE_DUE_WORK_INTERVAL_MS,M as StorageDueWorkReconciler,I as UNUSED_UPLOAD_TTL_MS};
+var S=Object.defineProperty;var u=(_,a)=>S(_,"name",{value:a,configurable:!0});import{blobStore as O}from"../../core/blobs/index.js";import{openDb as v,withCanvasWrite as w}from"../../core/canvas/db.js";import{emitEvent as h}from"../../core/events/emit.js";import{operationIdContext as E}from"../../core/events/operation-id.js";import{RunError as R}from"../../core/runs/operations.js";import{storageClient as b,storageConfig as N}from"../../core/storage/client.js";import{deleteUploadRow as T,expireShares as y,markShareObjectDeleted as k,markUploadsDeletePending as U,pendingUploadDeletes as A,sharesOwingObjectDelete as x,staleUnusedUploads as L}from"../../core/storage/tables.js";import{runThrottledLane as I}from"./throttled-lane.js";const j=6e4,W=1440*6e4,D=200,B=5*6e4,M=2160*60*6e4,m=25,F=["conversations","events","jobs","attachments"],G=3,C=u(_=>_ instanceof R&&_.code==="storage_unavailable","unavailable");class z{static{u(this,"StorageDueWorkReconciler")}deps;constructor(a={config:N,client:b}){this.deps=a}run(a,e){if(!e.lifecycle.acceptsDetachedWork())return;const t=E.fresh(()=>I("storage-due-work","",a,j,B,async l=>{await this.pass(a,l)})).then(()=>{}).catch(l=>{E.fresh(()=>h({level:"error",event:"storage.due_work.failed",error:l}))});e.lifecycle.registerDetached(t)}async pass(a,e){const t={marked_uploads:0,expired_shares:0,deleted_uploads:0,deleted_share_objects:0,purged_nodes:0,failed:0};return t.purged_nodes=await this.purgeDeletedNodes(a,e),this.deps.config()&&await this.settleFileTransfers(a,t),(t.purged_nodes||t.marked_uploads||t.expired_shares||t.deleted_uploads||t.deleted_share_objects)&&h({level:"info",event:"storage.due_work.settled",outcome:"succeeded",fields:{...t}}),t}async purgeDeletedNodes(a,e){const t=new Date(a-M).toISOString(),l=await v(),f=e?.state?JSON.parse(e.state):null,o=await l.prepare(`SELECT id, deleted_at FROM objects WHERE type = 'node' AND deleted_at < ?
+      AND (deleted_at, id) > (?, ?) ORDER BY deleted_at, id LIMIT ?`).all(t,f?.deleted_at??"",f?.id??"",m);if(o.length===0)return f&&await e?.save(null),0;const p=await(this.deps.blobs??O)();let d=0,s=-1;for(const[r,{id:c}]of o.entries())try{for(const i of F)for(const n of await p.list(`${i}/${c}/`))await p.delete(n);await w(async i=>{await i.prepare("DELETE FROM objects WHERE id = ? AND type = 'node' AND deleted_at < ?").run(c,t)},{savepoint:!0}),d++}catch(i){i.code!=="23503"&&(s<0&&(s=r),h({level:"warn",event:"storage.due_work.node_purge_failed",error:i,fields:{node_id:c}}))}if(s>=0){const r=o[s-1];await e?.save(r?JSON.stringify({deleted_at:r.deleted_at,id:r.id}):e.state)}else{const r=o[o.length-1];await e?.save(o.length<m?null:JSON.stringify({deleted_at:r.deleted_at,id:r.id}))}return d}async settleFileTransfers(a,e){const t=new Date(a).toISOString(),{marked:l,expired:f}=await w(async d=>{const s=await L(new Date(a-W).toISOString(),d);return await U(s,d,t),{marked:s.length,expired:(await y(t,d)).length}});e.marked_uploads=l,e.expired_shares=f;const o=(await A()).slice(0,D),p=(await x()).slice(0,D);if(o.length||p.length){const d=this.deps.client();let s,r=0,c=!1;const i=u(async n=>{if(c)return!1;try{return await n(),r=0,!0}catch(g){return e.failed++,s??=g,r=C(g)?r+1:0,r>=G&&(c=!0),!1}},"attempt");for(const n of o)await i(()=>d.deleteUpload(n.upload_id))&&(await T(n.upload_id),e.deleted_uploads++);for(const n of p)await i(()=>d.deleteShare(n.share_id))&&(await k(n.share_id),e.deleted_share_objects++);s!==void 0&&h({level:"warn",event:"storage.due_work.deletes_failed",error:s,fields:{failed:e.failed,storage_unavailable:c}})}}}export{M as NODE_BLOB_RETENTION_MS,m as NODE_PURGE_BUDGET,G as STORAGE_DOWN_AFTER,D as STORAGE_DUE_WORK_BUDGET,j as STORAGE_DUE_WORK_INTERVAL_MS,z as StorageDueWorkReconciler,W as UNUSED_UPLOAD_TTL_MS};

package/dist/daemon/reconcilers/storage-maintenance.d.ts

@@ -10,20 +10,9 @@ export interface StorageMaintenanceContext {
     /** A live engine handle is the sweep's proof that a node is NOT disposable. */
     fleet: FleetRegistry;
 }
-/** Recurring storage cleanup. The in-memory throttle clocks intentionally reset
- * on daemon boot, so the first tick runs every full sweep — which is also how
- * managed-worktree reconciliation gets its start-up pass. */
+/** Recurring storage cleanup. Each sweep's throttle and overlap guard is a `reconciler_leases` row (`runThrottledLane`),
+ * so two reconciler processes never run one sweep at once and a restart does not repeat a sweep early. */
 export declare class StorageMaintenanceReconciler {
-    private lastSpareSweepAt;
-    private lastCreateRequestSweepAt;
-    private lastEnvironmentTouchAt;
-    private lastRunExpiryAt;
-    private lastRunEventPassAt;
-    private runEventCursor;
-    private lastAnalyzeAt;
-    private lastRetentionSweepAt;
-    private lastContextStripAt;
-    private contextStripInFlight;
     private focusGcInFlight;
     /** Wall time of the last cache pass; null until the first one, which runs at daemon boot. */
     private lastCachePassAt;
@@ -59,6 +48,9 @@ export declare class StorageMaintenanceReconciler {
     private gcStaleFocuses;
     /** Delete create idempotency keys once their bounded recovery window ends. */
     private gcCreateRequests;
+    /** Delete expired `viewer_attachments` rows. A viewer that never detached cleanly (its API process died) leaves its
+     * row behind; readers already ignore it by `expires_at`, so this only bounds the table. */
+    private gcViewerAttachments;
     /** Stamp the machine environment as seen. A failed stamp is an event, never an error out of the tick. */
     private touchEnvironment;
     /** Collect dead or expired hidden warm-pool spares. */

package/dist/daemon/reconcilers/storage-maintenance.js

@@ -1 +1 @@
-var I=Object.defineProperty;var l=(f,t)=>I(f,"name",{value:t,configurable:!0});import{rm as y}from"node:fs/promises";import{closeFocusRow as A,getFocusById as S,listFocuses as N,listNodes as w}from"../../core/canvas/index.js";import{pruneExpiredIdempotencyKeys as R}from"../../core/runs/operations.js";import{getNode as g,lastActivityMs as u,listTerminalPlacementResidue as T,pruneExpiredCreateRequests as C,pruneNodes as F}from"../../core/canvas/canvas.js";import{lastContextStripMs as x,stripContextDir as P}from"../../core/canvas/context-strip.js";import{isAppRun as D}from"../../core/canvas/app-run.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{emitEvent as i}from"../../core/events/emit.js";import{operationIdContext as s}from"../../core/events/operation-id.js";import{listLivePanesAsync as G,tearDownNode as V}from"../../core/runtime/placement.js";import{reapStaleSpares as b}from"../../core/runtime/warm-pool.js";import{GroveInstanceSweepReconciler as M}from"./grove-instance-sweep.js";import{ManagedWorktreeSweepReconciler as Y}from"./managed-worktree-sweep.js";import{compactRunEvents as W,expireRunEvents as k}from"../../core/runs/events.js";import{openDb as q}from"../../core/canvas/db.js";import{touchMachineEnvironment as L}from"../../core/environments.js";import{retentionHold as _}from"./live-obligation.js";import{pruneMemoryReads as O}from"../../core/canvas/memory-reads.js";import{existsSync as z}from"node:fs";import{launcherRequest as U}from"../../launcher/client.js";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{cachePassDue as K}from"../../launcher/limits/policy.js";const $=10*6e4,m=60*1e3,B=60*1e3,v=5*6e4,Q=2,X=1440*6e4,Z=2e3;class Ae{static{l(this,"StorageMaintenanceReconciler")}lastSpareSweepAt=Number.NEGATIVE_INFINITY;lastCreateRequestSweepAt=Number.NEGATIVE_INFINITY;lastEnvironmentTouchAt=Number.NEGATIVE_INFINITY;lastRunExpiryAt=Number.NEGATIVE_INFINITY;lastRunEventPassAt=Number.NEGATIVE_INFINITY;runEventCursor=0;lastAnalyzeAt=Number.NEGATIVE_INFINITY;lastRetentionSweepAt=Number.NEGATIVE_INFINITY;lastContextStripAt=Number.NEGATIVE_INFINITY;contextStripInFlight=!1;focusGcInFlight=!1;lastCachePassAt=null;cachePassInFlight=!1;worktreeSweep=new Y;groveInstanceSweep=new M;async run(t,r){if(await this.reapDeadResidue(t),this.scheduleFocusGc(r),await this.gcWarmPool(t),await this.touchEnvironment(t),await this.gcCreateRequests(t),t-this.lastRunEventPassAt>=Z){this.lastRunEventPassAt=t;try{await k(t);const e=await W(t,this.runEventCursor);this.runEventCursor=e.cursor}catch(e){console.warn(`run event maintenance failed: ${String(e)}`)}}if(t-this.lastRunExpiryAt>=60*6e4){try{await O(t)}catch(e){console.warn(`memory-read log prune failed: ${String(e)}`)}this.lastRunExpiryAt=t}t-this.lastAnalyzeAt>=1440*6e4&&(await(await q()).exec("ANALYZE"),this.lastAnalyzeAt=t),await this.sweepRetention(t,r),this.scheduleCachePass(t,r),this.scheduleContextStrip(t,r),this.worktreeSweep.run(t,r),this.groveInstanceSweep.run(t,r)}scheduleCachePass(t,r){this.cachePassInFlight||!K(this.lastCachePassAt,t)||process.platform!=="linux"||!z(H)||r.lifecycle.acceptsDetachedWork()&&(this.lastCachePassAt=t,this.cachePassInFlight=!0,r.lifecycle.registerDetached(U({operation:"cache.purge"}).then(()=>{}).catch(e=>{s.fresh(()=>{i({level:"warn",event:"app.cache.pass_failed",error:e})})}).then(()=>{this.cachePassInFlight=!1})))}msSinceTerminalActivity(t,r){const e=u(t);return e===0?Number.POSITIVE_INFINITY:r-e}async sweepRetention(t,r){t-this.lastRetentionSweepAt<v||r.lifecycle.acceptsDetachedWork()&&(this.lastRetentionSweepAt=t,await s.fresh(async()=>{try{const e=await F({ttlDays:Q,held:await _(r.fleet),removeDirs:l(a=>{r.lifecycle.registerDetached(Promise.all(a.map(o=>y(o,{recursive:!0,force:!0}))).then(()=>{}).catch(o=>{s.fresh(()=>{i({level:"error",event:"canvas.retention.failed",error:o})})}))},"removeDirs")});e.pruned.length>0&&i({level:"info",event:"canvas.retention.pruned",outcome:"succeeded",fields:{pruned_count:e.pruned.length}})}catch(e){i({level:"error",event:"canvas.retention.failed",error:e})}}))}scheduleContextStrip(t,r){if(this.contextStripInFlight||t-this.lastContextStripAt<v||!r.lifecycle.acceptsDetachedWork())return;this.lastContextStripAt=t,this.contextStripInFlight=!0;const e=this.stripFinishedContexts(t,r).finally(()=>{this.contextStripInFlight=!1});r.lifecycle.registerDetached(e)}async stripFinishedContexts(t,r){const e=await _(r.fleet),a=l(async n=>n!==null&&["done","canceled","dead"].includes(n.status)&&(n.pi_pid??null)===null&&(n.frozen_at??null)===null&&!D(n)&&u(n)<t-X&&!await e(n.node_id),"eligible");let o=0,p=0;try{for(const n of await w({status:["done","canceled","dead"]})){if(!c(n.node_id)||!await a(n))continue;const h=await x(n.node_id);if(h!==null&&h>=u(n)||!await a(await g(n.node_id)))continue;let d;try{d=await P(n.node_id)}catch(E){s.fresh(()=>{i({level:"warn",event:"canvas.context_strip.node_failed",error:E,fields:{node_id:n.node_id}})});continue}d.removed.length>0&&(o+=1,p+=d.bytes)}}catch(n){s.fresh(()=>{i({level:"error",event:"canvas.context_strip.failed",error:n})})}o>0&&s.fresh(()=>{i({level:"info",event:"canvas.context_strip.completed",outcome:"succeeded",fields:{node_count:o,bytes:p}})})}async reapDeadResidue(t){let r;try{r=await T()}catch(e){s.fresh(()=>{i({level:"error",event:"residue.reap_scan.failed",error:e})});return}for(const e of r){try{if(this.msSinceTerminalActivity(e,t)<$)continue}catch(a){s.fresh(()=>{i({level:"error",event:"residue.reap.failed",...c(e.node_id)?{node_id:e.node_id}:{},error:a,fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})});continue}await s.fresh(async()=>{try{i({level:"info",event:"residue.reaped",...c(e.node_id)?{node_id:e.node_id}:{},fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}}),await V(e.node_id)}catch(a){i({level:"error",event:"residue.reap.failed",...c(e.node_id)?{node_id:e.node_id}:{},error:a,fields:{...c(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})}})}}scheduleFocusGc(t){if(this.focusGcInFlight||!t.lifecycle.acceptsDetachedWork())return;this.focusGcInFlight=!0;const r=this.gcStaleFocuses().finally(()=>{this.focusGcInFlight=!1});t.lifecycle.registerDetached(r)}async gcStaleFocuses(){let t;try{t=await N()}catch(e){s.fresh(()=>{i({level:"error",event:"focus.gc.failed",error:e})});return}if(t.length===0)return;let r;try{r=await G()}catch(e){s.fresh(()=>{i({level:"error",event:"focus.gc.failed",error:e})});return}if(r!==null)for(const e of t){if(e.pane===null||r.has(e.pane))continue;if(!await s.fresh(async()=>{try{return(await S(e.focus_id))?.pane!==e.pane||(i({level:"info",event:"focus.gc.removed",...c(e.node_id)?{node_id:e.node_id}:{},fields:{focus_id:e.focus_id,focus_node_id:e.node_id,pane:e.pane}}),await A(e.focus_id)),!0}catch(o){return i({level:"error",event:"focus.gc.failed",error:o}),!1}}))return}}async gcCreateRequests(t){t-this.lastCreateRequestSweepAt<B||(this.lastCreateRequestSweepAt=t,await s.fresh(async()=>{const r=await C(t),e=await R(t);e>0&&i({level:"info",event:"canvas.idempotency_keys.reaped",outcome:"succeeded",fields:{reaped_count:e}}),r>0&&i({level:"info",event:"canvas.create_requests.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async touchEnvironment(t){t-this.lastEnvironmentTouchAt<m||(this.lastEnvironmentTouchAt=t,await s.fresh(async()=>{try{await L(new Date(t).toISOString())}catch(r){i({level:"warn",event:"environment.touch.failed",error:r})}}))}async gcWarmPool(t){t-this.lastSpareSweepAt<m||(this.lastSpareSweepAt=t,await s.fresh(async()=>{const r=await b();r.length>0&&i({level:"info",event:"warm_pool.spares_reaped",outcome:"succeeded",fields:{reaped_count:r.length}})}))}}export{X as CONTEXT_STRIP_GRACE_MS,$ as DEAD_REAP_GRACE_MS,v as RETENTION_SWEEP_INTERVAL_MS,Q as RETENTION_TTL_DAYS,Ae as StorageMaintenanceReconciler};
+var S=Object.defineProperty;var f=(m,t)=>S(m,"name",{value:t,configurable:!0});import{rm as R}from"node:fs/promises";import{closeFocusRow as I,getFocusById as A,listFocuses as N,listNodes as T}from"../../core/canvas/index.js";import{pruneExpiredIdempotencyKeys as P}from"../../core/runs/operations.js";import{getNode as C,lastActivityMs as p,listTerminalPlacementResidue as D,pruneExpiredCreateRequests as x,pruneNodes as F}from"../../core/canvas/canvas.js";import{lastContextStripMs as M,stripContextDir as L}from"../../core/canvas/context-strip.js";import{isAppRun as W}from"../../core/canvas/app-run.js";import{isSafeNodeId as d}from"../../core/canvas/paths.js";import{emitEvent as n}from"../../core/events/emit.js";import{operationIdContext as i}from"../../core/events/operation-id.js";import{listLivePanesAsync as b,tearDownNode as G}from"../../core/runtime/placement.js";import{reapStaleSpares as V}from"../../core/runtime/warm-pool.js";import{GroveInstanceSweepReconciler as k}from"./grove-instance-sweep.js";import{ManagedWorktreeSweepReconciler as O}from"./managed-worktree-sweep.js";import{compactRunEvents as q}from"../../core/runs/events.js";import{archiveExpiredRunEvents as U}from"../../core/events/run-event-chunks.js";import{blobStore as H}from"../../core/blobs/index.js";import{runThrottledLane as c}from"./throttled-lane.js";import{openDb as w}from"../../core/canvas/db.js";import{touchMachineEnvironment as Y}from"../../core/environments.js";import{retentionHold as y}from"./live-obligation.js";import{pruneMemoryReads as z}from"../../core/canvas/memory-reads.js";import{existsSync as K}from"node:fs";import{launcherRequest as $}from"../../launcher/client.js";import{LAUNCHER_SOCKET as B}from"../../launcher/spec.js";import{cachePassDue as Q}from"../../launcher/limits/policy.js";const X=10*6e4,l=60*1e3,g=60*1e3,h=5*6e4,Z=2,j=1440*6e4,J=2e3,ee=6e4;class Te{static{f(this,"StorageMaintenanceReconciler")}focusGcInFlight=!1;lastCachePassAt=null;cachePassInFlight=!1;worktreeSweep=new O;groveInstanceSweep=new k;async run(t,r){await this.reapDeadResidue(t),this.scheduleFocusGc(r),await this.gcWarmPool(t),await this.touchEnvironment(t),await this.gcCreateRequests(t),await this.gcViewerAttachments(t);try{await c("run-event-maintenance","",t,J,ee,async e=>{await U(await H(),t);const s=await q(t,e.state===null?0:Number(e.state));await e.save(String(s.cursor))})}catch(e){console.warn(`run event maintenance failed: ${String(e)}`)}await c("memory-read-prune","",t,60*6e4,10*6e4,async()=>{try{await z(t)}catch(e){console.warn(`memory-read log prune failed: ${String(e)}`)}}),await c("storage-analyze","",t,1440*6e4,10*6e4,async()=>{await(await w()).exec("ANALYZE")}),await this.sweepRetention(t,r),this.scheduleCachePass(t,r),this.scheduleContextStrip(t,r),this.worktreeSweep.run(t,r),this.groveInstanceSweep.run(t,r)}scheduleCachePass(t,r){this.cachePassInFlight||!Q(this.lastCachePassAt,t)||process.platform!=="linux"||!K(B)||r.lifecycle.acceptsDetachedWork()&&(this.lastCachePassAt=t,this.cachePassInFlight=!0,r.lifecycle.registerDetached($({operation:"cache.purge"}).then(()=>{}).catch(e=>{i.fresh(()=>{n({level:"warn",event:"app.cache.pass_failed",error:e})})}).then(()=>{this.cachePassInFlight=!1})))}msSinceTerminalActivity(t,r){const e=p(t);return e===0?Number.POSITIVE_INFINITY:r-e}async sweepRetention(t,r){r.lifecycle.acceptsDetachedWork()&&await c("retention-sweep","",t,h,h,()=>i.fresh(async()=>{try{const e=await F({ttlDays:Z,held:await y(r.fleet),removeDirs:f(s=>{r.lifecycle.registerDetached(Promise.all(s.map(o=>R(o,{recursive:!0,force:!0}))).then(()=>{}).catch(o=>{i.fresh(()=>{n({level:"error",event:"canvas.retention.failed",error:o})})}))},"removeDirs")});e.pruned.length>0&&n({level:"info",event:"canvas.retention.pruned",outcome:"succeeded",fields:{pruned_count:e.pruned.length}})}catch(e){n({level:"error",event:"canvas.retention.failed",error:e})}}))}scheduleContextStrip(t,r){r.lifecycle.acceptsDetachedWork()&&r.lifecycle.registerDetached(c("context-strip","",t,h,10*6e4,()=>this.stripFinishedContexts(t,r)).then(()=>{},e=>{n({level:"error",event:"canvas.context_strip.failed",error:e})}))}async stripFinishedContexts(t,r){const e=await y(r.fleet),s=f(async a=>a!==null&&["done","canceled","dead"].includes(a.status)&&(a.pi_pid??null)===null&&(a.frozen_at??null)===null&&!W(a)&&p(a)<t-j&&!await e(a.node_id),"eligible");let o=0,_=0;try{for(const a of await T({status:["done","canceled","dead"]})){if(!d(a.node_id)||!await s(a))continue;const v=await M(a.node_id);if(v!==null&&v>=p(a)||!await s(await C(a.node_id)))continue;let u;try{u=await L(a.node_id)}catch(E){i.fresh(()=>{n({level:"warn",event:"canvas.context_strip.node_failed",error:E,fields:{node_id:a.node_id}})});continue}u.removed.length>0&&(o+=1,_+=u.bytes)}}catch(a){i.fresh(()=>{n({level:"error",event:"canvas.context_strip.failed",error:a})})}o>0&&i.fresh(()=>{n({level:"info",event:"canvas.context_strip.completed",outcome:"succeeded",fields:{node_count:o,bytes:_}})})}async reapDeadResidue(t){let r;try{r=await D()}catch(e){i.fresh(()=>{n({level:"error",event:"residue.reap_scan.failed",error:e})});return}for(const e of r){try{if(this.msSinceTerminalActivity(e,t)<X)continue}catch(s){i.fresh(()=>{n({level:"error",event:"residue.reap.failed",...d(e.node_id)?{node_id:e.node_id}:{},error:s,fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})});continue}await i.fresh(async()=>{try{n({level:"info",event:"residue.reaped",...d(e.node_id)?{node_id:e.node_id}:{},fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}}),await G(e.node_id)}catch(s){n({level:"error",event:"residue.reap.failed",...d(e.node_id)?{node_id:e.node_id}:{},error:s,fields:{...d(e.node_id)?{}:{affected_node_id:e.node_id},pane:e.pane,window:e.window,tmux_session:e.tmux_session}})}})}}scheduleFocusGc(t){if(this.focusGcInFlight||!t.lifecycle.acceptsDetachedWork())return;this.focusGcInFlight=!0;const r=this.gcStaleFocuses().finally(()=>{this.focusGcInFlight=!1});t.lifecycle.registerDetached(r)}async gcStaleFocuses(){let t;try{t=await N()}catch(e){i.fresh(()=>{n({level:"error",event:"focus.gc.failed",error:e})});return}if(t.length===0)return;let r;try{r=await b()}catch(e){i.fresh(()=>{n({level:"error",event:"focus.gc.failed",error:e})});return}if(r!==null)for(const e of t){if(e.pane===null||r.has(e.pane))continue;if(!await i.fresh(async()=>{try{return(await A(e.focus_id))?.pane!==e.pane||(n({level:"info",event:"focus.gc.removed",...d(e.node_id)?{node_id:e.node_id}:{},fields:{focus_id:e.focus_id,focus_node_id:e.node_id,pane:e.pane}}),await I(e.focus_id)),!0}catch(o){return n({level:"error",event:"focus.gc.failed",error:o}),!1}}))return}}async gcCreateRequests(t){await c("create-request-gc","",t,g,g,()=>i.fresh(async()=>{const r=await x(t),e=await P(t);e>0&&n({level:"info",event:"canvas.idempotency_keys.reaped",outcome:"succeeded",fields:{reaped_count:e}}),r>0&&n({level:"info",event:"canvas.create_requests.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async gcViewerAttachments(t){await c("viewer-attachment-gc","",t,l,l,()=>i.fresh(async()=>{const r=Number((await(await w()).prepare("DELETE FROM viewer_attachments WHERE expires_at <= ?").run(new Date(t).toISOString())).changes);r>0&&n({level:"info",event:"viewer_attachments.reaped",outcome:"succeeded",fields:{reaped_count:r}})}))}async touchEnvironment(t){await c("environment-touch","",t,l,l,()=>i.fresh(async()=>{try{await Y(new Date(t).toISOString())}catch(r){n({level:"warn",event:"environment.touch.failed",error:r})}}))}async gcWarmPool(t){await c("warm-pool-gc","",t,l,l,()=>i.fresh(async()=>{const r=await V();r.length>0&&n({level:"info",event:"warm_pool.spares_reaped",outcome:"succeeded",fields:{reaped_count:r.length}})}))}}export{j as CONTEXT_STRIP_GRACE_MS,X as DEAD_REAP_GRACE_MS,h as RETENTION_SWEEP_INTERVAL_MS,Z as RETENTION_TTL_DAYS,Te as StorageMaintenanceReconciler};

package/dist/daemon/reconcilers/structured-request-lane.js

@@ -1 +1 @@
-var n=Object.defineProperty;var i=(o,t)=>n(o,"name",{value:t,configurable:!0});import{withCanvasWrite as a}from"../../core/canvas/db.js";import{incompleteStructuredRequestEffects as u,pendingStructuredRequestsPast as d,settleStructuredRequestIn as f}from"../../core/canvas/structured-requests.js";import{emitEvent as l}from"../../core/events/emit.js";import{applySettlementEffects as c}from"../structured-requests/effects.js";class R{static{i(this,"StructuredRequestReconciler")}async run(t){try{for(const e of await d(t)){const r=await a(async s=>await f(s,e.request_id,{intended:"expired",now:t}));r.changed&&r.row!==null&&c(r.row)}for(const e of await u())c(e)}catch(e){l({level:"error",event:"structured_request.reconcile_failed",error:e})}}}export{R as StructuredRequestReconciler};
+var f=Object.defineProperty;var c=(t,r)=>f(t,"name",{value:r,configurable:!0});import{withCanvasWrite as w}from"../../core/canvas/db.js";import{incompleteStructuredRequestEffects as m,pendingStructuredRequestsPast as p,settleStructuredRequestIn as q}from"../../core/canvas/structured-requests.js";import{emitEvent as i}from"../../core/events/emit.js";import{applySettlementEffects as a}from"../structured-requests/effects.js";import{withRowClaim as u}from"./row-claim.js";const n=6e4,d=c(t=>{i({level:"error",event:"structured_request.reconcile_failed",error:t})},"reportFailure");class C{static{c(this,"StructuredRequestReconciler")}async run(r){try{for(const e of await p(r))u("structured_requests",e.request_id,n,async s=>{const o=await w(async l=>(await s.assertOwned(),await q(l,e.request_id,{intended:"expired",now:r})));o.changed&&o.row!==null&&await a(o.row,s)}).catch(d);for(const e of await m())u("structured_requests",e.request_id,n,async s=>{await a(e,s)}).catch(d)}catch(e){i({level:"error",event:"structured_request.reconcile_failed",error:e})}}}export{C as StructuredRequestReconciler};

package/dist/daemon/reconcilers/throttled-lane.d.ts

@@ -0,0 +1,11 @@
+export interface LanePass {
+    /** What the previous pass saved, or null. */
+    state: string | null;
+    /** Fenced: false when the lane was lost to another process, in which case the pass should stop. */
+    save(state: string | null): Promise<boolean>;
+    /** Renews the lane lease and says whether this pass still holds it. Call it before each external effect: false means another
+     *  process owns the lane (this one stalled or could not renew past the lease), and the pass must stop without enacting. */
+    holds(): Promise<boolean>;
+}
+/** Runs `pass` when this process wins the lane's claim and the last pass began at least `intervalMs` before `now`. Returns whether it ran. */
+export declare function runThrottledLane(lane: string, key: string, now: number, intervalMs: number, leaseMs: number, pass: (lane: LanePass) => Promise<void>): Promise<boolean>;

package/dist/daemon/reconcilers/throttled-lane.js

@@ -0,0 +1 @@
+var v=Object.defineProperty;var a=(r,e)=>v(r,"name",{value:e,configurable:!0});import{randomUUID as E}from"node:crypto";import{openDb as L}from"../../core/canvas/db.js";import{advanceReconcilerCursor as p,claimReconcilerLease as N,releaseReconcilerLease as O,renewReconcilerLease as S}from"../../core/canvas/reconciler-leases.js";function d(r){if(r===null)return null;const e=JSON.parse(r);if(typeof e.at!="number")throw new Error(`reconciler lease cursor is not a lane cursor: ${r}`);return e}a(d,"parse");async function b(r,e,n,i,s,h){const y=await(await L()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? AND key = ?").get(r,e),c=d(y?.cursor??null);if(c!==null&&n-c.at<i)return!1;const t=E(),u=await N(r,e,t,s);if(u===void 0)return!1;try{const l=d(u.cursor);if(l!==null&&n-l.at<i)return!1;const f=l?.state??null;if(!await p(r,e,t,JSON.stringify({at:n,state:f})))return!1;let o=!1;const w=a(async()=>{if(o)return!1;try{o=!await S(r,e,t,s)}catch{o=!0}return!o},"renewNow"),m=setInterval(()=>{w()},Math.max(1,Math.floor(s/3)));m.unref();try{await h({state:f,save:a(R=>p(r,e,t,JSON.stringify({at:n,state:R})),"save"),holds:w})}finally{clearInterval(m)}return!0}finally{await O(r,e,t)}}a(b,"runThrottledLane");export{b as runThrottledLane};

package/dist/daemon/review/finish.js

@@ -1 +1 @@
-var g=Object.defineProperty;var n=(e,t)=>g(e,"name",{value:t,configurable:!0});import{withCanvasWrite as j}from"../../core/canvas/db.js";import{get as S}from"../../core/graph/bodies.js";import{documentById as q,editDocument as C,heads as $}from"../../core/graph/documents.js";import{readerFor as k}from"../../core/graph/reader.js";import{checkoutsForCwd as J}from"../../core/graph/repo-sync/repos.js";import{snapshotCommentsLocked as x}from"../../core/review/comments.js";import{readReviewSource as N}from"../../core/review/document.js";import{projectReviewCancellation as O,projectReviewResult as T,buildReviewResult as A}from"../../core/review/project.js";import{signalReviewActivity as p}from"../../core/review/signal.js";import{clearReviewSubmit as D,getReview as E,getReviewByCompanion as B,requestReviewSubmit as F,requireVisibleReview as u}from"../../core/review/store.js";import{ReviewOperationError as w}from"../../core/review/types.js";import{settleHumanRequest as v}from"../human/settle.js";import{retireCompanion as h}from"../companion-retire.js";import{companionHasQueuedWork as W,companionMidTurn as H,companionStillWorking as M}from"./companion.js";import{renderApprovalMessage as L}from"./deliver.js";function f(){return new Date().toISOString()}n(f,"now");function l(e,t){throw new w("review_terminal","review already has a terminal outcome",{state:e.state,requested:t,review:e})}n(l,"terminalConflict");function _(e){if(e.result_json===null)throw new Error(`approved review lacks its immutable result: ${e.review_id}`);return JSON.parse(e.result_json)}n(_,"approvedResult");function y(e){return`${e.review_dir}/result.json`}n(y,"resultPath");function P(e){return{project:n(async t=>{T(await u(t.request_id),e)},"project"),retire:n(async(t,o)=>{await h(t,o)},"retire")}}n(P,"effects");function Q(e){return{label:"review approved by the user",disposition:null,body_json:{review_id:e.review_id,result_path:y(e),body:L(e,y(e))}}}n(Q,"approvalNotification");function V(e,t,o){const a=o==="dismissed";return{label:a?"review dismissed":"review canceled",disposition:"human-canceled",body_json:{review_id:e.review_id,disposition:o,body:a?`The review of \`${e.document_name??e.file}\` was dismissed without approval.`:`The review of \`${e.document_name??e.file}\` was canceled${t===""?"":` \u2014 ${t}`}.`}}}n(V,"cancellationNotification");async function z(e,t,o){if(t.document_id===null||t.document_base===null)return;const a=n(async d=>await J(e,d),"reposFn"),i=await q(e,t.document_id),s=await k(e,{class:"person"},a),c=await C(e,s,i,{body:o},`edited in review ${t.review_id}`,t.document_base,{},a);if(!("stale"in c))return;const r=await $(e,i);if(!(r.length===1&&S(r[0].body_hash)===o))throw new w("stale_revision",`${t.document_name??i.name} changed since the review began; the reviewed text was not saved`,{review_id:t.review_id,document:t.document_name??i.name,base:t.document_base,heads_key:c.stale.heads_key,revision:c.stale.revision,next:`Read the current revision with \`crtr canvas read ${i.id}\`, then start a new review of it.`})}n(z,"saveReviewedDocument");async function b(e){if(e.state==="canceled")return l(e,"submit");if(e.state==="approved")return{outcome:"already_settled",record:e,result:_(e)};const t=N(e.file,"submit"),{result:o,resultJson:a}=await j(async c=>{const r=await u(e.review_id);if(r.state==="canceled")return l(r,"submit");if(r.state==="approved")return{result:_(r),resultJson:r.result_json};const d=await x(c,r.review_id),m=A(r,d,t.sha256,f());return m.changed&&await z(c,r,new TextDecoder("utf-8").decode(t.bytes)),{result:m,resultJson:JSON.stringify(m)}}),i=await v({request_id:e.review_id,outcome:"approved",result_json:a,notification:Q({...e,changed:o.changed}),post_commit:P(o)});i.outcome==="settled"&&p();const s=await u(e.review_id);return i.outcome==="already_settled"&&s.state!=="approved"?l(s,"submit"):{outcome:i.outcome,record:s,result:i.outcome==="settled"?o:_(s)}}n(b,"approveReview");async function de(e){const t=await u(e);return t.state==="canceled"?l(t,"submit"):t.state==="open"&&await M(t.companion_node_id)?(await F(e,f())&&p(),{outcome:"awaiting_companion",record:await u(e)}):b(t)}n(de,"submitReviewApproval");async function G(e,t){const o=await E(e);if(o===null||o.state!=="open"||o.submit_requested_at===null||await W(o.companion_node_id)||!t.atCompanionTurnEnd&&await H(o.companion_node_id))return null;try{return await b(o)}catch(a){throw a instanceof w&&a.code==="stale_revision"&&(await D(e),p()),a}}n(G,"completeRequestedSubmit");async function me(e){const t=await B(e);return t===null?null:G(t.review_id,{atCompanionTurnEnd:!0})}n(me,"completeCompanionRequestedSubmit");async function le(e,t={}){const o=await u(e);if(o.state==="approved")return l(o,"cancel");const a=f(),i=t.reason??"",s=t.disposition??"canceled",c=JSON.stringify({schema:"humanloop.cancel/v1",kind:"canceled",canceledAt:a,...i===""?{}:{reason:i},...t.actor===void 0?{}:{actor:t.actor},...s==="dismissed"?{disposition:s}:{}}),r=await v({request_id:e,outcome:s==="dismissed"?"dismissed":"canceled",result_json:c,notification:V(o,i,s),post_commit:{project:n(async m=>{O(await u(m.request_id),JSON.parse(c))},"project"),retire:n(async(m,R)=>{await h(m,R)},"retire")}});r.outcome==="settled"&&p();const d=await u(e);return r.outcome==="already_settled"&&d.state==="approved"?l(d,"cancel"):{outcome:r.outcome,record:d}}n(le,"cancelReview");export{le as cancelReview,me as completeCompanionRequestedSubmit,G as completeRequestedSubmit,de as submitReviewApproval};
+var g=Object.defineProperty;var n=(e,t)=>g(e,"name",{value:t,configurable:!0});import{withCanvasWrite as j}from"../../core/canvas/db.js";import{get as S}from"../../core/graph/bodies.js";import{documentById as q,editDocument as C,heads as $}from"../../core/graph/documents.js";import{readerFor as k}from"../../core/graph/reader.js";import{warmDocumentWatchers as J}from"../api/reader.js";import{checkoutsForCwd as x}from"../../core/graph/repo-sync/repos.js";import{snapshotCommentsLocked as N}from"../../core/review/comments.js";import{readReviewSource as O}from"../../core/review/document.js";import{projectReviewCancellation as T,projectReviewResult as D,buildReviewResult as A}from"../../core/review/project.js";import{signalReviewActivity as p}from"../../core/review/signal.js";import{clearReviewSubmit as E,getReview as W,getReviewByCompanion as B,requestReviewSubmit as F,requireVisibleReview as u}from"../../core/review/store.js";import{ReviewOperationError as w}from"../../core/review/types.js";import{settleHumanRequest as v}from"../human/settle.js";import{retireCompanion as h}from"../companion-retire.js";import{companionHasQueuedWork as H,companionMidTurn as M,companionStillWorking as L}from"./companion.js";import{renderApprovalMessage as P}from"./deliver.js";function f(){return new Date().toISOString()}n(f,"now");function l(e,t){throw new w("review_terminal","review already has a terminal outcome",{state:e.state,requested:t,review:e})}n(l,"terminalConflict");function _(e){if(e.result_json===null)throw new Error(`approved review lacks its immutable result: ${e.review_id}`);return JSON.parse(e.result_json)}n(_,"approvedResult");function y(e){return`${e.review_dir}/result.json`}n(y,"resultPath");function Q(e){return{project:n(async t=>{D(await u(t.request_id),e)},"project"),retire:n(async(t,o)=>{await h(t,o)},"retire")}}n(Q,"effects");function V(e){return{label:"review approved by the user",disposition:null,body_json:{review_id:e.review_id,result_path:y(e),body:P(e,y(e))}}}n(V,"approvalNotification");function z(e,t,o){const a=o==="dismissed";return{label:a?"review dismissed":"review canceled",disposition:"human-canceled",body_json:{review_id:e.review_id,disposition:o,body:a?`The review of \`${e.document_name??e.file}\` was dismissed without approval.`:`The review of \`${e.document_name??e.file}\` was canceled${t===""?"":` \u2014 ${t}`}.`}}}n(z,"cancellationNotification");async function G(e,t,o){if(t.document_id===null||t.document_base===null)return;const a=n(async d=>await x(e,d),"reposFn"),i=await q(e,t.document_id),s=await k(e,{class:"person"},a),c=await C(e,s,i,{body:o},`edited in review ${t.review_id}`,t.document_base,{},a);if(!("stale"in c))return;const r=await $(e,i);if(!(r.length===1&&S(r[0].body_hash)===o))throw new w("stale_revision",`${t.document_name??i.name} changed since the review began; the reviewed text was not saved`,{review_id:t.review_id,document:t.document_name??i.name,base:t.document_base,heads_key:c.stale.heads_key,revision:c.stale.revision,next:`Read the current revision with \`crtr canvas read ${i.id}\`, then start a new review of it.`})}n(G,"saveReviewedDocument");async function b(e){if(e.state==="canceled")return l(e,"submit");if(e.state==="approved")return{outcome:"already_settled",record:e,result:_(e)};const t=O(e.file,"submit");e.document_id!==null&&await J(e.document_id);const{result:o,resultJson:a}=await j(async c=>{const r=await u(e.review_id);if(r.state==="canceled")return l(r,"submit");if(r.state==="approved")return{result:_(r),resultJson:r.result_json};const d=await N(c,r.review_id),m=A(r,d,t.sha256,f());return m.changed&&await G(c,r,new TextDecoder("utf-8").decode(t.bytes)),{result:m,resultJson:JSON.stringify(m)}}),i=await v({request_id:e.review_id,outcome:"approved",result_json:a,notification:V({...e,changed:o.changed}),post_commit:Q(o)});i.outcome==="settled"&&p();const s=await u(e.review_id);return i.outcome==="already_settled"&&s.state!=="approved"?l(s,"submit"):{outcome:i.outcome,record:s,result:i.outcome==="settled"?o:_(s)}}n(b,"approveReview");async function le(e){const t=await u(e);return t.state==="canceled"?l(t,"submit"):t.state==="open"&&await L(t.companion_node_id)?(await F(e,f())&&p(),{outcome:"awaiting_companion",record:await u(e)}):b(t)}n(le,"submitReviewApproval");async function K(e,t){const o=await W(e);if(o===null||o.state!=="open"||o.submit_requested_at===null||await H(o.companion_node_id)||!t.atCompanionTurnEnd&&await M(o.companion_node_id))return null;try{return await b(o)}catch(a){throw a instanceof w&&a.code==="stale_revision"&&(await E(e),p()),a}}n(K,"completeRequestedSubmit");async function pe(e){const t=await B(e);return t===null?null:K(t.review_id,{atCompanionTurnEnd:!0})}n(pe,"completeCompanionRequestedSubmit");async function we(e,t={}){const o=await u(e);if(o.state==="approved")return l(o,"cancel");const a=f(),i=t.reason??"",s=t.disposition??"canceled",c=JSON.stringify({schema:"humanloop.cancel/v1",kind:"canceled",canceledAt:a,...i===""?{}:{reason:i},...t.actor===void 0?{}:{actor:t.actor},...s==="dismissed"?{disposition:s}:{}}),r=await v({request_id:e,outcome:s==="dismissed"?"dismissed":"canceled",result_json:c,notification:z(o,i,s),post_commit:{project:n(async m=>{T(await u(m.request_id),JSON.parse(c))},"project"),retire:n(async(m,R)=>{await h(m,R)},"retire")}});r.outcome==="settled"&&p();const d=await u(e);return r.outcome==="already_settled"&&d.state==="approved"?l(d,"cancel"):{outcome:r.outcome,record:d}}n(we,"cancelReview");export{we as cancelReview,pe as completeCompanionRequestedSubmit,K as completeRequestedSubmit,le as submitReviewApproval};

package/dist/daemon/run-notices.d.ts

@@ -47,3 +47,6 @@ export declare class RunNoticeOutbox {
 export declare function databaseDeps(): RunNoticeDeps;
 /** The daemon's outbox. */
 export declare function runNotices(): RunNoticeOutbox;
+/** Starts the daemon's run notices: the local append hook, plus the `run_event` signal so an append made by another process
+ *  also owes a notice. `appended` coalesces the two calls an append makes in its own process. Returns the stop function. */
+export declare function startRunNotices(outbox?: RunNoticeOutbox): Promise<() => void>;

package/dist/daemon/run-notices.js

@@ -1 +1 @@
-var h=Object.defineProperty;var u=(n,t)=>h(n,"name",{value:t,configurable:!0});import{openDb as d}from"../core/canvas/db.js";import{runNoticeTarget as p}from"../core/grants/mirror.js";import{emitEvent as w}from"../core/events/emit.js";import{describeOutboundFailure as m}from"../core/events/outbound.js";import{discardFile as _}from"../core/providers/token-cache.js";import{outboundToken as g}from"../core/providers/mint.js";const y="crtr:run_notice",A=[3e3,5e3],E=5e3,T=6e4,k=1e4;class l extends Error{static{u(this,"Fault")}retryable;status;where;constructor(t,e,s,o,a={step:"run_notice"}){super(t,o===void 0?void 0:{cause:o}),this.retryable=e,this.status=s,this.where=a}}class b{static{u(this,"RunNoticeOutbox")}deps;runs=new Map;constructor(t){this.deps=t}state(t){let e=this.runs.get(t);return e||(e={streams:0,owed:!1,inFlight:!1,timer:void 0,nextAllowedAt:0},this.runs.set(t,e)),e}forget(t,e){e.streams===0&&!e.owed&&!e.inFlight&&e.timer===void 0&&e.nextAllowedAt<=this.deps.now()&&this.runs.delete(t)}async streamOpened(t){const e=this.state(t);e.streams++,e.owed=!1,e.nextAllowedAt=0,e.timer&&(clearTimeout(e.timer),e.timer=void 0);let s=!1;return async o=>{if(s)return;s=!0,e.streams--;let a=null;try{a=await this.deps.latest(t)}catch{}e.streams===0&&a!==null&&a>o?await this.owe(t):this.forget(t,e)}}async appended(t){const e=this.runs.get(t);e&&e.streams>0||await this.owe(t)}async owe(t){let e;try{e=await this.deps.target(t)}catch{return}if(e===null)return;const s=this.state(t);s.streams>0||(s.owed=!0,this.schedule(t,s))}schedule(t,e){if(!e.owed||e.inFlight||e.timer!==void 0||e.streams>0)return;const s=Math.max(0,e.nextAllowedAt-this.deps.now());e.timer=setTimeout(()=>{e.timer=void 0,this.deliver(t,e)},s),e.timer.unref()}async deliver(t,e){if(!e.owed||e.streams>0){this.forget(t,e);return}const s=await this.deps.target(t),o=await this.deps.latest(t);if(s===null||o===null){e.owed=!1,this.forget(t,e);return}e.owed=!1,e.inFlight=!0;let a=0,r;try{for(;;){a++;try{await this.attempt(s,t,o),r=void 0;break}catch(i){r=i instanceof l?i:new l(i instanceof Error?i.message:String(i),!0);const c=this.deps.retryWaitsMs[a-1];if(!r.retryable||c===void 0)break;if(await new Promise(f=>{setTimeout(f,c).unref()}),e.streams>0){r=void 0;break}}}}finally{e.inFlight=!1}r?(e.nextAllowedAt=this.deps.now()+this.deps.failGapMs,w({level:"error",event:"run_notice.failed",fields:{run_id:t,grantee:s.grantee,attempts:a,latest_sequence:o,...r.status===void 0?{}:{status:r.status},error:r.message,...m(r.cause,r.where.url===void 0&&r.where.step==="run_notice"?{step:"run_notice",url:s.url}:r.where)}})):e.streams===0&&(e.nextAllowedAt=this.deps.now()+this.deps.ackGapMs),this.schedule(t,e),this.forget(t,e)}async attempt(t,e,s){for(let o=0;;o++){let a;try{a=await this.deps.token(t)}catch(i){throw new l(`token mint failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"notice_token"})}let r;try{r=await this.deps.fetch(t.url,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(k),headers:{Authorization:`Bearer ${a.access_token}`,"Content-Type":"application/json"},body:JSON.stringify({run_id:e,latest_sequence:s})})}catch(i){throw new l(`request failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"run_notice",url:t.url})}if(r.body?.cancel().catch(()=>{}),r.status>=200&&r.status<300)return;if(r.status===401){if(this.deps.discard(a.key),o===0)continue;throw new l("the app refused the notice token twice (401)",!0,401)}throw r.status>=500?new l(`the app answered ${r.status}`,!0,r.status):new l(`contract_fault: the app answered ${r.status}`,!1,r.status)}}}function x(){return{target:u(async n=>{const t=await(await d()).prepare("SELECT grant_id FROM nodes WHERE node_id = ? AND run_id = ?").get(n,n);return t?.grant_id?await p(t.grant_id):null},"target"),latest:u(async n=>{const t=await(await d()).prepare("SELECT event_cursor FROM nodes WHERE node_id = ? AND run_id = ?").get(n,n);return t?.event_cursor==null?null:t.event_cursor-1},"latest"),token:u(async n=>{const t=await g({audience:n.url,sub:n.sub,grantee:n.grantee,initiating_app:n.grantee,scope:[y]});return{access_token:t.access_token,key:t.key}},"token"),discard:_,fetch:u((n,t)=>fetch(n,t),"fetch"),now:Date.now,retryWaitsMs:A,ackGapMs:E,failGapMs:T}}u(x,"databaseDeps");let M;function P(){return M??=new b(x())}u(P,"runNotices");export{E as RUN_NOTICE_ACK_GAP_MS,k as RUN_NOTICE_ATTEMPT_TIMEOUT_MS,T as RUN_NOTICE_FAIL_GAP_MS,A as RUN_NOTICE_RETRY_WAITS_MS,y as RUN_NOTICE_SCOPE,b as RunNoticeOutbox,x as databaseDeps,P as runNotices};
+var w=Object.defineProperty;var u=(s,t)=>w(s,"name",{value:t,configurable:!0});import{openDb as d}from"../core/canvas/db.js";import{onRunEventAppended as f}from"../core/runs/events.js";import{brokerSignalBus as m}from"./broker-signals/bus.js";import{runNoticeTarget as _}from"../core/grants/mirror.js";import{emitEvent as p}from"../core/events/emit.js";import{describeOutboundFailure as g}from"../core/events/outbound.js";import{discardFile as y}from"../core/providers/token-cache.js";import{outboundToken as A}from"../core/providers/mint.js";const E="crtr:run_notice",k=[3e3,5e3],b=5e3,T=6e4,x=1e4;class c extends Error{static{u(this,"Fault")}retryable;status;where;constructor(t,e,n,o,a={step:"run_notice"}){super(t,o===void 0?void 0:{cause:o}),this.retryable=e,this.status=n,this.where=a}}class N{static{u(this,"RunNoticeOutbox")}deps;runs=new Map;constructor(t){this.deps=t}state(t){let e=this.runs.get(t);return e||(e={streams:0,owed:!1,inFlight:!1,timer:void 0,nextAllowedAt:0},this.runs.set(t,e)),e}forget(t,e){e.streams===0&&!e.owed&&!e.inFlight&&e.timer===void 0&&e.nextAllowedAt<=this.deps.now()&&this.runs.delete(t)}async streamOpened(t){const e=this.state(t);e.streams++,e.owed=!1,e.nextAllowedAt=0,e.timer&&(clearTimeout(e.timer),e.timer=void 0);let n=!1;return async o=>{if(n)return;n=!0,e.streams--;let a=null;try{a=await this.deps.latest(t)}catch{}e.streams===0&&a!==null&&a>o?await this.owe(t):this.forget(t,e)}}async appended(t){const e=this.runs.get(t);e&&e.streams>0||await this.owe(t)}async owe(t){let e;try{e=await this.deps.target(t)}catch{return}if(e===null)return;const n=this.state(t);n.streams>0||(n.owed=!0,this.schedule(t,n))}schedule(t,e){if(!e.owed||e.inFlight||e.timer!==void 0||e.streams>0)return;const n=Math.max(0,e.nextAllowedAt-this.deps.now());e.timer=setTimeout(()=>{e.timer=void 0,this.deliver(t,e)},n),e.timer.unref()}async deliver(t,e){if(!e.owed||e.streams>0){this.forget(t,e);return}const n=await this.deps.target(t),o=await this.deps.latest(t);if(n===null||o===null){e.owed=!1,this.forget(t,e);return}e.owed=!1,e.inFlight=!0;let a=0,r;try{for(;;){a++;try{await this.attempt(n,t,o),r=void 0;break}catch(i){r=i instanceof c?i:new c(i instanceof Error?i.message:String(i),!0);const l=this.deps.retryWaitsMs[a-1];if(!r.retryable||l===void 0)break;if(await new Promise(h=>{setTimeout(h,l).unref()}),e.streams>0){r=void 0;break}}}}finally{e.inFlight=!1}r?(e.nextAllowedAt=this.deps.now()+this.deps.failGapMs,p({level:"error",event:"run_notice.failed",fields:{run_id:t,grantee:n.grantee,attempts:a,latest_sequence:o,...r.status===void 0?{}:{status:r.status},error:r.message,...g(r.cause,r.where.url===void 0&&r.where.step==="run_notice"?{step:"run_notice",url:n.url}:r.where)}})):e.streams===0&&(e.nextAllowedAt=this.deps.now()+this.deps.ackGapMs),this.schedule(t,e),this.forget(t,e)}async attempt(t,e,n){for(let o=0;;o++){let a;try{a=await this.deps.token(t)}catch(i){throw new c(`token mint failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"notice_token"})}let r;try{r=await this.deps.fetch(t.url,{method:"POST",redirect:"manual",signal:AbortSignal.timeout(x),headers:{Authorization:`Bearer ${a.access_token}`,"Content-Type":"application/json"},body:JSON.stringify({run_id:e,latest_sequence:n})})}catch(i){throw new c(`request failed: ${i instanceof Error?i.message:String(i)}`,!0,void 0,i,{step:"run_notice",url:t.url})}if(r.body?.cancel().catch(()=>{}),r.status>=200&&r.status<300)return;if(r.status===401){if(this.deps.discard(a.key),o===0)continue;throw new c("the app refused the notice token twice (401)",!0,401)}throw r.status>=500?new c(`the app answered ${r.status}`,!0,r.status):new c(`contract_fault: the app answered ${r.status}`,!1,r.status)}}}function M(){return{target:u(async s=>{const t=await(await d()).prepare("SELECT grant_id FROM nodes WHERE node_id = ? AND run_id = ?").get(s,s);return t?.grant_id?await _(t.grant_id):null},"target"),latest:u(async s=>{const t=await(await d()).prepare("SELECT event_cursor FROM nodes WHERE node_id = ? AND run_id = ?").get(s,s);return t?.event_cursor==null?null:t.event_cursor-1},"latest"),token:u(async s=>{const t=await A({audience:s.url,sub:s.sub,grantee:s.grantee,initiating_app:s.grantee,scope:[E]});return{access_token:t.access_token,key:t.key}},"token"),discard:y,fetch:u((s,t)=>fetch(s,t),"fetch"),now:Date.now,retryWaitsMs:k,ackGapMs:b,failGapMs:T}}u(M,"databaseDeps");let S;function v(){return S??=new N(M())}u(v,"runNotices");async function q(s=v()){f(e=>s.appended(e));const t=await m.subscribe(e=>{s.appended(e).catch(n=>p({level:"error",event:"runs.notice_hook_failed",error:n}))},{kinds:["run_event"]});return()=>{f(void 0),t()}}u(q,"startRunNotices");export{b as RUN_NOTICE_ACK_GAP_MS,x as RUN_NOTICE_ATTEMPT_TIMEOUT_MS,T as RUN_NOTICE_FAIL_GAP_MS,k as RUN_NOTICE_RETRY_WAITS_MS,E as RUN_NOTICE_SCOPE,N as RunNoticeOutbox,M as databaseDeps,v as runNotices,q as startRunNotices};