@north-light/cli 0.3.561 → 0.3.562

package/dist/daemon/due.js

@@ -1,11 +1,12 @@
-var D=Object.defineProperty;var a=(n,e)=>D(n,"name",{value:e,configurable:!0});import{openDb as f}from"../core/canvas/db.js";import{isPidPresent as R}from"../core/canvas/pid.js";import{isProfilePaused as p}from"../core/profiles/manifest.js";import{hasUnseenWakeMail as E}from"../core/canvas/deliveries.js";import{listPendingProviderRetries as T}from"../core/canvas/recovery.js";import{listSubmitRequestedRequests as x}from"../core/human/requests.js";import{storageConfig as I}from"../core/storage/client.js";import{emitEvent as A}from"../core/events/emit.js";import{wantsExecution as L}from"./reconcilers/node-lifecycle/wants-execution.js";import{dueAtMs as O}from"./reconcilers/fault-retry-backstop.js";import{UNUSED_UPLOAD_TTL_MS as y}from"./reconcilers/storage-due-work.js";import{busyOnlyLanes as M,collectDue as b,declareBusyOnly as l,dueSourceLanes as g,registerDueSource as r}from"./due-sources.js";import{filterAsync as h}from"../core/async-array.js";const c=a(async(n,...e)=>await(await f()).prepare(n).all(...e),"rows"),s=a(async n=>await(await f()).prepare(n).get(),"first"),o=a(n=>{if(n==null)return null;const e=Date.parse(n);return Number.isFinite(e)?e:null},"iso"),_=a(n=>{let e=null;for(const t of n)t!==null&&(e===null||t<e)&&(e=t);return e},"earliest"),N="node_id NOT IN (SELECT node_id FROM warm_pool)";let m=!1;async function S(){if(m)return;m=!0,l("brokerSupervision","acts only on live broker processes, which already make busy true"),l("controllerDeath","acts only on live controller processes"),l("pollRunObservers","follows live pids only"),l("bashDeadline","acts only on running bash jobs, which already make busy true"),l("storageMaintenance","retention and placement housekeeping: never time-critical, and it must not wake or hold a sleeping runtime"),r("nodeLifecycle",{nextDue:a(async()=>_((await h(await c(`SELECT node_id, status, intent, profile_id, respawn_not_before FROM node_rows
+var D=Object.defineProperty;var a=(n,e)=>D(n,"name",{value:e,configurable:!0});import{openDb as E}from"../core/canvas/db.js";import{isPidPresent as R}from"../core/canvas/pid.js";import{isProfilePaused as p}from"../core/profiles/manifest.js";import{hasUnseenWakeMail as f}from"../core/canvas/deliveries.js";import{listPendingProviderRetries as x}from"../core/canvas/recovery.js";import{listSubmitRequestedRequests as g}from"../core/human/requests.js";import{storageConfig as T}from"../core/storage/client.js";import{emitEvent as A}from"../core/events/emit.js";import{wantsExecution as m}from"./reconcilers/node-lifecycle/wants-execution.js";import{dueAtMs as I}from"./reconcilers/fault-retry-backstop.js";import{UNUSED_UPLOAD_TTL_MS as O}from"./reconcilers/storage-due-work.js";import{RECIPE_HAS_CONTENT_SQL as y}from"./api/environment-of.js";import{busyOnlyLanes as M,collectDue as b,declareBusyOnly as u,dueSourceLanes as h,registerDueSource as i}from"./due-sources.js";import{filterAsync as v}from"../core/async-array.js";const _=a(async(n,...e)=>await(await E()).prepare(n).all(...e),"rows"),s=a(async n=>await(await E()).prepare(n).get(),"first"),o=a(n=>{if(n==null)return null;const e=Date.parse(n);return Number.isFinite(e)?e:null},"iso"),c=a(n=>{let e=null;for(const t of n)t!==null&&(e===null||t<e)&&(e=t);return e},"earliest"),N="node_id NOT IN (SELECT node_id FROM warm_pool)";let L=!1;async function S(){if(L)return;L=!0,u("brokerSupervision","acts only on live broker processes, which already make busy true"),u("controllerDeath","acts only on live controller processes"),u("pollRunObservers","follows live pids only"),u("bashDeadline","acts only on running bash jobs, which already make busy true"),u("storageMaintenance","retention and placement housekeeping: never time-critical, and it must not wake or hold a sleeping runtime"),u("ledgerSandbox","bills awake time of executions that are busy or just exited; a sleeping runtime owes nothing until it wakes, and the lane must not wake it"),u("ledgerStorage","monthly storage billing: never time-critical, and it must not wake or hold a sleeping runtime"),u("ledgerR2","moves closed months to R2: housekeeping that must not wake or hold a sleeping runtime"),i("nodeLifecycle",{nextDue:a(async()=>c((await v(await _(`SELECT node_id, status, intent, profile_id, respawn_not_before FROM node_rows
         WHERE status IN ('active', 'idle') AND kind != 'human' AND ${N}
-          AND pi_pid IS NULL AND respawn_not_before IS NOT NULL`),async e=>!p(e.profile_id)&&await L({node_id:e.node_id,status:e.status,intent:e.intent}))).map(e=>o(e.respawn_not_before))),"nextDue")}),r("nodeDeadline",{nextDue:a(async()=>_((await c(`SELECT deadline_at, profile_id FROM node_rows
-        WHERE deadline_at IS NOT NULL AND outcome_at IS NULL AND status IN ('active', 'idle') AND kind != 'human'`)).filter(e=>!p(e.profile_id)).map(e=>o(e.deadline_at))),"nextDue")}),r("structuredRequests",{nextDue:a(async e=>await s("SELECT 1 FROM structured_requests WHERE state != 'pending' AND effects_done_at IS NULL LIMIT 1")!==void 0?e:(await s("SELECT MIN(deadline_at) AS at FROM structured_requests WHERE state = 'pending' AND deadline_at IS NOT NULL"))?.at??null,"nextDue")}),r("cronLane",{nextDue:a(async()=>{const e=(await c(`SELECT fire_at, profile FROM crons
+          AND pi_pid IS NULL AND respawn_not_before IS NOT NULL`),async e=>!p(e.profile_id)&&await m({node_id:e.node_id,status:e.status,intent:e.intent}))).map(e=>o(e.respawn_not_before))),"nextDue")}),i("nodeDeadline",{nextDue:a(async()=>c((await _(`SELECT deadline_at, profile_id FROM node_rows
+        WHERE deadline_at IS NOT NULL AND outcome_at IS NULL AND status IN ('active', 'idle') AND kind != 'human'`)).filter(e=>!p(e.profile_id)).map(e=>o(e.deadline_at))),"nextDue")}),i("structuredRequests",{nextDue:a(async e=>await s("SELECT 1 FROM structured_requests WHERE state != 'pending' AND effects_done_at IS NULL LIMIT 1")!==void 0?e:(await s("SELECT MIN(deadline_at) AS at FROM structured_requests WHERE state = 'pending' AND deadline_at IS NOT NULL"))?.at??null,"nextDue")}),i("recipeImage",{nextDue:a(async e=>c((await _(`SELECT image_building_at FROM environments
+        WHERE kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${y}`)).map(t=>t.image_building_at===null?e:Math.max(e,o(t.image_building_at)??e))),"nextDue")}),i("cronLane",{nextDue:a(async()=>{const e=(await _(`SELECT fire_at, profile FROM crons
           WHERE state = 'active' AND run_state = 'idle'
-            AND NOT (held = 1 AND recur IS NULL AND expires_at IS NULL)`)).filter(i=>!p(i.profile)).map(i=>o(i.fire_at)),t=(await c("SELECT expires_at FROM crons WHERE expires_at IS NOT NULL AND run_state = 'idle'")).map(i=>o(i.expires_at));return _([...e,...t])},"nextDue")});const n=a(async e=>async()=>(await s(`SELECT MIN(next_attempt_at) AS at FROM ${e} WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at IS NOT NULL`))?.at??null,"attempts");r("deliveryLane",{nextDue:await n("deliveries")}),r("actionLane",{nextDue:await n("daemon_actions")}),r("pendingReviewSubmit",{nextDue:a(async e=>(await x()).length>0?e:null,"nextDue")}),r("storageDueWork",{nextDue:a(async e=>{if(I()===null)return null;if(await s("SELECT 1 FROM uploads WHERE delete_pending_since IS NOT NULL LIMIT 1")!==void 0||await s("SELECT 1 FROM shares WHERE status = 'off' AND object_deleted_at IS NULL LIMIT 1")!==void 0)return e;const t=(await s("SELECT MIN(expires_at) AS at FROM shares WHERE status = 'on' AND expires_at IS NOT NULL"))?.at,i=(await s(`SELECT MIN(created_at) AS at FROM uploads u WHERE delete_pending_since IS NULL
-          AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`))?.at,u=o(i);return _([o(t),u===null?null:u+y])},"nextDue")}),r("faultRetryBackstop",{nextDue:a(async()=>_((await T()).map(e=>O(e))),"nextDue")})}a(S,"installDueSources");const w=12e4;async function U(n){const e=await c(`SELECT n.node_id, n.status, n.intent, n.profile_id, n.pi_pid, n.frozen_at, n.respawn_not_before, n.created, n.launched_at,
+            AND NOT (held = 1 AND recur IS NULL AND expires_at IS NULL)`)).filter(r=>!p(r.profile)).map(r=>o(r.fire_at)),t=(await _("SELECT expires_at FROM crons WHERE expires_at IS NOT NULL AND run_state = 'idle'")).map(r=>o(r.expires_at));return c([...e,...t])},"nextDue")});const n=a(async e=>async()=>(await s(`SELECT MIN(next_attempt_at) AS at FROM ${e} WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at IS NOT NULL`))?.at??null,"attempts");i("deliveryLane",{nextDue:await n("deliveries")}),i("actionLane",{nextDue:await n("daemon_actions")}),i("pendingReviewSubmit",{nextDue:a(async e=>(await g()).length>0?e:null,"nextDue")}),i("storageDueWork",{nextDue:a(async e=>{if(T()===null)return null;if(await s("SELECT 1 FROM uploads WHERE delete_pending_since IS NOT NULL LIMIT 1")!==void 0||await s("SELECT 1 FROM shares WHERE status = 'off' AND object_deleted_at IS NULL LIMIT 1")!==void 0)return e;const t=(await s("SELECT MIN(expires_at) AS at FROM shares WHERE status = 'on' AND expires_at IS NOT NULL"))?.at,r=(await s(`SELECT MIN(created_at) AS at FROM uploads u WHERE delete_pending_since IS NULL
+          AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`))?.at,l=o(r);return c([o(t),l===null?null:l+O])},"nextDue")}),i("faultRetryBackstop",{nextDue:a(async()=>c((await x()).map(e=>I(e))),"nextDue")})}a(S,"installDueSources");const w=12e4;async function C(n){const e=await _(`SELECT n.node_id, n.status, n.intent, n.profile_id, n.pi_pid, n.frozen_at, n.respawn_not_before, n.created, n.launched_at,
             n.turn_open_at, n.turn_closed_at,
             (SELECT x.started_at FROM executions x WHERE x.node_id = n.node_id AND x.state != 'exited') AS exec_started_at
        FROM node_rows n
-      WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${N}`);for(const t of e){if(p(t.profile_id)||!await L({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&R(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=o(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<w||await E(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null||t.status==="active")return!0;const u=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>o(d)??0));if(u>0&&n-u<w||t.status==="idle"&&await E(t.node_id))return!0}return!1}a(U,"nodesOweWork");async function X(n=Date.now()){await S();const{dueNow:e,nextDueMs:t}=await b(n,(u,d)=>{A({level:"warn",event:"daemon.due.source_failed",error:d,fields:{lane:u}})});return{busy:e||await U(n)||await s("SELECT 1 FROM crons WHERE run_state = 'running' LIMIT 1")!==void 0||await s("SELECT 1 FROM jobs j JOIN objects o ON o.id = j.object WHERE j.status = 'running' AND o.deleted_at IS NULL LIMIT 1")!==void 0,next_due_at:t===null?null:new Date(t).toISOString()}}a(X,"dueReport");async function K(){return await S(),[...g(),...M()]}a(K,"coveredLanes");export{w as START_GRACE_MS,K as coveredLanes,X as dueReport,S as installDueSources};
+      WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${N}`);for(const t of e){if(p(t.profile_id)||!await m({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&R(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=o(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<w||await f(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null||t.status==="active")return!0;const l=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>o(d)??0));if(l>0&&n-l<w||t.status==="idle"&&await f(t.node_id))return!0}return!1}a(C,"nodesOweWork");async function K(n=Date.now()){await S();const{dueNow:e,nextDueMs:t}=await b(n,(l,d)=>{A({level:"warn",event:"daemon.due.source_failed",error:d,fields:{lane:l}})});return{busy:e||await C(n)||await s("SELECT 1 FROM crons WHERE run_state = 'running' LIMIT 1")!==void 0||await s("SELECT 1 FROM jobs j JOIN objects o ON o.id = j.object WHERE j.status = 'running' AND o.deleted_at IS NULL LIMIT 1")!==void 0,next_due_at:t===null?null:new Date(t).toISOString()}}a(K,"dueReport");async function V(){return await S(),[...h(),...M()]}a(V,"coveredLanes");export{w as START_GRACE_MS,V as coveredLanes,K as dueReport,S as installDueSources};

package/dist/daemon/fleet.d.ts

@@ -55,16 +55,16 @@ export declare class DaemonFleet implements FleetRegistry {
     constructor(opts: DaemonFleetOpts);
     epoch(): string;
     register(nodeId: string, handle: HostHandle): void;
-    has(nodeId: string): boolean;
-    isClaimed(nodeId: string): boolean;
+    has(nodeId: string): Promise<boolean>;
+    isClaimed(nodeId: string): Promise<boolean>;
     get(nodeId: string): FleetEntry | undefined;
     size(): number;
-    occupancy(): number;
-    reserveSlot(nodeId: string): boolean;
+    occupancy(): Promise<number>;
+    reserveSlot(nodeId: string): Promise<boolean>;
     releaseReservation(nodeId: string): void;
     entries(): Iterable<[string, FleetEntry]>;
-    interruptForProfilePause(nodeId: string, pid?: number | undefined): void;
-    hasProfilePauseInterruption(nodeId: string): boolean;
+    interruptForProfilePause(nodeId: string, pid?: number | undefined): Promise<void>;
+    hasProfilePauseInterruption(nodeId: string): Promise<boolean>;
     forget(nodeId: string): void;
     exitsSettled(): Promise<void>;
     onChildExit(nodeId: string, status: FleetExitStatus, executionId?: string): Promise<void>;

package/dist/daemon/fleet.js

@@ -1,21 +1,21 @@
-var g=Object.defineProperty;var l=(i,e)=>g(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as x}from"../core/runtime/close.js";import{readFault as E,readInterruptedTurn as P,readTransportPoisonVerdict as N}from"../core/canvas/recovery.js";import{hasPendingWake as S}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as C,MAX_RESPAWN_ATTEMPTS as F,OFFLINE_RESPAWN_RETRY_MS as M,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{isStartRefusal as D}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as L}from"../core/runtime/fleet.js";import{isRecoveryHeld as W}from"../core/runtime/recovery-hold.js";async function _(i,e){const t=await E(i.node_id),r=t===null?"":`The broker reported a ${t.kind} fault on ${t.link} (${t.op}) before it exited: ${t.message}
+var P=Object.defineProperty;var l=(i,e)=>P(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as R}from"../core/canvas/index.js";import{revokeExitedNodeToken as T}from"../core/canvas/node-tokens.js";import{PROFILE_PAUSE_STOP_REASON as _,absorbProfilePauseStop as N,endExecution as y,hasLaunchingExecution as S,hasProfilePauseStop as F,hasRunningRemoteExecution as b,liveExecutionForNode as g,liveExecutionNodeIds as C,markProfilePauseStop as k,recordExecutionExit as O,takeObservedExit as M}from"../core/canvas/executions.js";import{isProfilePaused as z}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as A,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as L}from"../core/runtime/close.js";import{readFault as B,readInterruptedTurn as D,readTransportPoisonVerdict as W}from"../core/canvas/recovery.js";import{hasPendingWake as q}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as G,MAX_RESPAWN_ATTEMPTS as H,OFFLINE_RESPAWN_RETRY_MS as Y,recordRespawnPolicy as U,respawnBackoffMs as V}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as j}from"../core/runtime/connectivity.js";import{runnerForExecution as K}from"../core/runtime/runner/index.js";import{reviveNode as X}from"../core/runtime/revive.js";import{isStartRefusal as J}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as Q}from"../core/runtime/fleet.js";import{isRecoveryHeld as Z}from"../core/runtime/recovery-hold.js";async function $(i,e){const r=await B(i.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
 
 `,s=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${i.name}\` (${i.kind}) never started.
 
 Its pi vehicle exited before the session came up (no pi_session_id was ever recorded), so the node produced no output. ${s}
 
-`+r+`Inspect canonical event diagnostics with \`crtr sys logs --node ${i.node_id}\`.
+`+t+`Inspect canonical event diagnostics with \`crtr sys logs --node ${i.node_id}\`.
 
-If the work still needs doing, re-spawn it.`}l(_,"bootFailureBody");async function y(i,e){await m(i.node_id,await _(i,e),{from:i.node_id})}l(y,"surfaceBootFailure");function b(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
+If the work still needs doing, re-spawn it.`}l($,"bootFailureBody");async function E(i,e){await m(i.node_id,await $(i,e),{from:i.node_id})}l(E,"surfaceBootFailure");function x(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
 
-Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function q(i){const e=await N(i.node_id),t=e===null?"":`
+Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(x,"outOfMemoryBody");async function I(i){const e=await W(i.node_id),r=e===null?"":`
 
-Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,r=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
+Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
 
-Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+t+`
+Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
 
-Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(q,"surfaceCrashLoop");async function G(i,e){const t=e instanceof Error?e.message:String(e),r=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
+Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(I,"surfaceCrashLoop");async function ee(i,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
 
-The daemon refused this recovery before starting a broker: ${t}
+The daemon refused this recovery before starting a broker: ${r}
 
-The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(G,"surfaceRecoveryFailure");function H(i){return i?.cycle_pending!==!0}l(H,"retryResumeMode");function Y(i,e,t=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":R(i)?"dormant":i.lifecycle==="resident"&&!e&&!t&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?H(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(Y,"classifyDeadNode");const V=3e4;class ue{static{l(this,"DaemonFleet")}#o;#l;#i;#c;#e=new Map;#t=new Set;#n=new Set;#r=new Map;#u=Number.NEGATIVE_INFINITY;constructor(e){this.#o=e.epoch,this.#l=e.enqueue,this.#i=e.now??Date.now,this.#c=e.probeOnline??(()=>A())}epoch(){return this.#o}register(e,t){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(t.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#r.delete(e),this.#e.set(e,{pid:t.pid,handle:t})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#t.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#t.size}reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:t}=L();if(this.occupancy()<t)return this.#t.add(e),!0;const r=this.#i();return r-this.#u>=V&&(this.#u=r,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:t}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,t=this.#e.get(e)?.pid){if(t!==void 0){this.#n.add(e);try{process.kill(t,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#n.has(e)}forget(e){this.#e.delete(e),this.#t.delete(e),this.#r.delete(e)}#s=new Set;async exitsSettled(){for(;this.#s.size>0;)await Promise.allSettled([...this.#s])}async onChildExit(e,t,r){const s=this.#f(e,t,r);this.#s.add(s);try{await s}finally{this.#s.delete(s)}}async#f(e,t,r){r&&await $(r),this.#e.delete(e),this.#l(async()=>{const s=await w(e),a=s?.launched_at==null?Number.NaN:Date.parse(s.launched_at),n=Number.isFinite(a)?Math.max(0,this.#i()-a):null,d=s?.respawn_failures??0;s?.status==="done"||s?.status==="canceled"||s?.status==="dead"?this.#r.delete(e):this.#r.set(e,t),this.#h(e,t,n,d)})}async deliverExit(e,t){await this.onChildExit(e,t)}async applyDeadRowPolicy(e){const t=this.#r.get(e);try{await this.#d(e,t)}finally{this.#r.get(e)===t&&this.#r.delete(e)}}async terminalizeIfStillNonTerminal(e,t,r,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#a(e,n,{event:t,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>G(n,r),"surface")}),!0)}dispose(){}#h(e,t,r,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:t.code,signal:t.signal,uptime_ms:r,consecutive_failures:s}})}async#d(e,t){return this.#p(e,t)}async#p(e,t){const r=await w(e);if(r!==null&&T(r.profile_id))return await h(e),"forget";if(this.#n.delete(e))return await h(e),"forget";if(W(e))return"dormant";const s=(await P(e))?.interrupted===!0;if(t?.oomKilled===!0&&r!==null&&r.status!=="done"&&r.status!=="canceled"&&r.status!=="dead")return await h(e),await this.#a(e,r,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(r)},wake:`Child out of memory \u2014 ${p(r)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(r),{from:e})},"surface")}),"forget";const a=Y(r,s,await S(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#a(e,r,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await _(r,t)},wake:`Child crashed \u2014 ${p(r)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(r,t),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#m(e,r,a,s);break}return a}async#m(e,t,r,s){const a=await k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#c()){await C(e,n,this.#i()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,retry_in_ms:M,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await O(a,this.#i()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=F};if(f.exhausted){const c=`Child crashed \u2014 ${p(t)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#a(e,t,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...t.pi_session_id!=null?{surface:l(()=>q(t),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:r==="respawn-resume",wakeReason:r==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(D(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:r}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#a(e,t,r){u({level:"error",event:r.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:r.reason,outcome:{detail:r.detail}});try{await(r.surface??(()=>y(t)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await x(e,"notice.crash",r.wake,{reason:"child-crashed",child:e})}}export{ue as DaemonFleet,Y as classifyDeadNode,H as retryResumeMode,y as surfaceBootFailure};
+The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(ee,"surfaceRecoveryFailure");function te(i){return i?.cycle_pending!==!0}l(te,"retryResumeMode");function re(i,e,r=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":A(i)?"dormant":i.lifecycle==="resident"&&!e&&!r&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?te(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(re,"classifyDeadNode");const ie=3e4;class Ee{static{l(this,"DaemonFleet")}#a;#n;#r;#o;#e=new Map;#t=new Set;#l=Number.NEGATIVE_INFINITY;constructor(e){this.#a=e.epoch,this.#n=e.enqueue,this.#r=e.now??Date.now,this.#o=e.probeOnline??(()=>j())}epoch(){return this.#a}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}async has(e){return this.#e.has(e)||await b(e)}async isClaimed(e){return this.#e.has(e)||this.#t.has(e)||await b(e)||await S(e,this.#a)}get(e){return this.#e.get(e)}size(){return this.#e.size}async#c(){const e=await C();for(const r of this.#e.keys())e.add(r);for(const r of this.#t)e.add(r);return e}async occupancy(){return(await this.#c()).size}async reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=Q(),t=await this.#c();if(this.#t.has(e)||this.#e.has(e))return!0;if(t.delete(e),t.size<r)return this.#t.has(e)||this.#t.add(e),!0;const s=this.#r();return s-this.#l>=ie&&(this.#l=s,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:t.size,hard_cap:r}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}async interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r===void 0){const t=await g(e);if(t===null||t.runner==="process"||t.state!=="running")return;await k(e),(await K(t.runner).stop(t,{reason:_})).confirmed&&await y(t.executionId,{reason:_});return}await k(e);try{process.kill(r,"SIGTERM")}catch{}}async hasProfilePauseInterruption(e){return await F(e)}forget(e){this.#e.delete(e),this.#t.delete(e)}#i=new Set;async exitsSettled(){for(;this.#i.size>0;)await Promise.allSettled([...this.#i])}async onChildExit(e,r,t){const s=this.#u(e,r,t);this.#i.add(s);try{await s}finally{this.#i.delete(s)}}async#u(e,r,t){const s=t??(await g(e))?.executionId;t&&await T(t),this.#e.delete(e),s!==void 0&&(await O(s,r),await y(s,{reason:"exited"})),this.#n(async()=>{const a=await w(e),n=a?.launched_at==null?Number.NaN:Date.parse(a.launched_at),d=Number.isFinite(n)?Math.max(0,this.#r()-n):null,f=a?.respawn_failures??0;this.#f(e,r,d,f)})}async deliverExit(e,r){await this.onChildExit(e,r)}async applyDeadRowPolicy(e){await this.#h(e,await M(e))}async terminalizeIfStillNonTerminal(e,r,t,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#s(e,n,{event:r,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>ee(n,t),"surface")}),!0)}dispose(){}#f(e,r,t,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:s}})}async#h(e,r){return this.#d(e,r)}async#d(e,r){const t=await w(e);if(t!==null&&z(t.profile_id))return await h(e),"forget";if(await N(e))return await h(e),"forget";if(await Z(e))return"dormant";const s=(await D(e))?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return await h(e),await this.#s(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:x(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,x(t),{from:e})},"surface")}),"forget";const a=re(t,s,await q(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#s(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await $(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>E(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#p(e,t,a,s);break}return a}async#p(e,r,t,s){const a=await R(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#o()){await G(e,n,this.#r()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:Y,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await U(a,this.#r()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=H};if(f.exhausted){const c=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#s(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...r.pi_session_id!=null?{surface:l(()=>I(r),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:V(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await X(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(J(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#s(e,r,t){u({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>E(r)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await L(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{Ee as DaemonFleet,re as classifyDeadNode,te as retryResumeMode,E as surfaceBootFailure};

package/dist/daemon/park-activity.d.ts

@@ -1,4 +1,5 @@
 import type { NodeMeta } from '../core/canvas/types.js';
+import type { ViewerProbe } from '../core/runtime/placement.js';
 import type { PendingPark } from './park-pending.js';
 /** A main-session write after parking was armed is durable evidence of live
  * input even when the turn has since settled and the viewer detached. */
@@ -6,4 +7,4 @@ export declare function mainSessionActivitySince(sessionFile: string | null | un
 /** One verdict shared by normal completion and grace expiry. A broker records
  * accepted live input against the process-local marker before it drives Pi, so
  * the verdict cannot race a viewer, live delivery, or durable wake. */
-export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark): Promise<boolean>;
+export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark, viewer?: ViewerProbe): Promise<boolean>;

package/dist/daemon/park-activity.js

@@ -1 +1 @@
-var n=Object.defineProperty;var e=(r,t)=>n(r,"name",{value:t,configurable:!0});import{conversationIdFromPath as o,conversationStore as f,ConversationNotFoundError as c}from"../core/conversation-store/index.js";import{emitEvent as s}from"../core/events/emit.js";import{hasWakeMailSince as u}from"../core/canvas/deliveries.js";import{readLiveTurn as m}from"../core/canvas/recovery.js";import{parkEvent as l}from"./reconcilers/live-obligation.js";async function v(r,t){if(r==null)return!1;const a=o(r);if(a===null)return!1;try{return(await f().header(a)).mtime>=t}catch(i){return i instanceof c?!1:(s({level:"warn",event:"broker.park.session_activity_scan_failed",error:i}),!0)}}e(v,"mainSessionActivitySince");async function _(r,t,a){return!a.activity&&(await m(r))?.busy!==!0&&!await v(t.pi_session_file,a.startedAt)&&!await u(r,a.startedAt)&&await l(r,t.lifecycle)==="park"}e(_,"shouldParkAfterSummary");export{v as mainSessionActivitySince,_ as shouldParkAfterSummary};
+var n=Object.defineProperty;var i=(r,t)=>n(r,"name",{value:t,configurable:!0});import{conversationIdFromPath as o,conversationStore as f,ConversationNotFoundError as c}from"../core/conversation-store/index.js";import{emitEvent as s}from"../core/events/emit.js";import{hasWakeMailSince as u}from"../core/canvas/deliveries.js";import{readLiveTurn as m}from"../core/canvas/recovery.js";import{parkEvent as l}from"./reconcilers/live-obligation.js";async function v(r,t){if(r==null)return!1;const a=o(r);if(a===null)return!1;try{return(await f().header(a)).mtime>=t}catch(e){return e instanceof c?!1:(s({level:"warn",event:"broker.park.session_activity_scan_failed",error:e}),!0)}}i(v,"mainSessionActivitySince");async function _(r,t,a,e){return!a.activity&&(await m(r))?.busy!==!0&&!await v(t.pi_session_file,a.startedAt)&&!await u(r,a.startedAt)&&await l(r,t.lifecycle,e)==="park"}i(_,"shouldParkAfterSummary");export{v as mainSessionActivitySince,_ as shouldParkAfterSummary};

package/dist/daemon/park-pending.d.ts

@@ -1,18 +1,25 @@
+import { type CanvasDb } from '../core/canvas/db.js';
 export type PendingPark = {
     startedAt: number;
     activity: boolean;
 };
 /** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
-export declare function beginPendingPark(nodeId: string, now: number): void;
-/** Record main-engine input accepted while the isolated turn is in flight. */
-export declare function recordPendingParkActivity(nodeId: string): boolean;
-export declare function isPendingPark(nodeId: string): boolean;
-/** When the pending park was armed, or undefined when none is armed. */
-export declare function pendingParkStartedAt(nodeId: string): number | undefined;
-/** Delete-and-report the pending marker. Both the supervision tick and the
- * settle handler run on the daemon thread with no await between their check and
- * this call, so exactly one of them ever enacts a given park. */
-export declare function consumePendingPark(nodeId: string): PendingPark | undefined;
-/** Every node with a parking turn in flight — the reconciler's sweep drops the
- *  ones whose broker has left the fleet. */
-export declare function pendingParkIds(): string[];
+export declare function beginPendingPark(nodeId: string, now: number): Promise<void>;
+/** Record main-engine input accepted while the isolated turn is in flight. False when no park is armed. */
+export declare function recordPendingParkActivity(nodeId: string): Promise<boolean>;
+export declare function isPendingPark(nodeId: string): Promise<boolean>;
+/** When the pending park was armed (epoch ms), or undefined when none is armed. */
+export declare function pendingParkStartedAt(nodeId: string): Promise<number | undefined>;
+/** Clear-and-report the pending marker with one conditional update. Exactly one caller, in any process, gets the prior values back. */
+export declare function consumePendingPark(nodeId: string): Promise<PendingPark | undefined>;
+/** Every node with a parking turn in flight — the supervision sweep drops the ones whose broker has left the fleet. */
+export declare function pendingParkIds(): Promise<string[]>;
+/** The same clear inside the caller's transaction, for the execution leaving `running`. */
+export declare function clearPendingParkIn(db: CanvasDb, nodeId: string): Promise<void>;
+export type SupervisionMark = 'unattended_since' | 'stall_noticed_at' | 'wedge_noticed_at';
+/** The mark's time (epoch ms), or undefined when unset. */
+export declare function readSupervisionMark(nodeId: string, mark: SupervisionMark): Promise<number | undefined>;
+/** Set the mark to `at` (epoch ms). */
+export declare function writeSupervisionMark(nodeId: string, mark: SupervisionMark, at: number): Promise<void>;
+/** Clear the mark; a tick that finds it already clear takes no write. */
+export declare function clearSupervisionMark(nodeId: string, mark: SupervisionMark): Promise<void>;

package/dist/daemon/park-pending.js

@@ -1 +1,3 @@
-var i=Object.defineProperty;var n=(e,t)=>i(e,"name",{value:t,configurable:!0});const r=new Map;function o(e,t){r.set(e,{startedAt:t,activity:!1})}n(o,"beginPendingPark");function u(e){const t=r.get(e);return t===void 0?!1:(t.activity=!0,!0)}n(u,"recordPendingParkActivity");function d(e){return r.has(e)}n(d,"isPendingPark");function c(e){return r.get(e)?.startedAt}n(c,"pendingParkStartedAt");function f(e){const t=r.get(e);return t!==void 0&&r.delete(e),t}n(f,"consumePendingPark");function s(){return Array.from(r.keys())}n(s,"pendingParkIds");export{o as beginPendingPark,f as consumePendingPark,d as isPendingPark,s as pendingParkIds,c as pendingParkStartedAt,u as recordPendingParkActivity};
+var p=Object.defineProperty;var r=(e,a)=>p(e,"name",{value:a,configurable:!0});import{openDb as i,withCanvasWrite as n}from"../core/canvas/db.js";async function c(e,a){await n(async t=>{await t.prepare("UPDATE nodes SET park_requested_at = ?, park_activity = 0 WHERE node_id = ?").run(new Date(a).toISOString(),e)})}r(c,"beginPendingPark");async function E(e){return await n(async a=>Number((await a.prepare("UPDATE nodes SET park_activity = 1 WHERE node_id = ? AND park_requested_at IS NOT NULL").run(e)).changes)>0)}r(E,"recordPendingParkActivity");async function k(e){return await o(e)!==void 0}r(k,"isPendingPark");async function o(e){const a=await(await i()).prepare("SELECT park_requested_at FROM nodes WHERE node_id = ?").get(e);return a?.park_requested_at?Date.parse(a.park_requested_at):void 0}r(o,"pendingParkStartedAt");async function w(e){return await n(async a=>{const t=await a.prepare(`WITH prior AS (SELECT park_requested_at, park_activity FROM nodes WHERE node_id = ? AND park_requested_at IS NOT NULL FOR UPDATE)
+      UPDATE nodes n SET park_requested_at = NULL, park_activity = 0 FROM prior WHERE n.node_id = ?
+      RETURNING prior.park_requested_at AS park_requested_at, prior.park_activity AS park_activity`).get(e,e);return t===void 0?void 0:{startedAt:Date.parse(t.park_requested_at),activity:Number(t.park_activity)===1}})}r(w,"consumePendingPark");async function y(){return(await(await i()).prepare("SELECT node_id FROM nodes WHERE park_requested_at IS NOT NULL").all()).map(e=>e.node_id)}r(y,"pendingParkIds");async function S(e,a){await e.prepare("UPDATE nodes SET park_requested_at = NULL, park_activity = 0, unattended_since = NULL, stall_noticed_at = NULL, wedge_noticed_at = NULL WHERE node_id = ?").run(a)}r(S,"clearPendingParkIn");async function s(e,a){const t=await(await i()).prepare(`SELECT ${a} AS at FROM nodes WHERE node_id = ?`).get(e);return t?.at?Date.parse(t.at):void 0}r(s,"readSupervisionMark");async function L(e,a,t){await n(async d=>{await d.prepare(`UPDATE nodes SET ${a} = ? WHERE node_id = ?`).run(new Date(t).toISOString(),e)})}r(L,"writeSupervisionMark");async function T(e,a){await s(e,a)!==void 0&&await n(async t=>{await t.prepare(`UPDATE nodes SET ${a} = NULL WHERE node_id = ?`).run(e)})}r(T,"clearSupervisionMark");export{c as beginPendingPark,S as clearPendingParkIn,T as clearSupervisionMark,w as consumePendingPark,k as isPendingPark,y as pendingParkIds,o as pendingParkStartedAt,s as readSupervisionMark,E as recordPendingParkActivity,L as writeSupervisionMark};

package/dist/daemon/profile-delete.js

@@ -1 +1 @@
-var q=Object.defineProperty;var d=(e,t)=>q(e,"name",{value:t,configurable:!0});import{existsSync as w,readFileSync as E,readdirSync as g,rmSync as p}from"node:fs";import{join as u}from"node:path";import{cronIdsForProfileDeletion as R,cronIdsForProfileDetach as C,deleteCronsForProfile as T,deleteNodes as O,getNode as m,getRow as W,globalizeCronsForProfile as A,listNodesByProfile as M,nodesRoot as B,reviewDir as $}from"../core/canvas/index.js";import{withCanvasWrite as v}from"../core/canvas/db.js";import{deleteProfileWithOwned as _}from"../core/graph/deletion.js";import{usage as f}from"../core/errors.js";import{emitEvent as z}from"../core/events/emit.js";import{clearDefaultProfileId as k,defaultProfileDirs as L}from"../core/profiles/default-binding.js";import{reserveProfileDeletion as J}from"../core/profiles/deletion-reservation.js";import{deleteProfile as D,loadProfileManifest as G}from"../core/profiles/manifest.js";import{deleteReviewsByOriginNodes as H,reviewIdsByOriginNodes as K}from"../core/review/store.js";import{boundFleet as U}from"../core/runtime/fleet.js";import{headlessBrokerHost as b}from"../core/runtime/host.js";import{transition as Q}from"../core/runtime/lifecycle.js";import{tearDownNode as V}from"../core/runtime/placement.js";import{reshapeNode as X}from"../core/runtime/promote.js";import{reviveNode as Y}from"../core/runtime/revive.js";import{filterAsync as Z,flatMapAsync as ee}from"../core/async-array.js";const P=1e3,N=new Set(["page.json","page.tsx","page.html","page.js","review.json","response.json","progress.json","claim.json","reply-route.json","feedback-comments.json","feedback.json",".ticket-lock"]),h=new Map;function te(e){if(typeof e!="object"||e===null||Array.isArray(e))throw f("profile deletion requires a JSON request body",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});return e}d(te,"requestObject");function re(e){const t=te(e);if(t.force!==!0)throw f("profile deletion requires --force",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});if(t.detach!==void 0&&typeof t.detach!="boolean")throw f("detach must be a boolean",{error:"invalid_detach",message:"detach must be a boolean",next:"Use --detach as a presence-only flag, or omit it."});const r=t.caller_node_id;if(r!=null&&(typeof r!="string"||r===""))throw f("caller_node_id must be a nonempty string or null");return{force:!0,...t.detach===!0?{detach:!0}:{},...r===void 0?{}:{caller_node_id:r}}}d(re,"validateRequest");function j(e){if(!w(e))return null;try{const t=JSON.parse(E(e,"utf8"));return typeof t.source?.nodeId=="string"&&t.source.nodeId!==""?t.source.nodeId:null}catch{return null}}d(j,"manifestSourceNodeId");async function oe(e){const t=new Set(e),r=B();if(t.size===0||!w(r))return[];const o=[];for(const n of g(r,{withFileTypes:!0})){if(!n.isDirectory())continue;const s=u(r,n.name);[j(u(s,"page.json")),j(u(s,"review.json"))].some(a=>a!==null&&t.has(a))&&o.push({dir:s,wholeDirectory:t.has(n.name)||await W(n.name)===null})}return o.sort((n,s)=>n.dir.localeCompare(s.dir))}d(oe,"ticketCleanupForNodes");function ne(e){if(N.has(e))return!0;for(const t of N)if(e.startsWith(`${t}.`)&&(e.endsWith(".tmp")||e.endsWith(".lock")))return!0;return!1}d(ne,"isTicketOwnedResidue");function ie(e){let t=0;for(const r of e)if(w(r.dir)){if(r.wholeDirectory){p(r.dir,{recursive:!0,force:!0}),t+=1;continue}for(const o of g(r.dir,{withFileTypes:!0}))ne(o.name)&&p(u(r.dir,o.name),{recursive:!0,force:!0});t+=1}return t}d(ie,"cleanTickets");function S(e){const t=e.filter(o=>o.managed_worktree?.state==="open").map(o=>({node_id:o.node_id,path:o.managed_worktree.path}));if(t.length===0)return null;const r=`profile deletion is blocked by ${t.length} open managed worktree${t.length===1?"":"s"}`;return f(r,{error:"open_managed_worktree",message:r,nodes:t,next:`Close ${t.map(({node_id:o})=>o).join(", ")} with \`crtr node worktree close <node>\`, then retry.`})}d(S,"openWorktreeError");async function se(e,t){const r=await M(e.profileId),o=S(r);if(o!==null)throw o;const n=r.map(({node_id:c})=>c),s=t.detach===!0?"detached":"deleted";return{entry:e,mode:s,nodes:r,nodeIds:n,tickets:s==="deleted"?await oe(n):[],cronIds:s==="deleted"?await R(e.profileId,n):await C(e.profileId),reviewIds:s==="deleted"?await K(n):[],defaultDirs:L(e.profileId)}}d(se,"prepareDeletion");async function x(e){const t=await ee(e.nodeIds,async o=>{const n=await m(o);return n===null?[]:[n]}),r=S(t);if(r!==null)throw r}d(x,"assertDeletionPlanPreconditions");function y(e){return{profile_id:e.entry.profileId,name:e.entry.manifest.name,mode:e.mode,nodes:e.nodeIds.length,crons:e.cronIds.length,tickets:e.tickets.length,reviews:e.reviewIds.length,default_pins:e.defaultDirs.length}}d(y,"resultForPlan");async function de(e){await x(e);const t=await Z(e.nodeIds,async i=>await m(i)!==null),r=U(),o=t.flatMap(i=>{const l=r.get(i);return l===void 0?[]:[{nodeId:i,exited:l.handle.exited}]}),n=[];for(const{nodeId:i}of o){const l=await m(i);(l?.status==="active"||l?.status==="idle")&&(await Q(i,"release"),n.push(i)),b.teardown(i)}await Promise.all(o.map(({exited:i})=>i));const s=[];for(const i of t)await m(i)!==null&&(await X(i,{profileId:null}),s.push(i));const c=await A(e.entry.profileId),a=k(e.entry.profileId);await v(async i=>await _(i,e.entry.profileId)),D(e.entry.profileId);const I=new Set(s);for(const i of n)I.has(i)&&await Y(i,{resume:!0,recovery:!0,capacity:"freeze"});return{...y(e),nodes:s.length,crons:c.length,default_pins:a.length}}d(de,"detachProfile");async function ce(e){await x(e);const t=await T(e.entry.profileId,e.nodeIds);for(const c of e.nodeIds)b.teardown(c),await V(c);const r=await H(e.nodeIds);for(const c of r)p($(c),{recursive:!0,force:!0});const o=ie(e.tickets),n=await O(e.nodeIds),s=k(e.entry.profileId);return await v(async c=>await _(c,e.entry.profileId)),D(e.entry.profileId),{...y(e),nodes:n.length,crons:t.length,tickets:o,reviews:r.length,default_pins:s.length}}d(ce,"hardDeleteProfile");async function F(e){return e.mode==="detached"?de(e):await ce(e)}d(F,"executePlan");function ae(e,t){const r={...y(e),accepted:!0,grace_ms:P};return h.set(e.entry.profileId,{result:r}),setTimeout(()=>{(async()=>{try{await F(e)}catch(o){z({level:"error",event:"profile.delete.deferred_failed",error:o instanceof Error?o:new Error(String(o)),fields:{profile_id:e.entry.profileId,mode:e.mode}})}finally{h.delete(e.entry.profileId),t()}})()},P),r}d(ae,"scheduleSelfDeletion");async function xe(e,t){const r=re(t),o=G(e),n=h.get(o.profileId);if(n!==void 0)return{status:202,body:n.result};const s=J(o.profileId);let c=!1;try{const a=await se(o,r);if(r.caller_node_id!=null&&a.nodeIds.includes(r.caller_node_id)){const i=ae(a,s);return c=!0,{status:202,body:i}}return{status:200,body:await F(a)}}finally{c||s()}}d(xe,"requestProfileDeletion");export{xe as requestProfileDeletion};
+var q=Object.defineProperty;var d=(e,t)=>q(e,"name",{value:t,configurable:!0});import{existsSync as w,readFileSync as E,readdirSync as g,rmSync as p}from"node:fs";import{join as u}from"node:path";import{cronIdsForProfileDeletion as R,cronIdsForProfileDetach as C,deleteCronsForProfile as T,deleteNodes as O,getNode as m,getRow as W,globalizeCronsForProfile as A,listNodesByProfile as M,nodesRoot as B,reviewDir as $}from"../core/canvas/index.js";import{withCanvasWrite as v}from"../core/canvas/db.js";import{deleteProfileWithOwned as _}from"../core/graph/deletion.js";import{usage as f}from"../core/errors.js";import{emitEvent as z}from"../core/events/emit.js";import{clearDefaultProfileId as k,defaultProfileDirs as L}from"../core/profiles/default-binding.js";import{reserveProfileDeletion as J}from"../core/profiles/deletion-reservation.js";import{deleteProfile as D,loadProfileManifest as G}from"../core/profiles/manifest.js";import{deleteReviewsByOriginNodes as H,reviewIdsByOriginNodes as K}from"../core/review/store.js";import{boundFleet as U}from"../core/runtime/fleet.js";import{headlessBrokerHost as b}from"../core/runtime/host.js";import{transition as Q}from"../core/runtime/lifecycle.js";import{tearDownNode as V}from"../core/runtime/placement.js";import{reshapeNode as X}from"../core/runtime/promote.js";import{reviveNode as Y}from"../core/runtime/revive.js";import{filterAsync as Z,flatMapAsync as ee}from"../core/async-array.js";const P=1e3,N=new Set(["page.json","page.tsx","page.html","page.js","review.json","response.json","progress.json","claim.json","reply-route.json","feedback-comments.json","feedback.json",".ticket-lock"]),h=new Map;function te(e){if(typeof e!="object"||e===null||Array.isArray(e))throw f("profile deletion requires a JSON request body",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});return e}d(te,"requestObject");function re(e){const t=te(e);if(t.force!==!0)throw f("profile deletion requires --force",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});if(t.detach!==void 0&&typeof t.detach!="boolean")throw f("detach must be a boolean",{error:"invalid_detach",message:"detach must be a boolean",next:"Use --detach as a presence-only flag, or omit it."});const r=t.caller_node_id;if(r!=null&&(typeof r!="string"||r===""))throw f("caller_node_id must be a nonempty string or null");return{force:!0,...t.detach===!0?{detach:!0}:{},...r===void 0?{}:{caller_node_id:r}}}d(re,"validateRequest");function j(e){if(!w(e))return null;try{const t=JSON.parse(E(e,"utf8"));return typeof t.source?.nodeId=="string"&&t.source.nodeId!==""?t.source.nodeId:null}catch{return null}}d(j,"manifestSourceNodeId");async function oe(e){const t=new Set(e),r=B();if(t.size===0||!w(r))return[];const o=[];for(const n of g(r,{withFileTypes:!0})){if(!n.isDirectory())continue;const s=u(r,n.name);[j(u(s,"page.json")),j(u(s,"review.json"))].some(c=>c!==null&&t.has(c))&&o.push({dir:s,wholeDirectory:t.has(n.name)||await W(n.name)===null})}return o.sort((n,s)=>n.dir.localeCompare(s.dir))}d(oe,"ticketCleanupForNodes");function ne(e){if(N.has(e))return!0;for(const t of N)if(e.startsWith(`${t}.`)&&(e.endsWith(".tmp")||e.endsWith(".lock")))return!0;return!1}d(ne,"isTicketOwnedResidue");function ie(e){let t=0;for(const r of e)if(w(r.dir)){if(r.wholeDirectory){p(r.dir,{recursive:!0,force:!0}),t+=1;continue}for(const o of g(r.dir,{withFileTypes:!0}))ne(o.name)&&p(u(r.dir,o.name),{recursive:!0,force:!0});t+=1}return t}d(ie,"cleanTickets");function S(e){const t=e.filter(o=>o.managed_worktree?.state==="open").map(o=>({node_id:o.node_id,path:o.managed_worktree.path}));if(t.length===0)return null;const r=`profile deletion is blocked by ${t.length} open managed worktree${t.length===1?"":"s"}`;return f(r,{error:"open_managed_worktree",message:r,nodes:t,next:`Close ${t.map(({node_id:o})=>o).join(", ")} with \`crtr node worktree close <node>\`, then retry.`})}d(S,"openWorktreeError");async function se(e,t){const r=await M(e.profileId),o=S(r);if(o!==null)throw o;const n=r.map(({node_id:a})=>a),s=t.detach===!0?"detached":"deleted";return{entry:e,mode:s,nodes:r,nodeIds:n,tickets:s==="deleted"?await oe(n):[],cronIds:s==="deleted"?await R(e.profileId,n):await C(e.profileId),reviewIds:s==="deleted"?await K(n):[],defaultDirs:L(e.profileId)}}d(se,"prepareDeletion");async function x(e){const t=await ee(e.nodeIds,async o=>{const n=await m(o);return n===null?[]:[n]}),r=S(t);if(r!==null)throw r}d(x,"assertDeletionPlanPreconditions");function y(e){return{profile_id:e.entry.profileId,name:e.entry.manifest.name,mode:e.mode,nodes:e.nodeIds.length,crons:e.cronIds.length,tickets:e.tickets.length,reviews:e.reviewIds.length,default_pins:e.defaultDirs.length}}d(y,"resultForPlan");async function de(e){await x(e);const t=await Z(e.nodeIds,async i=>await m(i)!==null),r=U(),o=t.flatMap(i=>{const l=r.get(i);return l===void 0?[]:[{nodeId:i,exited:l.handle.exited}]}),n=[];for(const{nodeId:i}of o){const l=await m(i);(l?.status==="active"||l?.status==="idle")&&(await Q(i,"release"),n.push(i)),b.teardown(i)}await Promise.all(o.map(({exited:i})=>i));const s=[];for(const i of t)await m(i)!==null&&(await X(i,{profileId:null}),s.push(i));const a=await A(e.entry.profileId),c=await k(e.entry.profileId);await v(async i=>await _(i,e.entry.profileId)),await D(e.entry.profileId);const I=new Set(s);for(const i of n)I.has(i)&&await Y(i,{resume:!0,recovery:!0,capacity:"freeze"});return{...y(e),nodes:s.length,crons:a.length,default_pins:c.length}}d(de,"detachProfile");async function ae(e){await x(e);const t=await T(e.entry.profileId,e.nodeIds);for(const a of e.nodeIds)b.teardown(a),await V(a);const r=await H(e.nodeIds);for(const a of r)p($(a),{recursive:!0,force:!0});const o=ie(e.tickets),n=await O(e.nodeIds),s=await k(e.entry.profileId);return await v(async a=>await _(a,e.entry.profileId)),await D(e.entry.profileId),{...y(e),nodes:n.length,crons:t.length,tickets:o,reviews:r.length,default_pins:s.length}}d(ae,"hardDeleteProfile");async function F(e){return e.mode==="detached"?de(e):await ae(e)}d(F,"executePlan");function ce(e,t){const r={...y(e),accepted:!0,grace_ms:P};return h.set(e.entry.profileId,{result:r}),setTimeout(()=>{(async()=>{try{await F(e)}catch(o){z({level:"error",event:"profile.delete.deferred_failed",error:o instanceof Error?o:new Error(String(o)),fields:{profile_id:e.entry.profileId,mode:e.mode}})}finally{h.delete(e.entry.profileId),t()}})()},P),r}d(ce,"scheduleSelfDeletion");async function xe(e,t){const r=re(t),o=G(e),n=h.get(o.profileId);if(n!==void 0)return{status:202,body:n.result};const s=J(o.profileId);let a=!1;try{const c=await se(o,r);if(r.caller_node_id!=null&&c.nodeIds.includes(r.caller_node_id)){const i=ce(c,s);return a=!0,{status:202,body:i}}return{status:200,body:await F(c)}}finally{a||s()}}d(xe,"requestProfileDeletion");export{xe as requestProfileDeletion};

package/dist/daemon/reconcilers/bash-deadline.d.ts

@@ -17,25 +17,30 @@ export interface BashDeadlineContext {
  * a job the broker backgrounded while exiting (SIGTERM/SIGHUP), which could not
  * record its own job object. */
 export declare class BashDeadlineReconciler {
-    private lastScanAt;
     /** An exit written before this daemon started is history, not a job to adopt. */
     private readonly startedAt;
+    /** Bytes of each running job's log already in the blob store. */
+    private readonly uploadedBytes;
     run(now: number, ctx: BashDeadlineContext): Promise<void>;
+    private scan;
     /** Tell the owner about a live background job that has shown no activity (see the supervisor's
      *  activity sampler) for 15 minutes, again at 30 and 60, then hourly. Any activity restarts the
      *  schedule; a job marked as an expected wait is skipped. Messages only: the job is never stopped. */
     private warnSilent;
-    /** A running job whose supervisor wrote job.exit has exited: `finishJob` publishes `exited`
-     *  once (the row leaves `running`). A job being stopped (job.stop) is ended by its killer. */
+    /** Mirror what each live node's job supervisors wrote beside the jobs (process identity, activity,
+     *  exit) into the job rows, upload the output written since, and report a process confirmed gone
+     *  without an exit record as vanished. The supervisor files live on this host's disk; a broker on
+     *  another host reports the same things through the `report` route. */
+    private mirrorSupervisors;
+    private mirror;
+    /** Put the job's log in the blob store as 64 KiB chunks under its output key; chunks that were already
+     *  full are kept, the last one is rewritten as it grows. */
+    private uploadOutput;
+    /** End every job whose exit the supervisor reported: `finishJob` publishes `exited` once (the row leaves `running`). */
     private publishExits;
     /** Give every backgrounded job dir (job.bg) that has no job object one; the next tick's
-     *  `publishExits` then ends it like any other job. */
+     *  mirror then reports its process and exit like any other job's. */
     private adoptUnrecorded;
-    /** A running job whose process is confirmed gone without job.exit (killed outright, so its
-     *  supervisor never wrote one) is ended once as killed, with a null exit code, through
-     *  `finishJob`. job.exit is then written so the file-based listing and stop guard stop counting
-     *  the job. A job.exit that appears by the re-check is a real exit: `publishExits` reports it.
-     *  A job being stopped (job.stop) is ended by its killer. */
-    private settleVanished;
+    /** Cancel a job whose deadline passed: signal its process (a sandbox job's reporter does that once the row ends), then end its row as killed by the deadline. */
     private cancel;
 }

package/dist/daemon/reconcilers/bash-deadline.js

@@ -1,3 +1 @@
-var E=Object.defineProperty;var m=(s,t)=>E(s,"name",{value:t,configurable:!0});import{existsSync as a,readFileSync as S,readdirSync as x,statSync as B}from"node:fs";import{join as l}from"node:path";import{bashJobPaths as j,bashJobsDir as J,activeBackgroundBashJobs as v,expiredAutoBackgroundBashJobs as g,formatBashElapsed as M,isJobQuietOk as N,readJobActivity as I,readJobSilence as P,writeJobSilence as T,isBashJobProcessConfirmedGone as k,readJobPgid as C,readJobPgidIdentity as L,stopBackgroundBashJob as D}from"../../core/bash-jobs.js";import{killBashJobObject as O,recordBashJobObject as F}from"../../core/spaces/stop.js";import{createFileBeneath as R}from"../../core/spaces/open-beneath.js";import{withCanvasWrite as f}from"../../core/canvas/db.js";import{finishJob as A,jobRow as V}from"../../core/graph/jobs.js";import{publish as G}from"../../core/graph/events.js";import{envBashSilenceMs as U}from"../../shared/env.js";import{getById as h}from"../../core/graph/objects.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as W}from"../../core/canvas/node-agent-paths.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as b}from"../../core/events/operation-id.js";import{isProfilePaused as X}from"../../core/profiles/manifest.js";const Y=300*1e3;function H(s,t,o){const e=s<o?0:s<2*o?o:s<4*o?2*o:4*o*Math.floor(s/(4*o));return e>t?e:null}m(H,"dueSilenceThresholdMs");const Q=5*1e3;class ft{static{m(this,"BashDeadlineReconciler")}lastScanAt=Number.NEGATIVE_INFINITY;startedAt=Date.now();async run(t,o){if(await this.publishExits(),!(t-this.lastScanAt<Q)){this.lastScanAt=t;for(const e of o.rows){if(e.status==="dead"||X(e.profile_id))continue;const r=await W(e.node_id);let i;try{await this.adoptUnrecorded(e.node_id,r),await this.settleVanished(r),i=g(r,t)}catch(n){b.fresh(()=>{c({level:"error",event:"bash_deadline.scan_failed",...u(e.node_id)?{node_id:e.node_id}:{},error:n})});continue}for(const n of i)await this.cancel(e.node_id,r,n.jobId);try{await this.warnSilent(r,t)}catch(n){b.fresh(()=>{c({level:"error",event:"bash_silence.scan_failed",...u(e.node_id)?{node_id:e.node_id}:{},error:n})})}}}}async warnSilent(t,o){const e=U();for(const{jobId:r}of v(t)){if(N(t,r))continue;const i=I(t,r);if(i===void 0||o-i.sampledAtMs>Y)continue;const n=o-i.lastActivityMs,y=P(t,r),_=H(n,y?.lastActivityMs===i.lastActivityMs?y.thresholdMs:0,e);if(_===null)continue;const w=j(t,r);await f(async p=>{const d=await h(p,r);d===null||d.type!=="job"||(await V(p,d))?.status!=="running"||await G(p,{event:"silent",object:d,actor:null,attrs:{"silent-for":M(n),files:i.files.join(", "),"log-path":w.jobLog}})}),T(w,i.lastActivityMs,_)}}async publishExits(){try{await f(async t=>{const o=await t.prepare(`SELECT j.object, j.job_dir FROM jobs j JOIN objects o ON o.id = j.object
-          WHERE j.status = 'running' AND o.deleted_at IS NULL`).all();for(const{object:e,job_dir:r}of o){const i=l(r,"job.exit");if(!a(i)||a(l(r,"job.stop")))continue;const n=Number(S(i,"utf8").trim());Number.isSafeInteger(n)&&await A(t,await h(t,e),{exitCode:n})}})}catch(t){b.fresh(()=>c({level:"error",event:"bash_completion.delivery_failed",error:t}))}}async adoptUnrecorded(t,o){let e;try{e=x(J(o))}catch{return}await f(async r=>{for(const i of e){if(await h(r,i)!==null)continue;const n=j(o,i);!a(n.jobBg)||!a(n.cmdSh)||a(n.jobExit)&&B(n.jobExit).mtimeMs<this.startedAt||await F(t,i,{command:S(n.cmdSh,"utf8"),jobDir:n.dir,logPath:n.jobLog})}})}async settleVanished(t){for(const{jobId:o}of v(t)){const e=j(t,o);a(e.jobExit)||a(l(e.dir,"job.stop"))||k(C(t,o),L(t,o))&&await f(async r=>{const i=await h(r,o);if(!(i===null||i.type!=="job")&&!(a(e.jobExit)||a(l(e.dir,"job.stop")))){await A(r,i,{killedBy:null,reason:"process disappeared without an exit record"});try{R(e.root,e.jobExit,`-1
-`)}catch(n){if(n.code!=="EEXIST")throw n}}})}}async cancel(t,o,e){(await D(o,e,!1,t)).kind==="stopped"&&await b.fresh(async()=>{try{c({level:"info",event:"bash_deadline.canceled",...u(t)?{node_id:t}:{},fields:{job_id:e}}),await O(e,async()=>null,"deadline")}catch(i){c({level:"error",event:"bash_deadline.notice_failed",...u(t)?{node_id:t}:{},error:i,fields:{job_id:e}})}})}}export{ft as BashDeadlineReconciler,H as dueSilenceThresholdMs};
+var A=Object.defineProperty;var m=(s,t)=>A(s,"name",{value:t,configurable:!0});import{existsSync as _,readFileSync as B,readdirSync as J,statSync as I}from"node:fs";import{bashJobPaths as g,bashJobsDir as C,clearJobDeadlineFile as E,formatBashElapsed as T,isBashJobProcessConfirmedGone as D,readJobSupervisorState as S,signalBashJob as O}from"../../core/bash-jobs.js";import{killBashJobObject as R,recordBashJobObject as j}from"../../core/spaces/stop.js";import{readFileBeneath as L}from"../../core/spaces/open-beneath.js";import{blobStore as P}from"../../core/blobs/index.js";import{openDb as c,withCanvasWrite as f}from"../../core/canvas/db.js";import{applyJobReport as N,endJobsOfExitedSandboxes as U,endReportedJobs as k,expiredJobs as F,jobRow as K,ownerRunsRemotely as v,recordJobSilence as q,runningJobs as b,setJobDeadline as z}from"../../core/graph/jobs.js";import{publish as G}from"../../core/graph/events.js";import{envBashSilenceMs as V}from"../../shared/env.js";import{getById as M}from"../../core/graph/objects.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as x}from"../../core/canvas/node-agent-paths.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{isProfilePaused as Y}from"../../core/profiles/manifest.js";import{runThrottledLane as $}from"./throttled-lane.js";const H=300*1e3;function W(s,t,e){const i=s<e?0:s<2*e?e:s<4*e?2*e:4*e*Math.floor(s/(4*e));return i>t?i:null}m(W,"dueSilenceThresholdMs");const X=5*1e3,Q=60*1e3,y=64*1024;class ye{static{m(this,"BashDeadlineReconciler")}startedAt=Date.now();uploadedBytes=new Map;async run(t,e){await this.mirrorSupervisors(e),await this.publishExits(),await $("bash-deadline","",t,X,Q,i=>this.scan(t,e,i))}async scan(t,e,i){const o=new Set;for(const n of e.rows)if(n.status!=="dead"&&!Y(n.profile_id)&&(o.add(n.node_id),!await v(await c(),n.node_id)))try{await this.adoptUnrecorded(n.node_id,await x(n.node_id))}catch(a){p.fresh(()=>{l({level:"error",event:"bash_deadline.scan_failed",...u(n.node_id)?{node_id:n.node_id}:{},error:a})})}for(const n of await F(await c(),t))if(o.has(n.owner)){if(!await i.holds())return;await this.cancel(n)}for(const n of o)try{await this.warnSilent(n,t)}catch(a){p.fresh(()=>{l({level:"error",event:"bash_silence.scan_failed",...u(n)?{node_id:n}:{},error:a})})}}async warnSilent(t,e){const i=V();for(const o of await b(await c(),t)){const{activity:n}=o;if(o.quietOk||n===null||e-n.sampledAtMs>H)continue;const a=e-n.lastActivityMs,r=o.silence,d=W(a,r?.lastActivityMs===n.lastActivityMs?r.thresholdMs:0,i);d!==null&&await f(async h=>{const w=await M(h,o.jobId);w===null||w.type!=="job"||(await K(h,w))?.status!=="running"||(await G(h,{event:"silent",object:w,actor:null,attrs:{"silent-for":T(a),files:n.files.join(", "),"output-key":o.outputKey}}),await q(h,o.jobId,n.lastActivityMs,d))})}}async mirrorSupervisors(t){for(const e of t.rows)if(e.status!=="dead")try{const i=await b(await c(),e.node_id);if(i.length===0||await v(await c(),e.node_id))continue;const o=await x(e.node_id);for(const n of i)await this.mirror(o,n)}catch(i){p.fresh(()=>{l({level:"error",event:"bash_mirror.failed",...u(e.node_id)?{node_id:e.node_id}:{},error:i})})}}async mirror(t,e){const i=S(t,e.jobId),o=i.pgid??e.pgid??void 0,n=i.pgidIdentity??e.pgidIdentity??void 0,a=i.exitCode===void 0&&D(o,n)&&S(t,e.jobId).exitCode===void 0;i.deadlineMs!==void 0&&(e.deadlineAtMs===null&&await f(r=>z(r,e.jobId,i.deadlineMs)),E(g(t,e.jobId))),await this.uploadOutput(t,e),(i.exitCode!==void 0||a)&&this.uploadedBytes.delete(e.jobId),await f(async r=>N(r,e.jobId,{...i.pgid===void 0||i.pgid===e.pgid?{}:{pgid:i.pgid},...i.pgidIdentity===void 0||i.pgidIdentity===e.pgidIdentity?{}:{pgidIdentity:i.pgidIdentity},...i.activity===void 0?{}:{activity:i.activity},...i.exitCode===void 0?{}:{exitCode:i.exitCode},...a?{vanished:!0}:{}}))}async uploadOutput(t,e){const i=g(t,e.jobId);let o;try{o=I(i.jobLog).size}catch{return}const n=this.uploadedBytes.get(e.jobId)??0;if(o<=n)return;const a=await P(),r=L(i.root,i.jobLog);for(let d=Math.floor(n/y);d*y<r.length;d++)await a.put(`${e.outputKey}/${String(d).padStart(8,"0")}`,r.subarray(d*y,(d+1)*y));this.uploadedBytes.set(e.jobId,r.length)}async publishExits(){try{await f(async t=>{await k(t),await U(t)})}catch(t){p.fresh(()=>l({level:"error",event:"bash_completion.delivery_failed",error:t}))}}async adoptUnrecorded(t,e){let i;try{i=J(C(e))}catch{return}await f(async o=>{for(const n of i){if(await M(o,n)!==null)continue;const a=g(e,n);!_(a.jobBg)||!_(a.cmdSh)||_(a.jobExit)&&I(a.jobExit).mtimeMs<this.startedAt||await j(t,n,{command:B(a.cmdSh,"utf8")})}})}async cancel(t){const e=await v(await c(),t.owner);!e&&(t.pgid===null||t.pgidIdentity===null)||await p.fresh(async()=>{try{if(e||await O(t.pgid,t.pgidIdentity,t.owner),!await R(t.jobId,async()=>null,"deadline"))return;l({level:"info",event:"bash_deadline.canceled",...u(t.owner)?{node_id:t.owner}:{},fields:{job_id:t.jobId}})}catch(i){l({level:"error",event:"bash_deadline.notice_failed",...u(t.owner)?{node_id:t.owner}:{},error:i,fields:{job_id:t.jobId}})}})}}export{ye as BashDeadlineReconciler,W as dueSilenceThresholdMs};

package/dist/daemon/reconcilers/broker-supervision.d.ts

@@ -59,9 +59,8 @@ export declare function wedgeVerdict(input: {
 export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
 export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
 export declare class BrokerSupervisionReconciler {
-    private readonly yieldStalls;
-    private readonly wedgeNotifiedAt;
-    private readonly unattendedSince;
+    /** SIGTERM times of yield-stalled brokers this process signalled; the stall's start is `nodes.stall_noticed_at`. */
+    private readonly yieldTermedAt;
     private readonly sampleTree;
     private readonly signal;
     private lastOverCapWarnAt;
@@ -88,10 +87,21 @@ export declare class BrokerSupervisionReconciler {
      * starts the same isolated turn, so a concurrent main turn stays passive. */
     private requestParkSummary;
     private warnOverCapInTmux;
+    /** Stop a node's broker. A remote broker's stop is recorded on its row, enacted through its runner, and ends exactly that
+     *  execution once confirmed; any other broker is torn down by the local host. */
+    private teardownBroker;
+    private stopRemote;
+    /** A refresh that never reached its replacement: the same stall clock as a local broker, enacted through the runner. */
+    private handleRemoteYieldStall;
+    /** A remote broker has no process this daemon can sample, so its only evidence is the turn's progress clock. A quiet turn is
+     *  reported once and the broker is left running: with no view of its subprocesses the daemon cannot tell a runaway from a long command. */
+    private handleRemoteWedge;
     private handleYieldStall;
     private signalYieldStall;
     private readTreeSample;
     private handleWedgeDetection;
+    /** Record and announce a wedged broker: the event, the fault, and the notice to its managers. */
+    private reportWedge;
     /** Publish a fatal provider fault's notice once per episode and, for a node that settles on it, end the node.
      *  The notices, the `notified_since` mark and the terminal transition commit in one write, so a daemon exit
      *  leaves the fault either fully announced or fully unannounced. The daemon registers this as the announcer

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ee,listUnnotifiedFatalFaultNodes as te,markFaultNotified as ne,readFault as O,readLiveTurn as E,recordFault as re}from"../../core/canvas/recovery.js";import{errorClassFromFault as ie}from"../../core/fault-classifier.js";import{nextFaultRetry as ae}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as G,withCanvasWrite as se}from"../../core/canvas/db.js";import{getById as oe}from"../../core/graph/objects.js";import{watcherIdsOf as le}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as de,resetStaleExecutionMail as ce}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ue}from"../../core/bash-jobs.js";import{focusOf as fe}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as pe}from"../api/viewer-presence.js";import{deliverLive as me}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as he}from"../../core/runtime/broker-request.js";import{formatCard as _e,PARK_SUMMARY_PROMPT as ge}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ve}from"../../core/runtime/fleet.js";import{readConfig as we}from"../../core/config.js";import{beginPendingPark as ke,consumePendingPark as N,pendingParkIds as ye,pendingParkStartedAt as be}from"../park-pending.js";import{shouldParkAfterSummary as Se}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Ne}from"./live-obligation.js";import{readGoal as Pe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as $e}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Te,envTestUnattendedParkMs as Ae,parkSummaryGraceMs as Me,envUnattendedParkMs as Ee}from"../../shared/env.js";const A=j(z);async function Fe(s){return!await ee(s)&&((await Pe(s))?.trim()??"")!==""}p(Fe,"awaitsBirthTurn");async function xe(s){return(await E(s))?.busy===!0}p(xe,"liveBusy");async function Nt(s){const e=(await E(s))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(Nt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");async function x(s){const e=await E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(x,"busySinceForIncarnation");const W=3*6e4,B=2e4,I=20*6e4,Ie=2,Ce=5*6e4;function De(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(De,"settlesOnFatalProviderFault");function Oe(){const s=Ee();if(s!==void 0)return s;const e=we("user").lifecycle.unattendedParkMs;return Te()?Ae()??e:e}p(Oe,"unattendedParkMsForDaemon");function U(){return Me()}p(U,"parkSummaryGraceMsForDaemon");function Re(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Re,"yieldStallVerdict");function Ge(s){return!s.busy||s.quietForMs===null||s.quietForMs<I?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ie?"pending":"wedged"}p(Ge,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
-`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Le(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Le,"sumTreeCpu");function We(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(We,"treeHasDescendants");class Pt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ve()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Oe();for(const i of await te())await F.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),await this.handleUnattendedParking(n,await T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ye())t.fleet.has(i)||N(i);await ce(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ue(await X(e)).length>0||await fe(e)!==null||pe(e))return!0;const n=await O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ae(n,i)!==null)?!0:await de(e)}async handleUnattendedParking(e,t,r,i,n,l){if(await J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=await this.parkResetCondition(e,t,r,i),a=be(e);if(a!==void 0){if(i-a<U())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!await Se(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:U()}}),await P(e,"park"),S.teardown(e);return}if(o||await xe(e)||await Fe(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=await Ne(e,t.lifecycle),h=d!=="cancel"&&$e(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){await P(e,d),S.teardown(e);return}if(d==="cancel"){await P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ke(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await me(e,_e("park",{},ge),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof he){await P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ce)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async h=>{if(h!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await x(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Re(!0,e.intent,!1,i-o.since)!=="kill")return;const a=await T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=B&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:B,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Le(t,e),i=We(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity),o=await x(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=I?await this.sampleTree(t.pid):null,d=await T(n),h=d===null?null:await x(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(Ge({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const w=C(d),g=Math.round((a??I)/6e4),k=$??!0,v=!k&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,y=!1,b;if(k)b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),y=!0,b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(M){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:M,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:k,kick_attempted:f,kicked:y,...v===null?{}:{kick_suppression_reason:v}}});try{await re(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:b})}catch(M){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:M})}await R(n,"notice.crash",b,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:y,...v===null?{}:{kick_suppression_reason:v}})}async handleFatalFault(e,t=!0){const r=await O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
-`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=De(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,w=d?"failed":"hit a fatal provider fault",g=new Set((await Q(e)).map(f=>f.node_id)),k=await le(await G(),await oe(await G(),e)),v=[{relation:"Child",recipients:k.filter(f=>g.has(f))},{relation:"Watched node",recipients:k.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),await se(async()=>{for(const{relation:f,recipients:y}of v)y.length!==0&&await R(e,"notice.crash",`${f} ${w} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,y);await ne(r.faultId),t&&_&&await Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:ie(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{Pt as BrokerSupervisionReconciler,Ie as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,De as settlesOnFatalProviderFault,Le as sumTreeCpu,We as treeHasDescendants,Oe as unattendedParkMsForDaemon,Ge as wedgeVerdict,Re as yieldStallVerdict};
+var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,parkEvent as Ie}from"./live-obligation.js";import{readGoal as xe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Fe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Oe,envTestUnattendedParkMs as Ce,parkSummaryGraceMs as De,envUnattendedParkMs as We}from"../../shared/env.js";const R=ee(Z);async function Be(a){return!await ie(a)&&((await xe(a))?.trim()??"")!==""}m(Be,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Ge(a){return(await F(a))?.busy===!0}m(Ge,"liveBusy");async function It(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(It,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,Le=2,qe=5*6e4;function Ue(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ue,"settlesOnFatalProviderFault");function Ye(){const a=We();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Oe()?Ce()??e:e}m(Ye,"unattendedParkMsForDaemon");function H(){return De()}m(H,"parkSummaryGraceMsForDaemon");function K(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(K,"yieldStallVerdict");function He(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>Le?"pending":"wedged"}m(He,"wedgeVerdict");function V(a){const e=new Map,t=new Map;for(const r of a.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(V,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=V(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function Ve(a,e){const{byPid:t,childrenOf:r}=V(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(Ve,"treeHasDescendants");class xt{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Ye();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Ge(e)||await Be(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const u=await Ie(e,t.lifecycle),w=u!=="cancel"&&Fe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<w)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<qe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
+`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}K(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(K(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=Ve(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,k=s?.hasDescendants??null;if(He({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:k})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const v=x(p),y=Math.round((d??I)/6e4),b=k??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
+`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ue(i,typeof s?.code=="string"?s.code:void 0),k=u||w,v=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${v}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&k&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&k&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{xt as BrokerSupervisionReconciler,Le as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ue as settlesOnFatalProviderFault,Ke as sumTreeCpu,Ve as treeHasDescendants,Ye as unattendedParkMsForDaemon,He as wedgeVerdict,K as yieldStallVerdict};

package/dist/daemon/reconcilers/cron-lane.d.ts

@@ -3,6 +3,5 @@ export interface CronLaneContext {
     lifecycle: DetachedWorkLifecycle;
 }
 export declare class CronLaneReconciler {
-    private lastFolderSweep;
     run(now: number, ctx: CronLaneContext): Promise<void>;
 }

package/dist/daemon/reconcilers/cron-lane.js

@@ -1 +1 @@
-var l=Object.defineProperty;var t=(i,r)=>l(i,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as n}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";class w{static{t(this,"CronLaneReconciler")}lastFolderSweep=0;async run(r,a){await c.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}if(r-this.lastFolderSweep>=6e4){this.lastFolderSweep=r;try{await n()}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{w as CronLaneReconciler};
+var i=Object.defineProperty;var t=(c,r)=>i(c,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{runThrottledLane as m}from"./throttled-lane.js";class _{static{t(this,"CronLaneReconciler")}async run(r,a){await n.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}try{await m("cron-folder-sweep","",r,6e4,6e4,async()=>{await l()})}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{_ as CronLaneReconciler};

package/dist/daemon/reconcilers/fault-retry-backstop.d.ts

@@ -18,8 +18,6 @@ export interface FaultRetryBackstopContext {
 }
 export interface FaultRetryBackstopDeps {
     probeOnline?: () => Promise<boolean>;
-    /** Signal a node's broker; false when it holds no open signal stream. */
-    nudge?: (nodeId: string) => Promise<boolean>;
     list?: () => Promise<PendingProviderRetry[]>;
     busy?: (nodeId: string) => Promise<boolean>;
 }
@@ -28,14 +26,9 @@ export interface FaultRetryBackstopDeps {
 export declare function dueAtMs(pending: PendingProviderRetry): number | null;
 export declare class FaultRetryBackstopReconciler {
     private readonly deps;
-    private lastScanAt;
-    private scanning;
-    /** fault id → when the daemon last nudged it. Pruned to current candidates. */
-    private readonly nudgedAt;
     constructor(deps?: FaultRetryBackstopDeps);
-    /** Starts a detached scan (throttled, never overlapping). The returned promise
-     *  settles with that scan so a caller that wants to observe it can; the tick
-     *  does not await it. */
+    /** Starts a detached scan, throttled and exclusive across processes by the lane lease. The returned promise settles with the scan;
+     *  the tick does not await it. */
     run(now: number, ctx: FaultRetryBackstopContext): Promise<void>;
     private scan;
 }