@north-light/cli 0.3.574 → 0.3.576

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,parkEvent as Ie}from"./live-obligation.js";import{readGoal as xe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Fe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Oe,envTestUnattendedParkMs as Ce,parkSummaryGraceMs as De,envUnattendedParkMs as We}from"../../shared/env.js";const R=ee(Z);async function Be(a){return!await ie(a)&&((await xe(a))?.trim()??"")!==""}m(Be,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Ge(a){return(await F(a))?.busy===!0}m(Ge,"liveBusy");async function It(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(It,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,Le=2,qe=5*6e4;function Ue(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ue,"settlesOnFatalProviderFault");function Ye(){const a=We();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Oe()?Ce()??e:e}m(Ye,"unattendedParkMsForDaemon");function H(){return De()}m(H,"parkSummaryGraceMsForDaemon");function K(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(K,"yieldStallVerdict");function He(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>Le?"pending":"wedged"}m(He,"wedgeVerdict");function V(a){const e=new Map,t=new Map;for(const r of a.split(`
-`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(V,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=V(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function Ve(a,e){const{byPid:t,childrenOf:r}=V(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(Ve,"treeHasDescendants");class xt{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Ye();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Ge(e)||await Be(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const u=await Ie(e,t.lifecycle),w=u!=="cancel"&&Fe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<w)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<qe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
-`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}K(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(K(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=Ve(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,k=s?.hasDescendants??null;if(He({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:k})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const v=x(p),y=Math.round((d??I)/6e4),b=k??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
-`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ue(i,typeof s?.code=="string"?s.code:void 0),k=u||w,v=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${v}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&k&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&k&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{xt as BrokerSupervisionReconciler,Le as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ue as settlesOnFatalProviderFault,Ke as sumTreeCpu,Ve as treeHasDescendants,Ye as unattendedParkMsForDaemon,He as wedgeVerdict,K as yieldStallVerdict};
+var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,noticeUnfinishedIdle as Ie,parkVerdict as xe}from"./live-obligation.js";import{readGoal as Fe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Oe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Ce,envTestUnattendedParkMs as De,parkSummaryGraceMs as We,envUnattendedParkMs as Be}from"../../shared/env.js";const R=ee(Z);async function Ge(a){return!await ie(a)&&((await Fe(a))?.trim()??"")!==""}m(Ge,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Le(a){return(await F(a))?.busy===!0}m(Le,"liveBusy");async function xt(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(xt,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,qe=2,Ue=5*6e4;function Ye(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ye,"settlesOnFatalProviderFault");function He(){const a=Be();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Ce()?De()??e:e}m(He,"unattendedParkMsForDaemon");function H(){return We()}m(H,"parkSummaryGraceMsForDaemon");function V(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(V,"yieldStallVerdict");function Ve(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>qe?"pending":"wedged"}m(Ve,"wedgeVerdict");function K(a){const e=new Map,t=new Map;for(const r of a.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(K,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=K(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function ze(a,e){const{byPid:t,childrenOf:r}=K(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(ze,"treeHasDescendants");class Ft{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=He();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Le(e)||await Ge(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const{event:u,heldOnlyBySubscriber:w}=await xe(e,t.lifecycle),v=u!=="cancel"&&Oe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<v)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),w&&await Ie(e),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ue)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
+`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}V(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(V(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=ze(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,v=s?.hasDescendants??null;if(Ve({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:v})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const k=x(p),y=Math.round((d??I)/6e4),b=v??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
+`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ye(i,typeof s?.code=="string"?s.code:void 0),v=u||w,k=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${k}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&v&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&v&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{Ft as BrokerSupervisionReconciler,qe as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ye as settlesOnFatalProviderFault,Ke as sumTreeCpu,ze as treeHasDescendants,He as unattendedParkMsForDaemon,Ve as wedgeVerdict,V as yieldStallVerdict};

package/dist/daemon/reconcilers/live-obligation.d.ts

@@ -50,6 +50,13 @@ export declare function hasHumanViewer(nodeId: string, viewer?: ViewerProbe): Pr
  */
 export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
 export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<ParkEvent>;
+/** `parkEvent` plus whether a live subscriber awaiting the report is the ONLY thing holding a
+ *  terminal node's `release`: no viewer, no wake source, no unread mail, no fatal fault. Such a
+ *  node stopped unfinished with nothing that will wake it, and its subscriber knows nothing. */
+export declare function parkVerdict(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<{
+    event: ParkEvent;
+    heldOnlyBySubscriber: boolean;
+}>;
 /** Apply an unattended decision and notify subscribers of automatic completion.
  *  This is the DEGRADED enactment: no summary turn ran, so subscribers get the
  *  doctrine notice instead of the node's own report. The live clock's good path
@@ -64,3 +71,7 @@ export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?:
  *  resident throughout: its conversation ended without a final report, its state
  *  lives in whatever report it did leave, and a message reopens it. */
 export declare function applyParkEvent(nodeId: string, event: ParkEvent): Promise<void>;
+/** Tell a terminal node's subscribers it went idle unfinished: it pushed no final, holds no wait
+ *  that will wake it, and only their subscription kept it from being canceled. The cause of the
+ *  stop is not known here, so the notice claims none. */
+export declare function noticeUnfinishedIdle(nodeId: string): Promise<void>;

package/dist/daemon/reconcilers/live-obligation.js

@@ -1 +1 @@
-var l=Object.defineProperty;var r=(t,a)=>l(t,"name",{value:a,configurable:!0});import{hasActiveLiveSubscription as f,hasLiveSubscriber as p,hasPendingCronWake as w}from"../../core/canvas/index.js";import{fullName as m}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as n}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/output-result.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await f(t)||await h(t)||await k(t)||await w(t)||await b(t)>0||await g(t)!==null}r(s,"hasLiveObligation");async function B(t){return await s(t)||await n(t)?!0:await o(t)}r(B,"hasPendingWake");async function F(t){return async a=>await t.isClaimed(a)||await s(a)||await o(a)}r(F,"retentionHold");async function o(t,a){return(a!==void 0?a.live!==null:await v(t)!==null)||await x(t)}r(o,"hasHumanViewer");async function M(t,a,e){if(await o(t,e)||await s(t))return"release";const i=await W(t);return i?.link==="pi\u2192provider"&&i.retry.disposition==="fatal"?"release":a==="resident"?await n(t)?"release":"park":await n(t)||await p(t)?"release":"cancel"}r(M,"parkEvent");async function T(t,a){const e=a==="cancel"?await c(t,a,{reason:"stranded"}):await c(t,a);if(a!=="park")return;const i=await y(t),u=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${m(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}r(T,"applyParkEvent");export{T as applyParkEvent,o as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};
+var w=Object.defineProperty;var a=(t,e)=>w(t,"name",{value:e,configurable:!0});import{getNode as h,hasActiveLiveSubscription as m,hasLiveSubscriber as y,hasPendingCronWake as g}from"../../core/canvas/index.js";import{fullName as l}from"../../core/canvas/labels.js";import{hasLiveControllerWait as k,hasLiveObjectWait as b}from"../../core/canvas/waits.js";import{newestReportName as v}from"../../core/feed/reports.js";import{countTickets as x}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as s}from"../../core/canvas/deliveries.js";import{readOutputRequest as $}from"../../core/runtime/output-result.js";import{focusOf as O}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as d}from"../api/viewer-presence.js";import{publishNotice as u}from"../../core/runtime/close.js";import{readFault as L}from"../../core/canvas/recovery.js";import{transition as f}from"../../core/runtime/lifecycle.js";async function o(t){return await m(t)||await k(t)||await b(t)||await g(t)||await x(t)>0||await $(t)!==null}a(o,"hasLiveObligation");async function F(t){return await o(t)||await s(t)?!0:await c(t)}a(F,"hasPendingWake");async function M(t){return async e=>await t.isClaimed(e)||await o(e)||await c(e)}a(M,"retentionHold");async function c(t,e){return(e!==void 0?e.live!==null:await O(t)!==null)||await d(t)}a(c,"hasHumanViewer");async function T(t,e,r){return(await W(t,e,r)).event}a(T,"parkEvent");async function W(t,e,r){const i=a(p=>({event:p,heldOnlyBySubscriber:!1}),"held");if(await c(t,r)||await o(t))return i("release");const n=await L(t);return n?.link==="pi\u2192provider"&&n.retry.disposition==="fatal"?i("release"):e==="resident"?i(await s(t)?"release":"park"):await s(t)?i("release"):await y(t)?{event:"release",heldOnlyBySubscriber:!0}:i("cancel")}a(W,"parkVerdict");async function z(t,e){const r=e==="cancel"?await f(t,e,{reason:"stranded"}):await f(t,e);if(e!=="park")return;const i=await v(t),n=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await u(t,"notice.close",`Resident concluded \u2014 ${l(r)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${n} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}a(z,"applyParkEvent");async function D(t){const e=await h(t);e!==null&&await u(t,"notice.crash",`Child went idle unfinished \u2014 ${l(e)} (${t}) stopped without pushing a final report and has no wait that will wake it. Its engine was released; the cause is not recorded. \`crtr node inspect show ${t}\` shows where it stands; a message to it revives it.`,{reason:"child-idle-unfinished",child:t},"urgent")}a(D,"noticeUnfinishedIdle");export{z as applyParkEvent,c as hasHumanViewer,o as hasLiveObligation,F as hasPendingWake,D as noticeUnfinishedIdle,T as parkEvent,W as parkVerdict,M as retentionHold};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var E=Object.defineProperty;var l=(r,t)=>E(r,"name",{value:t,configurable:!0});import{clearFrozen as m,endLiveExecution as h,getNode as R}from"../../../core/canvas/index.js";import{isSafeNodeId as b}from"../../../core/canvas/paths.js";import{recordedPidLiveness as _}from"../../../core/canvas/pid.js";import{hasObservedExit as L,hasAbsorbedProfilePauseStop as z}from"../../../core/canvas/executions.js";import{emitEvent as y}from"../../../core/events/emit.js";import{operationIdContext as I}from"../../../core/events/operation-id.js";import{readInterruptedTurn as g}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as w,isParkedResident as N}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as M}from"../../fleet.js";import{applyParkEvent as F,hasPendingWake as x,parkEvent as A}from"../live-obligation.js";import{FreezeLaneReconciler as C}from"./freeze-lane.js";import{isInBackoff as P}from"./respawn-policy.js";import{asNodePerson as D,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as S}from"./terminating.js";import{wantsExecution as T}from"./wants-execution.js";import{enactRequestedStops as U}from"./requested-stops.js";import{endRemoteExecutionIfGone as G,recoverLaunching as H,wakeResident as W}from"./remote-wake.js";import{liveExecutionForNode as q}from"../../../core/canvas/executions.js";function B(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}l(B,"isTerminal");const O=12e4;function j(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<O}l(j,"isLaunchInFlight");class me{static{l(this,"NodeLifecycleReconciler")}freezeLane=new C;async run(t,e){const a=[];await U(t);for(const n of e.rows)await I.fresh(async()=>{try{await D(n,()=>this.classifyRow(t,n,e,a))}catch(i){const s=n.node_id;y({level:i instanceof k?"warn":"error",event:i instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...b(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(B(e)){if(S(e,await a.fleet.has(i)))return;if(N(e)&&!await a.fleet.has(i)&&!d(e.profile_id)&&!await v(i)&&!P(e,t)&&await p(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead"&&await h(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await m(i):n.push({row:e,enact:"thaw"}));return}if(await a.fleet.has(i)){const f=await q(i);if(f?.runner==="blaxel"&&f.state==="running"&&!d(e.profile_id)&&await p(i)){if(await W(f)==="lost")return}else f?.runner==="blaxel"&&f.state==="running"&&await G(f);if(await a.fleet.has(i)){d(e.profile_id)?await a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}}if(await H(i,t),d(e.profile_id)){await this.freezeLane.endEpisode(i);return}if(await v(i))return;const s=e.pi_pid==null&&await L(i),u=e.pi_pid==null&&!s&&await z(i),c=s||u||e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead";if(c&&!await a.fleet.hasProfilePauseInterruption(i)&&await h(i,{reason:"observed-dead"}),!await T(e)){await this.thawMark(e),await this.settleParked(e);return}if(P(e,t))return;if(e.status==="idle"&&w(e)&&await p(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!s||e.frozen_at!=null)&&!u){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(await a.fleet.isClaimed(i)||j(e,t))return;y({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!c)return;const o=M(await R(i),(await g(i))?.interrupted===!0,await x(i));if(o==="respawn-fresh"||o==="respawn-cycle"||o==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await m(t.node_id)}async settleParked(t){if(t.status!=="idle")return;await this.freezeLane.endEpisode(t.node_id);const e=await A(t.node_id,t.lifecycle);e!=="release"&&await F(t.node_id,e)}}export{O as LAUNCH_GRACE_MS,me as NodeLifecycleReconciler,j as isLaunchInFlight};
+var b=Object.defineProperty;var o=(r,a)=>b(r,"name",{value:a,configurable:!0});import{clearFrozen as P,getNode as x}from"../../../core/canvas/index.js";import{isSafeNodeId as z}from"../../../core/canvas/paths.js";import{recordedPidLiveness as L}from"../../../core/canvas/pid.js";import{endExecution as v,hasAbsorbedProfilePauseStop as g,hasObservedExit as N,latestExecutionForNode as M,liveExecutionForNode as c}from"../../../core/canvas/executions.js";import{emitEvent as w}from"../../../core/events/emit.js";import{operationIdContext as F}from"../../../core/events/operation-id.js";import{readInterruptedTurn as D}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as I}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as A,isParkedResident as C}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as S}from"../../fleet.js";import{applyParkEvent as T,hasPendingWake as U,parkEvent as W}from"../live-obligation.js";import{FreezeLaneReconciler as B}from"./freeze-lane.js";import{isInBackoff as E}from"./respawn-policy.js";import{asNodePerson as G,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as H}from"./terminating.js";import{wantsExecution as q}from"./wants-execution.js";import{enactRequestedStops as O}from"./requested-stops.js";import{endRemoteExecutionIfGone as j,recoverLaunching as J,wakeResident as K}from"./remote-wake.js";function Q(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}o(Q,"isTerminal");const V=12e4;function X(r,a){const e=Math.max(...[r.created,r.launched_at].map(i=>i==null?0:Date.parse(i)||0));return e>0&&a-e<V}o(X,"isLaunchInFlight");async function R(r){const a=await c(r);return a!==null&&a.state==="running"&&a.pid!=null&&L(a.pid,a.processIdentity)==="dead"?a:null}o(R,"executionWithDeadPid");class Pe{static{o(this,"NodeLifecycleReconciler")}freezeLane=new B;async run(a,e){const i=[];await O(a);for(const s of e.rows)await F.fresh(async()=>{try{await G(s,()=>this.classifyRow(a,s,e,i))}catch(t){const l=s.node_id;w({level:t instanceof k?"warn":"error",event:t instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...z(l)?{node_id:l}:{fields:{affected_node_id:l}},error:t})}});return this.freezeLane.run(a,{fleet:e.fleet,candidates:i})}async classifyRow(a,e,i,s){const t=e.node_id;if(Q(e)){if(H(e,await i.fleet.has(t)))return;if(C(e)&&!await i.fleet.has(t)&&!d(e.profile_id)&&!await I(t)&&!E(e,a)&&await p(t)){s.push({row:e,enact:"reopen"});return}if(e.pi_pid!=null){const n=await R(t);n!==null&&await v(n.executionId,{reason:"observed-dead"})}e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await P(t):s.push({row:e,enact:"thaw"}));return}if(await i.fleet.has(t)){const n=await c(t);if(n?.runner==="blaxel"&&n.state==="running"&&!d(e.profile_id)&&await p(t)){if(await K(n)==="lost")return}else n?.runner==="blaxel"&&n.state==="running"&&await j(n);if(await i.fleet.has(t)){d(e.profile_id)?await i.fleet.interruptForProfilePause(t):await this.thawMark(e);return}}if(await J(t,a),d(e.profile_id)){await this.freezeLane.endEpisode(t);return}if(await I(t))return;const l=e.pi_pid==null?await M(t):null,f=l!==null&&N(l),m=l!==null&&!f&&g(l),h=e.pi_pid!=null?await R(t):null,_=f||m||h!==null;if(_&&!await i.fleet.hasProfilePauseInterruption(t)){const n=h??l;n!==null&&await v(n.executionId,{reason:"observed-dead"});const y=await c(t);if(y!==null&&y.executionId!==n?.executionId)return}if(!await q(e)){await this.thawMark(e),await this.settleParked(e);return}if(E(e,a))return;if(e.status==="idle"&&A(e)&&await p(t)){s.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!f||e.frozen_at!=null)&&!m){if(e.respawn_not_before!=null){s.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){s.push({row:e,enact:"thaw"});return}if(await i.fleet.isClaimed(t)||X(e,a))return;w({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),s.push({row:e,enact:"thaw"});return}if(!_)return;const u=S(await x(t),(await D(t))?.interrupted===!0,await U(t));if(u==="respawn-fresh"||u==="respawn-cycle"||u==="respawn-resume"){s.push({row:e,enact:"policy"});return}await i.fleet.applyDeadRowPolicy(t),await this.thawMark(e)}async thawMark(a){a.frozen_at!=null&&await P(a.node_id)}async settleParked(a){if(a.status!=="idle")return;await this.freezeLane.endEpisode(a.node_id);const e=await W(a.node_id,a.lifecycle);e!=="release"&&await T(a.node_id,e)}}export{V as LAUNCH_GRACE_MS,Pe as NodeLifecycleReconciler,X as isLaunchInFlight};

package/dist/launcher/run.js

@@ -1,3 +1,3 @@
-var Q=Object.defineProperty;var h=(_,k)=>Q(_,"name",{value:k,configurable:!0});import{spawn as W}from"node:child_process";import{createHash as X}from"node:crypto";import{chownSync as K,chmodSync as H,closeSync as Y,existsSync as Z,mkdirSync as $,readFileSync as ee,rmSync as te,statSync as re}from"node:fs";import{createServer as ne}from"node:net";import{dirname as F,join as E}from"node:path";import{fileURLToPath as oe}from"node:url";import{loadLinuxNative as se}from"../native/linux.js";import{envHomeOverride as ie}from"../shared/env.js";import{launchSandbox as ae}from"./bwrap.js";import{LaunchGate as de}from"./launch-gate.js";import{serveAccepted as ce}from"./accepted-socket.js";import{installNetworkRule as ue}from"./netrule.js";import{LAUNCHER_SOCKET as S,LaunchRefused as u}from"./spec.js";import{systemIdentity as pe,AppUsers as le}from"./users.js";import{SourceValidator as fe}from"./validate.js";import{prepareDaemonScopeDirectories as me}from"./scope-directories.js";import{LimitsManager as he}from"./limits/manager.js";import{CRTRD_OOM_SCORE_ADJ as we}from"./limits/policy.js";import{writeLimitsLog as ge}from"./limits/log.js";const P=se();function J(_){const k=_._handle?.fd;if(k===void 0||k<0)throw new Error("launcher socket fd unavailable");return k}h(J,"socketFd");async function Pe(_={}){const k=_.superviseDaemon??!0;if(process.platform!=="linux"||process.getuid?.()!==0||!P)throw new Error("launcher requires Linux root and its native addon");const{daemonUid:L,daemonGid:x,dataGid:M}=pe(),T=F(S);if($(T,{recursive:!0,mode:493}),(re(T).mode&511)!==493)throw new Error(`${T} must be mode 0755`);K(T,L,x),process.umask(7);const N=E(T,"nodes");$(N,{recursive:!0,mode:504}),K(N,0,x),H(N,504);const v=new fe(ie()||"/",x);me(E(v.root,"runtime"),L,x);const b=new le(M);ue();const d=new he({runtimeRoot:v.runtime,appsRoot:E(v.root,"apps"),stateDir:E(v.runtime,"limits-state"),dataGid:M,users:b});d.init();const G=new Map,j=h(r=>{const p=d.takeEvents();for(const o of p)ge(o);return p.length===0?r:{...r,events:p}},"reply_"),B=new de,I=new Map,R=new Map,q=h((r,p)=>{const o=p.capture,c=X("sha256"),w=h((l,i)=>{let f=0,a=!1;const m=[];return{take(y){i?.update(y);const D=Math.max(0,Math.min(y.length,l-f));D&&(m.push(y.subarray(0,D)),f+=D),D!==y.length&&(a=!0)},result:h(()=>({text:Buffer.concat(m).toString("utf8"),truncated:a}),"result")}},"chunks"),e=w(o.stdoutBytes,c),s=w(o.stderrBytes);let t=Buffer.alloc(0),n=0;return r.stdout?.on("data",l=>e.take(l)),r.stderr?.on("data",l=>{const i=l;o.stderrTail?(n+=i.length,t=i.length>=o.stderrBytes?i.subarray(i.length-o.stderrBytes):Buffer.concat([t,i]).subarray(Math.max(0,t.length+i.length-o.stderrBytes))):s.take(i)}),new Promise(l=>r.once("close",(i,f)=>{const a=e.result(),m=o.stderrTail?{text:t.toString("utf8"),truncated:n>o.stderrBytes}:s.result();l({exitCode:i,signal:f,stdout:a.text,stderr:m.text,stdoutTruncated:a.truncated,stderrTruncated:m.truncated,stdoutSha256:c.digest("hex")})}))},"captureExit"),V=h(r=>{const p=new Set;for(;r>1&&!p.has(r);){const o=I.get(r);if(o)return o;p.add(r);try{const c=ee(`/proc/${r}/status`,"utf8");r=Number(/^PPid:\s*(\d+)/m.exec(c)?.[1]??0)}catch{return null}}return null},"sandboxForPid");let g,C=!1;const z=E(F(oe(import.meta.url)),"../daemon/crtrd-cli.js"),U=h(()=>{C||(g=W("/bin/setpriv",["--reuid",String(L),"--regid",String(x),"--groups",String(M),"--",process.execPath,z,...process.argv.slice(2)],{stdio:"inherit",env:{...process.env,CRTR_LAUNCHER_SOCKET:S,CRTR_HOME:v.root}}),g.pid&&d.protectDaemon(g.pid,we),g.once("exit",()=>{g=void 0,C||U()}),g.once("error",r=>{console.error("crtrd spawn failed:",r),process.exitCode=1,C=!0}))},"startDaemon");Z(S)&&te(S);const O=ne({pauseOnConnect:!0},r=>ce(r,p=>{const o=[];return(async()=>{try{const c=P.peerCred(J(r)).uid;if(c!==L&&c!==0)throw new u("invalid_spec","launcher peer is not crtrd or root");let w;for(;;)try{w=P.recvFds(J(r));break}catch(t){if(t.code!=="EINTR")throw t}p(),o.push(...w.fds);const e=JSON.parse(w.data);let s;switch(e.operation){case"launch":{if(o.length>1)throw new u("invalid_uid","app uid is not registered");let t,n,l;try{n=await B.launch(async()=>{if(!b.has(e.spec.uid))throw new u("invalid_uid","app uid is not registered");const i=l=d.prepareRun(e.spec.uid,e.spec.nodeId,e.spec.broker===!0);return ae(e.spec,o[0],v,M,f=>{if(!f.pid)throw new u("launch_failed","bwrap has no pid");t=f.pid;const a=f.pid;if(I.set(a,{child:f,uid:e.spec.uid,nodeId:e.spec.nodeId,mounts:e.spec.mounts,plan:i}),f.once("exit",()=>{I.delete(a);const m=d.finishRun(i);G.set(a,m.oomKilled),setTimeout(()=>G.delete(a),5*6e4).unref()}),e.spec.capture){const m=q(f,e.spec).then(y=>({...y,oomKilled:G.get(a)===!0}));R.set(a,m),m.then(()=>{setTimeout(()=>{R.get(a)===m&&R.delete(a)},3e5).unref()})}},i)})}catch(i){throw t!==void 0?R.delete(t):l&&d.finishRun(l),i}if(!n.pid)throw new u("launch_failed","bwrap has no pid");d.confirmStarted(l,n.pid),s={pid:n.pid};break}case"wait":{const t=R.get(e.pid);if(!t)throw new u("invalid_spec","unknown captured sandbox pid");const n=await t;r.writable&&R.delete(e.pid),s={result:n};break}case"stop":{const t=Number.isInteger(e.pid)?V(e.pid):null;if(!t||e.nodeId!==void 0&&t.nodeId!==e.nodeId)throw new u("invalid_spec","unknown sandbox pid");if(e.signal!==void 0&&e.signal!=="SIGTERM"&&e.signal!=="SIGKILL")throw new u("invalid_spec","invalid stop signal");const n=e.signal??"SIGKILL";t.child.kill(n),n==="SIGKILL"&&d.killRun(t.plan),s={ok:!0};break}case"user.create":s={uid:await B.write(()=>b.create(e.app,e.uid))};break;case"user.delete":{await B.write(()=>{if([...I.values()].some(t=>t.uid===e.uid))throw new u("invalid_spec","stop this app\u2019s sandboxes before deleting its user");b.delete(e.uid)}),s={ok:!0};break}case"users.sync":await B.write(()=>b.sync(e.users)),s={ok:!0};break;case"list":s={sandboxes:[...I].map(([t,n])=>({pid:t,uid:n.uid,nodeId:n.nodeId,mounts:n.mounts}))};break;case"limits.apply":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new u("invalid_uid","app is not registered");d.applyApp(t.app,t.uid,!0),s={ok:!0};break}case"memory.admit":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new u("invalid_uid","app is not registered");s=d.admitBroker(t.app,e.nodeId);break}case"limits.show":s={limits:d.show()};break;case"disk.usage":s={disk:await d.diskUsageReport()};break;case"cache.purge":s={purged:await d.cachePass()};break;case"exit.status":s={oomKilled:G.get(e.pid)===!0};break;case"daemon.handover":g?.kill("SIGTERM"),s={ok:!0};break;default:throw new u("invalid_spec","unknown launcher operation")}return`${JSON.stringify(j(s))}
+var W=Object.defineProperty;var h=(L,k)=>W(L,"name",{value:k,configurable:!0});import{spawn as X}from"node:child_process";import{createHash as Y}from"node:crypto";import{chownSync as P,chmodSync as $,closeSync as Z,existsSync as ee,mkdirSync as F,readFileSync as te,rmSync as re,statSync as ne}from"node:fs";import{createServer as oe}from"node:net";import{dirname as J,join as M}from"node:path";import{fileURLToPath as se}from"node:url";import{loadLinuxNative as ie}from"../native/linux.js";import{envHomeOverride as ae}from"../shared/env.js";import{launchSandbox as de}from"./bwrap.js";import{LaunchGate as ce}from"./launch-gate.js";import{serveAccepted as ue}from"./accepted-socket.js";import{installNetworkRule as pe}from"./netrule.js";import{LAUNCHER_SOCKET as S,LaunchRefused as u}from"./spec.js";import{systemIdentity as le,AppUsers as fe}from"./users.js";import{SourceValidator as me}from"./validate.js";import{prepareDaemonScopeDirectories as he}from"./scope-directories.js";import{LimitsManager as we}from"./limits/manager.js";import{CRTRD_OOM_SCORE_ADJ as ge}from"./limits/policy.js";import{writeLimitsLog as ye}from"./limits/log.js";const U=ie();function j(L){const k=L._handle?.fd;if(k===void 0||k<0)throw new Error("launcher socket fd unavailable");return k}h(j,"socketFd");async function Ae(L={}){const k=L.superviseDaemon??!0;if(process.platform!=="linux"||process.getuid?.()!==0||!U)throw new Error("launcher requires Linux root and its native addon");const{daemonUid:T,daemonGid:x,dataGid:G}=le(),E=J(S);if(F(E,{recursive:!0,mode:493}),(ne(E).mode&511)!==493)throw new Error(`${E} must be mode 0755`);P(E,T,x),process.umask(7);const O=M(E,"nodes");F(O,{recursive:!0,mode:504}),P(O,0,x),$(O,504);const v=new me(ae()||"/",x);he(M(v.root,"runtime"),T,x);const b=new fe(G);pe();const d=new we({runtimeRoot:v.runtime,appsRoot:M(v.root,"apps"),stateDir:M(v.runtime,"limits-state"),dataGid:G,users:b});d.init();const B=new Map,C=new Map,q=h(r=>{const p=d.takeEvents();for(const o of p)ye(o);return p.length===0?r:{...r,events:p}},"reply_"),D=new ce,I=new Map,R=new Map,V=h((r,p)=>{const o=p.capture,c=Y("sha256"),w=h((l,i)=>{let f=0,a=!1;const m=[];return{take(g){i?.update(g);const _=Math.max(0,Math.min(g.length,l-f));_&&(m.push(g.subarray(0,_)),f+=_),_!==g.length&&(a=!0)},result:h(()=>({text:Buffer.concat(m).toString("utf8"),truncated:a}),"result")}},"chunks"),e=w(o.stdoutBytes,c),s=w(o.stderrBytes);let t=Buffer.alloc(0),n=0;return r.stdout?.on("data",l=>e.take(l)),r.stderr?.on("data",l=>{const i=l;o.stderrTail?(n+=i.length,t=i.length>=o.stderrBytes?i.subarray(i.length-o.stderrBytes):Buffer.concat([t,i]).subarray(Math.max(0,t.length+i.length-o.stderrBytes))):s.take(i)}),new Promise(l=>r.once("close",(i,f)=>{const a=e.result(),m=o.stderrTail?{text:t.toString("utf8"),truncated:n>o.stderrBytes}:s.result();l({exitCode:i,signal:f,stdout:a.text,stderr:m.text,stdoutTruncated:a.truncated,stderrTruncated:m.truncated,stdoutSha256:c.digest("hex")})}))},"captureExit"),z=h(r=>{const p=new Set;for(;r>1&&!p.has(r);){const o=I.get(r);if(o)return o;p.add(r);try{const c=te(`/proc/${r}/status`,"utf8");r=Number(/^PPid:\s*(\d+)/m.exec(c)?.[1]??0)}catch{return null}}return null},"sandboxForPid");let y,N=!1;const Q=M(J(se(import.meta.url)),"../daemon/crtrd-cli.js"),A=h(()=>{N||(y=X("/bin/setpriv",["--reuid",String(T),"--regid",String(x),"--groups",String(G),"--",process.execPath,Q,...process.argv.slice(2)],{stdio:"inherit",env:{...process.env,CRTR_LAUNCHER_SOCKET:S,CRTR_HOME:v.root}}),y.pid&&d.protectDaemon(y.pid,ge),y.once("exit",()=>{y=void 0,N||A()}),y.once("error",r=>{console.error("crtrd spawn failed:",r),process.exitCode=1,N=!0}))},"startDaemon");ee(S)&&re(S);const K=oe({pauseOnConnect:!0},r=>ue(r,p=>{const o=[];return(async()=>{try{const c=U.peerCred(j(r)).uid;if(c!==T&&c!==0)throw new u("invalid_spec","launcher peer is not crtrd or root");let w;for(;;)try{w=U.recvFds(j(r));break}catch(t){if(t.code!=="EINTR")throw t}p(),o.push(...w.fds);const e=JSON.parse(w.data);let s;switch(e.operation){case"launch":{if(o.length>1)throw new u("invalid_uid","app uid is not registered");let t,n,l;try{n=await D.launch(async()=>{if(!b.has(e.spec.uid))throw new u("invalid_uid","app uid is not registered");const i=l=d.prepareRun(e.spec.uid,e.spec.nodeId,e.spec.broker===!0);return de(e.spec,o[0],v,G,f=>{if(!f.pid)throw new u("launch_failed","bwrap has no pid");t=f.pid;const a=f.pid;if(I.set(a,{child:f,uid:e.spec.uid,nodeId:e.spec.nodeId,mounts:e.spec.mounts,plan:i}),f.once("exit",(m,g)=>{I.delete(a),C.set(a,{code:m,signal:g});const _=d.finishRun(i);B.set(a,_.oomKilled),setTimeout(()=>{B.delete(a),C.delete(a)},5*6e4).unref()}),e.spec.capture){const m=V(f,e.spec).then(g=>({...g,oomKilled:B.get(a)===!0}));R.set(a,m),m.then(()=>{setTimeout(()=>{R.get(a)===m&&R.delete(a)},3e5).unref()})}},i)})}catch(i){throw t!==void 0?R.delete(t):l&&d.finishRun(l),i}if(!n.pid)throw new u("launch_failed","bwrap has no pid");d.confirmStarted(l,n.pid),s={pid:n.pid};break}case"wait":{const t=R.get(e.pid);if(!t)throw new u("invalid_spec","unknown captured sandbox pid");const n=await t;r.writable&&R.delete(e.pid),s={result:n};break}case"stop":{const t=Number.isInteger(e.pid)?z(e.pid):null;if(!t||e.nodeId!==void 0&&t.nodeId!==e.nodeId)throw new u("invalid_spec","unknown sandbox pid");if(e.signal!==void 0&&e.signal!=="SIGTERM"&&e.signal!=="SIGKILL")throw new u("invalid_spec","invalid stop signal");const n=e.signal??"SIGKILL";t.child.kill(n),n==="SIGKILL"&&d.killRun(t.plan),s={ok:!0};break}case"user.create":s={uid:await D.write(()=>b.create(e.app,e.uid))};break;case"user.delete":{await D.write(()=>{if([...I.values()].some(t=>t.uid===e.uid))throw new u("invalid_spec","stop this app\u2019s sandboxes before deleting its user");b.delete(e.uid)}),s={ok:!0};break}case"users.sync":await D.write(()=>b.sync(e.users)),s={ok:!0};break;case"list":s={sandboxes:[...I].map(([t,n])=>({pid:t,uid:n.uid,nodeId:n.nodeId,mounts:n.mounts}))};break;case"limits.apply":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new u("invalid_uid","app is not registered");d.applyApp(t.app,t.uid,!0),s={ok:!0};break}case"memory.admit":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new u("invalid_uid","app is not registered");s=d.admitBroker(t.app,e.nodeId);break}case"limits.show":s={limits:d.show()};break;case"disk.usage":s={disk:await d.diskUsageReport()};break;case"cache.purge":s={purged:await d.cachePass()};break;case"exit.status":s={oomKilled:B.get(e.pid)===!0,code:C.get(e.pid)?.code??null,signal:C.get(e.pid)?.signal??null};break;case"daemon.handover":y?.kill("SIGTERM"),s={ok:!0};break;default:throw new u("invalid_spec","unknown launcher operation")}return`${JSON.stringify(q(s))}
 `}catch(c){const w=c instanceof u?c.refusal:"launch_failed";return console.error(`launcher refusal ${w}:`,c),`${JSON.stringify({refusal:w})}
-`}finally{for(const c of o)Y(c)}})()}));await new Promise((r,p)=>{O.once("error",p),O.listen(S,r)}),K(S,L,x),H(S,384);const A=h(()=>{C=!0,g?.kill("SIGTERM");for(const{child:r}of I.values())r.kill("SIGKILL");O.close(),v.close()},"shutdown");process.once("SIGTERM",A),process.once("SIGINT",A),k&&U()}h(Pe,"runLauncher");export{Pe as runLauncher};
+`}finally{for(const c of o)Z(c)}})()}));await new Promise((r,p)=>{K.once("error",p),K.listen(S,r)}),P(S,T,x),$(S,384);const H=h(()=>{N=!0,y?.kill("SIGTERM");for(const{child:r}of I.values())r.kill("SIGKILL");K.close(),v.close()},"shutdown");process.once("SIGTERM",H),process.once("SIGINT",H),k&&A()}h(Ae,"runLauncher");export{Ae as runLauncher};

package/dist/launcher/spec.d.ts

@@ -75,7 +75,7 @@ export type LauncherRequest = {
  | {
     operation: 'cache.purge';
 }
-/** Whether the sandbox with this first-process pid ended because the kernel OOM-killed its run. */
+/** How the sandbox with this first-process pid ended: whether the kernel OOM-killed its run, and the code and signal its first process exited with. */
  | {
     operation: 'exit.status';
     pid: number;
@@ -167,6 +167,8 @@ export type LauncherReply = ({
     purged: CachePurged[];
 } | {
     oomKilled: boolean;
+    code: number | null;
+    signal: NodeJS.Signals | null;
 }) & {
     events?: LimitEvent[];
 } | {

package/dist/pi-extensions/canvas-inbox-watcher.js

@@ -1,2 +1,2 @@
 var U=Object.defineProperty;var f=(s,t)=>U(s,"name",{value:t,configurable:!0});import{envInboxBetweenTurns as q,envNodeId as B,envProfileId as J,envWatcherDebounceMs as W,envWatcherTickMs as $,envWatcherBackstopMs as G}from"../shared/env.js";import{coalesceBrokerInbox as X,entryCards as j}from"../core/runtime/broker/inbox.js";import{acknowledgeBrokerMail as z,claimBrokerMail as L,brokerExtensionState as F}from"../core/runtime/broker/daemon-ops.js";import{materializeSpilledBody as V}from"../core/runtime/broker/spilled-body.js";import{brokerSignalClient as S}from"../core/runtime/broker/signal-stream.js";import{wasTurnRefused as Q}from"../core/runtime/broker/extension-abort.js";import{isDaemonTransportApiError as R,isTokenRevokedApiError as Y}from"../api/errors.js";import{announceMessageOrigin as K,messageSignature as _}from"../core/runtime/stamp/channel.js";import{captureGoalIfAbsent as Z,generateAndCommitInitialName as ee}from"./broker-local.js";import{emitEvent as O}from"../core/events/emit.js";import{errorClassFromError as te}from"../core/events/errors.js";import{operationIdContext as m}from"../core/events/operation-id.js";import{SITUATIONAL_CONTEXT_CUSTOM_TYPE as I}from"../shared/generated-context.js";import{isProfilePaused as ie}from"../core/profiles/manifest.js";import{brokerDisposeAndExit as N,brokerIdleTurnStart as ne,brokerRunTurnBusy as re,HELD_DEFERRED_INBOX_PROMPT_JOIN as se}from"../core/runtime/broker/held-deferred-inbox.js";const b=64,A=16;let p,ae=0;class D extends Error{static{f(this,"WatcherFatalError")}constructor(t,e){super(t,{cause:e}),this.name="WatcherFatalError"}}function de(){return{tickMs:$(),debounceMs:W()}}f(de,"resolveCadence");function E(){const s=B();return s===void 0||s.trim()===""?void 0:s}f(E,"currentNodeId");function P(){const s=J();return s!==void 0&&s!==""&&ie(s)}f(P,"profilePaused");function k(s){try{return te(s)}catch{return{class:"unknown"}}}f(k,"safeErrorClass");function v(s){try{O(s)}catch{}}f(v,"emitBestEffort");function w(s){try{m.fresh(t=>{v({...s,operation_id:t})})}catch{}}f(w,"emitFreshBestEffort");function C(s,t){try{m.run(s,()=>{v({...t,operation_id:s})})}catch{}}f(C,"emitStoredBestEffort");function oe(s){return s.kind==="human"?{type:"human"}:{type:"runtime",kind:"inbox",entries:s.entries.map(t=>({...t.from===null?{}:{from:t.from},...t.from_name===void 0?{}:{fromName:t.from_name},kind:t.kind,tier:t.tier,...t.ref===void 0?{}:{ref:t.ref}}))}}f(oe,"deliveryOrigin");function x(s,t){if(s.length===0)throw new Error(`cannot create an empty ${t} delivery`);const e=[...new Set(s.map(r=>r.operation_id))].sort(),i=e.length-A;return{channel:t,entryIds:s.map(r=>r.entry_id),throughEntryId:s.at(-1).entry_id,sourceOperationIds:e.slice(0,A),...i>0?{sourceOperationIdsOmitted:i}:{}}}f(x,"deliveryFacts");function y(s){return{channel:s.channel,entry_ids:s.entryIds,through_entry_id:s.throughEntryId,source_operation_ids:s.sourceOperationIds,...s.sourceOperationIdsOmitted===void 0?{}:{source_operation_ids_omitted:s.sourceOperationIdsOmitted}}}f(y,"deliveryFields");function le(s){C(s.operationId,{level:"info",event:"inbox.delivery.completed",outcome:"succeeded",fields:y(s)})}f(le,"emitDeliveryCompleted");function T(s,t,e,i){C(s,{level:"error",event:"inbox.delivery.failed",outcome:"failed",error:i,error_class:k(i),fields:{stage:e,...y(t)}})}f(T,"emitDeliveryFailure");function M(s,t){C(s.operationId,{level:"error",event:"inbox.delivery.failed",outcome:"failed",error_class:{class:"unknown"},fields:{stage:"abandoned",reason:t,...y(s)}})}f(M,"emitDeliveryAbandoned");function fe(s){return typeof s=="string"?s:Array.isArray(s)?s.filter(t=>typeof t=="object"&&t!==null&&typeof t.text=="string").map(t=>t.text).join(`
-`):""}f(fe,"messageText");function H(s){return s==="quit"||s==="reload"||s==="new"||s==="resume"||s==="fork"}f(H,"isSessionShutdownReason");function he(s){switch(s){case"quit":return"shutdown";case"reload":case"new":case"resume":case"fork":return"reload";default:return s}}f(he,"disposalReasonForShutdown");function ue(s){return typeof s=="string"?s.slice(0,128):s===null?"null":s===void 0?"undefined":typeof s=="number"||typeof s=="boolean"||typeof s=="bigint"?String(s).slice(0,128):typeof s}f(ue,"boundedRuntimeReason");class ce{static{f(this,"CanvasInboxWatcher")}pi;cadence;generation;timer;backstop;debounceTimer;signalUnsubscribers=[];rerun=!1;retryNeeded=!1;polledOk=!1;armed=!1;lastCtx;streaming=!1;runNode=q();nodeId;claims=new Map;reportBodies=new Map;tickInFlight;buffer=[];lastArrival=0;pendingHandoffs=[];pendingCommit;promptJoinInFlight=!1;replayUnseenInherited=!1;seeded=!1;disposed=!1;disposalReason;fatalError;fatalEscaped=!1;constructor(t,e,i){this.pi=t,this.cadence=e,this.generation=i}start(){const t=f((e,i)=>{this.canMutate()&&this.captureCtx(e,i)},"capture");this.pi.on("session_start",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.arm())}),this.pi.on("turn_end",t),this.pi.on("agent_start",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.streaming=!0)}),this.pi.on("agent_end",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.streaming=!1)}),this.pi.on("message_end",e=>{if(!this.canMutate())return;const i=e?.message;if(i?.role!=="user"&&i?.role!=="custom"||!this.pendingHandoffs.some(d=>d.awaitingDequeue!==void 0))return;const r=_(fe(i.content)),n=this.pendingHandoffs.find(d=>d.awaitingDequeue===r);n!==void 0&&delete n.awaitingDequeue}),this.pi.on("agent_settled",async(e,i)=>{if(!this.canMutate())return;this.captureCtx(e,i);const r=E();if(r!==void 0)try{this.assertNodeOwnership(r,"settle"),await this.tickInFlight,this.seeded||await this.claimMail(r);const n=await this.settle(r,i,!1,!0);n==="refresh_held"&&this.dispose("refresh",!1),this.runNode&&n==="settled"&&this.canMutate()&&this.buffer.length>0&&await this.flushOrdinary(r),this.syncTimers()}catch(n){this.dispose("shutdown",!1,void 0,{error:n,stage:"settle"})}}),this.pi.on("session_shutdown",async e=>{if(!this.canMutate())return;let i;try{i=e?.reason}catch{i=void 0}const r=H(i)?he(i):"reload";H(i)||w({level:"warn",event:"inbox.lifecycle.unknown_shutdown_reason",outcome:"failed",fields:{reason:ue(i)}});try{await this.disposeAtSettlement(r,this.lastCtx)}catch(n){this.dispose(r,!1,void 0,{error:n,stage:"settle"})}})}arm(){if(this.armed)return;this.armed=!0;const t=S();this.signalUnsubscribers.push(t.subscribe("state",()=>this.syncTimers()),t.subscribe("mail",()=>this.runTick("schedule")),t.subscribe("resync",()=>this.runTick("schedule"))),this.syncTimers(),t.state!=="connected"&&this.runTick("schedule")}async disposeAtSettlement(t,e){const i=E();if(i===void 0){this.dispose(t,!1);return}try{this.assertNodeOwnership(i,"settle");const r=await this.settle(i,e,t!=="refresh");this.dispose(r==="refresh_held"?"refresh":t,!1)}catch(r){this.dispose(t,!1,void 0,{error:r,stage:"settle"})}}dispose(t,e,i,r){if(this.disposed)return;this.disposed=!0,this.disposalReason=t,this.clearTimer();for(const d of this.signalUnsubscribers.splice(0))d();const n=r;{const d=[...new Set([...this.pendingCommit?.handoffs??[],...this.pendingHandoffs.splice(0)])];this.buffer.length=0,this.claims.clear(),this.reportBodies.clear(),this.pendingCommit=void 0,this.promptJoinInFlight=!1,this.replayUnseenInherited=!1,this.nodeId=void 0,this.lastCtx=void 0,this.lastArrival=0,this.streaming=!1,this.seeded=!1,this.ownsCurrentGeneration()&&(p=void 0);for(const a of d)M(a,t)}if(n!==void 0){this.scheduleHandlerFatal(n.error,n.stage);return}}ownsCurrentGeneration(){return p===this&&p.generation===this.generation}canMutate(){return this.ownsCurrentGeneration()&&!this.disposed&&this.fatalError===void 0}captureCtx(t,e){e!==void 0&&(this.lastCtx=e)}clearTimer(){this.timer!==void 0&&clearInterval(this.timer),this.backstop!==void 0&&clearInterval(this.backstop),this.debounceTimer!==void 0&&clearTimeout(this.debounceTimer),this.timer=this.backstop=this.debounceTimer=void 0}syncTimers(){if(!this.armed||!this.canMutate())return;const t=S().state==="connected",e=!t||this.buffer.length>0||this.pendingCommit!==void 0||this.retryNeeded;e&&this.timer===void 0?(this.timer=setInterval(()=>this.runTick("schedule"),this.cadence.tickMs),this.timer.unref?.()):!e&&this.timer!==void 0&&(clearInterval(this.timer),this.timer=void 0),t&&this.backstop===void 0?(this.backstop=setInterval(()=>this.runTick("schedule"),G()),this.backstop.unref?.()):!t&&this.backstop!==void 0&&(clearInterval(this.backstop),this.backstop=void 0),this.buffer.length===0&&this.debounceTimer!==void 0&&(clearTimeout(this.debounceTimer),this.debounceTimer=void 0)}armDebounce(){this.debounceTimer!==void 0&&clearTimeout(this.debounceTimer);const t=Math.max(0,this.lastArrival+this.cadence.debounceMs-Date.now());this.debounceTimer=setTimeout(()=>{this.debounceTimer=void 0,this.runTick("schedule")},t),this.debounceTimer.unref?.()}isIdle(){if(this.runNode&&re())return!1;try{if(typeof this.lastCtx?.isIdle=="function")return this.lastCtx.isIdle()===!0}catch{}return!this.streaming}assertNodeOwnership(t,e){!this.seeded||this.nodeId===void 0||this.nodeId===t||this.raiseReadFatal(e,new Error(`watcher node changed from ${this.nodeId} to ${t}`))}raiseFatal(t,e,i,r){throw this.fatalError===void 0&&(this.fatalError=new D(e,i),this.clearTimer(),w({level:"error",event:t,outcome:"failed",error:i,error_class:k(i),fields:r})),this.fatalError}raiseReadFatal(t,e){return this.raiseFatal("inbox.read.failed",`inbox ${t} state is corrupt`,e,{stage:t})}raiseSituationalFatal(t,e){return this.raiseFatal("inbox.situational_context.missing","situational context marker has no readable sidecar",e,{entry_id:t})}escapeFatal(t,e){if(!this.fatalEscaped){if(this.fatalEscaped=!0,e==="throw")throw t;queueMicrotask(()=>{throw t})}}scheduleHandlerFatal(t,e){let i;if(t instanceof D)i=t;else try{this.raiseReadFatal(e,t)}catch(r){i=r}this.escapeFatal(i,"schedule")}runTick(t="throw"){if(!this.canMutate())return;if(this.tickInFlight!==void 0){this.rerun=!0;return}const e=this.tick().then(()=>{this.retryNeeded=!1,this.polledOk=!0}).catch(i=>{if(this.canMutate()){if(R(i)){this.retryNeeded=!0,w({level:"warn",event:"inbox.read.failed",outcome:"failed",error:i,error_class:k(i),fields:{stage:"poll"}});return}if(this.polledOk&&Y(i)){this.clearTimer(),w({level:"info",event:"inbox.poll.token_revoked",outcome:"succeeded",fields:{stage:"poll"}}),N()?.("token-revoked");return}this.scheduleHandlerFatal(i,"poll")}}).finally(()=>{this.tickInFlight===e&&(this.tickInFlight=void 0),this.syncTimers(),this.rerun&&this.canMutate()&&(this.rerun=!1,this.runTick("schedule"))});this.tickInFlight=e}async claimMail(t){this.assertNodeOwnership(t,"poll");let e;try{e=await L(t,"wake",this.reportBodies)}catch(a){throw R(a)&&(this.replayUnseenInherited=!0),a}if(!this.canMutate()||e.action==="hold")return!1;const i=this.replayUnseenInherited;this.replayUnseenInherited=!1,this.nodeId=t,this.seeded=!0;const r=new Set(e.deliveries.map(a=>a.entry.entry_id)),n=this.buffer.filter(a=>!r.has(a.entry_id));for(const a of n)this.claims.delete(a.entry_id);if(this.buffer.splice(0,this.buffer.length,...this.buffer.filter(a=>r.has(a.entry_id))),n.length>0){const a=globalThis[Symbol.for("@crouton-kit/crtr:watcher-cancel-probe")];typeof a=="function"&&a(n.map(l=>l.entry_id),process.hrtime.bigint())}let d=!1;for(const a of e.deliveries){const l=a.entry.entry_id;this.claims.has(l)||(a.entry=V(a.entry),this.claims.set(l,a),a.handoff==="inherited"&&!i?m.fresh(o=>{this.pendingHandoffs.push({...x([a.entry],"visible"),operationId:o,settleAtAgentSettled:!0})}):(this.buffer.push(a.entry),this.lastArrival=Date.now(),d=!0))}return d&&this.armDebounce(),this.syncTimers(),!0}async tick(){const t=E();if(t!==void 0){if(this.pendingCommit!==void 0){await this.retryPendingCommit(t)==="refresh_held"&&this.canMutate()&&this.dispose("refresh",!1);return}this.promptJoinInFlight||P()||this.pendingHandoffs.length>=b||await this.claimMail(t)&&this.buffer.length>0&&(!this.runNode||this.isIdle()||this.buffer.some(e=>e.tier==="critical"))&&Date.now()-this.lastArrival>=this.cadence.debounceMs&&await this.flushOrdinary(t)}}missingChannelEntries(t){const e=t.filter(n=>this.claims.has(n.entry_id)),i=this.coveredEntryIds("hidden"),r=this.coveredEntryIds("visible");return{hiddenEntries:e.filter(n=>(n.data?.situational===!0||n.data?.situationalOnly===!0)&&!i.has(n.entry_id)),visibleEntries:e.filter(n=>n.data?.situationalOnly!==!0&&!r.has(n.entry_id))}}visibleRoute(t){return t.length===0?"none":this.isIdle()?"idle":this.runNode?t.some(e=>e.tier==="critical")?"critical":"none":t.some(e=>e.tier==="critical")?"critical":t.some(e=>e.tier==="urgent"||e.kind==="final")?"steer":"followUp"}buildPlan(t){const{hiddenEntries:e,visibleEntries:i}=this.missingChannelEntries(t),r=new Set(e.map(u=>u.entry_id)),n=new Set(i.map(u=>u.entry_id)),d=t.filter(u=>r.has(u.entry_id)||n.has(u.entry_id)),a=this.isIdle()&&d.length>0&&d.every(u=>u.tier==="deferred"),l=this.visibleRoute(i)==="critical",o=!a&&!l&&e.length>0,c=!a&&!l&&i.length>0;return{hiddenEntries:e,visibleEntries:i,deferredHold:a,sendCount:Number(o)+Number(c)}}ordinaryPlan(){return this.buildPlan(this.buffer)}async contextCards(t,e){const i=[];if(e.some(r=>r.data?.situational===!0)){const r=e.at(-1).entry_id;let n;try{n=(await F(t)).situational_card}catch(d){this.raiseSituationalFatal(r,d)}n===null&&this.raiseSituationalFatal(r,new Error(`missing situational context for entry ${r}`)),i.push(n)}for(const r of e)r.data?.situationalOnly===!0&&i.push(...j(r));return i}digest(t,e){try{return X(t)}catch(i){this.raiseReadFatal(e,i)}}sendDelivery(t,e,i,r,n){this.pendingHandoffs.length>=b&&this.raiseReadFatal(r,new Error(`pending handoff capacity exceeded before ${t} send`));let d;try{d=x(e,t)}catch(o){this.raiseReadFatal(r,o)}let a;try{m.fresh(o=>{a=o,v({level:"info",event:"inbox.delivery.started",operation_id:o,fields:y(d)}),i()})}catch(o){return a===void 0&&this.raiseReadFatal(r,o),T(a,d,"send",o),"failed"}a===void 0&&this.raiseReadFatal(r,new Error(`${t} delivery returned without an operation id`));const l={operationId:a,...d,...n===void 0?{}:{awaitingDequeue:n}};return this.canMutate()?(this.pendingHandoffs.push(l),"handed-off"):(this.disposed&&this.disposalReason!==void 0&&M(l,this.disposalReason),"inactive")}sendContext(t,e,i,r,n,d){const a=r==="steer"||r==="followUp"?r:this.isIdle()?void 0:"followUp",l=a===void 0||i.length===0?void 0:_(i.at(-1));return this.sendDelivery("hidden",e,()=>{this.runNode&&n&&this.pi.appendEntry("crtr-turn-input",{message_ids:e.map(o=>o.entry_id)}),i.forEach((o,c)=>{const u=n&&c===i.length-1;this.pi.sendMessage({customType:I,content:o,display:!0,details:{nodeId:t}},a!==void 0?{deliverAs:a}:u?{deliverAs:"followUp",triggerTurn:!0}:{})})},d,l)}sendVisible(t,e,i,r,n,d){return this.sendDelivery("visible",t,()=>{if(n!=="idle"&&n!=="steer"&&n!=="followUp")throw new Error(`visible send requested for non-send route ${n}`);for(const a of e)this.pi.sendMessage({customType:I,content:a,display:!0},n==="idle"?{}:{deliverAs:n});if(K(i,r),n==="idle"){const a=ne();if(a===void 0)throw new Error("broker idle-turn admission is unavailable");a(i,this.runNode?t.map(l=>l.entry_id):[])}else this.pi.sendUserMessage(i,{deliverAs:n})},d,n==="steer"||n==="followUp"?_(i):void 0)}abortCritical(){try{this.lastCtx?.abort?.()}catch{}}requeueFromBatch(t,e){if(e.size===0)return;const i=t.filter(d=>e.has(d.entry_id)),r=new Set(i.map(d=>d.entry_id)),n=this.buffer.filter(d=>!r.has(d.entry_id));this.buffer.splice(0,this.buffer.length,...i,...n)}async executePlan(t,e,i){const r=new Set,n=this.visibleRoute(e.visibleEntries);if(n==="critical"){if(this.abortCritical(),!this.canMutate())return{inactive:!0,requeueEntryIds:r};for(const o of e.visibleEntries)r.add(o.entry_id);for(const o of e.hiddenEntries)r.add(o.entry_id);return{inactive:!1,requeueEntryIds:r}}if(this.runNode&&n==="none")return{inactive:!1,requeueEntryIds:new Set([...e.hiddenEntries,...e.visibleEntries].map(o=>o.entry_id))};const d=e.visibleEntries.length>0?this.digest(e.visibleEntries,i):[],[a,...l]=d;for(const o of l)for(const c of o.entries)r.add(c.entry_id);if(e.hiddenEntries.length>0){const o=await this.contextCards(t,e.hiddenEntries),c=this.sendContext(t,e.hiddenEntries,o,n,a===void 0,i);if(c==="inactive")return{inactive:!0,requeueEntryIds:r};if(c==="failed")for(const u of e.hiddenEntries)r.add(u.entry_id)}if(!this.canMutate())return{inactive:!0,requeueEntryIds:r};if(a!==void 0){const o=this.sendVisible(a.entries,a.cards,a.text,oe(a),n,i);if(o==="inactive")return{inactive:!0,requeueEntryIds:r};if(o==="failed")for(const c of a.entries)r.add(c.entry_id);else a.kind==="human"&&(await Z(t,a.text).catch(()=>{}),ee(t,a.text))}return{inactive:!1,requeueEntryIds:r}}async flushOrdinary(t){if(this.buffer.length===0)return"empty";const e=this.ordinaryPlan();if(e.deferredHold)return"held";if(this.pendingHandoffs.length+e.sendCount>b)return"capacity-blocked";const i=[...this.buffer],r=await this.executePlan(t,e,"poll");return r.inactive||(this.buffer.splice(0,i.length),this.requeueFromBatch(i,r.requeueEntryIds)),"attempted"}async prependHeldDeferredInbox(t){if(!this.canMutate()||this.nodeId===void 0||P())return;const e=this.ordinaryPlan();if(!e.deferredHold)return;const i=e.visibleEntries.length===0?[]:this.digest(e.visibleEntries,"poll"),r=+(e.hiddenEntries.length>0)+i.length;if(this.pendingHandoffs.length+r>b)return;const n=[...this.buffer];this.promptJoinInFlight=!0;try{this.runNode&&this.pi.appendEntry("crtr-turn-input",{message_ids:[...e.hiddenEntries,...e.visibleEntries].map(d=>d.entry_id)}),e.hiddenEntries.length>0&&await this.sendPromptJoinDelivery(t,"hidden",e.hiddenEntries,await this.contextCards(this.nodeId,e.hiddenEntries));for(const d of i)await this.sendPromptJoinDelivery(t,"visible",d.entries,[...d.cards,d.text]);if(!this.canMutate())return;this.buffer.splice(0,n.length)}finally{this.promptJoinInFlight=!1}}async sendPromptJoinDelivery(t,e,i,r){let n;try{n=x(i,e)}catch(l){this.raiseReadFatal("poll",l)}let d;try{await m.fresh(async l=>{d=l,v({level:"info",event:"inbox.delivery.started",operation_id:l,fields:y(n)});for(const o of r)await t.sendCustomMessage({customType:I,content:o,display:!0})})}catch(l){throw d===void 0&&this.raiseReadFatal("poll",l),T(d,n,"send",l),l}d===void 0&&this.raiseReadFatal("poll",new Error(`${e} prompt-join delivery returned without an operation id`));const a={operationId:d,...n,settleAtAgentSettled:!0};if(!this.canMutate()){this.disposed&&this.disposalReason!==void 0&&M(a,this.disposalReason);return}this.pendingHandoffs.push(a)}coveredEntryIds(t){const e=new Set;for(const i of this.pendingHandoffs)if(i.channel===t)for(const r of i.entryIds)e.add(r);return e}retryPendingCommit(t){const e=this.pendingCommit;if(e===void 0)return Promise.resolve("settled");if(e.attempt!==void 0)return e.attempt;const i=this.attemptPendingCommit(t,e).finally(()=>{e.attempt===i&&(e.attempt=void 0)});return e.attempt=i,i}async attemptPendingCommit(t,e){let i=!1;try{const n=await z(t,e.receipts);if(n.action==="hold")return"refresh_held";for(const d of e.receipts){const a=n.results.find(l=>l.delivery_id===d.delivery_id);if(a?.outcome!==d.disposition&&a?.outcome!=="stale")throw new Error("daemon returned no terminal mail settlement outcome")}i=n.results.every(d=>d.outcome==="acknowledged")}catch(n){if(this.pendingCommit===e)for(const d of e.handoffs)T(d.operationId,d,"receipt_commit",n);return String(n?.message).startsWith("conversation_write_failed")&&(O({level:"error",event:"broker.exited",node_id:t,error:n,fields:{fault:"conversation_write_failed"}}),N()?.("conversation_write_failed",1)),"held"}if(this.pendingCommit!==e)return"settled";for(const n of e.entryIds)this.claims.delete(n);if(e.receipts.some(n=>n.disposition==="released")){const n=new Set(e.entryIds);this.buffer.splice(0,this.buffer.length,...this.buffer.filter(d=>!n.has(d.entry_id)))}this.pendingCommit=void 0;const r=this.detachPending(n=>n.entryIds.every(d=>!this.claims.has(d)));for(const n of r)i&&le(n);return"settled"}async settle(t,e,i,r=!1){if(i)try{if(e?.hasPendingMessages?.()===!0)return"held"}catch{}if(this.pendingCommit!==void 0)return this.retryPendingCommit(t);const n=Q(),d=!n&&this.pendingHandoffs.some(h=>h.awaitingDequeue!==void 0)&&await this.nodeRefreshing(t);if(this.pendingCommit!==void 0)return this.retryPendingCommit(t);const a=new Set(this.buffer.map(h=>h.entry_id)),l=new Set(d?this.pendingHandoffs.filter(h=>h.awaitingDequeue!==void 0).flatMap(h=>[...h.entryIds]):[]),o=this.pendingHandoffs.filter(h=>(r||h.settleAtAgentSettled!==!0)&&h.entryIds.every(g=>!l.has(g))),c=new Set(o.flatMap(h=>[...h.entryIds])),u=[...this.claims.keys()].filter(h=>n||c.has(h)&&!a.has(h));return u.length===0?"settled":(this.pendingCommit={handoffs:o,entryIds:u,receipts:u.map(h=>{const g=this.claims.get(h);return{delivery_id:g.delivery_id,attempts:g.attempts,disposition:n?"released":"acknowledged"}})},this.retryPendingCommit(t))}async nodeRefreshing(t){try{return(await F(t)).node.intent==="refresh"}catch{return!0}}detachPending(t){const e=[];for(let i=this.pendingHandoffs.length-1;i>=0;i-=1){const r=this.pendingHandoffs[i];t(r)&&e.push(...this.pendingHandoffs.splice(i,1))}return e.reverse(),e}}function me(s,t){p?.dispose("reload",!1);const e=new ce(s,t,++ae);p=e;const i=f(r=>e.prependHeldDeferredInbox(r),"promptJoin");globalThis[se]=i;try{e.start()}catch(r){throw e.dispose("reload",!1),r}return e}f(me,"installWatcher");function pe(s){const t=me(s,de());return()=>t.dispose("shutdown",!1)}f(pe,"registerCanvasInboxWatcher");var Oe=pe;export{Oe as default,pe as registerCanvasInboxWatcher};
+`):""}f(fe,"messageText");function H(s){return s==="quit"||s==="reload"||s==="new"||s==="resume"||s==="fork"}f(H,"isSessionShutdownReason");function he(s){switch(s){case"quit":return"shutdown";case"reload":case"new":case"resume":case"fork":return"reload";default:return s}}f(he,"disposalReasonForShutdown");function ue(s){return typeof s=="string"?s.slice(0,128):s===null?"null":s===void 0?"undefined":typeof s=="number"||typeof s=="boolean"||typeof s=="bigint"?String(s).slice(0,128):typeof s}f(ue,"boundedRuntimeReason");class ce{static{f(this,"CanvasInboxWatcher")}pi;cadence;generation;timer;backstop;debounceTimer;signalUnsubscribers=[];rerun=!1;retryNeeded=!1;polledOk=!1;armed=!1;lastCtx;streaming=!1;runNode=q();nodeId;claims=new Map;reportBodies=new Map;tickInFlight;buffer=[];lastArrival=0;pendingHandoffs=[];pendingCommit;promptJoinInFlight=!1;replayUnseenInherited=!1;seeded=!1;disposed=!1;disposalReason;fatalError;fatalEscaped=!1;constructor(t,e,i){this.pi=t,this.cadence=e,this.generation=i}start(){const t=f((e,i)=>{this.canMutate()&&this.captureCtx(e,i)},"capture");this.pi.on("session_start",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.arm())}),this.pi.on("turn_end",t),this.pi.on("agent_start",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.streaming=!0)}),this.pi.on("agent_end",(e,i)=>{this.canMutate()&&(this.captureCtx(e,i),this.streaming=!1)}),this.pi.on("message_end",e=>{if(!this.canMutate())return;const i=e?.message;if(i?.role!=="user"&&i?.role!=="custom"||!this.pendingHandoffs.some(d=>d.awaitingDequeue!==void 0))return;const r=_(fe(i.content)),n=this.pendingHandoffs.find(d=>d.awaitingDequeue===r);n!==void 0&&delete n.awaitingDequeue}),this.pi.on("agent_settled",async(e,i)=>{if(!this.canMutate())return;this.captureCtx(e,i);const r=E();if(r!==void 0)try{this.assertNodeOwnership(r,"settle"),await this.tickInFlight,this.seeded||await this.claimMail(r);const n=await this.settle(r,i,!1,!0);n==="refresh_held"&&this.dispose("refresh",!1),this.runNode&&n==="settled"&&this.canMutate()&&this.buffer.length>0&&await this.flushOrdinary(r),this.syncTimers()}catch(n){this.dispose("shutdown",!1,void 0,{error:n,stage:"settle"})}}),this.pi.on("session_shutdown",async e=>{if(!this.canMutate())return;let i;try{i=e?.reason}catch{i=void 0}const r=H(i)?he(i):"reload";H(i)||w({level:"warn",event:"inbox.lifecycle.unknown_shutdown_reason",outcome:"failed",fields:{reason:ue(i)}});try{await this.disposeAtSettlement(r,this.lastCtx)}catch(n){this.dispose(r,!1,void 0,{error:n,stage:"settle"})}})}arm(){if(this.armed)return;this.armed=!0;const t=S();this.signalUnsubscribers.push(t.subscribe("state",()=>this.syncTimers()),t.subscribe("mail",()=>this.runTick("schedule")),t.subscribe("resync",()=>this.runTick("schedule"))),this.syncTimers(),t.state!=="connected"&&this.runTick("schedule")}async disposeAtSettlement(t,e){const i=E();if(i===void 0){this.dispose(t,!1);return}try{this.assertNodeOwnership(i,"settle");const r=await this.settle(i,e,t!=="refresh");this.dispose(r==="refresh_held"?"refresh":t,!1)}catch(r){this.dispose(t,!1,void 0,{error:r,stage:"settle"})}}dispose(t,e,i,r){if(this.disposed)return;this.disposed=!0,this.disposalReason=t,this.clearTimer();for(const d of this.signalUnsubscribers.splice(0))d();const n=r;{const d=[...new Set([...this.pendingCommit?.handoffs??[],...this.pendingHandoffs.splice(0)])];this.buffer.length=0,this.claims.clear(),this.reportBodies.clear(),this.pendingCommit=void 0,this.promptJoinInFlight=!1,this.replayUnseenInherited=!1,this.nodeId=void 0,this.lastCtx=void 0,this.lastArrival=0,this.streaming=!1,this.seeded=!1,this.ownsCurrentGeneration()&&(p=void 0);for(const a of d)M(a,t)}if(n!==void 0){this.scheduleHandlerFatal(n.error,n.stage);return}}ownsCurrentGeneration(){return p===this&&p.generation===this.generation}canMutate(){return this.ownsCurrentGeneration()&&!this.disposed&&this.fatalError===void 0}captureCtx(t,e){e!==void 0&&(this.lastCtx=e)}clearTimer(){this.timer!==void 0&&clearInterval(this.timer),this.backstop!==void 0&&clearInterval(this.backstop),this.debounceTimer!==void 0&&clearTimeout(this.debounceTimer),this.timer=this.backstop=this.debounceTimer=void 0}syncTimers(){if(!this.armed||!this.canMutate())return;const t=S().state,e=t==="connected",i=t!=="stopped"&&(t!=="connected"||this.buffer.length>0||this.pendingCommit!==void 0||this.retryNeeded);i&&this.timer===void 0?(this.timer=setInterval(()=>this.runTick("schedule"),this.cadence.tickMs),this.timer.unref?.()):!i&&this.timer!==void 0&&(clearInterval(this.timer),this.timer=void 0),e&&this.backstop===void 0?(this.backstop=setInterval(()=>this.runTick("schedule"),G()),this.backstop.unref?.()):!e&&this.backstop!==void 0&&(clearInterval(this.backstop),this.backstop=void 0),this.buffer.length===0&&this.debounceTimer!==void 0&&(clearTimeout(this.debounceTimer),this.debounceTimer=void 0)}armDebounce(){this.debounceTimer!==void 0&&clearTimeout(this.debounceTimer);const t=Math.max(0,this.lastArrival+this.cadence.debounceMs-Date.now());this.debounceTimer=setTimeout(()=>{this.debounceTimer=void 0,this.runTick("schedule")},t),this.debounceTimer.unref?.()}isIdle(){if(this.runNode&&re())return!1;try{if(typeof this.lastCtx?.isIdle=="function")return this.lastCtx.isIdle()===!0}catch{}return!this.streaming}assertNodeOwnership(t,e){!this.seeded||this.nodeId===void 0||this.nodeId===t||this.raiseReadFatal(e,new Error(`watcher node changed from ${this.nodeId} to ${t}`))}raiseFatal(t,e,i,r){throw this.fatalError===void 0&&(this.fatalError=new D(e,i),this.clearTimer(),w({level:"error",event:t,outcome:"failed",error:i,error_class:k(i),fields:r})),this.fatalError}raiseReadFatal(t,e){return this.raiseFatal("inbox.read.failed",`inbox ${t} state is corrupt`,e,{stage:t})}raiseSituationalFatal(t,e){return this.raiseFatal("inbox.situational_context.missing","situational context marker has no readable sidecar",e,{entry_id:t})}escapeFatal(t,e){if(!this.fatalEscaped){if(this.fatalEscaped=!0,e==="throw")throw t;queueMicrotask(()=>{throw t})}}scheduleHandlerFatal(t,e){let i;if(t instanceof D)i=t;else try{this.raiseReadFatal(e,t)}catch(r){i=r}this.escapeFatal(i,"schedule")}runTick(t="throw"){if(!this.canMutate())return;if(this.tickInFlight!==void 0){this.rerun=!0;return}const e=this.tick().then(()=>{this.retryNeeded=!1,this.polledOk=!0}).catch(i=>{if(this.canMutate()){if(R(i)){this.retryNeeded=!0,w({level:"warn",event:"inbox.read.failed",outcome:"failed",error:i,error_class:k(i),fields:{stage:"poll"}});return}if(this.polledOk&&Y(i)){this.clearTimer(),w({level:"info",event:"inbox.poll.token_revoked",outcome:"succeeded",fields:{stage:"poll"}}),N()?.("token-revoked");return}this.scheduleHandlerFatal(i,"poll")}}).finally(()=>{this.tickInFlight===e&&(this.tickInFlight=void 0),this.syncTimers(),this.rerun&&this.canMutate()&&(this.rerun=!1,this.runTick("schedule"))});this.tickInFlight=e}async claimMail(t){this.assertNodeOwnership(t,"poll");let e;try{e=await L(t,"wake",this.reportBodies)}catch(a){throw R(a)&&(this.replayUnseenInherited=!0),a}if(!this.canMutate()||e.action==="hold")return!1;const i=this.replayUnseenInherited;this.replayUnseenInherited=!1,this.nodeId=t,this.seeded=!0;const r=new Set(e.deliveries.map(a=>a.entry.entry_id)),n=this.buffer.filter(a=>!r.has(a.entry_id));for(const a of n)this.claims.delete(a.entry_id);if(this.buffer.splice(0,this.buffer.length,...this.buffer.filter(a=>r.has(a.entry_id))),n.length>0){const a=globalThis[Symbol.for("@crouton-kit/crtr:watcher-cancel-probe")];typeof a=="function"&&a(n.map(l=>l.entry_id),process.hrtime.bigint())}let d=!1;for(const a of e.deliveries){const l=a.entry.entry_id;this.claims.has(l)||(a.entry=V(a.entry),this.claims.set(l,a),a.handoff==="inherited"&&!i?m.fresh(o=>{this.pendingHandoffs.push({...x([a.entry],"visible"),operationId:o,settleAtAgentSettled:!0})}):(this.buffer.push(a.entry),this.lastArrival=Date.now(),d=!0))}return d&&this.armDebounce(),this.syncTimers(),!0}async tick(){const t=E();if(t!==void 0){if(this.pendingCommit!==void 0){await this.retryPendingCommit(t)==="refresh_held"&&this.canMutate()&&this.dispose("refresh",!1);return}this.promptJoinInFlight||P()||this.pendingHandoffs.length>=b||await this.claimMail(t)&&this.buffer.length>0&&(!this.runNode||this.isIdle()||this.buffer.some(e=>e.tier==="critical"))&&Date.now()-this.lastArrival>=this.cadence.debounceMs&&await this.flushOrdinary(t)}}missingChannelEntries(t){const e=t.filter(n=>this.claims.has(n.entry_id)),i=this.coveredEntryIds("hidden"),r=this.coveredEntryIds("visible");return{hiddenEntries:e.filter(n=>(n.data?.situational===!0||n.data?.situationalOnly===!0)&&!i.has(n.entry_id)),visibleEntries:e.filter(n=>n.data?.situationalOnly!==!0&&!r.has(n.entry_id))}}visibleRoute(t){return t.length===0?"none":this.isIdle()?"idle":this.runNode?t.some(e=>e.tier==="critical")?"critical":"none":t.some(e=>e.tier==="critical")?"critical":t.some(e=>e.tier==="urgent"||e.kind==="final")?"steer":"followUp"}buildPlan(t){const{hiddenEntries:e,visibleEntries:i}=this.missingChannelEntries(t),r=new Set(e.map(u=>u.entry_id)),n=new Set(i.map(u=>u.entry_id)),d=t.filter(u=>r.has(u.entry_id)||n.has(u.entry_id)),a=this.isIdle()&&d.length>0&&d.every(u=>u.tier==="deferred"),l=this.visibleRoute(i)==="critical",o=!a&&!l&&e.length>0,c=!a&&!l&&i.length>0;return{hiddenEntries:e,visibleEntries:i,deferredHold:a,sendCount:Number(o)+Number(c)}}ordinaryPlan(){return this.buildPlan(this.buffer)}async contextCards(t,e){const i=[];if(e.some(r=>r.data?.situational===!0)){const r=e.at(-1).entry_id;let n;try{n=(await F(t)).situational_card}catch(d){this.raiseSituationalFatal(r,d)}n===null&&this.raiseSituationalFatal(r,new Error(`missing situational context for entry ${r}`)),i.push(n)}for(const r of e)r.data?.situationalOnly===!0&&i.push(...j(r));return i}digest(t,e){try{return X(t)}catch(i){this.raiseReadFatal(e,i)}}sendDelivery(t,e,i,r,n){this.pendingHandoffs.length>=b&&this.raiseReadFatal(r,new Error(`pending handoff capacity exceeded before ${t} send`));let d;try{d=x(e,t)}catch(o){this.raiseReadFatal(r,o)}let a;try{m.fresh(o=>{a=o,v({level:"info",event:"inbox.delivery.started",operation_id:o,fields:y(d)}),i()})}catch(o){return a===void 0&&this.raiseReadFatal(r,o),T(a,d,"send",o),"failed"}a===void 0&&this.raiseReadFatal(r,new Error(`${t} delivery returned without an operation id`));const l={operationId:a,...d,...n===void 0?{}:{awaitingDequeue:n}};return this.canMutate()?(this.pendingHandoffs.push(l),"handed-off"):(this.disposed&&this.disposalReason!==void 0&&M(l,this.disposalReason),"inactive")}sendContext(t,e,i,r,n,d){const a=r==="steer"||r==="followUp"?r:this.isIdle()?void 0:"followUp",l=a===void 0||i.length===0?void 0:_(i.at(-1));return this.sendDelivery("hidden",e,()=>{this.runNode&&n&&this.pi.appendEntry("crtr-turn-input",{message_ids:e.map(o=>o.entry_id)}),i.forEach((o,c)=>{const u=n&&c===i.length-1;this.pi.sendMessage({customType:I,content:o,display:!0,details:{nodeId:t}},a!==void 0?{deliverAs:a}:u?{deliverAs:"followUp",triggerTurn:!0}:{})})},d,l)}sendVisible(t,e,i,r,n,d){return this.sendDelivery("visible",t,()=>{if(n!=="idle"&&n!=="steer"&&n!=="followUp")throw new Error(`visible send requested for non-send route ${n}`);for(const a of e)this.pi.sendMessage({customType:I,content:a,display:!0},n==="idle"?{}:{deliverAs:n});if(K(i,r),n==="idle"){const a=ne();if(a===void 0)throw new Error("broker idle-turn admission is unavailable");a(i,this.runNode?t.map(l=>l.entry_id):[])}else this.pi.sendUserMessage(i,{deliverAs:n})},d,n==="steer"||n==="followUp"?_(i):void 0)}abortCritical(){try{this.lastCtx?.abort?.()}catch{}}requeueFromBatch(t,e){if(e.size===0)return;const i=t.filter(d=>e.has(d.entry_id)),r=new Set(i.map(d=>d.entry_id)),n=this.buffer.filter(d=>!r.has(d.entry_id));this.buffer.splice(0,this.buffer.length,...i,...n)}async executePlan(t,e,i){const r=new Set,n=this.visibleRoute(e.visibleEntries);if(n==="critical"){if(this.abortCritical(),!this.canMutate())return{inactive:!0,requeueEntryIds:r};for(const o of e.visibleEntries)r.add(o.entry_id);for(const o of e.hiddenEntries)r.add(o.entry_id);return{inactive:!1,requeueEntryIds:r}}if(this.runNode&&n==="none")return{inactive:!1,requeueEntryIds:new Set([...e.hiddenEntries,...e.visibleEntries].map(o=>o.entry_id))};const d=e.visibleEntries.length>0?this.digest(e.visibleEntries,i):[],[a,...l]=d;for(const o of l)for(const c of o.entries)r.add(c.entry_id);if(e.hiddenEntries.length>0){const o=await this.contextCards(t,e.hiddenEntries),c=this.sendContext(t,e.hiddenEntries,o,n,a===void 0,i);if(c==="inactive")return{inactive:!0,requeueEntryIds:r};if(c==="failed")for(const u of e.hiddenEntries)r.add(u.entry_id)}if(!this.canMutate())return{inactive:!0,requeueEntryIds:r};if(a!==void 0){const o=this.sendVisible(a.entries,a.cards,a.text,oe(a),n,i);if(o==="inactive")return{inactive:!0,requeueEntryIds:r};if(o==="failed")for(const c of a.entries)r.add(c.entry_id);else a.kind==="human"&&(await Z(t,a.text).catch(()=>{}),ee(t,a.text))}return{inactive:!1,requeueEntryIds:r}}async flushOrdinary(t){if(this.buffer.length===0)return"empty";const e=this.ordinaryPlan();if(e.deferredHold)return"held";if(this.pendingHandoffs.length+e.sendCount>b)return"capacity-blocked";const i=[...this.buffer],r=await this.executePlan(t,e,"poll");return r.inactive||(this.buffer.splice(0,i.length),this.requeueFromBatch(i,r.requeueEntryIds)),"attempted"}async prependHeldDeferredInbox(t){if(!this.canMutate()||this.nodeId===void 0||P())return;const e=this.ordinaryPlan();if(!e.deferredHold)return;const i=e.visibleEntries.length===0?[]:this.digest(e.visibleEntries,"poll"),r=+(e.hiddenEntries.length>0)+i.length;if(this.pendingHandoffs.length+r>b)return;const n=[...this.buffer];this.promptJoinInFlight=!0;try{this.runNode&&this.pi.appendEntry("crtr-turn-input",{message_ids:[...e.hiddenEntries,...e.visibleEntries].map(d=>d.entry_id)}),e.hiddenEntries.length>0&&await this.sendPromptJoinDelivery(t,"hidden",e.hiddenEntries,await this.contextCards(this.nodeId,e.hiddenEntries));for(const d of i)await this.sendPromptJoinDelivery(t,"visible",d.entries,[...d.cards,d.text]);if(!this.canMutate())return;this.buffer.splice(0,n.length)}finally{this.promptJoinInFlight=!1}}async sendPromptJoinDelivery(t,e,i,r){let n;try{n=x(i,e)}catch(l){this.raiseReadFatal("poll",l)}let d;try{await m.fresh(async l=>{d=l,v({level:"info",event:"inbox.delivery.started",operation_id:l,fields:y(n)});for(const o of r)await t.sendCustomMessage({customType:I,content:o,display:!0})})}catch(l){throw d===void 0&&this.raiseReadFatal("poll",l),T(d,n,"send",l),l}d===void 0&&this.raiseReadFatal("poll",new Error(`${e} prompt-join delivery returned without an operation id`));const a={operationId:d,...n,settleAtAgentSettled:!0};if(!this.canMutate()){this.disposed&&this.disposalReason!==void 0&&M(a,this.disposalReason);return}this.pendingHandoffs.push(a)}coveredEntryIds(t){const e=new Set;for(const i of this.pendingHandoffs)if(i.channel===t)for(const r of i.entryIds)e.add(r);return e}retryPendingCommit(t){const e=this.pendingCommit;if(e===void 0)return Promise.resolve("settled");if(e.attempt!==void 0)return e.attempt;const i=this.attemptPendingCommit(t,e).finally(()=>{e.attempt===i&&(e.attempt=void 0)});return e.attempt=i,i}async attemptPendingCommit(t,e){let i=!1;try{const n=await z(t,e.receipts);if(n.action==="hold")return"refresh_held";for(const d of e.receipts){const a=n.results.find(l=>l.delivery_id===d.delivery_id);if(a?.outcome!==d.disposition&&a?.outcome!=="stale")throw new Error("daemon returned no terminal mail settlement outcome")}i=n.results.every(d=>d.outcome==="acknowledged")}catch(n){if(this.pendingCommit===e)for(const d of e.handoffs)T(d.operationId,d,"receipt_commit",n);return String(n?.message).startsWith("conversation_write_failed")&&(O({level:"error",event:"broker.exited",node_id:t,error:n,fields:{fault:"conversation_write_failed"}}),N()?.("conversation_write_failed",1)),"held"}if(this.pendingCommit!==e)return"settled";for(const n of e.entryIds)this.claims.delete(n);if(e.receipts.some(n=>n.disposition==="released")){const n=new Set(e.entryIds);this.buffer.splice(0,this.buffer.length,...this.buffer.filter(d=>!n.has(d.entry_id)))}this.pendingCommit=void 0;const r=this.detachPending(n=>n.entryIds.every(d=>!this.claims.has(d)));for(const n of r)i&&le(n);return"settled"}async settle(t,e,i,r=!1){if(i)try{if(e?.hasPendingMessages?.()===!0)return"held"}catch{}if(this.pendingCommit!==void 0)return this.retryPendingCommit(t);const n=Q(),d=!n&&this.pendingHandoffs.some(h=>h.awaitingDequeue!==void 0)&&await this.nodeRefreshing(t);if(this.pendingCommit!==void 0)return this.retryPendingCommit(t);const a=new Set(this.buffer.map(h=>h.entry_id)),l=new Set(d?this.pendingHandoffs.filter(h=>h.awaitingDequeue!==void 0).flatMap(h=>[...h.entryIds]):[]),o=this.pendingHandoffs.filter(h=>(r||h.settleAtAgentSettled!==!0)&&h.entryIds.every(g=>!l.has(g))),c=new Set(o.flatMap(h=>[...h.entryIds])),u=[...this.claims.keys()].filter(h=>n||c.has(h)&&!a.has(h));return u.length===0?"settled":(this.pendingCommit={handoffs:o,entryIds:u,receipts:u.map(h=>{const g=this.claims.get(h);return{delivery_id:g.delivery_id,attempts:g.attempts,disposition:n?"released":"acknowledged"}})},this.retryPendingCommit(t))}async nodeRefreshing(t){try{return(await F(t)).node.intent==="refresh"}catch{return!0}}detachPending(t){const e=[];for(let i=this.pendingHandoffs.length-1;i>=0;i-=1){const r=this.pendingHandoffs[i];t(r)&&e.push(...this.pendingHandoffs.splice(i,1))}return e.reverse(),e}}function me(s,t){p?.dispose("reload",!1);const e=new ce(s,t,++ae);p=e;const i=f(r=>e.prependHeldDeferredInbox(r),"promptJoin");globalThis[se]=i;try{e.start()}catch(r){throw e.dispose("reload",!1),r}return e}f(me,"installWatcher");function pe(s){const t=me(s,de());return()=>t.dispose("shutdown",!1)}f(pe,"registerCanvasInboxWatcher");var Oe=pe;export{Oe as default,pe as registerCanvasInboxWatcher};

package/docs/sdk/release-notes.md

@@ -7,6 +7,10 @@ description: Behavior changes in @north-light/sdk that an app must act on, newes
 
 Changes to `@north-light/sdk` that can change what an existing app sees. Newest first.
 
+## 0.3.576
+
+On a cloud role, `provider.tools` now reports a granted provider as `available` once its bundle is installed in the deployment's shared plugin directory. It no longer answers `500` when a person has no materialized bundle; a granted provider still being installed reports `installing` or `install_failed`. Providers without that person's grant remain hidden. The Mac's plugin settings and provider tools behavior are unchanged.
+
 ## 0.3.572
 
 `POST /v1/runs` works on a cloud role (a runtime started with `--role api` or `--role reconcile`). It answered `500 server_failure` because the role has no root launcher and the run start tried to prepare an app OS user and folder. Behavior on a person's Mac and on a hosted guest with a launcher is unchanged.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.574",
+  "version": "0.3.576",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -145,7 +145,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.574",
+    "@north-light/identity": "0.3.576",
     "@earendil-works/chord": "1.1.0",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.574",
+  "version": "0.3.576",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.574",
+  "version": "0.3.576",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -28,7 +28,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.574",
+        "@north-light/identity": "0.3.576",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12459,17 +12459,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12477,21 +12477,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.574",
-        "@north-light/identity": "0.3.574"
+        "@north-light/api": "0.3.576",
+        "@north-light/identity": "0.3.576"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.574",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.574",
-        "@north-light/identity": "0.3.574",
-        "@north-light/plugin": "0.3.574",
+        "@north-light/api": "0.3.576",
+        "@north-light/identity": "0.3.576",
+        "@north-light/plugin": "0.3.576",
         "jose": "^6.2.1"
       },
       "bin": {

package/scripts/sandbox-start.mjs

@@ -300,7 +300,14 @@ async function start(env) {
   return await exited;
 }
 
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+// Both sides are real paths: Node resolves import.meta.url through the `current` symlink the wake command runs from.
+// An importer's argv[1] may not exist; that is simply "not the entry point", never an error.
+function runAsEntryPoint() {
+  if (!process.argv[1]) return false;
+  try { return realpathSync(resolve(process.argv[1])) === realpathSync(fileURLToPath(import.meta.url)); }
+  catch (error) { if (error?.code === 'ENOENT') return false; throw error; }
+}
+if (runAsEntryPoint()) {
   const args = process.argv.slice(2);
   try {
     if (args[0] === '--wake') {