@north-light/cli 0.3.575 → 0.3.576

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,parkEvent as Ie}from"./live-obligation.js";import{readGoal as xe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Fe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Oe,envTestUnattendedParkMs as Ce,parkSummaryGraceMs as De,envUnattendedParkMs as We}from"../../shared/env.js";const R=ee(Z);async function Be(a){return!await ie(a)&&((await xe(a))?.trim()??"")!==""}m(Be,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Ge(a){return(await F(a))?.busy===!0}m(Ge,"liveBusy");async function It(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(It,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,Le=2,qe=5*6e4;function Ue(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ue,"settlesOnFatalProviderFault");function Ye(){const a=We();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Oe()?Ce()??e:e}m(Ye,"unattendedParkMsForDaemon");function H(){return De()}m(H,"parkSummaryGraceMsForDaemon");function K(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(K,"yieldStallVerdict");function He(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>Le?"pending":"wedged"}m(He,"wedgeVerdict");function V(a){const e=new Map,t=new Map;for(const r of a.split(`
-`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(V,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=V(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function Ve(a,e){const{byPid:t,childrenOf:r}=V(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(Ve,"treeHasDescendants");class xt{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Ye();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Ge(e)||await Be(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const u=await Ie(e,t.lifecycle),w=u!=="cancel"&&Fe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<w)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<qe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
-`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}K(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(K(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=Ve(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,k=s?.hasDescendants??null;if(He({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:k})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const v=x(p),y=Math.round((d??I)/6e4),b=k??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
-`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ue(i,typeof s?.code=="string"?s.code:void 0),k=u||w,v=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${v}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&k&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&k&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{xt as BrokerSupervisionReconciler,Le as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ue as settlesOnFatalProviderFault,Ke as sumTreeCpu,Ve as treeHasDescendants,Ye as unattendedParkMsForDaemon,He as wedgeVerdict,K as yieldStallVerdict};
+var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,noticeUnfinishedIdle as Ie,parkVerdict as xe}from"./live-obligation.js";import{readGoal as Fe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Oe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Ce,envTestUnattendedParkMs as De,parkSummaryGraceMs as We,envUnattendedParkMs as Be}from"../../shared/env.js";const R=ee(Z);async function Ge(a){return!await ie(a)&&((await Fe(a))?.trim()??"")!==""}m(Ge,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Le(a){return(await F(a))?.busy===!0}m(Le,"liveBusy");async function xt(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(xt,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,qe=2,Ue=5*6e4;function Ye(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ye,"settlesOnFatalProviderFault");function He(){const a=Be();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Ce()?De()??e:e}m(He,"unattendedParkMsForDaemon");function H(){return We()}m(H,"parkSummaryGraceMsForDaemon");function V(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(V,"yieldStallVerdict");function Ve(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>qe?"pending":"wedged"}m(Ve,"wedgeVerdict");function K(a){const e=new Map,t=new Map;for(const r of a.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(K,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=K(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function ze(a,e){const{byPid:t,childrenOf:r}=K(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(ze,"treeHasDescendants");class Ft{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=He();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Le(e)||await Ge(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const{event:u,heldOnlyBySubscriber:w}=await xe(e,t.lifecycle),v=u!=="cancel"&&Oe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<v)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),w&&await Ie(e),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ue)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
+`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}V(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(V(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=ze(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,v=s?.hasDescendants??null;if(Ve({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:v})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const k=x(p),y=Math.round((d??I)/6e4),b=v??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
+`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ye(i,typeof s?.code=="string"?s.code:void 0),v=u||w,k=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${k}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&v&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&v&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{Ft as BrokerSupervisionReconciler,qe as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ye as settlesOnFatalProviderFault,Ke as sumTreeCpu,ze as treeHasDescendants,He as unattendedParkMsForDaemon,Ve as wedgeVerdict,V as yieldStallVerdict};

package/dist/daemon/reconcilers/live-obligation.d.ts

@@ -50,6 +50,13 @@ export declare function hasHumanViewer(nodeId: string, viewer?: ViewerProbe): Pr
  */
 export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
 export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<ParkEvent>;
+/** `parkEvent` plus whether a live subscriber awaiting the report is the ONLY thing holding a
+ *  terminal node's `release`: no viewer, no wake source, no unread mail, no fatal fault. Such a
+ *  node stopped unfinished with nothing that will wake it, and its subscriber knows nothing. */
+export declare function parkVerdict(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<{
+    event: ParkEvent;
+    heldOnlyBySubscriber: boolean;
+}>;
 /** Apply an unattended decision and notify subscribers of automatic completion.
  *  This is the DEGRADED enactment: no summary turn ran, so subscribers get the
  *  doctrine notice instead of the node's own report. The live clock's good path
@@ -64,3 +71,7 @@ export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?:
  *  resident throughout: its conversation ended without a final report, its state
  *  lives in whatever report it did leave, and a message reopens it. */
 export declare function applyParkEvent(nodeId: string, event: ParkEvent): Promise<void>;
+/** Tell a terminal node's subscribers it went idle unfinished: it pushed no final, holds no wait
+ *  that will wake it, and only their subscription kept it from being canceled. The cause of the
+ *  stop is not known here, so the notice claims none. */
+export declare function noticeUnfinishedIdle(nodeId: string): Promise<void>;

package/dist/daemon/reconcilers/live-obligation.js

@@ -1 +1 @@
-var l=Object.defineProperty;var r=(t,a)=>l(t,"name",{value:a,configurable:!0});import{hasActiveLiveSubscription as f,hasLiveSubscriber as p,hasPendingCronWake as w}from"../../core/canvas/index.js";import{fullName as m}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as n}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/output-result.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await f(t)||await h(t)||await k(t)||await w(t)||await b(t)>0||await g(t)!==null}r(s,"hasLiveObligation");async function B(t){return await s(t)||await n(t)?!0:await o(t)}r(B,"hasPendingWake");async function F(t){return async a=>await t.isClaimed(a)||await s(a)||await o(a)}r(F,"retentionHold");async function o(t,a){return(a!==void 0?a.live!==null:await v(t)!==null)||await x(t)}r(o,"hasHumanViewer");async function M(t,a,e){if(await o(t,e)||await s(t))return"release";const i=await W(t);return i?.link==="pi\u2192provider"&&i.retry.disposition==="fatal"?"release":a==="resident"?await n(t)?"release":"park":await n(t)||await p(t)?"release":"cancel"}r(M,"parkEvent");async function T(t,a){const e=a==="cancel"?await c(t,a,{reason:"stranded"}):await c(t,a);if(a!=="park")return;const i=await y(t),u=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${m(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}r(T,"applyParkEvent");export{T as applyParkEvent,o as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};
+var w=Object.defineProperty;var a=(t,e)=>w(t,"name",{value:e,configurable:!0});import{getNode as h,hasActiveLiveSubscription as m,hasLiveSubscriber as y,hasPendingCronWake as g}from"../../core/canvas/index.js";import{fullName as l}from"../../core/canvas/labels.js";import{hasLiveControllerWait as k,hasLiveObjectWait as b}from"../../core/canvas/waits.js";import{newestReportName as v}from"../../core/feed/reports.js";import{countTickets as x}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as s}from"../../core/canvas/deliveries.js";import{readOutputRequest as $}from"../../core/runtime/output-result.js";import{focusOf as O}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as d}from"../api/viewer-presence.js";import{publishNotice as u}from"../../core/runtime/close.js";import{readFault as L}from"../../core/canvas/recovery.js";import{transition as f}from"../../core/runtime/lifecycle.js";async function o(t){return await m(t)||await k(t)||await b(t)||await g(t)||await x(t)>0||await $(t)!==null}a(o,"hasLiveObligation");async function F(t){return await o(t)||await s(t)?!0:await c(t)}a(F,"hasPendingWake");async function M(t){return async e=>await t.isClaimed(e)||await o(e)||await c(e)}a(M,"retentionHold");async function c(t,e){return(e!==void 0?e.live!==null:await O(t)!==null)||await d(t)}a(c,"hasHumanViewer");async function T(t,e,r){return(await W(t,e,r)).event}a(T,"parkEvent");async function W(t,e,r){const i=a(p=>({event:p,heldOnlyBySubscriber:!1}),"held");if(await c(t,r)||await o(t))return i("release");const n=await L(t);return n?.link==="pi\u2192provider"&&n.retry.disposition==="fatal"?i("release"):e==="resident"?i(await s(t)?"release":"park"):await s(t)?i("release"):await y(t)?{event:"release",heldOnlyBySubscriber:!0}:i("cancel")}a(W,"parkVerdict");async function z(t,e){const r=e==="cancel"?await f(t,e,{reason:"stranded"}):await f(t,e);if(e!=="park")return;const i=await v(t),n=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await u(t,"notice.close",`Resident concluded \u2014 ${l(r)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${n} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}a(z,"applyParkEvent");async function D(t){const e=await h(t);e!==null&&await u(t,"notice.crash",`Child went idle unfinished \u2014 ${l(e)} (${t}) stopped without pushing a final report and has no wait that will wake it. Its engine was released; the cause is not recorded. \`crtr node inspect show ${t}\` shows where it stands; a message to it revives it.`,{reason:"child-idle-unfinished",child:t},"urgent")}a(D,"noticeUnfinishedIdle");export{z as applyParkEvent,c as hasHumanViewer,o as hasLiveObligation,F as hasPendingWake,D as noticeUnfinishedIdle,T as parkEvent,W as parkVerdict,M as retentionHold};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var E=Object.defineProperty;var l=(r,t)=>E(r,"name",{value:t,configurable:!0});import{clearFrozen as m,endLiveExecution as h,getNode as R}from"../../../core/canvas/index.js";import{isSafeNodeId as b}from"../../../core/canvas/paths.js";import{recordedPidLiveness as _}from"../../../core/canvas/pid.js";import{hasObservedExit as L,hasAbsorbedProfilePauseStop as z}from"../../../core/canvas/executions.js";import{emitEvent as y}from"../../../core/events/emit.js";import{operationIdContext as I}from"../../../core/events/operation-id.js";import{readInterruptedTurn as g}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as w,isParkedResident as N}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as M}from"../../fleet.js";import{applyParkEvent as F,hasPendingWake as x,parkEvent as A}from"../live-obligation.js";import{FreezeLaneReconciler as C}from"./freeze-lane.js";import{isInBackoff as P}from"./respawn-policy.js";import{asNodePerson as D,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as S}from"./terminating.js";import{wantsExecution as T}from"./wants-execution.js";import{enactRequestedStops as U}from"./requested-stops.js";import{endRemoteExecutionIfGone as G,recoverLaunching as H,wakeResident as W}from"./remote-wake.js";import{liveExecutionForNode as q}from"../../../core/canvas/executions.js";function B(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}l(B,"isTerminal");const O=12e4;function j(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<O}l(j,"isLaunchInFlight");class me{static{l(this,"NodeLifecycleReconciler")}freezeLane=new C;async run(t,e){const a=[];await U(t);for(const n of e.rows)await I.fresh(async()=>{try{await D(n,()=>this.classifyRow(t,n,e,a))}catch(i){const s=n.node_id;y({level:i instanceof k?"warn":"error",event:i instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...b(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(B(e)){if(S(e,await a.fleet.has(i)))return;if(N(e)&&!await a.fleet.has(i)&&!d(e.profile_id)&&!await v(i)&&!P(e,t)&&await p(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead"&&await h(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await m(i):n.push({row:e,enact:"thaw"}));return}if(await a.fleet.has(i)){const f=await q(i);if(f?.runner==="blaxel"&&f.state==="running"&&!d(e.profile_id)&&await p(i)){if(await W(f)==="lost")return}else f?.runner==="blaxel"&&f.state==="running"&&await G(f);if(await a.fleet.has(i)){d(e.profile_id)?await a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}}if(await H(i,t),d(e.profile_id)){await this.freezeLane.endEpisode(i);return}if(await v(i))return;const s=e.pi_pid==null&&await L(i),u=e.pi_pid==null&&!s&&await z(i),c=s||u||e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead";if(c&&!await a.fleet.hasProfilePauseInterruption(i)&&await h(i,{reason:"observed-dead"}),!await T(e)){await this.thawMark(e),await this.settleParked(e);return}if(P(e,t))return;if(e.status==="idle"&&w(e)&&await p(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!s||e.frozen_at!=null)&&!u){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(await a.fleet.isClaimed(i)||j(e,t))return;y({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!c)return;const o=M(await R(i),(await g(i))?.interrupted===!0,await x(i));if(o==="respawn-fresh"||o==="respawn-cycle"||o==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await m(t.node_id)}async settleParked(t){if(t.status!=="idle")return;await this.freezeLane.endEpisode(t.node_id);const e=await A(t.node_id,t.lifecycle);e!=="release"&&await F(t.node_id,e)}}export{O as LAUNCH_GRACE_MS,me as NodeLifecycleReconciler,j as isLaunchInFlight};
+var b=Object.defineProperty;var o=(r,a)=>b(r,"name",{value:a,configurable:!0});import{clearFrozen as P,getNode as x}from"../../../core/canvas/index.js";import{isSafeNodeId as z}from"../../../core/canvas/paths.js";import{recordedPidLiveness as L}from"../../../core/canvas/pid.js";import{endExecution as v,hasAbsorbedProfilePauseStop as g,hasObservedExit as N,latestExecutionForNode as M,liveExecutionForNode as c}from"../../../core/canvas/executions.js";import{emitEvent as w}from"../../../core/events/emit.js";import{operationIdContext as F}from"../../../core/events/operation-id.js";import{readInterruptedTurn as D}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as I}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as A,isParkedResident as C}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as S}from"../../fleet.js";import{applyParkEvent as T,hasPendingWake as U,parkEvent as W}from"../live-obligation.js";import{FreezeLaneReconciler as B}from"./freeze-lane.js";import{isInBackoff as E}from"./respawn-policy.js";import{asNodePerson as G,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as H}from"./terminating.js";import{wantsExecution as q}from"./wants-execution.js";import{enactRequestedStops as O}from"./requested-stops.js";import{endRemoteExecutionIfGone as j,recoverLaunching as J,wakeResident as K}from"./remote-wake.js";function Q(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}o(Q,"isTerminal");const V=12e4;function X(r,a){const e=Math.max(...[r.created,r.launched_at].map(i=>i==null?0:Date.parse(i)||0));return e>0&&a-e<V}o(X,"isLaunchInFlight");async function R(r){const a=await c(r);return a!==null&&a.state==="running"&&a.pid!=null&&L(a.pid,a.processIdentity)==="dead"?a:null}o(R,"executionWithDeadPid");class Pe{static{o(this,"NodeLifecycleReconciler")}freezeLane=new B;async run(a,e){const i=[];await O(a);for(const s of e.rows)await F.fresh(async()=>{try{await G(s,()=>this.classifyRow(a,s,e,i))}catch(t){const l=s.node_id;w({level:t instanceof k?"warn":"error",event:t instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...z(l)?{node_id:l}:{fields:{affected_node_id:l}},error:t})}});return this.freezeLane.run(a,{fleet:e.fleet,candidates:i})}async classifyRow(a,e,i,s){const t=e.node_id;if(Q(e)){if(H(e,await i.fleet.has(t)))return;if(C(e)&&!await i.fleet.has(t)&&!d(e.profile_id)&&!await I(t)&&!E(e,a)&&await p(t)){s.push({row:e,enact:"reopen"});return}if(e.pi_pid!=null){const n=await R(t);n!==null&&await v(n.executionId,{reason:"observed-dead"})}e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await P(t):s.push({row:e,enact:"thaw"}));return}if(await i.fleet.has(t)){const n=await c(t);if(n?.runner==="blaxel"&&n.state==="running"&&!d(e.profile_id)&&await p(t)){if(await K(n)==="lost")return}else n?.runner==="blaxel"&&n.state==="running"&&await j(n);if(await i.fleet.has(t)){d(e.profile_id)?await i.fleet.interruptForProfilePause(t):await this.thawMark(e);return}}if(await J(t,a),d(e.profile_id)){await this.freezeLane.endEpisode(t);return}if(await I(t))return;const l=e.pi_pid==null?await M(t):null,f=l!==null&&N(l),m=l!==null&&!f&&g(l),h=e.pi_pid!=null?await R(t):null,_=f||m||h!==null;if(_&&!await i.fleet.hasProfilePauseInterruption(t)){const n=h??l;n!==null&&await v(n.executionId,{reason:"observed-dead"});const y=await c(t);if(y!==null&&y.executionId!==n?.executionId)return}if(!await q(e)){await this.thawMark(e),await this.settleParked(e);return}if(E(e,a))return;if(e.status==="idle"&&A(e)&&await p(t)){s.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!f||e.frozen_at!=null)&&!m){if(e.respawn_not_before!=null){s.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){s.push({row:e,enact:"thaw"});return}if(await i.fleet.isClaimed(t)||X(e,a))return;w({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),s.push({row:e,enact:"thaw"});return}if(!_)return;const u=S(await x(t),(await D(t))?.interrupted===!0,await U(t));if(u==="respawn-fresh"||u==="respawn-cycle"||u==="respawn-resume"){s.push({row:e,enact:"policy"});return}await i.fleet.applyDeadRowPolicy(t),await this.thawMark(e)}async thawMark(a){a.frozen_at!=null&&await P(a.node_id)}async settleParked(a){if(a.status!=="idle")return;await this.freezeLane.endEpisode(a.node_id);const e=await W(a.node_id,a.lifecycle);e!=="release"&&await T(a.node_id,e)}}export{V as LAUNCH_GRACE_MS,Pe as NodeLifecycleReconciler,X as isLaunchInFlight};

package/docs/sdk/release-notes.md

@@ -7,6 +7,10 @@ description: Behavior changes in @north-light/sdk that an app must act on, newes
 
 Changes to `@north-light/sdk` that can change what an existing app sees. Newest first.
 
+## 0.3.576
+
+On a cloud role, `provider.tools` now reports a granted provider as `available` once its bundle is installed in the deployment's shared plugin directory. It no longer answers `500` when a person has no materialized bundle; a granted provider still being installed reports `installing` or `install_failed`. Providers without that person's grant remain hidden. The Mac's plugin settings and provider tools behavior are unchanged.
+
 ## 0.3.572
 
 `POST /v1/runs` works on a cloud role (a runtime started with `--role api` or `--role reconcile`). It answered `500 server_failure` because the role has no root launcher and the run start tried to prepare an app OS user and folder. Behavior on a person's Mac and on a hosted guest with a launcher is unchanged.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.575",
+  "version": "0.3.576",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -145,7 +145,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.575",
+    "@north-light/identity": "0.3.576",
     "@earendil-works/chord": "1.1.0",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.575",
+  "version": "0.3.576",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.575",
+  "version": "0.3.576",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -28,7 +28,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.575",
+        "@north-light/identity": "0.3.576",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12459,17 +12459,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12477,21 +12477,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.575",
-        "@north-light/identity": "0.3.575"
+        "@north-light/api": "0.3.576",
+        "@north-light/identity": "0.3.576"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.575",
+      "version": "0.3.576",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.575",
-        "@north-light/identity": "0.3.575",
-        "@north-light/plugin": "0.3.575",
+        "@north-light/api": "0.3.576",
+        "@north-light/identity": "0.3.576",
+        "@north-light/plugin": "0.3.576",
         "jose": "^6.2.1"
       },
       "bin": {

package/scripts/sandbox-start.mjs

@@ -300,7 +300,14 @@ async function start(env) {
   return await exited;
 }
 
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+// Both sides are real paths: Node resolves import.meta.url through the `current` symlink the wake command runs from.
+// An importer's argv[1] may not exist; that is simply "not the entry point", never an error.
+function runAsEntryPoint() {
+  if (!process.argv[1]) return false;
+  try { return realpathSync(resolve(process.argv[1])) === realpathSync(fileURLToPath(import.meta.url)); }
+  catch (error) { if (error?.code === 'ENOENT') return false; throw error; }
+}
+if (runAsEntryPoint()) {
   const args = process.argv.slice(2);
   try {
     if (args[0] === '--wake') {