@north-light/cli 0.3.524 → 0.3.526

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var q=Object.defineProperty;var p=(s,e)=>q(s,"name",{value:e,configurable:!0});import{existsSync as Y}from"node:fs";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{launcherRequest as K}from"../../launcher/client.js";import{execFile as V}from"node:child_process";import{promisify as z}from"node:util";import{getNode as N,isWarmSpare as j}from"../../core/canvas/index.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as J}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Q}from"../../core/runtime/lifecycle.js";import{readFault as I,readLiveTurn as E,recordFault as X}from"../../core/canvas/recovery.js";import{errorClassFromFault as Z}from"../../core/fault-classifier.js";import{nextFaultRetry as ee}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{headlessBrokerHost as k}from"../../core/runtime/host.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as O}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as te,resetStaleExecutionMail as ne}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as re}from"../../core/bash-jobs.js";import{focusOf as ie}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ae}from"../api/viewer-presence.js";import{deliverLive as se}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as oe}from"../../core/runtime/broker-request.js";import{formatCard as le,PARK_SUMMARY_PROMPT as de}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ue}from"../../core/runtime/fleet.js";import{readConfig as ce}from"../../core/config.js";import{beginPendingPark as fe,consumePendingPark as y,pendingParkIds as pe,pendingParkStartedAt as me}from"../park-pending.js";import{shouldParkAfterSummary as he}from"../park-activity.js";import{applyParkEvent as b,parkEvent as _e}from"./live-obligation.js";import{memoryDeniedRecently as ge}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as ke,envTestUnattendedParkMs as ve,parkSummaryGraceMs as ye,envUnattendedParkMs as be}from"../../shared/env.js";const P=z(V);function Se(s){return E(s)?.busy===!0}p(Se,"liveBusy");function ut(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(ut,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function F(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}p(F,"busySinceForIncarnation");const G=3*6e4,W=2e4,x=20*6e4,we=2,Ne=5*6e4;function Pe(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Pe,"settlesOnFatalProviderFault");function Ae(){const s=be();if(s!==void 0)return s;const e=ce("user").lifecycle.unattendedParkMs;return ke()?ve()??e:e}p(Ae,"unattendedParkMsForDaemon");function U(){return ye()}p(U,"parkSummaryGraceMsForDaemon");function $e(s,e,t,a){return s!==!0||e!=="refresh"||t?"leave":a===null||a<G?"pending":"kill"}p($e,"yieldStallVerdict");function Me(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>we?"pending":"wedged"}p(Me,"wedgeVerdict");function B(s){const e=new Map,t=new Map;for(const a of s.split(`
-`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),r=Number(n[1]),l=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(r)||!Number.isFinite(l))continue;e.set(o,l);const c=t.get(r);c===void 0?t.set(r,[o]):c.push(o)}return{byPid:e,childrenOf:t}}p(B,"parsePsTree");function Te(s,e){const{byPid:t,childrenOf:a}=B(s);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const r=n.pop();if(o.has(r))continue;o.add(r),i+=t.get(r)??0;const l=a.get(r);l!==void 0&&n.push(...l)}return i}p(Te,"sumTreeCpu");function Ee(s,e){const{byPid:t,childrenOf:a}=B(s);return t.has(e)?(a.get(e)??[]).length>0:null}p(Ee,"treeHasDescendants");class ct{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&Y(H)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return K({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return ue()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=O.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Ae();for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n);if(o===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await O.fresh(async()=>{try{await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,N(n),o.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),f({level:"error",event:"broker.supervision.failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of pe())t.fleet.has(i)||y(i);ne(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||re(J(e)).length>0||ie(e)!==null||ae(e))return!0;const n=I(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ee(n,i)!==null)?!0:te(e)}handleUnattendedParking(e,t,a,i,n,o){if(j(e)){this.unattendedSince.delete(e),y(e);return}if(t===null){this.unattendedSince.delete(e),y(e);return}const r=this.parkResetCondition(e,t,a,i),l=me(e);if(l!==void 0){if(i-l<U())return;const h=y(e);if(h===void 0||(this.unattendedSince.delete(e),!he(e,t,h)))return;f({level:"warn",event:"broker.park.summary_expired",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},grace_ms:U()}}),b(e,"park"),k.teardown(e);return}if(r||Se(e)){this.unattendedSince.delete(e);return}const c=this.unattendedSince.get(e);if(c===void 0){this.unattendedSince.set(e,i);return}const d=_e(e,t.lifecycle),m=d!=="cancel"&&ge(t.grantee??"app:terminal",i)?0:o;if(!(i-c<m)){if(this.unattendedSince.delete(e),d==="release"){b(e,d),k.teardown(e);return}if(d==="cancel"){b(e,d),k.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){b(e,"park"),k.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(fe(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await se(e,le("park",{},de),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(y(e)===void 0)return;if(t instanceof oe){b(e,"park"),k.teardown(e);return}f({level:"error",event:"broker.park.summary_failed",...u(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Ne)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,r=a.stalled+a.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const l=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,c=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${l} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await P("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async m=>{if(m!=="")try{await P("tmux",["display-message","-C","-d","10000","-c",m,c],{encoding:"utf8",timeout:2e3})}catch{try{await P("tmux",["display-message","-c",m,c],{encoding:"utf8",timeout:2e3})}catch(h){f({level:"error",event:"broker.capacity.tmux_display_failed",error:h,fields:{client:m}})}}}))}catch(d){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||o===null){o!==null&&this.yieldStalls.delete(o);return}let r=this.yieldStalls.get(o);if(r===void 0){r={since:i},this.yieldStalls.set(o,r);return}if($e(!0,e.intent,!1,i-r.since)!=="kill")return;const l=N(n);if(!(a.get(n)!==t||l?.pi_pid!==t.pid||l.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:G}})}else i-r.termedAt>=W&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:W,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...u(e)?{node_id:e}:{},error:i,fields:{...u(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await P("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=Te(t,e),i=Ee(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity),r=F(e);if(r===null||o===null){o!==null&&this.wedgeNotifiedAt.delete(o);return}const l=i-r,c=l>=x?await this.sampleTree(t.pid):null,d=N(n),m=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||m!==r)return;const h=c?.cpuPercent??null,_=c?.hasDescendants??null;if(Me({busy:!0,quietForMs:l,cpuPercent:h,hasDescendants:_})!=="wedged"||this.wedgeNotifiedAt.get(o)===r)return;this.wedgeNotifiedAt.set(o,r);const S=C(d),w=Math.round((l??x)/6e4),A=_??!0,v=!A&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let $=!1,M=!1,g;if(A)g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{$=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){f({level:"error",event:"broker.wedge.signal_failed",...u(n)?{node_id:n}:{},error:T,fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}f({level:"error",event:"broker.wedge.detected",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:l,cpu_percent:h,has_descendants:A,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}}});try{X(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:g})}catch(T){f({level:"error",event:"broker.wedge.fault_record_failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}R(n,"notice.crash",g,{reason:"child-wedged",child:n,quiet_ms:l,cpu_percent:h,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}})}handleFatalFault(e){const t=I(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=N(e),i=a!==null?C(a):e,n=t.kind==="auth",o=(t.message.split(`
-`)[0]??"").slice(0,200),r=t.providerError,l=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),c=t.kind==="context-overflow",d=Pe(a,typeof r?.code=="string"?r.code:void 0),m=c||d,h=c?`Child failed \u2014 ${i} (${e}) exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`Child hit a fatal provider fault \u2014 ${i} (${e}) got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); the child recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${l}`;m&&(Q(e,"crash",{reason:c?"context_overflow":r?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:Z(t).class}}}),k.teardown(e).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))),this.fatalFaultNotifiedAt.set(e,t.since),f({level:"error",event:"fault.fatal.fanout",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:c,terminal_provider_fault:d}}),R(e,"notice.crash",h,{reason:c?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:o,...r===void 0?{}:{provider_error:r}})}}export{ct as BrokerSupervisionReconciler,we as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,G as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,Pe as settlesOnFatalProviderFault,Te as sumTreeCpu,Ee as treeHasDescendants,Ae as unattendedParkMsForDaemon,Me as wedgeVerdict,$e as yieldStallVerdict};
+var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{listUnnotifiedFatalFaultNodes as ee,markFaultNotified as te,readFault as O,readLiveTurn as E,recordFault as ne}from"../../core/canvas/recovery.js";import{errorClassFromFault as re}from"../../core/fault-classifier.js";import{nextFaultRetry as ie}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as L,withCanvasWrite as ae}from"../../core/canvas/db.js";import{getById as se}from"../../core/graph/objects.js";import{watcherIdsOf as oe}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as le,resetStaleExecutionMail as de}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ce}from"../../core/bash-jobs.js";import{focusOf as ue}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as fe}from"../api/viewer-presence.js";import{deliverLive as pe}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as me}from"../../core/runtime/broker-request.js";import{formatCard as he,PARK_SUMMARY_PROMPT as _e}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ge}from"../../core/runtime/fleet.js";import{readConfig as ke}from"../../core/config.js";import{beginPendingPark as ve,consumePendingPark as N,pendingParkIds as ye,pendingParkStartedAt as be}from"../park-pending.js";import{shouldParkAfterSummary as we}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Se}from"./live-obligation.js";import{memoryDeniedRecently as Ne}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Pe,envTestUnattendedParkMs as $e,parkSummaryGraceMs as Te,envUnattendedParkMs as Me}from"../../shared/env.js";const M=j(z);function Ae(s){return E(s)?.busy===!0}p(Ae,"liveBusy");function yt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(yt,"liveTurnProgressMs");function G(s,e){return e===null?null:`${s}:${e}`}p(G,"incarnationOf");function I(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(I,"busySinceForIncarnation");const W=3*6e4,U=2e4,x=20*6e4,Ee=2,Fe=5*6e4;function Ie(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Ie,"settlesOnFatalProviderFault");function xe(){const s=Me();if(s!==void 0)return s;const e=ke("user").lifecycle.unattendedParkMs;return Pe()?$e()??e:e}p(xe,"unattendedParkMsForDaemon");function B(){return Te()}p(B,"parkSummaryGraceMsForDaemon");function Ce(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Ce,"yieldStallVerdict");function De(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ee?"pending":"wedged"}p(De,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Oe(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Oe,"sumTreeCpu");function Re(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(Re,"treeHasDescendants");class bt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ge()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=xe();for(const i of ee())await F.fresh(async()=>{try{this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),this.handleUnattendedParking(n,T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ye())t.fleet.has(i)||N(i);de(e)}parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ce(X(e)).length>0||ue(e)!==null||fe(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ie(n,i)!==null)?!0:le(e)}handleUnattendedParking(e,t,r,i,n,l){if(J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=this.parkResetCondition(e,t,r,i),a=be(e);if(a!==void 0){if(i-a<B())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!we(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:B()}}),P(e,"park"),S.teardown(e);return}if(o||Ae(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=Se(e,t.lifecycle),h=d!=="cancel"&&Ne(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){P(e,d),S.teardown(e);return}if(d==="cancel"){P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ve(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await pe(e,he("park",{},_e),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof me){P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Fe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await M("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
+`).map(async h=>{if(h!=="")try{await M("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await M("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||I(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Ce(!0,e.intent,!1,i-o.since)!=="kill")return;const a=T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=U&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:U,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await M("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Oe(t,e),i=Re(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity),o=I(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=x?await this.sampleTree(t.pid):null,d=T(n),h=d===null?null:I(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(De({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const v=C(d),g=Math.round((a??x)/6e4),y=$??!0,k=!y&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,b=!1,w;if(y)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(k!==null)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),b=!0,w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(A){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:A,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:y,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}}});try{ne(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:w})}catch(A){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:A})}R(n,"notice.crash",w,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}})}handleFatalFault(e,t=!0){const r=O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
+`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=Ie(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,v=d?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),y=oe(L(),se(L(),e)),k=[{relation:"Child",recipients:y.filter(f=>g.has(f))},{relation:"Watched node",recipients:y.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),ae(()=>{for(const{relation:f,recipients:b}of k)b.length!==0&&R(e,"notice.crash",`${f} ${v} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,b);te(r.faultId),t&&_&&Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:re(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{bt as BrokerSupervisionReconciler,Ee as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,B as parkSummaryGraceMsForDaemon,Ie as settlesOnFatalProviderFault,Oe as sumTreeCpu,Re as treeHasDescendants,xe as unattendedParkMsForDaemon,De as wedgeVerdict,Ce as yieldStallVerdict};

package/dist/daemon/runs/observer.js

@@ -1,4 +1,4 @@
-var D=Object.defineProperty;var r=(n,t)=>D(n,"name",{value:t,configurable:!0});import{prepared as d}from"../../core/sqlite-statements.js";import{randomUUID as O}from"node:crypto";import{openDb as l}from"../../core/canvas/db.js";import{getNode as p}from"../../core/canvas/canvas.js";import{BrokerClient as T,SocketTransport as I}from"../../core/broker-client/client.js";import{emitRunEvent as c,emitRunSettlement as m}from"../../core/runs/events.js";import{cancelOpenQuestions as R}from"../../core/runs/questions.js";import{emitEvent as _}from"../../core/events/emit.js";import{toRunOutcomeDTO as v}from"../api/map.js";import{readOutputResult as N}from"../../core/runtime/structured-output.js";const A=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),h=new Map,y=250;class b{static{r(this,"RunObserver")}nodeId;runId;client;timer;status;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;constructor(t,e){this.nodeId=t,this.runId=e;const i=l(),s=d(i,"SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(t,t===e?"node.settled":"child.settled");s&&(this.outcomeRevision=JSON.parse(s.body_json).outcome.revision);const o=d(i,"SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(t);o&&(this.status=JSON.parse(o.body_json).status),this.timer=setInterval(()=>this.pollOrClose(),250),this.timer.unref(),h.set(t,this),this.poll()}pollOrClose(){try{this.poll()}catch(t){this.close(),_({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:t})}}poll(){const t=p(this.nodeId);if(!t||t.run_id!==this.runId||!g(this.runId)){this.close();return}const e=t.pi_pid===null?"dormant":t.status;this.status!==e&&c(this.runId,this.nodeId,"node.status.changed",{status:e,reason:t.intent??null}),this.status=e;const i=v(t);if(i&&i.revision!==this.outcomeRevision&&Date.now()-Date.parse(i.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const s=d(l(),"SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const o of s)E(o.node_id,this.runId),h.get(o.node_id)?.poll()}if(this.outcomeRevision=i.revision,this.nodeId===this.runId){const s=i.kind==="result"?N(this.nodeId,t.grantee):void 0;m(this.runId,this.nodeId,"node.settled",{outcome:i,usage:JSON.parse(t.usage_json??A),...s===void 0?{}:{result:s}})}else m(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:i});if(t.grantee&&t.grantee!=="app:terminal"){const s=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};R(s,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(o=>_({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:o}))}}t.pi_pid===null?this.close():this.connect()}get assistantMessageStartSequence(){return this.openAssistant&&this.client&&p(this.nodeId)?.pi_pid!=null?this.assistantStartSequence:null}get openBlock(){return this.assistantMessageStartSequence===null?null:this.block}connect(){if(this.client||this.closed)return;const t=new T(this.nodeId,{transport:new I(this.nodeId,{recordsViewerFaults:!1})});this.client=t,t.on("connect",()=>t.send({type:"hello",role:"observer",client_id:O(),attends:!1})),t.on("frame",e=>{if(e.type==="message_start"&&e.message.role==="assistant"&&(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",this.assistantStartSequence=d(l(),"SELECT event_cursor FROM nodes WHERE node_id = ?").get(this.runId).event_cursor),e.type==="message_update"&&e.message.role==="assistant"&&this.openAssistant){const i=e.message.content;if(!Array.isArray(i))return;const s=i.at(-1);s&&typeof s=="object"&&"type"in s&&(s.type==="toolCall"?this.block={kind:"tool_call",name:"name"in s&&typeof s.name=="string"?s.name:""}:(s.type==="text"||s.type==="thinking")&&(this.block={kind:s.type}));const o=i.filter(u=>u?.type==="thinking").map(u=>u.thinking).join(""),a=i.filter(u=>u?.type==="text").map(u=>u.text).join(""),S=o.startsWith(this.thinking)?o.slice(this.thinking.length):o,k=a.startsWith(this.output)?a.slice(this.output.length):a;this.thinking=o,this.output=a,this.pendingThinking+=S,this.pendingOutput+=k,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>this.flushDeltas(),y),this.flushTimer.unref())}e.type==="message_end"&&e.message.role==="assistant"&&(this.flushDeltas(),this.openAssistant=!1,this.block=null)}),t.on("error",()=>{}),t.on("close",()=>{this.client===t&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null)}),t.connect()}flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const t=this.pendingThinking,e=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!t&&!e)return;if(d(l(),"SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}t&&c(this.runId,this.nodeId,"node.thinking.delta",{delta:t}),e&&c(this.runId,this.nodeId,"node.output_text.delta",{delta:e})}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){this.closed=!0,this.discardDeltas(),clearInterval(this.timer),h.delete(this.nodeId);const t=this.client;this.client=void 0,t?.close()}}function J(n){return h.get(n)?.assistantMessageStartSequence??null}r(J,"openAssistantMessageStartSequence");function Y(n){return h.get(n)?.openBlock??null}r(Y,"openAssistantBlock");function g(n){return d(l(),"SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(n,n)!==void 0}r(g,"runRootExists");function E(n,t){if(!h.has(n)&&g(t)){if(n!==t){const e=p(n);!d(l(),"SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(t,n)&&e&&Date.now()-Date.parse(e.created)<720*60*6e4&&c(t,n,"child.started",{child_node_id:n,kind:e.kind})}new b(n,t)}}r(E,"ensureRunObserver");function G(){const n=new Date(Date.now()-2592e6).toISOString(),t=d(l(),`SELECT n.node_id, n.run_id FROM node_rows n WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ?
+var D=Object.defineProperty;var r=(n,t)=>D(n,"name",{value:t,configurable:!0});import{prepared as d}from"../../core/sqlite-statements.js";import{randomUUID as O}from"node:crypto";import{openDb as l}from"../../core/canvas/db.js";import{getNode as f}from"../../core/canvas/canvas.js";import{BrokerClient as T,SocketTransport as I}from"../../core/broker-client/client.js";import{emitRunEvent as c,emitRunSettlement as m}from"../../core/runs/events.js";import{cancelOpenQuestions as R}from"../../core/runs/questions.js";import{emitEvent as p}from"../../core/events/emit.js";import{toRunOutcomeDTO as v}from"../api/map.js";import{readOutputResult as N}from"../../core/runtime/structured-output.js";const A=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),h=new Map,y=250;class b{static{r(this,"RunObserver")}nodeId;runId;client;timer;status;piPid=null;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;constructor(t,e){this.nodeId=t,this.runId=e;const i=l(),s=d(i,"SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(t,t===e?"node.settled":"child.settled");s&&(this.outcomeRevision=JSON.parse(s.body_json).outcome.revision);const o=d(i,"SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(t);o&&(this.status=JSON.parse(o.body_json).status),this.timer=setInterval(()=>this.pollOrClose(),250),this.timer.unref(),h.set(t,this),this.poll()}pollOrClose(){try{this.poll()}catch(t){this.close(),p({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:t})}}poll(){const t=f(this.nodeId);if(!t||t.run_id!==this.runId||!g(this.runId)){this.close();return}this.piPid=t.pi_pid??null;const e=t.pi_pid===null?"dormant":t.status;this.status!==e&&c(this.runId,this.nodeId,"node.status.changed",{status:e,reason:t.intent??null}),this.status=e;const i=v(t);if(i&&i.revision!==this.outcomeRevision&&Date.now()-Date.parse(i.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const s=d(l(),"SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const o of s)E(o.node_id,this.runId),h.get(o.node_id)?.poll()}if(this.outcomeRevision=i.revision,this.nodeId===this.runId){const s=i.kind==="result"?N(this.nodeId,t.grantee):void 0;m(this.runId,this.nodeId,"node.settled",{outcome:i,usage:JSON.parse(t.usage_json??A),...s===void 0?{}:{result:s}})}else m(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:i});if(t.grantee&&t.grantee!=="app:terminal"){const s=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};R(s,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(o=>p({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:o}))}}t.pi_pid===null?this.close():this.connect()}get assistantMessageStartSequence(){return this.openAssistant&&this.client&&this.piPid!==null?this.assistantStartSequence:null}get openBlock(){return this.assistantMessageStartSequence===null?null:this.block}connect(){if(this.client||this.closed)return;const t=new T(this.nodeId,{transport:new I(this.nodeId,{recordsViewerFaults:!1})});this.client=t,t.on("connect",()=>t.send({type:"hello",role:"observer",client_id:O(),attends:!1})),t.on("frame",e=>{if(e.type==="message_start"&&e.message.role==="assistant"&&(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",this.assistantStartSequence=d(l(),"SELECT event_cursor FROM nodes WHERE node_id = ?").get(this.runId).event_cursor),e.type==="message_update"&&e.message.role==="assistant"&&this.openAssistant){const i=e.message.content;if(!Array.isArray(i))return;const s=i.at(-1);s&&typeof s=="object"&&"type"in s&&(s.type==="toolCall"?this.block={kind:"tool_call",name:"name"in s&&typeof s.name=="string"?s.name:""}:(s.type==="text"||s.type==="thinking")&&(this.block={kind:s.type}));const o=i.filter(u=>u?.type==="thinking").map(u=>u.thinking).join(""),a=i.filter(u=>u?.type==="text").map(u=>u.text).join(""),S=o.startsWith(this.thinking)?o.slice(this.thinking.length):o,k=a.startsWith(this.output)?a.slice(this.output.length):a;this.thinking=o,this.output=a,this.pendingThinking+=S,this.pendingOutput+=k,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>this.flushDeltas(),y),this.flushTimer.unref())}e.type==="message_end"&&e.message.role==="assistant"&&(this.flushDeltas(),this.openAssistant=!1,this.block=null)}),t.on("error",()=>{}),t.on("close",()=>{this.client===t&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null)}),t.connect()}flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const t=this.pendingThinking,e=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!t&&!e)return;if(d(l(),"SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}t&&c(this.runId,this.nodeId,"node.thinking.delta",{delta:t}),e&&c(this.runId,this.nodeId,"node.output_text.delta",{delta:e})}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){this.closed=!0,this.discardDeltas(),clearInterval(this.timer),h.delete(this.nodeId);const t=this.client;this.client=void 0,t?.close()}}function J(n){return h.get(n)?.assistantMessageStartSequence??null}r(J,"openAssistantMessageStartSequence");function P(n){return h.get(n)?.openBlock??null}r(P,"openAssistantBlock");function g(n){return d(l(),"SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(n,n)!==void 0}r(g,"runRootExists");function E(n,t){if(!h.has(n)&&g(t)){if(n!==t){const e=f(n);!d(l(),"SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(t,n)&&e&&Date.now()-Date.parse(e.created)<720*60*6e4&&c(t,n,"child.started",{child_node_id:n,kind:e.kind})}new b(n,t)}}r(E,"ensureRunObserver");function Y(){const n=new Date(Date.now()-2592e6).toISOString(),t=d(l(),`SELECT n.node_id, n.run_id FROM node_rows n WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ?
     AND n.outcome_reason != 'parked' AND NOT EXISTS (
     SELECT 1 FROM run_events e WHERE e.run_id = n.run_id AND e.node_id = n.node_id AND e.type = CASE WHEN n.node_id = n.run_id THEN 'node.settled' ELSE 'child.settled' END
-      AND json_extract(e.body_json, '$.outcome.revision') = COALESCE(n.outcome_revision, 1))))`).all(n);for(const e of t)try{E(e.node_id,e.run_id),f.delete(e.node_id)}catch(i){L(e.node_id,i)}}r(G,"pollRunObservers");const f=new Map;function L(n,t){const e=t instanceof Error?t.message:String(t);f.get(n)!==e&&(f.set(n,e),_({level:"error",event:"daemon.runs.observer_failed",node_id:n,error:t}))}r(L,"reportObserverFailure");export{E as ensureRunObserver,Y as openAssistantBlock,J as openAssistantMessageStartSequence,G as pollRunObservers};
+      AND json_extract(e.body_json, '$.outcome.revision') = COALESCE(n.outcome_revision, 1))))`).all(n);for(const e of t)try{E(e.node_id,e.run_id),_.delete(e.node_id)}catch(i){L(e.node_id,i)}}r(Y,"pollRunObservers");const _=new Map;function L(n,t){const e=t instanceof Error?t.message:String(t);_.get(n)!==e&&(_.set(n,e),p({level:"error",event:"daemon.runs.observer_failed",node_id:n,error:t}))}r(L,"reportObserverFailure");export{E as ensureRunObserver,P as openAssistantBlock,J as openAssistantMessageStartSequence,Y as pollRunObservers};

package/docs/plugin/deploying.md

@@ -62,8 +62,8 @@ A definition with `version: N`, or a list of definitions with one per supported
 - gains paging where it returned everything: a limit flag whose constraint states a default page size ("default N"), or a cursor input together with a `next_cursor`-style output. The answer keeps its shape, so callers that do not page silently get a partial list. A changed stated page size is breaking too. A new `--cursor`, `--limit` with no stated default, or `next_cursor` output alone is additive, and so is a declared `default`, which crtr applies while parsing and does not send;
 - changes how the request is built: method, route (apart from the `/v<N>` segment), body root, constant body fields, a parameter's placement or wire name, or streaming;
 - removes an output field, changes its type, makes it optional, or adds a `file` output;
-- widens an output so a client typed against the previous archive could read a value outside its type: adds an enum value or a union variant, adds `| null`, turns `int` into `number`, replaces a declared type with `unknown`, or opens a closed object by dropping its `children`;
-- narrows a `json` flag's shape so a value the previous archive accepted is refused: removes an enum value or a union variant, adds a required member, or closes an open object by adding `children`.
+- widens an output so a client typed against the previous archive could read a value outside its type: adds an enum value or a union variant, adds `| null`, turns `int` into `number`, replaces a declared type with `unknown`, opens a closed object by dropping its `children` or adding an `extra`, or adds an open object member whose shape its old `extra` did not accept;
+- narrows a `json` flag's shape so a value the previous archive accepted is refused: removes an enum value or a union variant, adds a required member, closes an open object by adding `children` or removing its `extra`, or adds a member to an open object that its old `extra` did not accept. Removing an output member is always breaking, open or closed.
 
 A new optional flag, a new leaf, a new optional output field and changed prose are additive.
 

package/docs/plugin/output.md

@@ -26,12 +26,14 @@ Each leaf declares an `output` object. Field object keys are returned verbatim i
 
 Fields are required by default. Pass `{ required: false }` for a field that can be absent. `field.nullable(field.string('...'))` declares a present field that can be `string | null`; it does not make the field optional.
 
-The handler result type is inferred from `output`, including structured members and elements. TypeScript reports a missing required field or a wrong value type at the `defineLeaf` call. `definePlugin` throws `PluginDefinitionError` for a malformed structured field, a file field that is not top-level, or a second file output. The Fetch handler checks the returned object's top-level fields again before sending it. It rejects missing fields, known type mismatches, and keys that were not declared, returning a non-2xx `handler_output_invalid` error instead of a JSON success response. It also checks nested values against the declared shape: an object with declared `children` is closed, so a member it does not declare, a missing required member, an array element or map value of the wrong shape, or an enum value outside `values` is rejected. An `object` declared without `children` accepts any members.
+The handler result type is inferred from `output`, including structured members and elements. TypeScript reports a missing required field or a wrong value type at the `defineLeaf` call. `definePlugin` throws `PluginDefinitionError` for a malformed structured field, a file field that is not top-level, or a second file output. The Fetch handler checks the returned object's top-level fields again before sending it. It rejects missing fields, known type mismatches, and keys that were not declared, returning a non-2xx `handler_output_invalid` error instead of a JSON success response. It also checks nested values against the declared shape: an object with declared `children` is closed unless it declares `extra` (see Structured fields), so a member it does not declare, a missing required member, an array element or map value of the wrong shape, or an enum value outside `values` is rejected. An `object` declared without `children` accepts any members, and so does one whose `extra` is `field.unknown(...)`.
 
 ## Structured fields
 
 `field.object({ ... })` types an object's members. Each member is a `field.*` builder, and members can be structured themselves. `field.array(field.string('A tag.'))` types each element; the element's `required` is ignored and the array's constraint defaults to the element's. `field.enum(['open', 'closed'], 'Thread state.')` restricts a string to fixed values, which the handler sees as literal types. The generated manifest carries the whole shape: object members as ordered `children`, an array element as `items`, enum `values`. `crtr` command help renders it. `field.nullable` keeps the members, element, or values of the field it wraps.
 
+An object with members is closed: a member it does not declare is rejected. To accept members it does not declare, pass `extra` as the third argument, `field.object({ id: field.string('Id.') }, 'The row.', { extra: field.unknown('Any other column.') })`. Every undeclared member must then have the `extra` field's shape (`field.unknown` accepts any value; `field.string(...)` only strings; the `extra` field's own `required` is ignored). The manifest carries it as `extra`, `crtr` command help lists it as an `other members` row, and the generated client types it as an index signature. The handler's inferred type for an open object is the declared members plus `{ readonly [key: string]: unknown }`, whatever the `extra` shape is, so a handler can return an object literal with extra members; the runtime check enforces the `extra` shape. `extra` is only valid on an object with members (an object without members already accepts anything; a `map` takes a value shape), and a `file` cannot be inside it.
+
 ```ts
 output: {
   thread: field.object({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.524",
+  "version": "0.3.526",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -139,7 +139,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.524",
+    "@north-light/identity": "0.3.526",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.524",
+  "version": "0.3.526",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.524",
+  "version": "0.3.526",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -24,7 +24,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.524",
+        "@north-light/identity": "0.3.526",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12468,17 +12468,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12486,20 +12486,20 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.524",
-        "@north-light/identity": "0.3.524"
+        "@north-light/api": "0.3.526",
+        "@north-light/identity": "0.3.526"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.524",
+      "version": "0.3.526",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.524",
-        "@north-light/identity": "0.3.524",
+        "@north-light/api": "0.3.526",
+        "@north-light/identity": "0.3.526",
         "jose": "^6.2.1"
       },
       "bin": {