@north-light/cli 0.3.528 → 0.3.529
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/daemon/fleet.js
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
|
-
var g=Object.defineProperty;var l=(s,e)=>g(s,"name",{value:e,configurable:!0});import{getNode as v,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as _}from"../core/runtime/lifecycle.js";import{publishNotice as E}from"../core/runtime/close.js";import{
|
|
1
|
+
var g=Object.defineProperty;var l=(s,e)=>g(s,"name",{value:e,configurable:!0});import{getNode as v,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as _}from"../core/runtime/lifecycle.js";import{publishNotice as E}from"../core/runtime/close.js";import{readFault as P,readInterruptedTurn as N,readTransportPoisonVerdict as x}from"../core/canvas/recovery.js";import{hasPendingWake as C}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as F,MAX_RESPAWN_ATTEMPTS as M,OFFLINE_RESPAWN_RETRY_MS as S,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as c}from"../core/events/emit.js";import{brokerThresholdsForDaemon as D}from"../core/runtime/fleet.js";import{isRecoveryHeld as L}from"../core/runtime/recovery-hold.js";function w(s,e){const r=P(s.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`,i=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${s.name}\` (${s.kind}) never started.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Its pi vehicle exited before the session came up (no pi_session_id was ever recorded), so the node produced no output. ${i}
|
|
6
|
+
|
|
7
|
+
`+t+`Inspect canonical event diagnostics with \`crtr sys logs --node ${s.node_id}\`.
|
|
6
8
|
|
|
7
9
|
If the work still needs doing, re-spawn it.`}l(w,"bootFailureBody");async function y(s,e){await m(s.node_id,w(s,e),{from:s.node_id})}l(y,"surfaceBootFailure");function b(s){return`\u26A0 Out of memory \u2014 \`${s.name}\` (${s.kind}) was killed because its app's processes together passed the app's memory limit.
|
|
8
10
|
|
|
9
|
-
Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${s.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function
|
|
11
|
+
Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${s.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function W(s){const e=x(s.node_id),r=e===null?"":`
|
|
10
12
|
|
|
11
13
|
Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${s.name}\` (${s.kind}) kept dying without settling a turn.
|
|
12
14
|
|
|
13
15
|
Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
|
|
14
16
|
|
|
15
|
-
Fix the crash cause, then revive the node if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(
|
|
17
|
+
Fix the crash cause, then revive the node if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(W,"surfaceCrashLoop");async function q(s,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${s.name}\` (${s.kind}) could not be relaunched.
|
|
16
18
|
|
|
17
19
|
The daemon refused this recovery before starting a broker: ${r}
|
|
18
20
|
|
|
19
|
-
The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(
|
|
21
|
+
The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(q,"surfaceRecoveryFailure");function G(s){return s?.cycle_pending!==!0}l(G,"retryResumeMode");function H(s,e,r=!0){return s===null||s.status==="done"||s.status==="canceled"||s.status==="dead"?"forget":s.intent==="refresh"?"respawn-fresh":R(s)?"dormant":s.lifecycle==="resident"&&!e&&!r&&s.pi_session_id!=null?"release-idle":s.pi_session_id!=null?G(s)?"respawn-resume":"respawn-cycle":e||s.fork_from!=null?"respawn-fresh":"boot-failure"}l(H,"classifyDeadNode");const Y=3e4;class le{static{l(this,"DaemonFleet")}#n;#o;#s;#l;#e=new Map;#r=new Set;#a=new Set;#t=new Map;#c=Number.NEGATIVE_INFINITY;constructor(e){this.#n=e.epoch,this.#o=e.enqueue,this.#s=e.now??Date.now,this.#l=e.probeOnline??(()=>A())}epoch(){return this.#n}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#r.delete(e),this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#r.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#r.size}reserveSlot(e){if(this.#r.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=D();if(this.occupancy()<r)return this.#r.add(e),!0;const t=this.#s();return t-this.#c>=Y&&(this.#c=t,c({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:r}})),!1}releaseReservation(e){this.#r.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r!==void 0){this.#a.add(e);try{process.kill(r,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#a.has(e)}forget(e){this.#e.delete(e),this.#r.delete(e),this.#t.delete(e)}onChildExit(e,r,t){t&&$(t),this.#e.delete(e),this.#o(()=>{const i=v(e),a=i?.launched_at==null?Number.NaN:Date.parse(i.launched_at),n=Number.isFinite(a)?Math.max(0,this.#s()-a):null,d=i?.respawn_failures??0;i?.status==="done"||i?.status==="canceled"||i?.status==="dead"?this.#t.delete(e):this.#t.set(e,r),this.#u(e,r,n,d)})}deliverExit(e,r){this.onChildExit(e,r)}async applyDeadRowPolicy(e){const r=this.#t.get(e);try{await this.#f(e,r)}finally{this.#t.get(e)===r&&this.#t.delete(e)}}async terminalizeIfStillNonTerminal(e,r,t,i,a){const n=v(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#i(e,n,{event:r,reason:i,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>q(n,t),"surface")}),!0)}dispose(){}#u(e,r,t,i){c({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:i}})}async#f(e,r){return this.#h(e,r)}async#h(e,r){const t=v(e);if(t!==null&&T(t.profile_id))return h(e),"forget";if(this.#a.delete(e))return h(e),"forget";if(L(e))return"dormant";const i=N(e)?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return h(e),await this.#i(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(t),{from:e})},"surface")}),"forget";const a=H(t,i,C(e));switch(a){case"forget":h(e);break;case"dormant":h(e);break;case"release-idle":h(e),_(e,"release"),c({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#i(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:w(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#d(e,t,a,i);break}return a}async#d(e,r,t,i){const a=k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#l()){F(e,n,this.#s()),c({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:S,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?O(a,this.#s()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=M};if(f.exhausted){const u=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#i(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:u,respawn_failures:f.failures},wake:u,...r.pi_session_id!=null?{surface:l(()=>W(r),"surface")}:{}});return}if(d&&f.notBefore!==null){c({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}c({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&i?"runtime-restart-abort":void 0,recovery:!0})}catch(u){c({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:u,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",u,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:u instanceof Error?u.message:String(u)})}}async#i(e,r,t){c({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),_(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>y(r)))()}catch(i){c({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:i})}E(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{le as DaemonFleet,H as classifyDeadNode,G as retryResumeMode,y as surfaceBootFailure};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{
|
|
2
|
-
`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function
|
|
3
|
-
`).map(async h=>{if(h!=="")try{await
|
|
4
|
-
`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=
|
|
1
|
+
var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ee,listUnnotifiedFatalFaultNodes as te,markFaultNotified as ne,readFault as O,readLiveTurn as E,recordFault as re}from"../../core/canvas/recovery.js";import{errorClassFromFault as ie}from"../../core/fault-classifier.js";import{nextFaultRetry as ae}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as G,withCanvasWrite as se}from"../../core/canvas/db.js";import{getById as oe}from"../../core/graph/objects.js";import{watcherIdsOf as le}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as de,resetStaleExecutionMail as ce}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ue}from"../../core/bash-jobs.js";import{focusOf as fe}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as pe}from"../api/viewer-presence.js";import{deliverLive as me}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as he}from"../../core/runtime/broker-request.js";import{formatCard as _e,PARK_SUMMARY_PROMPT as ge}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ke}from"../../core/runtime/fleet.js";import{readConfig as ve}from"../../core/config.js";import{beginPendingPark as ye,consumePendingPark as N,pendingParkIds as be,pendingParkStartedAt as we}from"../park-pending.js";import{shouldParkAfterSummary as Se}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Ne}from"./live-obligation.js";import{readGoal as Pe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as $e}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Te,envTestUnattendedParkMs as Ae,parkSummaryGraceMs as Me,envUnattendedParkMs as Ee}from"../../shared/env.js";const A=j(z);function Fe(s){return!ee(s)&&(Pe(s)?.trim()??"")!==""}p(Fe,"awaitsBirthTurn");function xe(s){return E(s)?.busy===!0}p(xe,"liveBusy");function Nt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(Nt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function x(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(x,"busySinceForIncarnation");const W=3*6e4,B=2e4,I=20*6e4,Ie=2,Ce=5*6e4;function De(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(De,"settlesOnFatalProviderFault");function Oe(){const s=Ee();if(s!==void 0)return s;const e=ve("user").lifecycle.unattendedParkMs;return Te()?Ae()??e:e}p(Oe,"unattendedParkMsForDaemon");function U(){return Me()}p(U,"parkSummaryGraceMsForDaemon");function Re(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Re,"yieldStallVerdict");function Ge(s){return!s.busy||s.quietForMs===null||s.quietForMs<I?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ie?"pending":"wedged"}p(Ge,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
|
|
2
|
+
`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Le(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Le,"sumTreeCpu");function We(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(We,"treeHasDescendants");class Pt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ke()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Oe();for(const i of te())await F.fresh(async()=>{try{this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),this.handleUnattendedParking(n,T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of be())t.fleet.has(i)||N(i);ce(e)}parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ue(X(e)).length>0||fe(e)!==null||pe(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ae(n,i)!==null)?!0:de(e)}handleUnattendedParking(e,t,r,i,n,l){if(J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=this.parkResetCondition(e,t,r,i),a=we(e);if(a!==void 0){if(i-a<U())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!Se(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:U()}}),P(e,"park"),S.teardown(e);return}if(o||xe(e)||Fe(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=Ne(e,t.lifecycle),h=d!=="cancel"&&$e(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){P(e,d),S.teardown(e);return}if(d==="cancel"){P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ye(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await me(e,_e("park",{},ge),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof he){P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ce)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
|
|
3
|
+
`).map(async h=>{if(h!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||x(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Re(!0,e.intent,!1,i-o.since)!=="kill")return;const a=T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=B&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:B,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Le(t,e),i=We(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity),o=x(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=I?await this.sampleTree(t.pid):null,d=T(n),h=d===null?null:x(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(Ge({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const v=C(d),g=Math.round((a??I)/6e4),y=$??!0,k=!y&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,b=!1,w;if(y)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(k!==null)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),b=!0,w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(M){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:M,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:y,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}}});try{re(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:w})}catch(M){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:M})}R(n,"notice.crash",w,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}})}handleFatalFault(e,t=!0){const r=O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
|
|
4
|
+
`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=De(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,v=d?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),y=le(G(),oe(G(),e)),k=[{relation:"Child",recipients:y.filter(f=>g.has(f))},{relation:"Watched node",recipients:y.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),se(()=>{for(const{relation:f,recipients:b}of k)b.length!==0&&R(e,"notice.crash",`${f} ${v} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,b);ne(r.faultId),t&&_&&Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:ie(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{Pt as BrokerSupervisionReconciler,Ie as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,De as settlesOnFatalProviderFault,Le as sumTreeCpu,We as treeHasDescendants,Oe as unattendedParkMsForDaemon,Ge as wedgeVerdict,Re as yieldStallVerdict};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.529",
|
|
4
4
|
"description": "crtr — agent runtime with memory, plugins, and marketplaces",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -139,7 +139,7 @@
|
|
|
139
139
|
},
|
|
140
140
|
"license": "UNLICENSED",
|
|
141
141
|
"dependencies": {
|
|
142
|
-
"@north-light/identity": "0.3.
|
|
142
|
+
"@north-light/identity": "0.3.529",
|
|
143
143
|
"@earendil-works/pi-agent-core": "1.1.0",
|
|
144
144
|
"@earendil-works/pi-ai": "1.1.0",
|
|
145
145
|
"@earendil-works/pi-coding-agent": "1.1.0",
|
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.529",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/cli",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.529",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "UNLICENSED",
|
|
12
12
|
"workspaces": [
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"@milkdown/preset-gfm": "^7.22.3",
|
|
25
25
|
"@milkdown/prose": "^7.22.3",
|
|
26
26
|
"@milkdown/utils": "^7.22.3",
|
|
27
|
-
"@north-light/identity": "0.3.
|
|
27
|
+
"@north-light/identity": "0.3.529",
|
|
28
28
|
"cron-parser": "^5.6.0",
|
|
29
29
|
"esbuild": "^0.27.7",
|
|
30
30
|
"jose": "^6.2.12",
|
|
@@ -12468,17 +12468,17 @@
|
|
|
12468
12468
|
},
|
|
12469
12469
|
"packages/crouter-api": {
|
|
12470
12470
|
"name": "@north-light/api",
|
|
12471
|
-
"version": "0.3.
|
|
12471
|
+
"version": "0.3.529",
|
|
12472
12472
|
"license": "UNLICENSED"
|
|
12473
12473
|
},
|
|
12474
12474
|
"packages/crouter-env-docker": {
|
|
12475
12475
|
"name": "@north-light/env-docker",
|
|
12476
|
-
"version": "0.3.
|
|
12476
|
+
"version": "0.3.529",
|
|
12477
12477
|
"license": "UNLICENSED"
|
|
12478
12478
|
},
|
|
12479
12479
|
"packages/crouter-identity": {
|
|
12480
12480
|
"name": "@north-light/identity",
|
|
12481
|
-
"version": "0.3.
|
|
12481
|
+
"version": "0.3.529",
|
|
12482
12482
|
"license": "UNLICENSED",
|
|
12483
12483
|
"dependencies": {
|
|
12484
12484
|
"jose": "^6.2.1"
|
|
@@ -12486,20 +12486,20 @@
|
|
|
12486
12486
|
},
|
|
12487
12487
|
"packages/crouter-plugin": {
|
|
12488
12488
|
"name": "@north-light/plugin",
|
|
12489
|
-
"version": "0.3.
|
|
12489
|
+
"version": "0.3.529",
|
|
12490
12490
|
"license": "UNLICENSED",
|
|
12491
12491
|
"dependencies": {
|
|
12492
|
-
"@north-light/api": "0.3.
|
|
12493
|
-
"@north-light/identity": "0.3.
|
|
12492
|
+
"@north-light/api": "0.3.529",
|
|
12493
|
+
"@north-light/identity": "0.3.529"
|
|
12494
12494
|
}
|
|
12495
12495
|
},
|
|
12496
12496
|
"packages/crouter-sdk": {
|
|
12497
12497
|
"name": "@north-light/sdk",
|
|
12498
|
-
"version": "0.3.
|
|
12498
|
+
"version": "0.3.529",
|
|
12499
12499
|
"license": "UNLICENSED",
|
|
12500
12500
|
"dependencies": {
|
|
12501
|
-
"@north-light/api": "0.3.
|
|
12502
|
-
"@north-light/identity": "0.3.
|
|
12501
|
+
"@north-light/api": "0.3.529",
|
|
12502
|
+
"@north-light/identity": "0.3.529",
|
|
12503
12503
|
"jose": "^6.2.1"
|
|
12504
12504
|
},
|
|
12505
12505
|
"bin": {
|