@north-light/cli 0.3.527 → 0.3.529
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
var Z=Object.defineProperty;var r=(e,n)=>Z(e,"name",{value:n,configurable:!0});import{prepared as d}from"../sqlite-statements.js";import{envNodeId as
|
|
1
|
+
var Z=Object.defineProperty;var r=(e,n)=>Z(e,"name",{value:n,configurable:!0});import{prepared as d}from"../sqlite-statements.js";import{envNodeId as x}from"../../shared/env.js";import{rmdirSync as ee,rmSync as ne,statSync as te}from"node:fs";import{dirname as oe,join as j,resolve as re}from"node:path";import{removeTreeBeneath as W}from"../spaces/open-beneath.js";import{emitEvent as ie}from"../events/emit.js";import{openDb as a,prepared as b,withCanvasWrite as p}from"./db.js";import{conversationStore as se,ConversationNotFoundError as de}from"../conversation-store/index.js";import{hasUnseenWakeMail as ae}from"./deliveries.js";import{isAppRun as I}from"./app-run.js";import{registerOutcomeAction as F}from"./outcome-actions.js";import{publish as le}from"../graph/events.js";import{nodeObject as ue}from"../graph/objects.js";import{findNodeBySessionFileSql as _e}from"./sessions.js";import{hydrateFault as ce}from"./recovery.js";import{hasUnreconciledWorktree as pe,openWorktree as fe,readWorktree as Ee,worktreeFromRow as me}from"./worktrees.js";import{isPidAlive as Ne,isPidPresent as H,recordedPidLiveness as Re}from"./pid.js";import{ensureHome as ye,jobDir as Se,nodeDir as Oe,nodeRunDir as C}from"./paths.js";import{appForNode as we,ensureNodeDirs as Te,nodeAgentDir as he}from"./node-agent-paths.js";import{appSpace as U}from"../layout.js";import{createObject as Le,getById as M,ownerForGrantee as w,setObjectKind as ge,setObjectName as De}from"../graph/objects.js";import{deleteNodeWithOwned as ve}from"../graph/deletion.js";function m(e,n){if(e==null)return null;const t=JSON.stringify(e);if(t===void 0)throw new TypeError(`${n} must be JSON-serializable`);return t}r(m,"jsonValue");function rn(e,n){if(d(e,"SELECT deleting_at FROM nodes WHERE node_id = ?").get(n)?.deleting_at)throw new Error("run is deleting")}r(rn,"assertRunNotDeleting");function ke(e,n){const t=w(n,e.grantee??"app:terminal");Le(n,{id:e.node_id,type:"node",kind:e.kind,owner:t.rowid,privacy:"public",name:e.name,preview:""}),d(n,`INSERT INTO nodes (node_id, description, title, icon, cycles, mode, lifecycle, cwd, host_kind, profile_id, parent, spawned_by, creator, fork_from, fork_source_file, created, passive_default, cycle_pending, model_override, persona_ack_json, review_binding_json, launch_config_json, scopes_json, isolated, status, intent, final_report, finalized_at, deadline_at, "window", tmux_session, pane, sub, grant_id, client_id, metadata_json, run_id, event_cursor, usage_json, deleting_at) VALUES (${Array(40).fill("?").join(", ")})`).run(e.node_id,e.description??null,e.title??null,e.icon??null,e.cycles??null,e.mode,e.lifecycle,e.cwd,e.host_kind??null,e.profile_id??null,e.parent??null,e.spawned_by??null,e.creator??null,e.fork_from??null,e.fork_source_file??null,e.created,e.passive_default===!0?1:0,e.cycle_pending===!0?1:0,e.model_override??null,m(e.persona_ack,"persona_ack"),m(e.review_binding,"review_binding"),m(e.launch,"launch"),m(e.scopes,"scopes"),e.isolated===!0?1:0,e.status??"active",e.intent??null,e.final_report??null,e.finalized_at??null,e.deadline_at??null,e.window??null,e.tmux_session??null,e.pane??null,e.sub??null,e.grant_id??null,e.client_id??null,e.metadata_json??null,e.run_id??null,e.event_cursor??null,e.usage_json??null,e.deleting_at??null)}r(ke,"seedRow");function P(e){if(e.fault_id==null)return null;const n=Object.entries(e).filter(([t])=>t.startsWith("fault_")).map(([t,o])=>[t==="fault_id"?t:t.slice(6),o]);return ce(Object.fromEntries(n))}r(P,"faultFrom");function E(e){const n={node_id:e.node_id,name:e.name,description:e.description??void 0,cycles:e.cycles??void 0,kind:e.kind,mode:e.mode,lifecycle:e.lifecycle,status:e.status,cwd:e.cwd,host_kind:e.host_kind??null,profile_id:e.profile_id??null,scopes:e.scopes_json==null?null:JSON.parse(e.scopes_json),isolated:e.isolated===1,grantee:e.grantee,sub:e.sub??null,grant_id:e.grant_id??null,client_id:e.client_id??null,metadata_json:e.metadata_json??null,run_id:e.run_id??null,event_cursor:e.event_cursor??null,last_activity_at:e.last_activity_at??null,usage_json:e.usage_json??null,last_turn_error_json:e.last_turn_error_json??null,deleting_at:e.deleting_at??null,parent:e.parent??null,created:e.created,intent:e.intent??null,frozen_at:e.frozen_at??null,respawn_failures:e.respawn_failures??0,respawn_not_before:e.respawn_not_before??null,launched_at:e.launched_at??null,last_settled_at:e.last_settled_at??null,terminal_reason:e.terminal_reason??null,final_report:e.final_report??null,finalized_at:e.finalized_at??null,outcome_kind:e.outcome_kind??null,outcome_reason:e.outcome_reason??null,outcome_at:e.outcome_at??null,outcome_revision:e.outcome_revision??0,outcome_detail_json:e.outcome_detail_json??null,deadline_at:e.deadline_at??null,telemetry_tokens_in:e.telemetry_tokens_in??null,telemetry_context_tokens:e.telemetry_context_tokens??null,telemetry_last_activity:e.telemetry_last_activity??null,telemetry_updated_at:e.telemetry_updated_at??null,pi_pid:e.pi_pid??null,pi_pid_identity:e.pi_pid_identity??null,window:e.window??null,tmux_session:e.tmux_session??null,pane:e.pane??null,fault:null,streaming:!1},t=H(n.pi_pid),o=P(e);return n.fault=o?.link==="pi\u2192provider"&&o.retry.by==="daemon"&&o.retry.disposition==="auto"&&!t?null:o,n.streaming=e.turn_open_at!==null&&e.turn_closed_at===null&&t,n}r(E,"rowFrom");function q(e,n=Ee(e.node_id)){return{...E(e),title:e.title??void 0,icon:e.icon??void 0,spawned_by:e.spawned_by??null,creator:e.creator??null,fork_from:e.fork_from??null,fork_source_file:e.fork_source_file??null,passive_default:e.passive_default===1,cycle_pending:e.cycle_pending===1,model_override:e.model_override??null,scopes:e.scopes_json==null?null:JSON.parse(e.scopes_json),persona_ack:e.persona_ack_json==null?void 0:JSON.parse(e.persona_ack_json),review_binding:e.review_binding_json==null?void 0:JSON.parse(e.review_binding_json),launch:e.launch_config_json==null?void 0:JSON.parse(e.launch_config_json),pi_session_id:e.pi_session_id??null,pi_session_file:e.pi_session_file??null,...n===null?{}:{managed_worktree:n}}}r(q,"metaFrom");function sn(e,n={}){return ye(),p(t=>{ke(e,t),e.managed_worktree!==void 0&&e.managed_worktree!==null&&fe(e.node_id,e.managed_worktree,t),n.outcomeDelivery!==void 0&&F(t,e.node_id,n.outcomeDelivery),n.withinCreate?.(t,e)}),Te(e.node_id),y(e.node_id)}r(sn,"createNode");function y(e){const n=b(a(),"SELECT * FROM node_rows WHERE node_id = ?").get(e);return n===void 0?null:q(n)}r(y,"getNode");const Ae=1440*60*1e3;function $(e){return{requestKey:e.request_key,nodeId:e.node_id,state:e.state}}r($,"createRequestRecord");function dn(e){const n=a().prepare("SELECT request_key, node_id, state FROM create_requests WHERE request_key = ?").get(e);return n===void 0?null:$(n)}r(dn,"findCreateRequest");function an(e){return p(n=>{const t=Date.now();n.prepare("DELETE FROM create_requests WHERE expires_at <= ?").run(t);const o=n.prepare("SELECT request_key, node_id, state FROM create_requests WHERE request_key = ?").get(e);return o!==void 0?$(o):(n.prepare("INSERT INTO create_requests (request_key, node_id, expires_at, state) VALUES (?, NULL, ?, 'reserved')").run(e,t+Ae),{requestKey:e,nodeId:null,state:"reserved"})})}r(an,"reserveCreateRequest");function ln(e=Date.now()){return p(n=>Number(n.prepare("DELETE FROM create_requests WHERE expires_at <= ?").run(e).changes))}r(ln,"pruneExpiredCreateRequests");function un(e,n){p(t=>{t.prepare("UPDATE create_requests SET node_id = ?, state = 'complete' WHERE request_key = ?").run(n,e)})}r(un,"recordCreateRequestNode");function _n(e,n){p(t=>{t.prepare("UPDATE create_requests SET node_id = ?, state = 'reserved' WHERE request_key = ?").run(n,e)})}r(_n,"reserveCreateRequestNode");function cn(e,n){p(t=>{t.prepare("UPDATE create_requests SET state = 'complete' WHERE request_key = ? AND node_id = ?").run(e,n)})}r(cn,"completeCreateRequest");function pn(e,n){p(t=>{if(n===null){t.prepare("DELETE FROM create_requests WHERE request_key = ? AND node_id IS NULL").run(e);return}t.prepare("DELETE FROM create_requests WHERE request_key = ? AND node_id = ?").run(e,n)})}r(pn,"abandonCreateRequest");function fn(e){return re(e)}r(fn,"canonicalSessionFile");function En(e){const n=_e(e);return n===null?null:y(n)}r(En,"findNodeBySessionFile");function mn(e){const n=d(a(),"SELECT * FROM node_rows WHERE node_id = ?").get(e);return n?E(n):null}r(mn,"getRow");function Nn(e){const n=d(a(),"SELECT * FROM node_rows WHERE pane = ?").get(e);return n?E(n):null}r(Nn,"getRowByPane");function Rn(e){return d(a(),`SELECT * FROM node_rows
|
|
2
2
|
WHERE deadline_at IS NOT NULL AND outcome_at IS NULL
|
|
3
3
|
AND deadline_at <= ? AND status IN ('active','idle') AND kind != 'human'
|
|
4
4
|
ORDER BY deadline_at, created, node_id`).all(e).map(E)}r(Rn,"dueNodeDeadlines");function yn(e,n){return Number(d(a(),"UPDATE nodes SET deadline_at = ? WHERE node_id = ?").run(n,e).changes)===1}r(yn,"setNodeDeadline");function Sn(e,n){return!Number.isFinite(n)||n<=0?0:p(t=>{const o=d(t,`SELECT node_id, deadline_at FROM nodes
|
|
5
|
-
WHERE profile_id = ? AND deadline_at IS NOT NULL AND status IN ('active', 'idle')`).all(e),i=d(t,"UPDATE nodes SET deadline_at = ? WHERE node_id = ? AND deadline_at = ?");let s=0;for(const l of o){const
|
|
6
|
-
WHERE o.type = 'node' AND i.scheme = 'local-user' AND i.external_id = 'user:local'`,s=e.prepare(`${o} LIMIT 1`).get()!==void 0?Number(e.prepare(`UPDATE objects SET owner = ? WHERE rowid IN (${o})`).run(
|
|
5
|
+
WHERE profile_id = ? AND deadline_at IS NOT NULL AND status IN ('active', 'idle')`).all(e),i=d(t,"UPDATE nodes SET deadline_at = ? WHERE node_id = ? AND deadline_at = ?");let s=0;for(const l of o){const _=Date.parse(l.deadline_at);Number.isNaN(_)||(s+=Number(i.run(new Date(_+n).toISOString(),l.node_id,l.deadline_at).changes))}return s})}r(Sn,"shiftProfileNodeDeadlines");function On(e,n,t){const o=`SELECT o.rowid FROM objects o JOIN identities i ON i.object = o.owner JOIN nodes n ON n.node_id = o.id
|
|
6
|
+
WHERE o.type = 'node' AND i.scheme = 'local-user' AND i.external_id = 'user:local'`,s=e.prepare(`${o} LIMIT 1`).get()!==void 0?Number(e.prepare(`UPDATE objects SET owner = ? WHERE rowid IN (${o})`).run(w(e,"app:terminal").rowid).changes):0,l=`SELECT o.rowid FROM objects o JOIN peer_stubs s ON s.object = o.rowid JOIN identities i ON i.object = o.owner
|
|
7
|
+
WHERE o.type = 'node' AND o.reception = 2 AND json_extract(s.envelope_json, '$.app') IS NULL
|
|
8
|
+
AND i.scheme = 'app' AND i.external_id = 'app:terminal'`,_=e.prepare(`${l} LIMIT 1`).get()!==void 0?Number(e.prepare(`UPDATE objects SET owner = ? WHERE rowid IN (${l})`).run(w(e,"user:local").rowid).changes):0;return Number(d(e,"UPDATE nodes SET sub = ? WHERE sub = 'user:local'").run(n).changes)+s+_+Number(d(e,`UPDATE nodes SET sub = ? WHERE sub IS NULL AND node_id IN (
|
|
7
9
|
SELECT o.id FROM objects o JOIN identities i ON i.object = o.owner
|
|
8
|
-
WHERE o.type = 'node' AND i.scheme = 'app' AND i.external_id = 'app:terminal')`).run(n).changes)+Number(d(e,"UPDATE nodes SET grant_id = NULL WHERE grant_id = ?").run(t).changes)}r(On,"rewriteLocalNodeIdentity");function
|
|
10
|
+
WHERE o.type = 'node' AND i.scheme = 'app' AND i.external_id = 'app:terminal')`).run(n).changes)+Number(d(e,"UPDATE nodes SET grant_id = NULL WHERE grant_id = ?").run(t).changes)}r(On,"rewriteLocalNodeIdentity");function T(e,n){if(e==null)return null;if(typeof e=="string"||typeof e=="number"||typeof e=="bigint"||e instanceof Uint8Array)return e;throw new TypeError(`${n} must be a SQLite scalar`)}r(T,"sqlInput");function wn(e,n){const t=[],o=r((s,l=s)=>{Object.prototype.hasOwnProperty.call(n,s)&&t.push([l,T(n[s],s)])},"add");if(o("description"),o("title"),o("icon"),o("cycles"),o("mode"),o("lifecycle"),o("cwd"),o("host_kind"),o("profile_id"),o("parent"),o("spawned_by"),o("creator"),o("fork_from"),o("fork_source_file"),o("model_override"),Object.prototype.hasOwnProperty.call(n,"passive_default")&&t.push(["passive_default",n.passive_default===!0?1:0]),Object.prototype.hasOwnProperty.call(n,"cycle_pending")&&t.push(["cycle_pending",n.cycle_pending===!0?1:0]),Object.prototype.hasOwnProperty.call(n,"persona_ack")&&t.push(["persona_ack_json",m(n.persona_ack,"persona_ack")]),Object.prototype.hasOwnProperty.call(n,"review_binding")&&t.push(["review_binding_json",m(n.review_binding,"review_binding")]),Object.prototype.hasOwnProperty.call(n,"isolated"))throw new TypeError("isolated cannot change after node creation");Object.prototype.hasOwnProperty.call(n,"scopes")&&t.push(["scopes_json",m(n.scopes,"scopes")]),Object.prototype.hasOwnProperty.call(n,"launch")&&t.push(["launch_config_json",m(n.launch,"launch")]);const i=r(s=>Object.prototype.hasOwnProperty.call(n,s),"has");return p(s=>{const l=M(s,e);if(l===null||l.type!=="node")throw new Error(`unknown node: ${e}`);if(i("name")&&De(s,l,T(n.name,"name")),i("kind")&&ge(s,l,T(n.kind,"kind")),t.length===0)return;if(s.prepare(`UPDATE nodes SET ${t.map(([u])=>`${u} = ?`).join(", ")} WHERE node_id = ?`).run(...t.map(([,u])=>u),e).changes!==1)throw new Error(`unknown node: ${e}`)}),y(e)}r(wn,"updateNode");function Tn(e,n){d(a(),"UPDATE nodes SET status = ? WHERE node_id = ?").run(n,e)}r(Tn,"setStatus");function hn(e,n){d(a(),"UPDATE nodes SET intent = ? WHERE node_id = ?").run(n??null,e)}r(hn,"setIntent");function Ln(e,n,t={}){if(t.outcome===void 0){const i=[],s=[];Object.prototype.hasOwnProperty.call(n,"status")&&(i.push("status = ?"),s.push(n.status??null)),Object.prototype.hasOwnProperty.call(n,"intent")&&(i.push("intent = ?"),s.push(n.intent??null)),Object.prototype.hasOwnProperty.call(n,"terminal_reason")&&(i.push("terminal_reason = ?"),s.push(n.terminal_reason??null));const l=["node_id = ?"];s.push(e),t.fromStatuses!==void 0&&(l.push(`status IN (${t.fromStatuses.map(()=>"?").join(", ")})`),s.push(...t.fromStatuses)),t.requireNoFinalReport===!0&&l.push("final_report IS NULL");const _=a().prepare(`UPDATE nodes SET ${i.join(", ")} WHERE ${l.join(" AND ")}`).run(...s);return Number(_.changes)}const o=t.outcome;return p(i=>{const s=[],l=[];Object.prototype.hasOwnProperty.call(n,"status")&&(s.push("status = ?"),l.push(n.status??null)),Object.prototype.hasOwnProperty.call(n,"intent")&&(s.push("intent = ?"),l.push(n.intent??null)),Object.prototype.hasOwnProperty.call(n,"terminal_reason")&&(s.push("terminal_reason = ?"),l.push(n.terminal_reason??null));const _=r((R,c)=>o.supersede===!0?"?":`CASE WHEN outcome_at IS NULL THEN ? ELSE ${c} END`,"outcomeValue");s.push(`outcome_kind = ${_(o.kind,"outcome_kind")}`,`outcome_reason = ${_(o.reason,"outcome_reason")}`,`outcome_detail_json = ${_(o.detailJson,"outcome_detail_json")}`,`outcome_revision = ${o.supersede===!0?"outcome_revision + 1":"CASE WHEN outcome_at IS NULL THEN outcome_revision + 1 ELSE outcome_revision END"}`,`outcome_at = ${_(o.at,"outcome_at")}`),l.push(o.kind,o.reason,o.detailJson,o.at);const u=["node_id = ?"];l.push(e),t.fromStatuses!==void 0&&(u.push(`status IN (${t.fromStatuses.map(()=>"?").join(", ")})`),l.push(...t.fromStatuses)),t.requireNoFinalReport===!0&&u.push("final_report IS NULL");const f=i.prepare(`UPDATE nodes SET ${s.join(", ")} WHERE ${u.join(" AND ")}`).run(...l);if(Number(f.changes)===0)return 0;const O=d(i,"SELECT outcome_at FROM nodes WHERE node_id = ?").get(e);return(o.supersede===!0||O.outcome_at===o.at)&&o.documentJson!==null&&o.documentJson!==void 0&&le(i,{event:"outcome",object:ue(i,e),actor:null,attrs:{revision:Number(d(i,"SELECT outcome_revision FROM nodes WHERE node_id = ?").get(e).outcome_revision)},messageBody:JSON.parse(o.documentJson)}),Number(f.changes)})}r(Ln,"transitionStatusIntent");function gn(e){return d(a(),"SELECT final_report FROM nodes WHERE node_id = ?").get(e)?.final_report??null}r(gn,"getFinalReport");function Dn(e,n){return d(a(),"UPDATE nodes SET final_report = NULL, finalized_at = NULL WHERE node_id = ? AND final_report = ?").run(e,n).changes>0}r(Dn,"clearFinalReportIfMatches");function vn(e,n){d(a(),'UPDATE nodes SET tmux_session = ?, "window" = ?, pane = ? WHERE node_id = ?').run(n.tmux_session??null,n.window??null,n.pane??null,e)}r(vn,"setPresence");function kn(e,n){d(a(),"UPDATE nodes SET launched_at = ?, frozen_at = NULL, respawn_not_before = NULL WHERE node_id = ?").run(n,e)}r(kn,"recordLaunch");function An(e,n){d(a(),"UPDATE nodes SET last_settled_at = ? WHERE node_id = ?").run(n,e)}r(An,"recordSettledTurn");function xn(e,n){d(a(),"UPDATE nodes SET frozen_at = ? WHERE node_id = ?").run(n,e)}r(xn,"setFrozen");function jn(e){d(a(),"UPDATE nodes SET frozen_at = NULL WHERE node_id = ?").run(e)}r(jn,"clearFrozen");function Wn(e,n){d(a(),"UPDATE nodes SET respawn_failures = ?, respawn_not_before = ? WHERE node_id = ?").run(n.failures,n.notBefore,e)}r(Wn,"recordRespawnAttempt");const N="node_id NOT IN (SELECT node_id FROM warm_pool)",z=["state","cleanup_state","path","branch","repo_root","base_ref","base_sha","created_at","closed_at","next_attempt_at","attempts","quarantined","sweep_reason","sweep_detail","sweep_last_at","sweep_observed_json","abandoned_json","land_intent_json","disposal_json"];function xe(e){return e.worktree_state===null?null:me(Object.fromEntries(z.map(n=>[n,e[`worktree_${n}`]])))}r(xe,"worktreeFromProjection");function bn(e){const n=z.map(o=>`w.${o} AS worktree_${o}`).join(", ");return a().prepare(`SELECT n.*, ${n} FROM node_rows n LEFT JOIN worktrees w ON w.node_id = n.node_id WHERE n.profile_id = ? ORDER BY n.created`).all(e).map(o=>q(o,xe(o)))}r(bn,"listNodesByProfile");function In(){const e=a();return b(e,`SELECT
|
|
9
11
|
node_id, name, description, kind, mode, lifecycle, status, cwd, profile_id,
|
|
10
12
|
grantee, parent, created, frozen_at, terminal_reason, final_report,
|
|
11
13
|
telemetry_tokens_in, telemetry_updated_at, pi_pid, turn_open_at, turn_closed_at,
|
|
@@ -13,13 +15,13 @@ var Z=Object.defineProperty;var r=(e,n)=>Z(e,"name",{value:n,configurable:!0});i
|
|
|
13
15
|
fault_operation_id, fault_anchor_entry_id, fault_retry_disposition,
|
|
14
16
|
fault_retry_by, fault_retry_attempt, fault_retry_max, fault_retry_next_at,
|
|
15
17
|
fault_provider_error_json
|
|
16
|
-
FROM node_rows WHERE ${N} ORDER BY created`).all().map(E)}r(
|
|
18
|
+
FROM node_rows WHERE ${N} ORDER BY created`).all().map(E)}r(In,"browseNodes");function Fn(e,n){d(a(),`UPDATE nodes
|
|
17
19
|
SET telemetry_tokens_in = ?,
|
|
18
20
|
telemetry_context_tokens = COALESCE(?, telemetry_context_tokens),
|
|
19
21
|
telemetry_last_activity = COALESCE(?, telemetry_last_activity),
|
|
20
22
|
telemetry_updated_at = ?
|
|
21
23
|
WHERE node_id = ?
|
|
22
|
-
AND (telemetry_updated_at IS NULL OR telemetry_updated_at < ?)`).run(n.tokensIn,n.contextTokens,n.lastActivity,n.updatedAt,e,n.updatedAt)}r(
|
|
24
|
+
AND (telemetry_updated_at IS NULL OR telemetry_updated_at < ?)`).run(n.tokensIn,n.contextTokens,n.lastActivity,n.updatedAt,e,n.updatedAt)}r(Fn,"updateNodeTelemetry");function Hn(){return d(a(),`SELECT
|
|
23
25
|
node_id, name, description, cycles, kind, status, frozen_at,
|
|
24
26
|
telemetry_context_tokens, telemetry_last_activity,
|
|
25
27
|
pi_pid, turn_open_at, turn_closed_at,
|
|
@@ -27,7 +29,7 @@ var Z=Object.defineProperty;var r=(e,n)=>Z(e,"name",{value:n,configurable:!0});i
|
|
|
27
29
|
fault_operation_id, fault_anchor_entry_id, fault_retry_disposition,
|
|
28
30
|
fault_retry_by, fault_retry_attempt, fault_retry_max, fault_retry_next_at,
|
|
29
31
|
fault_provider_error_json
|
|
30
|
-
FROM node_rows WHERE ${N} ORDER BY created`).all().map(n=>{const t=n.pi_pid??null,o=H(t),i=P(n);return{node_id:n.node_id,name:n.name,description:n.description??null,cycles:n.cycles??null,kind:n.kind,status:n.status,frozen_at:n.frozen_at??null,pi_pid:t,telemetry_context_tokens:n.telemetry_context_tokens??null,telemetry_last_activity:n.telemetry_last_activity??null,fault:i?.link==="pi\u2192provider"&&i.retry.by==="daemon"&&i.retry.disposition==="auto"&&!o?null:i,streaming:n.turn_open_at!==null&&n.turn_closed_at===null&&o}})}r(
|
|
32
|
+
FROM node_rows WHERE ${N} ORDER BY created`).all().map(n=>{const t=n.pi_pid??null,o=H(t),i=P(n);return{node_id:n.node_id,name:n.name,description:n.description??null,cycles:n.cycles??null,kind:n.kind,status:n.status,frozen_at:n.frozen_at??null,pi_pid:t,telemetry_context_tokens:n.telemetry_context_tokens??null,telemetry_last_activity:n.telemetry_last_activity??null,fault:i?.link==="pi\u2192provider"&&i.retry.by==="daemon"&&i.retry.disposition==="auto"&&!o?null:i,streaming:n.turn_open_at!==null&&n.turn_closed_at===null&&o}})}r(Hn,"graphNodes");function je(e){const n=[N],t=[];if(e?.status!==void 0){const i=Array.isArray(e.status)?e.status:[e.status];n.push(`status IN (${i.map(()=>"?").join(",")})`),t.push(...i)}return e?.withRecordedPid===!0&&n.push("pi_pid IS NOT NULL"),e?.withRecordedPidOrFrozen===!0&&n.push("(pi_pid IS NOT NULL OR frozen_at IS NOT NULL)"),a().prepare(`SELECT * FROM node_rows WHERE ${n.join(" AND ")} ORDER BY created`).all(...t).map(E)}r(je,"listNodes");const We=`SELECT
|
|
31
33
|
node_id, created, launched_at, last_settled_at, finalized_at,
|
|
32
34
|
outcome_at, last_activity_at, pane, "window", tmux_session
|
|
33
35
|
FROM nodes INDEXED BY idx_nodes_terminal_placement_created
|
|
@@ -48,16 +50,16 @@ var Z=Object.defineProperty;var r=(e,n)=>Z(e,"name",{value:n,configurable:!0});i
|
|
|
48
50
|
AND v.node_id NOT IN (SELECT node_id FROM warm_pool)
|
|
49
51
|
AND v.kind != 'human'
|
|
50
52
|
AND (v.status IN ('active', 'idle') OR (v.status IN ('done', 'canceled', 'dead') AND (v.pi_pid IS NOT NULL OR v.frozen_at IS NOT NULL)))
|
|
51
|
-
ORDER BY v.created`;function Un(){return d(a(),be).all().map(E)}r(Un,"listNodesForSupervision");function Mn(){return d(a(),`SELECT node_id, name, created, grantee FROM node_rows WHERE ${N} ORDER BY created`).all()}r(Mn,"listNodeIdentities");function Pn(e,n){p(t=>{d(t,"INSERT OR REPLACE INTO warm_pool (node_id, recipe_key, created) VALUES (?, ?, ?)").run(e,n,new Date().toISOString())})}r(Pn,"registerWarmSpare");function qn(e){return p(n=>d(n,"DELETE FROM warm_pool WHERE node_id = ?").run(e).changes>0)}r(qn,"unregisterWarmSpare");function
|
|
53
|
+
ORDER BY v.created`;function Un(){return d(a(),be).all().map(E)}r(Un,"listNodesForSupervision");function Mn(){return d(a(),`SELECT node_id, name, created, grantee FROM node_rows WHERE ${N} ORDER BY created`).all()}r(Mn,"listNodeIdentities");function Pn(e,n){p(t=>{d(t,"INSERT OR REPLACE INTO warm_pool (node_id, recipe_key, created) VALUES (?, ?, ?)").run(e,n,new Date().toISOString())})}r(Pn,"registerWarmSpare");function qn(e){return p(n=>d(n,"DELETE FROM warm_pool WHERE node_id = ?").run(e).changes>0)}r(qn,"unregisterWarmSpare");function h(e,n){const t=`SELECT w.node_id AS node_id, w.recipe_key AS recipe_key, w.created AS created,
|
|
52
54
|
n.status AS status, n.pi_pid AS pi_pid, n.pi_pid_identity AS pi_pid_identity
|
|
53
|
-
FROM warm_pool w JOIN node_rows n ON n.node_id = w.node_id`+(n===void 0?"":" WHERE w.recipe_key = ?")+" ORDER BY w.created";return(n===void 0?e.prepare(t).all():e.prepare(t).all(n)).map(i=>({node_id:i.node_id,recipe_key:i.recipe_key,created:i.created,status:i.status,pi_pid:i.pi_pid??null,pi_pid_identity:i.pi_pid_identity??null}))}r(
|
|
55
|
+
FROM warm_pool w JOIN node_rows n ON n.node_id = w.node_id`+(n===void 0?"":" WHERE w.recipe_key = ?")+" ORDER BY w.created";return(n===void 0?e.prepare(t).all():e.prepare(t).all(n)).map(i=>({node_id:i.node_id,recipe_key:i.recipe_key,created:i.created,status:i.status,pi_pid:i.pi_pid??null,pi_pid_identity:i.pi_pid_identity??null}))}r(h,"warmSpareRowsFor");function $n(){return h(a())}r($n,"listWarmSpares");function zn(e){return d(a(),"SELECT 1 FROM warm_pool WHERE node_id = ?").get(e)!==void 0}r(zn,"isWarmSpare");function B(e){return e.status!=="active"&&e.status!=="idle"?!1:Re(e.pi_pid,e.pi_pid_identity)==="alive"}r(B,"spareIsClaimable");function Bn(e,n,t={}){return p(o=>{const i=h(o,e).find(B);return i===void 0||t.requestKey!==void 0&&o.prepare("UPDATE create_requests SET node_id = ?, state = 'warm_claimed' WHERE request_key = ? AND node_id IS NULL").run(i.node_id,t.requestKey).changes===0||d(o,"DELETE FROM warm_pool WHERE node_id = ?").run(i.node_id).changes===0?null:(d(o,"UPDATE nodes SET created = ?, deadline_at = ? WHERE node_id = ?").run(n,t.deadlineAt??null,i.node_id),t.outcomeDelivery!==void 0&&F(o,i.node_id,t.outcomeDelivery),i.node_id)})}r(Bn,"claimWarmSpare");function Jn(e){return h(a(),e).filter(B).length}r(Jn,"warmSpareCount");function Yn(e){return d(a(),"SELECT node_id FROM nodes WHERE parent = ? ORDER BY created").all(e).map(n=>n.node_id)}r(Yn,"childrenOf");function Vn(){return a().prepare(`SELECT node_id, name, kind, mode, lifecycle, status, cwd, host_kind, parent, created
|
|
54
56
|
FROM node_rows WHERE ${N} ORDER BY created`).all().map(e=>({node_id:e.node_id,name:e.name,kind:e.kind,mode:e.mode,lifecycle:e.lifecycle,status:e.status,cwd:e.cwd,host_kind:e.host_kind??null,parent:e.parent??null,created:e.created}))}r(Vn,"rosterNodes");function Xn(e){return a().prepare(`SELECT * FROM node_rows
|
|
55
57
|
WHERE parent IS NULL AND kind != 'human'
|
|
56
58
|
AND status IN ('active','idle') AND cwd = ?
|
|
57
59
|
AND ${N}
|
|
58
|
-
ORDER BY created DESC`).all(e).map(E)}r(Xn,"listResumableRoots");export*from"./tree.js";function Gn(e,n,t){return p(o=>{const i=d(o,"SELECT status, final_report, frozen_at FROM nodes WHERE node_id = ?").get(e);return i===void 0?{kind:"missing"}:n(i)?{kind:"terminal",status:i.status,final_report:i.final_report}:{kind:"ran",result:t()}})}r(Gn,"withFreshTerminalGuard");function S(e){return pe(e)}r(S,"hasUnreconciledManagedWorktree");function J(e,n={}){return!["done","dead","canceled"].includes(e.status)||e.pi_pid!==null||e.frozen_at!==null||S(e.node_id)||e.node_id===
|
|
59
|
-
AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.node_id = n.node_id) LIMIT 1`);for(const i of e)if(!(t.get(i)!==void 0||o.get(i)!==void 0))try{se().delete(i)}catch(s){if(!(s instanceof de))throw s}}r(
|
|
60
|
+
ORDER BY created DESC`).all(e).map(E)}r(Xn,"listResumableRoots");export*from"./tree.js";function Gn(e,n,t){return p(o=>{const i=d(o,"SELECT status, final_report, frozen_at FROM nodes WHERE node_id = ?").get(e);return i===void 0?{kind:"missing"}:n(i)?{kind:"terminal",status:i.status,final_report:i.final_report}:{kind:"ran",result:t()}})}r(Gn,"withFreshTerminalGuard");function S(e){return pe(e)}r(S,"hasUnreconciledManagedWorktree");function J(e,n={}){return!["done","dead","canceled"].includes(e.status)||e.pi_pid!==null||e.frozen_at!==null||S(e.node_id)||e.node_id===x()||I(e)?!1:n.pastTtl===!0||!["crashed","stranded",null].includes(e.terminal_reason??null)}r(J,"isCollectible");function L(e){if(e.length===0)return[];const n=e.map(()=>"?").join(",");return a().prepare(`SELECT DISTINCT session_id FROM sessions WHERE node_id IN (${n}) AND session_id IS NOT NULL`).all(...e).map(({session_id:o})=>o)}r(L,"nodeConversationIds");function g(e){const n=a(),t=d(n,"SELECT 1 FROM sessions WHERE session_id = ? LIMIT 1"),o=d(n,`SELECT 1 FROM nodes n WHERE n.fork_source_file LIKE '%\\_' || ? || '.jsonl' ESCAPE '\\'
|
|
61
|
+
AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.node_id = n.node_id) LIMIT 1`);for(const i of e)if(!(t.get(i)!==void 0||o.get(i)!==void 0))try{se().delete(i)}catch(s){if(!(s instanceof de))throw s}}r(g,"deleteNodeConversations");function D(e,n){const t=M(e,n);return t===null||t.deleted_at!==null?!1:(ve(e,t),!0)}r(D,"deleteNodeObject");function v(e){return{nodeId:e,app:y(e)===null?null:we(e)}}r(v,"nodeFiles");function Y({nodeId:e,app:n}){n!==null&&W(U(n).nodes,he(e,n)),W(oe(C(e)),C(e))}r(Y,"removeNodeAppTrees");function V({nodeId:e}){return[Oe(e)]}r(V,"nodeOwnedDirs");function X({nodeId:e,app:n}){if(n!==null)try{ee(j(U(n).conversations,e))}catch(t){if(!["ENOENT","ENOTEMPTY"].includes(t.code??""))throw t}}r(X,"removeNodeConversationDir");function k(e){Y(e);for(const n of V(e))ne(n,{recursive:!0,force:!0});X(e)}r(k,"removeNodeFiles");function Qn(e){if(S(e))return{deleted:!1};const n=v(e),t=L([e]),o=p(i=>(D(i,e),{deleted:!0}));return g(t),k(n),o}r(Qn,"deleteNode");function Kn(e){const n=[...new Set(e)];if(n.length===0)return[];const t=n.filter(S);if(t.length>0)throw new Error(`refusing to delete nodes with unreconciled managed worktrees: ${t.join(", ")}`);const o=n.map(v),i=L(n),s=n.map(()=>"?").join(","),_=a().prepare(`SELECT node_id FROM nodes WHERE node_id IN (${s}) ORDER BY created`).all(...n).map(({node_id:u})=>u).filter(u=>p(f=>D(f,u)));g(i);for(const u of _){const f=o[n.indexOf(u)];A(f,()=>k(f))}return _}r(Kn,"deleteNodes");function Zn(e){const n=e.dryRun??!1,t=e.includeStale??!1,o=Date.now()-e.ttlDays*864e5,i=new Date(o).toISOString(),s=x()??"",l=a(),_=r(c=>K(c)<o,"quiet"),u=r(c=>({node_id:c.node_id,status:c.status,created:c.created}),"toPruned"),f=d(l,`SELECT * FROM node_rows
|
|
60
62
|
WHERE status IN ('dead', 'done', 'canceled') AND created < ?
|
|
61
|
-
ORDER BY created`).all(i).map(E).filter(
|
|
63
|
+
ORDER BY created`).all(i).map(E).filter(c=>J(c,{pastTtl:!0})&&_(c)&&!e.held(c.node_id)).map(u),O=t?d(l,`SELECT * FROM node_rows
|
|
62
64
|
WHERE status IN ('active', 'idle') AND created < ?
|
|
63
|
-
ORDER BY created`).all(i).map(E).filter(
|
|
65
|
+
ORDER BY created`).all(i).map(E).filter(c=>c.node_id!==s&&!Ne(c.pi_pid??null)&&(c.frozen_at??null)===null&&(c.respawn_not_before??null)===null&&c.intent!=="idle-release"&&!I(c)&&!ae(c.node_id)&&!S(c.node_id)&&_(c)&&!e.held(c.node_id)).map(u):[],R=G([...f,...O]);return n||R.length===0?{pruned:R,dryRun:n}:(Q(R,e.removeDirs),{pruned:R,dryRun:n})}r(Zn,"pruneNodes");function G(e){const n=a(),t=d(n,"SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ?"),o=new Set(e.map(i=>i.node_id));return e.filter(i=>t.all(i.node_id,i.node_id).every(s=>o.has(s.node_id)))}r(G,"retainRunRootsWithMembers");function Q(e,n){if(e.length===0)return;const t=e.map(i=>v(i.node_id)),o=L(e.map(i=>i.node_id));for(const i of e)p(s=>D(s,i.node_id));if(g(o),n!==void 0){for(const i of t)A(i,()=>{Y(i),X(i)});n(t.flatMap(V));return}for(const i of t)A(i,()=>k(i))}r(Q,"deletePrunedNodes");function A(e,n){try{n()}catch(t){ie({level:"error",event:"canvas.node_files.remove_failed",error:t,fields:{node_id:e.nodeId}})}}r(A,"eachNodeBestEffort");function K(e){let n=0;for(const t of[e.created,e.launched_at,e.last_settled_at,e.finalized_at,e.outcome_at,e.last_activity_at]){if(t==null)continue;const o=Date.parse(t);Number.isFinite(o)&&o>n&&(n=o)}try{const t=te(j(Se(e.node_id),"telemetry.json"),{throwIfNoEntry:!1});t!==void 0&&t.mtimeMs>n&&(n=t.mtimeMs)}catch{}return n}r(K,"lastActivityMs");function et(e){const n=e.dryRun??!1,t=je(),o=t.filter(u=>J(u)&&!e.held(u.node_id)),i=t.length-o.length,s=Math.max(0,e.maxNodes-i),l=o.map(u=>({row:u,recency:K(u)})).sort((u,f)=>f.recency-u.recency).slice(s).map(({row:u})=>({node_id:u.node_id,status:u.status,created:u.created})),_=G(l);return n||_.length===0?{pruned:_,dryRun:n}:(Q(_),{pruned:_,dryRun:n})}r(et,"pruneToLimit");export{pn as abandonCreateRequest,rn as assertRunNotDeleting,In as browseNodes,fn as canonicalSessionFile,Yn as childrenOf,Bn as claimWarmSpare,Dn as clearFinalReportIfMatches,jn as clearFrozen,cn as completeCreateRequest,sn as createNode,Qn as deleteNode,Kn as deleteNodes,Rn as dueNodeDeadlines,dn as findCreateRequest,En as findNodeBySessionFile,gn as getFinalReport,y as getNode,mn as getRow,Nn as getRowByPane,Hn as graphNodes,J as isCollectible,zn as isWarmSpare,K as lastActivityMs,Mn as listNodeIdentities,je as listNodes,bn as listNodesByProfile,Un as listNodesForSupervision,Xn as listResumableRoots,Cn as listTerminalPlacementResidue,$n as listWarmSpares,v as nodeFiles,be as nodesForSupervisionSql,ln as pruneExpiredCreateRequests,Zn as pruneNodes,et as pruneToLimit,un as recordCreateRequestNode,kn as recordLaunch,Wn as recordRespawnAttempt,An as recordSettledTurn,Pn as registerWarmSpare,k as removeNodeFiles,an as reserveCreateRequest,_n as reserveCreateRequestNode,On as rewriteLocalNodeIdentity,Vn as rosterNodes,xn as setFrozen,hn as setIntent,yn as setNodeDeadline,vn as setPresence,Tn as setStatus,Sn as shiftProfileNodeDeadlines,We as terminalPlacementResidueSql,Ln as transitionStatusIntent,qn as unregisterWarmSpare,wn as updateNode,Fn as updateNodeTelemetry,Jn as warmSpareCount,Gn as withFreshTerminalGuard};
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
|
-
var g=Object.defineProperty;var l=(s,e)=>g(s,"name",{value:e,configurable:!0});import{getNode as v,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as _}from"../core/runtime/lifecycle.js";import{publishNotice as E}from"../core/runtime/close.js";import{
|
|
1
|
+
var g=Object.defineProperty;var l=(s,e)=>g(s,"name",{value:e,configurable:!0});import{getNode as v,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as _}from"../core/runtime/lifecycle.js";import{publishNotice as E}from"../core/runtime/close.js";import{readFault as P,readInterruptedTurn as N,readTransportPoisonVerdict as x}from"../core/canvas/recovery.js";import{hasPendingWake as C}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as F,MAX_RESPAWN_ATTEMPTS as M,OFFLINE_RESPAWN_RETRY_MS as S,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as c}from"../core/events/emit.js";import{brokerThresholdsForDaemon as D}from"../core/runtime/fleet.js";import{isRecoveryHeld as L}from"../core/runtime/recovery-hold.js";function w(s,e){const r=P(s.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`,i=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${s.name}\` (${s.kind}) never started.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Its pi vehicle exited before the session came up (no pi_session_id was ever recorded), so the node produced no output. ${i}
|
|
6
|
+
|
|
7
|
+
`+t+`Inspect canonical event diagnostics with \`crtr sys logs --node ${s.node_id}\`.
|
|
6
8
|
|
|
7
9
|
If the work still needs doing, re-spawn it.`}l(w,"bootFailureBody");async function y(s,e){await m(s.node_id,w(s,e),{from:s.node_id})}l(y,"surfaceBootFailure");function b(s){return`\u26A0 Out of memory \u2014 \`${s.name}\` (${s.kind}) was killed because its app's processes together passed the app's memory limit.
|
|
8
10
|
|
|
9
|
-
Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${s.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function
|
|
11
|
+
Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${s.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function W(s){const e=x(s.node_id),r=e===null?"":`
|
|
10
12
|
|
|
11
13
|
Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${s.name}\` (${s.kind}) kept dying without settling a turn.
|
|
12
14
|
|
|
13
15
|
Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
|
|
14
16
|
|
|
15
|
-
Fix the crash cause, then revive the node if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(
|
|
17
|
+
Fix the crash cause, then revive the node if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(W,"surfaceCrashLoop");async function q(s,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${s.name}\` (${s.kind}) could not be relaunched.
|
|
16
18
|
|
|
17
19
|
The daemon refused this recovery before starting a broker: ${r}
|
|
18
20
|
|
|
19
|
-
The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(
|
|
21
|
+
The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(s.node_id,t,{from:s.node_id})}l(q,"surfaceRecoveryFailure");function G(s){return s?.cycle_pending!==!0}l(G,"retryResumeMode");function H(s,e,r=!0){return s===null||s.status==="done"||s.status==="canceled"||s.status==="dead"?"forget":s.intent==="refresh"?"respawn-fresh":R(s)?"dormant":s.lifecycle==="resident"&&!e&&!r&&s.pi_session_id!=null?"release-idle":s.pi_session_id!=null?G(s)?"respawn-resume":"respawn-cycle":e||s.fork_from!=null?"respawn-fresh":"boot-failure"}l(H,"classifyDeadNode");const Y=3e4;class le{static{l(this,"DaemonFleet")}#n;#o;#s;#l;#e=new Map;#r=new Set;#a=new Set;#t=new Map;#c=Number.NEGATIVE_INFINITY;constructor(e){this.#n=e.epoch,this.#o=e.enqueue,this.#s=e.now??Date.now,this.#l=e.probeOnline??(()=>A())}epoch(){return this.#n}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#r.delete(e),this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#r.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#r.size}reserveSlot(e){if(this.#r.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=D();if(this.occupancy()<r)return this.#r.add(e),!0;const t=this.#s();return t-this.#c>=Y&&(this.#c=t,c({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:r}})),!1}releaseReservation(e){this.#r.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r!==void 0){this.#a.add(e);try{process.kill(r,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#a.has(e)}forget(e){this.#e.delete(e),this.#r.delete(e),this.#t.delete(e)}onChildExit(e,r,t){t&&$(t),this.#e.delete(e),this.#o(()=>{const i=v(e),a=i?.launched_at==null?Number.NaN:Date.parse(i.launched_at),n=Number.isFinite(a)?Math.max(0,this.#s()-a):null,d=i?.respawn_failures??0;i?.status==="done"||i?.status==="canceled"||i?.status==="dead"?this.#t.delete(e):this.#t.set(e,r),this.#u(e,r,n,d)})}deliverExit(e,r){this.onChildExit(e,r)}async applyDeadRowPolicy(e){const r=this.#t.get(e);try{await this.#f(e,r)}finally{this.#t.get(e)===r&&this.#t.delete(e)}}async terminalizeIfStillNonTerminal(e,r,t,i,a){const n=v(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#i(e,n,{event:r,reason:i,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>q(n,t),"surface")}),!0)}dispose(){}#u(e,r,t,i){c({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:i}})}async#f(e,r){return this.#h(e,r)}async#h(e,r){const t=v(e);if(t!==null&&T(t.profile_id))return h(e),"forget";if(this.#a.delete(e))return h(e),"forget";if(L(e))return"dormant";const i=N(e)?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return h(e),await this.#i(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(t),{from:e})},"surface")}),"forget";const a=H(t,i,C(e));switch(a){case"forget":h(e);break;case"dormant":h(e);break;case"release-idle":h(e),_(e,"release"),c({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#i(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:w(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#d(e,t,a,i);break}return a}async#d(e,r,t,i){const a=k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#l()){F(e,n,this.#s()),c({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:S,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?O(a,this.#s()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=M};if(f.exhausted){const u=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#i(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:u,respawn_failures:f.failures},wake:u,...r.pi_session_id!=null?{surface:l(()=>W(r),"surface")}:{}});return}if(d&&f.notBefore!==null){c({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}c({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&i?"runtime-restart-abort":void 0,recovery:!0})}catch(u){c({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:u,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",u,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:u instanceof Error?u.message:String(u)})}}async#i(e,r,t){c({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),_(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>y(r)))()}catch(i){c({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:i})}E(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{le as DaemonFleet,H as classifyDeadNode,G as retryResumeMode,y as surfaceBootFailure};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{
|
|
2
|
-
`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function
|
|
3
|
-
`).map(async h=>{if(h!=="")try{await
|
|
4
|
-
`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=
|
|
1
|
+
var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ee,listUnnotifiedFatalFaultNodes as te,markFaultNotified as ne,readFault as O,readLiveTurn as E,recordFault as re}from"../../core/canvas/recovery.js";import{errorClassFromFault as ie}from"../../core/fault-classifier.js";import{nextFaultRetry as ae}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as G,withCanvasWrite as se}from"../../core/canvas/db.js";import{getById as oe}from"../../core/graph/objects.js";import{watcherIdsOf as le}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as de,resetStaleExecutionMail as ce}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ue}from"../../core/bash-jobs.js";import{focusOf as fe}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as pe}from"../api/viewer-presence.js";import{deliverLive as me}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as he}from"../../core/runtime/broker-request.js";import{formatCard as _e,PARK_SUMMARY_PROMPT as ge}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ke}from"../../core/runtime/fleet.js";import{readConfig as ve}from"../../core/config.js";import{beginPendingPark as ye,consumePendingPark as N,pendingParkIds as be,pendingParkStartedAt as we}from"../park-pending.js";import{shouldParkAfterSummary as Se}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Ne}from"./live-obligation.js";import{readGoal as Pe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as $e}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Te,envTestUnattendedParkMs as Ae,parkSummaryGraceMs as Me,envUnattendedParkMs as Ee}from"../../shared/env.js";const A=j(z);function Fe(s){return!ee(s)&&(Pe(s)?.trim()??"")!==""}p(Fe,"awaitsBirthTurn");function xe(s){return E(s)?.busy===!0}p(xe,"liveBusy");function Nt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(Nt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function x(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(x,"busySinceForIncarnation");const W=3*6e4,B=2e4,I=20*6e4,Ie=2,Ce=5*6e4;function De(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(De,"settlesOnFatalProviderFault");function Oe(){const s=Ee();if(s!==void 0)return s;const e=ve("user").lifecycle.unattendedParkMs;return Te()?Ae()??e:e}p(Oe,"unattendedParkMsForDaemon");function U(){return Me()}p(U,"parkSummaryGraceMsForDaemon");function Re(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Re,"yieldStallVerdict");function Ge(s){return!s.busy||s.quietForMs===null||s.quietForMs<I?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ie?"pending":"wedged"}p(Ge,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
|
|
2
|
+
`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Le(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Le,"sumTreeCpu");function We(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(We,"treeHasDescendants");class Pt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ke()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Oe();for(const i of te())await F.fresh(async()=>{try{this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),this.handleUnattendedParking(n,T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of be())t.fleet.has(i)||N(i);ce(e)}parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ue(X(e)).length>0||fe(e)!==null||pe(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ae(n,i)!==null)?!0:de(e)}handleUnattendedParking(e,t,r,i,n,l){if(J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=this.parkResetCondition(e,t,r,i),a=we(e);if(a!==void 0){if(i-a<U())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!Se(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:U()}}),P(e,"park"),S.teardown(e);return}if(o||xe(e)||Fe(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=Ne(e,t.lifecycle),h=d!=="cancel"&&$e(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){P(e,d),S.teardown(e);return}if(d==="cancel"){P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ye(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await me(e,_e("park",{},ge),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof he){P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ce)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
|
|
3
|
+
`).map(async h=>{if(h!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||x(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Re(!0,e.intent,!1,i-o.since)!=="kill")return;const a=T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=B&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:B,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Le(t,e),i=We(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity),o=x(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=I?await this.sampleTree(t.pid):null,d=T(n),h=d===null?null:x(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(Ge({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const v=C(d),g=Math.round((a??I)/6e4),y=$??!0,k=!y&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,b=!1,w;if(y)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(k!==null)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),b=!0,w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(M){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:M,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:y,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}}});try{re(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:w})}catch(M){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:M})}R(n,"notice.crash",w,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}})}handleFatalFault(e,t=!0){const r=O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
|
|
4
|
+
`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=De(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,v=d?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),y=le(G(),oe(G(),e)),k=[{relation:"Child",recipients:y.filter(f=>g.has(f))},{relation:"Watched node",recipients:y.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),se(()=>{for(const{relation:f,recipients:b}of k)b.length!==0&&R(e,"notice.crash",`${f} ${v} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,b);ne(r.faultId),t&&_&&Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:ie(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{Pt as BrokerSupervisionReconciler,Ie as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,De as settlesOnFatalProviderFault,Le as sumTreeCpu,We as treeHasDescendants,Oe as unattendedParkMsForDaemon,Ge as wedgeVerdict,Re as yieldStallVerdict};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.529",
|
|
4
4
|
"description": "crtr — agent runtime with memory, plugins, and marketplaces",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -139,7 +139,7 @@
|
|
|
139
139
|
},
|
|
140
140
|
"license": "UNLICENSED",
|
|
141
141
|
"dependencies": {
|
|
142
|
-
"@north-light/identity": "0.3.
|
|
142
|
+
"@north-light/identity": "0.3.529",
|
|
143
143
|
"@earendil-works/pi-agent-core": "1.1.0",
|
|
144
144
|
"@earendil-works/pi-ai": "1.1.0",
|
|
145
145
|
"@earendil-works/pi-coding-agent": "1.1.0",
|
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.529",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/cli",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.529",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "UNLICENSED",
|
|
12
12
|
"workspaces": [
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"@milkdown/preset-gfm": "^7.22.3",
|
|
25
25
|
"@milkdown/prose": "^7.22.3",
|
|
26
26
|
"@milkdown/utils": "^7.22.3",
|
|
27
|
-
"@north-light/identity": "0.3.
|
|
27
|
+
"@north-light/identity": "0.3.529",
|
|
28
28
|
"cron-parser": "^5.6.0",
|
|
29
29
|
"esbuild": "^0.27.7",
|
|
30
30
|
"jose": "^6.2.12",
|
|
@@ -12468,17 +12468,17 @@
|
|
|
12468
12468
|
},
|
|
12469
12469
|
"packages/crouter-api": {
|
|
12470
12470
|
"name": "@north-light/api",
|
|
12471
|
-
"version": "0.3.
|
|
12471
|
+
"version": "0.3.529",
|
|
12472
12472
|
"license": "UNLICENSED"
|
|
12473
12473
|
},
|
|
12474
12474
|
"packages/crouter-env-docker": {
|
|
12475
12475
|
"name": "@north-light/env-docker",
|
|
12476
|
-
"version": "0.3.
|
|
12476
|
+
"version": "0.3.529",
|
|
12477
12477
|
"license": "UNLICENSED"
|
|
12478
12478
|
},
|
|
12479
12479
|
"packages/crouter-identity": {
|
|
12480
12480
|
"name": "@north-light/identity",
|
|
12481
|
-
"version": "0.3.
|
|
12481
|
+
"version": "0.3.529",
|
|
12482
12482
|
"license": "UNLICENSED",
|
|
12483
12483
|
"dependencies": {
|
|
12484
12484
|
"jose": "^6.2.1"
|
|
@@ -12486,20 +12486,20 @@
|
|
|
12486
12486
|
},
|
|
12487
12487
|
"packages/crouter-plugin": {
|
|
12488
12488
|
"name": "@north-light/plugin",
|
|
12489
|
-
"version": "0.3.
|
|
12489
|
+
"version": "0.3.529",
|
|
12490
12490
|
"license": "UNLICENSED",
|
|
12491
12491
|
"dependencies": {
|
|
12492
|
-
"@north-light/api": "0.3.
|
|
12493
|
-
"@north-light/identity": "0.3.
|
|
12492
|
+
"@north-light/api": "0.3.529",
|
|
12493
|
+
"@north-light/identity": "0.3.529"
|
|
12494
12494
|
}
|
|
12495
12495
|
},
|
|
12496
12496
|
"packages/crouter-sdk": {
|
|
12497
12497
|
"name": "@north-light/sdk",
|
|
12498
|
-
"version": "0.3.
|
|
12498
|
+
"version": "0.3.529",
|
|
12499
12499
|
"license": "UNLICENSED",
|
|
12500
12500
|
"dependencies": {
|
|
12501
|
-
"@north-light/api": "0.3.
|
|
12502
|
-
"@north-light/identity": "0.3.
|
|
12501
|
+
"@north-light/api": "0.3.529",
|
|
12502
|
+
"@north-light/identity": "0.3.529",
|
|
12503
12503
|
"jose": "^6.2.1"
|
|
12504
12504
|
},
|
|
12505
12505
|
"bin": {
|