@north-light/cli 0.3.525 → 0.3.527

package/dist/core/canvas/recovery.js

@@ -1,4 +1,4 @@
-var re=Object.defineProperty;var o=(e,n)=>re(e,"name",{value:n,configurable:!0});import{prepared as s}from"../sqlite-statements.js";import{randomUUID as C}from"node:crypto";import{existsSync as p,readdirSync as ie,readFileSync as oe,realpathSync as se,statSync as de}from"node:fs";import{join as N,resolve as ae}from"node:path";import{openDb as E,prepared as ue,withCanvasWrite as l}from"./db.js";import{jobDir as ce,nodesRoot as b}from"./paths.js";import{errorClassFromFault as w}from"../fault-classifier.js";import{parseProviderRateLimit as _e}from"../provider-failure.js";import{emitEvent as X}from"../events/emit.js";import{operationIdContext as $,validateOperationId as j}from"../events/operation-id.js";const We=5*6e4,le=500,B=["pi\u2192provider","viewer\u2194broker","relay\u2194broker","viewer\u2194crtrd","daemon\u2192node","crtr\u2192pi"],pe=[...B,"broker\u2194crtrd"],Ee=["rate-limit","overloaded","connection","auth","protocol","context-overflow","other","wedged","model-not-found"],fe=["auto","manual","fatal"],ye=["sdk","daemon","client"],Te=["pending","admitted","invalidated"];function c(){return new Date().toISOString()}o(c,"now");function J(e,n){return new Set(e.prepare(`PRAGMA table_info(${n})`).all().map(({name:t})=>t))}o(J,"tableColumns");function Ne(e){const n=J(e,"executions"),t=[["turn_open_at","TEXT"],["turn_progress_at","TEXT"],["turn_closed_at","TEXT"],["turn_end_reason","TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],["exit_diagnostic_json","TEXT"]];for(const[r,i]of t)n.has(r)||e.exec(`ALTER TABLE executions ADD COLUMN ${r} ${i}`)}o(Ne,"addExecutionColumns");function ge(e){e.exec(`
+var ie=Object.defineProperty;var o=(e,n)=>ie(e,"name",{value:n,configurable:!0});import{prepared as s}from"../sqlite-statements.js";import{randomUUID as F}from"node:crypto";import{existsSync as f,readdirSync as oe,readFileSync as se,realpathSync as de,statSync as ae}from"node:fs";import{join as N,resolve as ue}from"node:path";import{openDb as E,prepared as ce,withCanvasWrite as l}from"./db.js";import{jobDir as _e,nodesRoot as C}from"./paths.js";import{errorClassFromFault as $}from"../fault-classifier.js";import{parseProviderRateLimit as le}from"../provider-failure.js";import{emitEvent as j}from"../events/emit.js";import{operationIdContext as B,validateOperationId as J}from"../events/operation-id.js";const Me=5*6e4,pe=500,K=["pi\u2192provider","viewer\u2194broker","relay\u2194broker","viewer\u2194crtrd","daemon\u2192node","crtr\u2192pi"],Ee=[...K,"broker\u2194crtrd"],fe=["rate-limit","overloaded","connection","auth","protocol","context-overflow","other","wedged","model-not-found"],ye=["auto","manual","fatal"],Te=["sdk","daemon","client"],Ne=["pending","admitted","invalidated"];function c(){return new Date().toISOString()}o(c,"now");function b(e,n){return new Set(e.prepare(`PRAGMA table_info(${n})`).all().map(({name:t})=>t))}o(b,"tableColumns");function ge(e){const n=b(e,"executions"),t=[["turn_open_at","TEXT"],["turn_progress_at","TEXT"],["turn_closed_at","TEXT"],["turn_end_reason","TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],["exit_diagnostic_json","TEXT"]];for(const[r,i]of t)n.has(r)||e.exec(`ALTER TABLE executions ADD COLUMN ${r} ${i}`)}o(ge,"addExecutionColumns");function Le(e){e.exec(`
 CREATE TABLE IF NOT EXISTS node_faults (
   fault_id TEXT PRIMARY KEY,
   node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
@@ -23,4 +23,4 @@ CREATE TABLE IF NOT EXISTS node_faults (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_active ON node_faults(node_id) WHERE active = 1;
 CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_episode ON node_faults(node_id) WHERE episode_state IS NOT NULL;
-`)}o(ge,"createFaultTable");function g(e,n){return typeof n=="string"&&e.includes(n)}o(g,"contains");function F(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}o(F,"isRecord");function K(e){if(!F(e)||typeof e.provider!="string"||e.provider===""||typeof e.model!="string"||e.model==="")return;const n=typeof e.status=="number"&&Number.isFinite(e.status)?e.status:void 0,t=typeof e.code=="string"||typeof e.code=="number"?e.code:void 0,r=typeof e.errno=="string"?e.errno:void 0,i=typeof e.body=="string"?e.body:void 0,d=_e(e.ratelimit),a=Array.isArray(e.causes)?e.causes.flatMap(u=>{if(!F(u))return[];const f={...typeof u.message=="string"?{message:u.message}:{},...typeof u.status=="number"&&Number.isFinite(u.status)?{status:u.status}:{},...typeof u.code=="string"||typeof u.code=="number"?{code:u.code}:{},...typeof u.errno=="string"?{errno:u.errno}:{},...typeof u.body=="string"?{body:u.body}:{}};return Object.keys(f).length===0?[]:[f]}):void 0;return{provider:e.provider,model:e.model,...n===void 0?{}:{status:n},...t===void 0?{}:{code:t},...r===void 0?{}:{errno:r},...i===void 0?{}:{body:i},...a===void 0||a.length===0?{}:{causes:a},...d===void 0?{}:{ratelimit:d}}}o(K,"parseProviderError");function Y(e){try{const n=JSON.parse(e);if(!g(pe,n.link)||typeof n.op!="string"||n.op===""||!g(Ee,n.kind)||typeof n.message!="string"||typeof n.since!="string")return null;const t=j(n.operation_id);if(!F(n.retry)||!g(fe,n.retry.disposition)||n.retry.by!==void 0&&!g(ye,n.retry.by)||n.retry.attempt!==void 0&&typeof n.retry.attempt!="number"||n.retry.max!==void 0&&typeof n.retry.max!="number"||n.retry.nextAt!==void 0&&typeof n.retry.nextAt!="string"||n.anchorEntryId!==void 0&&(typeof n.anchorEntryId!="string"||n.anchorEntryId===""))return null;const r=n.providerError===void 0?void 0:K(n.providerError);return n.providerError!==void 0&&r===void 0?null:{link:n.link,op:n.op,kind:n.kind,retry:{disposition:n.retry.disposition,...n.retry.by===void 0?{}:{by:n.retry.by},...n.retry.attempt===void 0?{}:{attempt:n.retry.attempt},...n.retry.max===void 0?{}:{max:n.retry.max},...n.retry.nextAt===void 0?{}:{nextAt:n.retry.nextAt}},message:n.message,since:n.since,operation_id:t,...n.anchorEntryId===void 0?{}:{anchorEntryId:n.anchorEntryId},...r===void 0?{}:{providerError:r}}}catch{return null}}o(Y,"parseLegacyFault");function Le(e){const n=ae(e);try{return se(n)}catch{return n}}o(Le,"canonicalPath");function me(e){try{const n=JSON.parse(e);if(!g(Te,n.state)||typeof n.sessionFile!="string"||n.sessionFile==="")return null;const t=JSON.stringify(n.fault),r=Y(t);if(r===null||r.link!=="pi\u2192provider"||r.retry.disposition!=="auto"||r.retry.by!=="daemon")return null;const i=n.anchorEntryId;return i!==null&&typeof i!="string"||(r.anchorEntryId??null)!==i?null:{state:n.state,sessionFile:Le(n.sessionFile),fault:r}}catch{return null}}o(me,"parseLegacyEpisode");function G(e){return e!==void 0?j(e):$.current()??$.fresh(t=>t)}o(G,"resolveOperationId");function Se(e){return e.slice(0,le)}o(Se,"clampMessage");function R(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="auto"&&e.retry.by==="daemon"?3:e.retry.disposition==="fatal"?2:1}o(R,"faultPriority");function V(e){return{link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operationId:e.operation_id,anchorEntryId:e.anchorEntryId??null,retryDisposition:e.retry.disposition,retryBy:e.retry.by??null,retryAttempt:e.retry.attempt??null,retryMax:e.retry.max??null,retryNextAt:e.retry.nextAt??null,providerError:e.providerError===void 0?null:JSON.stringify(e.providerError)}}o(V,"encodeFault");function H(e){const n=e.provider_error_json,t=typeof n=="string"?K(JSON.parse(n)):void 0;if(typeof n=="string"&&t===void 0)throw new Error(`invalid provider error on stored fault ${e.fault_id}`);const r={disposition:e.retry_disposition,...e.retry_by===null?{}:{by:e.retry_by},...e.retry_attempt===null?{}:{attempt:e.retry_attempt},...e.retry_max===null?{}:{max:e.retry_max},...e.retry_next_at===null?{}:{nextAt:e.retry_next_at}};return{faultId:e.fault_id,link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operation_id:e.operation_id,retry:r,...e.anchor_entry_id===null?{}:{anchorEntryId:e.anchor_entry_id},...t===void 0?{}:{providerError:t}}}o(H,"hydrateFault");function L(e,n){const t=ue(e,"SELECT * FROM node_faults WHERE node_id = ? AND active = 1").get(n);return t===void 0?null:H(t)}o(L,"activeFault");function Q(e){if(!g(B,e.link))throw new Error(`fault link is not durable: ${e.link}`);return{link:e.link,op:e.op,kind:e.kind,retry:e.retry,message:Se(e.message),since:e.since??c(),operation_id:G(e.operation_id),...typeof e.anchorEntryId=="string"&&e.anchorEntryId!==""?{anchorEntryId:e.anchorEntryId}:{},...e.providerError===void 0?{}:{providerError:e.providerError}}}o(Q,"storedFault");function q(e,n){X({level:n.retry.disposition==="fatal"?"error":"warn",event:"fault.recorded",operation_id:n.operation_id,node_id:e,error_class:w(n),fields:{link:n.link,op:n.op,kind:n.kind,retry:n.retry,message:n.message,...n.anchorEntryId===void 0?{}:{anchor_entry_id:n.anchorEntryId},...n.providerError===void 0?{}:{provider_error:n.providerError}}})}o(q,"emitRecordedFault");function W(e,n,t,r,i){const d=C(),a=V(t),u=c();return s(e,"INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(d,n,a.link,a.op,a.kind,a.message,a.since,a.operationId,a.anchorEntryId,a.retryDisposition,a.retryBy,a.retryAttempt,a.retryMax,a.retryNextAt,a.providerError,r?1:0,i?.state??null,i?.bindingId??null,u,u),{...t,faultId:d}}o(W,"insertFault");function z(e,n,t,r){const i=V(t);return s(e,"UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?").run(i.link,i.op,i.kind,i.message,i.since,i.operationId,i.anchorEntryId,i.retryDisposition,i.retryBy,i.retryAttempt,i.retryMax,i.retryNextAt,i.providerError,r?.state??null,r?.bindingId??null,c(),n),{...t,faultId:n}}o(z,"updateFault");function D(e,n){s(e,"DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL").run(n)}o(D,"deleteIfUnretained");function Z(e,n,t,r){const i=L(e,n);if(i===null){const d=s(e,"SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL").get(n);return d!==void 0&&d.link===t.link&&d.kind===t.kind&&d.since===t.since&&d.operation_id===t.operation_id?(s(e,"UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?").run(c(),d.fault_id),z(e,d.fault_id,t)):W(e,n,t,!0,r)}if(i.link===t.link&&i.kind===t.kind){const d=t.retry.disposition==="auto"?{...i.retry,...t.retry}:t.retry,u=i.retry.disposition===t.retry.disposition&&(t.retry.disposition!=="fatal"||i.message===t.message)?i.since:t.since;return z(e,i.faultId,{...t,since:u,anchorEntryId:i.anchorEntryId??t.anchorEntryId,retry:d},r)}return i.link!==t.link&&R(i)>R(t)?null:(s(e,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(e,i.faultId),W(e,n,t,!0,r))}o(Z,"replaceActiveFault");function m(e,n,t){return s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL").get(n,t)?.binding_id??null}o(m,"liveBinding");function Pe(e,n){return l(t=>{const r=c();return s(t,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,e).changes>0?!0:s(t,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(r,r,e).changes>0})}o(Pe,"openTurn");function Me(e){return l(n=>s(n,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),e).changes>0)}o(Me,"recordTurnProgress");function we(e,n){return l(t=>{const r=c(),i=s(t,"UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,n.endReason,e).changes>0;if(i&&n.endReason==="settled"){const d="(SELECT node_id FROM executions WHERE execution_id = ?)";n.viewerPresent===!0?s(t,`UPDATE nodes SET reply_at = ?, read_at = ? WHERE node_id = ${d}`).run(r,r,e):s(t,`UPDATE nodes SET reply_at = ? WHERE node_id = ${d}`).run(r,e)}return i})}o(we,"closeTurn");function Xe(e){return l(n=>s(n,"UPDATE nodes SET read_at = ? WHERE node_id = ?").run(c(),e).changes>0)}o(Xe,"markNodeRead");function $e(e,n){if(!Number.isInteger(n.consecutivePasses)||n.consecutivePasses<1||n.routes.length===0||!n.routes.every(t=>t!==""))throw new Error("invalid transport-poison verdict");return l(t=>s(t,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:n.consecutivePasses,routes:n.routes}}),e).changes>0)}o($e,"recordTransportPoisonVerdict");function je(e){const n=s(E(),"SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);if(n===void 0||n.exit_diagnostic_json===null)return null;try{const t=JSON.parse(n.exit_diagnostic_json),r=t.transport_poison;return t.schema!=="crtr.execution-exit-diagnostic/v1"||r===void 0||!Number.isInteger(r.consecutive_passes)||r.consecutive_passes<1||!Array.isArray(r.routes)||r.routes.length===0||!r.routes.every(i=>typeof i=="string"&&i!=="")?null:{consecutivePasses:r.consecutive_passes,routes:r.routes}}catch{return null}}o(je,"readTransportPoisonVerdict");function Be(e){const n=s(E(),"SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(e);return n===void 0?null:{busy:n.turn_open_at!==null&&n.turn_closed_at===null,progressAt:n.turn_progress_at}}o(Be,"readLiveTurn");function Je(e){const n=E(),t=s(n,"SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(e);if(t===void 0)return null;const r=s(n,"SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);return{interrupted:t.turn_closed_at===null||r?.turn_end_reason==="teardown-aborted",progressAt:t.turn_progress_at}}o(Je,"readInterruptedTurn");function Ke(e){return s(E(),"SELECT 1 FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL LIMIT 1").get(e)!==void 0}o(Ke,"hasAdmittedTurn");function Ye(e,n){const t=Q(n);return q(e,t),l(r=>Z(r,e,t))}o(Ye,"recordFault");function Ge(e,n){const t=L(E(),e);return t===null||n?.link!==void 0&&t.link!==n.link?!1:l(r=>{const i=L(r,e);if(i===null||n?.link!==void 0&&i.link!==n.link)return!1;const d=s(r,"SELECT episode_state FROM node_faults WHERE fault_id = ?").get(i.faultId);s(r,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),d.episode_state==="pending"&&n?.preserveEpisode!==!0&&s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(r,i.faultId);const a=G();return X({level:"info",event:"fault.cleared",operation_id:a,node_id:e,error_class:w(i),fields:{cleared_fault_operation_id:i.operation_id,link:i.link,op:i.op,kind:i.kind,retry:i.retry,message:i.message,since:i.since,...i.anchorEntryId===void 0?{}:{anchor_entry_id:i.anchorEntryId}}}),!0})}o(Ge,"clearFault");function ee(e){return L(E(),e)}o(ee,"readFault");function Ve(e,n){const t=Q(n.input);if(t.link!=="pi\u2192provider"||t.retry.disposition!=="auto"||t.retry.by!=="daemon")throw new Error("provider retry episode requires an automatic daemon provider fault");return q(e,t),l(r=>{const i=m(r,e,n.sessionId);if(i===null)return null;const d=L(r,e);if(d!==null&&(d.link!==t.link||d.kind!==t.kind)&&R(d)>R(t))return null;const a=s(r,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL").all(e);s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL").run(c(),e);for(const f of a)D(r,f.fault_id);const u=Z(r,e,t,{state:"pending",bindingId:i});return u===null?null:(s(r,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run("pending",i,c(),u.faultId),u)})}o(Ve,"recordPendingEpisode");function Qe(e,n){return l(t=>{if(s(t,"SELECT 1 FROM nodes WHERE node_id = ? AND intent IS 'refresh'").get(e)!==void 0)return!1;const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(n.retry.disposition,n.retry.by??null,n.retry.attempt??null,n.retry.max??null,n.retry.nextAt??null,c(),e,n.faultId,r).changes>0})}o(Qe,"admitEpisode");function qe(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(qe,"reopenEpisode");function ze(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(ze,"invalidateEpisode");function Ze(e,n){return l(t=>{const r=m(t,e,n.sessionId);if(r===null)return!1;const i=s(t,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").all(e,r);if(i.length===0)return!1;s(t,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").run(c(),e,r);for(const d of i)D(t,d.fault_id);return!0})}o(Ze,"clearEpisode");function en(e){const n=s(E(),"SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL").get(e);return n===void 0?null:{state:n.episode_state,sessionId:n.session_id,fault:H(n)}}o(en,"readEpisode");function nn(){return s(E(),"SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all().map(n=>({nodeId:n.node_id,sessionId:n.bound_session_id,fault:H(n),updatedAt:n.updated_at}))}o(nn,"listPendingProviderRetries");function tn(e){const n=ee(e)===null;return{explanation(){if(!n)return null;const t=ee(e);if(t===null||t.link!=="crtr\u2192pi"||t.retry.disposition!=="fatal")return null;const r=t.op.trim(),i=t.message.trim();return r===""||i===""?null:`${r}: ${i}`}}}o(tn,"beginBootFaultAttempt");function ne(e,n){try{return de(e).mtime.toISOString()}catch{return n}}o(ne,"markerMtime");function P(e,n){if(!p(e))return null;try{return oe(e,"utf8")}catch(t){if(!n)return null;throw new Error(`cannot import recovery input ${e}: ${t.message}`)}}o(P,"legacyFile");function Ae(e,n){return s(e,"SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(n)??null}o(Ae,"latestExecution");function xe(e,n){const r=J(e,"nodes").has("launched_at")?s(e,"SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?").get(n):s(e,"SELECT created AS started_at FROM nodes WHERE node_id = ?").get(n);if(r===void 0)throw new Error(`cannot import recovery witness for unknown node ${n}`);const i=C();return s(e,"INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(i,n,r.started_at,r.started_at),{execution_id:i,started_at:r.started_at}}o(xe,"syntheticExecution");function te(e,n,t,r,i){return W(e,n,t,r,i)}o(te,"insertMigratedFault");function Ie(e,n,t,r){const i=s(e,"SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL").get(t);if(i!==void 0){if(i.node_id!==n)throw new Error(`cannot import provider retry ${r}: transcript is live-owned by ${i.node_id}`);return i.binding_id}const d=s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1").get(n,t);if(d!==void 0)return d.binding_id;const a=s(e,"SELECT created FROM nodes WHERE node_id = ?").get(n);if(a===void 0)throw new Error(`cannot import provider retry ${r}: unknown node ${n}`);const u=C();return s(e,"INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)").run(u,n,t,a.created,a.created,c()),u}o(Ie,"migrationBinding");function rn(e){if(Ne(e),ge(e),!p(b()))return;const n=ie(b(),{withFileTypes:!0}).filter(r=>r.isDirectory()).sort((r,i)=>r.name.localeCompare(i.name)),t=s(e,"SELECT 1 FROM nodes WHERE node_id = ?");for(const r of n){const i=r.name,d=ce(i),a=N(d,"busy"),u=N(d,"clean-abort"),f=N(d,"transport-poison.json"),h=N(d,"fault"),S=N(d,"provider-retry"),O=p(a),U=p(u);if(!O&&!U&&!p(f)&&!p(h)&&!p(S)||!p(N(b(),i,"meta.json"))&&t.get(i)===void 0)continue;const k=P(f,!1),A=P(h,!0),x=P(S,!0);if(!O&&!U&&k===null&&A===null&&x===null)continue;let y=Ae(e,i);if(y===null&&(y=xe(e,i)),O){const _=ne(a,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?").run(_,_,y.execution_id)}else if(U){const _=ne(u,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(_,_,_,y.execution_id)}if(k!==null)try{const _=JSON.parse(k);Number.isInteger(_.consecutivePasses)&&_.consecutivePasses>=1&&Array.isArray(_.routes)&&_.routes.length>0&&_.routes.every(M=>typeof M=="string"&&M!=="")&&s(e,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:_.consecutivePasses,routes:_.routes}}),y.execution_id)}catch{}const I=A===null?null:Y(A);if(A!==null&&I===null)throw new Error(`cannot import recovery fault ${h}`);const T=x===null?null:me(x);if(x!==null&&T===null)throw new Error(`cannot import provider retry ${S}`);let v=I===null||I.link==="broker\u2194crtrd"?null:te(e,i,I,!0);if(T!==null){const _=Ie(e,i,T.sessionFile,S);v!==null&&v.since===T.fault.since?s(e,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run(T.state,_,c(),v.faultId):te(e,i,T.fault,!1,{state:T.state,bindingId:_})}}}o(rn,"migrateRecoveryWitnesses");export{We as FAULT_QUIET_MS,Qe as admitEpisode,tn as beginBootFaultAttempt,Ze as clearEpisode,Ge as clearFault,we as closeTurn,Ke as hasAdmittedTurn,H as hydrateFault,ze as invalidateEpisode,nn as listPendingProviderRetries,Xe as markNodeRead,rn as migrateRecoveryWitnesses,Pe as openTurn,K as parseProviderError,en as readEpisode,ee as readFault,Je as readInterruptedTurn,Be as readLiveTurn,je as readTransportPoisonVerdict,Ye as recordFault,Ve as recordPendingEpisode,$e as recordTransportPoisonVerdict,Me as recordTurnProgress,qe as reopenEpisode};
+`)}o(Le,"createFaultTable");function g(e,n){return typeof n=="string"&&e.includes(n)}o(g,"contains");function H(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}o(H,"isRecord");function Y(e){if(!H(e)||typeof e.provider!="string"||e.provider===""||typeof e.model!="string"||e.model==="")return;const n=typeof e.status=="number"&&Number.isFinite(e.status)?e.status:void 0,t=typeof e.code=="string"||typeof e.code=="number"?e.code:void 0,r=typeof e.errno=="string"?e.errno:void 0,i=typeof e.body=="string"?e.body:void 0,d=le(e.ratelimit),a=Array.isArray(e.causes)?e.causes.flatMap(u=>{if(!H(u))return[];const p={...typeof u.message=="string"?{message:u.message}:{},...typeof u.status=="number"&&Number.isFinite(u.status)?{status:u.status}:{},...typeof u.code=="string"||typeof u.code=="number"?{code:u.code}:{},...typeof u.errno=="string"?{errno:u.errno}:{},...typeof u.body=="string"?{body:u.body}:{}};return Object.keys(p).length===0?[]:[p]}):void 0;return{provider:e.provider,model:e.model,...n===void 0?{}:{status:n},...t===void 0?{}:{code:t},...r===void 0?{}:{errno:r},...i===void 0?{}:{body:i},...a===void 0||a.length===0?{}:{causes:a},...d===void 0?{}:{ratelimit:d}}}o(Y,"parseProviderError");function G(e){try{const n=JSON.parse(e);if(!g(Ee,n.link)||typeof n.op!="string"||n.op===""||!g(fe,n.kind)||typeof n.message!="string"||typeof n.since!="string")return null;const t=J(n.operation_id);if(!H(n.retry)||!g(ye,n.retry.disposition)||n.retry.by!==void 0&&!g(Te,n.retry.by)||n.retry.attempt!==void 0&&typeof n.retry.attempt!="number"||n.retry.max!==void 0&&typeof n.retry.max!="number"||n.retry.nextAt!==void 0&&typeof n.retry.nextAt!="string"||n.anchorEntryId!==void 0&&(typeof n.anchorEntryId!="string"||n.anchorEntryId===""))return null;const r=n.providerError===void 0?void 0:Y(n.providerError);return n.providerError!==void 0&&r===void 0?null:{link:n.link,op:n.op,kind:n.kind,retry:{disposition:n.retry.disposition,...n.retry.by===void 0?{}:{by:n.retry.by},...n.retry.attempt===void 0?{}:{attempt:n.retry.attempt},...n.retry.max===void 0?{}:{max:n.retry.max},...n.retry.nextAt===void 0?{}:{nextAt:n.retry.nextAt}},message:n.message,since:n.since,operation_id:t,...n.anchorEntryId===void 0?{}:{anchorEntryId:n.anchorEntryId},...r===void 0?{}:{providerError:r}}}catch{return null}}o(G,"parseLegacyFault");function me(e){const n=ue(e);try{return de(n)}catch{return n}}o(me,"canonicalPath");function Ae(e){try{const n=JSON.parse(e);if(!g(Ne,n.state)||typeof n.sessionFile!="string"||n.sessionFile==="")return null;const t=JSON.stringify(n.fault),r=G(t);if(r===null||r.link!=="pi\u2192provider"||r.retry.disposition!=="auto"||r.retry.by!=="daemon")return null;const i=n.anchorEntryId;return i!==null&&typeof i!="string"||(r.anchorEntryId??null)!==i?null:{state:n.state,sessionFile:me(n.sessionFile),fault:r}}catch{return null}}o(Ae,"parseLegacyEpisode");function V(e){return e!==void 0?J(e):B.current()??B.fresh(t=>t)}o(V,"resolveOperationId");function Se(e){return e.slice(0,pe)}o(Se,"clampMessage");function I(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="auto"&&e.retry.by==="daemon"?3:e.retry.disposition==="fatal"?2:1}o(I,"faultPriority");function Q(e){return{link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operationId:e.operation_id,anchorEntryId:e.anchorEntryId??null,retryDisposition:e.retry.disposition,retryBy:e.retry.by??null,retryAttempt:e.retry.attempt??null,retryMax:e.retry.max??null,retryNextAt:e.retry.nextAt??null,providerError:e.providerError===void 0?null:JSON.stringify(e.providerError)}}o(Q,"encodeFault");function W(e){const n=e.provider_error_json,t=typeof n=="string"?Y(JSON.parse(n)):void 0;if(typeof n=="string"&&t===void 0)throw new Error(`invalid provider error on stored fault ${e.fault_id}`);const r={disposition:e.retry_disposition,...e.retry_by===null?{}:{by:e.retry_by},...e.retry_attempt===null?{}:{attempt:e.retry_attempt},...e.retry_max===null?{}:{max:e.retry_max},...e.retry_next_at===null?{}:{nextAt:e.retry_next_at}};return{faultId:e.fault_id,link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operation_id:e.operation_id,retry:r,...e.anchor_entry_id===null?{}:{anchorEntryId:e.anchor_entry_id},...t===void 0?{}:{providerError:t},...typeof e.notified_since=="string"?{notifiedSince:e.notified_since}:{}}}o(W,"hydrateFault");function L(e,n){const t=ce(e,"SELECT * FROM node_faults WHERE node_id = ? AND active = 1").get(n);return t===void 0?null:W(t)}o(L,"activeFault");function q(e){if(!g(K,e.link))throw new Error(`fault link is not durable: ${e.link}`);return{link:e.link,op:e.op,kind:e.kind,retry:e.retry,message:Se(e.message),since:e.since??c(),operation_id:V(e.operation_id),...typeof e.anchorEntryId=="string"&&e.anchorEntryId!==""?{anchorEntryId:e.anchorEntryId}:{},...e.providerError===void 0?{}:{providerError:e.providerError}}}o(q,"storedFault");function z(e,n){j({level:n.retry.disposition==="fatal"?"error":"warn",event:"fault.recorded",operation_id:n.operation_id,node_id:e,error_class:$(n),fields:{link:n.link,op:n.op,kind:n.kind,retry:n.retry,message:n.message,...n.anchorEntryId===void 0?{}:{anchor_entry_id:n.anchorEntryId},...n.providerError===void 0?{}:{provider_error:n.providerError}}})}o(z,"emitRecordedFault");function P(e,n,t,r,i){const d=F(),a=Q(t),u=c();return s(e,"INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(d,n,a.link,a.op,a.kind,a.message,a.since,a.operationId,a.anchorEntryId,a.retryDisposition,a.retryBy,a.retryAttempt,a.retryMax,a.retryNextAt,a.providerError,r?1:0,i?.state??null,i?.bindingId??null,u,u),{...t,faultId:d}}o(P,"insertFault");function Z(e,n,t,r){const i=Q(t);return s(e,"UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?").run(i.link,i.op,i.kind,i.message,i.since,i.operationId,i.anchorEntryId,i.retryDisposition,i.retryBy,i.retryAttempt,i.retryMax,i.retryNextAt,i.providerError,r?.state??null,r?.bindingId??null,c(),n),{...t,faultId:n}}o(Z,"updateFault");function D(e,n){s(e,"DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL").run(n)}o(D,"deleteIfUnretained");function ee(e,n,t,r){const i=L(e,n);if(i===null){const d=s(e,"SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL").get(n);return d!==void 0&&d.link===t.link&&d.kind===t.kind&&d.since===t.since&&d.operation_id===t.operation_id?(s(e,"UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?").run(c(),d.fault_id),Z(e,d.fault_id,t)):P(e,n,t,!0,r)}if(i.link===t.link&&i.kind===t.kind){const d=t.retry.disposition==="auto"?{...i.retry,...t.retry}:t.retry,a=i.retry.disposition===t.retry.disposition&&(t.retry.disposition!=="fatal"||i.message===t.message),u=a?i.since:t.since,p=Z(e,i.faultId,{...t,since:u,anchorEntryId:i.anchorEntryId??t.anchorEntryId,retry:d},r);return a||s(e,"UPDATE node_faults SET notified_since = NULL WHERE fault_id = ?").run(i.faultId),p}return i.link!==t.link&&I(i)>I(t)?null:(s(e,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(e,i.faultId),P(e,n,t,!0,r))}o(ee,"replaceActiveFault");function m(e,n,t){return s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL").get(n,t)?.binding_id??null}o(m,"liveBinding");function we(e,n){return l(t=>{const r=c();return s(t,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,e).changes>0?!0:s(t,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(r,r,e).changes>0})}o(we,"openTurn");function Xe(e){return l(n=>s(n,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),e).changes>0)}o(Xe,"recordTurnProgress");function $e(e,n){return l(t=>{const r=c(),i=s(t,"UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,n.endReason,e).changes>0;if(i&&n.endReason==="settled"){const d="(SELECT node_id FROM executions WHERE execution_id = ?)";n.viewerPresent===!0?s(t,`UPDATE nodes SET reply_at = ?, read_at = ? WHERE node_id = ${d}`).run(r,r,e):s(t,`UPDATE nodes SET reply_at = ? WHERE node_id = ${d}`).run(r,e)}return i})}o($e,"closeTurn");function je(e){return l(n=>s(n,"UPDATE nodes SET read_at = ? WHERE node_id = ?").run(c(),e).changes>0)}o(je,"markNodeRead");function Be(e,n){if(!Number.isInteger(n.consecutivePasses)||n.consecutivePasses<1||n.routes.length===0||!n.routes.every(t=>t!==""))throw new Error("invalid transport-poison verdict");return l(t=>s(t,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:n.consecutivePasses,routes:n.routes}}),e).changes>0)}o(Be,"recordTransportPoisonVerdict");function Je(e){const n=s(E(),"SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);if(n===void 0||n.exit_diagnostic_json===null)return null;try{const t=JSON.parse(n.exit_diagnostic_json),r=t.transport_poison;return t.schema!=="crtr.execution-exit-diagnostic/v1"||r===void 0||!Number.isInteger(r.consecutive_passes)||r.consecutive_passes<1||!Array.isArray(r.routes)||r.routes.length===0||!r.routes.every(i=>typeof i=="string"&&i!=="")?null:{consecutivePasses:r.consecutive_passes,routes:r.routes}}catch{return null}}o(Je,"readTransportPoisonVerdict");function Ke(e){const n=s(E(),"SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(e);return n===void 0?null:{busy:n.turn_open_at!==null&&n.turn_closed_at===null,progressAt:n.turn_progress_at}}o(Ke,"readLiveTurn");function Ye(e){const n=E(),t=s(n,"SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(e);if(t===void 0)return null;const r=s(n,"SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);return{interrupted:t.turn_closed_at===null||r?.turn_end_reason==="teardown-aborted",progressAt:t.turn_progress_at}}o(Ye,"readInterruptedTurn");function Ge(e){return s(E(),"SELECT 1 FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL LIMIT 1").get(e)!==void 0}o(Ge,"hasAdmittedTurn");function Ve(e,n){const t=q(n);return z(e,t),l(r=>ee(r,e,t))}o(Ve,"recordFault");function Qe(e){return l(n=>s(n,"UPDATE node_faults SET notified_since = since, updated_at = ? WHERE fault_id = ?").run(c(),e).changes>0)}o(Qe,"markFaultNotified");function xe(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="fatal"&&e.notifiedSince!==e.since}o(xe,"isUnnotifiedFatalProviderFault");function qe(){return s(E(),"SELECT f.node_id FROM node_faults f JOIN nodes n ON n.node_id = f.node_id WHERE f.active = 1 AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'fatal' AND (f.notified_since IS NULL OR f.notified_since != f.since) AND n.status IN ('active', 'idle')").all().map(e=>e.node_id)}o(qe,"listUnnotifiedFatalFaultNodes");let M=null;function ze(e){M=e}o(ze,"setFatalFaultAnnouncer");function Ze(e,n){const t=L(E(),e);return t===null||n?.link!==void 0&&t.link!==n.link?!1:l(r=>{const i=L(r,e);if(i===null||n?.link!==void 0&&i.link!==n.link)return!1;M!==null&&xe(i)&&M(e);const d=s(r,"SELECT episode_state FROM node_faults WHERE fault_id = ?").get(i.faultId);s(r,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),d.episode_state==="pending"&&n?.preserveEpisode!==!0&&s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(r,i.faultId);const a=V();return j({level:"info",event:"fault.cleared",operation_id:a,node_id:e,error_class:$(i),fields:{cleared_fault_operation_id:i.operation_id,link:i.link,op:i.op,kind:i.kind,retry:i.retry,message:i.message,since:i.since,...i.anchorEntryId===void 0?{}:{anchor_entry_id:i.anchorEntryId}}}),!0})}o(Ze,"clearFault");function ne(e){return L(E(),e)}o(ne,"readFault");function en(e,n){const t=q(n.input);if(t.link!=="pi\u2192provider"||t.retry.disposition!=="auto"||t.retry.by!=="daemon")throw new Error("provider retry episode requires an automatic daemon provider fault");return z(e,t),l(r=>{const i=m(r,e,n.sessionId);if(i===null)return null;const d=L(r,e);if(d!==null&&(d.link!==t.link||d.kind!==t.kind)&&I(d)>I(t))return null;const a=s(r,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL").all(e);s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL").run(c(),e);for(const p of a)D(r,p.fault_id);const u=ee(r,e,t,{state:"pending",bindingId:i});return u===null?null:(s(r,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run("pending",i,c(),u.faultId),u)})}o(en,"recordPendingEpisode");function nn(e,n){return l(t=>{if(s(t,"SELECT 1 FROM nodes WHERE node_id = ? AND intent IS 'refresh'").get(e)!==void 0)return!1;const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(n.retry.disposition,n.retry.by??null,n.retry.attempt??null,n.retry.max??null,n.retry.nextAt??null,c(),e,n.faultId,r).changes>0})}o(nn,"admitEpisode");function tn(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(tn,"reopenEpisode");function rn(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(rn,"invalidateEpisode");function on(e,n){return l(t=>{const r=m(t,e,n.sessionId);if(r===null)return!1;const i=s(t,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").all(e,r);if(i.length===0)return!1;s(t,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").run(c(),e,r);for(const d of i)D(t,d.fault_id);return!0})}o(on,"clearEpisode");function sn(e){const n=s(E(),"SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL").get(e);return n===void 0?null:{state:n.episode_state,sessionId:n.session_id,fault:W(n)}}o(sn,"readEpisode");function dn(){return s(E(),"SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all().map(n=>({nodeId:n.node_id,sessionId:n.bound_session_id,fault:W(n),updatedAt:n.updated_at}))}o(dn,"listPendingProviderRetries");function an(e){const n=ne(e)===null;return{explanation(){if(!n)return null;const t=ne(e);if(t===null||t.link!=="crtr\u2192pi"||t.retry.disposition!=="fatal")return null;const r=t.op.trim(),i=t.message.trim();return r===""||i===""?null:`${r}: ${i}`}}}o(an,"beginBootFaultAttempt");function te(e,n){try{return ae(e).mtime.toISOString()}catch{return n}}o(te,"markerMtime");function w(e,n){if(!f(e))return null;try{return se(e,"utf8")}catch(t){if(!n)return null;throw new Error(`cannot import recovery input ${e}: ${t.message}`)}}o(w,"legacyFile");function Re(e,n){return s(e,"SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(n)??null}o(Re,"latestExecution");function Ie(e,n){const r=b(e,"nodes").has("launched_at")?s(e,"SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?").get(n):s(e,"SELECT created AS started_at FROM nodes WHERE node_id = ?").get(n);if(r===void 0)throw new Error(`cannot import recovery witness for unknown node ${n}`);const i=F();return s(e,"INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(i,n,r.started_at,r.started_at),{execution_id:i,started_at:r.started_at}}o(Ie,"syntheticExecution");function re(e,n,t,r,i){return P(e,n,t,r,i)}o(re,"insertMigratedFault");function De(e,n,t,r){const i=s(e,"SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL").get(t);if(i!==void 0){if(i.node_id!==n)throw new Error(`cannot import provider retry ${r}: transcript is live-owned by ${i.node_id}`);return i.binding_id}const d=s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1").get(n,t);if(d!==void 0)return d.binding_id;const a=s(e,"SELECT created FROM nodes WHERE node_id = ?").get(n);if(a===void 0)throw new Error(`cannot import provider retry ${r}: unknown node ${n}`);const u=F();return s(e,"INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)").run(u,n,t,a.created,a.created,c()),u}o(De,"migrationBinding");function un(e){e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'node_faults'").get()!==void 0&&(b(e,"node_faults").has("notified_since")||e.exec("ALTER TABLE node_faults ADD COLUMN notified_since TEXT"))}o(un,"addFaultNotifiedSince");function cn(e){if(ge(e),Le(e),!f(C()))return;const n=oe(C(),{withFileTypes:!0}).filter(r=>r.isDirectory()).sort((r,i)=>r.name.localeCompare(i.name)),t=s(e,"SELECT 1 FROM nodes WHERE node_id = ?");for(const r of n){const i=r.name,d=_e(i),a=N(d,"busy"),u=N(d,"clean-abort"),p=N(d,"transport-poison.json"),U=N(d,"fault"),A=N(d,"provider-retry"),O=f(a),h=f(u);if(!O&&!h&&!f(p)&&!f(U)&&!f(A)||!f(N(C(),i,"meta.json"))&&t.get(i)===void 0)continue;const k=w(p,!1),S=w(U,!0),x=w(A,!0);if(!O&&!h&&k===null&&S===null&&x===null)continue;let y=Re(e,i);if(y===null&&(y=Ie(e,i)),O){const _=te(a,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?").run(_,_,y.execution_id)}else if(h){const _=te(u,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(_,_,_,y.execution_id)}if(k!==null)try{const _=JSON.parse(k);Number.isInteger(_.consecutivePasses)&&_.consecutivePasses>=1&&Array.isArray(_.routes)&&_.routes.length>0&&_.routes.every(X=>typeof X=="string"&&X!=="")&&s(e,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:_.consecutivePasses,routes:_.routes}}),y.execution_id)}catch{}const R=S===null?null:G(S);if(S!==null&&R===null)throw new Error(`cannot import recovery fault ${U}`);const T=x===null?null:Ae(x);if(x!==null&&T===null)throw new Error(`cannot import provider retry ${A}`);let v=R===null||R.link==="broker\u2194crtrd"?null:re(e,i,R,!0);if(T!==null){const _=De(e,i,T.sessionFile,A);v!==null&&v.since===T.fault.since?s(e,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run(T.state,_,c(),v.faultId):re(e,i,T.fault,!1,{state:T.state,bindingId:_})}}}o(cn,"migrateRecoveryWitnesses");export{Me as FAULT_QUIET_MS,un as addFaultNotifiedSince,nn as admitEpisode,an as beginBootFaultAttempt,on as clearEpisode,Ze as clearFault,$e as closeTurn,Ge as hasAdmittedTurn,W as hydrateFault,rn as invalidateEpisode,dn as listPendingProviderRetries,qe as listUnnotifiedFatalFaultNodes,Qe as markFaultNotified,je as markNodeRead,cn as migrateRecoveryWitnesses,we as openTurn,Y as parseProviderError,sn as readEpisode,ne as readFault,Ye as readInterruptedTurn,Ke as readLiveTurn,Je as readTransportPoisonVerdict,Ve as recordFault,en as recordPendingEpisode,Be as recordTransportPoisonVerdict,Xe as recordTurnProgress,tn as reopenEpisode,ze as setFatalFaultAnnouncer};

package/dist/core/runtime/broker/bash-job-launch.d.ts

@@ -0,0 +1,10 @@
+import { type BashJobPaths } from '../../bash-jobs.js';
+/** True when jobs are started by the daemon's launcher rather than this process. */
+export declare const BASH_JOBS_USE_LAUNCHER: boolean;
+/** Start the supervisor for a job whose `cmd.sh` and `job.log` are already written.
+ *  Resolves with the supervisor's process group and its launch-time identity, which
+ *  lets a later cancellation reject a stale pid; it is not a containment guarantee. */
+export declare function launchBashJobSupervisor(nodeId: string, paths: BashJobPaths, contextDir: string, cwd: string, env: NodeJS.ProcessEnv): Promise<{
+    pgid: number;
+    identity: string;
+}>;

package/dist/core/runtime/broker/bash-job-launch.js

@@ -0,0 +1 @@
+var u=Object.defineProperty;var m=(e,o)=>u(e,"name",{value:o,configurable:!0});import{spawn as S}from"node:child_process";import{writeFileSync as a}from"node:fs";import{BASH_JOB_SUPERVISOR_SCRIPT as l}from"../../bash-job-supervisor.js";import{readBashNiceness as h}from"../../bash-jobs.js";import{capturePidIdentities as j}from"../../canvas/pid.js";import{freshChildNodeExecPath as g}from"../branded-host.js";import{resolveAdmittedHostCommand as B}from"../spawn-env.js";import{startBrokerBashJob as I}from"./daemon-ops.js";const P=process.platform==="linux";async function J(e,o,f,n,d){const b=await g();if(P){const{pid:t}=await I(e,{job_id:o.jobId,cwd:n,env:d});return{pgid:t,identity:`launcher:${t}`}}const c=B("bash",d,n),r=S(c,["-c",l,"--",o.cmdSh,o.jobLog,o.jobExit,o.jobRun,o.jobDone,c,String(h(f)),b],{cwd:n,detached:!0,stdio:"ignore",env:d}),i=await new Promise((t,p)=>{r.once("error",p),r.pid!==void 0&&t(r.pid)});r.unref();const s=j([i])?.get(i);if(s===void 0){try{process.kill(-i,"SIGKILL")}catch{}throw new Error(`bash job ${o.jobId} started without a process identity`)}return a(o.jobPgid,String(i)),a(o.jobPgidIdentity,s),{pgid:i,identity:s}}m(J,"launchBashJobSupervisor");export{P as BASH_JOBS_USE_LAUNCHER,J as launchBashJobSupervisor};

package/dist/daemon/api/handlers/bash-jobs.js

@@ -1 +1 @@
-var S=Object.defineProperty;var h=(e,o)=>S(e,"name",{value:o,configurable:!0});import{killBashJobObject as B,recordBashJobObject as E}from"../../../core/spaces/stop.js";import{readerFor as J}from"../reader.js";import{join as P}from"node:path";import{createFileBeneath as m,existsBeneath as w,unlinkBeneath as $}from"../../../core/spaces/open-beneath.js";import{launchAppProcess as A,appProcessSandboxed as q}from"../../../core/spaces/process.js";import{BASH_JOB_SUPERVISOR_SCRIPT as O}from"../../../core/bash-job-supervisor.js";import{resolveAdmittedHostCommand as N}from"../../../core/runtime/spawn-env.js";import{requireCurrentExecution as _}from"./broker-recovery.js";import{isSafeNodeId as L}from"../../../core/canvas/paths.js";import{contextDir as y}from"../../../core/canvas/node-agent-paths.js";import{getNode as j}from"../../../core/canvas/canvas.js";import{isBrokerLive as T}from"../../../core/runtime/model-swap.js";import{assertNotFinalized as D}from"../../../core/runtime/reopen.js";import{reviveNode as R}from"../../../core/runtime/revive.js";import{activeBackgroundBashJobs as C,bashJobPaths as v,isBashJobProcessGroupAlive as z,readJobPgid as F,readJobPgidIdentity as M,isSafeBashJobId as I,readBashNiceness as G,stopBackgroundBashJob as H}from"../../../core/bash-jobs.js";import{ApiError as i}from"../../../api/index.js";import{notFound as Z}from"../../../core/errors.js";import{freshChildNodeExecPath as U}from"../../../core/runtime/branded-host.js";function f(e){const o=e.params.id;if(o===void 0||!L(o))throw new i(400,"invalid_node_id",`invalid node id: ${JSON.stringify(o)}`);if(j(o)===null)throw Z(`unknown node: ${o}`,{received:o});const d=e.params.jobId;if(d!==void 0){if(!I(d))throw new i(400,"invalid_job_id",`invalid background job id: ${JSON.stringify(d)}`);return{nodeId:o,jobId:d,dir:y(o)}}return{nodeId:o,dir:y(o)}}h(f,"requireTarget");async function V(e){const{nodeId:o,dir:t}=f(e);if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may start a job");const d=e.body;if(!d||typeof d!="object"||Array.isArray(d))throw new i(400,"invalid_request","job.start requires a JSON body");const s=d;_(o,s.expected_execution_id);const r=s.job_id,a=s.cwd,u=s.env;if(typeof r!="string"||!I(r)||typeof a!="string"||!u||typeof u!="object"||Array.isArray(u)||Object.entries(u).some(([c,b])=>!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c)||typeof b!="string"))throw new i(400,"invalid_request","invalid job id, cwd or environment");if(!q())throw new i(400,"invalid_request","job.start is only used with the root launcher");const n=v(t,r);if(![n.jobRun,n.cmdSh,n.jobLog].every(c=>w(n.root,c)))throw new i(404,"job_not_found","job was not prepared by this node");const p=P(n.dir,"job.starting");try{m(n.root,p,"")}catch(c){throw c.code==="EEXIST"?new i(409,"job_already_started","job was already started"):c}let l;try{const c=u,b=N("bash",c,a),g=j(o),k=await U();return l=await A({nodeId:o,app:g.grantee??"app:terminal",isolated:g.isolated,cwd:a,env:c,capture:!1,command:[b,"-c",O,"--",n.cmdSh,n.jobLog,n.jobExit,n.jobRun,n.jobDone,b,String(G(t)),k]}),m(n.root,n.jobPgid,String(l.pid),384),m(n.root,n.jobPgidIdentity,`launcher:${l.pid}`,384),{status:201,body:{pid:l.pid}}}catch(c){if(l!==void 0)try{await l.stop()}catch(b){throw new AggregateError([c,b],`job ${r} launched but could not be recorded or stopped`)}try{$(n.root,p)}catch(b){if(b.code!=="ENOENT")throw b}throw c}}h(V,"handleStart");function X(e){const{nodeId:o,dir:t}=f(e),d=Date.now();return{status:200,body:C(t).map(r=>{const a=F(t,r.jobId)??null;return{job_id:r.jobId,command:r.command,purpose:r.purpose,started_at:new Date(r.startedAtMs).toISOString(),elapsed_ms:Math.max(0,d-r.startedAtMs),pgid:a,pgid_alive:a!==null&&z(a,M(t,r.jobId))}})}}h(X,"handleList");async function x(e){const{nodeId:o,jobId:t,dir:d}=f(e);if(t===void 0)throw new i(400,"invalid_request","a job id is required");const s=e.query.get("foreground")==="1";if(s){if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may abort its foreground job");_(o,e.body?.expected_execution_id)}const r=await H(d,t,s,o);if(r.kind==="missing")throw new i(404,"job_not_found",`no background job ${t} for ${o}`);if(r.kind==="retired")throw new i(404,"job_not_found",`background job ${t} for ${o} is no longer running`,{log_path:r.logPath});if(r.kind==="not-stoppable")throw new i(409,"job_not_stoppable",`job ${t} recorded no trusted process identity`,{log_path:r.logPath});if(s)return{status:200,body:{node_id:o,job_id:t,signaled:r.signaled}};D(o),B(t,n=>{const p=J(n,e);return p.class==="person"?p.person:p.class==="app"?p.app:p.node});const a=j(o);if(a!==null&&!T(a))try{await R(o,{resume:!0,capacity:"freeze"})}catch{}return{status:200,body:{node_id:o,job_id:t,signaled:r.signaled}}}h(x,"handleStop");function K(e){const{nodeId:o,jobId:t,dir:d}=f(e);if(t===void 0)throw new i(400,"invalid_request","a job id is required");if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may record its job");const s=e.body,r=v(d,t);if(typeof s?.command!="string"||s.job_dir!==r.dir||s.log_path!==r.jobLog)throw new i(400,"invalid_request",`job.background requires command, job_dir ${r.dir} and log_path ${r.jobLog}`);if(!w(r.root,r.jobBg))throw new i(404,"job_not_found",`job ${t} of ${o} is not backgrounded`);const a=s.command;return{status:201,body:{id:E(o,t,{command:a,jobDir:r.dir,logPath:r.jobLog}).id}}}h(K,"handleBackground");const jo=[{method:"POST",pattern:"/v1/nodes/:id/jobs/:jobId/background",handler:K},{method:"POST",pattern:"/v1/nodes/:id/jobs/start",handler:V},{method:"GET",pattern:"/v1/nodes/:id/jobs",handler:X},{method:"DELETE",pattern:"/v1/nodes/:id/jobs/:jobId",handler:x}];export{jo as bashJobRoutes};
+var B=Object.defineProperty;var h=(e,o)=>B(e,"name",{value:o,configurable:!0});import{killBashJobObject as E,recordBashJobObject as J}from"../../../core/spaces/stop.js";import{readerFor as P}from"../reader.js";import{join as $}from"node:path";import{createFileBeneath as j,existsBeneath as m,unlinkBeneath as A}from"../../../core/spaces/open-beneath.js";import{launchAppProcess as q,appProcessSandboxed as O}from"../../../core/spaces/process.js";import{BASH_JOB_SUPERVISOR_SCRIPT as N}from"../../../core/bash-job-supervisor.js";import{resolveAdmittedHostCommand as L}from"../../../core/runtime/spawn-env.js";import{requireCurrentExecution as y}from"./broker-recovery.js";import{isSafeNodeId as T}from"../../../core/canvas/paths.js";import{contextDir as v}from"../../../core/canvas/node-agent-paths.js";import{getNode as g}from"../../../core/canvas/canvas.js";import{isBrokerLive as D}from"../../../core/runtime/model-swap.js";import{assertNotFinalized as R}from"../../../core/runtime/reopen.js";import{reviveNode as C}from"../../../core/runtime/revive.js";import{activeBackgroundBashJobs as z,bashJobPaths as k,isBashJobProcessGroupAlive as F,readJobPgid as M,readJobPgidIdentity as G,isSafeBashJobId as I,readBashNiceness as H,stopBackgroundBashJob as Z}from"../../../core/bash-jobs.js";import{ApiError as i}from"../../../api/index.js";import{notFound as U}from"../../../core/errors.js";import{freshChildNodeExecPath as V}from"../../../core/runtime/branded-host.js";function f(e){const o=e.params.id;if(o===void 0||!T(o))throw new i(400,"invalid_node_id",`invalid node id: ${JSON.stringify(o)}`);if(g(o)===null)throw U(`unknown node: ${o}`,{received:o});const d=e.params.jobId;if(d!==void 0){if(!I(d))throw new i(400,"invalid_job_id",`invalid background job id: ${JSON.stringify(d)}`);return{nodeId:o,jobId:d,dir:v(o)}}return{nodeId:o,dir:v(o)}}h(f,"requireTarget");async function X(e){const{nodeId:o,dir:t}=f(e);if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may start a job");const d=e.body;if(!d||typeof d!="object"||Array.isArray(d))throw new i(400,"invalid_request","job.start requires a JSON body");const a=d;y(o,a.expected_execution_id);const r=a.job_id,s=a.cwd,p=a.env;if(typeof r!="string"||!I(r)||typeof s!="string"||!p||typeof p!="object"||Array.isArray(p)||Object.entries(p).some(([c,b])=>!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c)||typeof b!="string"))throw new i(400,"invalid_request","invalid job id, cwd or environment");if(!O())throw new i(400,"invalid_request","job.start is only used with the root launcher");const n=k(t,r);if(!(m(n.root,n.jobRun)||m(n.root,n.jobBg))||![n.cmdSh,n.jobLog].every(c=>m(n.root,c)))throw new i(404,"job_not_found","job was not prepared by this node");const w=$(n.dir,"job.starting");try{j(n.root,w,"")}catch(c){throw c.code==="EEXIST"?new i(409,"job_already_started","job was already started"):c}let l;try{const c=p,b=L("bash",c,s),_=g(o),S=await V();return l=await q({nodeId:o,app:_.grantee??"app:terminal",isolated:_.isolated,cwd:s,env:c,capture:!1,command:[b,"-c",N,"--",n.cmdSh,n.jobLog,n.jobExit,n.jobRun,n.jobDone,b,String(H(t)),S]}),j(n.root,n.jobPgid,String(l.pid),384),j(n.root,n.jobPgidIdentity,`launcher:${l.pid}`,384),{status:201,body:{pid:l.pid}}}catch(c){if(l!==void 0)try{await l.stop()}catch(b){throw new AggregateError([c,b],`job ${r} launched but could not be recorded or stopped`)}try{A(n.root,w)}catch(b){if(b.code!=="ENOENT")throw b}throw c}}h(X,"handleStart");function x(e){const{nodeId:o,dir:t}=f(e),d=Date.now();return{status:200,body:z(t).map(r=>{const s=M(t,r.jobId)??null;return{job_id:r.jobId,command:r.command,purpose:r.purpose,started_at:new Date(r.startedAtMs).toISOString(),elapsed_ms:Math.max(0,d-r.startedAtMs),pgid:s,pgid_alive:s!==null&&F(s,G(t,r.jobId))}})}}h(x,"handleList");async function K(e){const{nodeId:o,jobId:t,dir:d}=f(e);if(t===void 0)throw new i(400,"invalid_request","a job id is required");const a=e.query.get("foreground")==="1";if(a){if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may abort its foreground job");y(o,e.body?.expected_execution_id)}const r=await Z(d,t,a,o);if(r.kind==="missing")throw new i(404,"job_not_found",`no background job ${t} for ${o}`);if(r.kind==="retired")throw new i(404,"job_not_found",`background job ${t} for ${o} is no longer running`,{log_path:r.logPath});if(r.kind==="not-stoppable")throw new i(409,"job_not_stoppable",`job ${t} recorded no trusted process identity`,{log_path:r.logPath});if(a)return{status:200,body:{node_id:o,job_id:t,signaled:r.signaled}};R(o),E(t,n=>{const u=P(n,e);return u.class==="person"?u.person:u.class==="app"?u.app:u.node});const s=g(o);if(s!==null&&!D(s))try{await C(o,{resume:!0,capacity:"freeze"})}catch{}return{status:200,body:{node_id:o,job_id:t,signaled:r.signaled}}}h(K,"handleStop");function Q(e){const{nodeId:o,jobId:t,dir:d}=f(e);if(t===void 0)throw new i(400,"invalid_request","a job id is required");if(e.principal.class!=="broker"||e.principal.node!==o)throw new i(403,"permission_error","only this node\u2019s broker may record its job");const a=e.body,r=k(d,t);if(typeof a?.command!="string"||a.job_dir!==r.dir||a.log_path!==r.jobLog)throw new i(400,"invalid_request",`job.background requires command, job_dir ${r.dir} and log_path ${r.jobLog}`);if(!m(r.root,r.jobBg))throw new i(404,"job_not_found",`job ${t} of ${o} is not backgrounded`);const s=a.command;return{status:201,body:{id:J(o,t,{command:s,jobDir:r.dir,logPath:r.jobLog}).id}}}h(Q,"handleBackground");const go=[{method:"POST",pattern:"/v1/nodes/:id/jobs/:jobId/background",handler:Q},{method:"POST",pattern:"/v1/nodes/:id/jobs/start",handler:X},{method:"GET",pattern:"/v1/nodes/:id/jobs",handler:x},{method:"DELETE",pattern:"/v1/nodes/:id/jobs/:jobId",handler:K}];export{go as bashJobRoutes};

package/dist/daemon/crtrd.js

@@ -1,16 +1,16 @@
-var Pe=Object.defineProperty;var a=(t,n)=>Pe(t,"name",{value:n,configurable:!0});import{startPeerOutbox as xe}from"./peer/outbox.js";import{reportMachineStatus as Ie}from"../core/peer/directory.js";import{createPeerRelay as De}from"./peer/relay-service.js";import{writeFileSync as Te,readFileSync as Ce,rmSync as $e,mkdirSync as ee,existsSync as E,chmodSync as Le}from"node:fs";import{createServer as re}from"node:http";import{dirname as Ae}from"node:path";import{envCrtrdTcp as Ne,envDocumentPort as Me}from"../shared/env.js";import{serveCanvasConversationStore as Oe}from"../core/canvas/sessions.js";import{pollRunObservers as Ge}from"./runs/observer.js";import{randomBytes as Fe}from"node:crypto";import{apiSocketPath as He,crtrHome as je}from"../core/canvas/paths.js";import{activeAppUsers as Be}from"../core/spaces/users.js";import{ensureAppSpace as qe,ensureUserFiles as ze}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as D}from"../launcher/client.js";import{installLauncherEventSink as We}from"./launcher-events.js";import{initializeEmptyLayout as Ue,layoutNeedsMigration as Ke,socketDir as te}from"../core/layout.js";import{migrateLooseKeys as Je}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as Ve}from"../core/profiles/relocate.js";import{getNode as Qe,listNodesForSupervision as ne,setFrozen as oe}from"../core/canvas/index.js";import{createApiServer as Ye}from"./api/server.js";import{createDocumentListener as Xe}from"./api/document-listener.js";import{documentSecret as Ze}from"./api/document-secret.js";import{createAppListener as er}from"./api/app-listener.js";import{installLocalTerminalGrant as rr}from"../core/grants/local.js";import{onRunEventAppended as tr}from"../core/runs/events.js";import{runNotices as nr}from"./run-notices.js";import{createGrantSync as or,GRANT_SYNC_INTERVAL_S as ir}from"../core/grants/sync.js";import{runRemovalCascades as sr}from"../core/grants/remove.js";import{sweepSandboxes as ar}from"../core/grants/on-sync.js";import{rebuildCronExecutions as cr}from"./cron/run-socket.js";import{hasSyncedGrants as dr}from"../core/grants/mirror.js";import{readConfig as ie}from"../core/config.js";import{reconcileUndeliveredTickets as lr}from"./human/sweep.js";import{resetHandedOffDeliveries as pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as mr}from"../core/canvas/daemon-actions.js";import{sweepReviews as fr}from"./review/sweep.js";import{pidfilePath as j}from"./pidfile.js";import{rendererWarning as ur}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as hr}from"./supervise-cadence.js";import{createResumeDetector as vr}from"./resume-detector.js";import{BrokerSupervisionReconciler as wr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as gr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as yr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as Sr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as _r}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as br}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as kr}from"./reconcilers/action-lane.js";import{endLiveExecution as Er,migrateLegacyExecutionIdentities as Rr}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as Pr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as xr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Ir}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Dr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Tr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Cr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as $r}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Lr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as se,capturePidCommand as Ar,captureTeardownSnapshot as ae,isPidAlive as Nr,isAnyPidAlive as ce,killProcessTreePids as de}from"../core/canvas/pid.js";import{bindFleet as Mr,boundFleet as Or}from"../core/runtime/fleet.js";import{bindDaemonControl as Gr,clearDaemonControl as Fr}from"./control.js";import{execDaemon as Hr,spawnDaemon as jr}from"./manage.js";import{reconcileAndMigrateBoot as Br}from"../core/canvas/boot.js";import{resolveInstallId as qr}from"../core/canvas/install-id.js";import{setInstallId as zr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Wr,releaseDaemonOwnership as le}from"../core/canvas/daemon-owner.js";import{migrate as Ur,openDb as pe,withCanvasWrite as Kr,bindDaemonCanvasOpenHook as Jr,clearDaemonCanvasOpenHook as me}from"../core/canvas/db.js";import{brokerSignalBus as B}from"./broker-signals/bus.js";import{brokerSignalStreams as Vr}from"./api/handlers/broker-signals.js";import{reconcilePackages as Qr}from"../core/graph/package-reconcile.js";import{startRepoSync as Yr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Xr,reapStaleSpares as Zr}from"../core/runtime/warm-pool.js";import{DaemonFleet as et}from"./fleet.js";import{BOOT_MIGRATIONS as rt}from"./boot-migrations.js";import{applyJoinedOwner as tt}from"./joined-owner.js";import{emitEvent as f}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as nt}from"../core/events/source.js";import{activateCanvasImport as ot,MigrationBlockedError as it,runForcedCanvasImport as st}from"../migrations/activation.js";import{convertLegacyProviderStore as at,convertLegacyScopedStore as ct}from"../core/scoped-state/migrate.js";import{userScopeRoot as dt}from"../core/scope.js";import{secretsDir as lt}from"../core/layout.js";function q(t,n){return t.includes(` --canvas-home ${n} `)}a(q,"brokerCommandDeclaresCanvasHome");function z(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}a(z,"brokerCommandEpoch");const pt=5e3,fe=150;async function mt(t){const n=se();if(n===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const c=je(),u=[];for(const[s,m]of n.processes){const y=m.command;if(m.zombie||!y.includes("broker-cli.js")||!q(y,c)||z(y)===t)continue;const b=y.trim().split(/\s+/).at(-1)??"";b===""||b.endsWith("broker-cli.js")||u.push({pid:s,nodeId:b})}let h=0;const o=[];for(const s of u){const m=Ar(s.pid);if(m===null||!m.includes("broker-cli.js")||!q(m,c)||z(m)===t||m.trim().split(/\s+/).at(-1)!==s.nodeId)continue;const y=ae(s.pid,null);if(!y.reused){try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):process.kill(s.pid,"SIGTERM")}catch{continue}h++,o.push({pid:s.pid,snap:y})}}const g=Date.now()+pt;for(;Date.now()<g&&o.some(s=>ce(s.snap.tree.length>0?s.snap.tree:[s.pid]));)await new Promise(s=>setTimeout(s,fe));let w=0;for(const s of o){const m=s.snap.tree.length>0?s.snap.tree:[s.pid];ce(m)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):de(m,"SIGKILL",s.snap.identities),w++)}w>0&&await new Promise(s=>setTimeout(s,fe));const d=se();if(d===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const p=[];for(const[s,m]of d.processes){const y=m.command;m.zombie||!y.includes("broker-cli.js")||q(y,c)&&z(y)!==t&&p.push(s)}if(p.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${p.join(", ")}`);f({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:u.length,sigtermed:h,sigkilled:w}})}a(mt,"reapPriorEpochBrokers");const ft=5e3;async function ut(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=Qe(o)?.status;(g==="active"||g==="idle")&&oe(o,new Date().toISOString())}const c=n.map(([,o])=>({pid:o.pid,snap:ae(o.pid,null),exited:o.handle.exited}));for(const o of c)if(!o.snap.reused)try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const u=new Promise(o=>{setTimeout(o,ft).unref()});await Promise.allSettled(c.map(o=>Promise.race([o.exited.then(()=>{}),u])));const h=[];for(const o of c)Nr(o.pid)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):de(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),h.push(o));await Promise.all(h.map(o=>o.exited));for(const[o]of n)Er(o,{reason:"daemon-teardown"});f({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:c.length,sigkilled:h.length}})}a(ut,"teardownFleet");function ue(){ee(Ae(j()),{recursive:!0}),Te(j(),String(process.pid),"utf8")}a(ue,"writePidfile");function he(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}a(he,"isMissingPathError");function ve(t){const n=j();let c;try{c=Ce(n,"utf8")}catch(u){if(he(u))return;throw u}if(c.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{$e(n)}catch(u){if(he(u))return;throw u}}a(ve,"removeOwnedPidfileMirror");const ht={acceptsDetachedWork:a(()=>!0,"acceptsDetachedWork"),registerDetached:a(t=>{t.catch(()=>{})},"registerDetached")},W=new wr,vt=new gr,wt=new yr,gt=new Sr,yt=new _r,St=new br,_t=new kr,bt=new xr,kt=new Ir,Et=new Dr,Rt=new Tr,Pt=new Cr,xt=new $r;async function It(t=Date.now(),n=ht){const c=Or(),u=await W.runFleetPreamble(c);let h,o;try{const w=ne();h=w.filter(d=>d.status==="active"||d.status==="idle"),o=w.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(w){f({level:"error",event:"canvas.node_list.failed",error:w});return}await W.run(t,{rows:h,fleet:c,lifecycle:n}),vt.run(t,{rows:h});const g=await wt.run(t,{rows:[...h,...o],fleet:c});gt.run(t),xt.run(t),yt.run(t,{lifecycle:n}),St.run(t,{lifecycle:n}),_t.run(t,{lifecycle:n}),bt.run(t,{lifecycle:n}),kt.run(t,{lifecycle:n,fleet:c}),Et.run(t,{lifecycle:n}),Ge(),Pt.run(t,{fleet:c,lifecycle:n}),await Rt.run(t,{rows:h}),W.warnIfSaturated({fleet:c,lifecycle:n,thresholds:u,frozen:g})}a(It,"superviseTick");async function Dt(t){ee(te(),{recursive:!0,mode:493}),process.platform==="linux"&&Le(te(),493);const n=He();if(E(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const c=re((d,p)=>{if(d.method==="GET"&&d.url==="/healthz"){p.writeHead(200,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}p.writeHead(503,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,p)=>{c.once("error",p),c.listen(n,()=>{c.off("error",p),d()})});let u;const h=t.tcp??Ne();if(h){const d=/^(.*):(\d+)$/.exec(h);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${h}`);const p=re(c.listeners("request")[0]);await new Promise((s,m)=>{p.once("error",m),p.listen(Number(d[2]),d[1],()=>{p.off("error",m),s()})}),u=p}ue();let o;const g=a(()=>(o??=(async()=>{c.closeAllConnections(),u?.closeAllConnections(),await Promise.all([c,u].filter(d=>d!==void 0).map(d=>new Promise((p,s)=>d.close(m=>m?s(m):p())))),ve(process.pid),process.off("SIGINT",w),process.off("SIGTERM",w)})(),o),"cleanup"),w=a(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",w),process.on("SIGTERM",w),g}a(Dt,"serveLayoutRefusal");async function Tt(t={}){if(Ke())return Dt(t);Ue();const n=Wr(Fe(16).toString("hex"));if(!n.claimed){if(n.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${n.incumbentPid??"?"})
+var Pe=Object.defineProperty;var a=(t,n)=>Pe(t,"name",{value:n,configurable:!0});import{startPeerOutbox as xe}from"./peer/outbox.js";import{reportMachineStatus as Ie}from"../core/peer/directory.js";import{createPeerRelay as De}from"./peer/relay-service.js";import{writeFileSync as Te,readFileSync as Ce,rmSync as $e,mkdirSync as ee,existsSync as E,chmodSync as Le}from"node:fs";import{createServer as re}from"node:http";import{dirname as Ae}from"node:path";import{envCrtrdTcp as Ne,envDocumentPort as Me}from"../shared/env.js";import{serveCanvasConversationStore as Oe}from"../core/canvas/sessions.js";import{pollRunObservers as Fe}from"./runs/observer.js";import{randomBytes as Ge}from"node:crypto";import{apiSocketPath as He,crtrHome as je}from"../core/canvas/paths.js";import{activeAppUsers as Be}from"../core/spaces/users.js";import{ensureAppSpace as qe,ensureUserFiles as ze}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as D}from"../launcher/client.js";import{installLauncherEventSink as We}from"./launcher-events.js";import{initializeEmptyLayout as Ue,layoutNeedsMigration as Ke,socketDir as te}from"../core/layout.js";import{migrateLooseKeys as Je}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as Ve}from"../core/profiles/relocate.js";import{getNode as Qe,listNodesForSupervision as ne,setFrozen as oe}from"../core/canvas/index.js";import{createApiServer as Ye}from"./api/server.js";import{createDocumentListener as Xe}from"./api/document-listener.js";import{documentSecret as Ze}from"./api/document-secret.js";import{createAppListener as er}from"./api/app-listener.js";import{installLocalTerminalGrant as rr}from"../core/grants/local.js";import{onRunEventAppended as tr}from"../core/runs/events.js";import{runNotices as nr}from"./run-notices.js";import{createGrantSync as or,GRANT_SYNC_INTERVAL_S as ir}from"../core/grants/sync.js";import{runRemovalCascades as sr}from"../core/grants/remove.js";import{sweepSandboxes as ar}from"../core/grants/on-sync.js";import{rebuildCronExecutions as cr}from"./cron/run-socket.js";import{hasSyncedGrants as dr}from"../core/grants/mirror.js";import{readConfig as ie}from"../core/config.js";import{reconcileUndeliveredTickets as lr}from"./human/sweep.js";import{resetHandedOffDeliveries as pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as mr}from"../core/canvas/daemon-actions.js";import{sweepReviews as fr}from"./review/sweep.js";import{pidfilePath as B}from"./pidfile.js";import{rendererWarning as ur}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as hr}from"./supervise-cadence.js";import{createResumeDetector as vr}from"./resume-detector.js";import{setFatalFaultAnnouncer as wr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as gr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as yr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as Sr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as _r}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as br}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as kr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as Er}from"./reconcilers/action-lane.js";import{endLiveExecution as Rr,migrateLegacyExecutionIdentities as Pr}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as xr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as Ir}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Dr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Tr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Cr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as $r}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Lr}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Ar}from"./structured-requests/boot.js";import{captureLivenessSnapshot as se,capturePidCommand as Nr,captureTeardownSnapshot as ae,isPidAlive as Mr,isAnyPidAlive as ce,killProcessTreePids as de}from"../core/canvas/pid.js";import{bindFleet as Or,boundFleet as Fr}from"../core/runtime/fleet.js";import{bindDaemonControl as Gr,clearDaemonControl as Hr}from"./control.js";import{execDaemon as jr,spawnDaemon as Br}from"./manage.js";import{reconcileAndMigrateBoot as qr}from"../core/canvas/boot.js";import{resolveInstallId as zr}from"../core/canvas/install-id.js";import{setInstallId as Wr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Ur,releaseDaemonOwnership as le}from"../core/canvas/daemon-owner.js";import{migrate as Kr,openDb as pe,withCanvasWrite as Jr,bindDaemonCanvasOpenHook as Vr,clearDaemonCanvasOpenHook as me}from"../core/canvas/db.js";import{brokerSignalBus as q}from"./broker-signals/bus.js";import{brokerSignalStreams as Qr}from"./api/handlers/broker-signals.js";import{reconcilePackages as Yr}from"../core/graph/package-reconcile.js";import{startRepoSync as Xr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Zr,reapStaleSpares as et}from"../core/runtime/warm-pool.js";import{DaemonFleet as rt}from"./fleet.js";import{BOOT_MIGRATIONS as tt}from"./boot-migrations.js";import{applyJoinedOwner as nt}from"./joined-owner.js";import{emitEvent as f}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as ot}from"../core/events/source.js";import{activateCanvasImport as it,MigrationBlockedError as st,runForcedCanvasImport as at}from"../migrations/activation.js";import{convertLegacyProviderStore as ct,convertLegacyScopedStore as dt}from"../core/scoped-state/migrate.js";import{userScopeRoot as lt}from"../core/scope.js";import{secretsDir as pt}from"../core/layout.js";function z(t,n){return t.includes(` --canvas-home ${n} `)}a(z,"brokerCommandDeclaresCanvasHome");function W(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}a(W,"brokerCommandEpoch");const mt=5e3,fe=150;async function ft(t){const n=se();if(n===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const c=je(),u=[];for(const[s,m]of n.processes){const y=m.command;if(m.zombie||!y.includes("broker-cli.js")||!z(y,c)||W(y)===t)continue;const b=y.trim().split(/\s+/).at(-1)??"";b===""||b.endsWith("broker-cli.js")||u.push({pid:s,nodeId:b})}let h=0;const o=[];for(const s of u){const m=Nr(s.pid);if(m===null||!m.includes("broker-cli.js")||!z(m,c)||W(m)===t||m.trim().split(/\s+/).at(-1)!==s.nodeId)continue;const y=ae(s.pid,null);if(!y.reused){try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):process.kill(s.pid,"SIGTERM")}catch{continue}h++,o.push({pid:s.pid,snap:y})}}const g=Date.now()+mt;for(;Date.now()<g&&o.some(s=>ce(s.snap.tree.length>0?s.snap.tree:[s.pid]));)await new Promise(s=>setTimeout(s,fe));let w=0;for(const s of o){const m=s.snap.tree.length>0?s.snap.tree:[s.pid];ce(m)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):de(m,"SIGKILL",s.snap.identities),w++)}w>0&&await new Promise(s=>setTimeout(s,fe));const d=se();if(d===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const p=[];for(const[s,m]of d.processes){const y=m.command;m.zombie||!y.includes("broker-cli.js")||z(y,c)&&W(y)!==t&&p.push(s)}if(p.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${p.join(", ")}`);f({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:u.length,sigtermed:h,sigkilled:w}})}a(ft,"reapPriorEpochBrokers");const ut=5e3;async function ht(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=Qe(o)?.status;(g==="active"||g==="idle")&&oe(o,new Date().toISOString())}const c=n.map(([,o])=>({pid:o.pid,snap:ae(o.pid,null),exited:o.handle.exited}));for(const o of c)if(!o.snap.reused)try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const u=new Promise(o=>{setTimeout(o,ut).unref()});await Promise.allSettled(c.map(o=>Promise.race([o.exited.then(()=>{}),u])));const h=[];for(const o of c)Mr(o.pid)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):de(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),h.push(o));await Promise.all(h.map(o=>o.exited));for(const[o]of n)Rr(o,{reason:"daemon-teardown"});f({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:c.length,sigkilled:h.length}})}a(ht,"teardownFleet");function ue(){ee(Ae(B()),{recursive:!0}),Te(B(),String(process.pid),"utf8")}a(ue,"writePidfile");function he(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}a(he,"isMissingPathError");function ve(t){const n=B();let c;try{c=Ce(n,"utf8")}catch(u){if(he(u))return;throw u}if(c.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{$e(n)}catch(u){if(he(u))return;throw u}}a(ve,"removeOwnedPidfileMirror");const vt={acceptsDetachedWork:a(()=>!0,"acceptsDetachedWork"),registerDetached:a(t=>{t.catch(()=>{})},"registerDetached")},O=new gr;wr(t=>O.handleFatalFault(t,!1));const wt=new yr,gt=new Sr,yt=new _r,St=new br,_t=new kr,bt=new Er,kt=new Ir,Et=new Dr,Rt=new Tr,Pt=new Cr,xt=new $r,It=new Lr;async function Dt(t=Date.now(),n=vt){const c=Fr(),u=await O.runFleetPreamble(c);let h,o;try{const w=ne();h=w.filter(d=>d.status==="active"||d.status==="idle"),o=w.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(w){f({level:"error",event:"canvas.node_list.failed",error:w});return}await O.run(t,{rows:h,fleet:c,lifecycle:n}),wt.run(t,{rows:h});const g=await gt.run(t,{rows:[...h,...o],fleet:c});yt.run(t),It.run(t),St.run(t,{lifecycle:n}),_t.run(t,{lifecycle:n}),bt.run(t,{lifecycle:n}),kt.run(t,{lifecycle:n}),Et.run(t,{lifecycle:n,fleet:c}),Rt.run(t,{lifecycle:n}),Fe(),xt.run(t,{fleet:c,lifecycle:n}),await Pt.run(t,{rows:h}),O.warnIfSaturated({fleet:c,lifecycle:n,thresholds:u,frozen:g})}a(Dt,"superviseTick");async function Tt(t){ee(te(),{recursive:!0,mode:493}),process.platform==="linux"&&Le(te(),493);const n=He();if(E(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const c=re((d,p)=>{if(d.method==="GET"&&d.url==="/healthz"){p.writeHead(200,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}p.writeHead(503,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,p)=>{c.once("error",p),c.listen(n,()=>{c.off("error",p),d()})});let u;const h=t.tcp??Ne();if(h){const d=/^(.*):(\d+)$/.exec(h);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${h}`);const p=re(c.listeners("request")[0]);await new Promise((s,m)=>{p.once("error",m),p.listen(Number(d[2]),d[1],()=>{p.off("error",m),s()})}),u=p}ue();let o;const g=a(()=>(o??=(async()=>{c.closeAllConnections(),u?.closeAllConnections(),await Promise.all([c,u].filter(d=>d!==void 0).map(d=>new Promise((p,s)=>d.close(m=>m?s(m):p())))),ve(process.pid),process.off("SIGINT",w),process.off("SIGTERM",w)})(),o),"cleanup"),w=a(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",w),process.on("SIGTERM",w),g}a(Tt,"serveLayoutRefusal");async function Ct(t={}){if(Ke())return Tt(t);Ue();const n=Ur(Ge(16).toString("hex"));if(!n.claimed){if(n.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${n.incumbentPid??"?"})
 `);else{const r=[n.incumbentPid!==void 0?`pid ${n.incumbentPid}`:void 0,n.verdict!==void 0?`verdict ${n.verdict}`:void 0].filter(i=>i!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${n.reason}${e}
-`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const c=n.claim;let u=null;try{await mt(c.token),ct({kind:"user",root:dt()});for(const e of Ve())process.stderr.write(`[crtrd] ${e}
-`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:i}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:l}=await import("node:os");i("user",S=>{S.paths??={},S.paths.user_files??=l()})}}await at(lt());for(const e of Je())process.stderr.write(`[crtrd] ${e}
-`);Ur(pe()),Jr(e=>B.install(e));const r=Kr(e=>Qr(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
+`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const c=n.claim;let u=null;try{await ft(c.token),dt({kind:"user",root:lt()});for(const e of Ve())process.stderr.write(`[crtrd] ${e}
+`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:i}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:l}=await import("node:os");i("user",S=>{S.paths??={},S.paths.user_files??=l()})}}await ct(pt());for(const e of Je())process.stderr.write(`[crtrd] ${e}
+`);Kr(pe()),Vr(e=>q.install(e));const r=Jr(e=>Yr(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
 `);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
 `);for(const e of r.quarantined)process.stderr.write(`[crtrd] plugin "${e.name}" has a damaged ${e.file} (${e.reason}); ${e.movedTo!==void 0?`moved to ${e.movedTo}`:"left in place and skipped"}
 `);for(const e of r.kinds.refused)process.stderr.write(`[crtrd] plugin "${e.plugin}" kind ${e.kind} not registered: ${e.message}
 `);for(const e of r.kinds.invalid)process.stderr.write(`[crtrd] plugin "${e.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${e.error}
-`);u=tt(),process.platform==="linux"&&E(x)&&(We(),await D({operation:"users.sync",users:Be()})),Pr(c.token)}catch(r){me(),B.close();try{const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let h=!0,o,g=Promise.resolve(),w=!1,d,p;const s=new Set;let m=!1,y=!1,b,A,O,U=!1,N,T,C,$,G;const we=vr();let R,v={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},I="initial startup activation",ge=null;function ye(){return v.phase==="ready"?null:{phase:v.phase,message:v.message,blockerPaths:v.blockerPaths}}a(ye,"startupBlock");function k(r){s.add(r),r.finally(()=>{s.delete(r)}).catch(()=>{})}a(k,"registerDetached");const K={acceptsDetachedWork:a(()=>h,"acceptsDetachedWork"),registerDetached:k};Yr(K,pe);async function Se(){try{h=!1,Vr.close(),me(),B.close(),d!==void 0&&(clearInterval(d),d=void 0),C!==void 0&&(clearInterval(C),C=void 0),G?.(),G=void 0;let r;do r=g,await r;while(r!==g);for(;s.size>0;)await Promise.allSettled([...s]);if(p!==void 0&&(await ut(p),p.dispose()),O!==void 0&&await O.close(),A!==void 0&&!U&&(U=!0,await A.close()),$!==void 0){const e=$;$=void 0,await e.stop()}if(T!==void 0){const e=T;T=void 0,await e.close()}if(N!==void 0){const e=N;N=void 0,await e.close()}if(m||(ve(process.pid),m=!0),m&&!y){const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released");y=!0}}finally{Fr(),process.off("SIGINT",Q),process.off("SIGTERM",Y),process.off("exit",X)}}a(Se,"cleanupAttempt");const M=1e3;let J=!1;function _e(){return J?{graceMs:M}:(J=!0,_.fresh(()=>{f({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:M}})}),setTimeout(()=>{be()},M),{graceMs:M})}a(_e,"requestHandover");async function be(){try{await P()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+`);u=nt(),process.platform==="linux"&&E(x)&&(We(),await D({operation:"users.sync",users:Be()})),xr(c.token)}catch(r){me(),q.close();try{const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let h=!0,o,g=Promise.resolve(),w=!1,d,p;const s=new Set;let m=!1,y=!1,b,A,F,U=!1,N,T,C,$,G;const we=vr();let R,v={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},I="initial startup activation",ge=null;function ye(){return v.phase==="ready"?null:{phase:v.phase,message:v.message,blockerPaths:v.blockerPaths}}a(ye,"startupBlock");function k(r){s.add(r),r.finally(()=>{s.delete(r)}).catch(()=>{})}a(k,"registerDetached");const K={acceptsDetachedWork:a(()=>h,"acceptsDetachedWork"),registerDetached:k};Xr(K,pe);async function Se(){try{h=!1,Qr.close(),me(),q.close(),d!==void 0&&(clearInterval(d),d=void 0),C!==void 0&&(clearInterval(C),C=void 0),G?.(),G=void 0;let r;do r=g,await r;while(r!==g);for(;s.size>0;)await Promise.allSettled([...s]);if(p!==void 0&&(await ht(p),p.dispose()),F!==void 0&&await F.close(),A!==void 0&&!U&&(U=!0,await A.close()),$!==void 0){const e=$;$=void 0,await e.stop()}if(T!==void 0){const e=T;T=void 0,await e.close()}if(N!==void 0){const e=N;N=void 0,await e.close()}if(m||(ve(process.pid),m=!0),m&&!y){const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released");y=!0}}finally{Hr(),process.off("SIGINT",Q),process.off("SIGTERM",Y),process.off("exit",X)}}a(Se,"cleanupAttempt");const M=1e3;let J=!1;function _e(){return J?{graceMs:M}:(J=!0,_.fresh(()=>{f({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:M}})}),setTimeout(()=>{be()},M),{graceMs:M})}a(_e,"requestHandover");async function be(){try{await P()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
 ${e}
-`),process.exit(1)}try{process.platform==="linux"&&E(x)&&(await D({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Hr();const r=await jr({waitForJoinLock:!0});_.fresh(()=>{f({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
+`),process.exit(1)}try{process.platform==="linux"&&E(x)&&(await D({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await jr();const r=await Br({waitForJoinLock:!0});_.fresh(()=>{f({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
 `),process.exit(1)}process.exit(0)}a(be,"performHandover");function P(){if(b!==void 0)return b;const r=Se();return b=r,r.then(()=>{b===r&&(b=void 0)},()=>{b===r&&(b=void 0)}),r}a(P,"requestCleanup");async function V(r){try{await P(),process.exit(0)}catch(e){process.exitCode=1;const i=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
 ${i}
-`),process.exit(1)}}a(V,"handleSignal");function Q(){V("SIGINT")}a(Q,"onSigint");function Y(){V("SIGTERM")}a(Y,"onSigterm");function X(){P().catch(()=>{})}a(X,"onExit");function F(r){h&&(g=g.then(r).then(()=>{},e=>{_.fresh(()=>{f({level:"error",event:"daemon.lane_job.failed",error:e})})}))}a(F,"enqueue");function Z(){!h||w||(we.check()&&R&&(f({level:"info",event:"daemon.resume.detected"}),k(R().catch(()=>{}))),w=!0,F(async()=>{if(w=!1,!!h)try{await It(Date.now(),K)}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.tick.failed",error:r})})}}))}a(Z,"enqueueTick");let L=null;async function ke(){if(L!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${L instanceof Error?L.message:String(L)}`};if(v.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(v.phase!=="prepared"&&v.phase!=="recovering")throw{status:409,code:"not_prepared",message:v.message};const r=v.phase==="prepared";try{return await H(),v={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw L=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}a(ke,"admitStartup");async function Ee(r){if(I!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${I})`};I="repair";try{const e=await st({dryRun:r.dryRun});if(r.dryRun||e.pending||v.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await H(),v={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(i){const l=i instanceof Error?i.message:String(i);return setTimeout(()=>{P().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:l}}}finally{I=null}}a(Ee,"repairMigrations");async function Re(){o=t.intervalMs??hr;const r=c.token;p=new et({epoch:r,enqueue:F}),Mr(p),v={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{f({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:o}})});const e=qr();if(zr(e),_.fresh(()=>{f({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),_.fresh(()=>{let i;try{i=ur()}catch{return}i!==null&&f({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:i.slice(0,2048)}})}),await _.fresh(async()=>{await fr(),await lr(),pr(Date.now()),mr(Date.now())}),_.fresh(()=>{const{reconcile:i,migrated:l}=Br(Rr);i.changed&&f({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:i.cleared.length,same_boot_proven:i.sameBootProven}}),l.length>0&&f({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:l.length}})}),_.fresh(()=>{for(const i of ne())i.status!=="active"||i.pi_pid!=null||i.frozen_at!=null||i.respawn_not_before!=null||(oe(i.node_id,new Date().toISOString()),f({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:i.node_id}))}),_.fresh(()=>{Lr();for(const i of rt){const l=i.apply();l.changed&&f({level:"info",event:"daemon.boot_migration.applied",fields:{migration:i.id,...l.summary}})}}),k(cr()),R){const i=a(()=>{k(R().catch(()=>{})),k(Ie().then(l=>$?.observe(l)).then(()=>{},l=>{f({level:"warn",event:"daemon.peer.status_failed",error:l})}))},"refresh");k(sr().catch(()=>{})),k(ar().then(({completion:l})=>l).catch(l=>{f({level:"error",event:"daemon.grants.sweep_failed",error:l})})),i(),C=setInterval(i,ir*1e3),C.unref(),G=xe({joined:!0,register:k})}F(()=>{Zr(),Xr()}),d=setInterval(Z,o),Z()}a(Re,"completeStartup");function H(){return ge??=Re().catch(r=>{throw v={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}a(H,"resumeStartup");try{process.on("SIGINT",Q),process.on("SIGTERM",Y),process.on("exit",X),nt(),u!==null&&f({level:"info",event:"daemon.joined_owner.applied",fields:{owner:u.owner,rows:u.rows}}),N=await Oe();const r=c.token;Gr({epoch:r,requestHandover:_e,registerDetached:k,startupBlock:ye,admitStartup:ke,repairMigrations:Ee}),ue(),A=Ye({tcp:t.tcp}),await A.ready;const e=Me()??ie("user").document_port,i=Xe(e,Ze());try{await i.ready,O=i}catch(S){f({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:S instanceof Error?S.message:String(S)}})}const l=ie("user").runtime;l?.issuer?(R=or(l.issuer,k),l.unsandboxed===!0&&($=De({syncGrants:R})),tr(S=>nr().appended(S))):rr(),l?.unsandboxed===!0&&l.listener!==void 0&&f({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),l!==void 0&&l.listener!==void 0&&l.unsandboxed!==!0&&(T=er(l,()=>dr(l.issuer.replace(/\/$/,"")),R),await T.ready);try{await ot()}catch(S){if(I=null,S instanceof it)return v={phase:"blocked",message:S.message,blockerPaths:S.blockerPaths},P;throw S}process.platform==="linux"&&E(x)&&(ze(),await qe("app:terminal"));try{t.prepare?v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await H(),v={phase:"ready",message:"",blockerPaths:[]})}finally{I=null}return P}catch(r){try{await P()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}a(Tt,"runDaemon");var so=Tt;export{q as brokerCommandDeclaresCanvasHome,z as brokerCommandEpoch,so as default,Tt as runDaemon,It as superviseTick,ut as teardownFleet};
+`),process.exit(1)}}a(V,"handleSignal");function Q(){V("SIGINT")}a(Q,"onSigint");function Y(){V("SIGTERM")}a(Y,"onSigterm");function X(){P().catch(()=>{})}a(X,"onExit");function H(r){h&&(g=g.then(r).then(()=>{},e=>{_.fresh(()=>{f({level:"error",event:"daemon.lane_job.failed",error:e})})}))}a(H,"enqueue");function Z(){!h||w||(we.check()&&R&&(f({level:"info",event:"daemon.resume.detected"}),k(R().catch(()=>{}))),w=!0,H(async()=>{if(w=!1,!!h)try{await Dt(Date.now(),K)}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.tick.failed",error:r})})}}))}a(Z,"enqueueTick");let L=null;async function ke(){if(L!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${L instanceof Error?L.message:String(L)}`};if(v.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(v.phase!=="prepared"&&v.phase!=="recovering")throw{status:409,code:"not_prepared",message:v.message};const r=v.phase==="prepared";try{return await j(),v={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw L=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}a(ke,"admitStartup");async function Ee(r){if(I!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${I})`};I="repair";try{const e=await at({dryRun:r.dryRun});if(r.dryRun||e.pending||v.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await j(),v={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(i){const l=i instanceof Error?i.message:String(i);return setTimeout(()=>{P().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:l}}}finally{I=null}}a(Ee,"repairMigrations");async function Re(){o=t.intervalMs??hr;const r=c.token;p=new rt({epoch:r,enqueue:H}),Or(p),v={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{f({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:o}})});const e=zr();if(Wr(e),_.fresh(()=>{f({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),_.fresh(()=>{let i;try{i=ur()}catch{return}i!==null&&f({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:i.slice(0,2048)}})}),await _.fresh(async()=>{await fr(),await lr(),pr(Date.now()),mr(Date.now())}),_.fresh(()=>{const{reconcile:i,migrated:l}=qr(Pr);i.changed&&f({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:i.cleared.length,same_boot_proven:i.sameBootProven}}),l.length>0&&f({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:l.length}})}),_.fresh(()=>{for(const i of ne())i.status!=="active"||i.pi_pid!=null||i.frozen_at!=null||i.respawn_not_before!=null||(oe(i.node_id,new Date().toISOString()),f({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:i.node_id}))}),_.fresh(()=>{Ar();for(const i of tt){const l=i.apply();l.changed&&f({level:"info",event:"daemon.boot_migration.applied",fields:{migration:i.id,...l.summary}})}}),k(cr()),R){const i=a(()=>{k(R().catch(()=>{})),k(Ie().then(l=>$?.observe(l)).then(()=>{},l=>{f({level:"warn",event:"daemon.peer.status_failed",error:l})}))},"refresh");k(sr().catch(()=>{})),k(ar().then(({completion:l})=>l).catch(l=>{f({level:"error",event:"daemon.grants.sweep_failed",error:l})})),i(),C=setInterval(i,ir*1e3),C.unref(),G=xe({joined:!0,register:k})}H(()=>{et(),Zr()}),d=setInterval(Z,o),Z()}a(Re,"completeStartup");function j(){return ge??=Re().catch(r=>{throw v={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}a(j,"resumeStartup");try{process.on("SIGINT",Q),process.on("SIGTERM",Y),process.on("exit",X),ot(),u!==null&&f({level:"info",event:"daemon.joined_owner.applied",fields:{owner:u.owner,rows:u.rows}}),N=await Oe();const r=c.token;Gr({epoch:r,requestHandover:_e,registerDetached:k,startupBlock:ye,admitStartup:ke,repairMigrations:Ee}),ue(),A=Ye({tcp:t.tcp}),await A.ready;const e=Me()??ie("user").document_port,i=Xe(e,Ze());try{await i.ready,F=i}catch(S){f({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:S instanceof Error?S.message:String(S)}})}const l=ie("user").runtime;l?.issuer?(R=or(l.issuer,k),l.unsandboxed===!0&&($=De({syncGrants:R})),tr(S=>nr().appended(S))):rr(),l?.unsandboxed===!0&&l.listener!==void 0&&f({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),l!==void 0&&l.listener!==void 0&&l.unsandboxed!==!0&&(T=er(l,()=>dr(l.issuer.replace(/\/$/,"")),R),await T.ready);try{await it()}catch(S){if(I=null,S instanceof st)return v={phase:"blocked",message:S.message,blockerPaths:S.blockerPaths},P;throw S}process.platform==="linux"&&E(x)&&(ze(),await qe("app:terminal"));try{t.prepare?v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await j(),v={phase:"ready",message:"",blockerPaths:[]})}finally{I=null}return P}catch(r){try{await P()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}a(Ct,"runDaemon");var co=Ct;export{z as brokerCommandDeclaresCanvasHome,W as brokerCommandEpoch,co as default,Ct as runDaemon,Dt as superviseTick,ht as teardownFleet};

package/dist/daemon/reconcilers/broker-supervision.d.ts

@@ -61,7 +61,6 @@ export declare function treeHasDescendants(psOutput: string, rootPid: number): b
 export declare class BrokerSupervisionReconciler {
     private readonly yieldStalls;
     private readonly wedgeNotifiedAt;
-    private readonly fatalFaultNotifiedAt;
     private readonly unattendedSince;
     private readonly sampleTree;
     private readonly signal;
@@ -93,6 +92,11 @@ export declare class BrokerSupervisionReconciler {
     private signalYieldStall;
     private readTreeSample;
     private handleWedgeDetection;
-    private handleFatalFault;
+    /** Publish a fatal provider fault's notice once per episode and, for a node that settles on it, end the node.
+     *  The notices, the `notified_since` mark and the terminal transition commit in one write, so a daemon exit
+     *  leaves the fault either fully announced or fully unannounced. The daemon registers this as the announcer
+     *  every clear of an unannounced fatal fault runs first with `settle` false: a clear ends the fault on purpose
+     *  (a revive, a completed turn), so it announces but does not end the node. */
+    handleFatalFault(id: string, settle?: boolean): void;
 }
 export {};

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var Y=Object.defineProperty;var m=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as $,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as I}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{readFault as O,readLiveTurn as E,recordFault as ee}from"../../core/canvas/recovery.js";import{errorClassFromFault as te}from"../../core/fault-classifier.js";import{nextFaultRetry as ne}from"../../core/runtime/fault-recovery.js";import{publishNotice as C}from"../../core/runtime/close.js";import{openDb as R}from"../../core/canvas/db.js";import{getById as re}from"../../core/graph/objects.js";import{watcherIdsOf as ie}from"../../core/graph/watches.js";import{headlessBrokerHost as b}from"../../core/runtime/host.js";import{emitEvent as p}from"../../core/events/emit.js";import{operationIdContext as L}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as ae,resetStaleExecutionMail as se}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as oe}from"../../core/bash-jobs.js";import{focusOf as le}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as de}from"../api/viewer-presence.js";import{deliverLive as ce}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as ue}from"../../core/runtime/broker-request.js";import{formatCard as fe,PARK_SUMMARY_PROMPT as pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as me}from"../../core/runtime/fleet.js";import{readConfig as he}from"../../core/config.js";import{beginPendingPark as _e,consumePendingPark as S,pendingParkIds as ge,pendingParkStartedAt as ke}from"../park-pending.js";import{shouldParkAfterSummary as ve}from"../park-activity.js";import{applyParkEvent as N,parkEvent as ye}from"./live-obligation.js";import{memoryDeniedRecently as be}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as we,envTestUnattendedParkMs as Se,parkSummaryGraceMs as Ne,envUnattendedParkMs as Pe}from"../../shared/env.js";const A=j(z);function $e(s){return E(s)?.busy===!0}m($e,"liveBusy");function gt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(gt,"liveTurnProgressMs");function G(s,e){return e===null?null:`${s}:${e}`}m(G,"incarnationOf");function F(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}m(F,"busySinceForIncarnation");const W=3*6e4,B=2e4,x=20*6e4,Ae=2,Me=5*6e4;function Te(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}m(Te,"settlesOnFatalProviderFault");function Ee(){const s=Pe();if(s!==void 0)return s;const e=he("user").lifecycle.unattendedParkMs;return we()?Se()??e:e}m(Ee,"unattendedParkMsForDaemon");function U(){return Ne()}m(U,"parkSummaryGraceMsForDaemon");function Fe(s,e,t,a){return s!==!0||e!=="refresh"||t?"leave":a===null||a<W?"pending":"kill"}m(Fe,"yieldStallVerdict");function xe(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ae?"pending":"wedged"}m(xe,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const a of s.split(`
-`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),r=Number(n[1]),l=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(r)||!Number.isFinite(l))continue;e.set(o,l);const u=t.get(r);u===void 0?t.set(r,[o]):u.push(o)}return{byPid:e,childrenOf:t}}m(q,"parsePsTree");function Ie(s,e){const{byPid:t,childrenOf:a}=q(s);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const r=n.pop();if(o.has(r))continue;o.add(r),i+=t.get(r)??0;const l=a.get(r);l!==void 0&&n.push(...l)}return i}m(Ie,"sumTreeCpu");function De(s,e){const{byPid:t,childrenOf:a}=q(s);return t.has(e)?(a.get(e)??[]).length>0:null}m(De,"treeHasDescendants");class kt{static{m(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&H(K)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return V({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return me()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=L.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Ee();for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n);if(o===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await L.fresh(async()=>{try{await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,$(n),o.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),p({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ge())t.fleet.has(i)||S(i);se(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||oe(X(e)).length>0||le(e)!==null||de(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ne(n,i)!==null)?!0:ae(e)}handleUnattendedParking(e,t,a,i,n,o){if(J(e)){this.unattendedSince.delete(e),S(e);return}if(t===null){this.unattendedSince.delete(e),S(e);return}const r=this.parkResetCondition(e,t,a,i),l=ke(e);if(l!==void 0){if(i-l<U())return;const _=S(e);if(_===void 0||(this.unattendedSince.delete(e),!ve(e,t,_)))return;p({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:U()}}),N(e,"park"),b.teardown(e);return}if(r||$e(e)){this.unattendedSince.delete(e);return}const u=this.unattendedSince.get(e);if(u===void 0){this.unattendedSince.set(e,i);return}const d=ye(e,t.lifecycle),h=d!=="cancel"&&be(t.grantee??"app:terminal",i)?0:o;if(!(i-u<h)){if(this.unattendedSince.delete(e),d==="release"){N(e,d),b.teardown(e);return}if(d==="cancel"){N(e,d),b.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){N(e,"park"),b.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(_e(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await ce(e,fe("park",{},pe),{origin:{type:"runtime",kind:"park"}});p({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(S(e)===void 0)return;if(t instanceof ue){N(e,"park"),b.teardown(e);return}p({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Me)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,r=a.stalled+a.waitingOnMail;p({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const l=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,u=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${l} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async h=>{if(h!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",h,u],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",h,u],{encoding:"utf8",timeout:2e3})}catch(_){p({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){p({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,o=G(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||o===null){o!==null&&this.yieldStalls.delete(o);return}let r=this.yieldStalls.get(o);if(r===void 0){r={since:i},this.yieldStalls.set(o,r);return}if(Fe(!0,e.intent,!1,i-r.since)!=="kill")return;const l=$(n);if(!(a.get(n)!==t||l?.pi_pid!==t.pid||l.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,p({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:W}})}else i-r.termedAt>=B&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&p({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:B,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return p({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=Ie(t,e),i=De(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,o=G(t.pid,e.pi_pid_identity),r=F(e);if(r===null||o===null){o!==null&&this.wedgeNotifiedAt.delete(o);return}const l=i-r,u=l>=x?await this.sampleTree(t.pid):null,d=$(n),h=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==r)return;const _=u?.cpuPercent??null,P=u?.hasDescendants??null;if(xe({busy:!0,quietForMs:l,cpuPercent:_,hasDescendants:P})!=="wedged"||this.wedgeNotifiedAt.get(o)===r)return;this.wedgeNotifiedAt.set(o,r);const g=I(d),k=Math.round((l??x)/6e4),w=P??!0,f=!w&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let v=!1,M=!1,y;if(w)y=`Child wedged \u2014 ${g} (${n}) is alive and mid-turn but has shown NO engine progress for ${k}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(f!==null)y=`Child wedged \u2014 ${g} (${n}) is alive and mid-turn but has shown NO engine progress for ${k}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{v=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,y=`Child wedged \u2014 ${g} (${n}) is alive and mid-turn but has shown NO engine progress for ${k}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){p({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:T,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),y=`Child wedged \u2014 ${g} (${n}) is alive and mid-turn but has shown NO engine progress for ${k}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}p({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:l,cpu_percent:_,has_descendants:w,kick_attempted:v,kicked:M,...f===null?{}:{kick_suppression_reason:f}}});try{ee(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:y})}catch(T){p({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}C(n,"notice.crash",y,{reason:"child-wedged",child:n,quiet_ms:l,cpu_percent:_,kick_attempted:v,kicked:M,...f===null?{}:{kick_suppression_reason:f}})}handleFatalFault(e){const t=O(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=$(e),i=a!==null?I(a):e,n=t.kind==="auth",o=(t.message.split(`
-`)[0]??"").slice(0,200),r=t.providerError,l=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),u=t.kind==="context-overflow",d=Te(a,typeof r?.code=="string"?r.code:void 0),h=u||d,_=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${l}`,P=u?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),k=ie(R(),re(R(),e)),w=[{relation:"Child",recipients:k.filter(f=>g.has(f))},{relation:"Watched node",recipients:k.filter(f=>!g.has(f))}];this.fatalFaultNotifiedAt.set(e,t.since),p({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:u,terminal_provider_fault:d}});for(const{relation:f,recipients:v}of w)v.length!==0&&C(e,"notice.crash",`${f} ${P} \u2014 ${i} (${e}) ${_}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:o,...r===void 0?{}:{provider_error:r}},"normal",null,v);h&&(Z(e,"crash",{reason:u?"context_overflow":r?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:te(t).class}}}),b.teardown(e).then(f=>{f.confirmed||p({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>p({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f})))}}export{kt as BrokerSupervisionReconciler,Ae as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,Te as settlesOnFatalProviderFault,Ie as sumTreeCpu,De as treeHasDescendants,Ee as unattendedParkMsForDaemon,xe as wedgeVerdict,Fe as yieldStallVerdict};
+var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{listUnnotifiedFatalFaultNodes as ee,markFaultNotified as te,readFault as O,readLiveTurn as E,recordFault as ne}from"../../core/canvas/recovery.js";import{errorClassFromFault as re}from"../../core/fault-classifier.js";import{nextFaultRetry as ie}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as L,withCanvasWrite as ae}from"../../core/canvas/db.js";import{getById as se}from"../../core/graph/objects.js";import{watcherIdsOf as oe}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as le,resetStaleExecutionMail as de}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ce}from"../../core/bash-jobs.js";import{focusOf as ue}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as fe}from"../api/viewer-presence.js";import{deliverLive as pe}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as me}from"../../core/runtime/broker-request.js";import{formatCard as he,PARK_SUMMARY_PROMPT as _e}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ge}from"../../core/runtime/fleet.js";import{readConfig as ke}from"../../core/config.js";import{beginPendingPark as ve,consumePendingPark as N,pendingParkIds as ye,pendingParkStartedAt as be}from"../park-pending.js";import{shouldParkAfterSummary as we}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Se}from"./live-obligation.js";import{memoryDeniedRecently as Ne}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Pe,envTestUnattendedParkMs as $e,parkSummaryGraceMs as Te,envUnattendedParkMs as Me}from"../../shared/env.js";const M=j(z);function Ae(s){return E(s)?.busy===!0}p(Ae,"liveBusy");function yt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(yt,"liveTurnProgressMs");function G(s,e){return e===null?null:`${s}:${e}`}p(G,"incarnationOf");function I(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(I,"busySinceForIncarnation");const W=3*6e4,U=2e4,x=20*6e4,Ee=2,Fe=5*6e4;function Ie(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Ie,"settlesOnFatalProviderFault");function xe(){const s=Me();if(s!==void 0)return s;const e=ke("user").lifecycle.unattendedParkMs;return Pe()?$e()??e:e}p(xe,"unattendedParkMsForDaemon");function B(){return Te()}p(B,"parkSummaryGraceMsForDaemon");function Ce(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Ce,"yieldStallVerdict");function De(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ee?"pending":"wedged"}p(De,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Oe(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Oe,"sumTreeCpu");function Re(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(Re,"treeHasDescendants");class bt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ge()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=xe();for(const i of ee())await F.fresh(async()=>{try{this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),this.handleUnattendedParking(n,T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ye())t.fleet.has(i)||N(i);de(e)}parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ce(X(e)).length>0||ue(e)!==null||fe(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ie(n,i)!==null)?!0:le(e)}handleUnattendedParking(e,t,r,i,n,l){if(J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=this.parkResetCondition(e,t,r,i),a=be(e);if(a!==void 0){if(i-a<B())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!we(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:B()}}),P(e,"park"),S.teardown(e);return}if(o||Ae(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=Se(e,t.lifecycle),h=d!=="cancel"&&Ne(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){P(e,d),S.teardown(e);return}if(d==="cancel"){P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ve(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await pe(e,he("park",{},_e),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof me){P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Fe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await M("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
+`).map(async h=>{if(h!=="")try{await M("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await M("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||I(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Ce(!0,e.intent,!1,i-o.since)!=="kill")return;const a=T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=U&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:U,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await M("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Oe(t,e),i=Re(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity),o=I(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=x?await this.sampleTree(t.pid):null,d=T(n),h=d===null?null:I(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(De({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const v=C(d),g=Math.round((a??x)/6e4),y=$??!0,k=!y&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,b=!1,w;if(y)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(k!==null)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),b=!0,w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(A){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:A,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:y,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}}});try{ne(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:w})}catch(A){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:A})}R(n,"notice.crash",w,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}})}handleFatalFault(e,t=!0){const r=O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
+`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=Ie(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,v=d?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),y=oe(L(),se(L(),e)),k=[{relation:"Child",recipients:y.filter(f=>g.has(f))},{relation:"Watched node",recipients:y.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),ae(()=>{for(const{relation:f,recipients:b}of k)b.length!==0&&R(e,"notice.crash",`${f} ${v} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,b);te(r.faultId),t&&_&&Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:re(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{bt as BrokerSupervisionReconciler,Ee as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,B as parkSummaryGraceMsForDaemon,Ie as settlesOnFatalProviderFault,Oe as sumTreeCpu,Re as treeHasDescendants,xe as unattendedParkMsForDaemon,De as wedgeVerdict,Ce as yieldStallVerdict};

package/dist/pi-extensions/canvas-bash-valve.js

@@ -1,7 +1,7 @@
-var ce=Object.defineProperty;var r=(t,n)=>ce(t,"name",{value:n,configurable:!0});import{envBashAutoBgDeadlineMs as ue,envBashValveMs as fe,envNodeId as le}from"../shared/env.js";import{spawn as be}from"node:child_process";import{closeSync as he,existsSync as B,mkdirSync as q,openSync as pe,readFileSync as X,readSync as me,rmSync as Y,statSync as Q,writeFileSync as j}from"node:fs";import{homedir as ge}from"node:os";import{dirname as we}from"node:path";import{performance as P}from"node:perf_hooks";import{backgroundBashJob as Z,bashJobPaths as ye,cancelBashJobProcessTree as je,countJobLogLines as Ie,formatBashElapsed as ee,newBashJobId as Se,normalizeBashJobPurpose as Ee,readBashNiceness as ve,tailJobLog as Be,writeJobDeadlineMs as _e}from"../core/bash-jobs.js";import{BASH_JOB_SUPERVISOR_SCRIPT as Le}from"../core/bash-job-supervisor.js";import{MAX_BACKGROUND_BASH_LOG_BYTES as gt,BACKGROUND_BASH_LOG_TRUNCATION_MARKER as wt}from"../core/bash-job-supervisor.js";import{recordBackgroundedJob as xe,startBrokerBashJob as Re,stopBrokerBashJob as ke}from"../core/runtime/broker/daemon-ops.js";import{bashCommandRefusal as Te}from"../core/bash-guard.js";import{capturePidIdentities as Oe}from"../core/canvas/pid.js";import{readConfig as $e}from"../core/config.js";import{PREVIEW_RESULT_PATH_ENV as Ae,previewResultPath as Pe}from"../core/preview-result-path.js";import{resolveAdmittedHostCommand as Ce}from"../core/runtime/spawn-env.js";import{freshChildNodeExecPath as Ne}from"../core/runtime/branded-host.js";import{Type as C}from"typebox";import{createBashToolDefinition as te}from"@earendil-works/pi-coding-agent";const Je=200;function Me(){return fe()}r(Me,"resolveValveMs");function De(t){return ye(t,Se())}r(De,"makeJobPaths");const _=new Set;let ne=!1;function oe(){for(const t of[..._])try{Z(t),_.delete(t)}catch(n){process.stderr.write(`crtr: failed to background bash job ${t.jobId} during broker termination: ${n.message}
-`)}}r(oe,"backgroundLiveJobsForTermination");function He(){ne||(ne=!0,process.prependListener("SIGTERM",oe),process.platform!=="win32"&&process.prependListener("SIGHUP",oe))}r(He,"installTerminationHandoff");const Ue=64*1024;function N(t,n,i){let s;try{s=Q(t)}catch{return n}if(s.size<=n)return n;const u=pe(t,"r");try{for(;n<s.size;){const m=Math.min(Ue,s.size-n),g=Buffer.allocUnsafe(m),d=me(u,g,0,m,n);if(d===0)break;i(g.subarray(0,d)),n+=d}return n}finally{he(u)}}r(N,"tailOnce");function p(t){try{Y(t,{recursive:!0,force:!0})}catch{}}r(p,"cleanupJobDir");function Fe(t){try{const n=X(t,"utf8").trim(),i=Number(n);return Number.isFinite(i)?i:null}catch{return null}}r(Fe,"readExitCode");function Ge(t){try{if(Q(t).isDirectory())return{cwd:t}}catch{}const n=ge();return{cwd:n,warning:`[working directory no longer exists: ${t}; running from ${n}]
-`}}r(Ge,"resolveExecutionCwd");function re(t,n,i=()=>null,s,u=()=>{}){return{exec:r(async(m,g,{onData:d,signal:b,timeout:w,env:T})=>{const O=await Ne();return new Promise((L,c)=>{const $=i();if(b?.aborted){c(new Error("aborted"));return}const x=Te(m,g);if(x!==null){c(new Error(x));return}const e=De(n);q(e.dir,{recursive:!0}),j(e.cmdSh,m),j(e.jobLog,""),j(e.jobRun,""),$!==null&&j(e.jobPurpose,$);let h=0,R=!1,f,y,I,J=!1;const M=process.platform==="linux",D=P.now(),ie=Me();let S;const se=r(()=>{S&&clearInterval(S),b&&b.removeEventListener("abort",G),_.delete(e)},"teardown"),l=r(o=>{R||(R=!0,se(),o())},"finish");let H=!1;function A(o=!1){R||H||(H=!0,S&&clearInterval(S),xe(t,e.jobId,{command:X(e.cmdSh,"utf8"),job_dir:e.dir,log_path:e.jobLog}).then(()=>U(o,""),a=>U(o,`
-[crtr could not record job ${e.jobId} (${a.message}); it will NOT message you when it exits. Watch its log.]`)))}r(A,"finishBackground");function U(o,a){const de=ee(P.now()-D);let K="";if(o){const W=ue();_e(e,Date.now()+W),K=`It will be CANCELED in ${ee(W)} unless you claim it: crtr node bash extend ${e.jobId} --for <duration> (or --keep to run to completion). `}const z=`
-[backgrounded as job ${e.jobId} after ${de} in the foreground${o?" (auto)":""}. `+K+(a===""?"It will interrupt you with a message when it exits. ":"")+`Watch: tail -f ${e.jobLog} ; stop: crtr node bash kill ${e.jobId}]`;d(Buffer.from(z+a)),u({jobId:e.jobId,logPath:e.jobLog,notice:z+a}),l(()=>L({exitCode:0}))}r(U,"handOff");function F(){h=N(e.jobLog,h,d);const o=Fe(e.jobExit);l(()=>L({exitCode:o})),p(e.dir)}r(F,"finishForeground");function ae(o=!1){const a=Z(e);if(a==="backgrounded"||a==="already-backgrounded"){A(o);return}if(a==="finished"){F();return}l(()=>c(new Error(`bash job ${e.jobId} disappeared before it could be backgrounded`))),p(e.dir)}r(ae,"requestBackground");function k(o){if(!(R||J)){if(h=N(e.jobLog,h,d),B(e.jobBg)){A();return}if(!B(e.jobExit)){if(f===void 0){I??=o;return}if(M){J=!0,ke(t,e.jobId).then(()=>{l(()=>c(o)),p(e.dir)},a=>l(()=>c(a)));return}f!==void 0&&y!==void 0&&je(f,y),l(()=>c(o)),p(e.dir)}}}r(k,"requestStop");function G(){k(new Error("aborted"))}r(G,"onAbort"),b&&b.addEventListener("abort",G,{once:!0});const E=Ge(g);E.warning&&d(Buffer.from(E.warning));let v=T??process.env;s!==void 0&&(q(we(s),{recursive:!0}),Y(s,{force:!0}),v={...v,[Ae]:s});const V=Ce("bash",v,E.cwd);if(M)Re(t,{job_id:e.jobId,cwd:E.cwd,env:v}).then(({pid:o})=>{f=o,y=`launcher:${o}`,_.add(e),I!==void 0&&k(I)},o=>{l(()=>c(o)),p(e.dir)});else{const o=be(V,["-c",Le,"--",e.cmdSh,e.jobLog,e.jobExit,e.jobRun,e.jobDone,V,String(ve(n)),O],{cwd:E.cwd,detached:!0,stdio:"ignore",env:v});if(o.on("error",a=>{l(()=>c(a)),p(e.dir)}),f=o.pid,f!==void 0){if(y=Oe([f])?.get(f),y===void 0){try{process.kill(-f,"SIGKILL")}catch{}l(()=>c(new Error(`bash job ${e.jobId} started without a process identity`))),p(e.dir);return}j(e.jobPgid,String(f)),j(e.jobPgidIdentity,y),_.add(e),I!==void 0&&k(I)}o.unref()}S=setInterval(()=>{if(h=N(e.jobLog,h,d),B(e.jobDone)){F();return}if(B(e.jobBg)){A();return}if(B(e.jobExit))return;const o=P.now()-D;if(w!==void 0&&w>0){o>=w*1e3&&k(new Error(`timeout:${w}`));return}o>=ie&&ae(!0)},Je)})},"exec")}}r(re,"createValveOperations");const Ve=["One short clause naming what this step is FOR, in the words of the person you",'are working for \u2014 "Reading the Acme thread", not "Fetching emails" and never','"Running GMAIL_FETCH_EMAILS". A person reads this while the command runs; it',"is the only thing they see of this step. Under about eight words. Omit it for","steps that only touch your own scratch work."].join(" "),Ke=3;function ze(t){const n=Be(t.logPath,Ke),i=Math.max(0,Ie(t.logPath)-n.length),s=n.length===0?"(no output yet)":n.join(`
-`),u=i===0?"":`
-[${i} earlier output line${i===1?"":"s"} are in the job log only]`;return{content:[{type:"text",text:`${s}${u}${t.notice}`}],details:void 0}}r(ze,"handoffResult");function We(t){return{...t,parameters:C.Object({purpose:C.Optional(C.String({description:Ve})),...t.parameters.properties})}}r(We,"withPurpose");function qe(t,n,i,s){const u=te(i,{operations:re(t,n)});return{...s?We(u):u,execute:r(async(d,b,w,T,O)=>{const L=s?Ee(b.purpose):null;let c=null;const x=await te(i,{operations:re(t,n,()=>L,Pe(d),h=>{c=h})}).execute(d,b,w,T,O),e=c;return e===null?x:ze(e)},"execute")}}r(qe,"createValveToolDefinition");function Xe(t){const n=le();if(n===void 0||n.trim()==="")return;const i=process.env.CRTR_CONTEXT_DIR;i===void 0||i.trim()===""||(He(),t.on("session_start",(s,u)=>{t.registerTool(qe(n,i,u.cwd,$e("user").bash_tool_purpose))}))}r(Xe,"default");export{wt as BACKGROUND_BASH_LOG_TRUNCATION_MARKER,gt as MAX_BACKGROUND_BASH_LOG_BYTES,re as createValveOperations,qe as createValveToolDefinition,Xe as default};
+var ae=Object.defineProperty;var o=(t,n)=>ae(t,"name",{value:n,configurable:!0});import{envBashAutoBgDeadlineMs as ce,envBashValveMs as ue,envNodeId as de}from"../shared/env.js";import{closeSync as le,existsSync as j,mkdirSync as z,openSync as fe,readFileSync as K,readSync as he,rmSync as W,statSync as q,writeFileSync as S}from"node:fs";import{homedir as be}from"node:os";import{dirname as me}from"node:path";import{performance as J}from"node:perf_hooks";import{backgroundBashJob as X,bashJobPaths as pe,cancelBashJobProcessTree as ge,countJobLogLines as we,formatBashElapsed as Y,newBashJobId as ye,normalizeBashJobPurpose as je,tailJobLog as ve,writeJobDeadlineMs as Ee}from"../core/bash-jobs.js";import{MAX_BACKGROUND_BASH_LOG_BYTES as ct,BACKGROUND_BASH_LOG_TRUNCATION_MARKER as ut}from"../core/bash-job-supervisor.js";import{recordBackgroundedJob as _e,stopBrokerBashJob as Se}from"../core/runtime/broker/daemon-ops.js";import{BASH_JOBS_USE_LAUNCHER as Be,launchBashJobSupervisor as Ie}from"../core/runtime/broker/bash-job-launch.js";import{bashCommandRefusal as Le}from"../core/bash-guard.js";import{readConfig as ke}from"../core/config.js";import{PREVIEW_RESULT_PATH_ENV as Ae,previewResultPath as Oe}from"../core/preview-result-path.js";import{Type as B}from"typebox";import{createBashToolDefinition as Q}from"@earendil-works/pi-coding-agent";const xe=200;function Re(){return ue()}o(Re,"resolveValveMs");function Te(t){return pe(t,ye())}o(Te,"makeJobPaths");const I=new Set;let Z=!1;function ee(){for(const t of[...I])try{X(t),I.delete(t)}catch(n){process.stderr.write(`crtr: failed to background bash job ${t.jobId} during broker termination: ${n.message}
+`)}}o(ee,"backgroundLiveJobsForTermination");function $e(){Z||(Z=!0,process.prependListener("SIGTERM",ee),process.platform!=="win32"&&process.prependListener("SIGHUP",ee))}o($e,"installTerminationHandoff");const Je=64*1024;function N(t,n,i){let s;try{s=q(t)}catch{return n}if(s.size<=n)return n;const c=fe(t,"r");try{for(;n<s.size;){const b=Math.min(Je,s.size-n),m=Buffer.allocUnsafe(b),a=he(c,m,0,b,n);if(a===0)break;i(m.subarray(0,a)),n+=a}return n}finally{le(c)}}o(N,"tailOnce");function v(t){try{W(t,{recursive:!0,force:!0})}catch{}}o(v,"cleanupJobDir");function Ne(t){try{const n=K(t,"utf8").trim(),i=Number(n);return Number.isFinite(i)?i:null}catch{return null}}o(Ne,"readExitCode");function Ce(t){try{if(q(t).isDirectory())return{cwd:t}}catch{}const n=be();return{cwd:n,warning:`[working directory no longer exists: ${t}; running from ${n}]
+`}}o(Ce,"resolveExecutionCwd");function te(t,n,i=()=>null,s,c=()=>{}){return{exec:o(async(b,m,{onData:a,signal:f,timeout:p,env:L})=>new Promise((E,d)=>{const w=i();if(f?.aborted){d(new Error("aborted"));return}const k=Le(b,m);if(k!==null){d(new Error(k));return}const e=Te(n);z(e.dir,{recursive:!0}),S(e.cmdSh,b),S(e.jobLog,""),S(e.jobRun,""),w!==null&&S(e.jobPurpose,w);let l=0,g=!1,_,A,O,C=!1;const ne=Be,M=J.now(),oe=Re();let y;const re=o(()=>{y&&clearInterval(y),f&&f.removeEventListener("abort",H),I.delete(e)},"teardown"),h=o(r=>{g||(g=!0,re(),r())},"finish");let D=!1;function x(r=!1){g||D||(D=!0,y&&clearInterval(y),_e(t,e.jobId,{command:K(e.cmdSh,"utf8"),job_dir:e.dir,log_path:e.jobLog}).then(()=>P(r,""),u=>P(r,`
+[crtr could not record job ${e.jobId} (${u.message}); it will NOT message you when it exits. Watch its log.]`)))}o(x,"finishBackground");function P(r,u){const se=Y(J.now()-M);let F="";if(r){const V=ce();Ee(e,Date.now()+V),F=`It will be CANCELED in ${Y(V)} unless you claim it: crtr node bash extend ${e.jobId} --for <duration> (or --keep to run to completion). `}const G=`
+[backgrounded as job ${e.jobId} after ${se} in the foreground${r?" (auto)":""}. `+F+(u===""?"It will interrupt you with a message when it exits. ":"")+`Watch: tail -f ${e.jobLog} ; stop: crtr node bash kill ${e.jobId}]`;a(Buffer.from(G+u)),c({jobId:e.jobId,logPath:e.jobLog,notice:G+u}),h(()=>E({exitCode:0}))}o(P,"handOff");function U(){l=N(e.jobLog,l,a);const r=Ne(e.jobExit);h(()=>E({exitCode:r})),v(e.dir)}o(U,"finishForeground");function ie(r=!1){const u=X(e);if(u==="backgrounded"||u==="already-backgrounded"){x(r);return}if(u==="finished"){U();return}h(()=>d(new Error(`bash job ${e.jobId} disappeared before it could be backgrounded`))),v(e.dir)}o(ie,"requestBackground");function R(r){if(!(g||C)){if(l=N(e.jobLog,l,a),j(e.jobBg)){x();return}if(!j(e.jobExit)){if(_===void 0){O??=r;return}if(ne){C=!0,Se(t,e.jobId).then(()=>{h(()=>d(r)),v(e.dir)},u=>h(()=>d(u)));return}_!==void 0&&A!==void 0&&ge(_,A),h(()=>d(r)),v(e.dir)}}}o(R,"requestStop");function H(){R(new Error("aborted"))}o(H,"onAbort"),f&&f.addEventListener("abort",H,{once:!0});const T=Ce(m);T.warning&&a(Buffer.from(T.warning));let $=L??process.env;s!==void 0&&(z(me(s),{recursive:!0}),W(s,{force:!0}),$={...$,[Ae]:s}),Ie(t,e,n,T.cwd,$).then(r=>{_=r.pgid,A=r.identity,I.add(e),O!==void 0&&R(O)},r=>{h(()=>d(r)),v(e.dir)}),y=setInterval(()=>{if(l=N(e.jobLog,l,a),j(e.jobDone)){U();return}if(j(e.jobBg)){x();return}if(j(e.jobExit))return;const r=J.now()-M;if(p!==void 0&&p>0){r>=p*1e3&&R(new Error(`timeout:${p}`));return}r>=oe&&ie(!0)},xe)}),"exec")}}o(te,"createValveOperations");const Me=["One short clause naming what this step is FOR, in the words of the person you",'are working for \u2014 "Reading the Acme thread", not "Fetching emails" and never','"Running GMAIL_FETCH_EMAILS". A person reads this while the command runs; it',"is the only thing they see of this step. Under about eight words. Omit it for","steps that only touch your own scratch work."].join(" "),De=3;function Pe(t){const n=ve(t.logPath,De),i=Math.max(0,we(t.logPath)-n.length),s=n.length===0?"(no output yet)":n.join(`
+`),c=i===0?"":`
+[${i} earlier output line${i===1?"":"s"} are in the job log only]`;return{content:[{type:"text",text:`${s}${c}${t.notice}`}],details:void 0}}o(Pe,"handoffResult");function Ue(t){return{...t,parameters:B.Object({purpose:B.Optional(B.String({description:Me})),...t.parameters.properties})}}o(Ue,"withPurpose");const He=["A command that needs longer than one call \u2014 a test lane, a build, a soak \u2014 is started with","`crtr node bash run '<command>'`, which returns at once and wakes you with the exit code; the job is a canvas","object others can watch. Do not use `&`, `nohup`, `setsid`, or a cron that polls a log. A foreground command","past 5 minutes is moved to a job automatically; a `timeout` kills it and everything it started."].join(" ");function Fe(t){const{command:n,...i}=t.parameters.properties;return{...t,parameters:B.Object({command:{...n,description:`${n.description??""} ${He}`.trim()},...i})}}o(Fe,"withJobGuidance");function Ge(t,n,i,s){const c=Fe(Q(i,{operations:te(t,n)}));return{...s?Ue(c):c,execute:o(async(a,f,p,L,E)=>{const d=s?je(f.purpose):null;let w=null;const e=await Q(i,{operations:te(t,n,()=>d,Oe(a),g=>{w=g})}).execute(a,f,p,L,E),l=w;return l===null?e:Pe(l)},"execute")}}o(Ge,"createValveToolDefinition");function Ve(t){const n=de();if(n===void 0||n.trim()==="")return;const i=process.env.CRTR_CONTEXT_DIR;i===void 0||i.trim()===""||($e(),t.on("session_start",(s,c)=>{t.registerTool(Ge(n,i,c.cwd,ke("user").bash_tool_purpose))}))}o(Ve,"default");export{ut as BACKGROUND_BASH_LOG_TRUNCATION_MARKER,ct as MAX_BACKGROUND_BASH_LOG_BYTES,te as createValveOperations,Ge as createValveToolDefinition,Ve as default};

package/docs/sdk/release-notes.md

@@ -7,6 +7,12 @@ description: Behavior changes in @north-light/sdk that an app must act on, newes
 
 Changes to `@north-light/sdk` that can change what an existing app sees. Newest first.
 
+## 0.3.527
+
+`POST /v1/nodes/:id/jobs/start` accepts a job prepared as already backgrounded. Additive: no request or response an existing app sends or reads changes.
+
+- The route used to refuse a job directory without `job.run` (404 `job_not_found`). It now also accepts one with `job.bg`, which `crtr node bash run` writes before launching. A job directory with neither marker is still refused.
+
 ## 0.3.521
 
 `GET /v1/status` names the runtime generation the daemon is serving. Additive: no request or response an existing app sends or reads changes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.525",
+  "version": "0.3.527",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -139,7 +139,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.525",
+    "@north-light/identity": "0.3.527",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.525",
+  "version": "0.3.527",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",