@north-light/cli 0.3.414 → 0.3.416

package/dist/daemon/cron/authority.js

@@ -0,0 +1 @@
+var c=Object.defineProperty;var o=(r,e)=>c(r,"name",{value:e,configurable:!0});import{covers as g}from"@north-light/identity";import{activeRecord as u,grantRecord as l}from"../../core/grants/mirror.js";import{readConfig as d}from"../../core/config.js";import{SCOPE_VOCABULARY as f,scopeAllowed as i}from"../../core/scopes.js";function n(r,e){return{allowed:!1,reason:r,message:e}}o(n,"refuse");function _(r){let e;if(r.grant_id===null){if(r.grantee!=="app:terminal"||d("user").runtime?.issuer)return n("grant_missing",`cron has no grant for ${r.grantee}; re-arm it from a node of that app`);e=f}else{const t=l(r.grant_id);if(t===null)return n("grant_missing",`grant ${r.grant_id} is not in this runtime's grant mirror`);if(t.status!=="active")return n("grant_revoked",`grant ${r.grant_id} is ${t.status}`);if(t.expires_at!==null&&Date.parse(t.expires_at)<=Date.now())return n("grant_revoked",`grant ${r.grant_id} expired at ${t.expires_at}`);if(t.grantee!==r.grantee||t.sub!==r.sub)return n("grant_revoked",`grant ${r.grant_id} no longer belongs to ${r.grantee} for ${r.sub}`);const s=u(t);if(s===null)return n("grant_revoked",`grant ${r.grant_id} cannot be read`);e=s.resolved}if(r.scopes!==null){const t=r.scopes.find(s=>!e.some(a=>g(a,s)));if(t!==void 0)return n("scope_narrowed",`scope ${t} is no longer covered by the grant`)}return!i(e,"crtr:schedule")||!i(r.scopes,"crtr:schedule")?n("scope_missing","the cron no longer holds crtr:schedule"):{allowed:!0,effectiveScopes:e}}o(_,"checkCronFire");function b(r){const e=_(r);return e.allowed&&(!i(e.effectiveScopes,"crtr:act")||!i(r.scopes,"crtr:act"))?n("scope_missing","the cron no longer holds crtr:act"):e}o(b,"checkCronBirth");function w(r){return{grantee:r.grantee,sub:r.sub,grant_id:r.grant_id,scopes:r.scopes}}o(w,"cronPrincipalOf");export{b as checkCronBirth,_ as checkCronFire,w as cronPrincipalOf};

package/dist/daemon/cron/disposition.d.ts

@@ -16,6 +16,8 @@ export interface Disposition {
     delivered: string;
     /** A paused row is retained so `cron resume` can re-arm it. */
     paused: boolean;
+    /** The sink refused the birth: the run did not deliver, so the on-change baseline must not move. */
+    refused?: boolean;
     obligation?: CronSinkResolution;
 }
 export declare function disposeSettledRun(c: Cron, ctx: RunSettleContext, outOfBand: boolean, selfCanceled?: boolean): Promise<Disposition>;

package/dist/daemon/cron/disposition.js

@@ -1 +1 @@
-var p=Object.defineProperty;var l=(n,e)=>p(n,"name",{value:e,configurable:!0});import{getCron as v,setCronLastOutputHash as c}from"../../core/canvas/crons.js";import{escalateFailure as u,failureSummary as h}from"./escalation.js";import{resolveCronSink as m}from"./sinks.js";async function w(n,e,t,s=!1){const r=t?"out-of-band: ":"",a=v(n.cron_id);if(a===null&&!s)return{delivered:`${r}cron canceled mid-run \u2014 nothing delivered`,paused:!1};const d=a??n,f=e.timedOut||e.processErrorMessage!==null||e.exitCode!==0;let i=!0;if(d.on_output==="on-change"&&(i=d.last_output_hash!==e.stdoutSha256,!t&&a!==null&&c(n.cron_id,e.stdoutSha256)),f)return d.on_output==="silent"?{delivered:`${r}silent`,paused:!1}:t?{delivered:`out-of-band: ${h(e,null)} \u2014 not escalated`,paused:!1}:{delivered:await u(d,e,null),paused:!0};switch(d.on_output){case"silent":return{delivered:`${r}silent`,paused:!1};case"on-failure":return{delivered:`${r}ok \u2014 nothing to deliver (on-failure)`,paused:!1};case"on-change":return i?o(d,e,t,r):{delivered:`${r}unchanged \u2014 not delivered (on-change)`,paused:!1};case"always":return o(d,e,t,r)}}l(w,"disposeSettledRun");async function o(n,e,t,s){if(e.stdoutHead.trim()==="")return{delivered:`${s}empty stdout \u2014 nothing delivered`,paused:!1};try{const r=m(n,e.runId,e.stdoutHead);return r.kind==="none"?{delivered:`${s}${r.delivered}`,paused:!1}:{delivered:"",paused:!1,obligation:r}}catch(r){const a=`delivery failed: ${r.message}`;return t?{delivered:`out-of-band: ${a} \u2014 not escalated`,paused:!1}:{delivered:await u(n,e,a),paused:!0}}}l(o,"resolveOrEscalate");export{w as disposeSettledRun};
+var c=Object.defineProperty;var u=(n,e)=>c(n,"name",{value:e,configurable:!0});import{getCron as i,setCronLastOutputHash as v}from"../../core/canvas/crons.js";import{escalateFailure as l,failureSummary as h}from"./escalation.js";import{resolveCronSink as g}from"./sinks.js";async function S(n,e,t,a=!1){const r=await m(n,e,t,a);if(!r.refused&&!t&&n.on_output==="on-change"){const s=i(n.cron_id);s!==null&&s.on_output==="on-change"&&v(n.cron_id,e.stdoutSha256)}return r}u(S,"disposeSettledRun");async function m(n,e,t,a=!1){const r=t?"out-of-band: ":"",s=i(n.cron_id);if(s===null&&!a)return{delivered:`${r}cron canceled mid-run \u2014 nothing delivered`,paused:!1};const d=s??n,p=e.timedOut||e.processErrorMessage!==null||e.exitCode!==0;let o=!0;if(d.on_output==="on-change"&&(o=d.last_output_hash!==e.stdoutSha256),p)return d.on_output==="silent"?{delivered:`${r}silent`,paused:!1}:t?{delivered:`out-of-band: ${h(e,null)} \u2014 not escalated`,paused:!1}:{delivered:await l(d,e,null),paused:!0};switch(d.on_output){case"silent":return{delivered:`${r}silent`,paused:!1};case"on-failure":return{delivered:`${r}ok \u2014 nothing to deliver (on-failure)`,paused:!1};case"on-change":return o?f(d,e,t,r):{delivered:`${r}unchanged \u2014 not delivered (on-change)`,paused:!1};case"always":return f(d,e,t,r)}}u(m,"disposeRun");async function f(n,e,t,a){if(e.stdoutHead.trim()==="")return{delivered:`${a}empty stdout \u2014 nothing delivered`,paused:!1};try{const r=g(n,e.runId,e.stdoutHead);return r.kind==="none"?{delivered:`${a}${r.delivered}`,paused:!1,...r.refused===!0?{refused:!0}:{}}:{delivered:"",paused:!1,obligation:r}}catch(r){const s=`delivery failed: ${r.message}`;return t?{delivered:`out-of-band: ${s} \u2014 not escalated`,paused:!1}:{delivered:await l(n,e,s),paused:!0}}}u(f,"resolveOrEscalate");export{S as disposeSettledRun};

package/dist/daemon/cron/escalation.js

@@ -1,3 +1,3 @@
-var m=Object.defineProperty;var o=(e,r)=>m(e,"name",{value:r,configurable:!0});import{getNode as f}from"../../core/canvas/canvas.js";import{nodeAgentDir as h}from"../../core/canvas/node-agent-paths.js";import{newNodeId as p}from"../../core/runtime/nodes.js";import{fullName as $}from"../../core/canvas/labels.js";import{listCronRuns as w,setCronState as y}from"../../core/canvas/crons.js";import{emitEvent as u}from"../../core/events/emit.js";import{spawnChild as _}from"../../core/runtime/spawn.js";import{cronWakeOrigin as g}from"../../core/runtime/bearings.js";import{cadenceDisplay as b}from"../../core/wake.js";import{cronBirthParent as v,resolveLiveProfileOrRoot as O}from"./armed-context.js";function k(e,r){return r!==null?r:e.timedOut?`run timed out after ${e.runTimeoutS}s (killed; recorded exit -1)`:e.processErrorMessage!==null?`run process error: ${e.processErrorMessage}`:`command exited ${e.exitCode??"(no exit observed)"}`}o(k,"failureSummary");function S(e,r,d){const t=e.created_by!=null?f(e.created_by):null,i=e.created_by==null?"created by: (no creating node recorded)":t!==null?`created by: node ${e.created_by} ("${$(t)}") \u2014 still on the canvas; consider telling it what happened`:`created by: node ${e.created_by} (now gone)`,a=e.recur!=null?`every ${b(e.recur)}`:`one-shot at ${e.fire_at}`,s=w(e.cron_id).slice(0,10).map(n=>{const c=n.exit_code===null?"no exit":`exit ${n.exit_code}`,l=(n.stderr_head??n.stdout_head??"").split(`
-`)[0]?.slice(0,120)??"";return`- ${n.started} \u2014 ${c}, ${n.duration_ms}ms \u2014 ${n.delivered??""}${l!==""?` \u2014 ${l}`:""}`});return["A scheduled cron FAILED and has paused itself. You were spawned to deal with it.","",`- cron: "${e.name}" (${e.cron_id})`,`- schedule: ${a}${e.tz!=null?` (tz ${e.tz})`:""}`,`- context: cwd ${e.cwd}${e.profile!=null?`, profile ${e.profile}`:""}`,`- ${i}`,`- this failure: ${k(r,d)}`,"","## The command","","```bash",e.command,"```","","## This run","",`- exit: ${r.timedOut?"-1 (timeout)":String(r.exitCode)}`,`- stdout (head): ${r.stdoutHead===""?"(empty)":""}`,...r.stdoutHead!==""?["```",r.stdoutHead,"```"]:[],`- stderr (head): ${r.stderrHead===""?"(empty)":""}`,...r.stderrHead!==""?["```",r.stderrHead,"```"]:[],"","## Recent runs (most recent first, this failure not yet recorded)","",...s.length>0?s:["(no prior runs recorded)"],"","## Your mandate","","Diagnose why it failed and decide what to do:",`- Transient or fixable: fix the environment/command context, verify with \`crtr cron run ${e.cron_id}\` (fires now, out of band, shows you the result), then \`crtr cron resume ${e.cron_id}\` to re-arm it.`,`- Obsolete: \`crtr cron cancel ${e.cron_id}\`.`,"- The user's call (or the fix needs a change you cannot make): `crtr human send`.","","The cron stays paused until someone resumes it \u2014 nothing refires while you work."].join(`
-`)}o(S,"buildEscalationKickoff");async function I(e,r,d){y(e.cron_id,"paused");try{const t=v(e),i=p(),a=await _({nodeId:i,kind:"general",isolated:e.isolated,cwd:e.isolated?h(i,e.grantee):e.cwd,prompt:S(e,r,d),...t!==null?{parent:t}:{root:!0,rootLifecycle:"terminal"},readiness:"launch",profile:O(e),wakeOrigin:g(e,"node_birth")});return u({level:"info",event:"cron.failure.escalated",fields:{cron_id:e.cron_id,name:e.name,escalation_node:a.node.node_id}}),`failure: cron paused, escalation node ${a.node.node_id} spawned`}catch(t){return u({level:"error",event:"cron.escalation.spawn_failed",error:t,fields:{cron_id:e.cron_id,name:e.name}}),`failure: cron paused; escalation spawn failed: ${t.message}`}}o(I,"escalateFailure");function P(e){return[`${e}.`,"This is a ONE-SHOT, so it has been PAUSED rather than released: its side effects may have completed in full,","and re-firing would enact them a second time. Determine whether the work actually landed, then either","`crtr cron run` it to redo it deliberately, or `crtr cron cancel` it."].join(" ")}o(P,"crashedOneShotSummary");export{P as crashedOneShotSummary,I as escalateFailure,k as failureSummary};
+var m=Object.defineProperty;var a=(e,r)=>m(e,"name",{value:r,configurable:!0});import{getNode as f}from"../../core/canvas/canvas.js";import{nodeAgentDir as h}from"../../core/canvas/node-agent-paths.js";import{newNodeId as p}from"../../core/runtime/nodes.js";import{fullName as $}from"../../core/canvas/labels.js";import{listCronRuns as _,setCronState as w}from"../../core/canvas/crons.js";import{emitEvent as l}from"../../core/events/emit.js";import{spawnChild as y}from"../../core/runtime/spawn.js";import{cronWakeOrigin as g}from"../../core/runtime/bearings.js";import{cadenceDisplay as b}from"../../core/wake.js";import{cronBirthPlacement as v,cronBirthPrincipalOf as O,resolveLiveProfileOrRoot as k}from"./armed-context.js";import{checkCronBirth as S}from"./authority.js";function T(e,r){return r!==null?r:e.timedOut?`run timed out after ${e.runTimeoutS}s (killed; recorded exit -1)`:e.processErrorMessage!==null?`run process error: ${e.processErrorMessage}`:`command exited ${e.exitCode??"(no exit observed)"}`}a(T,"failureSummary");function x(e,r,s){const t=e.created_by!=null?f(e.created_by):null,o=e.created_by==null?"created by: (no creating node recorded)":t!==null?`created by: node ${e.created_by} ("${$(t)}") \u2014 still on the canvas; consider telling it what happened`:`created by: node ${e.created_by} (now gone)`,i=e.recur!=null?`every ${b(e.recur)}`:`one-shot at ${e.fire_at}`,d=_(e.cron_id).slice(0,10).map(n=>{const c=n.exit_code===null?"no exit":`exit ${n.exit_code}`,u=(n.stderr_head??n.stdout_head??"").split(`
+`)[0]?.slice(0,120)??"";return`- ${n.started} \u2014 ${c}, ${n.duration_ms}ms \u2014 ${n.delivered??""}${u!==""?` \u2014 ${u}`:""}`});return["A scheduled cron FAILED and has paused itself. You were spawned to deal with it.","",`- cron: "${e.name}" (${e.cron_id})`,`- schedule: ${i}${e.tz!=null?` (tz ${e.tz})`:""}`,`- context: cwd ${e.cwd}${e.profile!=null?`, profile ${e.profile}`:""}`,`- ${o}`,`- this failure: ${T(r,s)}`,"","## The command","","```bash",e.command,"```","","## This run","",`- exit: ${r.timedOut?"-1 (timeout)":String(r.exitCode)}`,`- stdout (head): ${r.stdoutHead===""?"(empty)":""}`,...r.stdoutHead!==""?["```",r.stdoutHead,"```"]:[],`- stderr (head): ${r.stderrHead===""?"(empty)":""}`,...r.stderrHead!==""?["```",r.stderrHead,"```"]:[],"","## Recent runs (most recent first, this failure not yet recorded)","",...d.length>0?d:["(no prior runs recorded)"],"","## Your mandate","","Diagnose why it failed and decide what to do:",`- Transient or fixable: fix the environment/command context, verify with \`crtr cron run ${e.cron_id}\` (fires now, out of band, shows you the result), then \`crtr cron resume ${e.cron_id}\` to re-arm it.`,`- Obsolete: \`crtr cron cancel ${e.cron_id}\`.`,"- The user's call (or the fix needs a change you cannot make): `crtr human send`.","","The cron stays paused until someone resumes it \u2014 nothing refires while you work."].join(`
+`)}a(x,"buildEscalationKickoff");async function j(e,r,s){w(e.cron_id,"paused");try{const t=O(e),o=S(t);if(!o.allowed)return l({level:"warn",event:"cron.birth.refused",fields:{cron_id:e.cron_id,grant_id:e.grant_id,reason:o.reason,stage:"escalation"}}),`failure: cron paused; no escalation node (refused: ${o.reason})`;const i=p(),d=v(t,i),n=await y({nodeId:i,kind:"general",isolated:e.isolated,cwd:e.isolated?h(i,e.grantee):e.cwd,prompt:x(e,r,s),...d,readiness:"launch",profile:k(e),wakeOrigin:g(e,"node_birth")});return l({level:"info",event:"cron.failure.escalated",fields:{cron_id:e.cron_id,name:e.name,escalation_node:n.node.node_id}}),`failure: cron paused, escalation node ${n.node.node_id} spawned`}catch(t){return l({level:"error",event:"cron.escalation.spawn_failed",error:t,fields:{cron_id:e.cron_id,name:e.name}}),`failure: cron paused; escalation spawn failed: ${t.message}`}}a(j,"escalateFailure");function M(e){return[`${e}.`,"This is a ONE-SHOT, so it has been PAUSED rather than released: its side effects may have completed in full,","and re-firing would enact them a second time. Determine whether the work actually landed, then either","`crtr cron run` it to redo it deliberately, or `crtr cron cancel` it."].join(" ")}a(M,"crashedOneShotSummary");export{M as crashedOneShotSummary,j as escalateFailure,T as failureSummary};

package/dist/daemon/cron/folders.d.ts

@@ -0,0 +1,5 @@
+/** Create (idempotent) the cron folder and its `home`. Where the app's space is shared through the
+ *  crtr-data group (Linux sandbox), the folder takes the group of its parent and stays group-writable. */
+export declare function ensureCronFolder(app: string, cronId: string): string;
+/** Delete every `<app space>/crons/<id>` whose cron row no longer exists. */
+export declare function sweepOrphanCronFolders(): number;

package/dist/daemon/cron/folders.js

@@ -0,0 +1 @@
+var l=Object.defineProperty;var c=(r,e)=>l(r,"name",{value:e,configurable:!0});import{chmodSync as m,chownSync as u,existsSync as f,mkdirSync as a,readdirSync as s,rmSync as y,statSync as v}from"node:fs";import{dirname as S,join as i}from"node:path";import{getCron as h}from"../../core/canvas/crons.js";import{emitEvent as d}from"../../core/events/emit.js";import{appsRoot as w,cronFolder as x}from"../../core/layout.js";function E(r,e){const n=x(r,e);for(const o of[n,i(n,"home")])if(a(o,{recursive:!0}),process.platform==="linux"){const t=v(S(o));try{u(o,process.getuid(),t.gid),m(o,1528)}catch{}}return n}c(E,"ensureCronFolder");function O(){let r=0;const e=w();if(!f(e))return 0;for(const n of s(e)){const o=i(e,n,"crons");if(f(o)){for(const t of s(o))if(h(t)===null)try{y(i(o,t),{recursive:!0,force:!0}),r++}catch(p){d({level:"warn",event:"cron.folder.sweep_failed",error:p,fields:{cron_id:t,app:n}})}}}return r>0&&d({level:"info",event:"cron.folder.swept",fields:{removed:r}}),r}c(O,"sweepOrphanCronFolders");export{E as ensureCronFolder,O as sweepOrphanCronFolders};

package/dist/daemon/cron/lease-recovery.js

@@ -1 +1 @@
-var v=Object.defineProperty;var d=(e,r)=>v(e,"name",{value:r,configurable:!0});import{randomUUID as m}from"node:crypto";import{setTimeout as S}from"node:timers/promises";import{CRON_DEATH_POLL_MS as w,CRON_KILL_GRACE_MS as A,EMPTY_STDOUT_SHA256 as f}from"./capture.js";import{cronDaemonInstanceId as C}from"./live-runs.js";import{crashedOneShotSummary as L,escalateFailure as H}from"./escalation.js";import{settleRun as h}from"./settle.js";import{runningCrons as M,stopLeasedCronRun as p,setCronState as g,verifyLeasedCronRun as l}from"../../core/canvas/crons.js";import{emitEvent as c}from"../../core/events/emit.js";const i=new Set;function K(e,r={}){for(const t of M()){if(t.run_lease_owner===C||i.has(t.cron_id))continue;if(t.run_pid===null||!l(t)){i.add(t.cron_id),D(t,e,r);continue}const o=t.run_started_at!=null?new Date(t.run_started_at).getTime():e;if(e-o<t.run_timeout_s*1e3)continue;const a={runId:m(),startedAtIso:t.run_started_at??new Date(e).toISOString(),exitCode:-1,timedOut:!0,runTimeoutS:t.run_timeout_s,processErrorMessage:null,stdoutHead:"",stdoutSha256:f,stderrHead:`run timed out after ${t.run_timeout_s}s (recovered stale lease from a prior daemon instance)`};i.add(t.cron_id);const n=(async()=>{await p(t),await y(t),await h(t,{outOfBand:!1,outcome:{kind:"disposed",ctx:a},record:{runId:a.runId,startedAtIso:a.startedAtIso,finishedAtMs:e,durationMs:e-o,exitCode:-1,stdoutHead:null,stderrHead:a.stderrHead},lease:{kind:"clear"}})})().finally(()=>i.delete(t.cron_id));r.register?.(n),n.catch(s=>{c({level:"error",event:"cron.stale_lease.settle_failed",error:s,fields:{cron_id:t.cron_id}})})}}d(K,"recoverStaleCronLeases");async function y(e){if(!await I(e)&&(await p(e,"SIGKILL"),!await I(e)))throw new Error(`cron ${e.cron_id} run ${e.run_pid} survived SIGKILL`)}d(y,"awaitLeasedRunDeath");async function I(e){const r=Date.now()+A;for(;Date.now()<r;){if(!l(e))return!0;await S(w)}return!l(e)}d(I,"pollLeasedRunGone");function D(e,r,t){const o=e.recur==null,a=e.run_started_at!=null?new Date(e.run_started_at).getTime():r,n=e.run_pid===null?"the daemon life that claimed this run ended before its launch pid was recorded \u2014 the command may have started and its outcome is unknown":"the daemon life that owned this run ended before it settled \u2014 the process is gone and its outcome was never observed",s={runId:m(),startedAtIso:e.run_started_at??new Date(r).toISOString(),exitCode:null,timedOut:!1,runTimeoutS:e.run_timeout_s,processErrorMessage:n,stdoutHead:"",stdoutSha256:f,stderrHead:`run outcome unobserved: ${n}`},_=(async()=>{let u=`crashed: ${n}${o?"":" \u2014 the next occurrence is unaffected"}`;o&&(g(e.cron_id,"paused"),u=e.on_output==="silent"?`crashed: ${n} \u2014 one-shot paused, not re-fired (silent: not escalated)`:`crashed one-shot: ${await H(e,s,L(n))}`),await h(e,{outOfBand:!1,outcome:{kind:"stated",delivered:u,paused:o},record:{runId:s.runId,startedAtIso:s.startedAtIso,finishedAtMs:r,durationMs:Math.max(0,r-a),exitCode:null,stdoutHead:null,stderrHead:s.stderrHead},lease:"release"}),c({level:"warn",event:"cron.run.unobserved",fields:{cron_id:e.cron_id,name:e.name,one_shot:o}})})().finally(()=>i.delete(e.cron_id));t.register?.(_),_.catch(u=>{c({level:"error",event:"cron.crashed_run.settle_failed",error:u,fields:{cron_id:e.cron_id}})})}d(D,"settleCrashedRun");export{K as recoverStaleCronLeases};
+var v=Object.defineProperty;var d=(e,r)=>v(e,"name",{value:r,configurable:!0});import{randomUUID as m}from"node:crypto";import{setTimeout as S}from"node:timers/promises";import{CRON_DEATH_POLL_MS as w,CRON_KILL_GRACE_MS as A,EMPTY_STDOUT_SHA256 as f}from"./capture.js";import{cronDaemonInstanceId as C}from"./live-runs.js";import{crashedOneShotSummary as L,escalateFailure as H}from"./escalation.js";import{settleRun as h}from"./settle.js";import{runningCrons as M,stopLeasedCronRun as p,setCronState as g,verifyLeasedCronRun as l}from"../../core/canvas/crons.js";import{emitEvent as c}from"../../core/events/emit.js";const i=new Set;function K(e,r={}){for(const t of M()){if(t.run_lease_owner===C||i.has(t.cron_id))continue;if(t.run_pid===null||!l(t)){i.add(t.cron_id),y(t,e,r);continue}const o=t.run_started_at!=null?new Date(t.run_started_at).getTime():e;if(e-o<t.run_timeout_s*1e3)continue;const a={runId:t.run_execution_id??m(),startedAtIso:t.run_started_at??new Date(e).toISOString(),exitCode:-1,timedOut:!0,runTimeoutS:t.run_timeout_s,processErrorMessage:null,stdoutHead:"",stdoutSha256:f,stderrHead:`run timed out after ${t.run_timeout_s}s (recovered stale lease from a prior daemon instance)`};i.add(t.cron_id);const n=(async()=>{await p(t),await x(t),await h(t,{outOfBand:!1,outcome:{kind:"disposed",ctx:a},record:{runId:a.runId,startedAtIso:a.startedAtIso,finishedAtMs:e,durationMs:e-o,exitCode:-1,stdoutHead:null,stderrHead:a.stderrHead},lease:{kind:"clear"}})})().finally(()=>i.delete(t.cron_id));r.register?.(n),n.catch(s=>{c({level:"error",event:"cron.stale_lease.settle_failed",error:s,fields:{cron_id:t.cron_id}})})}}d(K,"recoverStaleCronLeases");async function x(e){if(!await I(e)&&(await p(e,"SIGKILL"),!await I(e)))throw new Error(`cron ${e.cron_id} run ${e.run_pid} survived SIGKILL`)}d(x,"awaitLeasedRunDeath");async function I(e){const r=Date.now()+A;for(;Date.now()<r;){if(!l(e))return!0;await S(w)}return!l(e)}d(I,"pollLeasedRunGone");function y(e,r,t){const o=e.recur==null,a=e.run_started_at!=null?new Date(e.run_started_at).getTime():r,n=e.run_pid===null?"the daemon life that claimed this run ended before its launch pid was recorded \u2014 the command may have started and its outcome is unknown":"the daemon life that owned this run ended before it settled \u2014 the process is gone and its outcome was never observed",s={runId:e.run_execution_id??m(),startedAtIso:e.run_started_at??new Date(r).toISOString(),exitCode:null,timedOut:!1,runTimeoutS:e.run_timeout_s,processErrorMessage:n,stdoutHead:"",stdoutSha256:f,stderrHead:`run outcome unobserved: ${n}`},_=(async()=>{let u=`crashed: ${n}${o?"":" \u2014 the next occurrence is unaffected"}`;o&&(g(e.cron_id,"paused"),u=e.on_output==="silent"?`crashed: ${n} \u2014 one-shot paused, not re-fired (silent: not escalated)`:`crashed one-shot: ${await H(e,s,L(n))}`),await h(e,{outOfBand:!1,outcome:{kind:"stated",delivered:u,paused:o},record:{runId:s.runId,startedAtIso:s.startedAtIso,finishedAtMs:r,durationMs:Math.max(0,r-a),exitCode:null,stdoutHead:null,stderrHead:s.stderrHead},lease:"release"}),c({level:"warn",event:"cron.run.unobserved",fields:{cron_id:e.cron_id,name:e.name,one_shot:o}})})().finally(()=>i.delete(e.cron_id));t.register?.(_),_.catch(u=>{c({level:"error",event:"cron.crashed_run.settle_failed",error:u,fields:{cron_id:e.cron_id}})})}d(y,"settleCrashedRun");export{K as recoverStaleCronLeases};

package/dist/daemon/cron/live-runs.d.ts

@@ -1,3 +1,4 @@
+import type { CronRefusalReason } from './authority.js';
 export declare const cronDaemonInstanceId: `${string}-${string}-${string}-${string}-${string}`;
 export interface LiveRun {
     runId: string;
@@ -9,3 +10,30 @@ export declare function liveRunFor(cronId: string): LiveRun | undefined;
 export declare function registerLiveRun(cronId: string, run: LiveRun): void;
 export declare function clearLiveRun(cronId: string, runId: string): void;
 export declare function markSelfCanceledCronRun(cronId: string, runId: string): boolean;
+/** One cron execution's identity while it runs: the stored cron principal keyed by execution id.
+ *  Its socket answers as this principal; the grant sweep and the router read it. A principal is
+ *  never read off the cron row after the fire, so two executions of one run keep separate scopes. */
+export interface CronExecution {
+    executionId: string;
+    cronId: string;
+    grantee: string;
+    sub: string;
+    grantId: string | null;
+    scopes: string[] | null;
+    originRunId: string | null;
+    isolated: boolean;
+    anchor: string | null;
+    /** The app's OS uid: the only peer the execution socket serves on Linux. */
+    uid: number;
+    /** Rebuilt from a row and a live sandbox after a daemon restart: no executor promise owns its close. */
+    rebuilt: boolean;
+    /** Set by the grant sweep when it stopped this execution; the executor settles the run `refused` with it. */
+    refusal: {
+        reason: CronRefusalReason;
+        message: string;
+    } | null;
+}
+export declare function registerCronExecution(execution: CronExecution): void;
+export declare function cronExecutionFor(executionId: string): CronExecution | undefined;
+export declare function dropCronExecution(executionId: string): void;
+export declare function cronExecutions(): CronExecution[];

package/dist/daemon/cron/live-runs.js

@@ -1 +1 @@
-var u=Object.defineProperty;var r=(e,n)=>u(e,"name",{value:n,configurable:!0});import{randomUUID as i}from"node:crypto";const l=i(),o=new Map;function d(e){return o.get(e)}r(d,"liveRunFor");function s(e,n){o.set(e,n)}r(s,"registerLiveRun");function a(e,n){const t=o.get(e);t!==void 0&&t.runId===n&&o.delete(e)}r(a,"clearLiveRun");function p(e,n){const t=o.get(e);return t===void 0||t.runId!==n?!1:(t.selfCanceled=!0,!0)}r(p,"markSelfCanceledCronRun");export{a as clearLiveRun,l as cronDaemonInstanceId,d as liveRunFor,p as markSelfCanceledCronRun,s as registerLiveRun};
+var i=Object.defineProperty;var n=(e,t)=>i(e,"name",{value:t,configurable:!0});import{randomUUID as c}from"node:crypto";const x=c(),o=new Map;function d(e){return o.get(e)}n(d,"liveRunFor");function l(e,t){o.set(e,t)}n(l,"registerLiveRun");function p(e,t){const r=o.get(e);r!==void 0&&r.runId===t&&o.delete(e)}n(p,"clearLiveRun");function a(e,t){const r=o.get(e);return r===void 0||r.runId!==t?!1:(r.selfCanceled=!0,!0)}n(a,"markSelfCanceledCronRun");const u=new Map;function v(e){u.set(e.executionId,e)}n(v,"registerCronExecution");function g(e){return u.get(e)}n(g,"cronExecutionFor");function m(e){u.delete(e)}n(m,"dropCronExecution");function C(){return[...u.values()]}n(C,"cronExecutions");export{p as clearLiveRun,x as cronDaemonInstanceId,g as cronExecutionFor,C as cronExecutions,m as dropCronExecution,d as liveRunFor,a as markSelfCanceledCronRun,v as registerCronExecution,l as registerLiveRun};

package/dist/daemon/cron/run-socket.d.ts

@@ -0,0 +1,30 @@
+import { type IncomingMessage, type ServerResponse } from 'node:http';
+import type { CronPrincipalClass } from '../api/router.js';
+import { type CronExecution } from './live-runs.js';
+/** Installed by the API server: runs one request through the router as `principal`. */
+export type CronRequestHandler = (req: IncomingMessage, res: ServerResponse, principal: CronPrincipalClass) => void;
+export declare function setCronRequestHandler(handler: CronRequestHandler | null): void;
+type PeerUid = (req: IncomingMessage) => number | null;
+/** Test seam: the peer uid of a request (null = unchecked). */
+export declare function __setCronPeerUidForTest(fn: PeerUid | undefined): void;
+export declare function cronSocketPath(executionId: string): string;
+export declare function cronPrincipalClassOf(e: CronExecution): CronPrincipalClass;
+/** Register the execution and open its socket. The folder is created daemon-owned, group daemon, mode 0770 — the state
+ *  the root launcher's `SourceValidator.pin` accepts and re-owns to the app uid for a launch whose node id is
+ *  `cron-<execution_id>` (`launcher/validate.ts`); `cron-launch.json` carries the config snapshot a sandboxed
+ *  `crtr` installs (the shape of `broker-launch.json`'s). */
+export declare function openCronExecution(execution: CronExecution, cwd: string, profileId: string | null): Promise<void>;
+/** Close the listener, delete the execution folder, drop the registry entry. Idempotent. */
+export declare function closeCronExecution(executionId: string): void;
+/** Resolves once a restart rebuild finished; the grant sweep waits on it so it never stops a live execution it has not yet re-registered. */
+export declare function cronExecutionsReady(): Promise<void>;
+/** After a daemon restart the listener died with the daemon but the folder and the sandbox survived. Rebuild the registry from
+ *  `running` rows whose recorded execution id still has a live `cron-<id>` sandbox and re-listen on the same path; delete
+ *  `nodes/cron-*` folders with no live sandbox. A self-canceled execution (row deleted) cannot be rebuilt: it stays without
+ *  an API path and the grant sweep stops it. */
+export declare function rebuildCronExecutions(list?: () => Promise<Array<{
+    nodeId: string;
+}>>): Promise<void>;
+/** Close rebuilt executions whose row no longer records them (their lease settled elsewhere). Own executions close in the executor. */
+export declare function reapRebuiltCronExecutions(): void;
+export {};

package/dist/daemon/cron/run-socket.js

@@ -0,0 +1 @@
+var k=Object.defineProperty;var i=(r,n)=>k(r,"name",{value:n,configurable:!0});import{createServer as F}from"node:http";import{chmodSync as g,existsSync as x,mkdirSync as A,readdirSync as O,rmSync as v,unlinkSync as z}from"node:fs";import{dirname as H,join as f}from"node:path";import{peerUid as M}from"../../native/linux.js";import{nodeRunDir as a}from"../../core/canvas/paths.js";import{readConfig as U,readMergedLaunchConfig as j}from"../../core/config.js";import{readRotationConfig as q}from"../../core/subscription-state.js";import{runningCrons as _}from"../../core/canvas/crons.js";import{emitEvent as u}from"../../core/events/emit.js";import{cronLaunchId as s}from"../../core/spaces/process.js";import{ensureAppSpace as B}from"../../core/spaces/space.js";import{launcherPresent as y,listSandboxes as b}from"../../core/grants/sandboxes.js";import{uidForApp as J}from"../../core/spaces/users.js";import{setModeBeneath as L,writeFileBeneath as N}from"../../core/spaces/open-beneath.js";import{cronExecutionFor as S,cronExecutions as D,dropCronExecution as C,registerCronExecution as w}from"./live-runs.js";let p=null;function ar(r){p=r}i(ar,"setCronRequestHandler");const E=i(r=>process.platform==="linux"?M(r.socket):null,"linuxPeerUid");let I=E;function dr(r){I=r??E}i(dr,"__setCronPeerUidForTest");const m=new Map;function T(r){return f(a(s(r)),"cli.sock")}i(T,"cronSocketPath");function W(r){return{class:"cron",app:r.grantee,sub:r.sub,grant:r.grantId,scopes:r.scopes,cron:r.cronId,execution:r.executionId,origin_run:r.originRunId,isolated:r.isolated,anchor:r.anchor,uid:r.uid}}i(W,"cronPrincipalClassOf");function d(r,n,t,o){r.writeHead(n,{"content-type":"application/json; charset=utf-8"}),r.end(JSON.stringify({error:{code:t,message:o,type:"invalid_request_error",origin:"runtime",retryable:!1}}))}i(d,"refuse");function G(r){return(n,t)=>{try{const o=S(r);if(o===void 0){d(t,401,"unauthorized","this cron execution has ended");return}const e=I(n);if(e!==null&&e!==o.uid){u({event:"api.principal.denied",level:"warn",fields:{method:n.method,execution:r}}),d(t,401,"unauthorized","missing or invalid bearer token");return}if(p===null){d(t,503,"daemon_unavailable","the API server is not running");return}p(n,t,W(o))}catch(o){u({event:"api.principal.failed",level:"error",error:o instanceof Error?o:new Error(String(o))}),t.headersSent||d(t,401,"unauthorized","missing or invalid bearer token")}}}i(G,"onCronRequest");async function P(r){const n=T(r);x(n)&&z(n);const t=F(G(r));await new Promise((o,e)=>{t.once("error",e),t.listen(n,()=>{t.off("error",e),o()})}),g(n,438),m.set(r,t)}i(P,"listen");async function lr(r,n,t){const o=a(s(r.executionId));w(r);try{A(o,{recursive:!0,mode:504}),g(o,504);const e={user:U("user"),launch:j(n,t),rotation:q()},c=f(o,"cron-launch.json");if(N(o,c,JSON.stringify({configSnapshot:e}),{atomic:!0,mode:416}),y()){const l=await B(r.grantee);L(o,c,416,{uid:process.getuid(),gid:l.gid})}await P(r.executionId)}catch(e){throw R(r.executionId),e}}i(lr,"openCronExecution");function R(r){C(r);const n=m.get(r);m.delete(r),n!==void 0&&(n.closeAllConnections(),n.close());try{v(a(s(r)),{recursive:!0,force:!0})}catch{}}i(R,"closeCronExecution");let h=Promise.resolve();function fr(){return h}i(fr,"cronExecutionsReady");function pr(r=b){return h=(async()=>{if(r===b&&!y())return;const t=new Set((await r()).map(e=>e.nodeId));for(const e of _()){const c=e.run_execution_id;if(!(c==null||!t.has(s(c))||S(c)!==void 0))try{w({executionId:c,cronId:e.cron_id,grantee:e.grantee,sub:e.sub,grantId:e.grant_id,scopes:e.scopes,originRunId:e.origin_run_id,isolated:e.isolated,anchor:e.anchor_node,uid:J(e.grantee),rebuilt:!0,refusal:null}),await P(c),u({level:"info",event:"cron.execution.rebuilt",fields:{cron_id:e.cron_id,execution_id:c}})}catch(l){C(c),u({level:"error",event:"cron.execution.rebuild_failed",error:l,fields:{cron_id:e.cron_id,execution_id:c}})}}if(process.platform!=="linux")return;const o=H(a(s("x")));if(x(o)){for(const e of O(o))if(!(!e.startsWith("cron-")||t.has(e)))try{v(f(o,e),{recursive:!0,force:!0})}catch{}}})().catch(t=>{u({level:"error",event:"cron.execution.rebuild_failed",error:t})}),h}i(pr,"rebuildCronExecutions");function mr(){const r=new Set(_().map(n=>n.run_execution_id));for(const n of D())n.rebuilt&&!r.has(n.executionId)&&R(n.executionId)}i(mr,"reapRebuiltCronExecutions");export{dr as __setCronPeerUidForTest,R as closeCronExecution,fr as cronExecutionsReady,W as cronPrincipalClassOf,T as cronSocketPath,lr as openCronExecution,mr as reapRebuiltCronExecutions,pr as rebuildCronExecutions,ar as setCronRequestHandler};

package/dist/daemon/cron/settle.d.ts

@@ -1,4 +1,5 @@
 import { type Cron, type CronRunRecord, type HeldSettle } from '../../core/canvas/crons.js';
+import type { CronRefusalReason } from './authority.js';
 import { type RunSettleContext } from './disposition.js';
 export type LeaseSettlement = 'none' | 'release' | HeldSettle;
 export type SettleOutcome = {
@@ -9,6 +10,15 @@ export type SettleOutcome = {
     kind: 'stated';
     delivered: string;
     paused?: boolean;
+}
+/** A fire refused by the cron authority check (or a run stopped because the grant no
+ *  longer covers it). One record with exit null; no disposition, delivery, escalation or
+ *  `on-change` hash. A recurrence stays scheduled; a scheduled one-shot is paused, not
+ *  consumed, so a reactivated grant plus `cron resume` can fire it. */
+ | {
+    kind: 'refused';
+    reason: CronRefusalReason;
+    message: string;
 };
 export interface SettleRecord {
     runId: string;

package/dist/daemon/cron/settle.js

@@ -1 +1 @@
-var m=Object.defineProperty;var u=(e,s)=>m(e,"name",{value:s,configurable:!0});import{consumeCron as y,recordCronRunLocked as I,releaseCronRunLease as b}from"../../core/canvas/crons.js";import{withCanvasWrite as k}from"../../core/canvas/db.js";import{enqueueDaemonAction as _}from"../../core/canvas/daemon-actions.js";import{enqueueNodeMail as v}from"../../core/feed/messages.js";import{disposeSettledRun as g}from"./disposition.js";function w(e){return typeof e=="object"&&(e.kind==="park"||e.kind==="redue")}u(w,"occurrenceStaysOwed");async function S(e,s){const{outOfBand:c,outcome:o,record:n,lease:i}=s;let t,a,d;if(o.kind==="disposed"){const r=await g(e,o.ctx,c,o.selfCanceled??!1);t=r.delivered,a=r.paused,d=r.obligation}else t=o.delivered,a=o.paused??!1;const f=k(r=>{d!==void 0&&(d.kind==="node"?t=`delivered: ${v(d.destinationNodeId,{kind:"message",label:d.label,senderNodeId:d.senderNodeId,bodyJson:{body:d.stdout},urgency:d.urgency,wake:1,idempotencyKey:n.runId}).deliveryId}`:d.kind==="action"&&(t=`delivered: ${_({adapter:d.adapter,bodyJson:d.bodyJson,idempotencyKey:n.runId})}`));const l={run_id:n.runId,cron_id:e.cron_id,started:n.startedAtIso,finished:new Date(n.finishedAtMs).toISOString(),duration_ms:n.durationMs,exit_code:n.exitCode,stdout_head:n.stdoutHead,stderr_head:n.stderrHead,delivered:t};return I(r,l),l});return i!=="none"&&b(e.cron_id,i==="release"?void 0:i),!c&&e.recur==null&&!a&&!w(i)&&y(e.cron_id),f}u(S,"settleRun");export{S as settleRun};
+var p=Object.defineProperty;var l=(e,u)=>p(e,"name",{value:u,configurable:!0});import{consumeCron as _,recordCronRunLocked as y,releaseCronRunLease as I,setCronState as b}from"../../core/canvas/crons.js";import{withCanvasWrite as k}from"../../core/canvas/db.js";import{enqueueDaemonAction as v}from"../../core/canvas/daemon-actions.js";import{enqueueNodeMail as g}from"../../core/feed/messages.js";import{emitEvent as w}from"../../core/events/emit.js";import{disposeSettledRun as C}from"./disposition.js";function h(e){return typeof e=="object"&&(e.kind==="park"||e.kind==="redue")}l(h,"occurrenceStaysOwed");async function M(e,u){const{outOfBand:s,outcome:o,record:n,lease:a}=u;let r,t,d;if(o.kind==="disposed"){const i=await C(e,o.ctx,s,o.selfCanceled??!1);r=i.delivered,t=i.paused,d=i.obligation}else o.kind==="refused"?(r=`refused: ${o.reason}`,t=!s&&e.recur==null,t&&b(e.cron_id,"paused"),w({level:"warn",event:"cron.fire.refused",fields:{cron_id:e.cron_id,execution_id:n.runId,grant_id:e.grant_id,reason:o.reason,out_of_band:s}})):(r=o.delivered,t=o.paused??!1);const f=k(i=>{d!==void 0&&(d.kind==="node"?r=`delivered: ${g(d.destinationNodeId,{kind:"message",label:d.label,senderNodeId:d.senderNodeId,bodyJson:{body:d.stdout},urgency:d.urgency,wake:1,idempotencyKey:n.runId}).deliveryId}`:d.kind==="action"&&(r=`delivered: ${v({adapter:d.adapter,bodyJson:d.bodyJson,idempotencyKey:n.runId})}`));const c={run_id:n.runId,cron_id:e.cron_id,started:n.startedAtIso,finished:new Date(n.finishedAtMs).toISOString(),duration_ms:n.durationMs,exit_code:n.exitCode,stdout_head:n.stdoutHead,stderr_head:n.stderrHead,delivered:r};return y(i,c),c});return a!=="none"&&I(e.cron_id,a==="release"?void 0:a),!s&&e.recur==null&&!t&&!h(a)&&_(e.cron_id),f}l(M,"settleRun");export{M as settleRun};

package/dist/daemon/cron/sinks.d.ts

@@ -2,7 +2,8 @@ import { type Cron } from '../../core/canvas/crons.js';
 import type { DaemonActionAdapter } from '../../core/canvas/daemon-actions.js';
 export type CronSinkResolution = {
     kind: 'none';
-    delivered: string;
+    delivered: string; /** A spawn sink's birth was refused by the cron authority. */
+    refused?: true;
 } | {
     kind: 'node';
     destinationNodeId: string;

package/dist/daemon/cron/sinks.js

@@ -1,2 +1,2 @@
-var s=Object.defineProperty;var t=(e,r)=>s(e,"name",{value:r,configurable:!0});import{newNodeId as a}from"../../core/runtime/nodes.js";import{nodeAgentDir as u}from"../../core/canvas/node-agent-paths.js";import{getNode as l}from"../../core/canvas/canvas.js";import{parseCronSink as p}from"../../core/canvas/crons.js";import{hasNoNaturalCycle as f}from"../../core/runtime/revive-all.js";import{isParked as k}from"../../core/runtime/lifecycle.js";import{cronWakeOrigin as c}from"../../core/runtime/bearings.js";import{cronBirthParent as y,resolveLiveProfile as h,resolveLiveProfileName as w}from"./armed-context.js";function S(e,r,i){const n=p(e.sink);if(n===null)return{kind:"none",delivered:"no sink armed \u2014 nothing delivered"};if(n.kind==="node"){const o=l(n.node);if(o===null)throw new Error(`node sink ${n.node} is gone`);if(f(o)&&!k(o))throw new Error(`node sink ${n.node} has no natural cycle ahead (done or finalized)`);return{kind:"node",destinationNodeId:n.node,senderNodeId:e.created_by!==null&&l(e.created_by)!==null?e.created_by:null,label:`\u23F0 cron ${e.name}`,stdout:i,urgency:e.tier}}if(n.kind==="spawn"){const o=y(e),d=a(),m=[`Scheduled cron "${e.name}" (${e.cron_id}) fired and delivered its output to you as your kickoff.`,"","## Command output","",i].join(`
-`);return{kind:"action",adapter:{adapter:"cron_spawn",cronId:e.cron_id,nodeId:d,kind:n.spawn,cwd:e.isolated?u(d,e.grantee):e.cwd,isolated:e.isolated,...o!==null?{parent:o,root:!1}:{parent:null,root:!0,rootLifecycle:"terminal"},profile:h(e),wakeOrigin:c(e,"node_birth"),prompt:m},bodyJson:{runId:r,stdout:i}}}return{kind:"action",adapter:{adapter:"cron_human",cronId:e.cron_id,requestId:a(),title:`\u23F0 cron ${e.name}`,summary:`Scheduled cron "${e.name}" delivered output for your attention.`,sessionName:`cron ${e.name} (${e.cron_id})`,profileName:w(e)??null},bodyJson:{runId:r,stdout:i}}}t(S,"resolveCronSink");export{S as resolveCronSink};
+var u=Object.defineProperty;var a=(e,o)=>u(e,"name",{value:o,configurable:!0});import{newNodeId as l}from"../../core/runtime/nodes.js";import{nodeAgentDir as f}from"../../core/canvas/node-agent-paths.js";import{getNode as s}from"../../core/canvas/canvas.js";import{parseCronSink as p}from"../../core/canvas/crons.js";import{hasNoNaturalCycle as k}from"../../core/runtime/revive-all.js";import{isParked as c}from"../../core/runtime/lifecycle.js";import{cronWakeOrigin as h}from"../../core/runtime/bearings.js";import{emitEvent as _}from"../../core/events/emit.js";import{cronBirthPrincipalOf as v,resolveLiveProfile as w,resolveLiveProfileName as y}from"./armed-context.js";import{checkCronBirth as g}from"./authority.js";function J(e,o,i){const n=p(e.sink);if(n===null)return{kind:"none",delivered:"no sink armed \u2014 nothing delivered"};if(n.kind==="node"){const r=s(n.node);if(r===null)throw new Error(`node sink ${n.node} is gone`);if(k(r)&&!c(r))throw new Error(`node sink ${n.node} has no natural cycle ahead (done or finalized)`);return{kind:"node",destinationNodeId:n.node,senderNodeId:e.created_by!==null&&s(e.created_by)!==null?e.created_by:null,label:`\u23F0 cron ${e.name}`,stdout:i,urgency:e.tier}}if(n.kind==="spawn"){const r=v(e),d=g(r);if(!d.allowed)return _({level:"warn",event:"cron.birth.refused",fields:{cron_id:e.cron_id,grant_id:e.grant_id,reason:d.reason,stage:"enqueue"}}),{kind:"none",delivered:`refused: ${d.reason}`,refused:!0};const t=l(),m=[`Scheduled cron "${e.name}" (${e.cron_id}) fired and delivered its output to you as your kickoff.`,"","## Command output","",i].join(`
+`);return{kind:"action",adapter:{adapter:"cron_spawn",cronId:e.cron_id,nodeId:t,kind:n.spawn,cwd:e.isolated?f(t,e.grantee):e.cwd,isolated:e.isolated,principal:r,profile:w(e),wakeOrigin:h(e,"node_birth"),prompt:m},bodyJson:{runId:o,stdout:i}}}return{kind:"action",adapter:{adapter:"cron_human",cronId:e.cron_id,requestId:l(),title:`\u23F0 cron ${e.name}`,summary:`Scheduled cron "${e.name}" delivered output for your attention.`,sessionName:`cron ${e.name} (${e.cron_id})`,profileName:y(e)??null},bodyJson:{runId:o,stdout:i}}}a(J,"resolveCronSink");export{J as resolveCronSink};

package/dist/daemon/cron-run.js

@@ -1 +1,2 @@
-var A=Object.defineProperty;var f=(e,t)=>A(e,"name",{value:t,configurable:!0});import{randomUUID as R}from"node:crypto";import{CRON_OUTPUT_HEAD_BYTES as h,EMPTY_STDOUT_SHA256 as T,formatCronOutputHead as H}from"./cron/capture.js";import{clearLiveRun as _,cronDaemonInstanceId as E,registerLiveRun as g}from"./cron/live-runs.js";import{buildArmedRunContext as v}from"./cron/armed-context.js";import{decideHeldSettle as M,EXIT_HELD as k,HELD_DELIVERED as b}from"./cron/held.js";import{settleRun as D}from"./cron/settle.js";import{applyOverlapPolicies as B,expireDueCrons as L}from"./cron/passes.js";import{dueClockCrons as U,acquireCronRunLease as y,attachCronRunPid as P,advanceCron as $,setCronState as N}from"../core/canvas/crons.js";import{emitEvent as Y}from"../core/events/emit.js";import{runAppCommand as q}from"../core/runtime/run-command.js";import{nextSlotAfter as F}from"../core/wake.js";import{CRON_OUTPUT_HEAD_BYTES as ie}from"./cron/capture.js";import{cronDaemonInstanceId as ue,markSelfCanceledCronRun as ce}from"./cron/live-runs.js";import{recoverStaleCronLeases as me}from"./cron/lease-recovery.js";async function J(e,t){const{runId:s,outOfBand:r,startedAtIso:n,finishedAtMs:o,code:u,timedOut:i,processError:l}=t,p=!r&&!i&&l==null&&u===k&&!t.wasReplaced&&!t.wasSelfCanceled;let c,m;return t.wasReplaced?(c={kind:"stated",delivered:"replaced (overlap=replace)"},m="release"):p?(c={kind:"stated",delivered:b},m=M(e,n,o)??"release"):(c={kind:"disposed",ctx:{runId:s,startedAtIso:n,exitCode:i?-1:u,timedOut:i,runTimeoutS:e.run_timeout_s,processErrorMessage:l!=null?l.message:null,stdoutHead:t.stdoutHead,stdoutSha256:t.stdoutSha256,stderrHead:t.stderrHead},selfCanceled:t.wasSelfCanceled},m=r?"release":{kind:"clear"}),D(e,{outOfBand:r,outcome:c,record:{runId:s,startedAtIso:n,finishedAtMs:o,durationMs:o-t.startedAtMs,exitCode:i?-1:u,stdoutHead:t.stdoutHead===""?null:t.stdoutHead,stderrHead:l!=null?`run process error: ${l.message}`.slice(0,h):t.stderrHead===""?null:t.stderrHead},lease:m})}f(J,"settleClosedRun");function I(e,t,s,r){const n=Date.now(),o=`${s}: ${r.message}`.slice(0,h);return D(e,{outOfBand:t.outOfBand,outcome:{kind:"disposed",ctx:{runId:t.runId,startedAtIso:t.startedAtIso,exitCode:null,timedOut:!1,runTimeoutS:e.run_timeout_s,processErrorMessage:r.message,stdoutHead:"",stdoutSha256:T,stderrHead:o}},record:{runId:t.runId,startedAtIso:t.startedAtIso,finishedAtMs:n,durationMs:n-t.startedAtMs,exitCode:null,stdoutHead:null,stderrHead:o},lease:"release"})}f(I,"settleUnspawnedRun");function V(e,t){let s;try{s=F(e.recur,new Date(t))}catch(r){return Y({level:"error",event:"cron.recur.unparseable",error:r,fields:{cron_id:e.cron_id,name:e.name,recur:e.recur}}),N(e.cron_id,"paused"),new Error(`unparseable recurrence ${JSON.stringify(e.recur)}: ${r.message} \u2014 this cron cannot be rescheduled until its recurrence is fixed (\`crtr cron cancel\` and re-arm it), so it has been paused.`)}return $(e.cron_id,s),null}f(V,"advanceRecurrence");function X(e,t){const{outOfBand:s}=t;return new Promise((r,n)=>{const o=R(),u=Date.now(),i=new Date(u).toISOString(),l={runId:o,outOfBand:s,startedAtIso:i,startedAtMs:u};if(!y(e.cron_id,E,i,s)){r(null);return}let p;try{if({env:p}=v(e,o),!s&&e.recur!=null){const d=V(e,u);if(d!==null)throw d}}catch(d){I(e,l,"run could not be armed",d).then(r,n);return}let c=null,m=!1;const C={runId:o,replaced:!1,selfCanceled:!1,kill:f(()=>{c!==null?c():m=!0},"kill")};g(e.cron_id,C),q({command:e.command,cwd:e.cwd,env:p,app:e.grantee,isolated:e.isolated,...e.isolated?{nodeId:e.created_by}:e.anchor_node===null?{}:{nodeId:e.anchor_node},timeoutMs:e.run_timeout_s*1e3,headBytes:h}).then(d=>{if(d.pid===null){_(e.cron_id,o),d.completion.then(a=>{I(e,l,"run failed to spawn",a.processError??new Error("bash did not provide a process id")).then(r,n)},n);return}let S=!1;const w=f(()=>{d.completion.then(a=>{const O=C.replaced,x=C.selfCanceled;if(_(e.cron_id,o),S){r(null);return}J(e,{runId:o,outOfBand:s,startedAtIso:i,startedAtMs:u,finishedAtMs:Date.now(),code:a.exitCode,timedOut:a.timedOut,processError:a.processError,stdoutHead:H(a.stdout,a.stdoutTruncated),stderrHead:H(a.stderr,a.stderrTruncated),stdoutSha256:a.stdoutSha256,wasReplaced:O,wasSelfCanceled:x}).then(r,n)},n)},"settleCommand");if(!P(e.cron_id,E,i,d.pid)&&!C.selfCanceled){S=!0,d.terminate(),w();return}c=d.terminate,m&&d.terminate(),w()},d=>{_(e.cron_id,o),I(e,l,"run failed to spawn",d).then(r,n)})})}f(X,"executeCron");function se(e,t={}){const s=new Date(e).toISOString();L(s),B(e,s);const r=[];for(const n of U(s)){const o=X(n,{outOfBand:!1}).then(()=>{});r.push(o),t.register?.(o)}for(const n of r)n.catch(()=>{});return r}f(se,"runDueCrons");export{ie as CRON_OUTPUT_HEAD_BYTES,ue as cronDaemonInstanceId,X as executeCron,ce as markSelfCanceledCronRun,me as recoverStaleCronLeases,se as runDueCrons};
+var $=Object.defineProperty;var c=(e,t)=>$(e,"name",{value:t,configurable:!0});import{randomUUID as v}from"node:crypto";import{CRON_OUTPUT_HEAD_BYTES as g,EMPTY_STDOUT_SHA256 as y,formatCronOutputHead as I}from"./cron/capture.js";import{clearLiveRun as S,cronDaemonInstanceId as M,registerLiveRun as B}from"./cron/live-runs.js";import{closeCronExecution as L,openCronExecution as P}from"./cron/run-socket.js";import{appProcessSandboxed as U}from"../core/spaces/process.js";import{nodeAgentDir as F}from"../core/canvas/paths.js";import{uidForApp as N}from"../core/spaces/users.js";import{buildArmedRunContext as Y}from"./cron/armed-context.js";import{ensureCronFolder as q}from"./cron/folders.js";import{checkCronFire as J,cronPrincipalOf as V}from"./cron/authority.js";import{decideHeldSettle as X,EXIT_HELD as z,HELD_DELIVERED as G}from"./cron/held.js";import{settleRun as x}from"./cron/settle.js";import{applyOverlapPolicies as K,expireDueCrons as Q}from"./cron/passes.js";import{dueClockCrons as W,acquireCronRunLease as Z,attachCronRunPid as j,advanceCron as ee,setCronState as te}from"../core/canvas/crons.js";import{emitEvent as re}from"../core/events/emit.js";import{runAppCommand as ne}from"../core/runtime/run-command.js";import{nextSlotAfter as oe}from"../core/wake.js";import{CRON_OUTPUT_HEAD_BYTES as Me}from"./cron/capture.js";import{cronDaemonInstanceId as be,markSelfCanceledCronRun as ke}from"./cron/live-runs.js";import{recoverStaleCronLeases as ve}from"./cron/lease-recovery.js";async function se(e,t){const{runId:s,outOfBand:r,startedAtIso:o,finishedAtMs:n,code:i,timedOut:l,processError:u}=t,m=!r&&!l&&u==null&&i===z&&!t.wasReplaced&&!t.wasSelfCanceled;let p,f;return t.wasReplaced?(p={kind:"stated",delivered:"replaced (overlap=replace)"},f="release"):m?(p={kind:"stated",delivered:G},f=X(e,o,n)??"release"):(p={kind:"disposed",ctx:{runId:s,startedAtIso:o,exitCode:l?-1:i,timedOut:l,runTimeoutS:e.run_timeout_s,processErrorMessage:u!=null?u.message:null,stdoutHead:t.stdoutHead,stdoutSha256:t.stdoutSha256,stderrHead:t.stderrHead},selfCanceled:t.wasSelfCanceled},f=r?"release":{kind:"clear"}),x(e,{outOfBand:r,outcome:p,record:{runId:s,startedAtIso:o,finishedAtMs:n,durationMs:n-t.startedAtMs,exitCode:l?-1:i,stdoutHead:t.stdoutHead===""?null:t.stdoutHead,stderrHead:u!=null?`run process error: ${u.message}`.slice(0,g):t.stderrHead===""?null:t.stderrHead},lease:f})}c(se,"settleClosedRun");function H(e,t,s,r){const o=Date.now(),n=`${s}: ${r.message}`.slice(0,g);return x(e,{outOfBand:t.outOfBand,outcome:{kind:"disposed",ctx:{runId:t.runId,startedAtIso:t.startedAtIso,exitCode:null,timedOut:!1,runTimeoutS:e.run_timeout_s,processErrorMessage:r.message,stdoutHead:"",stdoutSha256:y,stderrHead:n}},record:{runId:t.runId,startedAtIso:t.startedAtIso,finishedAtMs:o,durationMs:o-t.startedAtMs,exitCode:null,stdoutHead:null,stderrHead:n},lease:"release"})}c(H,"settleUnspawnedRun");function T(e,t){let s;try{s=oe(e.recur,new Date(t))}catch(r){return re({level:"error",event:"cron.recur.unparseable",error:r,fields:{cron_id:e.cron_id,name:e.name,recur:e.recur}}),te(e.cron_id,"paused"),new Error(`unparseable recurrence ${JSON.stringify(e.recur)}: ${r.message} \u2014 this cron cannot be rescheduled until its recurrence is fixed (\`crtr cron cancel\` and re-arm it), so it has been paused.`)}return ee(e.cron_id,s),null}c(T,"advanceRecurrence");function de(e,t){const{outOfBand:s}=t;return new Promise((r,o)=>{const n=v(),i=Date.now(),l=new Date(i).toISOString(),u={runId:n,outOfBand:s,startedAtIso:l,startedAtMs:i};if(!Z(e.cron_id,M,l,s,n)){r(null);return}const m=J(V(e));if(!m.allowed){try{if(!s&&e.recur!=null){const h=T(e,i);if(h!==null)throw h}}catch(h){H(e,u,"run could not be armed",h).then(r,o);return}const d=Date.now();x(e,{outOfBand:s,outcome:{kind:"refused",reason:m.reason,message:m.message},record:{runId:n,startedAtIso:l,finishedAtMs:d,durationMs:d-i,exitCode:null,stdoutHead:null,stderrHead:`${m.reason}: ${m.message}`.slice(0,g)},lease:"release"}).then(r,o);return}let p;try{if(e.isolated&&q(e.grantee,e.cron_id),{env:p}=Y(e,n),!s&&e.recur!=null){const d=T(e,i);if(d!==null)throw d}}catch(d){H(e,u,"run could not be armed",d).then(r,o);return}let f=null,D=!1;const w={runId:n,replaced:!1,selfCanceled:!1,kill:c(()=>{f!==null?f():D=!0},"kill")};B(e.cron_id,w);const C=U()?{executionId:n,cronId:e.cron_id,grantee:e.grantee,sub:e.sub,grantId:e.grant_id,scopes:e.scopes,originRunId:e.origin_run_id,isolated:e.isolated,anchor:e.anchor_node,uid:N(e.grantee),rebuilt:!1,refusal:null}:null,E=c(()=>{C!==null&&L(n)},"closeExecution");c(async()=>(C!==null&&await P(C,e.cwd,e.profile),ne({command:e.command,cwd:e.cwd,env:p,app:e.grantee,isolated:e.isolated,principal:{sub:e.sub,grantId:e.grant_id,scopes:e.scopes},...e.isolated?{folder:e.cwd}:{},...C!==null?{cron:{executionId:n,...e.anchor_node===null||e.isolated?{}:{anchorDir:F(e.anchor_node,e.grantee)}}}:e.anchor_node===null||e.isolated?{}:{nodeId:e.anchor_node},timeoutMs:e.run_timeout_s*1e3,headBytes:g})),"launch")().then(d=>{if(d.pid===null){S(e.cron_id,n),E(),d.completion.then(a=>{H(e,u,"run failed to spawn",a.processError??new Error("bash did not provide a process id")).then(r,o)},o);return}let h=!1;const A=c(()=>{d.completion.then(a=>{const O=w.replaced,b=w.selfCanceled,_=C?.refusal??null;if(S(e.cron_id,n),E(),h){r(null);return}if(_!==null&&!O){const R=Date.now(),k=I(a.stderr,a.stderrTruncated);x(e,{outOfBand:s,outcome:{kind:"refused",reason:_.reason,message:_.message},record:{runId:n,startedAtIso:l,finishedAtMs:R,durationMs:R-i,exitCode:a.exitCode,stdoutHead:I(a.stdout,a.stdoutTruncated)||null,stderrHead:`${_.reason}: ${_.message}
+${k}`.trim().slice(0,g)},lease:"release"}).then(r,o);return}se(e,{runId:n,outOfBand:s,startedAtIso:l,startedAtMs:i,finishedAtMs:Date.now(),code:a.exitCode,timedOut:a.timedOut,processError:a.processError,stdoutHead:I(a.stdout,a.stdoutTruncated),stderrHead:I(a.stderr,a.stderrTruncated),stdoutSha256:a.stdoutSha256,wasReplaced:O,wasSelfCanceled:b}).then(r,o)},o)},"settleCommand");if(!j(e.cron_id,M,l,d.pid)&&!w.selfCanceled){h=!0,d.terminate(),A();return}f=d.terminate,D&&d.terminate(),A()},d=>{S(e.cron_id,n),E(),H(e,u,"run failed to spawn",d).then(r,o)})})}c(de,"executeCron");function Ae(e,t={}){const s=new Date(e).toISOString();Q(s),K(e,s);const r=[];for(const o of W(s)){const n=de(o,{outOfBand:!1}).then(()=>{});r.push(n),t.register?.(n)}for(const o of r)o.catch(()=>{});return r}c(Ae,"runDueCrons");export{Me as CRON_OUTPUT_HEAD_BYTES,be as cronDaemonInstanceId,de as executeCron,ke as markSelfCanceledCronRun,ve as recoverStaleCronLeases,Ae as runDueCrons};

package/dist/daemon/crtrd.js

@@ -1,12 +1,12 @@
-var Se=Object.defineProperty;var s=(t,o)=>Se(t,"name",{value:o,configurable:!0});import{writeFileSync as _e,readFileSync as ke,rmSync as be,mkdirSync as Y,existsSync as k,chmodSync as Ee}from"node:fs";import{createServer as X}from"node:http";import{dirname as Re}from"node:path";import{envCrtrdTcp as Ie}from"../shared/env.js";import{serveCanvasConversationStore as Pe}from"../core/canvas/sessions.js";import{pollRunObservers as xe}from"./runs/observer.js";import{randomBytes as De}from"node:crypto";import{apiSocketPath as Te,crtrHome as Ce}from"../core/canvas/paths.js";import{activeAppUsers as Le}from"../core/spaces/users.js";import{ensureAppSpace as Ae,ensureUserFiles as Ne}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as R}from"../launcher/spec.js";import{launcherRequest as x}from"../launcher/client.js";import{installLauncherEventSink as $e}from"./launcher-events.js";import{initializeEmptyLayout as Me,layoutNeedsMigration as Ge,socketDir as Z}from"../core/layout.js";import{migrateLooseKeys as Oe}from"../core/layout-migrate/loose-keys.js";import{getNode as Fe,listNodesForSupervision as He,setFrozen as Be}from"../core/canvas/index.js";import{createApiServer as je}from"./api/server.js";import{createAppListener as qe}from"./api/app-listener.js";import{installLocalTerminalGrant as ze}from"../core/grants/local.js";import{onRunEventAppended as We}from"../core/runs/events.js";import{runNotices as Ue}from"./run-notices.js";import{createGrantSync as Ke,GRANT_SYNC_INTERVAL_S as Je}from"../core/grants/sync.js";import{runRemovalCascades as Ve}from"../core/grants/remove.js";import{sweepSandboxes as Qe}from"../core/grants/on-sync.js";import{hasSyncedGrants as Ye}from"../core/grants/mirror.js";import{readConfig as Xe}from"../core/config.js";import{reconcileUndeliveredTickets as Ze}from"./human/sweep.js";import{resetHandedOffDeliveries as er}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as rr}from"../core/canvas/daemon-actions.js";import{sweepReviews as tr}from"./review/sweep.js";import{pidfilePath as G}from"./pidfile.js";import{rendererWarning as or}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as nr}from"./supervise-cadence.js";import{createResumeDetector as ir}from"./resume-detector.js";import{BrokerSupervisionReconciler as sr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as ar}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as cr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as lr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as dr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as pr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as mr}from"./reconcilers/action-lane.js";import{endLiveExecution as fr,migrateLegacyExecutionIdentities as ur}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as hr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as vr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as wr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as gr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as yr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Sr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as _r}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as kr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as ee,capturePidCommand as br,captureTeardownSnapshot as re,isPidAlive as Er,isAnyPidAlive as te,killProcessTreePids as oe}from"../core/canvas/pid.js";import{bindFleet as Rr,boundFleet as Ir}from"../core/runtime/fleet.js";import{bindDaemonControl as Pr,clearDaemonControl as xr}from"./control.js";import{execDaemon as Dr,spawnDaemon as Tr}from"./manage.js";import{reconcileAndMigrateBoot as Cr}from"../core/canvas/boot.js";import{resolveInstallId as Lr}from"../core/canvas/install-id.js";import{setInstallId as Ar}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Nr,releaseDaemonOwnership as ne}from"../core/canvas/daemon-owner.js";import{migrate as $r,openDb as ie,withCanvasWrite as Mr,bindDaemonCanvasOpenHook as Gr,clearDaemonCanvasOpenHook as se}from"../core/canvas/db.js";import{brokerSignalBus as O}from"./broker-signals/bus.js";import{brokerSignalStreams as Or}from"./api/handlers/broker-signals.js";import{reconcilePackageDocuments as Fr}from"../core/graph/package-docs.js";import{startRepoSync as Hr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Br,reapStaleSpares as jr}from"../core/runtime/warm-pool.js";import{DaemonFleet as qr}from"./fleet.js";import{BOOT_MIGRATIONS as zr}from"./boot-migrations.js";import{applyJoinedOwner as Wr}from"./joined-owner.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as Ur}from"../core/events/source.js";import{activateCanvasImport as Kr,MigrationBlockedError as Jr,runForcedCanvasImport as Vr}from"../migrations/activation.js";import{convertLegacyProviderStore as Qr,convertLegacyScopedStore as Yr}from"../core/scoped-state/migrate.js";import{userScopeRoot as Xr}from"../core/scope.js";import{secretsDir as Zr}from"../core/layout.js";function F(t,o){return t.includes(` --canvas-home ${o} `)}s(F,"brokerCommandDeclaresCanvasHome");function H(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}s(H,"brokerCommandEpoch");const et=5e3,ae=150;async function rt(t){const o=ee();if(o===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const a=Ce(),m=[];for(const[i,p]of o.processes){const g=p.command;if(p.zombie||!g.includes("broker-cli.js")||!F(g,a)||H(g)===t)continue;const S=g.trim().split(/\s+/).at(-1)??"";S===""||S.endsWith("broker-cli.js")||m.push({pid:i,nodeId:S})}let f=0;const n=[];for(const i of m){const p=br(i.pid);if(p===null||!p.includes("broker-cli.js")||!F(p,a)||H(p)===t||p.trim().split(/\s+/).at(-1)!==i.nodeId)continue;const g=re(i.pid,null);if(!g.reused){try{process.platform==="linux"&&k(R)?await x({operation:"stop",pid:i.pid}):process.kill(i.pid,"SIGTERM")}catch{continue}f++,n.push({pid:i.pid,snap:g})}}const w=Date.now()+et;for(;Date.now()<w&&n.some(i=>te(i.snap.tree.length>0?i.snap.tree:[i.pid]));)await new Promise(i=>setTimeout(i,ae));let v=0;for(const i of n){const p=i.snap.tree.length>0?i.snap.tree:[i.pid];te(p)&&(process.platform==="linux"&&k(R)?await x({operation:"stop",pid:i.pid}):oe(p,"SIGKILL",i.snap.identities),v++)}v>0&&await new Promise(i=>setTimeout(i,ae));const l=ee();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const d=[];for(const[i,p]of l.processes){const g=p.command;p.zombie||!g.includes("broker-cli.js")||F(g,a)&&H(g)!==t&&d.push(i)}if(d.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${d.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:f,sigkilled:v}})}s(rt,"reapPriorEpochBrokers");const tt=5e3;async function ot(t){const o=Array.from(t.entries());if(o.length===0)return;for(const[n]of o){const w=Fe(n)?.status;(w==="active"||w==="idle")&&Be(n,new Date().toISOString())}const a=o.map(([,n])=>({pid:n.pid,snap:re(n.pid,null),exited:n.handle.exited}));for(const n of a)if(!n.snap.reused)try{process.platform==="linux"&&k(R)?await x({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{}const m=new Promise(n=>{setTimeout(n,tt).unref()});await Promise.allSettled(a.map(n=>Promise.race([n.exited.then(()=>{}),m])));const f=[];for(const n of a)Er(n.pid)&&(process.platform==="linux"&&k(R)?await x({operation:"stop",pid:n.pid}):oe(n.snap.tree.length>0?n.snap.tree:[n.pid],"SIGKILL",n.snap.identities),f.push(n));await Promise.all(f.map(n=>n.exited));for(const[n]of o)fr(n,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:a.length,sigkilled:f.length}})}s(ot,"teardownFleet");function ce(){Y(Re(G()),{recursive:!0}),_e(G(),String(process.pid),"utf8")}s(ce,"writePidfile");function le(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}s(le,"isMissingPathError");function de(t){const o=G();let a;try{a=ke(o,"utf8")}catch(m){if(le(m))return;throw m}if(a.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{be(o)}catch(m){if(le(m))return;throw m}}s(de,"removeOwnedPidfileMirror");const nt={acceptsDetachedWork:s(()=>!0,"acceptsDetachedWork"),registerDetached:s(t=>{t.catch(()=>{})},"registerDetached")},B=new sr,it=new ar,st=new cr,at=new lr,ct=new dr,lt=new pr,dt=new mr,pt=new vr,mt=new wr,ft=new gr,ut=new yr,ht=new Sr,vt=new _r;async function wt(t=Date.now(),o=nt){const a=Ir(),m=await B.runFleetPreamble(a);let f,n;try{const v=He();f=v.filter(l=>l.status==="active"||l.status==="idle"),n=v.filter(l=>l.status==="done"||l.status==="canceled"||l.status==="dead")}catch(v){u({level:"error",event:"canvas.node_list.failed",error:v});return}await B.run(t,{rows:f,fleet:a,lifecycle:o}),it.run(t,{rows:f});const w=await st.run(t,{rows:[...f,...n],fleet:a});at.run(t),vt.run(t),ct.run(t,{lifecycle:o}),lt.run(t,{lifecycle:o}),dt.run(t,{lifecycle:o}),pt.run(t,{lifecycle:o}),mt.run(t,{lifecycle:o,fleet:a}),ft.run(t,{lifecycle:o}),xe(),ht.run(t,{fleet:a,lifecycle:o}),await ut.run(t,{rows:f}),B.warnIfSaturated({fleet:a,lifecycle:o,thresholds:m,frozen:w})}s(wt,"superviseTick");async function gt(t){Y(Z(),{recursive:!0,mode:493}),process.platform==="linux"&&Ee(Z(),493);const o=Te();if(k(o))throw new Error(`layout refusal cannot bind occupied socket: ${o}`);const a=X((l,d)=>{if(l.method==="GET"&&l.url==="/healthz"){d.writeHead(200,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}d.writeHead(503,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((l,d)=>{a.once("error",d),a.listen(o,()=>{a.off("error",d),l()})});let m;const f=t.tcp??Ie();if(f){const l=/^(.*):(\d+)$/.exec(f);if(!l||Number(l[2])>65535)throw new Error(`invalid refusal TCP address: ${f}`);const d=X(a.listeners("request")[0]);await new Promise((i,p)=>{d.once("error",p),d.listen(Number(l[2]),l[1],()=>{d.off("error",p),i()})}),m=d}ce();let n;const w=s(()=>(n??=(async()=>{a.closeAllConnections(),m?.closeAllConnections(),await Promise.all([a,m].filter(l=>l!==void 0).map(l=>new Promise((d,i)=>l.close(p=>p?i(p):d())))),de(process.pid),process.off("SIGINT",v),process.off("SIGTERM",v)})(),n),"cleanup"),v=s(()=>{w().then(()=>process.exit(0),l=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(l)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",v),process.on("SIGTERM",v),w}s(gt,"serveLayoutRefusal");async function yt(t={}){if(Ge())return gt(t);Me();const o=Nr(De(16).toString("hex"));if(!o.claimed){if(o.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${o.incumbentPid??"?"})
+var Se=Object.defineProperty;var s=(t,o)=>Se(t,"name",{value:o,configurable:!0});import{writeFileSync as _e,readFileSync as ke,rmSync as be,mkdirSync as Y,existsSync as b,chmodSync as Ee}from"node:fs";import{createServer as X}from"node:http";import{dirname as Re}from"node:path";import{envCrtrdTcp as Ie}from"../shared/env.js";import{serveCanvasConversationStore as Pe}from"../core/canvas/sessions.js";import{pollRunObservers as xe}from"./runs/observer.js";import{randomBytes as De}from"node:crypto";import{apiSocketPath as Te,crtrHome as Ce}from"../core/canvas/paths.js";import{activeAppUsers as Le}from"../core/spaces/users.js";import{ensureAppSpace as Ae,ensureUserFiles as Ne}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as R}from"../launcher/spec.js";import{launcherRequest as x}from"../launcher/client.js";import{installLauncherEventSink as $e}from"./launcher-events.js";import{initializeEmptyLayout as Me,layoutNeedsMigration as Ge,socketDir as Z}from"../core/layout.js";import{migrateLooseKeys as Oe}from"../core/layout-migrate/loose-keys.js";import{getNode as Fe,listNodesForSupervision as He,setFrozen as Be}from"../core/canvas/index.js";import{createApiServer as je}from"./api/server.js";import{createAppListener as qe}from"./api/app-listener.js";import{installLocalTerminalGrant as ze}from"../core/grants/local.js";import{onRunEventAppended as We}from"../core/runs/events.js";import{runNotices as Ue}from"./run-notices.js";import{createGrantSync as Ke,GRANT_SYNC_INTERVAL_S as Je}from"../core/grants/sync.js";import{runRemovalCascades as Ve}from"../core/grants/remove.js";import{sweepSandboxes as Qe}from"../core/grants/on-sync.js";import{rebuildCronExecutions as Ye}from"./cron/run-socket.js";import{hasSyncedGrants as Xe}from"../core/grants/mirror.js";import{readConfig as Ze}from"../core/config.js";import{reconcileUndeliveredTickets as er}from"./human/sweep.js";import{resetHandedOffDeliveries as rr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as tr}from"../core/canvas/daemon-actions.js";import{sweepReviews as or}from"./review/sweep.js";import{pidfilePath as G}from"./pidfile.js";import{rendererWarning as nr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as ir}from"./supervise-cadence.js";import{createResumeDetector as sr}from"./resume-detector.js";import{BrokerSupervisionReconciler as ar}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as cr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as lr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as dr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as pr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as mr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as fr}from"./reconcilers/action-lane.js";import{endLiveExecution as ur,migrateLegacyExecutionIdentities as hr}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as vr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as wr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as gr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as yr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Sr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as _r}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as kr}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as br}from"./structured-requests/boot.js";import{captureLivenessSnapshot as ee,capturePidCommand as Er,captureTeardownSnapshot as re,isPidAlive as Rr,isAnyPidAlive as te,killProcessTreePids as oe}from"../core/canvas/pid.js";import{bindFleet as Ir,boundFleet as Pr}from"../core/runtime/fleet.js";import{bindDaemonControl as xr,clearDaemonControl as Dr}from"./control.js";import{execDaemon as Tr,spawnDaemon as Cr}from"./manage.js";import{reconcileAndMigrateBoot as Lr}from"../core/canvas/boot.js";import{resolveInstallId as Ar}from"../core/canvas/install-id.js";import{setInstallId as Nr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as $r,releaseDaemonOwnership as ne}from"../core/canvas/daemon-owner.js";import{migrate as Mr,openDb as ie,withCanvasWrite as Gr,bindDaemonCanvasOpenHook as Or,clearDaemonCanvasOpenHook as se}from"../core/canvas/db.js";import{brokerSignalBus as O}from"./broker-signals/bus.js";import{brokerSignalStreams as Fr}from"./api/handlers/broker-signals.js";import{reconcilePackageDocuments as Hr}from"../core/graph/package-docs.js";import{startRepoSync as Br}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as jr,reapStaleSpares as qr}from"../core/runtime/warm-pool.js";import{DaemonFleet as zr}from"./fleet.js";import{BOOT_MIGRATIONS as Wr}from"./boot-migrations.js";import{applyJoinedOwner as Ur}from"./joined-owner.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as Kr}from"../core/events/source.js";import{activateCanvasImport as Jr,MigrationBlockedError as Vr,runForcedCanvasImport as Qr}from"../migrations/activation.js";import{convertLegacyProviderStore as Yr,convertLegacyScopedStore as Xr}from"../core/scoped-state/migrate.js";import{userScopeRoot as Zr}from"../core/scope.js";import{secretsDir as et}from"../core/layout.js";function F(t,o){return t.includes(` --canvas-home ${o} `)}s(F,"brokerCommandDeclaresCanvasHome");function H(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}s(H,"brokerCommandEpoch");const rt=5e3,ae=150;async function tt(t){const o=ee();if(o===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const a=Ce(),m=[];for(const[i,p]of o.processes){const g=p.command;if(p.zombie||!g.includes("broker-cli.js")||!F(g,a)||H(g)===t)continue;const S=g.trim().split(/\s+/).at(-1)??"";S===""||S.endsWith("broker-cli.js")||m.push({pid:i,nodeId:S})}let f=0;const n=[];for(const i of m){const p=Er(i.pid);if(p===null||!p.includes("broker-cli.js")||!F(p,a)||H(p)===t||p.trim().split(/\s+/).at(-1)!==i.nodeId)continue;const g=re(i.pid,null);if(!g.reused){try{process.platform==="linux"&&b(R)?await x({operation:"stop",pid:i.pid}):process.kill(i.pid,"SIGTERM")}catch{continue}f++,n.push({pid:i.pid,snap:g})}}const w=Date.now()+rt;for(;Date.now()<w&&n.some(i=>te(i.snap.tree.length>0?i.snap.tree:[i.pid]));)await new Promise(i=>setTimeout(i,ae));let v=0;for(const i of n){const p=i.snap.tree.length>0?i.snap.tree:[i.pid];te(p)&&(process.platform==="linux"&&b(R)?await x({operation:"stop",pid:i.pid}):oe(p,"SIGKILL",i.snap.identities),v++)}v>0&&await new Promise(i=>setTimeout(i,ae));const l=ee();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const d=[];for(const[i,p]of l.processes){const g=p.command;p.zombie||!g.includes("broker-cli.js")||F(g,a)&&H(g)!==t&&d.push(i)}if(d.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${d.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:f,sigkilled:v}})}s(tt,"reapPriorEpochBrokers");const ot=5e3;async function nt(t){const o=Array.from(t.entries());if(o.length===0)return;for(const[n]of o){const w=Fe(n)?.status;(w==="active"||w==="idle")&&Be(n,new Date().toISOString())}const a=o.map(([,n])=>({pid:n.pid,snap:re(n.pid,null),exited:n.handle.exited}));for(const n of a)if(!n.snap.reused)try{process.platform==="linux"&&b(R)?await x({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{}const m=new Promise(n=>{setTimeout(n,ot).unref()});await Promise.allSettled(a.map(n=>Promise.race([n.exited.then(()=>{}),m])));const f=[];for(const n of a)Rr(n.pid)&&(process.platform==="linux"&&b(R)?await x({operation:"stop",pid:n.pid}):oe(n.snap.tree.length>0?n.snap.tree:[n.pid],"SIGKILL",n.snap.identities),f.push(n));await Promise.all(f.map(n=>n.exited));for(const[n]of o)ur(n,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:a.length,sigkilled:f.length}})}s(nt,"teardownFleet");function ce(){Y(Re(G()),{recursive:!0}),_e(G(),String(process.pid),"utf8")}s(ce,"writePidfile");function le(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}s(le,"isMissingPathError");function de(t){const o=G();let a;try{a=ke(o,"utf8")}catch(m){if(le(m))return;throw m}if(a.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{be(o)}catch(m){if(le(m))return;throw m}}s(de,"removeOwnedPidfileMirror");const it={acceptsDetachedWork:s(()=>!0,"acceptsDetachedWork"),registerDetached:s(t=>{t.catch(()=>{})},"registerDetached")},B=new ar,st=new cr,at=new lr,ct=new dr,lt=new pr,dt=new mr,pt=new fr,mt=new wr,ft=new gr,ut=new yr,ht=new Sr,vt=new _r,wt=new kr;async function gt(t=Date.now(),o=it){const a=Pr(),m=await B.runFleetPreamble(a);let f,n;try{const v=He();f=v.filter(l=>l.status==="active"||l.status==="idle"),n=v.filter(l=>l.status==="done"||l.status==="canceled"||l.status==="dead")}catch(v){u({level:"error",event:"canvas.node_list.failed",error:v});return}await B.run(t,{rows:f,fleet:a,lifecycle:o}),st.run(t,{rows:f});const w=await at.run(t,{rows:[...f,...n],fleet:a});ct.run(t),wt.run(t),lt.run(t,{lifecycle:o}),dt.run(t,{lifecycle:o}),pt.run(t,{lifecycle:o}),mt.run(t,{lifecycle:o}),ft.run(t,{lifecycle:o,fleet:a}),ut.run(t,{lifecycle:o}),xe(),vt.run(t,{fleet:a,lifecycle:o}),await ht.run(t,{rows:f}),B.warnIfSaturated({fleet:a,lifecycle:o,thresholds:m,frozen:w})}s(gt,"superviseTick");async function yt(t){Y(Z(),{recursive:!0,mode:493}),process.platform==="linux"&&Ee(Z(),493);const o=Te();if(b(o))throw new Error(`layout refusal cannot bind occupied socket: ${o}`);const a=X((l,d)=>{if(l.method==="GET"&&l.url==="/healthz"){d.writeHead(200,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}d.writeHead(503,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((l,d)=>{a.once("error",d),a.listen(o,()=>{a.off("error",d),l()})});let m;const f=t.tcp??Ie();if(f){const l=/^(.*):(\d+)$/.exec(f);if(!l||Number(l[2])>65535)throw new Error(`invalid refusal TCP address: ${f}`);const d=X(a.listeners("request")[0]);await new Promise((i,p)=>{d.once("error",p),d.listen(Number(l[2]),l[1],()=>{d.off("error",p),i()})}),m=d}ce();let n;const w=s(()=>(n??=(async()=>{a.closeAllConnections(),m?.closeAllConnections(),await Promise.all([a,m].filter(l=>l!==void 0).map(l=>new Promise((d,i)=>l.close(p=>p?i(p):d())))),de(process.pid),process.off("SIGINT",v),process.off("SIGTERM",v)})(),n),"cleanup"),v=s(()=>{w().then(()=>process.exit(0),l=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(l)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",v),process.on("SIGTERM",v),w}s(yt,"serveLayoutRefusal");async function St(t={}){if(Ge())return yt(t);Me();const o=$r(De(16).toString("hex"));if(!o.claimed){if(o.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${o.incumbentPid??"?"})
 `);else{const r=[o.incumbentPid!==void 0?`pid ${o.incumbentPid}`:void 0,o.verdict!==void 0?`verdict ${o.verdict}`:void 0].filter(c=>c!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${o.reason}${e}
-`),o.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(o.reason==="own_identity_indeterminate"||o.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const a=o.claim;let m=null;try{if(await rt(a.token),Yr({kind:"user",root:Xr()}),process.platform==="darwin"){const{readConfig:e,updateConfig:c}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:y}=await import("node:os");c("user",Q=>{Q.paths??={},Q.paths.user_files??=y()})}}await Qr(Zr());for(const e of Oe())process.stderr.write(`[crtrd] ${e}
-`);$r(ie()),Gr(e=>O.install(e));const r=Mr(e=>Fr(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
+`),o.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(o.reason==="own_identity_indeterminate"||o.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const a=o.claim;let m=null;try{if(await tt(a.token),Xr({kind:"user",root:Zr()}),process.platform==="darwin"){const{readConfig:e,updateConfig:c}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:y}=await import("node:os");c("user",Q=>{Q.paths??={},Q.paths.user_files??=y()})}}await Yr(et());for(const e of Oe())process.stderr.write(`[crtrd] ${e}
+`);Mr(ie()),Or(e=>O.install(e));const r=Gr(e=>Hr(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
 `);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
-`);m=Wr(),process.platform==="linux"&&k(R)&&($e(),await x({operation:"users.sync",users:Le()})),hr(a.token)}catch(r){se(),O.close();try{const{released:e}=ne(a);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let f=!0,n,w=Promise.resolve(),v=!1,l,d;const i=new Set;let p=!1,g=!1,S,L,j=!1,A,D,T;const pe=ir();let I,h={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},P="initial startup activation",me=null;function fe(){return h.phase==="ready"?null:{phase:h.phase,message:h.message,blockerPaths:h.blockerPaths}}s(fe,"startupBlock");function b(r){i.add(r),r.finally(()=>{i.delete(r)}).catch(()=>{})}s(b,"registerDetached");const q={acceptsDetachedWork:s(()=>f,"acceptsDetachedWork"),registerDetached:b};Hr(q,ie);async function ue(){try{f=!1,Or.close(),se(),O.close(),l!==void 0&&(clearInterval(l),l=void 0),T!==void 0&&(clearInterval(T),T=void 0);let r;do r=w,await r;while(r!==w);for(;i.size>0;)await Promise.allSettled([...i]);if(d!==void 0&&(await ot(d),d.dispose()),L!==void 0&&!j&&(j=!0,await L.close()),D!==void 0){const e=D;D=void 0,await e.close()}if(A!==void 0){const e=A;A=void 0,await e.close()}if(p||(de(process.pid),p=!0),p&&!g){const{released:e}=ne(a);if(!e)throw new Error("daemon ownership claim was not released");g=!0}}finally{xr(),process.off("SIGINT",U),process.off("SIGTERM",K),process.off("exit",J)}}s(ue,"cleanupAttempt");const N=1e3;let z=!1;function he(){return z?{graceMs:N}:(z=!0,_.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:N}})}),setTimeout(()=>{ve()},N),{graceMs:N})}s(he,"requestHandover");async function ve(){try{await E()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+`);m=Ur(),process.platform==="linux"&&b(R)&&($e(),await x({operation:"users.sync",users:Le()})),vr(a.token)}catch(r){se(),O.close();try{const{released:e}=ne(a);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let f=!0,n,w=Promise.resolve(),v=!1,l,d;const i=new Set;let p=!1,g=!1,S,L,j=!1,A,D,T;const pe=sr();let I,h={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},P="initial startup activation",me=null;function fe(){return h.phase==="ready"?null:{phase:h.phase,message:h.message,blockerPaths:h.blockerPaths}}s(fe,"startupBlock");function k(r){i.add(r),r.finally(()=>{i.delete(r)}).catch(()=>{})}s(k,"registerDetached");const q={acceptsDetachedWork:s(()=>f,"acceptsDetachedWork"),registerDetached:k};Br(q,ie);async function ue(){try{f=!1,Fr.close(),se(),O.close(),l!==void 0&&(clearInterval(l),l=void 0),T!==void 0&&(clearInterval(T),T=void 0);let r;do r=w,await r;while(r!==w);for(;i.size>0;)await Promise.allSettled([...i]);if(d!==void 0&&(await nt(d),d.dispose()),L!==void 0&&!j&&(j=!0,await L.close()),D!==void 0){const e=D;D=void 0,await e.close()}if(A!==void 0){const e=A;A=void 0,await e.close()}if(p||(de(process.pid),p=!0),p&&!g){const{released:e}=ne(a);if(!e)throw new Error("daemon ownership claim was not released");g=!0}}finally{Dr(),process.off("SIGINT",U),process.off("SIGTERM",K),process.off("exit",J)}}s(ue,"cleanupAttempt");const N=1e3;let z=!1;function he(){return z?{graceMs:N}:(z=!0,_.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:N}})}),setTimeout(()=>{ve()},N),{graceMs:N})}s(he,"requestHandover");async function ve(){try{await E()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
 ${e}
-`),process.exit(1)}try{process.platform==="linux"&&k(R)&&(await x({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Dr();const r=await Tr();_.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
+`),process.exit(1)}try{process.platform==="linux"&&b(R)&&(await x({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Tr();const r=await Cr();_.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
 `),process.exit(1)}process.exit(0)}s(ve,"performHandover");function E(){if(S!==void 0)return S;const r=ue();return S=r,r.then(()=>{S===r&&(S=void 0)},()=>{S===r&&(S=void 0)}),r}s(E,"requestCleanup");async function W(r){try{await E(),process.exit(0)}catch(e){process.exitCode=1;const c=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
 ${c}
-`),process.exit(1)}}s(W,"handleSignal");function U(){W("SIGINT")}s(U,"onSigint");function K(){W("SIGTERM")}s(K,"onSigterm");function J(){E().catch(()=>{})}s(J,"onExit");function $(r){f&&(w=w.then(r).then(()=>{},e=>{_.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:e})})}))}s($,"enqueue");function V(){!f||v||(pe.check()&&I&&(u({level:"info",event:"daemon.resume.detected"}),b(I().catch(()=>{}))),v=!0,$(async()=>{if(v=!1,!!f)try{await wt(Date.now(),q)}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}s(V,"enqueueTick");let C=null;async function we(){if(C!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${C instanceof Error?C.message:String(C)}`};if(h.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(h.phase!=="prepared"&&h.phase!=="recovering")throw{status:409,code:"not_prepared",message:h.message};const r=h.phase==="prepared";try{return await M(),h={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw C=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}s(we,"admitStartup");async function ge(r){if(P!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${P})`};P="repair";try{const e=await Vr({dryRun:r.dryRun});if(r.dryRun||e.pending||h.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await M(),h={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(c){const y=c instanceof Error?c.message:String(c);return setTimeout(()=>{E().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:y}}}finally{P=null}}s(ge,"repairMigrations");async function ye(){n=t.intervalMs??nr;const r=a.token;d=new qr({epoch:r,enqueue:$}),Rr(d),h={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:n}})});const e=Lr();if(Ar(e),_.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),_.fresh(()=>{let c;try{c=or()}catch{return}c!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:c.slice(0,2048)}})}),await _.fresh(async()=>{await tr(),await Ze(),er(Date.now()),rr(Date.now())}),_.fresh(()=>{const{reconcile:c,migrated:y}=Cr(ur);c.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:c.cleared.length,same_boot_proven:c.sameBootProven}}),y.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:y.length}})}),_.fresh(()=>{kr();for(const c of zr){const y=c.apply();y.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:c.id,...y.summary}})}}),I){const c=s(()=>{b(I().catch(()=>{}))},"refresh");b(Ve().catch(()=>{})),b(Qe().then(({completion:y})=>y).catch(y=>{u({level:"error",event:"daemon.grants.sweep_failed",error:y})})),c(),T=setInterval(c,Je*1e3),T.unref()}$(()=>{jr(),Br()}),l=setInterval(V,n),V()}s(ye,"completeStartup");function M(){return me??=ye().catch(r=>{throw h={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}s(M,"resumeStartup");try{process.on("SIGINT",U),process.on("SIGTERM",K),process.on("exit",J),Ur(),m!==null&&u({level:"info",event:"daemon.joined_owner.applied",fields:{owner:m.owner,rows:m.rows}}),A=await Pe();const r=a.token;Pr({epoch:r,requestHandover:he,registerDetached:b,startupBlock:fe,admitStartup:we,repairMigrations:ge}),ce(),L=je({tcp:t.tcp}),await L.ready;const e=Xe("user").runtime;e?.issuer?(I=Ke(e.issuer,b),We(c=>Ue().appended(c))):ze(),e!==void 0&&(D=qe(e,()=>Ye(e.issuer.replace(/\/$/,"")),I),await D.ready);try{await Kr()}catch(c){if(P=null,c instanceof Jr)return h={phase:"blocked",message:c.message,blockerPaths:c.blockerPaths},E;throw c}process.platform==="linux"&&k(R)&&(Ne(),await Ae("app:terminal"));try{t.prepare?h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await M(),h={phase:"ready",message:"",blockerPaths:[]})}finally{P=null}return E}catch(r){try{await E()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}s(yt,"runDaemon");var jo=yt;export{F as brokerCommandDeclaresCanvasHome,H as brokerCommandEpoch,jo as default,yt as runDaemon,wt as superviseTick,ot as teardownFleet};
+`),process.exit(1)}}s(W,"handleSignal");function U(){W("SIGINT")}s(U,"onSigint");function K(){W("SIGTERM")}s(K,"onSigterm");function J(){E().catch(()=>{})}s(J,"onExit");function $(r){f&&(w=w.then(r).then(()=>{},e=>{_.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:e})})}))}s($,"enqueue");function V(){!f||v||(pe.check()&&I&&(u({level:"info",event:"daemon.resume.detected"}),k(I().catch(()=>{}))),v=!0,$(async()=>{if(v=!1,!!f)try{await gt(Date.now(),q)}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}s(V,"enqueueTick");let C=null;async function we(){if(C!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${C instanceof Error?C.message:String(C)}`};if(h.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(h.phase!=="prepared"&&h.phase!=="recovering")throw{status:409,code:"not_prepared",message:h.message};const r=h.phase==="prepared";try{return await M(),h={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw C=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}s(we,"admitStartup");async function ge(r){if(P!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${P})`};P="repair";try{const e=await Qr({dryRun:r.dryRun});if(r.dryRun||e.pending||h.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await M(),h={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(c){const y=c instanceof Error?c.message:String(c);return setTimeout(()=>{E().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:y}}}finally{P=null}}s(ge,"repairMigrations");async function ye(){n=t.intervalMs??ir;const r=a.token;d=new zr({epoch:r,enqueue:$}),Ir(d),h={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:n}})});const e=Ar();if(Nr(e),_.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),_.fresh(()=>{let c;try{c=nr()}catch{return}c!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:c.slice(0,2048)}})}),await _.fresh(async()=>{await or(),await er(),rr(Date.now()),tr(Date.now())}),_.fresh(()=>{const{reconcile:c,migrated:y}=Lr(hr);c.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:c.cleared.length,same_boot_proven:c.sameBootProven}}),y.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:y.length}})}),_.fresh(()=>{br();for(const c of Wr){const y=c.apply();y.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:c.id,...y.summary}})}}),k(Ye()),I){const c=s(()=>{k(I().catch(()=>{}))},"refresh");k(Ve().catch(()=>{})),k(Qe().then(({completion:y})=>y).catch(y=>{u({level:"error",event:"daemon.grants.sweep_failed",error:y})})),c(),T=setInterval(c,Je*1e3),T.unref()}$(()=>{qr(),jr()}),l=setInterval(V,n),V()}s(ye,"completeStartup");function M(){return me??=ye().catch(r=>{throw h={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}s(M,"resumeStartup");try{process.on("SIGINT",U),process.on("SIGTERM",K),process.on("exit",J),Kr(),m!==null&&u({level:"info",event:"daemon.joined_owner.applied",fields:{owner:m.owner,rows:m.rows}}),A=await Pe();const r=a.token;xr({epoch:r,requestHandover:he,registerDetached:k,startupBlock:fe,admitStartup:we,repairMigrations:ge}),ce(),L=je({tcp:t.tcp}),await L.ready;const e=Ze("user").runtime;e?.issuer?(I=Ke(e.issuer,k),We(c=>Ue().appended(c))):ze(),e!==void 0&&(D=qe(e,()=>Xe(e.issuer.replace(/\/$/,"")),I),await D.ready);try{await Jr()}catch(c){if(P=null,c instanceof Vr)return h={phase:"blocked",message:c.message,blockerPaths:c.blockerPaths},E;throw c}process.platform==="linux"&&b(R)&&(Ne(),await Ae("app:terminal"));try{t.prepare?h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await M(),h={phase:"ready",message:"",blockerPaths:[]})}finally{P=null}return E}catch(r){try{await E()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}s(St,"runDaemon");var zo=St;export{F as brokerCommandDeclaresCanvasHome,H as brokerCommandEpoch,zo as default,St as runDaemon,gt as superviseTick,nt as teardownFleet};

package/dist/daemon/reconcilers/action-lane.js

@@ -1 +1 @@
-var p=Object.defineProperty;var s=(n,r)=>p(n,"name",{value:r,configurable:!0});import{rmSync as h}from"node:fs";import{claimDaemonAction as y,dueDaemonActions as g,settleDaemonAction as I}from"../../core/canvas/daemon-actions.js";import{getNode as _}from"../../core/canvas/canvas.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{createPageRequest as v,getRequest as u}from"../../core/human/requests.js";import{synthesizedPageDocument as w,writeSynthesizedPage as k}from"../../core/human/page-synth.js";import{preparePage as $}from"../../core/human/page-authoring.js";import{resolvePageComponents as q}from"../../core/config.js";import{spawnChild as b}from"../../core/runtime/spawn.js";import{runDelivery as D}from"../delivery/run-delivery.js";import{deliveryDaemonInstanceId as f}from"./delivery-lane.js";function d(n,r,e){I(n.actionId,n.attempts,r,e,Date.now())}s(d,"settle");function a(n,r,e){d(n,r,{state:"permanent_failed",failure:{kind:"spawn_error",message:e}})}s(a,"failed");async function N(n,r,e){const t=_(e.nodeId);if(t!==null){if(t.frozen_at!==null||t.pi_session_id!==null||t.status==="idle"||t.status==="done"||t.status==="canceled"){d(n,r,{state:"acknowledged"});return}a(n,r,t.status==="dead"?`cron spawn ${e.nodeId} recorded a failed birth`:`cron spawn ${e.nodeId} found an incomplete birth`);return}try{await b({nodeId:e.nodeId,kind:e.kind,cwd:e.cwd,prompt:e.prompt,...e.root?{root:!0,rootLifecycle:e.rootLifecycle}:{parent:e.parent},readiness:"launch",profile:e.profile,isolated:e.isolated===!0,wakeOrigin:e.wakeOrigin}),d(n,r,{state:"acknowledged"})}catch(o){a(n,r,`cron spawn ${e.nodeId} failed: ${o instanceof Error?o.message:String(o)}`),c.fresh(()=>l({level:"error",event:"delivery.cron_spawn_failed",node_id:e.nodeId,error:o,fields:{action_id:n.actionId}}))}}s(N,"deliverSpawn");function P(n,r){const e=k(w(n.title,n.summary,r,!1));try{return $({sourceFile:e.path,source:{sessionName:n.sessionName,...n.profileName===null?{}:{profileName:n.profileName}},productKinds:q(),delivery:{placement:"inline",inbox:!0,reply:!1}})}finally{h(e.dir,{recursive:!0,force:!0})}}s(P,"preparedCronPage");function S(n,r,e){if(u(e.requestId)!==null){d(n,r,{state:"acknowledged"});return}const t=n.body,o=typeof t?.stdout=="string"?t.stdout:null;if(o===null){a(n,r,`cron human ${e.requestId} has no frozen stdout`);return}try{v({request_id:e.requestId,prepared:P(e,o)}),d(n,r,{state:"acknowledged"})}catch(i){a(n,r,`cron human ${e.requestId} failed: ${i instanceof Error?i.message:String(i)}`)}}s(S,"deliverHuman");async function z(n,r){const e=n.adapter;if(n.body===null){a(n,r,`daemon action ${n.actionId} has no body`);return}if(e.adapter==="subprocess"){let t=e.node_id;if(t===void 0){const o=n.idempotencyKey;if(!o.startsWith("human-action:"))throw new Error(`daemon action ${n.actionId} has no originating app`);const i=u(o.slice(13));if(!i)throw new Error(`daemon action ${n.actionId} has no originating human request`);t=i.requester_node_id}await D({argv:e.argv,cwd:e.cwd,document:n.body,attempt:n.attempts,nodeId:t,...e.app?{app:e.app}:{}},o=>d(n,r,o));return}if(e.adapter==="cron_spawn"){await N(n,r,e);return}if(e.adapter==="cron_human"){S(n,r,e);return}a(n,r,`daemon action ${n.actionId} has unknown adapter ${String(e.adapter)}`)}s(z,"deliverAction");class G{static{s(this,"ActionLaneReconciler")}run(r,e){if(!e.lifecycle.acceptsDetachedWork())return;let t;try{t=g(r)}catch(o){c.fresh(()=>l({level:"error",event:"daemon_action.scan_failed",error:o}));return}for(const o of t)try{const i=y(o.actionId,r,f);if(i===null)continue;e.lifecycle.registerDetached(z(i,f).catch(m=>{c.fresh(()=>l({level:"error",event:"daemon_action.failed",error:m,fields:{action_id:i.actionId}}))}))}catch(i){c.fresh(()=>l({level:"error",event:"daemon_action.claim_failed",error:i,fields:{action_id:o.actionId}}))}}}export{G as ActionLaneReconciler};
+var p=Object.defineProperty;var d=(n,r)=>p(n,"name",{value:r,configurable:!0});import{rmSync as h}from"node:fs";import{claimDaemonAction as g,dueDaemonActions as _,settleDaemonAction as y}from"../../core/canvas/daemon-actions.js";import{getNode as I}from"../../core/canvas/canvas.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{createPageRequest as v,getRequest as f}from"../../core/human/requests.js";import{synthesizedPageDocument as w,writeSynthesizedPage as k}from"../../core/human/page-synth.js";import{preparePage as $}from"../../core/human/page-authoring.js";import{resolvePageComponents as b}from"../../core/config.js";import{spawnChild as q}from"../../core/runtime/spawn.js";import{runDelivery as D}from"../delivery/run-delivery.js";import{deliveryDaemonInstanceId as m}from"./delivery-lane.js";import{cronBirthPlacement as P}from"../cron/armed-context.js";import{checkCronBirth as N}from"../cron/authority.js";function s(n,r,e){y(n.actionId,n.attempts,r,e,Date.now())}d(s,"settle");function u(n,r,e){s(n,r,{state:"permanent_failed",failure:{kind:"spawn_error",message:e}})}d(u,"failed");async function S(n,r,e){const i=I(e.nodeId);if(i!==null){if(i.frozen_at!==null||i.pi_session_id!==null||i.status==="idle"||i.status==="done"||i.status==="canceled"){s(n,r,{state:"acknowledged"});return}u(n,r,i.status==="dead"?`cron spawn ${e.nodeId} recorded a failed birth`:`cron spawn ${e.nodeId} found an incomplete birth`);return}const t=e.principal;if(t===void 0){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:e.nodeId,fields:{cron_id:e.cronId??null,action_id:n.actionId,reason:"grant_missing",stage:"delivery"}})),s(n,r,{state:"permanent_failed",failure:{kind:"grant_refused",message:"grant_missing: the queued birth carries no cron principal"}});return}const o=N(t);if(!o.allowed){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:e.nodeId,fields:{cron_id:t.cron_id,action_id:n.actionId,grant_id:t.grant_id,reason:o.reason,stage:"delivery"}})),s(n,r,{state:"permanent_failed",failure:{kind:"grant_refused",message:`${o.reason}: ${o.message}`}});return}try{await q({nodeId:e.nodeId,kind:e.kind,cwd:e.cwd,prompt:e.prompt,...P(t,e.nodeId),readiness:"launch",profile:e.profile,isolated:e.isolated===!0,wakeOrigin:e.wakeOrigin}),s(n,r,{state:"acknowledged"})}catch(a){u(n,r,`cron spawn ${e.nodeId} failed: ${a instanceof Error?a.message:String(a)}`),c.fresh(()=>l({level:"error",event:"delivery.cron_spawn_failed",node_id:e.nodeId,error:a,fields:{action_id:n.actionId}}))}}d(S,"deliverSpawn");function z(n,r){const e=k(w(n.title,n.summary,r,!1));try{return $({sourceFile:e.path,source:{sessionName:n.sessionName,...n.profileName===null?{}:{profileName:n.profileName}},productKinds:b(),delivery:{placement:"inline",inbox:!0,reply:!1}})}finally{h(e.dir,{recursive:!0,force:!0})}}d(z,"preparedCronPage");function A(n,r,e){if(f(e.requestId)!==null){s(n,r,{state:"acknowledged"});return}const i=n.body,t=typeof i?.stdout=="string"?i.stdout:null;if(t===null){u(n,r,`cron human ${e.requestId} has no frozen stdout`);return}try{v({request_id:e.requestId,prepared:z(e,t)}),s(n,r,{state:"acknowledged"})}catch(o){u(n,r,`cron human ${e.requestId} failed: ${o instanceof Error?o.message:String(o)}`)}}d(A,"deliverHuman");async function C(n,r){const e=n.adapter;if(n.body===null){u(n,r,`daemon action ${n.actionId} has no body`);return}if(e.adapter==="subprocess"){let i=e.node_id;if(i===void 0){const t=n.idempotencyKey;if(!t.startsWith("human-action:"))throw new Error(`daemon action ${n.actionId} has no originating app`);const o=f(t.slice(13));if(!o)throw new Error(`daemon action ${n.actionId} has no originating human request`);i=o.requester_node_id}await D({argv:e.argv,cwd:e.cwd,document:n.body,attempt:n.attempts,nodeId:i,...e.app?{app:e.app}:{}},t=>s(n,r,t));return}if(e.adapter==="cron_spawn"){await S(n,r,e);return}if(e.adapter==="cron_human"){A(n,r,e);return}u(n,r,`daemon action ${n.actionId} has unknown adapter ${String(e.adapter)}`)}d(C,"deliverAction");class T{static{d(this,"ActionLaneReconciler")}run(r,e){if(!e.lifecycle.acceptsDetachedWork())return;let i;try{i=_(r)}catch(t){c.fresh(()=>l({level:"error",event:"daemon_action.scan_failed",error:t}));return}for(const t of i)try{const o=g(t.actionId,r,m);if(o===null)continue;e.lifecycle.registerDetached(C(o,m).catch(a=>{c.fresh(()=>l({level:"error",event:"daemon_action.failed",error:a,fields:{action_id:o.actionId}}))}))}catch(o){c.fresh(()=>l({level:"error",event:"daemon_action.claim_failed",error:o,fields:{action_id:t.actionId}}))}}}export{T as ActionLaneReconciler};

package/dist/daemon/reconcilers/broker-supervision.d.ts

@@ -36,6 +36,16 @@ export interface SaturationContext {
 export declare const YIELD_STALL_GRACE_MS: number;
 export declare const WEDGE_QUIET_MS: number;
 export declare const WEDGE_CPU_MAX_PERCENT = 2;
+/** Whether a fatal provider fault ends the node: a parentless terminal node's run has failed,
+ *  and an app run's root (resident, `run_id` = its own id) settles once a model call is refused
+ *  `cap_exhausted` (D2-VOC-18), so the app sees the cap instead of a run left idle. Any other
+ *  node stays alive and recovers on its next wake. */
+export declare function settlesOnFatalProviderFault(meta: {
+    node_id: string;
+    parent?: string | null;
+    lifecycle?: NodeRow['lifecycle'] | null;
+    run_id?: string | null;
+} | null, code: string | undefined): boolean;
 export declare function unattendedParkMsForDaemon(): number;
 export declare function parkSummaryGraceMsForDaemon(): number;
 export type WedgeVerdict = 'leave' | 'pending' | 'wedged';

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var q=Object.defineProperty;var p=(l,e)=>q(l,"name",{value:e,configurable:!0});import{existsSync as Y}from"node:fs";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{launcherRequest as K}from"../../launcher/client.js";import{execFile as V}from"node:child_process";import{promisify as z}from"node:util";import{getNode as N,isWarmSpare as j}from"../../core/canvas/index.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as J}from"../../core/canvas/node-agent-paths.js";import{fullName as I}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Q}from"../../core/runtime/lifecycle.js";import{readFault as R,readLiveTurn as E,recordFault as X}from"../../core/canvas/recovery.js";import{errorClassFromFault as Z}from"../../core/fault-classifier.js";import{nextFaultRetry as ee}from"../../core/runtime/fault-recovery.js";import{publishNotice as x}from"../../core/runtime/close.js";import{headlessBrokerHost as k}from"../../core/runtime/host.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as O}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as te,resetStaleExecutionMail as ne}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as re}from"../../core/bash-jobs.js";import{focusOf as ie}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ae}from"../api/viewer-presence.js";import{deliverLive as se}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as le}from"../../core/runtime/broker-request.js";import{formatCard as oe,PARK_SUMMARY_PROMPT as de}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ue}from"../../core/runtime/fleet.js";import{readConfig as ce}from"../../core/config.js";import{beginPendingPark as fe,consumePendingPark as y,pendingParkIds as pe,pendingParkStartedAt as me}from"../park-pending.js";import{shouldParkAfterSummary as he}from"../park-activity.js";import{applyParkEvent as b,parkEvent as _e}from"./live-obligation.js";import{envNoDaemonAutostart as ge,envTestUnattendedParkMs as ke,parkSummaryGraceMs as ve,envUnattendedParkMs as ye}from"../../shared/env.js";const A=z(V);function be(l){return E(l)?.busy===!0}p(be,"liveBusy");function lt(l){const e=E(l)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(lt,"liveTurnProgressMs");function L(l,e){return e===null?null:`${l}:${e}`}p(L,"incarnationOf");function F(l){const e=E(l.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=l.launched_at===null||l.launched_at===void 0?Number.NaN:Date.parse(l.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}p(F,"busySinceForIncarnation");const G=3*6e4,W=2e4,C=20*6e4,we=2,Se=5*6e4;function Ne(){const l=ye();if(l!==void 0)return l;const e=ce("user").lifecycle.unattendedParkMs;return ge()?ke()??e:e}p(Ne,"unattendedParkMsForDaemon");function U(){return ve()}p(U,"parkSummaryGraceMsForDaemon");function Ae(l,e,t,a){return l!==!0||e!=="refresh"||t?"leave":a===null||a<G?"pending":"kill"}p(Ae,"yieldStallVerdict");function Pe(l){return!l.busy||l.quietForMs===null||l.quietForMs<C?"leave":l.cpuPercent===null||l.hasDescendants!==!1&&l.cpuPercent>we?"pending":"wedged"}p(Pe,"wedgeVerdict");function B(l){const e=new Map,t=new Map;for(const a of l.split(`
-`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const s=Number(n[0]),r=Number(n[1]),o=Number(n[2]);if(!Number.isFinite(s)||!Number.isFinite(r)||!Number.isFinite(o))continue;e.set(s,o);const c=t.get(r);c===void 0?t.set(r,[s]):c.push(s)}return{byPid:e,childrenOf:t}}p(B,"parsePsTree");function $e(l,e){const{byPid:t,childrenOf:a}=B(l);if(!t.has(e))return null;let i=0;const n=[e],s=new Set;for(;n.length>0;){const r=n.pop();if(s.has(r))continue;s.add(r),i+=t.get(r)??0;const o=a.get(r);o!==void 0&&n.push(...o)}return i}p($e,"sumTreeCpu");function Me(l,e){const{byPid:t,childrenOf:a}=B(l);return t.has(e)?(a.get(e)??[]).length>0:null}p(Me,"treeHasDescendants");class ot{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&Y(H)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return K({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return ue()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=O.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Ne();for(const i of t.rows){const n=i.node_id,s=t.fleet.get(n);if(s===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await O.fresh(async()=>{try{await this.handleYieldStall(i,s,t.fleet,e),await this.handleWedgeDetection(i,s,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,N(n),s.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),f({level:"error",event:"broker.supervision.failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of pe())t.fleet.has(i)||y(i);ne(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||re(J(e)).length>0||ie(e)!==null||ae(e))return!0;const n=R(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ee(n,i)!==null)?!0:te(e)}handleUnattendedParking(e,t,a,i,n,s){if(j(e)){this.unattendedSince.delete(e),y(e);return}if(t===null){this.unattendedSince.delete(e),y(e);return}const r=this.parkResetCondition(e,t,a,i),o=me(e);if(o!==void 0){if(i-o<U())return;const m=y(e);if(m===void 0||(this.unattendedSince.delete(e),!he(e,t,m)))return;f({level:"warn",event:"broker.park.summary_expired",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},grace_ms:U()}}),b(e,"park"),k.teardown(e);return}if(r||be(e)){this.unattendedSince.delete(e);return}const c=this.unattendedSince.get(e);if(c===void 0){this.unattendedSince.set(e,i);return}if(i-c<s)return;this.unattendedSince.delete(e);const d=_e(e,t.lifecycle);if(d==="release"){b(e,d),k.teardown(e);return}if(d==="cancel"){b(e,d),k.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){b(e,"park"),k.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(fe(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}async requestParkSummary(e){try{const t=await se(e,oe("park",{},de),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(y(e)===void 0)return;if(t instanceof le){b(e,"park"),k.teardown(e);return}f({level:"error",event:"broker.park.summary_failed",...u(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Se)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:s}=t,r=a.stalled+a.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:s,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const o=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,c=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${s})${o} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async m=>{if(m!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",m,c],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",m,c],{encoding:"utf8",timeout:2e3})}catch(h){f({level:"error",event:"broker.capacity.tmux_display_failed",error:h,fields:{client:m}})}}}))}catch(d){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,s=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||s===null){s!==null&&this.yieldStalls.delete(s);return}let r=this.yieldStalls.get(s);if(r===void 0){r={since:i},this.yieldStalls.set(s,r);return}if(Ae(!0,e.intent,!1,i-r.since)!=="kill")return;const o=N(n);if(!(a.get(n)!==t||o?.pi_pid!==t.pid||o.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:G}})}else i-r.termedAt>=W&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:W,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...u(e)?{node_id:e}:{},error:i,fields:{...u(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=$e(t,e),i=Me(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,s=L(t.pid,e.pi_pid_identity),r=F(e);if(r===null||s===null){s!==null&&this.wedgeNotifiedAt.delete(s);return}const o=i-r,c=o>=C?await this.sampleTree(t.pid):null,d=N(n),m=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||m!==r)return;const h=c?.cpuPercent??null,_=c?.hasDescendants??null;if(Pe({busy:!0,quietForMs:o,cpuPercent:h,hasDescendants:_})!=="wedged"||this.wedgeNotifiedAt.get(s)===r)return;this.wedgeNotifiedAt.set(s,r);const w=I(d),S=Math.round((o??C)/6e4),P=_??!0,v=!P&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let $=!1,M=!1,g;if(P)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{$=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){f({level:"error",event:"broker.wedge.signal_failed",...u(n)?{node_id:n}:{},error:T,fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}f({level:"error",event:"broker.wedge.detected",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:o,cpu_percent:h,has_descendants:P,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}}});try{X(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:g})}catch(T){f({level:"error",event:"broker.wedge.fault_record_failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}x(n,"notice.crash",g,{reason:"child-wedged",child:n,quiet_ms:o,cpu_percent:h,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}})}handleFatalFault(e){const t=R(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=N(e),i=a!==null?I(a):e,n=t.kind==="auth",s=(t.message.split(`
-`)[0]??"").slice(0,200),r=t.providerError,o=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),c=t.kind==="context-overflow",d=a?.parent==null&&a?.lifecycle==="terminal",m=c||d,h=c?`Child failed \u2014 ${i} (${e}) exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${s}`:`Child hit a fatal provider fault \u2014 ${i} (${e}) got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); the child recovers on its next wake once the auth reloads. ":""))+`Error: ${s}.${o}`;m&&(Q(e,"crash",{reason:c?"context_overflow":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:Z(t).class}}}),k.teardown(e).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))),this.fatalFaultNotifiedAt.set(e,t.since),f({level:"error",event:"fault.fatal.fanout",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:c,terminal_provider_fault:d}}),x(e,"notice.crash",h,{reason:c?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:s,...r===void 0?{}:{provider_error:r}})}}export{ot as BrokerSupervisionReconciler,we as WEDGE_CPU_MAX_PERCENT,C as WEDGE_QUIET_MS,G as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,$e as sumTreeCpu,Me as treeHasDescendants,Ne as unattendedParkMsForDaemon,Pe as wedgeVerdict,Ae as yieldStallVerdict};
+var q=Object.defineProperty;var p=(s,e)=>q(s,"name",{value:e,configurable:!0});import{existsSync as Y}from"node:fs";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{launcherRequest as K}from"../../launcher/client.js";import{execFile as V}from"node:child_process";import{promisify as z}from"node:util";import{getNode as N,isWarmSpare as j}from"../../core/canvas/index.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as J}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as I}from"../../core/canvas/pid.js";import{transition as Q}from"../../core/runtime/lifecycle.js";import{readFault as D,readLiveTurn as E,recordFault as X}from"../../core/canvas/recovery.js";import{errorClassFromFault as Z}from"../../core/fault-classifier.js";import{nextFaultRetry as ee}from"../../core/runtime/fault-recovery.js";import{publishNotice as O}from"../../core/runtime/close.js";import{headlessBrokerHost as k}from"../../core/runtime/host.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as te,resetStaleExecutionMail as ne}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as re}from"../../core/bash-jobs.js";import{focusOf as ie}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ae}from"../api/viewer-presence.js";import{deliverLive as se}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as oe}from"../../core/runtime/broker-request.js";import{formatCard as le,PARK_SUMMARY_PROMPT as de}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ue}from"../../core/runtime/fleet.js";import{readConfig as ce}from"../../core/config.js";import{beginPendingPark as fe,consumePendingPark as y,pendingParkIds as pe,pendingParkStartedAt as me}from"../park-pending.js";import{shouldParkAfterSummary as he}from"../park-activity.js";import{applyParkEvent as b,parkEvent as _e}from"./live-obligation.js";import{envNoDaemonAutostart as ge,envTestUnattendedParkMs as ke,parkSummaryGraceMs as ve,envUnattendedParkMs as ye}from"../../shared/env.js";const P=z(V);function be(s){return E(s)?.busy===!0}p(be,"liveBusy");function lt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(lt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function F(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}p(F,"busySinceForIncarnation");const G=3*6e4,W=2e4,x=20*6e4,we=2,Se=5*6e4;function Ne(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Ne,"settlesOnFatalProviderFault");function Pe(){const s=ye();if(s!==void 0)return s;const e=ce("user").lifecycle.unattendedParkMs;return ge()?ke()??e:e}p(Pe,"unattendedParkMsForDaemon");function U(){return ve()}p(U,"parkSummaryGraceMsForDaemon");function Ae(s,e,t,a){return s!==!0||e!=="refresh"||t?"leave":a===null||a<G?"pending":"kill"}p(Ae,"yieldStallVerdict");function $e(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>we?"pending":"wedged"}p($e,"wedgeVerdict");function B(s){const e=new Map,t=new Map;for(const a of s.split(`
+`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),r=Number(n[1]),l=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(r)||!Number.isFinite(l))continue;e.set(o,l);const c=t.get(r);c===void 0?t.set(r,[o]):c.push(o)}return{byPid:e,childrenOf:t}}p(B,"parsePsTree");function Me(s,e){const{byPid:t,childrenOf:a}=B(s);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const r=n.pop();if(o.has(r))continue;o.add(r),i+=t.get(r)??0;const l=a.get(r);l!==void 0&&n.push(...l)}return i}p(Me,"sumTreeCpu");function Te(s,e){const{byPid:t,childrenOf:a}=B(s);return t.has(e)?(a.get(e)??[]).length>0:null}p(Te,"treeHasDescendants");class dt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&Y(H)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return K({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return ue()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=R.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Pe();for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n);if(o===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await R.fresh(async()=>{try{await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,N(n),o.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),f({level:"error",event:"broker.supervision.failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of pe())t.fleet.has(i)||y(i);ne(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||re(J(e)).length>0||ie(e)!==null||ae(e))return!0;const n=D(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ee(n,i)!==null)?!0:te(e)}handleUnattendedParking(e,t,a,i,n,o){if(j(e)){this.unattendedSince.delete(e),y(e);return}if(t===null){this.unattendedSince.delete(e),y(e);return}const r=this.parkResetCondition(e,t,a,i),l=me(e);if(l!==void 0){if(i-l<U())return;const m=y(e);if(m===void 0||(this.unattendedSince.delete(e),!he(e,t,m)))return;f({level:"warn",event:"broker.park.summary_expired",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},grace_ms:U()}}),b(e,"park"),k.teardown(e);return}if(r||be(e)){this.unattendedSince.delete(e);return}const c=this.unattendedSince.get(e);if(c===void 0){this.unattendedSince.set(e,i);return}if(i-c<o)return;this.unattendedSince.delete(e);const d=_e(e,t.lifecycle);if(d==="release"){b(e,d),k.teardown(e);return}if(d==="cancel"){b(e,d),k.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){b(e,"park"),k.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(fe(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}async requestParkSummary(e){try{const t=await se(e,le("park",{},de),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(y(e)===void 0)return;if(t instanceof oe){b(e,"park"),k.teardown(e);return}f({level:"error",event:"broker.park.summary_failed",...u(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Se)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,r=a.stalled+a.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const l=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,c=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${l} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await P("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
+`).map(async m=>{if(m!=="")try{await P("tmux",["display-message","-C","-d","10000","-c",m,c],{encoding:"utf8",timeout:2e3})}catch{try{await P("tmux",["display-message","-c",m,c],{encoding:"utf8",timeout:2e3})}catch(h){f({level:"error",event:"broker.capacity.tmux_display_failed",error:h,fields:{client:m}})}}}))}catch(d){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||o===null){o!==null&&this.yieldStalls.delete(o);return}let r=this.yieldStalls.get(o);if(r===void 0){r={since:i},this.yieldStalls.set(o,r);return}if(Ae(!0,e.intent,!1,i-r.since)!=="kill")return;const l=N(n);if(!(a.get(n)!==t||l?.pi_pid!==t.pid||l.pi_pid_identity!==e.pi_pid_identity||I(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:G}})}else i-r.termedAt>=W&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:W,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...u(e)?{node_id:e}:{},error:i,fields:{...u(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await P("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=Me(t,e),i=Te(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity),r=F(e);if(r===null||o===null){o!==null&&this.wedgeNotifiedAt.delete(o);return}const l=i-r,c=l>=x?await this.sampleTree(t.pid):null,d=N(n),m=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||m!==r)return;const h=c?.cpuPercent??null,_=c?.hasDescendants??null;if($e({busy:!0,quietForMs:l,cpuPercent:h,hasDescendants:_})!=="wedged"||this.wedgeNotifiedAt.get(o)===r)return;this.wedgeNotifiedAt.set(o,r);const w=C(d),S=Math.round((l??x)/6e4),A=_??!0,v=!A&&I(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let $=!1,M=!1,g;if(A)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{$=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){f({level:"error",event:"broker.wedge.signal_failed",...u(n)?{node_id:n}:{},error:T,fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}f({level:"error",event:"broker.wedge.detected",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:l,cpu_percent:h,has_descendants:A,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}}});try{X(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:g})}catch(T){f({level:"error",event:"broker.wedge.fault_record_failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}O(n,"notice.crash",g,{reason:"child-wedged",child:n,quiet_ms:l,cpu_percent:h,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}})}handleFatalFault(e){const t=D(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=N(e),i=a!==null?C(a):e,n=t.kind==="auth",o=(t.message.split(`
+`)[0]??"").slice(0,200),r=t.providerError,l=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),c=t.kind==="context-overflow",d=Ne(a,typeof r?.code=="string"?r.code:void 0),m=c||d,h=c?`Child failed \u2014 ${i} (${e}) exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`Child hit a fatal provider fault \u2014 ${i} (${e}) got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); the child recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${l}`;m&&(Q(e,"crash",{reason:c?"context_overflow":r?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:Z(t).class}}}),k.teardown(e).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))),this.fatalFaultNotifiedAt.set(e,t.since),f({level:"error",event:"fault.fatal.fanout",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:c,terminal_provider_fault:d}}),O(e,"notice.crash",h,{reason:c?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:o,...r===void 0?{}:{provider_error:r}})}}export{dt as BrokerSupervisionReconciler,we as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,G as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,Ne as settlesOnFatalProviderFault,Me as sumTreeCpu,Te as treeHasDescendants,Pe as unattendedParkMsForDaemon,$e as wedgeVerdict,Ae as yieldStallVerdict};

package/dist/daemon/reconcilers/cron-lane.d.ts

@@ -3,5 +3,6 @@ export interface CronLaneContext {
     lifecycle: DetachedWorkLifecycle;
 }
 export declare class CronLaneReconciler {
+    private lastFolderSweep;
     run(now: number, ctx: CronLaneContext): void;
 }