@north-light/cli 0.3.493 → 0.3.495

package/dist/core/runtime/spawn.js

@@ -1,3 +1,3 @@
-var oe=Object.defineProperty;var a=(e,o)=>oe(e,"name",{value:o,configurable:!0});import{accessSync as re,constants as ie}from"node:fs";import{isAbsolute as te,resolve as de,join as ae}from"node:path";import{appSpace as se}from"../layout.js";import{ApiError as z}from"../../api/errors.js";import{spawnNode as ce,forkSourceNode as le,currentNodeContext as fe,rootOfSpine as ue,newNodeId as me,preflightNodeId as he}from"./nodes.js";import{resolveProfileOperand as we}from"../profiles/manifest.js";import{selectProfileForCwd as pe,selectProfileForCwdReadOnly as ke}from"../profiles/select.js";import{buildLaunchSpecAsync as U,buildPiArgv as _e}from"./launch.js";import{createManagedWorktreeAsync as ve,rollbackManagedWorktreeAsync as ge}from"../worktree-mutation-async.js";import{usage as u,brokerLaunchFailed as R}from"../errors.js";import{writeGoal as ye}from"./kickoff.js";import{appendSituationalContext as be,formatSituationalProse as xe}from"./situational-context.js";import{ensureRoadmap as Ce}from"./roadmap.js";import{buildWakeBearings as Fe}from"./bearings.js";import{encodeKickoffOrigin as $e,KICKOFF_ORIGIN_ENV as Se}from"./stamp/protocol.js";import{beginExecution as Ee,crtrHome as Oe,getNode as f,fullName as V,jobDir as Re,recordExecutionPid as Pe,abortExecution as D,setFrozen as Ie}from"../canvas/index.js";import{contextDir as B,nodeAgentDir as A}from"../canvas/node-agent-paths.js";import{copyTreeBeneath as Ne}from"../spaces/open-beneath.js";import{conversationIdFromPath as P,conversationStore as v}from"../conversation-store/index.js";import{boundFleet as De,brokerThresholdsForDaemon as Be}from"./fleet.js";import{emitEvent as Ae}from"../events/emit.js";import{openViewerWindow as Le,focusOf as Te,windowOfPane as je}from"./placement.js";import{waitForBrokerViewSocket as We}from"./placement-tmux.js";import{transition as g}from"./lifecycle.js";import{beginBootFaultAttempt as Me}from"../canvas/recovery.js";import{headlessBrokerHost as L}from"./host.js";import{ensureDaemon as He}from"../../daemon/manage.js";import{readConfig as Ke}from"../config.js";import{modelRequestFromConfig as qe,unregisteredConcreteModel as ze}from"../model-routes.js";import{openModelRegistry as Ue}from"./model-registry.js";import{MODEL_SPEC_NEXT as Ve}from"./model-selection.js";import{forkNodeLedger as Xe}from"./ledger.js";import{discoverLifecycleHookRegistry as Ge}from"../command-hooks/discovery.js";import{invokeLifecycleHooks as Je}from"../command-hooks/transport/exec-lifecycle.js";function y(e){return{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}a(y,"launchFailureOutcome");async function Qe(e,o,d){try{return await L.launch(e,o,d)}catch(r){try{g(e,"crash",{reason:"launch_failed",outcome:y(r)})}catch{}const t=r.message;throw R(t.startsWith("[broker]")?t:`[broker] refusing to launch ${e}: ${t}`,"Inspect the cause above, fix the launch recipe or environment, then retry.")}}a(Qe,"launchBrokerOrCrash");function Ye(e,o){try{g(e,"crash",{reason:"launch_failed",outcome:y(o)})}catch{}try{L.teardown(e)}catch{}}a(Ye,"crashAndTearDown");function Ze(e){const o=v().list().map(({conversation_id:t,path:s})=>({id:t,path:s})),d=o.filter(t=>t.id===e),r=d.length>0?d:o.filter(t=>t.id.startsWith(e));if(r.length===0)throw u(`--fork-from '${e}': no pi session with that id found.`,{next:"Pass a live node id, a real session uuid/prefix, or an absolute .jsonl session path."});if(r.length>1)throw u(`--fork-from '${e}': ambiguous \u2014 ${r.length} sessions match that id prefix; pass the full id or the .jsonl path.`,{next:"Pass the full session uuid or an absolute .jsonl path to disambiguate."});return r[0].path}a(Ze,"resolveSessionUuid");function X(e){const o=e.trim();if(o==="")throw u("--fork-from requires a node id, session file, or session uuid.",{next:"Pass a live node id, an absolute .jsonl session path, or a session uuid/prefix."});if(o.includes("/")||o.endsWith(".jsonl")){const r=P(te(o)?o:de(process.cwd(),o));if(r===null)throw u(`--fork-from '${o}': invalid conversation path.`,{next:"Pass a conversation file path or id."});return v().header(r).path}const d=f(o);if(d!==null){const r=d.pi_session_file;if(r==null||r==="")throw u(`node ${o} has no pi session yet \u2014 it has not captured a session FILE to fork from (the broker forks a .jsonl path, not a bare id).`,{next:"Pick a node that has captured a session, or drop --fork-from."});const t=P(r);if(t===null)throw u(`node ${o} has no valid conversation id.`,{next:"Pick a node with a captured conversation."});return v().header(t).path}return Ze(o)}a(X,"resolveForkSource");async function en(e,o,d,r={}){return e===null?null:e!==void 0&&e!==""?we(e).profileId:o!==null?f(o)?.profile_id??null:r.readOnlySelection===!0?ke(d):pe(d)}a(en,"resolveProfileId");function G(e){const o=e.modelIntent===void 0?void 0:`${e.modelIntent.family}/${e.modelIntent.strength}`;return qe(e.model,o,e.modelExact===!0)}a(G,"requestFromLaunch");async function J(e,o,d){if(e===void 0||e.kind==="logical")return;const r=ze(e,d??await Ue(),o);if(r!==void 0)throw u(`model '${r.request.spec}' is not registered in this daemon.`,{received:r.request.spec,next:Ve})}a(J,"assertLaunchModelRegistered");function nn(e,o,d){const r=e??o??null;if(!d&&r===null)throw new Error("spawnChild requires a calling node (CRTR_NODE_ID) or an explicit parent");return r}a(nn,"resolveSpawner");async function Q(e,o,d){try{He()}catch{}const r=fe(),t=e.root===!0,s=nn(e.parent,r.nodeId??null,t),c=e.mode??"base",h=e.rootLifecycle??(t?"resident":"terminal"),Y=t&&h==="resident",T=s===null?null:f(s),Z=e.creator===void 0||e.creator===null?null:f(e.creator);e.nodeId!==void 0&&he(e.nodeId);const w=e.forkFrom!==void 0?X(e.forkFrom):void 0,j=w===void 0?null:le(w),I=[T,Z,j].some(l=>l?.isolated===!0||l?.run_id!=null&&f(l.run_id)?.isolated===!0);if(I&&e.isolated===!1)throw new z(403,"isolated_run","isolation cannot be turned off under an isolated run",{param:"isolated"});const C=I||e.isolated===!0;if(C&&e.worktree===!0)throw new z(400,"invalid_request","isolated nodes cannot use a worktree",{param:"worktree"});const F=e.worktree===!0,p=C||F?e.nodeId??me():e.nodeId,$=C?e.run!==void 0&&t&&!I?e.cwd:ae(se(e.run?.grantee??T?.grantee??"app:terminal").nodes,p):e.cwd,b=await en(e.profile,s,$),W=(await U(e.kind,c,{lifecycle:h,hasManager:!t,model:e.model,modelOverride:e.model!==void 0,cwd:$,profileId:b})).launch;await J(G(W),{cwd:$,profileId:b});const M=j?.node_id??null;let k,S=$;try{F&&(k=await ve(e.worktreeCwd??e.cwd,p,e.worktreeBase),S=k.path);const l=F?(await U(e.kind,c,{lifecycle:h,hasManager:!t,model:e.model,modelOverride:e.model!==void 0,cwd:S,profileId:b})).launch:W;F&&await J(G(l),{cwd:S,profileId:b});const n=await ce({kind:e.kind,mode:c,lifecycle:h,cwd:S,name:e.name??e.kind,description:e.description,parent:t?null:s,creator:e.creator,...e.actor===void 0?{}:{actor:e.actor},forkSourceNode:M,spawnedBy:t?s??void 0:void 0,profile_id:b,...e.scopes===void 0?{}:{scopes:e.scopes},isolated:C,...e.run===void 0?{}:{run:e.run},...e.withinCreate===void 0?{}:{withinCreate:e.withinCreate},forkFrom:e.forkFrom,forkSourceFile:w,modelOverride:e.model===void 0?void 0:l.model,...k!==void 0?{managedWorktree:k}:{},...p!==void 0?{nodeId:p}:{},...e.reviewBinding===void 0?{}:{reviewBinding:e.reviewBinding},...e.deadlineAt===void 0?{}:{deadlineAt:e.deadlineAt},...e.outcomeDelivery===void 0?{}:{outcomeDelivery:e.outcomeDelivery},launch:l});if(d?.(n.node_id),e.forkFrom!==void 0)try{Xe(n.node_id,M??null)}catch(i){throw g(n.node_id,"crash",{reason:"launch_failed",outcome:y(i)}),i}if(ye(n.node_id,e.prompt),e.situationalContext!==void 0&&be(n.node_id,xe(e.situationalContext)),o!==void 0)try{o(n.node_id)}catch(i){throw g(n.node_id,"crash",{reason:"launch_failed",outcome:y(i)}),i}const N=e.prompt!=="",ee=`${N&&e.wakeOrigin!==void 0?`${Fe(e.wakeOrigin)}
+var ie=Object.defineProperty;var a=(e,o)=>ie(e,"name",{value:o,configurable:!0});import{accessSync as te,constants as de}from"node:fs";import{isAbsolute as ae,resolve as se,join as ce}from"node:path";import{appSpace as le}from"../layout.js";import{ApiError as V}from"../../api/errors.js";import{spawnNode as fe,forkSourceNode as ue,currentNodeContext as me,rootOfSpine as he,newNodeId as we,preflightNodeId as pe}from"./nodes.js";import{resolveProfileOperand as ke}from"../profiles/manifest.js";import{selectProfileForCwd as _e,selectProfileForCwdReadOnly as ve}from"../profiles/select.js";import{buildLaunchSpecAsync as X,buildPiArgv as ye}from"./launch.js";import{createManagedWorktreeAsync as ge,rollbackManagedWorktreeAsync as be}from"../worktree-mutation-async.js";import{usage as u,brokerLaunchFailed as P}from"../errors.js";import{writeGoal as xe}from"./kickoff.js";import{appendSituationalContext as Ce,formatSituationalProse as Fe}from"./situational-context.js";import{ensureRoadmap as Se}from"./roadmap.js";import{buildWakeBearings as $e}from"./bearings.js";import{encodeKickoffOrigin as Ee,KICKOFF_ORIGIN_ENV as Oe}from"./stamp/protocol.js";import{beginExecution as Re,crtrHome as Pe,getNode as f,fullName as G,jobDir as Ie,recordExecutionPid as Ne,abortExecution as D,setFrozen as Be}from"../canvas/index.js";import{contextDir as A,nodeAgentDir as L}from"../canvas/node-agent-paths.js";import{copyTreeBeneath as De}from"../spaces/open-beneath.js";import{conversationIdFromPath as I,conversationStore as x}from"../conversation-store/index.js";import{appMemoryTooSmall as Ae,boundFleet as Le,brokerThresholdsForDaemon as Te}from"./fleet.js";import{admitBrokerMemory as je}from"./memory-admission.js";import{emitEvent as Me}from"../events/emit.js";import{openViewerWindow as We,focusOf as He,windowOfPane as Ke}from"./placement.js";import{waitForBrokerViewSocket as qe}from"./placement-tmux.js";import{transition as w}from"./lifecycle.js";import{beginBootFaultAttempt as ze}from"../canvas/recovery.js";import{headlessBrokerHost as T}from"./host.js";import{ensureDaemon as Ue}from"../../daemon/manage.js";import{readConfig as Ve}from"../config.js";import{modelRequestFromConfig as Xe,unregisteredConcreteModel as Ge}from"../model-routes.js";import{openModelRegistry as Je}from"./model-registry.js";import{MODEL_SPEC_NEXT as Qe}from"./model-selection.js";import{forkNodeLedger as Ye}from"./ledger.js";import{discoverLifecycleHookRegistry as Ze}from"../command-hooks/discovery.js";import{invokeLifecycleHooks as en}from"../command-hooks/transport/exec-lifecycle.js";function p(e){return{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}a(p,"launchFailureOutcome");async function nn(e,o,d){try{return await T.launch(e,o,d)}catch(r){try{w(e,"crash",{reason:"launch_failed",outcome:p(r)})}catch{}const t=r.message;throw P(t.startsWith("[broker]")?t:`[broker] refusing to launch ${e}: ${t}`,"Inspect the cause above, fix the launch recipe or environment, then retry.")}}a(nn,"launchBrokerOrCrash");function on(e,o){try{w(e,"crash",{reason:"launch_failed",outcome:p(o)})}catch{}try{T.teardown(e)}catch{}}a(on,"crashAndTearDown");function rn(e){const o=x().list().map(({conversation_id:t,path:s})=>({id:t,path:s})),d=o.filter(t=>t.id===e),r=d.length>0?d:o.filter(t=>t.id.startsWith(e));if(r.length===0)throw u(`--fork-from '${e}': no pi session with that id found.`,{next:"Pass a live node id, a real session uuid/prefix, or an absolute .jsonl session path."});if(r.length>1)throw u(`--fork-from '${e}': ambiguous \u2014 ${r.length} sessions match that id prefix; pass the full id or the .jsonl path.`,{next:"Pass the full session uuid or an absolute .jsonl path to disambiguate."});return r[0].path}a(rn,"resolveSessionUuid");function J(e){const o=e.trim();if(o==="")throw u("--fork-from requires a node id, session file, or session uuid.",{next:"Pass a live node id, an absolute .jsonl session path, or a session uuid/prefix."});if(o.includes("/")||o.endsWith(".jsonl")){const r=I(ae(o)?o:se(process.cwd(),o));if(r===null)throw u(`--fork-from '${o}': invalid conversation path.`,{next:"Pass a conversation file path or id."});return x().header(r).path}const d=f(o);if(d!==null){const r=d.pi_session_file;if(r==null||r==="")throw u(`node ${o} has no pi session yet \u2014 it has not captured a session FILE to fork from (the broker forks a .jsonl path, not a bare id).`,{next:"Pick a node that has captured a session, or drop --fork-from."});const t=I(r);if(t===null)throw u(`node ${o} has no valid conversation id.`,{next:"Pick a node with a captured conversation."});return x().header(t).path}return rn(o)}a(J,"resolveForkSource");async function tn(e,o,d,r={}){return e===null?null:e!==void 0&&e!==""?ke(e).profileId:o!==null?f(o)?.profile_id??null:r.readOnlySelection===!0?ve(d):_e(d)}a(tn,"resolveProfileId");function Q(e){const o=e.modelIntent===void 0?void 0:`${e.modelIntent.family}/${e.modelIntent.strength}`;return Xe(e.model,o,e.modelExact===!0)}a(Q,"requestFromLaunch");async function Y(e,o,d){if(e===void 0||e.kind==="logical")return;const r=Ge(e,d??await Je(),o);if(r!==void 0)throw u(`model '${r.request.spec}' is not registered in this daemon.`,{received:r.request.spec,next:Qe})}a(Y,"assertLaunchModelRegistered");function dn(e,o,d){const r=e??o??null;if(!d&&r===null)throw new Error("spawnChild requires a calling node (CRTR_NODE_ID) or an explicit parent");return r}a(dn,"resolveSpawner");async function Z(e,o,d){try{Ue()}catch{}const r=me(),t=e.root===!0,s=dn(e.parent,r.nodeId??null,t),c=e.mode??"base",k=e.rootLifecycle??(t?"resident":"terminal"),ee=t&&k==="resident",j=s===null?null:f(s),ne=e.creator===void 0||e.creator===null?null:f(e.creator);e.nodeId!==void 0&&pe(e.nodeId);const _=e.forkFrom!==void 0?J(e.forkFrom):void 0,M=_===void 0?null:ue(_),N=[j,ne,M].some(l=>l?.isolated===!0||l?.run_id!=null&&f(l.run_id)?.isolated===!0);if(N&&e.isolated===!1)throw new V(403,"isolated_run","isolation cannot be turned off under an isolated run",{param:"isolated"});const S=N||e.isolated===!0;if(S&&e.worktree===!0)throw new V(400,"invalid_request","isolated nodes cannot use a worktree",{param:"worktree"});const $=e.worktree===!0,v=S||$?e.nodeId??we():e.nodeId,E=S?e.run!==void 0&&t&&!N?e.cwd:ce(le(e.run?.grantee??j?.grantee??"app:terminal").nodes,v):e.cwd,C=await tn(e.profile,s,E),W=(await X(e.kind,c,{lifecycle:k,hasManager:!t,model:e.model,modelOverride:e.model!==void 0,cwd:E,profileId:C})).launch;await Y(Q(W),{cwd:E,profileId:C});const H=M?.node_id??null;let y,O=E;try{$&&(y=await ge(e.worktreeCwd??e.cwd,v,e.worktreeBase),O=y.path);const l=$?(await X(e.kind,c,{lifecycle:k,hasManager:!t,model:e.model,modelOverride:e.model!==void 0,cwd:O,profileId:C})).launch:W;$&&await Y(Q(l),{cwd:O,profileId:C});const n=await fe({kind:e.kind,mode:c,lifecycle:k,cwd:O,name:e.name??e.kind,description:e.description,parent:t?null:s,creator:e.creator,...e.actor===void 0?{}:{actor:e.actor},forkSourceNode:H,spawnedBy:t?s??void 0:void 0,profile_id:C,...e.scopes===void 0?{}:{scopes:e.scopes},isolated:S,...e.run===void 0?{}:{run:e.run},...e.withinCreate===void 0?{}:{withinCreate:e.withinCreate},forkFrom:e.forkFrom,forkSourceFile:_,modelOverride:e.model===void 0?void 0:l.model,...y!==void 0?{managedWorktree:y}:{},...v!==void 0?{nodeId:v}:{},...e.reviewBinding===void 0?{}:{reviewBinding:e.reviewBinding},...e.deadlineAt===void 0?{}:{deadlineAt:e.deadlineAt},...e.outcomeDelivery===void 0?{}:{outcomeDelivery:e.outcomeDelivery},launch:l});if(d?.(n.node_id),e.forkFrom!==void 0)try{Ye(n.node_id,H??null)}catch(i){throw w(n.node_id,"crash",{reason:"launch_failed",outcome:p(i)}),i}if(xe(n.node_id,e.prompt),e.situationalContext!==void 0&&Ce(n.node_id,Fe(e.situationalContext)),o!==void 0)try{o(n.node_id)}catch(i){throw w(n.node_id,"crash",{reason:"launch_failed",outcome:p(i)}),i}const B=e.prompt!=="",oe=`${B&&e.wakeOrigin!==void 0?`${$e(e.wakeOrigin)}
 
-`:""}${e.prompt}`;c==="orchestrator"&&Ce(n.node_id,{goal:e.prompt.trim()}),await on(n);let H;if(w!==void 0)try{const i=P(w);if(i===null)throw new Error(`invalid fork source ${w}`);const m=v().fork(i,n.node_id);H=v().header(m).path}catch(i){throw g(n.node_id,"crash",{reason:"launch_failed",outcome:y(i)}),i}const _=_e(n,{prompt:ee,resumeSessionPath:H});if(_.env={..._.env,CRTR_SUBTREE:ue(n.node_id)},N){const i=e.wakeOrigin!==void 0?{type:"runtime",kind:"wake",wake:e.wakeOrigin}:{type:"runtime",kind:"kickoff",...n.spawned_by!=null&&n.spawned_by!==""?{spawnedBy:n.spawned_by}:{}};_.env[Se]=$e(i)}const E=De();if(!E.reserveSlot(n.node_id))return Ie(n.node_id,new Date().toISOString()),Ae({level:"info",event:"broker.frozen",node_id:n.node_id,fields:{live_count:E.size(),hard_cap:Be().automaticReviveCap}}),{node:f(n.node_id),frozen:!0,window:null,session:null};const ne=Me(n.node_id),x=Ee(n.node_id,{daemonEpoch:E.epoch()});_.env={..._.env,CRTR_EXECUTION_ID:x};let O;try{O=await Qe(n.node_id,_,{cwd:n.cwd,name:V(n),resuming:!1})}catch(i){throw D(x,{reason:"launch_failed"}),i}finally{E.releaseReservation(n.node_id)}if(O.pid===null){const i=new Error(`broker host returned no pid for ${n.node_id}`);throw D(x,{reason:"launch_failed"}),g(n.node_id,"crash",{reason:"launch_failed",outcome:y(i)}),R(`failed to launch the broker engine for ${n.node_id} (${n.name}) \u2014 the node was not started.`,`Inspect canonical event diagnostics with \`crtr sys logs --node ${n.node_id}\`, then verify the pi engine can start in this environment.`)}if(Pe(x,n.node_id,O.pid)==="shutdown")throw D(x,{reason:"superseded"}),L.teardown(n.node_id),R(`broker execution was superseded for ${n.node_id}`,"Retry spawning the node.");if(Y&&e.readiness==="await"&&!await We(n.node_id,O.exited)){const i=ne.explanation();throw Ye(n.node_id,new Error(i??`broker view socket never bound for ${n.node_id}`)),R(i!==null?`the root broker ${n.node_id} could not start \u2014 ${i}`:`the root broker ${n.node_id} never bound its view socket \u2014 it was not started.`,i!==null?"Resolve the cause named above, then retry.":`The broker engine exited before binding its view socket. Inspect canonical event diagnostics with \`crtr sys logs --node ${n.node_id}\`, then retry.`)}let K=null,q=null;if(!t&&s!==null&&Ke("user").auto_open_child_viewers)try{const i=Te(s);if(i!==null&&i.pane!==null&&i.pane!==""){const m=Le(n.node_id,i.session??"",{name:V(n),cwd:e.cwd,besidePane:i.pane,waitForBroker:!0,initialWhip:N});m!==null&&m.pane!==null&&m.pane!==""&&(q=m.session,K=je(m.pane))}}catch{}return{node:f(n.node_id),frozen:!1,window:K,session:q}}catch(l){throw k!==void 0&&p!==void 0&&f(p)===null&&await ge(k),l}}a(Q,"spawnChildPrepared");async function Hn(e){return Q(e)}a(Hn,"spawnChild");async function on(e){const o=Ge(e.cwd,e.profile_id).plans.get("node:start")??[];await Je(o,{node:{id:e.node_id,name:e.name,kind:e.kind,mode:e.mode,lifecycle:e.lifecycle,cwd:e.cwd,nodeDir:A(e.node_id),contextDir:B(e.node_id),jobDir:Re(e.node_id)},runtime:{isBirth:!0,canvasHome:Oe(),profile:e.profile_id??null}})}a(on,"invokeBirthLifecycleHooks");const rn=new Set(["initial-prompt.md","yield-message.md","result.json","jobs"]);async function Kn(e,o,d){const r=f(e);if(r===null)throw u(`unknown node: ${e}`,{received:e,next:"List nodes and pass the id of a node with a captured pi session."});const t=X(e),s=B(e);try{v().header(P(t)),re(s,ie.R_OK)}catch(c){throw new Error(`cannot fork ${e}: its conversation or context is unreadable \u2014 ${c.message}`)}return Q({kind:r.kind,mode:"base",cwd:r.cwd,prompt:"",root:!0,readiness:"await",forkFrom:e,...d===void 0?{}:{actor:d},profile:r.profile_id,model:r.model_override??void 0,...o===void 0?{}:{scopes:o}},c=>{const h=B(c);Ne(A(e),s,A(c),h,rn)})}a(Kn,"forkNode");export{J as assertLaunchModelRegistered,Kn as forkNode,on as invokeBirthLifecycleHooks,X as resolveForkSource,en as resolveProfileId,nn as resolveSpawner,Hn as spawnChild,Q as spawnChildPrepared};
+`:""}${e.prompt}`;c==="orchestrator"&&Se(n.node_id,{goal:e.prompt.trim()}),await an(n);let K;if(_!==void 0)try{const i=I(_);if(i===null)throw new Error(`invalid fork source ${_}`);const h=x().fork(i,n.node_id);K=x().header(h).path}catch(i){throw w(n.node_id,"crash",{reason:"launch_failed",outcome:p(i)}),i}const g=ye(n,{prompt:oe,resumeSessionPath:K});if(g.env={...g.env,CRTR_SUBTREE:he(n.node_id)},B){const i=e.wakeOrigin!==void 0?{type:"runtime",kind:"wake",wake:e.wakeOrigin}:{type:"runtime",kind:"kickoff",...n.spawned_by!=null&&n.spawned_by!==""?{spawnedBy:n.spawned_by}:{}};g.env[Oe]=Ee(i)}const b=Le(),q=b.reserveSlot(n.node_id),m=q?await je(n.node_id):null;if(m?.verdict==="too_small"){b.releaseReservation(n.node_id);const i=Ae(m);throw w(n.node_id,"crash",{reason:"launch_failed",outcome:p(i)}),i}if(!q||m?.verdict==="denied")return m?.verdict==="denied"&&b.releaseReservation(n.node_id),Be(n.node_id,new Date().toISOString()),Me({level:"info",event:"broker.frozen",node_id:n.node_id,fields:m?.verdict==="denied"?{reason:"app_memory",free_bytes:m.free_bytes,needed_bytes:m.needed_bytes}:{reason:"broker_cap",live_count:b.size(),hard_cap:Te().automaticReviveCap}}),{node:f(n.node_id),frozen:!0,window:null,session:null};const re=ze(n.node_id),F=Re(n.node_id,{daemonEpoch:b.epoch()});g.env={...g.env,CRTR_EXECUTION_ID:F};let R;try{R=await nn(n.node_id,g,{cwd:n.cwd,name:G(n),resuming:!1})}catch(i){throw D(F,{reason:"launch_failed"}),i}finally{b.releaseReservation(n.node_id)}if(R.pid===null){const i=new Error(`broker host returned no pid for ${n.node_id}`);throw D(F,{reason:"launch_failed"}),w(n.node_id,"crash",{reason:"launch_failed",outcome:p(i)}),P(`failed to launch the broker engine for ${n.node_id} (${n.name}) \u2014 the node was not started.`,`Inspect canonical event diagnostics with \`crtr sys logs --node ${n.node_id}\`, then verify the pi engine can start in this environment.`)}if(Ne(F,n.node_id,R.pid)==="shutdown")throw D(F,{reason:"superseded"}),T.teardown(n.node_id),P(`broker execution was superseded for ${n.node_id}`,"Retry spawning the node.");if(ee&&e.readiness==="await"&&!await qe(n.node_id,R.exited)){const i=re.explanation();throw on(n.node_id,new Error(i??`broker view socket never bound for ${n.node_id}`)),P(i!==null?`the root broker ${n.node_id} could not start \u2014 ${i}`:`the root broker ${n.node_id} never bound its view socket \u2014 it was not started.`,i!==null?"Resolve the cause named above, then retry.":`The broker engine exited before binding its view socket. Inspect canonical event diagnostics with \`crtr sys logs --node ${n.node_id}\`, then retry.`)}let z=null,U=null;if(!t&&s!==null&&Ve("user").auto_open_child_viewers)try{const i=He(s);if(i!==null&&i.pane!==null&&i.pane!==""){const h=We(n.node_id,i.session??"",{name:G(n),cwd:e.cwd,besidePane:i.pane,waitForBroker:!0,initialWhip:B});h!==null&&h.pane!==null&&h.pane!==""&&(U=h.session,z=Ke(h.pane))}}catch{}return{node:f(n.node_id),frozen:!1,window:z,session:U}}catch(l){throw y!==void 0&&v!==void 0&&f(v)===null&&await be(y),l}}a(Z,"spawnChildPrepared");async function Vn(e){return Z(e)}a(Vn,"spawnChild");async function an(e){const o=Ze(e.cwd,e.profile_id).plans.get("node:start")??[];await en(o,{node:{id:e.node_id,name:e.name,kind:e.kind,mode:e.mode,lifecycle:e.lifecycle,cwd:e.cwd,nodeDir:L(e.node_id),contextDir:A(e.node_id),jobDir:Ie(e.node_id)},runtime:{isBirth:!0,canvasHome:Pe(),profile:e.profile_id??null}})}a(an,"invokeBirthLifecycleHooks");const sn=new Set(["initial-prompt.md","yield-message.md","result.json","jobs"]);async function Xn(e,o,d){const r=f(e);if(r===null)throw u(`unknown node: ${e}`,{received:e,next:"List nodes and pass the id of a node with a captured pi session."});const t=J(e),s=A(e);try{x().header(I(t)),te(s,de.R_OK)}catch(c){throw new Error(`cannot fork ${e}: its conversation or context is unreadable \u2014 ${c.message}`)}return Z({kind:r.kind,mode:"base",cwd:r.cwd,prompt:"",root:!0,readiness:"await",forkFrom:e,...d===void 0?{}:{actor:d},profile:r.profile_id,model:r.model_override??void 0,...o===void 0?{}:{scopes:o}},c=>{const k=A(c);De(L(e),s,L(c),k,sn)})}a(Xn,"forkNode");export{Y as assertLaunchModelRegistered,Xn as forkNode,an as invokeBirthLifecycleHooks,J as resolveForkSource,tn as resolveProfileId,dn as resolveSpawner,Vn as spawnChild,Z as spawnChildPrepared};

package/dist/daemon/api/handlers/limits.js

@@ -1 +1 @@
-var h=Object.defineProperty;var n=(e,s)=>h(e,"name",{value:s,configurable:!0});import{existsSync as k}from"node:fs";import{runtimeRoot as i}from"../../../core/layout.js";import{launcherRequest as y}from"../../../launcher/client.js";import{LAUNCHER_SOCKET as w}from"../../../launcher/spec.js";import{bareApp as d,budgetFor as v}from"../../../launcher/limits/policy.js";import{LimitInputError as S,applyChange as g,readSettings as p,removeOverride as E,writeSettings as u}from"../../../launcher/limits/settings.js";import{ownerOnly as R}from"../router.js";function c(){return process.platform==="linux"&&k(w)}n(c,"hasLauncher");function m(e){throw{status:400,code:"invalid_request",message:e}}n(m,"bad");function l(e,s){const t=s?.apps.find(r=>r.app===d(e));if(t)return{app:`app:${t.app}`,memory_bytes:t.memory_bytes,disk_bytes:t.disk_bytes,memory_source:t.memory_source,disk_source:t.disk_source,disk_used_bytes:t.disk_used_bytes};const o=v(e,p(i()));return{app:`app:${d(e)}`,memory_bytes:o.memoryBytes,disk_bytes:o.diskBytes,memory_source:o.memorySource,disk_source:o.diskSource,disk_used_bytes:null}}n(l,"appRow");async function f(){if(!c())return null;const e=await y({operation:"limits.show"});return"limits"in e?e.limits:null}n(f,"report");async function A(){const e=await f(),s=p(i()),t=new Set([...e?.apps.map(r=>r.app)??[],...Object.keys(s.apps)]);return{status:200,body:{memory_enforced:e?.memory.available??!1,memory_reason:e?e.memory.reason:"this host has no launcher, so memory limits are not enforced",disk_enforced:e?.disk.available??!1,disk_reason:e?e.disk.reason:"this host has no launcher, so disk limits are not enforced",apps:[...t].sort().map(r=>l(r,e))}}}n(A,"handleShow");function b(e){const s=d(decodeURIComponent(e.params.app??""));return/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(s)||m(`app must be an app id such as app:terminal; received ${e.params.app??""}`),s}n(b,"appParam");async function _(e){let s=!1;return c()&&(await y({operation:"limits.apply",app:e}),s=!0),{app:`app:${e}`,applied:s,limits:l(e,await f())}}n(_,"apply");async function $(e){const s=b(e),t=e.body;(t===null||typeof t!="object"||Array.isArray(t))&&m("body must be an object with memory_bytes and/or disk_bytes");const{memory_bytes:o,disk_bytes:r}=t;o===void 0&&r===void 0&&m("set at least one of memory_bytes and disk_bytes");try{const a=g(p(i()),s,{...o===void 0?{}:{memory_bytes:o},...r===void 0?{}:{disk_bytes:r}});u(i(),a)}catch(a){throw a instanceof S&&m(a.message),a}return{status:200,body:await _(s)}}n($,"handleSet");async function C(e){const s=b(e),t=e.query.get("fields"),o=t===null||t===""?["memory","disk"]:t.split(",");return o.every(r=>r==="memory"||r==="disk")||m(`fields must be memory, disk or memory,disk; received ${t}`),u(i(),E(p(i()),s,o)),{status:200,body:await _(s)}}n(C,"handleRemove");const I=R([{method:"GET",pattern:"/v1/limits",handler:A},{method:"PUT",pattern:"/v1/limits/apps/:app",handler:$},{method:"DELETE",pattern:"/v1/limits/apps/:app",handler:C}]);export{I as limitsRoutes};
+var k=Object.defineProperty;var o=(e,t)=>k(e,"name",{value:t,configurable:!0});import{existsSync as w}from"node:fs";import{runtimeRoot as a}from"../../../core/layout.js";import{launcherRequest as l}from"../../../launcher/client.js";import{LAUNCHER_SOCKET as v}from"../../../launcher/spec.js";import{bareApp as u,budgetFor as S}from"../../../launcher/limits/policy.js";import{LimitInputError as g,applyChange as A,readSettings as d,removeOverride as E,writeSettings as c}from"../../../launcher/limits/settings.js";import{ownerOnly as R}from"../router.js";function y(){return process.platform==="linux"&&w(v)}o(y,"hasLauncher");function m(e){throw{status:400,code:"invalid_request",message:e}}o(m,"bad");function f(e,t,r=null){const s=t?.apps.find(p=>p.app===u(e)),i=r?.apps.find(p=>p.app===u(e))?.bytes??null;if(s)return{app:`app:${s.app}`,memory_bytes:s.memory_bytes,disk_bytes:s.disk_bytes,memory_source:s.memory_source,disk_source:s.disk_source,disk_used_bytes:s.disk_used_bytes??i};const n=S(e,d(a()));return{app:`app:${u(e)}`,memory_bytes:n.memoryBytes,disk_bytes:n.diskBytes,memory_source:n.memorySource,disk_source:n.diskSource,disk_used_bytes:i}}o(f,"appRow");async function b(){if(!y())return null;const e=await l({operation:"limits.show"});return"limits"in e?e.limits:null}o(b,"report");const T=5e3;async function $(){if(!y())return null;let e;try{const t=await Promise.race([l({operation:"disk.usage"}),new Promise(r=>{e=setTimeout(()=>r(null),T)})]);return t!==null&&"disk"in t?t.disk:null}catch{return null}finally{clearTimeout(e)}}o($,"diskUsage");function P(e){return e===null?void 0:{total_bytes:e.total_bytes,used_bytes:e.used_bytes,user_bytes:e.user_bytes,runtime_bytes:e.runtime_bytes}}o(P,"dataDisk");async function U(){const[e,t]=await Promise.all([b(),$()]),r=d(a()),s=new Set([...e?.apps.map(n=>n.app)??[],...Object.keys(r.apps)]);return{status:200,body:{memory_enforced:e?.memory.available??!1,memory_reason:e?e.memory.reason:"this host has no launcher, so memory limits are not enforced",disk_enforced:e?.disk.available??!1,disk_reason:e?e.disk.reason:"this host has no launcher, so disk limits are not enforced",apps:[...s].sort().map(n=>f(n,e,t)),...t===null?{}:{data_disk:P(t)}}}}o(U,"handleShow");function _(e){const t=u(decodeURIComponent(e.params.app??""));return/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(t)||m(`app must be an app id such as app:terminal; received ${e.params.app??""}`),t}o(_,"appParam");async function h(e){let t=!1;return y()&&(await l({operation:"limits.apply",app:e}),t=!0),{app:`app:${e}`,applied:t,limits:f(e,await b())}}o(h,"apply");async function C(e){const t=_(e),r=e.body;(r===null||typeof r!="object"||Array.isArray(r))&&m("body must be an object with memory_bytes and/or disk_bytes");const{memory_bytes:s,disk_bytes:i}=r;s===void 0&&i===void 0&&m("set at least one of memory_bytes and disk_bytes");try{const n=A(d(a()),t,{...s===void 0?{}:{memory_bytes:s},...i===void 0?{}:{disk_bytes:i}});c(a(),n)}catch(n){throw n instanceof g&&m(n.message),n}return{status:200,body:await h(t)}}o(C,"handleSet");async function I(e){const t=_(e),r=e.query.get("fields"),s=r===null||r===""?["memory","disk"]:r.split(",");return s.every(i=>i==="memory"||i==="disk")||m(`fields must be memory, disk or memory,disk; received ${r}`),c(a(),E(d(a()),t,s)),{status:200,body:await h(t)}}o(I,"handleRemove");const K=R([{method:"GET",pattern:"/v1/limits",handler:U},{method:"PUT",pattern:"/v1/limits/apps/:app",handler:C},{method:"DELETE",pattern:"/v1/limits/apps/:app",handler:I}]);export{K as limitsRoutes};

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var q=Object.defineProperty;var p=(s,e)=>q(s,"name",{value:e,configurable:!0});import{existsSync as Y}from"node:fs";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{launcherRequest as K}from"../../launcher/client.js";import{execFile as V}from"node:child_process";import{promisify as z}from"node:util";import{getNode as N,isWarmSpare as j}from"../../core/canvas/index.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as J}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as I}from"../../core/canvas/pid.js";import{transition as Q}from"../../core/runtime/lifecycle.js";import{readFault as D,readLiveTurn as E,recordFault as X}from"../../core/canvas/recovery.js";import{errorClassFromFault as Z}from"../../core/fault-classifier.js";import{nextFaultRetry as ee}from"../../core/runtime/fault-recovery.js";import{publishNotice as O}from"../../core/runtime/close.js";import{headlessBrokerHost as k}from"../../core/runtime/host.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as te,resetStaleExecutionMail as ne}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as re}from"../../core/bash-jobs.js";import{focusOf as ie}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ae}from"../api/viewer-presence.js";import{deliverLive as se}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as oe}from"../../core/runtime/broker-request.js";import{formatCard as le,PARK_SUMMARY_PROMPT as de}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ue}from"../../core/runtime/fleet.js";import{readConfig as ce}from"../../core/config.js";import{beginPendingPark as fe,consumePendingPark as y,pendingParkIds as pe,pendingParkStartedAt as me}from"../park-pending.js";import{shouldParkAfterSummary as he}from"../park-activity.js";import{applyParkEvent as b,parkEvent as _e}from"./live-obligation.js";import{envNoDaemonAutostart as ge,envTestUnattendedParkMs as ke,parkSummaryGraceMs as ve,envUnattendedParkMs as ye}from"../../shared/env.js";const P=z(V);function be(s){return E(s)?.busy===!0}p(be,"liveBusy");function lt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(lt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function F(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}p(F,"busySinceForIncarnation");const G=3*6e4,W=2e4,x=20*6e4,we=2,Se=5*6e4;function Ne(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Ne,"settlesOnFatalProviderFault");function Pe(){const s=ye();if(s!==void 0)return s;const e=ce("user").lifecycle.unattendedParkMs;return ge()?ke()??e:e}p(Pe,"unattendedParkMsForDaemon");function U(){return ve()}p(U,"parkSummaryGraceMsForDaemon");function Ae(s,e,t,a){return s!==!0||e!=="refresh"||t?"leave":a===null||a<G?"pending":"kill"}p(Ae,"yieldStallVerdict");function $e(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>we?"pending":"wedged"}p($e,"wedgeVerdict");function B(s){const e=new Map,t=new Map;for(const a of s.split(`
-`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),r=Number(n[1]),l=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(r)||!Number.isFinite(l))continue;e.set(o,l);const c=t.get(r);c===void 0?t.set(r,[o]):c.push(o)}return{byPid:e,childrenOf:t}}p(B,"parsePsTree");function Me(s,e){const{byPid:t,childrenOf:a}=B(s);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const r=n.pop();if(o.has(r))continue;o.add(r),i+=t.get(r)??0;const l=a.get(r);l!==void 0&&n.push(...l)}return i}p(Me,"sumTreeCpu");function Te(s,e){const{byPid:t,childrenOf:a}=B(s);return t.has(e)?(a.get(e)??[]).length>0:null}p(Te,"treeHasDescendants");class dt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&Y(H)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return K({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return ue()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=R.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Pe();for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n);if(o===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await R.fresh(async()=>{try{await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,N(n),o.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),f({level:"error",event:"broker.supervision.failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of pe())t.fleet.has(i)||y(i);ne(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||re(J(e)).length>0||ie(e)!==null||ae(e))return!0;const n=D(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ee(n,i)!==null)?!0:te(e)}handleUnattendedParking(e,t,a,i,n,o){if(j(e)){this.unattendedSince.delete(e),y(e);return}if(t===null){this.unattendedSince.delete(e),y(e);return}const r=this.parkResetCondition(e,t,a,i),l=me(e);if(l!==void 0){if(i-l<U())return;const m=y(e);if(m===void 0||(this.unattendedSince.delete(e),!he(e,t,m)))return;f({level:"warn",event:"broker.park.summary_expired",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},grace_ms:U()}}),b(e,"park"),k.teardown(e);return}if(r||be(e)){this.unattendedSince.delete(e);return}const c=this.unattendedSince.get(e);if(c===void 0){this.unattendedSince.set(e,i);return}if(i-c<o)return;this.unattendedSince.delete(e);const d=_e(e,t.lifecycle);if(d==="release"){b(e,d),k.teardown(e);return}if(d==="cancel"){b(e,d),k.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){b(e,"park"),k.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(fe(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}async requestParkSummary(e){try{const t=await se(e,le("park",{},de),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(y(e)===void 0)return;if(t instanceof oe){b(e,"park"),k.teardown(e);return}f({level:"error",event:"broker.park.summary_failed",...u(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Se)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,r=a.stalled+a.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const l=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,c=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${l} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await P("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async m=>{if(m!=="")try{await P("tmux",["display-message","-C","-d","10000","-c",m,c],{encoding:"utf8",timeout:2e3})}catch{try{await P("tmux",["display-message","-c",m,c],{encoding:"utf8",timeout:2e3})}catch(h){f({level:"error",event:"broker.capacity.tmux_display_failed",error:h,fields:{client:m}})}}}))}catch(d){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||o===null){o!==null&&this.yieldStalls.delete(o);return}let r=this.yieldStalls.get(o);if(r===void 0){r={since:i},this.yieldStalls.set(o,r);return}if(Ae(!0,e.intent,!1,i-r.since)!=="kill")return;const l=N(n);if(!(a.get(n)!==t||l?.pi_pid!==t.pid||l.pi_pid_identity!==e.pi_pid_identity||I(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:G}})}else i-r.termedAt>=W&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:W,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...u(e)?{node_id:e}:{},error:i,fields:{...u(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await P("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=Me(t,e),i=Te(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity),r=F(e);if(r===null||o===null){o!==null&&this.wedgeNotifiedAt.delete(o);return}const l=i-r,c=l>=x?await this.sampleTree(t.pid):null,d=N(n),m=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||m!==r)return;const h=c?.cpuPercent??null,_=c?.hasDescendants??null;if($e({busy:!0,quietForMs:l,cpuPercent:h,hasDescendants:_})!=="wedged"||this.wedgeNotifiedAt.get(o)===r)return;this.wedgeNotifiedAt.set(o,r);const w=C(d),S=Math.round((l??x)/6e4),A=_??!0,v=!A&&I(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let $=!1,M=!1,g;if(A)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{$=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){f({level:"error",event:"broker.wedge.signal_failed",...u(n)?{node_id:n}:{},error:T,fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${S}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}f({level:"error",event:"broker.wedge.detected",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:l,cpu_percent:h,has_descendants:A,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}}});try{X(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:g})}catch(T){f({level:"error",event:"broker.wedge.fault_record_failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}O(n,"notice.crash",g,{reason:"child-wedged",child:n,quiet_ms:l,cpu_percent:h,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}})}handleFatalFault(e){const t=D(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=N(e),i=a!==null?C(a):e,n=t.kind==="auth",o=(t.message.split(`
-`)[0]??"").slice(0,200),r=t.providerError,l=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),c=t.kind==="context-overflow",d=Ne(a,typeof r?.code=="string"?r.code:void 0),m=c||d,h=c?`Child failed \u2014 ${i} (${e}) exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`Child hit a fatal provider fault \u2014 ${i} (${e}) got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); the child recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${l}`;m&&(Q(e,"crash",{reason:c?"context_overflow":r?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:Z(t).class}}}),k.teardown(e).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))),this.fatalFaultNotifiedAt.set(e,t.since),f({level:"error",event:"fault.fatal.fanout",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:c,terminal_provider_fault:d}}),O(e,"notice.crash",h,{reason:c?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:o,...r===void 0?{}:{provider_error:r}})}}export{dt as BrokerSupervisionReconciler,we as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,G as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,Ne as settlesOnFatalProviderFault,Me as sumTreeCpu,Te as treeHasDescendants,Pe as unattendedParkMsForDaemon,$e as wedgeVerdict,Ae as yieldStallVerdict};
+var q=Object.defineProperty;var p=(s,e)=>q(s,"name",{value:e,configurable:!0});import{existsSync as Y}from"node:fs";import{LAUNCHER_SOCKET as H}from"../../launcher/spec.js";import{launcherRequest as K}from"../../launcher/client.js";import{execFile as V}from"node:child_process";import{promisify as z}from"node:util";import{getNode as N,isWarmSpare as j}from"../../core/canvas/index.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as J}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Q}from"../../core/runtime/lifecycle.js";import{readFault as I,readLiveTurn as E,recordFault as X}from"../../core/canvas/recovery.js";import{errorClassFromFault as Z}from"../../core/fault-classifier.js";import{nextFaultRetry as ee}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{headlessBrokerHost as k}from"../../core/runtime/host.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as O}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as te,resetStaleExecutionMail as ne}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as re}from"../../core/bash-jobs.js";import{focusOf as ie}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ae}from"../api/viewer-presence.js";import{deliverLive as se}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as oe}from"../../core/runtime/broker-request.js";import{formatCard as le,PARK_SUMMARY_PROMPT as de}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ue}from"../../core/runtime/fleet.js";import{readConfig as ce}from"../../core/config.js";import{beginPendingPark as fe,consumePendingPark as y,pendingParkIds as pe,pendingParkStartedAt as me}from"../park-pending.js";import{shouldParkAfterSummary as he}from"../park-activity.js";import{applyParkEvent as b,parkEvent as _e}from"./live-obligation.js";import{memoryDeniedRecently as ge}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as ke,envTestUnattendedParkMs as ve,parkSummaryGraceMs as ye,envUnattendedParkMs as be}from"../../shared/env.js";const P=z(V);function Se(s){return E(s)?.busy===!0}p(Se,"liveBusy");function ut(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(ut,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");function F(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,a=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(a)||t<a?null:t}p(F,"busySinceForIncarnation");const G=3*6e4,W=2e4,x=20*6e4,we=2,Ne=5*6e4;function Pe(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Pe,"settlesOnFatalProviderFault");function Ae(){const s=be();if(s!==void 0)return s;const e=ce("user").lifecycle.unattendedParkMs;return ke()?ve()??e:e}p(Ae,"unattendedParkMsForDaemon");function U(){return ye()}p(U,"parkSummaryGraceMsForDaemon");function $e(s,e,t,a){return s!==!0||e!=="refresh"||t?"leave":a===null||a<G?"pending":"kill"}p($e,"yieldStallVerdict");function Me(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>we?"pending":"wedged"}p(Me,"wedgeVerdict");function B(s){const e=new Map,t=new Map;for(const a of s.split(`
+`)){const i=a.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),r=Number(n[1]),l=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(r)||!Number.isFinite(l))continue;e.set(o,l);const c=t.get(r);c===void 0?t.set(r,[o]):c.push(o)}return{byPid:e,childrenOf:t}}p(B,"parsePsTree");function Te(s,e){const{byPid:t,childrenOf:a}=B(s);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const r=n.pop();if(o.has(r))continue;o.add(r),i+=t.get(r)??0;const l=a.get(r);l!==void 0&&n.push(...l)}return i}p(Te,"sumTreeCpu");function Ee(s,e){const{byPid:t,childrenOf:a}=B(s);return t.has(e)?(a.get(e)??[]).length>0:null}p(Ee,"treeHasDescendants");class ct{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;fatalFaultNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,a)=>{if(process.platform==="linux"&&Y(H)){if(a!=="SIGTERM"&&a!=="SIGKILL")throw new Error(`unsupported launcher signal ${a}`);return K({operation:"stop",pid:t,signal:a}).then(()=>{})}process.kill(t,a)})}async runFleetPreamble(e){return ue()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=O.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const a=Ae();for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n);if(o===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await O.fresh(async()=>{try{await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e),this.handleFatalFault(n),this.handleUnattendedParking(n,N(n),o.pid,e,t,a)}catch(r){this.unattendedSince.delete(n),f({level:"error",event:"broker.supervision.failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:r})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of pe())t.fleet.has(i)||y(i);ne(e)}parkResetCondition(e,t,a,i){if(t.pi_session_id==null||t.pi_pid!==a||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||re(J(e)).length>0||ie(e)!==null||ae(e))return!0;const n=I(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ee(n,i)!==null)?!0:te(e)}handleUnattendedParking(e,t,a,i,n,o){if(j(e)){this.unattendedSince.delete(e),y(e);return}if(t===null){this.unattendedSince.delete(e),y(e);return}const r=this.parkResetCondition(e,t,a,i),l=me(e);if(l!==void 0){if(i-l<U())return;const h=y(e);if(h===void 0||(this.unattendedSince.delete(e),!he(e,t,h)))return;f({level:"warn",event:"broker.park.summary_expired",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},grace_ms:U()}}),b(e,"park"),k.teardown(e);return}if(r||Se(e)){this.unattendedSince.delete(e);return}const c=this.unattendedSince.get(e);if(c===void 0){this.unattendedSince.set(e,i);return}const d=_e(e,t.lifecycle),m=d!=="cancel"&&ge(t.grantee??"app:terminal",i)?0:o;if(!(i-c<m)){if(this.unattendedSince.delete(e),d==="release"){b(e,d),k.teardown(e);return}if(d==="cancel"){b(e,d),k.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){b(e,"park"),k.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(fe(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await se(e,le("park",{},de),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(y(e)===void 0)return;if(t instanceof oe){b(e,"park"),k.teardown(e);return}f({level:"error",event:"broker.park.summary_failed",...u(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,a){const i=Date.now();if(i-this.lastOverCapWarnAt<Ne)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,r=a.stalled+a.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:a.stalled,frozen_waiting_on_mail:a.waitingOnMail}});const l=r===0?"":` \u2014 ${r} frozen (${a.stalled} stalled, ${a.waitingOnMail} waiting on mail)`,c=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${l} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await P("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
+`).map(async m=>{if(m!=="")try{await P("tmux",["display-message","-C","-d","10000","-c",m,c],{encoding:"utf8",timeout:2e3})}catch{try{await P("tmux",["display-message","-c",m,c],{encoding:"utf8",timeout:2e3})}catch(h){f({level:"error",event:"broker.capacity.tmux_display_failed",error:h,fields:{client:m}})}}}))}catch(d){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||F(e)!==null||o===null){o!==null&&this.yieldStalls.delete(o);return}let r=this.yieldStalls.get(o);if(r===void 0){r={since:i},this.yieldStalls.set(o,r);return}if($e(!0,e.intent,!1,i-r.since)!=="kill")return;const l=N(n);if(!(a.get(n)!==t||l?.pi_pid!==t.pid||l.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(r.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;r.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.since,grace_ms:G}})}else i-r.termedAt>=W&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-r.termedAt,grace_ms:W,stall_elapsed_ms:i-r.since}})}async signalYieldStall(e,t,a){try{return await this.signal(t,a),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...u(e)?{node_id:e}:{},error:i,fields:{...u(e)?{}:{affected_node_id:e},pid:t,signal:a}}),!1}}async readTreeSample(e){try{const{stdout:t}=await P("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const a=Te(t,e),i=Ee(t,e);return a===null||i===null?null:{cpuPercent:a,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,a,i){const n=e.node_id,o=L(t.pid,e.pi_pid_identity),r=F(e);if(r===null||o===null){o!==null&&this.wedgeNotifiedAt.delete(o);return}const l=i-r,c=l>=x?await this.sampleTree(t.pid):null,d=N(n),m=d===null?null:F(d);if(a.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||m!==r)return;const h=c?.cpuPercent??null,_=c?.hasDescendants??null;if(Me({busy:!0,quietForMs:l,cpuPercent:h,hasDescendants:_})!=="wedged"||this.wedgeNotifiedAt.get(o)===r)return;this.wedgeNotifiedAt.set(o,r);const S=C(d),w=Math.round((l??x)/6e4),A=_??!0,v=!A&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let $=!1,M=!1,g;if(A)g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{$=!0;try{await this.signal(t.pid,"SIGTERM"),M=!0,g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(T){f({level:"error",event:"broker.wedge.signal_failed",...u(n)?{node_id:n}:{},error:T,fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${S} (${n}) is alive and mid-turn but has shown NO engine progress for ${w}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}f({level:"error",event:"broker.wedge.detected",...u(n)?{node_id:n}:{},fields:{...u(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:l,cpu_percent:h,has_descendants:A,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}}});try{X(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:g})}catch(T){f({level:"error",event:"broker.wedge.fault_record_failed",...u(n)?{node_id:n}:{fields:{affected_node_id:n}},error:T})}R(n,"notice.crash",g,{reason:"child-wedged",child:n,quiet_ms:l,cpu_percent:h,kick_attempted:$,kicked:M,...v===null?{}:{kick_suppression_reason:v}})}handleFatalFault(e){const t=I(e);if(t===null){this.fatalFaultNotifiedAt.delete(e);return}if(t.retry.disposition!=="fatal"||t.link!=="pi\u2192provider"||this.fatalFaultNotifiedAt.get(e)===t.since)return;const a=N(e),i=a!==null?C(a):e,n=t.kind==="auth",o=(t.message.split(`
+`)[0]??"").slice(0,200),r=t.providerError,l=r===void 0?"":` Provider: ${r.provider}/${r.model}.`+(r.status===void 0?"":` HTTP status: ${r.status}.`)+(r.errno===void 0?"":` Errno: ${r.errno}.`)+(r.code===void 0?"":` Code: ${r.code}.`)+(r.body===void 0?"":` Body: ${r.body.slice(0,200)}.`)+(r.ratelimit===void 0?"":` Rate limit: ${r.ratelimit.scope??"account"}${r.ratelimit.claim===void 0?"":` (${r.ratelimit.claim})`}${r.ratelimit.model===void 0?"":` on ${r.ratelimit.model}`}.`),c=t.kind==="context-overflow",d=Pe(a,typeof r?.code=="string"?r.code:void 0),m=c||d,h=c?`Child failed \u2014 ${i} (${e}) exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`Child hit a fatal provider fault \u2014 ${i} (${e}) got a non-retryable ${t.kind} error and will NOT recover on its own; `+(d?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(n?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); the child recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${l}`;m&&(Q(e,"crash",{reason:c?"context_overflow":r?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t.message,fault_kind:t.kind,error_class:Z(t).class}}}),k.teardown(e).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))),this.fatalFaultNotifiedAt.set(e,t.since),f({level:"error",event:"fault.fatal.fanout",...u(e)?{node_id:e}:{},fields:{...u(e)?{}:{affected_node_id:e},kind:t.kind,link:t.link,terminal_overflow:c,terminal_provider_fault:d}}),R(e,"notice.crash",h,{reason:c?"child-context-overflow":"child-fault-fatal",child:e,kind:t.kind,message:o,...r===void 0?{}:{provider_error:r}})}}export{ct as BrokerSupervisionReconciler,we as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,G as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,Pe as settlesOnFatalProviderFault,Te as sumTreeCpu,Ee as treeHasDescendants,Ae as unattendedParkMsForDaemon,Me as wedgeVerdict,$e as yieldStallVerdict};

package/dist/launcher/limits/cgroup.d.ts

@@ -1,7 +1,7 @@
 import { type AppBudget } from './policy.js';
 export declare const CGROUP_ROOT = "/sys/fs/cgroup";
 export declare class Cgroups {
-    private readonly root;
+    readonly root: string;
     private readonly procMeminfo;
     /** Why per-app memory limits are off on this host, or null when they work. */
     unavailable: string | null;
@@ -15,6 +15,10 @@ export declare class Cgroups {
     /** Create a run group that kills as one when the kernel's OOM killer picks any member. */
     createRun(app: string, run: string): string;
     oomKilled(dir: string): boolean;
+    /** Bytes the group and its descendants hold now, or null when unreadable. */
+    memoryCurrent(dir: string): number | null;
+    /** The group's `memory.max` in bytes; null when it has no cap or the file is missing. */
+    memoryLimit(dir: string): number | null;
     kill(dir: string): void;
     /** Remove an empty run group; a group still dying is retried by the caller. */
     remove(dir: string): boolean;

package/dist/launcher/limits/cgroup.js

@@ -1,2 +1,2 @@
-var p=Object.defineProperty;var c=(s,r)=>p(s,"name",{value:r,configurable:!0});import{existsSync as l,mkdirSync as m,readdirSync as a,readFileSync as i,rmdirSync as y,writeFileSync as u}from"node:fs";import{appCgroupPath as h,appCgroupValues as f,appsMemoryMax as d,parseMemoryEvents as $,runCgroupPath as g,wasOomKilled as x}from"./policy.js";const b="/sys/fs/cgroup";function o(s,r){u(s,r)}c(o,"write");class S{static{c(this,"Cgroups")}root;procMeminfo;unavailable=null;constructor(r=b,t="/proc/meminfo"){this.root=r,this.procMeminfo=t}guestMemoryBytes(){try{const t=i(`${this.root}/memory.max`,"utf8").trim();if(/^\d+$/.test(t))return Number(t)}catch{}const r=/^MemTotal:\s+(\d+)\s+kB/m.exec(i(this.procMeminfo,"utf8"));if(!r)throw new Error("cannot read guest memory");return Number(r[1])*1024}init(){try{if(!i(`${this.root}/cgroup.controllers`,"utf8").split(/\s+/).includes("memory"))throw new Error("memory controller is not available");const t=`${this.root}/apps`;return m(t,{recursive:!0}),i(`${this.root}/cgroup.subtree_control`,"utf8").split(/\s+/).includes("memory")||o(`${this.root}/cgroup.subtree_control`,"+memory +pids"),o(`${t}/cgroup.subtree_control`,"+memory +pids"),o(`${t}/memory.max`,String(d(this.guestMemoryBytes()))),this.unavailable=null,!0}catch(r){return this.unavailable=r.message,!1}}applyApp(r,t){if(this.unavailable!==null)return;const e=h(this.root,r);m(e,{recursive:!0}),o(`${e}/cgroup.subtree_control`,"+memory +pids");const n=f(t);o(`${e}/memory.high`,n.memoryHigh);try{o(`${e}/memory.swap.max`,n.swapMax)}catch{}o(`${e}/memory.max`,n.memoryMax),o(`${e}/pids.max`,n.pidsMax)}createRun(r,t){const e=g(this.root,r,t);return m(e),o(`${e}/memory.oom.group`,"1"),e}oomKilled(r){try{return x($(i(`${r}/memory.events`,"utf8")))}catch{return!1}}kill(r){try{o(`${r}/cgroup.kill`,"1")}catch{}}remove(r){try{return y(r),!0}catch(t){return t.code==="ENOENT"}}exists(r){return l(r)}contains(r,t){try{return i(`${r}/cgroup.procs`,"utf8").split(`
-`).includes(String(t))}catch{return!1}}sweep(){let r=0;try{for(const t of a(`${this.root}/apps`,{withFileTypes:!0}))if(t.isDirectory())for(const e of a(`${this.root}/apps/${t.name}`,{withFileTypes:!0}))e.isDirectory()&&this.remove(`${this.root}/apps/${t.name}/${e.name}`)&&(r+=1)}catch{}return r}}function _(s,r){u(`/proc/${s}/oom_score_adj`,String(r))}c(_,"setOomScoreAdj");export{b as CGROUP_ROOT,S as Cgroups,_ as setOomScoreAdj};
+var l=Object.defineProperty;var m=(n,r)=>l(n,"name",{value:r,configurable:!0});import{existsSync as p,mkdirSync as u,readdirSync as c,readFileSync as s,rmdirSync as y,writeFileSync as a}from"node:fs";import{appCgroupPath as h,appCgroupValues as f,appsMemoryMax as d,parseMemoryEvents as $,runCgroupPath as g,wasOomKilled as x}from"./policy.js";const b="/sys/fs/cgroup";function o(n,r){a(n,r)}m(o,"write");class S{static{m(this,"Cgroups")}root;procMeminfo;unavailable=null;constructor(r=b,t="/proc/meminfo"){this.root=r,this.procMeminfo=t}guestMemoryBytes(){try{const t=s(`${this.root}/memory.max`,"utf8").trim();if(/^\d+$/.test(t))return Number(t)}catch{}const r=/^MemTotal:\s+(\d+)\s+kB/m.exec(s(this.procMeminfo,"utf8"));if(!r)throw new Error("cannot read guest memory");return Number(r[1])*1024}init(){try{if(!s(`${this.root}/cgroup.controllers`,"utf8").split(/\s+/).includes("memory"))throw new Error("memory controller is not available");const t=`${this.root}/apps`;return u(t,{recursive:!0}),s(`${this.root}/cgroup.subtree_control`,"utf8").split(/\s+/).includes("memory")||o(`${this.root}/cgroup.subtree_control`,"+memory +pids"),o(`${t}/cgroup.subtree_control`,"+memory +pids"),o(`${t}/memory.max`,String(d(this.guestMemoryBytes()))),this.unavailable=null,!0}catch(r){return this.unavailable=r.message,!1}}applyApp(r,t){if(this.unavailable!==null)return;const e=h(this.root,r);u(e,{recursive:!0}),o(`${e}/cgroup.subtree_control`,"+memory +pids");const i=f(t);o(`${e}/memory.high`,i.memoryHigh);try{o(`${e}/memory.swap.max`,i.swapMax)}catch{}o(`${e}/memory.max`,i.memoryMax),o(`${e}/pids.max`,i.pidsMax)}createRun(r,t){const e=g(this.root,r,t);return u(e),o(`${e}/memory.oom.group`,"1"),e}oomKilled(r){try{return x($(s(`${r}/memory.events`,"utf8")))}catch{return!1}}memoryCurrent(r){try{const t=Number(s(`${r}/memory.current`,"utf8").trim());return Number.isFinite(t)?t:null}catch{return null}}memoryLimit(r){try{const t=s(`${r}/memory.max`,"utf8").trim();return/^\d+$/.test(t)?Number(t):null}catch{return null}}kill(r){try{o(`${r}/cgroup.kill`,"1")}catch{}}remove(r){try{return y(r),!0}catch(t){return t.code==="ENOENT"}}exists(r){return p(r)}contains(r,t){try{return s(`${r}/cgroup.procs`,"utf8").split(`
+`).includes(String(t))}catch{return!1}}sweep(){let r=0;try{for(const t of c(`${this.root}/apps`,{withFileTypes:!0}))if(t.isDirectory())for(const e of c(`${this.root}/apps/${t.name}`,{withFileTypes:!0}))e.isDirectory()&&this.remove(`${this.root}/apps/${t.name}/${e.name}`)&&(r+=1)}catch{}return r}}function N(n,r){a(`/proc/${n}/oom_score_adj`,String(r))}m(N,"setOomScoreAdj");export{b as CGROUP_ROOT,S as Cgroups,N as setOomScoreAdj};

package/dist/launcher/limits/manager.d.ts

@@ -1,8 +1,8 @@
 import { Cgroups } from './cgroup.js';
-import { type AppBudget, type PurgeMode, type TmpfsSizes } from './policy.js';
+import { type AppBudget, type MemoryAdmission, type PurgeMode, type TmpfsSizes } from './policy.js';
 import type { PurgeResult } from './purge-worker.js';
 import { ProjectQuota } from './quota.js';
-import type { LimitEvent, LimitsReport } from '../spec.js';
+import type { DiskUsageReport, LimitEvent, LimitsReport } from '../spec.js';
 export interface RunPlan {
     /** The run's cgroup directory, or null when per-app memory limits are off. */
     cgroupDir: string | null;
@@ -37,10 +37,14 @@ export interface ManagerOptions {
     purge?: (cacheDir: string, mode: PurgeMode, uid: number, gid: number, now: number) => Promise<PurgeResult>;
     /** Fraction of the apps volume in use. */
     diskUsedFraction?: () => number;
+    /** Bytes under a directory; the default shells out to `du -x`. */
+    walk?: (dir: string) => Promise<number | null>;
     setOom?: (pid: number, value: number) => void;
     now?: () => number;
 }
 export declare function spawnPurge(cacheDir: string, mode: PurgeMode, uid: number, gid: number, now: number): Promise<PurgeResult>;
+/** Bytes under one directory on its own filesystem, or null when the walk failed or timed out. */
+export declare function walkBytes(dir: string): Promise<number | null>;
 export declare class LimitsManager {
     private readonly options;
     readonly cgroups: Cgroups;
@@ -53,6 +57,13 @@ export declare class LimitsManager {
     private readonly diskUsedFraction;
     private readonly setOom;
     private readonly now;
+    private readonly walk;
+    private diskUsage;
+    private diskUsageRun;
+    /** Admitted broker launches that have not reached `prepareRun` yet, by node. */
+    private readonly tickets;
+    /** Live broker runs by run group directory. Non-broker launches (tool calls, background jobs) never appear. */
+    private readonly brokerRuns;
     constructor(options: ManagerOptions);
     /** Queue an event for the daemon to emit with the next reply. */
     note(event: LimitEvent): void;
@@ -61,8 +72,14 @@ export declare class LimitsManager {
     budget(app: string): AppBudget;
     /** Write the app's memory cap and disk quota when they differ from what was last applied. */
     applyApp(app: string, uid: number, force?: boolean): AppBudget;
-    /** Everything a launch needs: the app's limits applied, a fresh run group, the sandbox tmpfs sizes. */
-    prepareRun(uid: number, nodeId: string): RunPlan;
+    /**
+     * Whether one more broker of `app` fits in its memory now. Measured here because only the launcher can read the cgroup
+     * files; an admit issues a ticket that holds the boot reservation until that node's broker launches or the ticket expires.
+     * Never throws and never writes a limit: cgroups that cannot be read admit.
+     */
+    admitBroker(app: string, nodeId: string): MemoryAdmission;
+    /** Everything a launch needs: the app's limits applied, a fresh run group, the sandbox tmpfs sizes. A broker launch (`broker`) joins the admission ledger and consumes its node's ticket. */
+    prepareRun(uid: number, nodeId: string, broker?: boolean): RunPlan;
     /** An app already past 80 % of its quota loses its cache before the sandbox starts. */
     private purgeIfNearQuota;
     /** Called when the sandbox's first process exists (its pid is bwrap's). */
@@ -76,6 +93,12 @@ export declare class LimitsManager {
     /** crtrd must outlive every sandbox when memory runs out. */
     protectDaemon(pid: number, value: number): void;
     show(): LimitsReport;
+    /**
+     * What the data volume holds, per app. A directory walk rather than project quota, so it works
+     * on hosts whose kernel cannot enforce quota. Cached for a minute; concurrent callers share one walk.
+     */
+    diskUsageReport(): Promise<DiskUsageReport>;
+    private measureDisk;
     purgeApp(app: string, uid: number, mode: PurgeMode): Promise<CachePassEntry | null>;
     /** One cache pass over every registered app. */
     cachePass(): Promise<CachePassEntry[]>;

package/dist/launcher/limits/manager.js

@@ -1 +1 @@
-var y=Object.defineProperty;var u=(l,e)=>y(l,"name",{value:e,configurable:!0});import{spawn as v}from"node:child_process";import{randomBytes as b}from"node:crypto";import{existsSync as k,statfsSync as w}from"node:fs";import{dirname as S,join as g}from"node:path";import{fileURLToPath as B}from"node:url";import{Cgroups as _,setOomScoreAdj as q}from"./cgroup.js";import{APP_QUOTA_CLEANUP_FRACTION as D,CONVERSATIONS_DISK_BYTES as A,appProjectId as c,bareApp as h,budgetFor as P,conversationsProjectId as O,passesFraction as N,runGroupName as R,selectCachePurges as F,tmpfsSizes as I}from"./policy.js";import{ProjectQuota as M}from"./quota.js";import{readSettings as U}from"./settings.js";function x(l,e,t,i,s){const r=g(S(B(import.meta.url)),"purge-worker.js");return new Promise((o,n)=>{const a=v("/bin/setpriv",["--reuid",String(t),"--regid",String(i),"--groups",String(i),"--no-new-privs","--",process.execPath,r,l,e,String(s)],{stdio:["ignore","pipe","pipe"]});let d="",m="";a.stdout.on("data",p=>{d+=String(p)}),a.stderr.on("data",p=>{m+=String(p)}),a.once("error",n),a.once("close",p=>{if(p!==0)return n(new Error(`purge worker exited ${p}: ${m.trim().slice(0,200)}`));try{o(JSON.parse(d))}catch(f){n(f)}})})}u(x,"spawnPurge");function T(l){const e=w(l),t=Number(e.blocks);return t>0?1-Number(e.bavail)/t:0}u(T,"appsDiskUsedFraction");class V{static{u(this,"LimitsManager")}options;cgroups;quota;events=[];appliedMemory=new Map;appliedDisk=new Map;purging=new Set;purge;diskUsedFraction;setOom;now;constructor(e){this.options=e,this.cgroups=e.cgroups??new _,this.quota=e.quota??new M(e.appsRoot,e.stateDir),this.purge=e.purge??x,this.diskUsedFraction=e.diskUsedFraction??(()=>T(e.appsRoot)),this.setOom=e.setOom??q,this.now=e.now??Date.now}note(e){this.events.push(e),this.events.length>200&&this.events.splice(0,this.events.length-200)}takeEvents(){return this.events.splice(0,this.events.length)}init(){this.cgroups.init()?this.cgroups.sweep():this.note({level:"warn",event:"limits.memory.unavailable",fields:{reason:this.cgroups.unavailable}}),this.quota.init()||this.note({level:"warn",event:"limits.disk.unavailable",fields:{reason:this.quota.unavailable}})}budget(e){return P(e,U(this.options.runtimeRoot))}applyApp(e,t,i=!1){const s=this.budget(e),r=h(e);if(this.cgroups.unavailable===null){const o=JSON.stringify([s.memoryBytes,s.pids]);if(i||this.appliedMemory.get(r)!==o)try{this.cgroups.applyApp(r,s),this.appliedMemory.set(r,o),this.note({level:"info",event:"limits.memory.applied",fields:{app:r,memory_bytes:s.memoryBytes,pids:s.pids}})}catch(n){this.note({level:"warn",event:"limits.memory.failed",fields:{app:r,error:String(n)}})}}if(this.quota.unavailable===null)try{if(this.quota.tag(r,t,{uid:0,gid:0})){const o=String(s.diskBytes);(i||this.appliedDisk.get(t)!==o)&&(this.quota.setLimit(c(t),s.diskBytes),this.quota.setLimit(O(t),A),this.appliedDisk.set(t,o),this.note({level:"info",event:"limits.disk.applied",fields:{app:r,disk_bytes:s.diskBytes}}))}}catch(o){this.note({level:"warn",event:"limits.disk.failed",fields:{app:r,error:String(o)}})}return s}prepareRun(e,t){const i=this.options.users.appFor(e)??String(e),s=this.applyApp(i,e);let r=null;if(this.cgroups.unavailable===null)try{r=this.cgroups.createRun(i,R(t,b(4).toString("hex")))}catch(o){this.note({level:"warn",event:"limits.memory.failed",nodeId:t,fields:{app:i,error:String(o)}})}return this.purgeIfNearQuota(i,e,s),{cgroupDir:r,app:h(i),nodeId:t,budget:s,tmpfs:I(s)}}purgeIfNearQuota(e,t,i){if(this.quota.unavailable!==null||i.diskBytes===null)return;const s=this.quota.usage().get(c(t))?.usedBytes??0;N(s,i.diskBytes,D)&&this.purgeApp(e,t,"all").catch(()=>{})}confirmStarted(e,t){e.cgroupDir!==null&&!this.cgroups.contains(e.cgroupDir,t)&&this.note({level:"warn",event:"limits.memory.not_joined",nodeId:e.nodeId,fields:{app:e.app,pid:t}})}finishRun(e){if(e.cgroupDir===null)return{oomKilled:!1};const t=e.cgroupDir,i=this.cgroups.oomKilled(t);this.cgroups.kill(t),i&&this.note({level:"warn",event:"run.oom_killed",nodeId:e.nodeId,fields:{app:e.app,memory_bytes:e.budget.memoryBytes}});let s=0;const r=u(()=>{this.cgroups.remove(t)||++s>=20||setTimeout(r,100).unref()},"retry");return r(),{oomKilled:i}}killRun(e){e?.cgroupDir&&this.cgroups.kill(e.cgroupDir)}protectDaemon(e,t){try{this.setOom(e,t)}catch(i){this.note({level:"warn",event:"limits.oom_score.failed",fields:{pid:e,error:String(i)}})}}show(){const e=this.quota.unavailable===null?this.quota.usage():new Map;return{memory:{available:this.cgroups.unavailable===null,reason:this.cgroups.unavailable},disk:{available:this.quota.unavailable===null,reason:this.quota.unavailable},apps:this.options.users.entries().map(({app:t,uid:i})=>{const s=this.budget(t);return{app:t,uid:i,memory_bytes:s.memoryBytes,disk_bytes:s.diskBytes,memory_source:s.memorySource,disk_source:s.diskSource,disk_used_bytes:e.get(c(i))?.usedBytes??null}})}}async purgeApp(e,t,i){const s=h(e),r=g(this.options.appsRoot,s,"cache");if(this.purging.has(s)||!k(r))return null;this.purging.add(s);try{const o=await this.purge(r,i,t,this.options.dataGid,this.now());return this.note({level:"info",event:"app.cache.purged",fields:{app:s,mode:i,files:o.files,bytes:o.bytes,dirs:o.dirs}}),{app:s,mode:i,files:o.files,bytes:o.bytes}}catch(o){return this.note({level:"warn",event:"app.cache.purge_failed",fields:{app:s,mode:i,error:String(o)}}),null}finally{this.purging.delete(s)}}async cachePass(){const e=this.quota.unavailable===null?this.quota.usage():new Map;let t=0;try{t=this.diskUsedFraction()}catch{}const i=this.options.users.entries(),s=F(i.map(({app:n,uid:a})=>({app:n,usedBytes:e.get(c(a))?.usedBytes??0,limitBytes:this.budget(n).diskBytes})),t),r=new Map(i.map(n=>[n.app,n.uid]));return(await Promise.all(s.map(({app:n,mode:a})=>this.purgeApp(n,r.get(n),a)))).filter(n=>n!==null)}}export{V as LimitsManager,x as spawnPurge};
+var D=Object.defineProperty;var d=(c,t)=>D(c,"name",{value:t,configurable:!0});import{spawn as w}from"node:child_process";import{randomBytes as A}from"node:crypto";import{existsSync as S,statfsSync as _}from"node:fs";import{dirname as B,join as b}from"node:path";import{fileURLToPath as N}from"node:url";import{Cgroups as O,setOomScoreAdj as P}from"./cgroup.js";import{ADMISSION_TICKET_MS as q,APP_QUOTA_CLEANUP_FRACTION as U,BOOT_RESERVATION_BYTES as M,BOOT_WINDOW_MS as T,CONVERSATIONS_DISK_BYTES as I,appCgroupPath as C,appProjectId as v,bareApp as g,bootPendingBytes as E,budgetFor as F,conversationsProjectId as x,memoryAdmission as L,passesFraction as K,runGroupName as j,selectCachePurges as Q,tmpfsSizes as G}from"./policy.js";import{ProjectQuota as $}from"./quota.js";import{readSettings as z}from"./settings.js";function J(c,t,s,i,e){const r=b(B(N(import.meta.url)),"purge-worker.js");return new Promise((o,n)=>{const a=w("/bin/setpriv",["--reuid",String(s),"--regid",String(i),"--groups",String(i),"--no-new-privs","--",process.execPath,r,c,t,String(e)],{stdio:["ignore","pipe","pipe"]});let h="",l="";a.stdout.on("data",u=>{h+=String(u)}),a.stderr.on("data",u=>{l+=String(u)}),a.once("error",n),a.once("close",u=>{if(u!==0)return n(new Error(`purge worker exited ${u}: ${l.trim().slice(0,200)}`));try{o(JSON.parse(h))}catch(y){n(y)}})})}d(J,"spawnPurge");function V(c){return new Promise(t=>{if(!S(c))return t(null);const s=w("du",["-x","-B1","-s",c],{stdio:["ignore","pipe","ignore"],timeout:6e4});let i="";s.stdout.on("data",e=>{i+=String(e)}),s.once("error",()=>t(null)),s.once("close",(e,r)=>{const o=Number(i.split(/\s/)[0]);t(r===null&&(e===0||e===1)&&i.length>0&&Number.isFinite(o)?o:null)})})}d(V,"walkBytes");const W=6e4;function Y(c){const t=_(c),s=Number(t.blocks);return s>0?1-Number(t.bavail)/s:0}d(Y,"appsDiskUsedFraction");class at{static{d(this,"LimitsManager")}options;cgroups;quota;events=[];appliedMemory=new Map;appliedDisk=new Map;purging=new Set;purge;diskUsedFraction;setOom;now;walk;diskUsage=null;diskUsageRun=null;tickets=new Map;brokerRuns=new Map;constructor(t){this.options=t,this.cgroups=t.cgroups??new O,this.quota=t.quota??new $(t.appsRoot,t.stateDir),this.purge=t.purge??J,this.diskUsedFraction=t.diskUsedFraction??(()=>Y(t.appsRoot)),this.setOom=t.setOom??P,this.now=t.now??Date.now,this.walk=t.walk??V}note(t){this.events.push(t),this.events.length>200&&this.events.splice(0,this.events.length-200)}takeEvents(){return this.events.splice(0,this.events.length)}init(){this.cgroups.init()?this.cgroups.sweep():this.note({level:"warn",event:"limits.memory.unavailable",fields:{reason:this.cgroups.unavailable}}),this.quota.init()||this.note({level:"warn",event:"limits.disk.unavailable",fields:{reason:this.quota.unavailable}})}budget(t){return F(t,z(this.options.runtimeRoot))}applyApp(t,s,i=!1){const e=this.budget(t),r=g(t);if(this.cgroups.unavailable===null){const o=JSON.stringify([e.memoryBytes,e.pids]);if(i||this.appliedMemory.get(r)!==o)try{this.cgroups.applyApp(r,e),this.appliedMemory.set(r,o),this.note({level:"info",event:"limits.memory.applied",fields:{app:r,memory_bytes:e.memoryBytes,pids:e.pids}})}catch(n){this.note({level:"warn",event:"limits.memory.failed",fields:{app:r,error:String(n)}})}}if(this.quota.unavailable===null)try{if(this.quota.tag(r,s,{uid:0,gid:0})){const o=String(e.diskBytes);(i||this.appliedDisk.get(s)!==o)&&(this.quota.setLimit(v(s),e.diskBytes),this.quota.setLimit(x(s),I),this.appliedDisk.set(s,o),this.note({level:"info",event:"limits.disk.applied",fields:{app:r,disk_bytes:e.diskBytes}}))}}catch(o){this.note({level:"warn",event:"limits.disk.failed",fields:{app:r,error:String(o)}})}return e}admitBroker(t,s){const i={verdict:"admitted"};if(this.cgroups.unavailable!==null)return i;const e=this.now();for(const[m,f]of this.tickets)e-f.issuedAt>=q&&this.tickets.delete(m);if(this.tickets.has(s))return i;const r=g(t),o=this.cgroups.root,n=`${o}/apps`,a=C(o,r),h=[...this.tickets.values()],l=[...this.brokerRuns].map(([m,f])=>({dir:m,...f})),u=d((m,f,k)=>({max:f,current:this.cgroups.memoryCurrent(m)??0,pending:h.filter(p=>k(p.app)).length*M+l.filter(p=>k(p.app)&&e-p.startedAt<T).reduce((p,R)=>p+E(this.cgroups.memoryCurrent(R.dir)??0),0),hasBroker:h.some(p=>k(p.app))||l.some(p=>k(p.app))}),"level"),y=L({app:u(a,this.cgroups.exists(a)?this.cgroups.memoryLimit(a):this.budget(r).memoryBytes,m=>m===r),apps:u(n,this.cgroups.memoryLimit(n),()=>!0)});return y.verdict==="admitted"&&this.tickets.set(s,{app:r,issuedAt:e}),y}prepareRun(t,s,i=!1){const e=this.options.users.appFor(t)??String(t),r=this.applyApp(e,t);let o=null;if(this.cgroups.unavailable===null)try{o=this.cgroups.createRun(e,j(s,A(4).toString("hex")))}catch(n){this.note({level:"warn",event:"limits.memory.failed",nodeId:s,fields:{app:e,error:String(n)}})}return i&&(this.tickets.delete(s),o!==null&&this.brokerRuns.set(o,{app:g(e),nodeId:s,startedAt:this.now()})),this.purgeIfNearQuota(e,t,r),{cgroupDir:o,app:g(e),nodeId:s,budget:r,tmpfs:G(r)}}purgeIfNearQuota(t,s,i){if(this.quota.unavailable!==null||i.diskBytes===null)return;const e=this.quota.usage().get(v(s))?.usedBytes??0;K(e,i.diskBytes,U)&&this.purgeApp(t,s,"all").catch(()=>{})}confirmStarted(t,s){t.cgroupDir!==null&&!this.cgroups.contains(t.cgroupDir,s)&&this.note({level:"warn",event:"limits.memory.not_joined",nodeId:t.nodeId,fields:{app:t.app,pid:s}})}finishRun(t){if(t.cgroupDir===null)return{oomKilled:!1};const s=t.cgroupDir;this.brokerRuns.delete(s);const i=this.cgroups.oomKilled(s);this.cgroups.kill(s),i&&this.note({level:"warn",event:"run.oom_killed",nodeId:t.nodeId,fields:{app:t.app,memory_bytes:t.budget.memoryBytes}});let e=0;const r=d(()=>{this.cgroups.remove(s)||++e>=20||setTimeout(r,100).unref()},"retry");return r(),{oomKilled:i}}killRun(t){t?.cgroupDir&&this.cgroups.kill(t.cgroupDir)}protectDaemon(t,s){try{this.setOom(t,s)}catch(i){this.note({level:"warn",event:"limits.oom_score.failed",fields:{pid:t,error:String(i)}})}}show(){const t=this.quota.unavailable===null?this.quota.usage():new Map;return{memory:{available:this.cgroups.unavailable===null,reason:this.cgroups.unavailable},disk:{available:this.quota.unavailable===null,reason:this.quota.unavailable},apps:this.options.users.entries().map(({app:s,uid:i})=>{const e=this.budget(s);return{app:s,uid:i,memory_bytes:e.memoryBytes,disk_bytes:e.diskBytes,memory_source:e.memorySource,disk_source:e.diskSource,disk_used_bytes:t.get(v(i))?.usedBytes??null}})}}async diskUsageReport(){const t=this.diskUsage;return t&&this.now()-t.at<W?t.report:(this.diskUsageRun??=this.measureDisk().finally(()=>{this.diskUsageRun=null}),this.diskUsageRun)}async measureDisk(){const t=_(this.options.appsRoot),s=Number(t.bsize),i=Number(t.blocks)*s,e=i-Number(t.bfree)*s,r=[...new Set(this.options.users.entries().map(l=>l.app))],[o,n,a]=await Promise.all([Promise.all(r.map(async l=>({app:l,bytes:await this.walk(b(this.options.appsRoot,g(l)))??0}))),this.walk(b(B(this.options.appsRoot),"user")),this.walk(this.options.runtimeRoot)]),h={total_bytes:i,used_bytes:e,apps:o.sort((l,u)=>u.bytes-l.bytes||l.app.localeCompare(u.app)),user_bytes:n,runtime_bytes:a,measured_at:this.now()};return this.diskUsage={at:this.now(),report:h},h}async purgeApp(t,s,i){const e=g(t),r=b(this.options.appsRoot,e,"cache");if(this.purging.has(e)||!S(r))return null;this.purging.add(e);try{const o=await this.purge(r,i,s,this.options.dataGid,this.now());return this.note({level:"info",event:"app.cache.purged",fields:{app:e,mode:i,files:o.files,bytes:o.bytes,dirs:o.dirs}}),{app:e,mode:i,files:o.files,bytes:o.bytes}}catch(o){return this.note({level:"warn",event:"app.cache.purge_failed",fields:{app:e,mode:i,error:String(o)}}),null}finally{this.purging.delete(e)}}async cachePass(){const t=this.quota.unavailable===null?this.quota.usage():new Map;let s=0;try{s=this.diskUsedFraction()}catch{}const i=this.options.users.entries(),e=Q(i.map(({app:n,uid:a})=>({app:n,usedBytes:t.get(v(a))?.usedBytes??0,limitBytes:this.budget(n).diskBytes})),s),r=new Map(i.map(n=>[n.app,n.uid]));return(await Promise.all(e.map(({app:n,mode:a})=>this.purgeApp(n,r.get(n),a)))).filter(n=>n!==null)}}export{at as LimitsManager,J as spawnPurge,V as walkBytes};

package/dist/launcher/limits/policy.d.ts

@@ -70,6 +70,49 @@ export declare function wasOomKilled(events: MemoryEvents): boolean;
 /** The terminal reason a node ends with when its run was OOM-killed. */
 export declare const OUT_OF_MEMORY_REASON: "out_of_memory";
 export declare function exitReasonFor(oomKilled: boolean, ordinary: 'crashed' | 'boot_failed' | 'crash_looped'): 'out_of_memory' | typeof ordinary;
+/** Largest kernel `memory.peak` of any broker run group in its first 20 s (498 MiB measured), rounded up. */
+export declare const BOOT_RESERVATION_BYTES: number;
+/** How long a launched broker still counts as booting; no measured group exceeded 261 MiB after it. */
+export declare const BOOT_WINDOW_MS = 20000;
+/** Growth the app's running nodes can still add: the largest measured (833 MiB), rounded up. */
+export declare const GROWTH_HEADROOM_BYTES: number;
+/** How long an admission holds its boot reservation while the launch it admitted has not reached the launcher. */
+export declare const ADMISSION_TICKET_MS = 30000;
+/** What a broker that started `BOOT_WINDOW_MS` ago or less still has to claim: its reservation less what it holds now, so it is not counted twice. */
+export declare function bootPendingBytes(currentBytes: number): number;
+/** One cgroup level's memory as the launcher measured it. */
+export interface MemoryLevel {
+    /** null means no cap at this level. */
+    max: number | null;
+    current: number;
+    /** Boot reservations not yet visible in `current`. */
+    pending: number;
+    /** A live broker run or an unexpired ticket exists at this level. */
+    hasBroker: boolean;
+}
+export type MemoryAdmissionLevel = 'app' | 'apps';
+export type MemoryAdmission = {
+    verdict: 'admitted';
+} | {
+    verdict: 'denied';
+    level: MemoryAdmissionLevel;
+    free_bytes: number;
+    needed_bytes: number;
+} | {
+    verdict: 'too_small';
+    level: MemoryAdmissionLevel;
+    max_bytes: number;
+    needed_bytes: number;
+};
+/**
+ * Whether one more broker may start. Each capped level must cover the boot reservation, plus the growth headroom
+ * once a broker already runs there: with none, the broker's own boot is the only risk, and waiting would never help.
+ * A cap below the reservation can never admit a broker. `app` is checked before `apps`.
+ */
+export declare function memoryAdmission(levels: {
+    app: MemoryLevel;
+    apps: MemoryLevel;
+}): MemoryAdmission;
 export interface TmpfsSizes {
     root: number;
     tmp: number;

package/dist/launcher/limits/policy.js

@@ -1 +1 @@
-var m=Object.defineProperty;var r=(o,t)=>m(o,"name",{value:t,configurable:!0});const I=1073741824,R=1048576,S="terminal",f=1073741824,N=2147483648,y=1073741824,D=2048,F=536870912,C=67108864,d=536870912,Y=-900,h=300,U=1e6,L=67108864,K=16777216;function i(o){return o.startsWith("app:")?o.slice(4):o}r(i,"bareApp");function $(o,t){const e=i(o),n=t.apps[e]??{},s=e===S,_=r((u,E)=>u===void 0?{bytes:s?null:E,source:"default"}:{bytes:u==="unlimited"?null:u,source:"override"},"resolve"),c=_(n.memory_bytes,2147483648),p=_(n.disk_bytes,1073741824);return{memoryBytes:c.bytes,diskBytes:p.bytes,pids:s?null:2048,memorySource:c.source,diskSource:p.source}}r($,"budgetFor");function g(o,t=1073741824){return o>t?Math.floor(o-t):Math.max(67108864,Math.floor(o/2))}r(g,"appsMemoryMax");function k(o){return{memoryMax:o.memoryBytes===null?"max":String(o.memoryBytes),memoryHigh:"max",swapMax:o.memoryBytes===null?"max":"0",pidsMax:o.pids===null?"max":String(o.pids)}}r(k,"appCgroupValues");function x(o,t){return`${o}/apps/${i(t)}`}r(x,"appCgroupPath");function G(o,t,e){return`${x(o,t)}/${e}`}r(G,"runCgroupPath");const a=/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;function V(o,t){const e=`${o}.${t}`;if(!a.test(e))throw new Error(`invalid run group name: ${e}`);return e}r(V,"runGroupName");function w(o){return o}r(w,"appProjectId");function X(o){return o+1e6}r(X,"conversationsProjectId");function J(o){return o===null?0:Math.ceil(o/1024)}r(J,"quotaBlocks");function M(o,t,e){return t!==null&&t>0&&o>t*e}r(M,"passesFraction");const l=.8,T=.85;function b(o){const t=r(e=>{const n=new RegExp(`^${e}\\s+(\\d+)\\s*$`,"m").exec(o);return n?Number(n[1]):0},"read");return{oomKill:t("oom_kill"),oomGroupKill:t("oom_group_kill")}}r(b,"parseMemoryEvents");function z(o){return o.oomKill>0||o.oomGroupKill>0}r(z,"wasOomKilled");const A="out_of_memory";function H(o,t){return o?A:t}r(H,"exitReasonFor");function j(o){return{root:67108864,tmp:o.memoryBytes===null?1073741824:Math.max(67108864,Math.min(536870912,Math.floor(o.memoryBytes/4)))}}r(j,"tmpfsSizes");const B=4320*6e4,O=60*6e4;function v(o,t){return t-Math.max(o.atimeMs,o.mtimeMs)>=B}r(v,"isStale");function Z(o,t){const e=t>T;return o.map(({app:n,usedBytes:s,limitBytes:_})=>({app:n,mode:e||M(s,_,l)?"all":"stale"}))}r(Z,"selectCachePurges");function q(o,t){return o===null||t-o>=O}r(q,"cachePassDue");export{l as APP_QUOTA_CLEANUP_FRACTION,O as CACHE_PASS_INTERVAL_MS,B as CACHE_STALE_MS,F as CONVERSATIONS_DISK_BYTES,U as CONVERSATIONS_PROJECT_OFFSET,Y as CRTRD_OOM_SCORE_ADJ,y as DEFAULT_APP_DISK_BYTES,N as DEFAULT_APP_MEMORY_BYTES,D as DEFAULT_APP_PIDS,T as DISK_CLEANUP_FRACTION,I as GIB,R as MIB,K as MIN_DISK_BYTES,L as MIN_MEMORY_BYTES,A as OUT_OF_MEMORY_REASON,f as RUNTIME_RESERVE_BYTES,h as SANDBOX_OOM_SCORE_ADJ,C as SANDBOX_ROOT_TMPFS_BYTES,d as SANDBOX_TMP_TMPFS_BYTES,S as UNCAPPED_BY_DEFAULT_APP,x as appCgroupPath,k as appCgroupValues,w as appProjectId,g as appsMemoryMax,i as bareApp,$ as budgetFor,q as cachePassDue,X as conversationsProjectId,H as exitReasonFor,v as isStale,b as parseMemoryEvents,M as passesFraction,J as quotaBlocks,G as runCgroupPath,V as runGroupName,Z as selectCachePurges,j as tmpfsSizes,z as wasOomKilled};
+var x=Object.defineProperty;var e=(t,o)=>x(t,"name",{value:o,configurable:!0});const P=1073741824,R=1048576,E="terminal",y=1073741824,N=2147483648,D=1073741824,C=2048,F=536870912,Y=67108864,h=536870912,U=-900,K=300,L=1e6,g=67108864,$=16777216;function a(t){return t.startsWith("app:")?t.slice(4):t}e(a,"bareApp");function b(t,o){const n=a(t),r=o.apps[n]??{},s=n===E,_=e((p,m)=>p===void 0?{bytes:s?null:m,source:"default"}:{bytes:p==="unlimited"?null:p,source:"override"},"resolve"),u=_(r.memory_bytes,2147483648),i=_(r.disk_bytes,1073741824);return{memoryBytes:u.bytes,diskBytes:i.bytes,pids:s?null:2048,memorySource:u.source,diskSource:i.source}}e(b,"budgetFor");function k(t,o=1073741824){return t>o?Math.floor(t-o):Math.max(67108864,Math.floor(t/2))}e(k,"appsMemoryMax");function G(t){return{memoryMax:t.memoryBytes===null?"max":String(t.memoryBytes),memoryHigh:"max",swapMax:t.memoryBytes===null?"max":"0",pidsMax:t.pids===null?"max":String(t.pids)}}e(G,"appCgroupValues");function S(t,o){return`${t}/apps/${a(o)}`}e(S,"appCgroupPath");function V(t,o,n){return`${S(t,o)}/${n}`}e(V,"runCgroupPath");const M=/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;function w(t,o){const n=`${t}.${o}`;if(!M.test(n))throw new Error(`invalid run group name: ${n}`);return n}e(w,"runGroupName");function H(t){return t}e(H,"appProjectId");function X(t){return t+1e6}e(X,"conversationsProjectId");function v(t){return t===null?0:Math.ceil(t/1024)}e(v,"quotaBlocks");function l(t,o,n){return o!==null&&o>0&&t>o*n}e(l,"passesFraction");const T=.8,O=.85;function J(t){const o=e(n=>{const r=new RegExp(`^${n}\\s+(\\d+)\\s*$`,"m").exec(t);return r?Number(r[1]):0},"read");return{oomKill:o("oom_kill"),oomGroupKill:o("oom_group_kill")}}e(J,"parseMemoryEvents");function W(t){return t.oomKill>0||t.oomGroupKill>0}e(W,"wasOomKilled");const B="out_of_memory";function z(t,o){return t?B:o}e(z,"exitReasonFor");const c=512*1048576,j=2e4,A=850*1048576,Z=3e4;function q(t){return Math.max(0,c-t)}e(q,"bootPendingBytes");function Q(t){const o=["app","apps"].flatMap(r=>t[r].max===null?[]:[{level:r,...t[r],max:t[r].max}]),n=o.find(r=>r.max<c);if(n)return{verdict:"too_small",level:n.level,max_bytes:n.max,needed_bytes:c};for(const r of o){const s=c+(r.hasBroker?A:0),_=r.max-r.current-r.pending;if(_<s)return{verdict:"denied",level:r.level,free_bytes:_,needed_bytes:s}}return{verdict:"admitted"}}e(Q,"memoryAdmission");function tt(t){return{root:67108864,tmp:t.memoryBytes===null?1073741824:Math.max(67108864,Math.min(536870912,Math.floor(t.memoryBytes/4)))}}e(tt,"tmpfsSizes");const I=4320*6e4,d=60*6e4;function ot(t,o){return o-Math.max(t.atimeMs,t.mtimeMs)>=I}e(ot,"isStale");function et(t,o){const n=o>O;return t.map(({app:r,usedBytes:s,limitBytes:_})=>({app:r,mode:n||l(s,_,T)?"all":"stale"}))}e(et,"selectCachePurges");function rt(t,o){return t===null||o-t>=d}e(rt,"cachePassDue");export{Z as ADMISSION_TICKET_MS,T as APP_QUOTA_CLEANUP_FRACTION,c as BOOT_RESERVATION_BYTES,j as BOOT_WINDOW_MS,d as CACHE_PASS_INTERVAL_MS,I as CACHE_STALE_MS,F as CONVERSATIONS_DISK_BYTES,L as CONVERSATIONS_PROJECT_OFFSET,U as CRTRD_OOM_SCORE_ADJ,D as DEFAULT_APP_DISK_BYTES,N as DEFAULT_APP_MEMORY_BYTES,C as DEFAULT_APP_PIDS,O as DISK_CLEANUP_FRACTION,P as GIB,A as GROWTH_HEADROOM_BYTES,R as MIB,$ as MIN_DISK_BYTES,g as MIN_MEMORY_BYTES,B as OUT_OF_MEMORY_REASON,y as RUNTIME_RESERVE_BYTES,K as SANDBOX_OOM_SCORE_ADJ,Y as SANDBOX_ROOT_TMPFS_BYTES,h as SANDBOX_TMP_TMPFS_BYTES,E as UNCAPPED_BY_DEFAULT_APP,S as appCgroupPath,G as appCgroupValues,H as appProjectId,k as appsMemoryMax,a as bareApp,q as bootPendingBytes,b as budgetFor,rt as cachePassDue,X as conversationsProjectId,z as exitReasonFor,ot as isStale,Q as memoryAdmission,J as parseMemoryEvents,l as passesFraction,v as quotaBlocks,V as runCgroupPath,w as runGroupName,et as selectCachePurges,tt as tmpfsSizes,W as wasOomKilled};

package/dist/launcher/main.js

@@ -1,3 +1,3 @@
-var V=Object.defineProperty;var w=(f,u)=>V(f,"name",{value:u,configurable:!0});import{spawn as z}from"node:child_process";import{createHash as Q}from"node:crypto";import{chownSync as B,chmodSync as A,closeSync as W,existsSync as X,mkdirSync as H,readFileSync as Y,rmSync as Z,statSync as ee}from"node:fs";import{createServer as te}from"node:net";import{dirname as U,join as _}from"node:path";import{fileURLToPath as re}from"node:url";import{loadLinuxNative as ne}from"../native/linux.js";import{envHomeOverride as oe}from"../shared/env.js";import{launchSandbox as se}from"./bwrap.js";import{LaunchGate as ie}from"./launch-gate.js";import{serveAccepted as ae}from"./accepted-socket.js";import{installNetworkRule as ce}from"./netrule.js";import{LAUNCHER_SOCKET as b,LaunchRefused as p}from"./spec.js";import{systemIdentity as de,AppUsers as ue}from"./users.js";import{SourceValidator as le}from"./validate.js";import{prepareDaemonScopeDirectories as pe}from"./scope-directories.js";import{LimitsManager as fe}from"./limits/manager.js";import{CRTRD_OOM_SCORE_ADJ as me}from"./limits/policy.js";import{writeLimitsLog as he}from"./limits/log.js";const D=ne();function $(f){const u=f._handle?.fd;if(u===void 0||u<0)throw new Error("launcher socket fd unavailable");return u}w($,"socketFd");async function we(){if(process.platform!=="linux"||process.getuid?.()!==0||!D)throw new Error("launcher requires Linux root and its native addon");const{daemonUid:f,daemonGid:u,dataGid:L}=de(),R=U(b);if(H(R,{recursive:!0,mode:493}),(ee(R).mode&511)!==493)throw new Error(`${R} must be mode 0755`);B(R,f,u),process.umask(7);const N=_(R,"nodes");H(N,{recursive:!0,mode:504}),B(N,0,u),A(N,504);const v=new le(oe()||"/",u);pe(_(v.root,"runtime"),f,u);const k=new ue(L);ce();const m=new fe({runtimeRoot:v.runtime,appsRoot:_(v.root,"apps"),stateDir:_(v.runtime,"limits-state"),dataGid:L,users:k});m.init();const T=new Map,F=w(r=>{const l=m.takeEvents();for(const o of l)he(o);return l.length===0?r:{...r,events:l}},"reply_"),E=new ie,x=new Map,I=new Map,J=w((r,l)=>{const o=l.capture,a=Q("sha256"),e=w((c,s)=>{let d=0,h=!1;const g=[];return{take(C){s?.update(C);const G=Math.max(0,Math.min(C.length,c-d));G&&(g.push(C.subarray(0,G)),d+=G),G!==C.length&&(h=!0)},result:w(()=>({text:Buffer.concat(g).toString("utf8"),truncated:h}),"result")}},"chunks"),i=e(o.stdoutBytes,a),t=e(o.stderrBytes);let n=Buffer.alloc(0),S=0;return r.stdout?.on("data",c=>i.take(c)),r.stderr?.on("data",c=>{const s=c;o.stderrTail?(S+=s.length,n=s.length>=o.stderrBytes?s.subarray(s.length-o.stderrBytes):Buffer.concat([n,s]).subarray(Math.max(0,n.length+s.length-o.stderrBytes))):t.take(s)}),new Promise(c=>r.once("close",(s,d)=>{const h=i.result(),g=o.stderrTail?{text:n.toString("utf8"),truncated:S>o.stderrBytes}:t.result();c({exitCode:s,signal:d,stdout:h.text,stderr:g.text,stdoutTruncated:h.truncated,stderrTruncated:g.truncated,stdoutSha256:a.digest("hex")})}))},"captureExit"),j=w(r=>{const l=new Set;for(;r>1&&!l.has(r);){const o=x.get(r);if(o)return o;l.add(r);try{const a=Y(`/proc/${r}/status`,"utf8");r=Number(/^PPid:\s*(\d+)/m.exec(a)?.[1]??0)}catch{return null}}return null},"sandboxForPid");let y,M=!1;const q=_(U(re(import.meta.url)),"../daemon/crtrd-cli.js"),K=w(()=>{M||(y=z("/bin/setpriv",["--reuid",String(f),"--regid",String(u),"--groups",String(L),"--",process.execPath,q,...process.argv.slice(2)],{stdio:"inherit",env:{...process.env,CRTR_LAUNCHER_SOCKET:b,CRTR_HOME:v.root}}),y.pid&&m.protectDaemon(y.pid,me),y.once("exit",()=>{y=void 0,M||K()}),y.once("error",r=>{console.error("crtrd spawn failed:",r),process.exitCode=1,M=!0}))},"startDaemon");X(b)&&Z(b);const O=te({pauseOnConnect:!0},r=>ae(r,l=>{const o=[];return(async()=>{try{if(D.peerCred($(r)).uid!==f)throw new p("invalid_spec","launcher peer is not crtrd");let a;for(;;)try{a=D.recvFds($(r));break}catch(t){if(t.code!=="EINTR")throw t}l(),o.push(...a.fds);const e=JSON.parse(a.data);let i;switch(e.operation){case"launch":{if(o.length>1)throw new p("invalid_uid","app uid is not registered");let t,n,S;try{n=await E.launch(async()=>{if(!k.has(e.spec.uid))throw new p("invalid_uid","app uid is not registered");const c=S=m.prepareRun(e.spec.uid,e.spec.nodeId);return se(e.spec,o[0],v,L,s=>{if(!s.pid)throw new p("launch_failed","bwrap has no pid");t=s.pid;const d=s.pid;if(x.set(d,{child:s,uid:e.spec.uid,nodeId:e.spec.nodeId,mounts:e.spec.mounts,plan:c}),s.once("exit",()=>{x.delete(d);const h=m.finishRun(c);T.set(d,h.oomKilled),setTimeout(()=>T.delete(d),5*6e4).unref()}),e.spec.capture){const h=J(s,e.spec).then(g=>({...g,oomKilled:T.get(d)===!0}));I.set(d,h),h.then(()=>{setTimeout(()=>{I.get(d)===h&&I.delete(d)},3e5).unref()})}},c)})}catch(c){throw t!==void 0?I.delete(t):S&&m.finishRun(S),c}if(!n.pid)throw new p("launch_failed","bwrap has no pid");m.confirmStarted(S,n.pid),i={pid:n.pid};break}case"wait":{const t=I.get(e.pid);if(!t)throw new p("invalid_spec","unknown captured sandbox pid");const n=await t;I.delete(e.pid),i={result:n};break}case"stop":{const t=Number.isInteger(e.pid)?j(e.pid):null;if(!t||e.nodeId!==void 0&&t.nodeId!==e.nodeId)throw new p("invalid_spec","unknown sandbox pid");if(e.signal!==void 0&&e.signal!=="SIGTERM"&&e.signal!=="SIGKILL")throw new p("invalid_spec","invalid stop signal");const n=e.signal??"SIGKILL";t.child.kill(n),n==="SIGKILL"&&m.killRun(t.plan),i={ok:!0};break}case"user.create":i={uid:await E.write(()=>k.create(e.app,e.uid))};break;case"user.delete":{await E.write(()=>{if([...x.values()].some(t=>t.uid===e.uid))throw new p("invalid_spec","stop this app\u2019s sandboxes before deleting its user");k.delete(e.uid)}),i={ok:!0};break}case"users.sync":await E.write(()=>k.sync(e.users)),i={ok:!0};break;case"list":i={sandboxes:[...x].map(([t,n])=>({pid:t,uid:n.uid,nodeId:n.nodeId,mounts:n.mounts}))};break;case"limits.apply":{const t=k.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new p("invalid_uid","app is not registered");m.applyApp(t.app,t.uid,!0),i={ok:!0};break}case"limits.show":i={limits:m.show()};break;case"cache.purge":i={purged:await m.cachePass()};break;case"exit.status":i={oomKilled:T.get(e.pid)===!0};break;case"daemon.handover":y?.kill("SIGTERM"),i={ok:!0};break;default:throw new p("invalid_spec","unknown launcher operation")}return`${JSON.stringify(F(i))}
+var V=Object.defineProperty;var w=(m,l)=>V(m,"name",{value:l,configurable:!0});import{spawn as z}from"node:child_process";import{createHash as Q}from"node:crypto";import{chownSync as O,chmodSync as A,closeSync as W,existsSync as X,mkdirSync as H,readFileSync as Y,rmSync as Z,statSync as ee}from"node:fs";import{createServer as te}from"node:net";import{dirname as U,join as _}from"node:path";import{fileURLToPath as re}from"node:url";import{loadLinuxNative as ne}from"../native/linux.js";import{envHomeOverride as oe}from"../shared/env.js";import{launchSandbox as se}from"./bwrap.js";import{LaunchGate as ie}from"./launch-gate.js";import{serveAccepted as ae}from"./accepted-socket.js";import{installNetworkRule as de}from"./netrule.js";import{LAUNCHER_SOCKET as S,LaunchRefused as p}from"./spec.js";import{systemIdentity as ce,AppUsers as ue}from"./users.js";import{SourceValidator as pe}from"./validate.js";import{prepareDaemonScopeDirectories as le}from"./scope-directories.js";import{LimitsManager as fe}from"./limits/manager.js";import{CRTRD_OOM_SCORE_ADJ as me}from"./limits/policy.js";import{writeLimitsLog as he}from"./limits/log.js";const D=ne();function $(m){const l=m._handle?.fd;if(l===void 0||l<0)throw new Error("launcher socket fd unavailable");return l}w($,"socketFd");async function we(){if(process.platform!=="linux"||process.getuid?.()!==0||!D)throw new Error("launcher requires Linux root and its native addon");const{daemonUid:m,daemonGid:l,dataGid:L}=ce(),R=U(S);if(H(R,{recursive:!0,mode:493}),(ee(R).mode&511)!==493)throw new Error(`${R} must be mode 0755`);O(R,m,l),process.umask(7);const B=_(R,"nodes");H(B,{recursive:!0,mode:504}),O(B,0,l),A(B,504);const k=new pe(oe()||"/",l);le(_(k.root,"runtime"),m,l);const b=new ue(L);de();const d=new fe({runtimeRoot:k.runtime,appsRoot:_(k.root,"apps"),stateDir:_(k.runtime,"limits-state"),dataGid:L,users:b});d.init();const T=new Map,F=w(r=>{const f=d.takeEvents();for(const o of f)he(o);return f.length===0?r:{...r,events:f}},"reply_"),E=new ie,x=new Map,I=new Map,J=w((r,f)=>{const o=f.capture,a=Q("sha256"),e=w((c,i)=>{let u=0,h=!1;const g=[];return{take(C){i?.update(C);const G=Math.max(0,Math.min(C.length,c-u));G&&(g.push(C.subarray(0,G)),u+=G),G!==C.length&&(h=!0)},result:w(()=>({text:Buffer.concat(g).toString("utf8"),truncated:h}),"result")}},"chunks"),s=e(o.stdoutBytes,a),t=e(o.stderrBytes);let n=Buffer.alloc(0),v=0;return r.stdout?.on("data",c=>s.take(c)),r.stderr?.on("data",c=>{const i=c;o.stderrTail?(v+=i.length,n=i.length>=o.stderrBytes?i.subarray(i.length-o.stderrBytes):Buffer.concat([n,i]).subarray(Math.max(0,n.length+i.length-o.stderrBytes))):t.take(i)}),new Promise(c=>r.once("close",(i,u)=>{const h=s.result(),g=o.stderrTail?{text:n.toString("utf8"),truncated:v>o.stderrBytes}:t.result();c({exitCode:i,signal:u,stdout:h.text,stderr:g.text,stdoutTruncated:h.truncated,stderrTruncated:g.truncated,stdoutSha256:a.digest("hex")})}))},"captureExit"),j=w(r=>{const f=new Set;for(;r>1&&!f.has(r);){const o=x.get(r);if(o)return o;f.add(r);try{const a=Y(`/proc/${r}/status`,"utf8");r=Number(/^PPid:\s*(\d+)/m.exec(a)?.[1]??0)}catch{return null}}return null},"sandboxForPid");let y,M=!1;const q=_(U(re(import.meta.url)),"../daemon/crtrd-cli.js"),K=w(()=>{M||(y=z("/bin/setpriv",["--reuid",String(m),"--regid",String(l),"--groups",String(L),"--",process.execPath,q,...process.argv.slice(2)],{stdio:"inherit",env:{...process.env,CRTR_LAUNCHER_SOCKET:S,CRTR_HOME:k.root}}),y.pid&&d.protectDaemon(y.pid,me),y.once("exit",()=>{y=void 0,M||K()}),y.once("error",r=>{console.error("crtrd spawn failed:",r),process.exitCode=1,M=!0}))},"startDaemon");X(S)&&Z(S);const N=te({pauseOnConnect:!0},r=>ae(r,f=>{const o=[];return(async()=>{try{if(D.peerCred($(r)).uid!==m)throw new p("invalid_spec","launcher peer is not crtrd");let a;for(;;)try{a=D.recvFds($(r));break}catch(t){if(t.code!=="EINTR")throw t}f(),o.push(...a.fds);const e=JSON.parse(a.data);let s;switch(e.operation){case"launch":{if(o.length>1)throw new p("invalid_uid","app uid is not registered");let t,n,v;try{n=await E.launch(async()=>{if(!b.has(e.spec.uid))throw new p("invalid_uid","app uid is not registered");const c=v=d.prepareRun(e.spec.uid,e.spec.nodeId,e.spec.broker===!0);return se(e.spec,o[0],k,L,i=>{if(!i.pid)throw new p("launch_failed","bwrap has no pid");t=i.pid;const u=i.pid;if(x.set(u,{child:i,uid:e.spec.uid,nodeId:e.spec.nodeId,mounts:e.spec.mounts,plan:c}),i.once("exit",()=>{x.delete(u);const h=d.finishRun(c);T.set(u,h.oomKilled),setTimeout(()=>T.delete(u),5*6e4).unref()}),e.spec.capture){const h=J(i,e.spec).then(g=>({...g,oomKilled:T.get(u)===!0}));I.set(u,h),h.then(()=>{setTimeout(()=>{I.get(u)===h&&I.delete(u)},3e5).unref()})}},c)})}catch(c){throw t!==void 0?I.delete(t):v&&d.finishRun(v),c}if(!n.pid)throw new p("launch_failed","bwrap has no pid");d.confirmStarted(v,n.pid),s={pid:n.pid};break}case"wait":{const t=I.get(e.pid);if(!t)throw new p("invalid_spec","unknown captured sandbox pid");const n=await t;I.delete(e.pid),s={result:n};break}case"stop":{const t=Number.isInteger(e.pid)?j(e.pid):null;if(!t||e.nodeId!==void 0&&t.nodeId!==e.nodeId)throw new p("invalid_spec","unknown sandbox pid");if(e.signal!==void 0&&e.signal!=="SIGTERM"&&e.signal!=="SIGKILL")throw new p("invalid_spec","invalid stop signal");const n=e.signal??"SIGKILL";t.child.kill(n),n==="SIGKILL"&&d.killRun(t.plan),s={ok:!0};break}case"user.create":s={uid:await E.write(()=>b.create(e.app,e.uid))};break;case"user.delete":{await E.write(()=>{if([...x.values()].some(t=>t.uid===e.uid))throw new p("invalid_spec","stop this app\u2019s sandboxes before deleting its user");b.delete(e.uid)}),s={ok:!0};break}case"users.sync":await E.write(()=>b.sync(e.users)),s={ok:!0};break;case"list":s={sandboxes:[...x].map(([t,n])=>({pid:t,uid:n.uid,nodeId:n.nodeId,mounts:n.mounts}))};break;case"limits.apply":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new p("invalid_uid","app is not registered");d.applyApp(t.app,t.uid,!0),s={ok:!0};break}case"memory.admit":{const t=b.entries().find(n=>n.app===e.app.replace(/^app:/,""));if(!t)throw new p("invalid_uid","app is not registered");s=d.admitBroker(t.app,e.nodeId);break}case"limits.show":s={limits:d.show()};break;case"disk.usage":s={disk:await d.diskUsageReport()};break;case"cache.purge":s={purged:await d.cachePass()};break;case"exit.status":s={oomKilled:T.get(e.pid)===!0};break;case"daemon.handover":y?.kill("SIGTERM"),s={ok:!0};break;default:throw new p("invalid_spec","unknown launcher operation")}return`${JSON.stringify(F(s))}
 `}catch(a){const e=a instanceof p?a.refusal:"launch_failed";return console.error(`launcher refusal ${e}:`,a),`${JSON.stringify({refusal:e})}
-`}finally{for(const a of o)W(a)}})()}));await new Promise((r,l)=>{O.once("error",l),O.listen(b,r)}),B(b,f,u),A(b,384);const P=w(()=>{M=!0,y?.kill("SIGTERM");for(const{child:r}of x.values())r.kill("SIGKILL");O.close(),v.close()},"shutdown");process.once("SIGTERM",P),process.once("SIGINT",P),K()}w(we,"runLauncher"),await we().catch(f=>{console.error(f),process.exitCode=1});export{we as runLauncher};
+`}finally{for(const a of o)W(a)}})()}));await new Promise((r,f)=>{N.once("error",f),N.listen(S,r)}),O(S,m,l),A(S,384);const P=w(()=>{M=!0,y?.kill("SIGTERM");for(const{child:r}of x.values())r.kill("SIGKILL");N.close(),k.close()},"shutdown");process.once("SIGTERM",P),process.once("SIGINT",P),K()}w(we,"runLauncher"),await we().catch(m=>{console.error(m),process.exitCode=1});export{we as runLauncher};

package/dist/launcher/spec.d.ts

@@ -1,3 +1,4 @@
+import type { MemoryAdmission } from './limits/policy.js';
 export declare const APP_UID_MIN = 200000;
 export declare const APP_UID_MAX = 299999;
 export declare const LAUNCHER_SOCKET = "/run/crtr/launcher.sock";
@@ -22,6 +23,8 @@ export interface LaunchSpec {
         stdin?: string;
     };
     pluginRoot?: string;
+    /** The launch is a node's broker engine. Only the daemon's broker host sets it; the launcher counts such runs for memory admission. */
+    broker?: true;
     token?: never;
 }
 export type LauncherRequest = {
@@ -58,6 +61,16 @@ export type LauncherRequest = {
 } | {
     operation: 'limits.show';
 }
+/** Whether one more broker of this app fits in its memory now; an admit holds the boot reservation until the node's broker launch. */
+ | {
+    operation: 'memory.admit';
+    app: string;
+    nodeId: string;
+}
+/** Bytes the data volume holds, per app and for the person's files and the runtime directory (a directory walk, cached briefly). */
+ | {
+    operation: 'disk.usage';
+}
 /** One cache pass over every app (stale files; whole cache for an app past 80 % of its quota or a disk past 85 %). */
  | {
     operation: 'cache.purge';
@@ -98,6 +111,21 @@ export interface LimitsReport {
         disk_used_bytes: number | null;
     }>;
 }
+export type MemoryAdmitReply = MemoryAdmission;
+export interface DiskUsageReport {
+    /** The data volume the apps live on, from statfs. */
+    total_bytes: number;
+    used_bytes: number;
+    /** Bytes under each registered app's folder, largest first. */
+    apps: Array<{
+        app: string;
+        bytes: number;
+    }>;
+    user_bytes: number | null;
+    runtime_bytes: number | null;
+    /** When the walk ran, in epoch milliseconds. */
+    measured_at: number;
+}
 export interface CachePurged {
     app: string;
     mode: 'stale' | 'all';
@@ -131,8 +159,10 @@ export type LauncherReply = ({
     uid: number;
 } | {
     ok: true;
-} | {
+} | MemoryAdmitReply | {
     limits: LimitsReport;
+} | {
+    disk: DiskUsageReport;
 } | {
     purged: CachePurged[];
 } | {

package/docs/sdk/release-notes.md

@@ -7,7 +7,15 @@ description: Behavior changes in @north-light/sdk that an app must act on, newes
 
 Changes to `@north-light/sdk` that can change what an existing app sees. Newest first.
 
-## 0.3.493
+## 0.3.494
+
+`GET /v1/limits` reports how much of the data disk each app uses on a host without disk quota, and the disk as a whole. Additive: no request or response an existing app sends or reads changes.
+
+### `disk_used_bytes` and `data_disk` on `LimitsDTO` (additive)
+
+- `AppLimitsDTO.disk_used_bytes` was `null` whenever the host could not enforce project quota (every hosted runtime on Blaxel). It now holds the bytes under the app's folder, measured by a directory walk at most a minute old. It stays `null` when the host has no launcher, the first walk has not finished (a read waits at most 5 s for it), or the walk failed.
+- `LimitsDTO.data_disk` is new and optional: `{total_bytes, used_bytes, user_bytes, runtime_bytes}` for the data volume. `crtr sys limits show` prints it.
+
 
 An app can act on a person's hosted runtime with no sign-in of theirs, and `@north-light/plugin` can declare canvas object kinds. Everything below is additive: no request or response an existing app sends or reads changes.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.493",
+  "version": "0.3.495",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -138,7 +138,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.493",
+    "@north-light/identity": "0.3.495",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.493",
+  "version": "0.3.495",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",