@north-light/cli 0.3.561 → 0.3.563

package/dist/daemon/reconcilers/bash-deadline.js

@@ -1,3 +1 @@
-var E=Object.defineProperty;var m=(s,t)=>E(s,"name",{value:t,configurable:!0});import{existsSync as a,readFileSync as S,readdirSync as x,statSync as B}from"node:fs";import{join as l}from"node:path";import{bashJobPaths as j,bashJobsDir as J,activeBackgroundBashJobs as v,expiredAutoBackgroundBashJobs as g,formatBashElapsed as M,isJobQuietOk as N,readJobActivity as I,readJobSilence as P,writeJobSilence as T,isBashJobProcessConfirmedGone as k,readJobPgid as C,readJobPgidIdentity as L,stopBackgroundBashJob as D}from"../../core/bash-jobs.js";import{killBashJobObject as O,recordBashJobObject as F}from"../../core/spaces/stop.js";import{createFileBeneath as R}from"../../core/spaces/open-beneath.js";import{withCanvasWrite as f}from"../../core/canvas/db.js";import{finishJob as A,jobRow as V}from"../../core/graph/jobs.js";import{publish as G}from"../../core/graph/events.js";import{envBashSilenceMs as U}from"../../shared/env.js";import{getById as h}from"../../core/graph/objects.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as W}from"../../core/canvas/node-agent-paths.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as b}from"../../core/events/operation-id.js";import{isProfilePaused as X}from"../../core/profiles/manifest.js";const Y=300*1e3;function H(s,t,o){const e=s<o?0:s<2*o?o:s<4*o?2*o:4*o*Math.floor(s/(4*o));return e>t?e:null}m(H,"dueSilenceThresholdMs");const Q=5*1e3;class ft{static{m(this,"BashDeadlineReconciler")}lastScanAt=Number.NEGATIVE_INFINITY;startedAt=Date.now();async run(t,o){if(await this.publishExits(),!(t-this.lastScanAt<Q)){this.lastScanAt=t;for(const e of o.rows){if(e.status==="dead"||X(e.profile_id))continue;const r=await W(e.node_id);let i;try{await this.adoptUnrecorded(e.node_id,r),await this.settleVanished(r),i=g(r,t)}catch(n){b.fresh(()=>{c({level:"error",event:"bash_deadline.scan_failed",...u(e.node_id)?{node_id:e.node_id}:{},error:n})});continue}for(const n of i)await this.cancel(e.node_id,r,n.jobId);try{await this.warnSilent(r,t)}catch(n){b.fresh(()=>{c({level:"error",event:"bash_silence.scan_failed",...u(e.node_id)?{node_id:e.node_id}:{},error:n})})}}}}async warnSilent(t,o){const e=U();for(const{jobId:r}of v(t)){if(N(t,r))continue;const i=I(t,r);if(i===void 0||o-i.sampledAtMs>Y)continue;const n=o-i.lastActivityMs,y=P(t,r),_=H(n,y?.lastActivityMs===i.lastActivityMs?y.thresholdMs:0,e);if(_===null)continue;const w=j(t,r);await f(async p=>{const d=await h(p,r);d===null||d.type!=="job"||(await V(p,d))?.status!=="running"||await G(p,{event:"silent",object:d,actor:null,attrs:{"silent-for":M(n),files:i.files.join(", "),"log-path":w.jobLog}})}),T(w,i.lastActivityMs,_)}}async publishExits(){try{await f(async t=>{const o=await t.prepare(`SELECT j.object, j.job_dir FROM jobs j JOIN objects o ON o.id = j.object
-          WHERE j.status = 'running' AND o.deleted_at IS NULL`).all();for(const{object:e,job_dir:r}of o){const i=l(r,"job.exit");if(!a(i)||a(l(r,"job.stop")))continue;const n=Number(S(i,"utf8").trim());Number.isSafeInteger(n)&&await A(t,await h(t,e),{exitCode:n})}})}catch(t){b.fresh(()=>c({level:"error",event:"bash_completion.delivery_failed",error:t}))}}async adoptUnrecorded(t,o){let e;try{e=x(J(o))}catch{return}await f(async r=>{for(const i of e){if(await h(r,i)!==null)continue;const n=j(o,i);!a(n.jobBg)||!a(n.cmdSh)||a(n.jobExit)&&B(n.jobExit).mtimeMs<this.startedAt||await F(t,i,{command:S(n.cmdSh,"utf8"),jobDir:n.dir,logPath:n.jobLog})}})}async settleVanished(t){for(const{jobId:o}of v(t)){const e=j(t,o);a(e.jobExit)||a(l(e.dir,"job.stop"))||k(C(t,o),L(t,o))&&await f(async r=>{const i=await h(r,o);if(!(i===null||i.type!=="job")&&!(a(e.jobExit)||a(l(e.dir,"job.stop")))){await A(r,i,{killedBy:null,reason:"process disappeared without an exit record"});try{R(e.root,e.jobExit,`-1
-`)}catch(n){if(n.code!=="EEXIST")throw n}}})}}async cancel(t,o,e){(await D(o,e,!1,t)).kind==="stopped"&&await b.fresh(async()=>{try{c({level:"info",event:"bash_deadline.canceled",...u(t)?{node_id:t}:{},fields:{job_id:e}}),await O(e,async()=>null,"deadline")}catch(i){c({level:"error",event:"bash_deadline.notice_failed",...u(t)?{node_id:t}:{},error:i,fields:{job_id:e}})}})}}export{ft as BashDeadlineReconciler,H as dueSilenceThresholdMs};
+var A=Object.defineProperty;var m=(s,t)=>A(s,"name",{value:t,configurable:!0});import{existsSync as _,readFileSync as B,readdirSync as J,statSync as I}from"node:fs";import{bashJobPaths as g,bashJobsDir as C,clearJobDeadlineFile as E,formatBashElapsed as T,isBashJobProcessConfirmedGone as D,readJobSupervisorState as S,signalBashJob as O}from"../../core/bash-jobs.js";import{killBashJobObject as R,recordBashJobObject as j}from"../../core/spaces/stop.js";import{readFileBeneath as L}from"../../core/spaces/open-beneath.js";import{blobStore as P}from"../../core/blobs/index.js";import{openDb as c,withCanvasWrite as f}from"../../core/canvas/db.js";import{applyJobReport as N,endJobsOfExitedSandboxes as U,endReportedJobs as k,expiredJobs as F,jobRow as K,ownerRunsRemotely as v,recordJobSilence as q,runningJobs as b,setJobDeadline as z}from"../../core/graph/jobs.js";import{publish as G}from"../../core/graph/events.js";import{envBashSilenceMs as V}from"../../shared/env.js";import{getById as M}from"../../core/graph/objects.js";import{isSafeNodeId as u}from"../../core/canvas/paths.js";import{contextDir as x}from"../../core/canvas/node-agent-paths.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{isProfilePaused as Y}from"../../core/profiles/manifest.js";import{runThrottledLane as $}from"./throttled-lane.js";const H=300*1e3;function W(s,t,e){const i=s<e?0:s<2*e?e:s<4*e?2*e:4*e*Math.floor(s/(4*e));return i>t?i:null}m(W,"dueSilenceThresholdMs");const X=5*1e3,Q=60*1e3,y=64*1024;class ye{static{m(this,"BashDeadlineReconciler")}startedAt=Date.now();uploadedBytes=new Map;async run(t,e){await this.mirrorSupervisors(e),await this.publishExits(),await $("bash-deadline","",t,X,Q,i=>this.scan(t,e,i))}async scan(t,e,i){const o=new Set;for(const n of e.rows)if(n.status!=="dead"&&!Y(n.profile_id)&&(o.add(n.node_id),!await v(await c(),n.node_id)))try{await this.adoptUnrecorded(n.node_id,await x(n.node_id))}catch(a){p.fresh(()=>{l({level:"error",event:"bash_deadline.scan_failed",...u(n.node_id)?{node_id:n.node_id}:{},error:a})})}for(const n of await F(await c(),t))if(o.has(n.owner)){if(!await i.holds())return;await this.cancel(n)}for(const n of o)try{await this.warnSilent(n,t)}catch(a){p.fresh(()=>{l({level:"error",event:"bash_silence.scan_failed",...u(n)?{node_id:n}:{},error:a})})}}async warnSilent(t,e){const i=V();for(const o of await b(await c(),t)){const{activity:n}=o;if(o.quietOk||n===null||e-n.sampledAtMs>H)continue;const a=e-n.lastActivityMs,r=o.silence,d=W(a,r?.lastActivityMs===n.lastActivityMs?r.thresholdMs:0,i);d!==null&&await f(async h=>{const w=await M(h,o.jobId);w===null||w.type!=="job"||(await K(h,w))?.status!=="running"||(await G(h,{event:"silent",object:w,actor:null,attrs:{"silent-for":T(a),files:n.files.join(", "),"output-key":o.outputKey}}),await q(h,o.jobId,n.lastActivityMs,d))})}}async mirrorSupervisors(t){for(const e of t.rows)if(e.status!=="dead")try{const i=await b(await c(),e.node_id);if(i.length===0||await v(await c(),e.node_id))continue;const o=await x(e.node_id);for(const n of i)await this.mirror(o,n)}catch(i){p.fresh(()=>{l({level:"error",event:"bash_mirror.failed",...u(e.node_id)?{node_id:e.node_id}:{},error:i})})}}async mirror(t,e){const i=S(t,e.jobId),o=i.pgid??e.pgid??void 0,n=i.pgidIdentity??e.pgidIdentity??void 0,a=i.exitCode===void 0&&D(o,n)&&S(t,e.jobId).exitCode===void 0;i.deadlineMs!==void 0&&(e.deadlineAtMs===null&&await f(r=>z(r,e.jobId,i.deadlineMs)),E(g(t,e.jobId))),await this.uploadOutput(t,e),(i.exitCode!==void 0||a)&&this.uploadedBytes.delete(e.jobId),await f(async r=>N(r,e.jobId,{...i.pgid===void 0||i.pgid===e.pgid?{}:{pgid:i.pgid},...i.pgidIdentity===void 0||i.pgidIdentity===e.pgidIdentity?{}:{pgidIdentity:i.pgidIdentity},...i.activity===void 0?{}:{activity:i.activity},...i.exitCode===void 0?{}:{exitCode:i.exitCode},...a?{vanished:!0}:{}}))}async uploadOutput(t,e){const i=g(t,e.jobId);let o;try{o=I(i.jobLog).size}catch{return}const n=this.uploadedBytes.get(e.jobId)??0;if(o<=n)return;const a=await P(),r=L(i.root,i.jobLog);for(let d=Math.floor(n/y);d*y<r.length;d++)await a.put(`${e.outputKey}/${String(d).padStart(8,"0")}`,r.subarray(d*y,(d+1)*y));this.uploadedBytes.set(e.jobId,r.length)}async publishExits(){try{await f(async t=>{await k(t),await U(t)})}catch(t){p.fresh(()=>l({level:"error",event:"bash_completion.delivery_failed",error:t}))}}async adoptUnrecorded(t,e){let i;try{i=J(C(e))}catch{return}await f(async o=>{for(const n of i){if(await M(o,n)!==null)continue;const a=g(e,n);!_(a.jobBg)||!_(a.cmdSh)||_(a.jobExit)&&I(a.jobExit).mtimeMs<this.startedAt||await j(t,n,{command:B(a.cmdSh,"utf8")})}})}async cancel(t){const e=await v(await c(),t.owner);!e&&(t.pgid===null||t.pgidIdentity===null)||await p.fresh(async()=>{try{if(e||await O(t.pgid,t.pgidIdentity,t.owner),!await R(t.jobId,async()=>null,"deadline"))return;l({level:"info",event:"bash_deadline.canceled",...u(t.owner)?{node_id:t.owner}:{},fields:{job_id:t.jobId}})}catch(i){l({level:"error",event:"bash_deadline.notice_failed",...u(t.owner)?{node_id:t.owner}:{},error:i,fields:{job_id:t.jobId}})}})}}export{ye as BashDeadlineReconciler,W as dueSilenceThresholdMs};

package/dist/daemon/reconcilers/broker-supervision.d.ts

@@ -59,9 +59,8 @@ export declare function wedgeVerdict(input: {
 export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
 export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
 export declare class BrokerSupervisionReconciler {
-    private readonly yieldStalls;
-    private readonly wedgeNotifiedAt;
-    private readonly unattendedSince;
+    /** SIGTERM times of yield-stalled brokers this process signalled; the stall's start is `nodes.stall_noticed_at`. */
+    private readonly yieldTermedAt;
     private readonly sampleTree;
     private readonly signal;
     private lastOverCapWarnAt;
@@ -88,10 +87,21 @@ export declare class BrokerSupervisionReconciler {
      * starts the same isolated turn, so a concurrent main turn stays passive. */
     private requestParkSummary;
     private warnOverCapInTmux;
+    /** Stop a node's broker. A remote broker's stop is recorded on its row, enacted through its runner, and ends exactly that
+     *  execution once confirmed; any other broker is torn down by the local host. */
+    private teardownBroker;
+    private stopRemote;
+    /** A refresh that never reached its replacement: the same stall clock as a local broker, enacted through the runner. */
+    private handleRemoteYieldStall;
+    /** A remote broker has no process this daemon can sample, so its only evidence is the turn's progress clock. A quiet turn is
+     *  reported once and the broker is left running: with no view of its subprocesses the daemon cannot tell a runaway from a long command. */
+    private handleRemoteWedge;
     private handleYieldStall;
     private signalYieldStall;
     private readTreeSample;
     private handleWedgeDetection;
+    /** Record and announce a wedged broker: the event, the fault, and the notice to its managers. */
+    private reportWedge;
     /** Publish a fatal provider fault's notice once per episode and, for a node that settles on it, end the node.
      *  The notices, the `notified_since` mark and the terminal transition commit in one write, so a daemon exit
      *  leaves the fault either fully announced or fully unannounced. The daemon registers this as the announcer

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ee,listUnnotifiedFatalFaultNodes as te,markFaultNotified as ne,readFault as O,readLiveTurn as E,recordFault as re}from"../../core/canvas/recovery.js";import{errorClassFromFault as ie}from"../../core/fault-classifier.js";import{nextFaultRetry as ae}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as G,withCanvasWrite as se}from"../../core/canvas/db.js";import{getById as oe}from"../../core/graph/objects.js";import{watcherIdsOf as le}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as de,resetStaleExecutionMail as ce}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ue}from"../../core/bash-jobs.js";import{focusOf as fe}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as pe}from"../api/viewer-presence.js";import{deliverLive as me}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as he}from"../../core/runtime/broker-request.js";import{formatCard as _e,PARK_SUMMARY_PROMPT as ge}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ve}from"../../core/runtime/fleet.js";import{readConfig as we}from"../../core/config.js";import{beginPendingPark as ke,consumePendingPark as N,pendingParkIds as ye,pendingParkStartedAt as be}from"../park-pending.js";import{shouldParkAfterSummary as Se}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Ne}from"./live-obligation.js";import{readGoal as Pe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as $e}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Te,envTestUnattendedParkMs as Ae,parkSummaryGraceMs as Me,envUnattendedParkMs as Ee}from"../../shared/env.js";const A=j(z);async function Fe(s){return!await ee(s)&&((await Pe(s))?.trim()??"")!==""}p(Fe,"awaitsBirthTurn");async function xe(s){return(await E(s))?.busy===!0}p(xe,"liveBusy");async function Nt(s){const e=(await E(s))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(Nt,"liveTurnProgressMs");function L(s,e){return e===null?null:`${s}:${e}`}p(L,"incarnationOf");async function x(s){const e=await E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(x,"busySinceForIncarnation");const W=3*6e4,B=2e4,I=20*6e4,Ie=2,Ce=5*6e4;function De(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(De,"settlesOnFatalProviderFault");function Oe(){const s=Ee();if(s!==void 0)return s;const e=we("user").lifecycle.unattendedParkMs;return Te()?Ae()??e:e}p(Oe,"unattendedParkMsForDaemon");function U(){return Me()}p(U,"parkSummaryGraceMsForDaemon");function Re(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Re,"yieldStallVerdict");function Ge(s){return!s.busy||s.quietForMs===null||s.quietForMs<I?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ie?"pending":"wedged"}p(Ge,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
-`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Le(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Le,"sumTreeCpu");function We(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(We,"treeHasDescendants");class Pt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ve()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Oe();for(const i of await te())await F.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),await this.handleUnattendedParking(n,await T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ye())t.fleet.has(i)||N(i);await ce(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ue(await X(e)).length>0||await fe(e)!==null||pe(e))return!0;const n=await O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ae(n,i)!==null)?!0:await de(e)}async handleUnattendedParking(e,t,r,i,n,l){if(await J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=await this.parkResetCondition(e,t,r,i),a=be(e);if(a!==void 0){if(i-a<U())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!await Se(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:U()}}),await P(e,"park"),S.teardown(e);return}if(o||await xe(e)||await Fe(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=await Ne(e,t.lifecycle),h=d!=="cancel"&&$e(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){await P(e,d),S.teardown(e);return}if(d==="cancel"){await P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ke(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await me(e,_e("park",{},ge),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof he){await P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ce)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await A("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
-`).map(async h=>{if(h!=="")try{await A("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await A("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await x(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Re(!0,e.intent,!1,i-o.since)!=="kill")return;const a=await T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=B&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:B,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await A("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Le(t,e),i=We(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=L(t.pid,e.pi_pid_identity),o=await x(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=I?await this.sampleTree(t.pid):null,d=await T(n),h=d===null?null:await x(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(Ge({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const w=C(d),g=Math.round((a??I)/6e4),k=$??!0,v=!k&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,y=!1,b;if(k)b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(v!==null)b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),y=!0,b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(M){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:M,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),b=`Child wedged \u2014 ${w} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:k,kick_attempted:f,kicked:y,...v===null?{}:{kick_suppression_reason:v}}});try{await re(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:b})}catch(M){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:M})}await R(n,"notice.crash",b,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:y,...v===null?{}:{kick_suppression_reason:v}})}async handleFatalFault(e,t=!0){const r=await O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
-`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=De(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,w=d?"failed":"hit a fatal provider fault",g=new Set((await Q(e)).map(f=>f.node_id)),k=await le(await G(),await oe(await G(),e)),v=[{relation:"Child",recipients:k.filter(f=>g.has(f))},{relation:"Watched node",recipients:k.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),await se(async()=>{for(const{relation:f,recipients:y}of v)y.length!==0&&await R(e,"notice.crash",`${f} ${w} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,y);await ne(r.faultId),t&&_&&await Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:ie(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{Pt as BrokerSupervisionReconciler,Ie as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,U as parkSummaryGraceMsForDaemon,De as settlesOnFatalProviderFault,Le as sumTreeCpu,We as treeHasDescendants,Oe as unattendedParkMsForDaemon,Ge as wedgeVerdict,Re as yieldStallVerdict};
+var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,parkEvent as Ie}from"./live-obligation.js";import{readGoal as xe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Fe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Oe,envTestUnattendedParkMs as Ce,parkSummaryGraceMs as De,envUnattendedParkMs as We}from"../../shared/env.js";const R=ee(Z);async function Be(a){return!await ie(a)&&((await xe(a))?.trim()??"")!==""}m(Be,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Ge(a){return(await F(a))?.busy===!0}m(Ge,"liveBusy");async function It(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(It,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,Le=2,qe=5*6e4;function Ue(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ue,"settlesOnFatalProviderFault");function Ye(){const a=We();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Oe()?Ce()??e:e}m(Ye,"unattendedParkMsForDaemon");function H(){return De()}m(H,"parkSummaryGraceMsForDaemon");function K(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(K,"yieldStallVerdict");function He(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>Le?"pending":"wedged"}m(He,"wedgeVerdict");function V(a){const e=new Map,t=new Map;for(const r of a.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(V,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=V(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function Ve(a,e){const{byPid:t,childrenOf:r}=V(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(Ve,"treeHasDescendants");class xt{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Ye();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Ge(e)||await Be(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const u=await Ie(e,t.lifecycle),w=u!=="cancel"&&Fe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<w)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<qe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
+`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}K(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(K(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=Ve(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,k=s?.hasDescendants??null;if(He({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:k})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const v=x(p),y=Math.round((d??I)/6e4),b=k??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
+`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ue(i,typeof s?.code=="string"?s.code:void 0),k=u||w,v=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${v}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&k&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&k&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{xt as BrokerSupervisionReconciler,Le as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ue as settlesOnFatalProviderFault,Ke as sumTreeCpu,Ve as treeHasDescendants,Ye as unattendedParkMsForDaemon,He as wedgeVerdict,K as yieldStallVerdict};

package/dist/daemon/reconcilers/cron-lane.d.ts

@@ -3,6 +3,5 @@ export interface CronLaneContext {
     lifecycle: DetachedWorkLifecycle;
 }
 export declare class CronLaneReconciler {
-    private lastFolderSweep;
     run(now: number, ctx: CronLaneContext): Promise<void>;
 }

package/dist/daemon/reconcilers/cron-lane.js

@@ -1 +1 @@
-var l=Object.defineProperty;var t=(i,r)=>l(i,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as n}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";class w{static{t(this,"CronLaneReconciler")}lastFolderSweep=0;async run(r,a){await c.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}if(r-this.lastFolderSweep>=6e4){this.lastFolderSweep=r;try{await n()}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{w as CronLaneReconciler};
+var i=Object.defineProperty;var t=(c,r)=>i(c,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{runThrottledLane as m}from"./throttled-lane.js";class _{static{t(this,"CronLaneReconciler")}async run(r,a){await n.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}try{await m("cron-folder-sweep","",r,6e4,6e4,async()=>{await l()})}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{_ as CronLaneReconciler};

package/dist/daemon/reconcilers/fault-retry-backstop.d.ts

@@ -18,8 +18,6 @@ export interface FaultRetryBackstopContext {
 }
 export interface FaultRetryBackstopDeps {
     probeOnline?: () => Promise<boolean>;
-    /** Signal a node's broker; false when it holds no open signal stream. */
-    nudge?: (nodeId: string) => Promise<boolean>;
     list?: () => Promise<PendingProviderRetry[]>;
     busy?: (nodeId: string) => Promise<boolean>;
 }
@@ -28,14 +26,9 @@ export interface FaultRetryBackstopDeps {
 export declare function dueAtMs(pending: PendingProviderRetry): number | null;
 export declare class FaultRetryBackstopReconciler {
     private readonly deps;
-    private lastScanAt;
-    private scanning;
-    /** fault id → when the daemon last nudged it. Pruned to current candidates. */
-    private readonly nudgedAt;
     constructor(deps?: FaultRetryBackstopDeps);
-    /** Starts a detached scan (throttled, never overlapping). The returned promise
-     *  settles with that scan so a caller that wants to observe it can; the tick
-     *  does not await it. */
+    /** Starts a detached scan, throttled and exclusive across processes by the lane lease. The returned promise settles with the scan;
+     *  the tick does not await it. */
     run(now: number, ctx: FaultRetryBackstopContext): Promise<void>;
     private scan;
 }

package/dist/daemon/reconcilers/fault-retry-backstop.js

@@ -1 +1 @@
-var A=Object.defineProperty;var o=(a,t)=>A(a,"name",{value:t,configurable:!0});import{filterAsync as g}from"../../core/async-array.js";import{listPendingProviderRetries as y,readLiveTurn as _}from"../../core/canvas/recovery.js";import{probeOnline as I}from"../../core/runtime/connectivity.js";import{nextFaultRetry as b}from"../../core/runtime/fault-recovery.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{isSafeNodeId as R}from"../../core/canvas/paths.js";import{brokerSignalStreams as v}from"../api/handlers/broker-signals.js";const h=2*6e4,S=15e3;function m(a){const{fault:t}=a;if(t.retry.nextAt!==void 0){const i=Date.parse(t.retry.nextAt);return Number.isFinite(i)?i:null}const n=Date.parse(a.updatedAt),s=b(t,n);return Number.isFinite(n)&&s!==null?n+s.delayMs:null}o(m,"dueAtMs");class D{static{o(this,"FaultRetryBackstopReconciler")}deps;lastScanAt=Number.NEGATIVE_INFINITY;scanning=null;nudgedAt=new Map;constructor(t={}){this.deps=t}run(t,n){if(this.scanning!==null)return this.scanning;if(!n.lifecycle.acceptsDetachedWork()||t-this.lastScanAt<S)return Promise.resolve();this.lastScanAt=t;const s=this.scan(t,n).catch(i=>{p.fresh(()=>{f({level:"warn",event:"fault_retry_backstop.scan_failed",error:i})})}).finally(()=>{this.scanning=null});return this.scanning=s,n.lifecycle.registerDetached(s),s}async scan(t,n){const s=await(this.deps.list??y)(),i=new Set(s.map(e=>e.fault.faultId));for(const e of this.nudgedAt.keys())i.has(e)||this.nudgedAt.delete(e);const d=this.deps.busy??(async e=>(await _(e))?.busy===!0),u=await g(s,async e=>{if(!n.fleet.has(e.nodeId)||await d(e.nodeId))return!1;const r=m(e);if(r===null||t<r+h)return!1;const l=this.nudgedAt.get(e.fault.faultId);return l===void 0||t-l>=h});if(u.length===0)return;let c;for(const e of u){if(e.fault.kind==="connection"&&(c??=await(this.deps.probeOnline??(()=>I()))(),!c)||!n.fleet.has(e.nodeId)||await d(e.nodeId))continue;this.nudgedAt.set(e.fault.faultId,t);const r=await(this.deps.nudge??(async l=>await v.nudgeFaultRetry(l)))(e.nodeId);p.fresh(()=>{f({level:"info",event:"fault_retry_backstop.nudged",...R(e.nodeId)?{node_id:e.nodeId}:{},fields:{kind:e.fault.kind,since:e.fault.since,delivered:r,overdue_ms:t-(m(e)??t)}})})}}}export{S as FAULT_RETRY_SCAN_INTERVAL_MS,h as FAULT_RETRY_STALE_GRACE_MS,D as FaultRetryBackstopReconciler,m as dueAtMs};
+var _=Object.defineProperty;var o=(i,t)=>_(i,"name",{value:t,configurable:!0});import{filterAsync as y}from"../../core/async-array.js";import{withCanvasWrite as h}from"../../core/canvas/db.js";import{claimFaultNudge as A,listPendingProviderRetries as g,readLiveTurn as b}from"../../core/canvas/recovery.js";import{probeOnline as E}from"../../core/runtime/connectivity.js";import{nextFaultRetry as I}from"../../core/runtime/fault-recovery.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as f}from"../../core/events/operation-id.js";import{isSafeNodeId as R}from"../../core/canvas/paths.js";import{runThrottledLane as v}from"./throttled-lane.js";const l=2*6e4,L=15e3;function p(i){const{fault:t}=i;if(t.retry.nextAt!==void 0){const a=Date.parse(t.retry.nextAt);return Number.isFinite(a)?a:null}const n=Date.parse(i.updatedAt),r=I(t,n);return Number.isFinite(n)&&r!==null?n+r.delayMs:null}o(p,"dueAtMs");const S=6e4,T="fault-retry-backstop";class U{static{o(this,"FaultRetryBackstopReconciler")}deps;constructor(t={}){this.deps=t}run(t,n){if(!n.lifecycle.acceptsDetachedWork())return Promise.resolve();const r=v(T,"",t,L,S,()=>this.scan(t,n)).then(()=>{}).catch(a=>{f.fresh(()=>{c({level:"warn",event:"fault_retry_backstop.scan_failed",error:a})})});return n.lifecycle.registerDetached(r),r}async scan(t,n){const r=await(this.deps.list??g)(),a=this.deps.busy??(async e=>(await b(e))?.busy===!0),d=await y(r,async e=>{if(!await n.fleet.has(e.nodeId)||await a(e.nodeId))return!1;const s=p(e);return s===null||t<s+l?!1:e.nudgedAt===null||t-Date.parse(e.nudgedAt)>=l});if(d.length===0)return;let u;for(const e of d)e.fault.kind==="connection"&&(u??=await(this.deps.probeOnline??(()=>E()))(),!u)||!await n.fleet.has(e.nodeId)||await a(e.nodeId)||!await h(async m=>await A(e.fault.faultId,t,l)?(await m.prepare("SELECT pg_notify(?, ?)").get("crtr_signal",`fault_retry:${e.nodeId}`),!0):!1)||f.fresh(()=>{c({level:"info",event:"fault_retry_backstop.nudged",...R(e.nodeId)?{node_id:e.nodeId}:{},fields:{kind:e.fault.kind,since:e.fault.since,overdue_ms:t-(p(e)??t)}})})}}export{L as FAULT_RETRY_SCAN_INTERVAL_MS,l as FAULT_RETRY_STALE_GRACE_MS,U as FaultRetryBackstopReconciler,p as dueAtMs};

package/dist/daemon/reconcilers/grant-effects.d.ts

@@ -0,0 +1,8 @@
+export interface GrantEffectsRoles {
+    serves: boolean;
+    runsLanes: boolean;
+}
+/** Do this process's part of whatever changed grant `grantId`, from its row as it reads now. Idempotent. */
+export declare function applyGrantEffects(roles: GrantEffectsRoles, grantId: string): Promise<void>;
+/** Subscribe this process to `grant:<id>` signals. Returns the unsubscribe. */
+export declare function startGrantEffects(roles: GrantEffectsRoles): Promise<() => void>;

package/dist/daemon/reconcilers/grant-effects.js

@@ -0,0 +1 @@
+var w=Object.defineProperty;var a=(t,e)=>w(t,"name",{value:e,configurable:!0});import{openDb as d}from"../../core/canvas/db.js";import{endLiveExecution as _,liveExecutionForNode as v}from"../../core/canvas/executions.js";import{emitEvent as s}from"../../core/events/emit.js";import{afterRevocationStop as E}from"../../core/grants/on-sync.js";import{allGrants as u,grantIdsWhere as S,grantStanding as m,mirrorOnly as y}from"../../core/grants/mirror.js";import{removeApp as h}from"../../core/grants/remove.js";import{wakeAllGrantWaiters as g,wakeGrantWaiters as k}from"../../core/grants/watch.js";import{discardOrphanedTokens as p,discardTokensForGrant as x}from"../../core/providers/token-cache.js";import{runnerForExecution as N}from"../../core/runtime/runner/index.js";import{closeGrantStreams as O,heldGrantIds as T}from"../../core/runs/events.js";import{stopNodeSandboxes as b}from"../../core/spaces/stop.js";import{brokerSignalBus as L}from"../broker-signals/bus.js";import{withRowClaim as R}from"./row-claim.js";const D=6e4;function c(t,e,r){s({level:"error",event:`daemon.grants.${t}`,error:e,fields:r})}a(c,"failed");function G(t,e,r){try{const o=e.sub===null||e.grantee===null?p(r):x({sub:e.sub,grantee:e.grantee,active:e.standing==="active"},r);o.tokens>0&&s({level:"info",event:"daemon.grants.outbound_tokens_discarded",fields:{grant:t,tokens:o.tokens}})}catch(o){c("outbound_tokens_discard_failed",o,{grant:t})}}a(G,"discardTokens");async function A(t,e){if(k(t),e.standing==="inactive"){const r=O(t);r>0&&s({level:"info",event:"daemon.grants.streams_closed",fields:{grant:t,grantee:e.grantee,streams:r}})}try{await(await import("../../core/providers/bundles.js")).syncProviderBundles()}catch(r){s({level:"error",event:"daemon.providers.bundle_sync_failed",error:r})}}a(A,"serveEffects");async function C(t,e){const r=await R("nodes",t,D,async o=>{const n=await v(t);if(n===null)return;await o.assertOwned(),s({level:"info",event:"daemon.grants.node_stop",node_id:t,fields:{grant:e,execution_id:n.executionId}});const i=a(async()=>{await o.assertOwned(),await _(t,{reason:"permission_revoked"}),await E(t)},"finish");if(n.runner==="process"){const l=await b(t,"permission_revoked",{holdRecovery:!0,after:i});if(!l.confirmed)throw l.error??new Error(`stop of ${t} was not confirmed`);return}const f=await N(n.runner).stop(n,{reason:"permission_revoked"});if(!f.confirmed)throw f.error??new Error(`stop of ${t} was not confirmed`);await i()})}a(C,"stopRevokedNode");async function P(t,e){if(!(e.grantee!==null&&y({grantee:e.grantee}))){if(e.standing==="inactive"){const r=await(await d()).prepare("SELECT DISTINCT n.node_id FROM nodes n JOIN executions e ON e.node_id = n.node_id AND e.state != 'exited' WHERE n.grant_id = ?").all(t);for(const{node_id:o}of r)try{await C(o,t)}catch(n){c("node_stop_failed",n,{grant:t,node_id:o})}}if(e.status==="removed"&&e.removal_done_at===null&&e.grantee!==null&&e.sub!==null)try{await h(e.grantee,e.sub),await(await d()).prepare("UPDATE grants SET removal_done_at = ? WHERE id = ? AND removal_done_at IS NULL").run(new Date().toISOString(),t)}catch(r){c("removal_failed",r,{grant:t,grantee:e.grantee})}}}a(P,"laneEffects");async function W(t,e){const r=await m(e);G(e,r,await u()),await Promise.all([t.serves?A(e,r):void 0,t.runsLanes&&r.standing!=="lagging"?P(e,r):void 0])}a(W,"applyGrantEffects");async function F(t){const e=new Set;if(t.serves)for(const o of T())e.add(o);if(t.runsLanes){const o=await(await d()).prepare("SELECT DISTINCT n.grant_id FROM nodes n JOIN executions e ON e.node_id = n.node_id AND e.state != 'exited' WHERE n.grant_id IS NOT NULL").all();for(const{grant_id:n}of o)e.add(n)}const r=[];for(const o of e)(await m(o)).status===null&&r.push(o);return r}a(F,"goneGrantIds");async function Z(t){const e=new Map,r=a(n=>{const i=(e.get(n)??Promise.resolve()).then(()=>W(t,n)).catch(f=>c("effects_failed",f,{grant:n})).finally(()=>{e.get(n)===i&&e.delete(n)});return e.set(n,i),i},"run"),o=a(async()=>{try{t.serves&&g(),p(await u());const n=new Set(await S(`status <> 'active' OR expires_at <= '${new Date().toISOString()}'`));for(const i of await F(t))n.add(i);await Promise.all([...n].map(r))}catch(n){c("catch_up_failed",n,{})}},"catchUp");return await L.subscribe(n=>{r(n)},{kinds:["grant"],onListenStart:a(()=>{o()},"onListenStart")})}a(Z,"startGrantEffects");export{W as applyGrantEffects,Z as startGrantEffects};

package/dist/daemon/reconcilers/grove-instance-sweep.d.ts

@@ -25,9 +25,7 @@ export interface GroveInstanceSweepContext {
 }
 export declare class GroveInstanceSweepReconciler {
     private readonly operations;
-    private lastSweepAt;
-    private inFlight;
-    private cursor;
+    /** Failed uproots wait here; a process that takes the lane next retries them at once, which is harmless (uproot is idempotent). */
     private readonly retryAfter;
     constructor(operations?: GroveInstanceSweepOperations);
     run(now: number, ctx: GroveInstanceSweepContext): void;

package/dist/daemon/reconcilers/grove-instance-sweep.js

@@ -1 +1 @@
-var A=Object.defineProperty;var c=(e,t)=>A(e,"name",{value:t,configurable:!0});import{spawn as S}from"node:child_process";import{statSync as I}from"node:fs";import{delimiter as T,join as j}from"node:path";import{getRow as G,listNodes as R}from"../../core/canvas/index.js";import{isSafeNodeId as w}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as b,recordedPidLiveness as x}from"../../core/canvas/pid.js";import{emitEvent as p}from"../../core/events/emit.js";import{operationIdContext as h}from"../../core/events/operation-id.js";import{isParked as C}from"../../core/runtime/lifecycle.js";const F=60*1e3,P=3,L=300*1e3,E=["done","dead","canceled"];function _(e){return E.includes(e.status)&&!C(e)}c(_,"isClosed");function M(){const e=process.env.PATH;if(e===void 0)return null;for(const t of e.split(T)){const i=j(t===""?process.cwd():t,"grove");try{const r=I(i);if(r.isFile()&&(r.mode&73)!==0)return i}catch{}}return null}c(M,"resolveGroveBinary");function O(e,t){return new Promise(i=>{let r;try{r=S(e,[...t],{stdio:["ignore","pipe","pipe"]})}catch(n){i({code:null,signal:null,stdout:"",stderr:"",error:n instanceof Error?n:new Error(String(n))});return}let a="",s="",l=!1;const u=c(n=>{l||(l=!0,i(n))},"settle");r.stdout.on("data",n=>{a+=n.toString()}),r.stderr.on("data",n=>{s+=n.toString()}),r.on("error",n=>u({code:null,signal:null,stdout:a,stderr:s,error:n})),r.on("close",(n,f)=>u({code:n,signal:f,stdout:a,stderr:s}))})}c(O,"executeGrove");const V={resolveBinary:M,listTerminalNodes:c(async()=>await R({status:[...E]}),"listTerminalNodes"),getNodeRow:G,execute:O,captureLiveness:b};function B(e){let t;try{t=JSON.parse(e)}catch{return null}if(typeof t!="object"||t===null||!Array.isArray(t.projects))return null;const i=[];for(const r of t.projects){if(typeof r!="object"||r===null)continue;const a=r.name,s=r.instances;if(!(typeof a!="string"||!Array.isArray(s)))for(const l of s){if(typeof l!="object"||l===null)continue;const u=l.name,n=l.slot,f=l.spec?.labels,m=typeof f=="object"&&f!==null?f.owner:void 0;typeof u=="string"&&typeof n=="number"&&typeof m=="string"&&i.push({project:a,name:u,slot:n,owner:m})}}return i}c(B,"instancesFromInventory");function k(e,t,i){return _(e)&&!t.isClaimed(e.node_id)&&e.frozen_at===null&&x(e.pi_pid,e.pi_pid_identity,i)==="dead"}c(k,"engineIsGone");async function D(e,t,i){if(t.isClaimed(e.node_id))return!1;const r=await i(e.node_id);return r!==null&&_(r)&&r.frozen_at===null&&r.pi_pid===e.pi_pid&&r.pi_pid_identity===e.pi_pid_identity}c(D,"stillEligible");function N(e){if(e.error!==void 0)return e.error;const t=e.stderr.trim()||e.stdout.trim()||`Grove exited with ${e.signal??e.code??"an unknown status"}`;return new Error(t)}c(N,"executionError");function g(e){return`${e.project}/${e.name}`}c(g,"instanceKey");class X{static{c(this,"GroveInstanceSweepReconciler")}operations;lastSweepAt=Number.NEGATIVE_INFINITY;inFlight=!1;cursor=0;retryAfter=new Map;constructor(t=V){this.operations=t}run(t,i){if(this.inFlight||t-this.lastSweepAt<F||!i.lifecycle.acceptsDetachedWork())return;this.lastSweepAt=t,this.inFlight=!0;const r=Promise.resolve().then(()=>this.sweep(t,i.fleet)).finally(()=>{this.inFlight=!1});i.lifecycle.registerDetached(r)}async sweep(t,i){const r=this.operations.resolveBinary();if(r===null)return;let a;try{a=(await this.operations.listTerminalNodes()).filter(_)}catch(o){h.fresh(()=>{p({level:"error",event:"grove.sweep.scan_failed",error:o})});return}if(a.length===0)return;const s=await this.operations.execute(r,["list","--json"]);if(s.error!==void 0||s.code!==0||s.signal!==null){h.fresh(()=>{p({level:"error",event:"grove.sweep.list_failed",error:N(s)})});return}const l=B(s.stdout);if(l===null){h.fresh(()=>{p({level:"error",event:"grove.sweep.list_failed",error:new Error("grove list --json returned invalid inventory JSON")})});return}const u=new Map(a.map(o=>[o.node_id,o])),n=l.filter(o=>u.get(o.owner)!==void 0&&(this.retryAfter.get(g(o))??Number.NEGATIVE_INFINITY)<=t).sort((o,d)=>o.project.localeCompare(d.project)||o.slot-d.slot||o.name.localeCompare(d.name));if(n.length===0)return;this.cursor>=n.length&&(this.cursor=0);const f=Math.min(P,n.length),m=Array.from({length:f},(o,d)=>n[(this.cursor+d)%n.length]);this.cursor=(this.cursor+f)%n.length;const v=await this.operations.captureLiveness();if(v!==null)for(const o of m){const d=u.get(o.owner);if(!k(d,i,v)||!await D(d,i,this.operations.getNodeRow))continue;const y=await this.operations.execute(r,["uproot",g(o),"--force"]);if(y.error===void 0&&y.code===0&&y.signal===null){this.retryAfter.delete(g(o)),h.fresh(()=>{p({level:"info",event:"grove.sweep.reconciled",outcome:"succeeded",...w(o.owner)?{node_id:o.owner}:{fields:{affected_node_id:o.owner}}})});continue}this.retryAfter.set(g(o),t+L),h.fresh(()=>{p({level:"error",event:"grove.sweep.failed",outcome:"failed",...w(o.owner)?{node_id:o.owner}:{fields:{affected_node_id:o.owner}},error:N(y)})})}}}export{P as GROVE_INSTANCE_SWEEP_BUDGET,F as GROVE_INSTANCE_SWEEP_INTERVAL_MS,L as GROVE_INSTANCE_SWEEP_RETRY_DELAY_MS,X as GroveInstanceSweepReconciler};
+var I=Object.defineProperty;var l=(e,t)=>I(e,"name",{value:t,configurable:!0});import{spawn as T}from"node:child_process";import{statSync as j}from"node:fs";import{delimiter as R,join as b}from"node:path";import{getRow as G,listNodes as C}from"../../core/canvas/index.js";import{isSafeNodeId as E}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as x,recordedPidLiveness as L}from"../../core/canvas/pid.js";import{emitEvent as m}from"../../core/events/emit.js";import{operationIdContext as y}from"../../core/events/operation-id.js";import{isParked as P}from"../../core/runtime/lifecycle.js";import{runThrottledLane as M}from"./throttled-lane.js";const O=60*1e3,V=3,B=300*1e3,W=300*1e3,N=["done","dead","canceled"];function g(e){return N.includes(e.status)&&!P(e)}l(g,"isClosed");function k(){const e=process.env.PATH;if(e===void 0)return null;for(const t of e.split(R)){const o=b(t===""?process.cwd():t,"grove");try{const r=j(o);if(r.isFile()&&(r.mode&73)!==0)return o}catch{}}return null}l(k,"resolveGroveBinary");function D(e,t){return new Promise(o=>{let r;try{r=T(e,[...t],{stdio:["ignore","pipe","pipe"]})}catch(i){o({code:null,signal:null,stdout:"",stderr:"",error:i instanceof Error?i:new Error(String(i))});return}let s="",d="",a=!1;const u=l(i=>{a||(a=!0,o(i))},"settle");r.stdout.on("data",i=>{s+=i.toString()}),r.stderr.on("data",i=>{d+=i.toString()}),r.on("error",i=>u({code:null,signal:null,stdout:s,stderr:d,error:i})),r.on("close",(i,c)=>u({code:i,signal:c,stdout:s,stderr:d}))})}l(D,"executeGrove");const F={resolveBinary:k,listTerminalNodes:l(async()=>await C({status:[...N]}),"listTerminalNodes"),getNodeRow:G,execute:D,captureLiveness:x};function Y(e){let t;try{t=JSON.parse(e)}catch{return null}if(typeof t!="object"||t===null||!Array.isArray(t.projects))return null;const o=[];for(const r of t.projects){if(typeof r!="object"||r===null)continue;const s=r.name,d=r.instances;if(!(typeof s!="string"||!Array.isArray(d)))for(const a of d){if(typeof a!="object"||a===null)continue;const u=a.name,i=a.slot,c=a.spec?.labels,p=typeof c=="object"&&c!==null?c.owner:void 0;typeof u=="string"&&typeof i=="number"&&typeof p=="string"&&o.push({project:s,name:u,slot:i,owner:p})}}return o}l(Y,"instancesFromInventory");async function $(e,t,o){return g(e)&&!await t.isClaimed(e.node_id)&&e.frozen_at===null&&L(e.pi_pid,e.pi_pid_identity,o)==="dead"}l($,"engineIsGone");async function z(e,t,o){if(await t.isClaimed(e.node_id))return!1;const r=await o(e.node_id);return r!==null&&g(r)&&r.frozen_at===null&&r.pi_pid===e.pi_pid&&r.pi_pid_identity===e.pi_pid_identity}l(z,"stillEligible");function S(e){if(e.error!==void 0)return e.error;const t=e.stderr.trim()||e.stdout.trim()||`Grove exited with ${e.signal??e.code??"an unknown status"}`;return new Error(t)}l(S,"executionError");function h(e){return`${e.project}/${e.name}`}l(h,"instanceKey");class ne{static{l(this,"GroveInstanceSweepReconciler")}operations;retryAfter=new Map;constructor(t=F){this.operations=t}run(t,o){if(!o.lifecycle.acceptsDetachedWork())return;const r=M("grove-instance-sweep","",t,O,B,s=>this.sweep(t,o.fleet,s)).then(()=>{}).catch(s=>{y.fresh(()=>m({level:"error",event:"grove.sweep.failed",error:s}))});o.lifecycle.registerDetached(r)}async sweep(t,o,r){const s=this.operations.resolveBinary();if(s===null)return;let d;try{d=(await this.operations.listTerminalNodes()).filter(g)}catch(n){y.fresh(()=>{m({level:"error",event:"grove.sweep.scan_failed",error:n})});return}if(d.length===0)return;const a=await this.operations.execute(s,["list","--json"]);if(a.error!==void 0||a.code!==0||a.signal!==null){y.fresh(()=>{m({level:"error",event:"grove.sweep.list_failed",error:S(a)})});return}const u=Y(a.stdout);if(u===null){y.fresh(()=>{m({level:"error",event:"grove.sweep.list_failed",error:new Error("grove list --json returned invalid inventory JSON")})});return}const i=new Map(d.map(n=>[n.node_id,n])),c=u.filter(n=>i.get(n.owner)!==void 0&&(this.retryAfter.get(h(n))??Number.NEGATIVE_INFINITY)<=t).sort((n,f)=>n.project.localeCompare(f.project)||n.slot-f.slot||n.name.localeCompare(f.name));if(c.length===0)return;let p=Number(r.state??0);(!Number.isInteger(p)||p>=c.length)&&(p=0);const v=Math.min(V,c.length),A=Array.from({length:v},(n,f)=>c[(p+f)%c.length]);if(!await r.save(String((p+v)%c.length)))return;const w=await this.operations.captureLiveness();if(w!==null)for(const n of A){const f=i.get(n.owner);if(!await $(f,o,w)||!await z(f,o,this.operations.getNodeRow))continue;if(!await r.holds())return;const _=await this.operations.execute(s,["uproot",h(n),"--force"]);if(_.error===void 0&&_.code===0&&_.signal===null){this.retryAfter.delete(h(n)),y.fresh(()=>{m({level:"info",event:"grove.sweep.reconciled",outcome:"succeeded",...E(n.owner)?{node_id:n.owner}:{fields:{affected_node_id:n.owner}}})});continue}this.retryAfter.set(h(n),t+W),y.fresh(()=>{m({level:"error",event:"grove.sweep.failed",outcome:"failed",...E(n.owner)?{node_id:n.owner}:{fields:{affected_node_id:n.owner}},error:S(_)})})}}}export{V as GROVE_INSTANCE_SWEEP_BUDGET,O as GROVE_INSTANCE_SWEEP_INTERVAL_MS,W as GROVE_INSTANCE_SWEEP_RETRY_DELAY_MS,ne as GroveInstanceSweepReconciler};

package/dist/daemon/reconcilers/ledger-r2.d.ts

@@ -0,0 +1,27 @@
+import { type BlobStore } from '../../core/blobs/index.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_R2_INTERVAL_MS: number;
+export declare const LEDGER_R2_GRACE_MS: number;
+/** (month, payer) pairs moved per pass. */
+export declare const LEDGER_R2_BUDGET = 25;
+/** The blob's key names its content (`hash` is the SHA-256 of the rows it holds), so two movers that read different versions of the
+ *  rows write different blobs and the committed `charge_chunks.blob_key` always matches the totals stored beside it. */
+export declare const chargeChunkKey: (month: string, payer: string, hash: string) => string;
+export interface LedgerR2Result {
+    moved: number;
+    deferred: number;
+}
+export declare class LedgerR2Reconciler {
+    private readonly blobs;
+    constructor(blobs?: () => Promise<BlobStore>);
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass. Exposed for tests; `afterBlob` runs between the blob write and the transaction (a test kills the process there). */
+    pass(now: number, lane?: LanePass, afterBlob?: () => Promise<void>): Promise<LedgerR2Result>;
+    private move;
+    /** The transaction half of a move: true when the `charge_chunks` row named `key` was inserted and the rows deleted. */
+    private commit;
+    private movable;
+}

package/dist/daemon/reconcilers/ledger-r2.js

@@ -0,0 +1,7 @@
+var g=Object.defineProperty;var u=(d,e)=>g(d,"name",{value:e,configurable:!0});import{createHash as _}from"node:crypto";import{zstdCompressSync as S}from"node:zlib";import{blobStore as A}from"../../core/blobs/index.js";import{openDb as E,withCanvasWrite as N}from"../../core/canvas/db.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{usageMonth as D}from"../../core/ledger/months.js";import{sandboxUnbilledIn as L}from"./ledger-sandbox.js";import{storageSettledThrough as w}from"./ledger-storage.js";import{runThrottledLane as y}from"./throttled-lane.js";const T=60*6e4,v=2880*6e4,C=25,M=10*6e4,I=u((d,e,t)=>`ledger/${d}/${e}.${t}.jsonl.zst`,"chargeChunkKey"),p="(billed_usd IS NULL OR billed_usd = 0 OR invoice_id IS NOT NULL)";class P{static{u(this,"LedgerR2Reconciler")}blobs;constructor(e=A){this.blobs=e}run(e,t){if(!t.lifecycle.acceptsDetachedWork())return;const a=R.fresh(()=>y("ledger-r2","",e,T,M,async r=>{await this.pass(e,r)})).then(()=>{}).catch(r=>{R.fresh(()=>f({level:"error",event:"ledger.r2.failed",error:r}))});t.lifecycle.registerDetached(a)}async pass(e,t,a){const r={moved:0,deferred:0};if(O("user").ledger?.r2_move!==!0)return r;const i=await w();if(i===null)return r;const o=D(new Date(e-v)),s=await(await E()).prepare(`SELECT c.month, c.payer FROM charges c
+      WHERE c.month < ? AND (?::text IS NULL OR c.month <= ?) AND NOT EXISTS (SELECT 1 FROM charge_chunks k WHERE k.month = c.month AND k.payer = c.payer)
+      GROUP BY c.month, c.payer HAVING bool_and(${p.replaceAll("billed_usd","c.billed_usd").replaceAll("invoice_id","c.invoice_id")})
+      ORDER BY c.month, c.payer LIMIT ?`).all(o,i??null,i??null,C),l=await this.blobs();for(const{month:n,payer:h}of s){if(t&&!await t.holds())break;if(await L(n,e)){r.deferred+=1;continue}await this.move(l,n,h,a)?r.moved+=1:r.deferred+=1}return r.moved>0&&f({level:"info",event:"ledger.r2.moved",outcome:"succeeded",fields:{...r}}),r}async move(e,t,a,r){const o=await(await E()).prepare("SELECT id, to_jsonb(charges)::text AS body FROM charges WHERE month = ? AND payer = ? ORDER BY id").all(t,a);if(o.length===0||!await this.movable(t,a))return!1;const c=o.map(n=>n.body).join(`
+`)+`
+`,s=I(t,a,_("sha256").update(c).digest("hex").slice(0,16));await e.put(s,S(Buffer.from(c)));const l=await this.commit(t,a,s,o,r);return!l&&await(await E()).prepare("SELECT 1 FROM charge_chunks WHERE blob_key = ?").get(s)===void 0&&await e.delete(s),l}async commit(e,t,a,r,i){return await i?.(),await N(async o=>{const c=await o.prepare("SELECT id FROM charges WHERE month = ? AND payer = ? ORDER BY id FOR UPDATE").all(e,t),s=await o.prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND ${p}`).get(e,t);if(c.length!==r.length||Number(s.n)!==r.length||c.some((m,b)=>m.id!==r[b].id)||(await o.prepare("SELECT string_agg(to_jsonb(c)::text, E'\\n' ORDER BY c.id) AS body FROM charges c WHERE month = ? AND payer = ?").get(e,t)).body!==r.map(m=>m.body).join(`
+`))return!1;const n=await o.prepare("SELECT sum(usd) AS usd, sum(billed_usd) AS billed, min(at) AS first_at, max(at) AS last_at FROM charges WHERE month = ? AND payer = ?").get(e,t);return(await o.prepare('INSERT INTO charge_chunks (month, payer, blob_key, "rows", usd, billed_usd, first_at, last_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (month, payer) DO NOTHING').run(e,t,a,r.length,n.usd,n.billed,n.first_at,n.last_at)).changes===0?!1:(await o.prepare("DELETE FROM charges WHERE month = ? AND payer = ?").run(e,t),!0)})}async movable(e,t){const a=await(await E()).prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND NOT ${p}`).get(e,t);return Number(a.n)===0}}export{C as LEDGER_R2_BUDGET,v as LEDGER_R2_GRACE_MS,T as LEDGER_R2_INTERVAL_MS,P as LedgerR2Reconciler,I as chargeChunkKey};

package/dist/daemon/reconcilers/ledger-sandbox.d.ts

@@ -0,0 +1,55 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_SANDBOX_INTERVAL_MS = 60000;
+export declare const DEFAULT_STANDBY_S = 30;
+/** Executions billed per pass; the rest wait for the next pass. */
+export declare const LEDGER_SANDBOX_BUDGET = 200;
+interface Rates {
+    usdPerGbHour: number | null;
+    standbyMs: number;
+}
+export declare function sandboxRates(): Rates;
+/** The `[from, to]` chunks to bill: every whole hour, plus the remainder when the period is closed. The hours stay aligned to `from`
+ *  whatever the month; an hour or remainder that crosses a UTC month boundary is cut there into one chunk per month. */
+export declare function chunksOf(from: number, end: number, closed: boolean): Array<[number, number]>;
+/** When a chunk's seconds accrued, for the row's `at` (and so its month): its last millisecond, so a chunk ending at exactly 00:00Z
+ *  of a month belongs to the month before it. */
+export declare const chunkAccruedAt: (chunkEnd: number) => Date;
+/** The row id of the billed chunk of an execution that starts at `chunkStart`: the start time in whole seconds makes it
+ *  the same in every process and unique across the awake periods of one execution. */
+export declare const sandboxChargeId: (executionId: string, chunkStart: number) => string;
+/** An awake period that a wake ended: whose it was, where it began, when the broker went idle and how far it had been billed. */
+interface EndedPeriod {
+    execution_id: string;
+    awake_since: string;
+    idle_at: string;
+    charged_through: string | null;
+}
+export interface LedgerSandboxResult {
+    executions: number;
+    rows: number;
+    failed: number;
+}
+export declare class LedgerSandboxReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass over the executions with awake time not yet billed. Exposed for tests. */
+    pass(now: number, lane?: LanePass): Promise<LedgerSandboxResult>;
+    /** Bills one execution; returns the rows it wrote. Payer and memory are read first; the rows and `charged_through` go in one
+     *  transaction that holds the execution row's lock, from which the period is recomputed. `settle` is the wake of a resident
+     *  broker: an idle period is closed at `min(now, idle_at + standby tail)` and billed whole, because the wake resets `awake_since`;
+     *  a period that is not idle is left to the lane. `ended` is a period a wake ended before its settle succeeded: it is billed from
+     *  the period saved with it, whatever the execution row says now, and its row is deleted in the same transaction. */
+    bill(executionId: string, now: number, usdPerGbHour: number, standbyMs: number, settle?: boolean, ended?: EndedPeriod): Promise<number>;
+}
+/** Whether any awake time that accrued in `month` (UTC) is not billed yet: an execution's current period, or an ended period a failed
+ *  settle saved, whose unbilled span `[max(awake_since, charged_through), end]` overlaps the month. `ledger-r2` moves no month while
+ *  this is true, because the month's allowance is summed from the live `charges` rows and a row written after the move would get it
+ *  again. With no sandbox rate the lane writes nothing, so nothing is unbilled. */
+export declare function sandboxUnbilledIn(month: string, now: number): Promise<boolean>;
+/** Before a wake resets `awake_since` (`recordExecutionAwake`): bills the idle period that is ending, which the lane would
+ *  otherwise lose. Never throws: a settle that fails must not block the wake, and the failure is logged. Settling the same
+ *  period twice charges once (`charged_through` and the deterministic row ids). */
+export declare function settleSandboxBeforeWake(executionId: string, now?: number): Promise<void>;
+export {};

package/dist/daemon/reconcilers/ledger-sandbox.js

@@ -0,0 +1,9 @@
+var W=Object.defineProperty;var d=(r,e)=>W(r,"name",{value:e,configurable:!0});import{openDb as x,withCanvasWrite as F}from"../../core/canvas/db.js";import{getNode as A}from"../../core/canvas/canvas.js";import{readConfig as z}from"../../core/config.js";import{emitEvent as E}from"../../core/events/emit.js";import{operationIdContext as T}from"../../core/events/operation-id.js";import{billingAccountOf as $}from"../../core/ledger/account.js";import{writeInfraCharge as B}from"../../core/ledger/infra-allowance.js";import{resetAt as G}from"../../core/ledger/months.js";import{startCandidateOf as P}from"../../core/runtime/start-check.js";import{runThrottledLane as J}from"./throttled-lane.js";const Y=6e4,V=5*6e4,b=36e5,X=30,y=200;function N(){const r=z("user").ledger;return{usdPerGbHour:r?.sandbox_usd_per_gb_hour??null,standbyMs:(r?.sandbox_standby_s??X)*1e3}}d(N,"sandboxRates");function j(r,e,n){const t=r.exited_at??r.idle_at;if(t===null)return{end:e,closed:!1};const a=Date.parse(t)+n;return a<=e?{end:a,closed:!0}:{end:e,closed:!1}}d(j,"periodEnd");const K=d(r=>{const e=new Date(r);return Date.UTC(e.getUTCFullYear(),e.getUTCMonth()+1,1)},"monthEndAfter");function R(r,e){const n=[];for(let t=r;t<e;){const a=Math.min(e,K(t));n.push([t,a]),t=a}return n}d(R,"withinMonths");function k(r,e,n){const t=[];let a=r;for(;e-a>=b;)t.push(...R(a,a+b)),a+=b;return n&&e>a&&t.push(...R(a,e)),t}d(k,"chunksOf");const Z=d(r=>new Date(r-1),"chunkAccruedAt"),q=d((r,e)=>`sandbox:${r}:${Math.floor(e/1e3)}`,"sandboxChargeId"),S="ledger-sandbox-unsettled",M=d((r,e)=>`${r}@${Date.parse(e)}`,"unsettledKey");class Q{static{d(this,"LedgerSandboxReconciler")}run(e,n){if(!n.lifecycle.acceptsDetachedWork())return;const t=T.fresh(()=>J("ledger-sandbox","",e,Y,V,async a=>{await this.pass(e,a)})).then(()=>{}).catch(a=>{T.fresh(()=>E({level:"error",event:"ledger.sandbox.failed",error:a}))});n.lifecycle.registerDetached(t)}async pass(e,n){const t={executions:0,rows:0,failed:0},{usdPerGbHour:a,standbyMs:l}=N();if(a===null)return t;const i=await x(),_=await i.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+      WHERE runner = 'blaxel' AND awake_since IS NOT NULL
+        AND (state <> 'exited' OR charged_through IS NULL OR charged_through::timestamptz < exited_at::timestamptz + make_interval(secs => ?::float8 / 1000))
+      ORDER BY execution_id LIMIT ?`).all(l,y);for(const s of _){if(n&&!await n.holds())break;try{const o=await this.bill(s.execution_id,e,a,l);o>0&&(t.executions+=1),t.rows+=o}catch(o){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:o,fields:{execution_id:s.execution_id}})}}const f=await i.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? ORDER BY key LIMIT ?").all(S,y);for(const{cursor:s}of f){const o=JSON.parse(s);if(n&&!await n.holds())break;try{const c=await this.bill(o.execution_id,e,a,l,!1,o);c>0&&(t.executions+=1),t.rows+=c}catch(c){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:c,fields:{execution_id:o.execution_id}})}}return(t.rows>0||t.failed>0)&&E({level:"info",event:"ledger.sandbox.settled",outcome:"succeeded",fields:{...t}}),t}async bill(e,n,t,a,l=!1,i){const _=await x(),f=await _.prepare("SELECT node_id FROM executions WHERE execution_id = ?").get(e);if(f===void 0)return 0;const s=await A(f.node_id);if(s===null)throw new Error(`execution ${e} names node ${f.node_id}, which has no row`);const o=await P(s);if(o===null)throw new Error(`node ${s.node_id} has no person yet, so its sandbox hours have no payer`);const c=await _.prepare("SELECT e.memory_mb AS mb FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ?").get(s.profile_id??null);if(c===void 0)throw new Error(`node ${s.node_id} has no environment, so its sandbox size is unknown`);const p=Number(c.mb),{account:C,billable:v}=await $(_,o.payer),H=(s.run_id?await A(s.run_id):null)?.caused_by??s.caused_by??null;return await F(async D=>{const u=await D.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+        WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL FOR UPDATE`).get(e);if(i===void 0&&u===void 0||i===void 0&&l&&u.idle_at===null)return 0;let g,h;if(i!==void 0)g=Math.max(Date.parse(i.awake_since),i.charged_through===null?0:Date.parse(i.charged_through)),h=k(g,Date.parse(i.idle_at)+a,!0);else{const{end:m,closed:w}=l?{end:Math.min(n,Date.parse(u.idle_at)+a),closed:!0}:j(u,n,a);g=Math.max(Date.parse(u.awake_since),u.charged_through===null?0:Date.parse(u.charged_through)),h=k(g,m,w)}let L=0;for(const[m,w]of h){const I=Math.round((w-m)/1e3),O=p/1024*((w-m)/b),U=O*t;(await B({id:q(e,m),at:Z(w),kind:"sandbox",what:`${p} MB`,units:{gb_hours:O,seconds:I,memory_mb:p},usd:U,payer:o.payer,funding:"bill",sub:o.sub,grantee:o.grantee,node:s.node_id,run:s.run_id??null,provider:"blaxel",causedBy:H,account:C},v)).written&&(L+=1)}return i!==void 0?await D.prepare("DELETE FROM reconciler_leases WHERE lane = ? AND key = ?").run(S,M(e,i.awake_since)):h.length>0&&await D.prepare("UPDATE executions SET charged_through = ? WHERE execution_id = ?").run(new Date(h[h.length-1][1]).toISOString(),e),L})}}async function pe(r,e){const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return!1;const a=await x(),l=new Date(`${r}-01T00:00:00Z`).toISOString(),i=G(r);return await a.prepare(`SELECT 1 AS found FROM executions e,
+      LATERAL (SELECT greatest(e.awake_since::timestamptz, COALESCE(e.charged_through::timestamptz, '-infinity'::timestamptz)) AS from_at,
+                      CASE WHEN COALESCE(e.exited_at, e.idle_at) IS NULL THEN ?::timestamptz
+                           ELSE least(COALESCE(e.exited_at, e.idle_at)::timestamptz + make_interval(secs => ?::float8 / 1000), ?::timestamptz) END AS end_at) p
+    WHERE e.runner = 'blaxel' AND e.awake_since IS NOT NULL AND p.from_at < p.end_at AND p.from_at < ?::timestamptz AND p.end_at > ?::timestamptz LIMIT 1`).get(new Date(e).toISOString(),t,new Date(e).toISOString(),i,l)!==void 0?!0:(await a.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ?").all(S)).some(({cursor:s})=>{const o=JSON.parse(s),c=Math.max(Date.parse(o.awake_since),o.charged_through===null?0:Date.parse(o.charged_through)),p=Date.parse(o.idle_at)+t;return c<p&&c<Date.parse(i)&&p>Date.parse(l)})}d(pe,"sandboxUnbilledIn");async function he(r,e=Date.now()){try{const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return;await new Q().bill(r,e,n,t,!0)}catch(n){E({level:"warn",event:"ledger.sandbox.settle_failed",error:n,fields:{execution_id:r}}),await ee(r)}}d(he,"settleSandboxBeforeWake");async function ee(r){try{const e=await x(),n=await e.prepare("SELECT execution_id, awake_since, idle_at, charged_through FROM executions WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL AND idle_at IS NOT NULL").get(r);if(n===void 0)return;await e.prepare("INSERT INTO reconciler_leases (lane, key, owner, expires_at, cursor) VALUES (?, ?, ?, ?, ?) ON CONFLICT (lane, key) DO NOTHING").run(S,M(r,n.awake_since),"settle",new Date(0).toISOString(),JSON.stringify(n))}catch(e){E({level:"error",event:"ledger.sandbox.unsettled_lost",error:e,fields:{execution_id:r}})}}d(ee,"keepUnsettled");export{X as DEFAULT_STANDBY_S,y as LEDGER_SANDBOX_BUDGET,Y as LEDGER_SANDBOX_INTERVAL_MS,Q as LedgerSandboxReconciler,Z as chunkAccruedAt,k as chunksOf,q as sandboxChargeId,N as sandboxRates,pe as sandboxUnbilledIn,he as settleSandboxBeforeWake};

package/dist/daemon/reconcilers/ledger-storage.d.ts

@@ -0,0 +1,22 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_STORAGE_INTERVAL_MS: number;
+/** The last month the storage lane settled (its saved cursor state), null when it has settled none, and `undefined` when no storage
+ *  rate is configured, so the lane writes nothing and there is nothing to wait for. `ledger-r2` moves no month past it. */
+export declare function storageSettledThrough(): Promise<string | null | undefined>;
+export interface LedgerStorageResult {
+    month: string | null;
+    rows: number;
+    failed: number;
+}
+export declare class LedgerStorageReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** Settles every closed month the lane has not: the months after the one its cursor saved, through the month before `now`'s
+     *  (with no saved month, only that one). A month's save follows its own success, so a failure leaves it and the later months
+     *  for the next pass. Exposed for tests. */
+    pass(now: number, lane?: LanePass): Promise<LedgerStorageResult>;
+    /** Writes the month's rows; false when a payer failed or the lane was lost, so the month is not saved as settled. */
+    private settleMonth;
+}

package/dist/daemon/reconcilers/ledger-storage.js

@@ -0,0 +1,5 @@
+var M=Object.defineProperty;var p=(s,e)=>M(s,"name",{value:e,configurable:!0});import{openDb as N}from"../../core/canvas/db.js";import{getNode as D}from"../../core/canvas/canvas.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as y}from"../../core/events/emit.js";import{operationIdContext as S}from"../../core/events/operation-id.js";import{billingAccountOf as T}from"../../core/ledger/account.js";import{writeInfraCharge as v}from"../../core/ledger/infra-allowance.js";import{usageMonth as w}from"../../core/ledger/months.js";import{startCandidateOf as I}from"../../core/runtime/start-check.js";import{runThrottledLane as L}from"./throttled-lane.js";const A=60*6e4,C=10*6e4,G=1024**3;async function Q(){const s=O("user").ledger;if((s?.storage_usd_per_gb_month??null)===null&&(s?.sandbox_disk_usd_per_gb_month??null)===null)return;const e=await(await N()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = 'ledger-storage' AND key = ''").get(),r=e?.cursor==null?null:JSON.parse(e.cursor).state;return r===null?null:r.slice(0,7)}p(Q,"storageSettledThrough");function U(s){const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r-2,1)))}p(U,"previousMonth");const x=p(s=>{const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r,1)))},"nextMonth"),F=p(s=>{const[e,r]=s.split("-").map(Number);return new Date(Date.UTC(e,r,1)).toISOString()},"endOf");class X{static{p(this,"LedgerStorageReconciler")}run(e,r){if(!r.lifecycle.acceptsDetachedWork())return;const u=S.fresh(()=>L("ledger-storage","",e,A,C,async o=>{await this.pass(e,o)})).then(()=>{}).catch(o=>{S.fresh(()=>y({level:"error",event:"ledger.storage.failed",error:o}))});r.lifecycle.registerDetached(u)}async pass(e,r){const u=U(w(new Date(e))),o={month:null,rows:0,failed:0},_=O("user").ledger,f={r2:_?.storage_usd_per_gb_month??null,disk:_?.sandbox_disk_usd_per_gb_month??null};if(f.r2===null&&f.disk===null)return o;const i=r?.state?.slice(0,7)??null,l=[];if(i===null)l.push(u);else for(let n=x(i);n<=u;n=x(n))l.push(n);for(const n of l){if(o.month=n,!await this.settleMonth(n,f,r,o))break;await r?.save(n)}return(o.rows>0||o.failed>0)&&y({level:"info",event:"ledger.storage.settled",outcome:"succeeded",fields:{...o}}),o}async settleMonth(e,r,u,o){const{r2:_,disk:f}=r,i=F(e),l=await N(),n=new Map,g=p(t=>{let a=n.get(t);return a||n.set(t,a={bytes:0,diskGb:0}),a},"entry");for(const t of await l.prepare("SELECT split_part(blob_key, '/', 2) AS node, sum(bytes) AS bytes FROM conversation_parts WHERE created < ? GROUP BY 1").all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node, sum(u.size) AS bytes FROM (
+        SELECT DISTINCT upload_id, split_part(blob_key, '/', 2) AS node FROM upload_uses WHERE blob_key LIKE 'attachments/%') x
+      JOIN uploads u ON u.upload_id = x.upload_id WHERE u.created_at < ? GROUP BY x.node`).all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node_id AS node, sum(e.memory_mb) AS mb FROM (
+        SELECT DISTINCT node_id, sandbox_name FROM executions WHERE runner = 'blaxel' AND sandbox_name IS NOT NULL AND started_at < ?) x
+      JOIN nodes n ON n.node_id = x.node_id JOIN objects o ON o.id = n.node_id AND (o.deleted_at IS NULL OR o.deleted_at >= ?) JOIN profiles p ON p.id = n.profile_id JOIN environments e ON e.object = p.environment_id GROUP BY x.node_id`).all(i,i))g(t.node).diskGb+=Number(t.mb)/2/1024;const h=new Map;for(const[t,a]of n){const b=await D(t),c=b===null?null:await I(b);if(c===null)continue;const m=h.get(c.payer)??{sub:c.sub,grantee:c.grantee,bytes:0,diskGb:0,node:t};m.bytes+=a.bytes,m.diskGb+=a.diskGb,h.set(c.payer,m)}const R=new Date(Date.parse(i)-1);let E=0;for(const[t,a]of h){if(u&&!await u.holds())return!1;try{const{account:b,billable:c}=await T(l,t),m=[{id:`storage:${t}:${e}:r2`,provider:"r2",gbMonths:a.bytes/G,rate:_},{id:`storage:${t}:${e}:blaxel`,provider:"blaxel",gbMonths:a.diskGb,rate:f}];for(const d of m){if(d.rate===null||d.gbMonths<=0)continue;const k=d.gbMonths*d.rate;(await v({id:d.id,at:R,kind:"storage",what:d.provider==="r2"?"object storage":"sandbox disk",units:{gb_months:d.gbMonths},usd:k,payer:t,funding:"bill",sub:a.sub,grantee:a.grantee,provider:d.provider,causedBy:null,account:b},c)).written&&(o.rows+=1)}}catch(b){o.failed+=1,E+=1,y({level:"warn",event:"ledger.storage.payer_failed",error:b,fields:{payer:t,month:e}})}}return E===0}}export{A as LEDGER_STORAGE_INTERVAL_MS,X as LedgerStorageReconciler,Q as storageSettledThrough};

package/dist/daemon/reconcilers/live-obligation.d.ts

@@ -1,4 +1,5 @@
 import { type Lifecycle } from '../../core/canvas/index.js';
+import { type ViewerProbe } from '../../core/runtime/placement.js';
 import { type LifecycleEvent } from '../../core/runtime/lifecycle.js';
 import type { FleetRegistry } from '../../core/runtime/fleet.js';
 /** Durable work that requires a node to remain automatically wakeable. */
@@ -23,7 +24,7 @@ export declare function retentionHold(fleet: Pick<FleetRegistry, 'isClaimed'>):
  *  `attends: false` is a system observer and deliberately does not count, which
  *  is how Northlight identifies itself — so an open product page is
  *  invisible here, while crouter's own local and remote viewers are not. */
-export declare function hasHumanViewer(nodeId: string): Promise<boolean>;
+export declare function hasHumanViewer(nodeId: string, viewer?: ViewerProbe): Promise<boolean>;
 /** What parking an unattended node actually amounts to — the one rule both the
  *  live-broker park clock and the lifecycle tick apply.
  *
@@ -48,7 +49,7 @@ export declare function hasHumanViewer(nodeId: string): Promise<boolean>;
  *  invariant is that content a sender already handed over is never dropped.
  */
 export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
-export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): Promise<ParkEvent>;
+export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<ParkEvent>;
 /** Apply an unattended decision and notify subscribers of automatic completion.
  *  This is the DEGRADED enactment: no summary turn ran, so subscribers get the
  *  doctrine notice instead of the node's own report. The live clock's good path

package/dist/daemon/reconcilers/live-obligation.js

@@ -1 +1 @@
-var f=Object.defineProperty;var a=(t,r)=>f(t,"name",{value:r,configurable:!0});import{hasActiveLiveSubscription as l,hasLiveSubscriber as p,hasPendingCronWake as m}from"../../core/canvas/index.js";import{fullName as w}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as i}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/structured-output.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await l(t)||await h(t)||await k(t)||await m(t)||await b(t)>0||g(t).state!=="absent"}a(s,"hasLiveObligation");async function B(t){return await s(t)||await i(t)?!0:await n(t)}a(B,"hasPendingWake");async function F(t){return async r=>t.isClaimed(r)||await s(r)||await n(r)}a(F,"retentionHold");async function n(t){return await v(t)!==null||x(t)}a(n,"hasHumanViewer");async function M(t,r){if(await n(t)||await s(t))return"release";const e=await W(t);return e?.link==="pi\u2192provider"&&e.retry.disposition==="fatal"?"release":r==="resident"?await i(t)?"release":"park":await i(t)||await p(t)?"release":"cancel"}a(M,"parkEvent");async function T(t,r){const e=r==="cancel"?await c(t,r,{reason:"stranded"}):await c(t,r);if(r!=="park")return;const o=await y(t),u=o===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${o}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${w(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}a(T,"applyParkEvent");export{T as applyParkEvent,n as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};
+var l=Object.defineProperty;var r=(t,a)=>l(t,"name",{value:a,configurable:!0});import{hasActiveLiveSubscription as f,hasLiveSubscriber as p,hasPendingCronWake as w}from"../../core/canvas/index.js";import{fullName as m}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as n}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/output-result.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await f(t)||await h(t)||await k(t)||await w(t)||await b(t)>0||await g(t)!==null}r(s,"hasLiveObligation");async function B(t){return await s(t)||await n(t)?!0:await o(t)}r(B,"hasPendingWake");async function F(t){return async a=>await t.isClaimed(a)||await s(a)||await o(a)}r(F,"retentionHold");async function o(t,a){return(a!==void 0?a.live!==null:await v(t)!==null)||await x(t)}r(o,"hasHumanViewer");async function M(t,a,e){if(await o(t,e)||await s(t))return"release";const i=await W(t);return i?.link==="pi\u2192provider"&&i.retry.disposition==="fatal"?"release":a==="resident"?await n(t)?"release":"park":await n(t)||await p(t)?"release":"cancel"}r(M,"parkEvent");async function T(t,a){const e=a==="cancel"?await c(t,a,{reason:"stranded"}):await c(t,a);if(a!=="park")return;const i=await y(t),u=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${m(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}r(T,"applyParkEvent");export{T as applyParkEvent,o as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};

package/dist/daemon/reconcilers/managed-worktree-sweep.d.ts

@@ -8,11 +8,6 @@ export interface ManagedWorktreeSweepContext {
     fleet: FleetRegistry;
 }
 export declare class ManagedWorktreeSweepReconciler {
-    private lastSweepAt;
-    private inFlight;
-    /** Round-robin cursor over candidate node ids, so a permanently stuck record
-     *  at the head of the list cannot starve the ones behind it. */
-    private cursor;
     run(now: number, ctx: ManagedWorktreeSweepContext): void;
     /** Terminal nodes whose managed worktree is unreconciled and whose backoff has
      *  elapsed. Canvas reads only — engine liveness is classified later, for the