@north-light/cli 0.3.560 → 0.3.562

package/dist/daemon/reconcilers/fault-retry-backstop.js

@@ -1 +1 @@
-var A=Object.defineProperty;var o=(a,t)=>A(a,"name",{value:t,configurable:!0});import{filterAsync as g}from"../../core/async-array.js";import{listPendingProviderRetries as y,readLiveTurn as _}from"../../core/canvas/recovery.js";import{probeOnline as I}from"../../core/runtime/connectivity.js";import{nextFaultRetry as b}from"../../core/runtime/fault-recovery.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as p}from"../../core/events/operation-id.js";import{isSafeNodeId as R}from"../../core/canvas/paths.js";import{brokerSignalStreams as v}from"../api/handlers/broker-signals.js";const h=2*6e4,S=15e3;function m(a){const{fault:t}=a;if(t.retry.nextAt!==void 0){const i=Date.parse(t.retry.nextAt);return Number.isFinite(i)?i:null}const n=Date.parse(a.updatedAt),s=b(t,n);return Number.isFinite(n)&&s!==null?n+s.delayMs:null}o(m,"dueAtMs");class D{static{o(this,"FaultRetryBackstopReconciler")}deps;lastScanAt=Number.NEGATIVE_INFINITY;scanning=null;nudgedAt=new Map;constructor(t={}){this.deps=t}run(t,n){if(this.scanning!==null)return this.scanning;if(!n.lifecycle.acceptsDetachedWork()||t-this.lastScanAt<S)return Promise.resolve();this.lastScanAt=t;const s=this.scan(t,n).catch(i=>{p.fresh(()=>{f({level:"warn",event:"fault_retry_backstop.scan_failed",error:i})})}).finally(()=>{this.scanning=null});return this.scanning=s,n.lifecycle.registerDetached(s),s}async scan(t,n){const s=await(this.deps.list??y)(),i=new Set(s.map(e=>e.fault.faultId));for(const e of this.nudgedAt.keys())i.has(e)||this.nudgedAt.delete(e);const d=this.deps.busy??(async e=>(await _(e))?.busy===!0),u=await g(s,async e=>{if(!n.fleet.has(e.nodeId)||await d(e.nodeId))return!1;const r=m(e);if(r===null||t<r+h)return!1;const l=this.nudgedAt.get(e.fault.faultId);return l===void 0||t-l>=h});if(u.length===0)return;let c;for(const e of u){if(e.fault.kind==="connection"&&(c??=await(this.deps.probeOnline??(()=>I()))(),!c)||!n.fleet.has(e.nodeId)||await d(e.nodeId))continue;this.nudgedAt.set(e.fault.faultId,t);const r=await(this.deps.nudge??(async l=>await v.nudgeFaultRetry(l)))(e.nodeId);p.fresh(()=>{f({level:"info",event:"fault_retry_backstop.nudged",...R(e.nodeId)?{node_id:e.nodeId}:{},fields:{kind:e.fault.kind,since:e.fault.since,delivered:r,overdue_ms:t-(m(e)??t)}})})}}}export{S as FAULT_RETRY_SCAN_INTERVAL_MS,h as FAULT_RETRY_STALE_GRACE_MS,D as FaultRetryBackstopReconciler,m as dueAtMs};
+var _=Object.defineProperty;var o=(i,t)=>_(i,"name",{value:t,configurable:!0});import{filterAsync as y}from"../../core/async-array.js";import{withCanvasWrite as h}from"../../core/canvas/db.js";import{claimFaultNudge as A,listPendingProviderRetries as g,readLiveTurn as b}from"../../core/canvas/recovery.js";import{probeOnline as E}from"../../core/runtime/connectivity.js";import{nextFaultRetry as I}from"../../core/runtime/fault-recovery.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as f}from"../../core/events/operation-id.js";import{isSafeNodeId as R}from"../../core/canvas/paths.js";import{runThrottledLane as v}from"./throttled-lane.js";const l=2*6e4,L=15e3;function p(i){const{fault:t}=i;if(t.retry.nextAt!==void 0){const a=Date.parse(t.retry.nextAt);return Number.isFinite(a)?a:null}const n=Date.parse(i.updatedAt),r=I(t,n);return Number.isFinite(n)&&r!==null?n+r.delayMs:null}o(p,"dueAtMs");const S=6e4,T="fault-retry-backstop";class U{static{o(this,"FaultRetryBackstopReconciler")}deps;constructor(t={}){this.deps=t}run(t,n){if(!n.lifecycle.acceptsDetachedWork())return Promise.resolve();const r=v(T,"",t,L,S,()=>this.scan(t,n)).then(()=>{}).catch(a=>{f.fresh(()=>{c({level:"warn",event:"fault_retry_backstop.scan_failed",error:a})})});return n.lifecycle.registerDetached(r),r}async scan(t,n){const r=await(this.deps.list??g)(),a=this.deps.busy??(async e=>(await b(e))?.busy===!0),d=await y(r,async e=>{if(!await n.fleet.has(e.nodeId)||await a(e.nodeId))return!1;const s=p(e);return s===null||t<s+l?!1:e.nudgedAt===null||t-Date.parse(e.nudgedAt)>=l});if(d.length===0)return;let u;for(const e of d)e.fault.kind==="connection"&&(u??=await(this.deps.probeOnline??(()=>E()))(),!u)||!await n.fleet.has(e.nodeId)||await a(e.nodeId)||!await h(async m=>await A(e.fault.faultId,t,l)?(await m.prepare("SELECT pg_notify(?, ?)").get("crtr_signal",`fault_retry:${e.nodeId}`),!0):!1)||f.fresh(()=>{c({level:"info",event:"fault_retry_backstop.nudged",...R(e.nodeId)?{node_id:e.nodeId}:{},fields:{kind:e.fault.kind,since:e.fault.since,overdue_ms:t-(p(e)??t)}})})}}export{L as FAULT_RETRY_SCAN_INTERVAL_MS,l as FAULT_RETRY_STALE_GRACE_MS,U as FaultRetryBackstopReconciler,p as dueAtMs};

package/dist/daemon/reconcilers/grant-effects.d.ts

@@ -0,0 +1,8 @@
+export interface GrantEffectsRoles {
+    serves: boolean;
+    runsLanes: boolean;
+}
+/** Do this process's part of whatever changed grant `grantId`, from its row as it reads now. Idempotent. */
+export declare function applyGrantEffects(roles: GrantEffectsRoles, grantId: string): Promise<void>;
+/** Subscribe this process to `grant:<id>` signals. Returns the unsubscribe. */
+export declare function startGrantEffects(roles: GrantEffectsRoles): Promise<() => void>;

package/dist/daemon/reconcilers/grant-effects.js

@@ -0,0 +1 @@
+var w=Object.defineProperty;var a=(t,e)=>w(t,"name",{value:e,configurable:!0});import{openDb as d}from"../../core/canvas/db.js";import{endLiveExecution as _,liveExecutionForNode as v}from"../../core/canvas/executions.js";import{emitEvent as s}from"../../core/events/emit.js";import{afterRevocationStop as E}from"../../core/grants/on-sync.js";import{allGrants as u,grantIdsWhere as S,grantStanding as m,mirrorOnly as y}from"../../core/grants/mirror.js";import{removeApp as h}from"../../core/grants/remove.js";import{wakeAllGrantWaiters as g,wakeGrantWaiters as k}from"../../core/grants/watch.js";import{discardOrphanedTokens as p,discardTokensForGrant as x}from"../../core/providers/token-cache.js";import{runnerForExecution as N}from"../../core/runtime/runner/index.js";import{closeGrantStreams as O,heldGrantIds as T}from"../../core/runs/events.js";import{stopNodeSandboxes as b}from"../../core/spaces/stop.js";import{brokerSignalBus as L}from"../broker-signals/bus.js";import{withRowClaim as R}from"./row-claim.js";const D=6e4;function c(t,e,r){s({level:"error",event:`daemon.grants.${t}`,error:e,fields:r})}a(c,"failed");function G(t,e,r){try{const o=e.sub===null||e.grantee===null?p(r):x({sub:e.sub,grantee:e.grantee,active:e.standing==="active"},r);o.tokens>0&&s({level:"info",event:"daemon.grants.outbound_tokens_discarded",fields:{grant:t,tokens:o.tokens}})}catch(o){c("outbound_tokens_discard_failed",o,{grant:t})}}a(G,"discardTokens");async function A(t,e){if(k(t),e.standing==="inactive"){const r=O(t);r>0&&s({level:"info",event:"daemon.grants.streams_closed",fields:{grant:t,grantee:e.grantee,streams:r}})}try{await(await import("../../core/providers/bundles.js")).syncProviderBundles()}catch(r){s({level:"error",event:"daemon.providers.bundle_sync_failed",error:r})}}a(A,"serveEffects");async function C(t,e){const r=await R("nodes",t,D,async o=>{const n=await v(t);if(n===null)return;await o.assertOwned(),s({level:"info",event:"daemon.grants.node_stop",node_id:t,fields:{grant:e,execution_id:n.executionId}});const i=a(async()=>{await o.assertOwned(),await _(t,{reason:"permission_revoked"}),await E(t)},"finish");if(n.runner==="process"){const l=await b(t,"permission_revoked",{holdRecovery:!0,after:i});if(!l.confirmed)throw l.error??new Error(`stop of ${t} was not confirmed`);return}const f=await N(n.runner).stop(n,{reason:"permission_revoked"});if(!f.confirmed)throw f.error??new Error(`stop of ${t} was not confirmed`);await i()})}a(C,"stopRevokedNode");async function P(t,e){if(!(e.grantee!==null&&y({grantee:e.grantee}))){if(e.standing==="inactive"){const r=await(await d()).prepare("SELECT DISTINCT n.node_id FROM nodes n JOIN executions e ON e.node_id = n.node_id AND e.state != 'exited' WHERE n.grant_id = ?").all(t);for(const{node_id:o}of r)try{await C(o,t)}catch(n){c("node_stop_failed",n,{grant:t,node_id:o})}}if(e.status==="removed"&&e.removal_done_at===null&&e.grantee!==null&&e.sub!==null)try{await h(e.grantee,e.sub),await(await d()).prepare("UPDATE grants SET removal_done_at = ? WHERE id = ? AND removal_done_at IS NULL").run(new Date().toISOString(),t)}catch(r){c("removal_failed",r,{grant:t,grantee:e.grantee})}}}a(P,"laneEffects");async function W(t,e){const r=await m(e);G(e,r,await u()),await Promise.all([t.serves?A(e,r):void 0,t.runsLanes&&r.standing!=="lagging"?P(e,r):void 0])}a(W,"applyGrantEffects");async function F(t){const e=new Set;if(t.serves)for(const o of T())e.add(o);if(t.runsLanes){const o=await(await d()).prepare("SELECT DISTINCT n.grant_id FROM nodes n JOIN executions e ON e.node_id = n.node_id AND e.state != 'exited' WHERE n.grant_id IS NOT NULL").all();for(const{grant_id:n}of o)e.add(n)}const r=[];for(const o of e)(await m(o)).status===null&&r.push(o);return r}a(F,"goneGrantIds");async function Z(t){const e=new Map,r=a(n=>{const i=(e.get(n)??Promise.resolve()).then(()=>W(t,n)).catch(f=>c("effects_failed",f,{grant:n})).finally(()=>{e.get(n)===i&&e.delete(n)});return e.set(n,i),i},"run"),o=a(async()=>{try{t.serves&&g(),p(await u());const n=new Set(await S(`status <> 'active' OR expires_at <= '${new Date().toISOString()}'`));for(const i of await F(t))n.add(i);await Promise.all([...n].map(r))}catch(n){c("catch_up_failed",n,{})}},"catchUp");return await L.subscribe(n=>{r(n)},{kinds:["grant"],onListenStart:a(()=>{o()},"onListenStart")})}a(Z,"startGrantEffects");export{W as applyGrantEffects,Z as startGrantEffects};

package/dist/daemon/reconcilers/grove-instance-sweep.d.ts

@@ -25,9 +25,7 @@ export interface GroveInstanceSweepContext {
 }
 export declare class GroveInstanceSweepReconciler {
     private readonly operations;
-    private lastSweepAt;
-    private inFlight;
-    private cursor;
+    /** Failed uproots wait here; a process that takes the lane next retries them at once, which is harmless (uproot is idempotent). */
     private readonly retryAfter;
     constructor(operations?: GroveInstanceSweepOperations);
     run(now: number, ctx: GroveInstanceSweepContext): void;

package/dist/daemon/reconcilers/grove-instance-sweep.js

@@ -1 +1 @@
-var A=Object.defineProperty;var c=(e,t)=>A(e,"name",{value:t,configurable:!0});import{spawn as S}from"node:child_process";import{statSync as I}from"node:fs";import{delimiter as T,join as j}from"node:path";import{getRow as G,listNodes as R}from"../../core/canvas/index.js";import{isSafeNodeId as w}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as b,recordedPidLiveness as x}from"../../core/canvas/pid.js";import{emitEvent as p}from"../../core/events/emit.js";import{operationIdContext as h}from"../../core/events/operation-id.js";import{isParked as C}from"../../core/runtime/lifecycle.js";const F=60*1e3,P=3,L=300*1e3,E=["done","dead","canceled"];function _(e){return E.includes(e.status)&&!C(e)}c(_,"isClosed");function M(){const e=process.env.PATH;if(e===void 0)return null;for(const t of e.split(T)){const i=j(t===""?process.cwd():t,"grove");try{const r=I(i);if(r.isFile()&&(r.mode&73)!==0)return i}catch{}}return null}c(M,"resolveGroveBinary");function O(e,t){return new Promise(i=>{let r;try{r=S(e,[...t],{stdio:["ignore","pipe","pipe"]})}catch(n){i({code:null,signal:null,stdout:"",stderr:"",error:n instanceof Error?n:new Error(String(n))});return}let a="",s="",l=!1;const u=c(n=>{l||(l=!0,i(n))},"settle");r.stdout.on("data",n=>{a+=n.toString()}),r.stderr.on("data",n=>{s+=n.toString()}),r.on("error",n=>u({code:null,signal:null,stdout:a,stderr:s,error:n})),r.on("close",(n,f)=>u({code:n,signal:f,stdout:a,stderr:s}))})}c(O,"executeGrove");const V={resolveBinary:M,listTerminalNodes:c(async()=>await R({status:[...E]}),"listTerminalNodes"),getNodeRow:G,execute:O,captureLiveness:b};function B(e){let t;try{t=JSON.parse(e)}catch{return null}if(typeof t!="object"||t===null||!Array.isArray(t.projects))return null;const i=[];for(const r of t.projects){if(typeof r!="object"||r===null)continue;const a=r.name,s=r.instances;if(!(typeof a!="string"||!Array.isArray(s)))for(const l of s){if(typeof l!="object"||l===null)continue;const u=l.name,n=l.slot,f=l.spec?.labels,m=typeof f=="object"&&f!==null?f.owner:void 0;typeof u=="string"&&typeof n=="number"&&typeof m=="string"&&i.push({project:a,name:u,slot:n,owner:m})}}return i}c(B,"instancesFromInventory");function k(e,t,i){return _(e)&&!t.isClaimed(e.node_id)&&e.frozen_at===null&&x(e.pi_pid,e.pi_pid_identity,i)==="dead"}c(k,"engineIsGone");async function D(e,t,i){if(t.isClaimed(e.node_id))return!1;const r=await i(e.node_id);return r!==null&&_(r)&&r.frozen_at===null&&r.pi_pid===e.pi_pid&&r.pi_pid_identity===e.pi_pid_identity}c(D,"stillEligible");function N(e){if(e.error!==void 0)return e.error;const t=e.stderr.trim()||e.stdout.trim()||`Grove exited with ${e.signal??e.code??"an unknown status"}`;return new Error(t)}c(N,"executionError");function g(e){return`${e.project}/${e.name}`}c(g,"instanceKey");class X{static{c(this,"GroveInstanceSweepReconciler")}operations;lastSweepAt=Number.NEGATIVE_INFINITY;inFlight=!1;cursor=0;retryAfter=new Map;constructor(t=V){this.operations=t}run(t,i){if(this.inFlight||t-this.lastSweepAt<F||!i.lifecycle.acceptsDetachedWork())return;this.lastSweepAt=t,this.inFlight=!0;const r=Promise.resolve().then(()=>this.sweep(t,i.fleet)).finally(()=>{this.inFlight=!1});i.lifecycle.registerDetached(r)}async sweep(t,i){const r=this.operations.resolveBinary();if(r===null)return;let a;try{a=(await this.operations.listTerminalNodes()).filter(_)}catch(o){h.fresh(()=>{p({level:"error",event:"grove.sweep.scan_failed",error:o})});return}if(a.length===0)return;const s=await this.operations.execute(r,["list","--json"]);if(s.error!==void 0||s.code!==0||s.signal!==null){h.fresh(()=>{p({level:"error",event:"grove.sweep.list_failed",error:N(s)})});return}const l=B(s.stdout);if(l===null){h.fresh(()=>{p({level:"error",event:"grove.sweep.list_failed",error:new Error("grove list --json returned invalid inventory JSON")})});return}const u=new Map(a.map(o=>[o.node_id,o])),n=l.filter(o=>u.get(o.owner)!==void 0&&(this.retryAfter.get(g(o))??Number.NEGATIVE_INFINITY)<=t).sort((o,d)=>o.project.localeCompare(d.project)||o.slot-d.slot||o.name.localeCompare(d.name));if(n.length===0)return;this.cursor>=n.length&&(this.cursor=0);const f=Math.min(P,n.length),m=Array.from({length:f},(o,d)=>n[(this.cursor+d)%n.length]);this.cursor=(this.cursor+f)%n.length;const v=await this.operations.captureLiveness();if(v!==null)for(const o of m){const d=u.get(o.owner);if(!k(d,i,v)||!await D(d,i,this.operations.getNodeRow))continue;const y=await this.operations.execute(r,["uproot",g(o),"--force"]);if(y.error===void 0&&y.code===0&&y.signal===null){this.retryAfter.delete(g(o)),h.fresh(()=>{p({level:"info",event:"grove.sweep.reconciled",outcome:"succeeded",...w(o.owner)?{node_id:o.owner}:{fields:{affected_node_id:o.owner}}})});continue}this.retryAfter.set(g(o),t+L),h.fresh(()=>{p({level:"error",event:"grove.sweep.failed",outcome:"failed",...w(o.owner)?{node_id:o.owner}:{fields:{affected_node_id:o.owner}},error:N(y)})})}}}export{P as GROVE_INSTANCE_SWEEP_BUDGET,F as GROVE_INSTANCE_SWEEP_INTERVAL_MS,L as GROVE_INSTANCE_SWEEP_RETRY_DELAY_MS,X as GroveInstanceSweepReconciler};
+var I=Object.defineProperty;var l=(e,t)=>I(e,"name",{value:t,configurable:!0});import{spawn as T}from"node:child_process";import{statSync as j}from"node:fs";import{delimiter as R,join as b}from"node:path";import{getRow as G,listNodes as C}from"../../core/canvas/index.js";import{isSafeNodeId as E}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as x,recordedPidLiveness as L}from"../../core/canvas/pid.js";import{emitEvent as m}from"../../core/events/emit.js";import{operationIdContext as y}from"../../core/events/operation-id.js";import{isParked as P}from"../../core/runtime/lifecycle.js";import{runThrottledLane as M}from"./throttled-lane.js";const O=60*1e3,V=3,B=300*1e3,W=300*1e3,N=["done","dead","canceled"];function g(e){return N.includes(e.status)&&!P(e)}l(g,"isClosed");function k(){const e=process.env.PATH;if(e===void 0)return null;for(const t of e.split(R)){const o=b(t===""?process.cwd():t,"grove");try{const r=j(o);if(r.isFile()&&(r.mode&73)!==0)return o}catch{}}return null}l(k,"resolveGroveBinary");function D(e,t){return new Promise(o=>{let r;try{r=T(e,[...t],{stdio:["ignore","pipe","pipe"]})}catch(i){o({code:null,signal:null,stdout:"",stderr:"",error:i instanceof Error?i:new Error(String(i))});return}let s="",d="",a=!1;const u=l(i=>{a||(a=!0,o(i))},"settle");r.stdout.on("data",i=>{s+=i.toString()}),r.stderr.on("data",i=>{d+=i.toString()}),r.on("error",i=>u({code:null,signal:null,stdout:s,stderr:d,error:i})),r.on("close",(i,c)=>u({code:i,signal:c,stdout:s,stderr:d}))})}l(D,"executeGrove");const F={resolveBinary:k,listTerminalNodes:l(async()=>await C({status:[...N]}),"listTerminalNodes"),getNodeRow:G,execute:D,captureLiveness:x};function Y(e){let t;try{t=JSON.parse(e)}catch{return null}if(typeof t!="object"||t===null||!Array.isArray(t.projects))return null;const o=[];for(const r of t.projects){if(typeof r!="object"||r===null)continue;const s=r.name,d=r.instances;if(!(typeof s!="string"||!Array.isArray(d)))for(const a of d){if(typeof a!="object"||a===null)continue;const u=a.name,i=a.slot,c=a.spec?.labels,p=typeof c=="object"&&c!==null?c.owner:void 0;typeof u=="string"&&typeof i=="number"&&typeof p=="string"&&o.push({project:s,name:u,slot:i,owner:p})}}return o}l(Y,"instancesFromInventory");async function $(e,t,o){return g(e)&&!await t.isClaimed(e.node_id)&&e.frozen_at===null&&L(e.pi_pid,e.pi_pid_identity,o)==="dead"}l($,"engineIsGone");async function z(e,t,o){if(await t.isClaimed(e.node_id))return!1;const r=await o(e.node_id);return r!==null&&g(r)&&r.frozen_at===null&&r.pi_pid===e.pi_pid&&r.pi_pid_identity===e.pi_pid_identity}l(z,"stillEligible");function S(e){if(e.error!==void 0)return e.error;const t=e.stderr.trim()||e.stdout.trim()||`Grove exited with ${e.signal??e.code??"an unknown status"}`;return new Error(t)}l(S,"executionError");function h(e){return`${e.project}/${e.name}`}l(h,"instanceKey");class ne{static{l(this,"GroveInstanceSweepReconciler")}operations;retryAfter=new Map;constructor(t=F){this.operations=t}run(t,o){if(!o.lifecycle.acceptsDetachedWork())return;const r=M("grove-instance-sweep","",t,O,B,s=>this.sweep(t,o.fleet,s)).then(()=>{}).catch(s=>{y.fresh(()=>m({level:"error",event:"grove.sweep.failed",error:s}))});o.lifecycle.registerDetached(r)}async sweep(t,o,r){const s=this.operations.resolveBinary();if(s===null)return;let d;try{d=(await this.operations.listTerminalNodes()).filter(g)}catch(n){y.fresh(()=>{m({level:"error",event:"grove.sweep.scan_failed",error:n})});return}if(d.length===0)return;const a=await this.operations.execute(s,["list","--json"]);if(a.error!==void 0||a.code!==0||a.signal!==null){y.fresh(()=>{m({level:"error",event:"grove.sweep.list_failed",error:S(a)})});return}const u=Y(a.stdout);if(u===null){y.fresh(()=>{m({level:"error",event:"grove.sweep.list_failed",error:new Error("grove list --json returned invalid inventory JSON")})});return}const i=new Map(d.map(n=>[n.node_id,n])),c=u.filter(n=>i.get(n.owner)!==void 0&&(this.retryAfter.get(h(n))??Number.NEGATIVE_INFINITY)<=t).sort((n,f)=>n.project.localeCompare(f.project)||n.slot-f.slot||n.name.localeCompare(f.name));if(c.length===0)return;let p=Number(r.state??0);(!Number.isInteger(p)||p>=c.length)&&(p=0);const v=Math.min(V,c.length),A=Array.from({length:v},(n,f)=>c[(p+f)%c.length]);if(!await r.save(String((p+v)%c.length)))return;const w=await this.operations.captureLiveness();if(w!==null)for(const n of A){const f=i.get(n.owner);if(!await $(f,o,w)||!await z(f,o,this.operations.getNodeRow))continue;if(!await r.holds())return;const _=await this.operations.execute(s,["uproot",h(n),"--force"]);if(_.error===void 0&&_.code===0&&_.signal===null){this.retryAfter.delete(h(n)),y.fresh(()=>{m({level:"info",event:"grove.sweep.reconciled",outcome:"succeeded",...E(n.owner)?{node_id:n.owner}:{fields:{affected_node_id:n.owner}}})});continue}this.retryAfter.set(h(n),t+W),y.fresh(()=>{m({level:"error",event:"grove.sweep.failed",outcome:"failed",...E(n.owner)?{node_id:n.owner}:{fields:{affected_node_id:n.owner}},error:S(_)})})}}}export{V as GROVE_INSTANCE_SWEEP_BUDGET,O as GROVE_INSTANCE_SWEEP_INTERVAL_MS,W as GROVE_INSTANCE_SWEEP_RETRY_DELAY_MS,ne as GroveInstanceSweepReconciler};

package/dist/daemon/reconcilers/ledger-r2.d.ts

@@ -0,0 +1,27 @@
+import { type BlobStore } from '../../core/blobs/index.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_R2_INTERVAL_MS: number;
+export declare const LEDGER_R2_GRACE_MS: number;
+/** (month, payer) pairs moved per pass. */
+export declare const LEDGER_R2_BUDGET = 25;
+/** The blob's key names its content (`hash` is the SHA-256 of the rows it holds), so two movers that read different versions of the
+ *  rows write different blobs and the committed `charge_chunks.blob_key` always matches the totals stored beside it. */
+export declare const chargeChunkKey: (month: string, payer: string, hash: string) => string;
+export interface LedgerR2Result {
+    moved: number;
+    deferred: number;
+}
+export declare class LedgerR2Reconciler {
+    private readonly blobs;
+    constructor(blobs?: () => Promise<BlobStore>);
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass. Exposed for tests; `afterBlob` runs between the blob write and the transaction (a test kills the process there). */
+    pass(now: number, lane?: LanePass, afterBlob?: () => Promise<void>): Promise<LedgerR2Result>;
+    private move;
+    /** The transaction half of a move: true when the `charge_chunks` row named `key` was inserted and the rows deleted. */
+    private commit;
+    private movable;
+}

package/dist/daemon/reconcilers/ledger-r2.js

@@ -0,0 +1,7 @@
+var g=Object.defineProperty;var u=(d,e)=>g(d,"name",{value:e,configurable:!0});import{createHash as _}from"node:crypto";import{zstdCompressSync as S}from"node:zlib";import{blobStore as A}from"../../core/blobs/index.js";import{openDb as E,withCanvasWrite as N}from"../../core/canvas/db.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{usageMonth as D}from"../../core/ledger/months.js";import{sandboxUnbilledIn as L}from"./ledger-sandbox.js";import{storageSettledThrough as w}from"./ledger-storage.js";import{runThrottledLane as y}from"./throttled-lane.js";const T=60*6e4,v=2880*6e4,C=25,M=10*6e4,I=u((d,e,t)=>`ledger/${d}/${e}.${t}.jsonl.zst`,"chargeChunkKey"),p="(billed_usd IS NULL OR billed_usd = 0 OR invoice_id IS NOT NULL)";class P{static{u(this,"LedgerR2Reconciler")}blobs;constructor(e=A){this.blobs=e}run(e,t){if(!t.lifecycle.acceptsDetachedWork())return;const a=R.fresh(()=>y("ledger-r2","",e,T,M,async r=>{await this.pass(e,r)})).then(()=>{}).catch(r=>{R.fresh(()=>f({level:"error",event:"ledger.r2.failed",error:r}))});t.lifecycle.registerDetached(a)}async pass(e,t,a){const r={moved:0,deferred:0};if(O("user").ledger?.r2_move!==!0)return r;const i=await w();if(i===null)return r;const o=D(new Date(e-v)),s=await(await E()).prepare(`SELECT c.month, c.payer FROM charges c
+      WHERE c.month < ? AND (?::text IS NULL OR c.month <= ?) AND NOT EXISTS (SELECT 1 FROM charge_chunks k WHERE k.month = c.month AND k.payer = c.payer)
+      GROUP BY c.month, c.payer HAVING bool_and(${p.replaceAll("billed_usd","c.billed_usd").replaceAll("invoice_id","c.invoice_id")})
+      ORDER BY c.month, c.payer LIMIT ?`).all(o,i??null,i??null,C),l=await this.blobs();for(const{month:n,payer:h}of s){if(t&&!await t.holds())break;if(await L(n,e)){r.deferred+=1;continue}await this.move(l,n,h,a)?r.moved+=1:r.deferred+=1}return r.moved>0&&f({level:"info",event:"ledger.r2.moved",outcome:"succeeded",fields:{...r}}),r}async move(e,t,a,r){const o=await(await E()).prepare("SELECT id, to_jsonb(charges)::text AS body FROM charges WHERE month = ? AND payer = ? ORDER BY id").all(t,a);if(o.length===0||!await this.movable(t,a))return!1;const c=o.map(n=>n.body).join(`
+`)+`
+`,s=I(t,a,_("sha256").update(c).digest("hex").slice(0,16));await e.put(s,S(Buffer.from(c)));const l=await this.commit(t,a,s,o,r);return!l&&await(await E()).prepare("SELECT 1 FROM charge_chunks WHERE blob_key = ?").get(s)===void 0&&await e.delete(s),l}async commit(e,t,a,r,i){return await i?.(),await N(async o=>{const c=await o.prepare("SELECT id FROM charges WHERE month = ? AND payer = ? ORDER BY id FOR UPDATE").all(e,t),s=await o.prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND ${p}`).get(e,t);if(c.length!==r.length||Number(s.n)!==r.length||c.some((m,b)=>m.id!==r[b].id)||(await o.prepare("SELECT string_agg(to_jsonb(c)::text, E'\\n' ORDER BY c.id) AS body FROM charges c WHERE month = ? AND payer = ?").get(e,t)).body!==r.map(m=>m.body).join(`
+`))return!1;const n=await o.prepare("SELECT sum(usd) AS usd, sum(billed_usd) AS billed, min(at) AS first_at, max(at) AS last_at FROM charges WHERE month = ? AND payer = ?").get(e,t);return(await o.prepare('INSERT INTO charge_chunks (month, payer, blob_key, "rows", usd, billed_usd, first_at, last_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (month, payer) DO NOTHING').run(e,t,a,r.length,n.usd,n.billed,n.first_at,n.last_at)).changes===0?!1:(await o.prepare("DELETE FROM charges WHERE month = ? AND payer = ?").run(e,t),!0)})}async movable(e,t){const a=await(await E()).prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND NOT ${p}`).get(e,t);return Number(a.n)===0}}export{C as LEDGER_R2_BUDGET,v as LEDGER_R2_GRACE_MS,T as LEDGER_R2_INTERVAL_MS,P as LedgerR2Reconciler,I as chargeChunkKey};

package/dist/daemon/reconcilers/ledger-sandbox.d.ts

@@ -0,0 +1,55 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_SANDBOX_INTERVAL_MS = 60000;
+export declare const DEFAULT_STANDBY_S = 30;
+/** Executions billed per pass; the rest wait for the next pass. */
+export declare const LEDGER_SANDBOX_BUDGET = 200;
+interface Rates {
+    usdPerGbHour: number | null;
+    standbyMs: number;
+}
+export declare function sandboxRates(): Rates;
+/** The `[from, to]` chunks to bill: every whole hour, plus the remainder when the period is closed. The hours stay aligned to `from`
+ *  whatever the month; an hour or remainder that crosses a UTC month boundary is cut there into one chunk per month. */
+export declare function chunksOf(from: number, end: number, closed: boolean): Array<[number, number]>;
+/** When a chunk's seconds accrued, for the row's `at` (and so its month): its last millisecond, so a chunk ending at exactly 00:00Z
+ *  of a month belongs to the month before it. */
+export declare const chunkAccruedAt: (chunkEnd: number) => Date;
+/** The row id of the billed chunk of an execution that starts at `chunkStart`: the start time in whole seconds makes it
+ *  the same in every process and unique across the awake periods of one execution. */
+export declare const sandboxChargeId: (executionId: string, chunkStart: number) => string;
+/** An awake period that a wake ended: whose it was, where it began, when the broker went idle and how far it had been billed. */
+interface EndedPeriod {
+    execution_id: string;
+    awake_since: string;
+    idle_at: string;
+    charged_through: string | null;
+}
+export interface LedgerSandboxResult {
+    executions: number;
+    rows: number;
+    failed: number;
+}
+export declare class LedgerSandboxReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass over the executions with awake time not yet billed. Exposed for tests. */
+    pass(now: number, lane?: LanePass): Promise<LedgerSandboxResult>;
+    /** Bills one execution; returns the rows it wrote. Payer and memory are read first; the rows and `charged_through` go in one
+     *  transaction that holds the execution row's lock, from which the period is recomputed. `settle` is the wake of a resident
+     *  broker: an idle period is closed at `min(now, idle_at + standby tail)` and billed whole, because the wake resets `awake_since`;
+     *  a period that is not idle is left to the lane. `ended` is a period a wake ended before its settle succeeded: it is billed from
+     *  the period saved with it, whatever the execution row says now, and its row is deleted in the same transaction. */
+    bill(executionId: string, now: number, usdPerGbHour: number, standbyMs: number, settle?: boolean, ended?: EndedPeriod): Promise<number>;
+}
+/** Whether any awake time that accrued in `month` (UTC) is not billed yet: an execution's current period, or an ended period a failed
+ *  settle saved, whose unbilled span `[max(awake_since, charged_through), end]` overlaps the month. `ledger-r2` moves no month while
+ *  this is true, because the month's allowance is summed from the live `charges` rows and a row written after the move would get it
+ *  again. With no sandbox rate the lane writes nothing, so nothing is unbilled. */
+export declare function sandboxUnbilledIn(month: string, now: number): Promise<boolean>;
+/** Before a wake resets `awake_since` (`recordExecutionAwake`): bills the idle period that is ending, which the lane would
+ *  otherwise lose. Never throws: a settle that fails must not block the wake, and the failure is logged. Settling the same
+ *  period twice charges once (`charged_through` and the deterministic row ids). */
+export declare function settleSandboxBeforeWake(executionId: string, now?: number): Promise<void>;
+export {};

package/dist/daemon/reconcilers/ledger-sandbox.js

@@ -0,0 +1,9 @@
+var W=Object.defineProperty;var d=(r,e)=>W(r,"name",{value:e,configurable:!0});import{openDb as x,withCanvasWrite as F}from"../../core/canvas/db.js";import{getNode as A}from"../../core/canvas/canvas.js";import{readConfig as z}from"../../core/config.js";import{emitEvent as E}from"../../core/events/emit.js";import{operationIdContext as T}from"../../core/events/operation-id.js";import{billingAccountOf as $}from"../../core/ledger/account.js";import{writeInfraCharge as B}from"../../core/ledger/infra-allowance.js";import{resetAt as G}from"../../core/ledger/months.js";import{startCandidateOf as P}from"../../core/runtime/start-check.js";import{runThrottledLane as J}from"./throttled-lane.js";const Y=6e4,V=5*6e4,b=36e5,X=30,y=200;function N(){const r=z("user").ledger;return{usdPerGbHour:r?.sandbox_usd_per_gb_hour??null,standbyMs:(r?.sandbox_standby_s??X)*1e3}}d(N,"sandboxRates");function j(r,e,n){const t=r.exited_at??r.idle_at;if(t===null)return{end:e,closed:!1};const a=Date.parse(t)+n;return a<=e?{end:a,closed:!0}:{end:e,closed:!1}}d(j,"periodEnd");const K=d(r=>{const e=new Date(r);return Date.UTC(e.getUTCFullYear(),e.getUTCMonth()+1,1)},"monthEndAfter");function R(r,e){const n=[];for(let t=r;t<e;){const a=Math.min(e,K(t));n.push([t,a]),t=a}return n}d(R,"withinMonths");function k(r,e,n){const t=[];let a=r;for(;e-a>=b;)t.push(...R(a,a+b)),a+=b;return n&&e>a&&t.push(...R(a,e)),t}d(k,"chunksOf");const Z=d(r=>new Date(r-1),"chunkAccruedAt"),q=d((r,e)=>`sandbox:${r}:${Math.floor(e/1e3)}`,"sandboxChargeId"),S="ledger-sandbox-unsettled",M=d((r,e)=>`${r}@${Date.parse(e)}`,"unsettledKey");class Q{static{d(this,"LedgerSandboxReconciler")}run(e,n){if(!n.lifecycle.acceptsDetachedWork())return;const t=T.fresh(()=>J("ledger-sandbox","",e,Y,V,async a=>{await this.pass(e,a)})).then(()=>{}).catch(a=>{T.fresh(()=>E({level:"error",event:"ledger.sandbox.failed",error:a}))});n.lifecycle.registerDetached(t)}async pass(e,n){const t={executions:0,rows:0,failed:0},{usdPerGbHour:a,standbyMs:l}=N();if(a===null)return t;const i=await x(),_=await i.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+      WHERE runner = 'blaxel' AND awake_since IS NOT NULL
+        AND (state <> 'exited' OR charged_through IS NULL OR charged_through::timestamptz < exited_at::timestamptz + make_interval(secs => ?::float8 / 1000))
+      ORDER BY execution_id LIMIT ?`).all(l,y);for(const s of _){if(n&&!await n.holds())break;try{const o=await this.bill(s.execution_id,e,a,l);o>0&&(t.executions+=1),t.rows+=o}catch(o){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:o,fields:{execution_id:s.execution_id}})}}const f=await i.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? ORDER BY key LIMIT ?").all(S,y);for(const{cursor:s}of f){const o=JSON.parse(s);if(n&&!await n.holds())break;try{const c=await this.bill(o.execution_id,e,a,l,!1,o);c>0&&(t.executions+=1),t.rows+=c}catch(c){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:c,fields:{execution_id:o.execution_id}})}}return(t.rows>0||t.failed>0)&&E({level:"info",event:"ledger.sandbox.settled",outcome:"succeeded",fields:{...t}}),t}async bill(e,n,t,a,l=!1,i){const _=await x(),f=await _.prepare("SELECT node_id FROM executions WHERE execution_id = ?").get(e);if(f===void 0)return 0;const s=await A(f.node_id);if(s===null)throw new Error(`execution ${e} names node ${f.node_id}, which has no row`);const o=await P(s);if(o===null)throw new Error(`node ${s.node_id} has no person yet, so its sandbox hours have no payer`);const c=await _.prepare("SELECT e.memory_mb AS mb FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ?").get(s.profile_id??null);if(c===void 0)throw new Error(`node ${s.node_id} has no environment, so its sandbox size is unknown`);const p=Number(c.mb),{account:C,billable:v}=await $(_,o.payer),H=(s.run_id?await A(s.run_id):null)?.caused_by??s.caused_by??null;return await F(async D=>{const u=await D.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+        WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL FOR UPDATE`).get(e);if(i===void 0&&u===void 0||i===void 0&&l&&u.idle_at===null)return 0;let g,h;if(i!==void 0)g=Math.max(Date.parse(i.awake_since),i.charged_through===null?0:Date.parse(i.charged_through)),h=k(g,Date.parse(i.idle_at)+a,!0);else{const{end:m,closed:w}=l?{end:Math.min(n,Date.parse(u.idle_at)+a),closed:!0}:j(u,n,a);g=Math.max(Date.parse(u.awake_since),u.charged_through===null?0:Date.parse(u.charged_through)),h=k(g,m,w)}let L=0;for(const[m,w]of h){const I=Math.round((w-m)/1e3),O=p/1024*((w-m)/b),U=O*t;(await B({id:q(e,m),at:Z(w),kind:"sandbox",what:`${p} MB`,units:{gb_hours:O,seconds:I,memory_mb:p},usd:U,payer:o.payer,funding:"bill",sub:o.sub,grantee:o.grantee,node:s.node_id,run:s.run_id??null,provider:"blaxel",causedBy:H,account:C},v)).written&&(L+=1)}return i!==void 0?await D.prepare("DELETE FROM reconciler_leases WHERE lane = ? AND key = ?").run(S,M(e,i.awake_since)):h.length>0&&await D.prepare("UPDATE executions SET charged_through = ? WHERE execution_id = ?").run(new Date(h[h.length-1][1]).toISOString(),e),L})}}async function pe(r,e){const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return!1;const a=await x(),l=new Date(`${r}-01T00:00:00Z`).toISOString(),i=G(r);return await a.prepare(`SELECT 1 AS found FROM executions e,
+      LATERAL (SELECT greatest(e.awake_since::timestamptz, COALESCE(e.charged_through::timestamptz, '-infinity'::timestamptz)) AS from_at,
+                      CASE WHEN COALESCE(e.exited_at, e.idle_at) IS NULL THEN ?::timestamptz
+                           ELSE least(COALESCE(e.exited_at, e.idle_at)::timestamptz + make_interval(secs => ?::float8 / 1000), ?::timestamptz) END AS end_at) p
+    WHERE e.runner = 'blaxel' AND e.awake_since IS NOT NULL AND p.from_at < p.end_at AND p.from_at < ?::timestamptz AND p.end_at > ?::timestamptz LIMIT 1`).get(new Date(e).toISOString(),t,new Date(e).toISOString(),i,l)!==void 0?!0:(await a.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ?").all(S)).some(({cursor:s})=>{const o=JSON.parse(s),c=Math.max(Date.parse(o.awake_since),o.charged_through===null?0:Date.parse(o.charged_through)),p=Date.parse(o.idle_at)+t;return c<p&&c<Date.parse(i)&&p>Date.parse(l)})}d(pe,"sandboxUnbilledIn");async function he(r,e=Date.now()){try{const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return;await new Q().bill(r,e,n,t,!0)}catch(n){E({level:"warn",event:"ledger.sandbox.settle_failed",error:n,fields:{execution_id:r}}),await ee(r)}}d(he,"settleSandboxBeforeWake");async function ee(r){try{const e=await x(),n=await e.prepare("SELECT execution_id, awake_since, idle_at, charged_through FROM executions WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL AND idle_at IS NOT NULL").get(r);if(n===void 0)return;await e.prepare("INSERT INTO reconciler_leases (lane, key, owner, expires_at, cursor) VALUES (?, ?, ?, ?, ?) ON CONFLICT (lane, key) DO NOTHING").run(S,M(r,n.awake_since),"settle",new Date(0).toISOString(),JSON.stringify(n))}catch(e){E({level:"error",event:"ledger.sandbox.unsettled_lost",error:e,fields:{execution_id:r}})}}d(ee,"keepUnsettled");export{X as DEFAULT_STANDBY_S,y as LEDGER_SANDBOX_BUDGET,Y as LEDGER_SANDBOX_INTERVAL_MS,Q as LedgerSandboxReconciler,Z as chunkAccruedAt,k as chunksOf,q as sandboxChargeId,N as sandboxRates,pe as sandboxUnbilledIn,he as settleSandboxBeforeWake};

package/dist/daemon/reconcilers/ledger-storage.d.ts

@@ -0,0 +1,22 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const LEDGER_STORAGE_INTERVAL_MS: number;
+/** The last month the storage lane settled (its saved cursor state), null when it has settled none, and `undefined` when no storage
+ *  rate is configured, so the lane writes nothing and there is nothing to wait for. `ledger-r2` moves no month past it. */
+export declare function storageSettledThrough(): Promise<string | null | undefined>;
+export interface LedgerStorageResult {
+    month: string | null;
+    rows: number;
+    failed: number;
+}
+export declare class LedgerStorageReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** Settles every closed month the lane has not: the months after the one its cursor saved, through the month before `now`'s
+     *  (with no saved month, only that one). A month's save follows its own success, so a failure leaves it and the later months
+     *  for the next pass. Exposed for tests. */
+    pass(now: number, lane?: LanePass): Promise<LedgerStorageResult>;
+    /** Writes the month's rows; false when a payer failed or the lane was lost, so the month is not saved as settled. */
+    private settleMonth;
+}

package/dist/daemon/reconcilers/ledger-storage.js

@@ -0,0 +1,5 @@
+var M=Object.defineProperty;var p=(s,e)=>M(s,"name",{value:e,configurable:!0});import{openDb as N}from"../../core/canvas/db.js";import{getNode as D}from"../../core/canvas/canvas.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as y}from"../../core/events/emit.js";import{operationIdContext as S}from"../../core/events/operation-id.js";import{billingAccountOf as T}from"../../core/ledger/account.js";import{writeInfraCharge as v}from"../../core/ledger/infra-allowance.js";import{usageMonth as w}from"../../core/ledger/months.js";import{startCandidateOf as I}from"../../core/runtime/start-check.js";import{runThrottledLane as L}from"./throttled-lane.js";const A=60*6e4,C=10*6e4,G=1024**3;async function Q(){const s=O("user").ledger;if((s?.storage_usd_per_gb_month??null)===null&&(s?.sandbox_disk_usd_per_gb_month??null)===null)return;const e=await(await N()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = 'ledger-storage' AND key = ''").get(),r=e?.cursor==null?null:JSON.parse(e.cursor).state;return r===null?null:r.slice(0,7)}p(Q,"storageSettledThrough");function U(s){const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r-2,1)))}p(U,"previousMonth");const x=p(s=>{const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r,1)))},"nextMonth"),F=p(s=>{const[e,r]=s.split("-").map(Number);return new Date(Date.UTC(e,r,1)).toISOString()},"endOf");class X{static{p(this,"LedgerStorageReconciler")}run(e,r){if(!r.lifecycle.acceptsDetachedWork())return;const u=S.fresh(()=>L("ledger-storage","",e,A,C,async o=>{await this.pass(e,o)})).then(()=>{}).catch(o=>{S.fresh(()=>y({level:"error",event:"ledger.storage.failed",error:o}))});r.lifecycle.registerDetached(u)}async pass(e,r){const u=U(w(new Date(e))),o={month:null,rows:0,failed:0},_=O("user").ledger,f={r2:_?.storage_usd_per_gb_month??null,disk:_?.sandbox_disk_usd_per_gb_month??null};if(f.r2===null&&f.disk===null)return o;const i=r?.state?.slice(0,7)??null,l=[];if(i===null)l.push(u);else for(let n=x(i);n<=u;n=x(n))l.push(n);for(const n of l){if(o.month=n,!await this.settleMonth(n,f,r,o))break;await r?.save(n)}return(o.rows>0||o.failed>0)&&y({level:"info",event:"ledger.storage.settled",outcome:"succeeded",fields:{...o}}),o}async settleMonth(e,r,u,o){const{r2:_,disk:f}=r,i=F(e),l=await N(),n=new Map,g=p(t=>{let a=n.get(t);return a||n.set(t,a={bytes:0,diskGb:0}),a},"entry");for(const t of await l.prepare("SELECT split_part(blob_key, '/', 2) AS node, sum(bytes) AS bytes FROM conversation_parts WHERE created < ? GROUP BY 1").all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node, sum(u.size) AS bytes FROM (
+        SELECT DISTINCT upload_id, split_part(blob_key, '/', 2) AS node FROM upload_uses WHERE blob_key LIKE 'attachments/%') x
+      JOIN uploads u ON u.upload_id = x.upload_id WHERE u.created_at < ? GROUP BY x.node`).all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node_id AS node, sum(e.memory_mb) AS mb FROM (
+        SELECT DISTINCT node_id, sandbox_name FROM executions WHERE runner = 'blaxel' AND sandbox_name IS NOT NULL AND started_at < ?) x
+      JOIN nodes n ON n.node_id = x.node_id JOIN objects o ON o.id = n.node_id AND (o.deleted_at IS NULL OR o.deleted_at >= ?) JOIN profiles p ON p.id = n.profile_id JOIN environments e ON e.object = p.environment_id GROUP BY x.node_id`).all(i,i))g(t.node).diskGb+=Number(t.mb)/2/1024;const h=new Map;for(const[t,a]of n){const b=await D(t),c=b===null?null:await I(b);if(c===null)continue;const m=h.get(c.payer)??{sub:c.sub,grantee:c.grantee,bytes:0,diskGb:0,node:t};m.bytes+=a.bytes,m.diskGb+=a.diskGb,h.set(c.payer,m)}const R=new Date(Date.parse(i)-1);let E=0;for(const[t,a]of h){if(u&&!await u.holds())return!1;try{const{account:b,billable:c}=await T(l,t),m=[{id:`storage:${t}:${e}:r2`,provider:"r2",gbMonths:a.bytes/G,rate:_},{id:`storage:${t}:${e}:blaxel`,provider:"blaxel",gbMonths:a.diskGb,rate:f}];for(const d of m){if(d.rate===null||d.gbMonths<=0)continue;const k=d.gbMonths*d.rate;(await v({id:d.id,at:R,kind:"storage",what:d.provider==="r2"?"object storage":"sandbox disk",units:{gb_months:d.gbMonths},usd:k,payer:t,funding:"bill",sub:a.sub,grantee:a.grantee,provider:d.provider,causedBy:null,account:b},c)).written&&(o.rows+=1)}}catch(b){o.failed+=1,E+=1,y({level:"warn",event:"ledger.storage.payer_failed",error:b,fields:{payer:t,month:e}})}}return E===0}}export{A as LEDGER_STORAGE_INTERVAL_MS,X as LedgerStorageReconciler,Q as storageSettledThrough};

package/dist/daemon/reconcilers/live-obligation.d.ts

@@ -1,4 +1,5 @@
 import { type Lifecycle } from '../../core/canvas/index.js';
+import { type ViewerProbe } from '../../core/runtime/placement.js';
 import { type LifecycleEvent } from '../../core/runtime/lifecycle.js';
 import type { FleetRegistry } from '../../core/runtime/fleet.js';
 /** Durable work that requires a node to remain automatically wakeable. */
@@ -23,7 +24,7 @@ export declare function retentionHold(fleet: Pick<FleetRegistry, 'isClaimed'>):
  *  `attends: false` is a system observer and deliberately does not count, which
  *  is how Northlight identifies itself — so an open product page is
  *  invisible here, while crouter's own local and remote viewers are not. */
-export declare function hasHumanViewer(nodeId: string): Promise<boolean>;
+export declare function hasHumanViewer(nodeId: string, viewer?: ViewerProbe): Promise<boolean>;
 /** What parking an unattended node actually amounts to — the one rule both the
  *  live-broker park clock and the lifecycle tick apply.
  *
@@ -48,7 +49,7 @@ export declare function hasHumanViewer(nodeId: string): Promise<boolean>;
  *  invariant is that content a sender already handed over is never dropped.
  */
 export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
-export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): Promise<ParkEvent>;
+export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<ParkEvent>;
 /** Apply an unattended decision and notify subscribers of automatic completion.
  *  This is the DEGRADED enactment: no summary turn ran, so subscribers get the
  *  doctrine notice instead of the node's own report. The live clock's good path

package/dist/daemon/reconcilers/live-obligation.js

@@ -1 +1 @@
-var f=Object.defineProperty;var a=(t,r)=>f(t,"name",{value:r,configurable:!0});import{hasActiveLiveSubscription as l,hasLiveSubscriber as p,hasPendingCronWake as m}from"../../core/canvas/index.js";import{fullName as w}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as i}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/structured-output.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await l(t)||await h(t)||await k(t)||await m(t)||await b(t)>0||g(t).state!=="absent"}a(s,"hasLiveObligation");async function B(t){return await s(t)||await i(t)?!0:await n(t)}a(B,"hasPendingWake");async function F(t){return async r=>t.isClaimed(r)||await s(r)||await n(r)}a(F,"retentionHold");async function n(t){return await v(t)!==null||x(t)}a(n,"hasHumanViewer");async function M(t,r){if(await n(t)||await s(t))return"release";const e=await W(t);return e?.link==="pi\u2192provider"&&e.retry.disposition==="fatal"?"release":r==="resident"?await i(t)?"release":"park":await i(t)||await p(t)?"release":"cancel"}a(M,"parkEvent");async function T(t,r){const e=r==="cancel"?await c(t,r,{reason:"stranded"}):await c(t,r);if(r!=="park")return;const o=await y(t),u=o===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${o}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${w(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}a(T,"applyParkEvent");export{T as applyParkEvent,n as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};
+var l=Object.defineProperty;var r=(t,a)=>l(t,"name",{value:a,configurable:!0});import{hasActiveLiveSubscription as f,hasLiveSubscriber as p,hasPendingCronWake as w}from"../../core/canvas/index.js";import{fullName as m}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as n}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/output-result.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await f(t)||await h(t)||await k(t)||await w(t)||await b(t)>0||await g(t)!==null}r(s,"hasLiveObligation");async function B(t){return await s(t)||await n(t)?!0:await o(t)}r(B,"hasPendingWake");async function F(t){return async a=>await t.isClaimed(a)||await s(a)||await o(a)}r(F,"retentionHold");async function o(t,a){return(a!==void 0?a.live!==null:await v(t)!==null)||await x(t)}r(o,"hasHumanViewer");async function M(t,a,e){if(await o(t,e)||await s(t))return"release";const i=await W(t);return i?.link==="pi\u2192provider"&&i.retry.disposition==="fatal"?"release":a==="resident"?await n(t)?"release":"park":await n(t)||await p(t)?"release":"cancel"}r(M,"parkEvent");async function T(t,a){const e=a==="cancel"?await c(t,a,{reason:"stranded"}):await c(t,a);if(a!=="park")return;const i=await y(t),u=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${m(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}r(T,"applyParkEvent");export{T as applyParkEvent,o as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};

package/dist/daemon/reconcilers/managed-worktree-sweep.d.ts

@@ -8,11 +8,6 @@ export interface ManagedWorktreeSweepContext {
     fleet: FleetRegistry;
 }
 export declare class ManagedWorktreeSweepReconciler {
-    private lastSweepAt;
-    private inFlight;
-    /** Round-robin cursor over candidate node ids, so a permanently stuck record
-     *  at the head of the list cannot starve the ones behind it. */
-    private cursor;
     run(now: number, ctx: ManagedWorktreeSweepContext): void;
     /** Terminal nodes whose managed worktree is unreconciled and whose backoff has
      *  elapsed. Canvas reads only — engine liveness is classified later, for the

package/dist/daemon/reconcilers/managed-worktree-sweep.js

@@ -1 +1 @@
-var w=Object.defineProperty;var o=(t,e)=>w(t,"name",{value:e,configurable:!0});import{getNode as p,getRow as h}from"../../core/canvas/index.js";import{managedWorktreeSweepCandidates as y}from"../../core/canvas/managed-worktree-sweep-candidates.js";import{isSafeNodeId as _}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as E,recordedPidLiveness as k}from"../../core/canvas/pid.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as u}from"../../core/events/operation-id.js";import{reconcileManagedWorktree as S}from"../../core/worktree-sweep.js";const I=60*1e3,v=3,T=["done","dead","canceled"];function m(t){return T.includes(t.status)}o(m,"isTerminal");function g(t,e){return e?.state==="closed"||e?.state==="open"&&t.status==="done"&&t.final_report!==null}o(g,"hasSweepDisposition");async function A(t,e,n){return e.isClaimed(t.node_id)||t.frozen_at!==null||!m(t)||!g(t,(await p(t.node_id))?.managed_worktree)?!1:k(t.pi_pid,t.pi_pid_identity,n)==="dead"}o(A,"engineIsGone");function N(t,e){const n=t.node_id,r=t.pi_pid,a=t.pi_pid_identity;return{stillEligible:o(async()=>{if(e.isClaimed(n))return!1;const s=await h(n);return s===null||!m(s)||s.frozen_at!==null||!g(s,(await p(n))?.managed_worktree)?!1:s.pi_pid===r&&s.pi_pid_identity===a},"stillEligible")}}o(N,"eligibilityGuard");class L{static{o(this,"ManagedWorktreeSweepReconciler")}lastSweepAt=Number.NEGATIVE_INFINITY;inFlight=!1;cursor=0;run(e,n){if(this.inFlight||e-this.lastSweepAt<I||!n.lifecycle.acceptsDetachedWork())return;this.lastSweepAt=e,this.inFlight=!0;const r=this.sweep(e,n.fleet).finally(()=>{this.inFlight=!1});n.lifecycle.registerDetached(r)}async candidates(e){try{return await y(e)}catch(n){return u.fresh(()=>{c({level:"error",event:"worktree.sweep.scan_failed",error:n})}),null}}async sweep(e,n){const r=await this.candidates(e);if(r===null||r.length===0)return;this.cursor>=r.length&&(this.cursor=0);const a=this.cursor,s=Math.min(v,r.length),d=[];for(let l=0;l<s;l+=1)d.push(r[(a+l)%r.length]);this.cursor=(a+s)%r.length;const i=await E();if(i!==null)for(const l of d){const f=await h(l);f===null||!await A(f,n,i)||await this.reconcileOne(f,e,n)}}async reconcileOne(e,n,r){const a=e.node_id,s=_(a)?{node_id:a}:{},d=_(a)?{}:{affected_node_id:a};try{const i=await S(a,N(e,r),n);u.fresh(()=>{if(i.status==="complete"){c({level:"info",event:"worktree.sweep.reconciled",outcome:"succeeded",...s,fields:{...d,...i.stash===void 0?{}:{base_checkout_stash:i.stash}}});return}i.status!=="skipped"&&(i.status==="refused"&&i.unchanged===!0||c({level:i.status==="failed"?"error":"warn",event:i.status==="failed"?"worktree.sweep.failed":"worktree.sweep.refused",outcome:i.status==="failed"?"failed":"skipped",...s,fields:{...d,reason:i.reason,...i.detail===void 0?{}:{detail:i.detail}}}))})}catch(i){u.fresh(()=>{c({level:"error",event:"worktree.sweep.failed",...s,error:i,fields:d})})}}}export{L as ManagedWorktreeSweepReconciler,v as WORKTREE_SWEEP_BUDGET,I as WORKTREE_SWEEP_INTERVAL_MS};
+var E=Object.defineProperty;var o=(e,t)=>E(e,"name",{value:t,configurable:!0});import{getNode as _,getRow as m}from"../../core/canvas/index.js";import{managedWorktreeSweepCandidates as k}from"../../core/canvas/managed-worktree-sweep-candidates.js";import{isSafeNodeId as h}from"../../core/canvas/paths.js";import{captureLivenessSnapshotAsync as y,recordedPidLiveness as S}from"../../core/canvas/pid.js";import{emitEvent as c}from"../../core/events/emit.js";import{operationIdContext as f}from"../../core/events/operation-id.js";import{reconcileManagedWorktree as v}from"../../core/worktree-sweep.js";import{runThrottledLane as R}from"./throttled-lane.js";const T=60*1e3,W=300*1e3,I=3,b=["done","dead","canceled"];function w(e){return b.includes(e.status)}o(w,"isTerminal");function g(e,t){return t?.state==="closed"||t?.state==="open"&&e.status==="done"&&e.final_report!==null}o(g,"hasSweepDisposition");async function L(e,t,i){return await t.isClaimed(e.node_id)||e.frozen_at!==null||!w(e)||!g(e,(await _(e.node_id))?.managed_worktree)?!1:S(e.pi_pid,e.pi_pid_identity,i)==="dead"}o(L,"engineIsGone");function M(e,t){const i=e.node_id,s=e.pi_pid,r=e.pi_pid_identity;return{stillEligible:o(async()=>{if(await t.isClaimed(i))return!1;const a=await m(i);return a===null||!w(a)||a.frozen_at!==null||!g(a,(await _(i))?.managed_worktree)?!1:a.pi_pid===s&&a.pi_pid_identity===r},"stillEligible")}}o(M,"eligibilityGuard");class z{static{o(this,"ManagedWorktreeSweepReconciler")}run(t,i){if(!i.lifecycle.acceptsDetachedWork())return;const s=R("managed-worktree-sweep","",t,T,W,r=>this.sweep(t,i.fleet,r)).then(()=>{}).catch(r=>{f.fresh(()=>c({level:"error",event:"worktree.sweep.failed",error:r}))});i.lifecycle.registerDetached(s)}async candidates(t){try{return await k(t)}catch(i){return f.fresh(()=>{c({level:"error",event:"worktree.sweep.scan_failed",error:i})}),null}}async sweep(t,i,s){const r=await this.candidates(t);if(r===null||r.length===0)return;let a=Number(s.state??0);(!Number.isInteger(a)||a>=r.length)&&(a=0);const d=Math.min(I,r.length),n=[];for(let l=0;l<d;l+=1)n.push(r[(a+l)%r.length]);if(!await s.save(String((a+d)%r.length)))return;const p=await y();if(p!==null)for(const l of n){const u=await m(l);if(!(u===null||!await L(u,i,p))){if(!await s.holds())return;await this.reconcileOne(u,t,i)}}}async reconcileOne(t,i,s){const r=t.node_id,a=h(r)?{node_id:r}:{},d=h(r)?{}:{affected_node_id:r};try{const n=await v(r,M(t,s),i);f.fresh(()=>{if(n.status==="complete"){c({level:"info",event:"worktree.sweep.reconciled",outcome:"succeeded",...a,fields:{...d,...n.stash===void 0?{}:{base_checkout_stash:n.stash}}});return}n.status!=="skipped"&&(n.status==="refused"&&n.unchanged===!0||c({level:n.status==="failed"?"error":"warn",event:n.status==="failed"?"worktree.sweep.failed":"worktree.sweep.refused",outcome:n.status==="failed"?"failed":"skipped",...a,fields:{...d,reason:n.reason,...n.detail===void 0?{}:{detail:n.detail}}}))})}catch(n){f.fresh(()=>{c({level:"error",event:"worktree.sweep.failed",...a,error:n,fields:d})})}}}export{z as ManagedWorktreeSweepReconciler,I as WORKTREE_SWEEP_BUDGET,T as WORKTREE_SWEEP_INTERVAL_MS};

package/dist/daemon/reconcilers/node-deadline.d.ts

@@ -1,7 +1,8 @@
+import { type RowClaim } from './row-claim.js';
 /** Enforce absolute spawn deadlines independently of the tick's live-row scan. */
 export declare class NodeDeadlineReconciler {
     run(now: number): Promise<void>;
 }
 /** The one deadline terminalization path. Structured-request expiry reuses it
  * even when a profile is paused, because its dedicated broker must not resume. */
-export declare function expireNodeDeadline(nodeId: string, deadlineAt: string, now: number): Promise<void>;
+export declare function expireNodeDeadline(nodeId: string, deadlineAt: string, now: number, claim?: RowClaim): Promise<void>;

package/dist/daemon/reconcilers/node-deadline.js

@@ -1 +1 @@
-var m=Object.defineProperty;var s=(r,a)=>m(r,"name",{value:a,configurable:!0});import{dueNodeDeadlines as _,getNode as u}from"../../core/canvas/index.js";import{emitEvent as i}from"../../core/events/emit.js";import{isProfilePaused as p}from"../../core/profiles/manifest.js";import{operationIdContext as w}from"../../core/events/operation-id.js";import{errorClassFromFault as h}from"../../core/fault-classifier.js";import{publishNotice as v}from"../../core/runtime/close.js";import{readFault as x}from"../../core/canvas/recovery.js";import{requestBrokerTeardown as f}from"../../core/runtime/host.js";import{transition as y}from"../../core/runtime/lifecycle.js";class z{static{s(this,"NodeDeadlineReconciler")}async run(a){let o;try{o=await _(new Date(a).toISOString())}catch(e){i({level:"error",event:"node_deadline.scan_failed",error:e});return}for(const e of o)p(e.profile_id)||e.deadline_at===null||e.deadline_at===void 0||w.fresh(()=>{D(e.node_id,e.deadline_at,a).catch(d=>{i({level:"error",event:"node_deadline.expire_failed",node_id:e.node_id,error:d,fields:{deadline_at:e.deadline_at}})})})}}async function D(r,a,o){const e=await u(r);if(e===null||e.status==="done")return;if(e.status==="canceled"||e.status==="dead"){const t=await f(r);if(!t.confirmed)throw t.error??new Error("broker teardown was not confirmed");return}const d=await x(r),n=d?.retry.disposition==="fatal"?d:null;let l;try{l=await y(r,"cancel",{reason:"deadline_exceeded",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",deadline:{deadline_at:a,elapsed_ms:o-Date.parse(a)},...n===null?{}:{message:n.message,fault_kind:n.kind,error_class:h(n).class}}}})}catch(t){throw i({level:"error",event:"node_deadline.terminalize_failed",node_id:r,error:t,fields:{deadline_at:a}}),t}if(l.status!=="canceled"||l.outcome_reason!=="deadline_exceeded")return;const c=await f(r);if(!c.confirmed)throw c.error??new Error("broker teardown was not confirmed");try{await v(r,"notice.deadline","child-deadline-exceeded",{reason:"child-deadline-exceeded",child:r})}catch(t){i({level:"error",event:"node_deadline.notify_failed",node_id:r,error:t,fields:{deadline_at:a}})}try{i({level:"error",event:"node_deadline.expired",node_id:r,fields:{deadline_at:a,elapsed_ms:o-Date.parse(a)}})}catch{}}s(D,"expireNodeDeadline");export{z as NodeDeadlineReconciler,D as expireNodeDeadline};
+var m=Object.defineProperty;var s=(r,t)=>m(r,"name",{value:t,configurable:!0});import{dueNodeDeadlines as w,getNode as u}from"../../core/canvas/index.js";import{emitEvent as d}from"../../core/events/emit.js";import{isProfilePaused as p}from"../../core/profiles/manifest.js";import{operationIdContext as h}from"../../core/events/operation-id.js";import{errorClassFromFault as v}from"../../core/fault-classifier.js";import{publishNotice as x}from"../../core/runtime/close.js";import{readFault as y}from"../../core/canvas/recovery.js";import{requestBrokerTeardown as _}from"../../core/runtime/host.js";import{transition as D}from"../../core/runtime/lifecycle.js";import{withRowClaim as N}from"./row-claim.js";const O=6e4;class z{static{s(this,"NodeDeadlineReconciler")}async run(t){let i;try{i=await w(new Date(t).toISOString())}catch(e){d({level:"error",event:"node_deadline.scan_failed",error:e});return}for(const e of i)p(e.profile_id)||e.deadline_at===null||e.deadline_at===void 0||h.fresh(()=>{N("nodes",e.node_id,O,a=>k(e.node_id,e.deadline_at,t,a)).catch(a=>{d({level:"error",event:"node_deadline.expire_failed",node_id:e.node_id,error:a,fields:{deadline_at:e.deadline_at}})})})}}async function k(r,t,i,e){await e?.assertOwned();const a=await u(r);if(a===null||a.status==="done")return;if(a.status==="canceled"||a.status==="dead"){await e?.assertOwned();const o=await _(r);if(!o.confirmed)throw o.error??new Error("broker teardown was not confirmed");return}const c=await y(r),n=c?.retry.disposition==="fatal"?c:null;let l;await e?.assertOwned();try{l=await D(r,"cancel",{reason:"deadline_exceeded",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",deadline:{deadline_at:t,elapsed_ms:i-Date.parse(t)},...n===null?{}:{message:n.message,fault_kind:n.kind,error_class:v(n).class}}}})}catch(o){throw d({level:"error",event:"node_deadline.terminalize_failed",node_id:r,error:o,fields:{deadline_at:t}}),o}if(l.status!=="canceled"||l.outcome_reason!=="deadline_exceeded")return;await e?.assertOwned();const f=await _(r);if(!f.confirmed)throw f.error??new Error("broker teardown was not confirmed");await e?.assertOwned();try{await x(r,"notice.deadline","child-deadline-exceeded",{reason:"child-deadline-exceeded",child:r})}catch(o){d({level:"error",event:"node_deadline.notify_failed",node_id:r,error:o,fields:{deadline_at:t}})}try{d({level:"error",event:"node_deadline.expired",node_id:r,fields:{deadline_at:t,elapsed_ms:i-Date.parse(t)}})}catch{}}s(k,"expireNodeDeadline");export{z as NodeDeadlineReconciler,k as expireNodeDeadline};

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts

@@ -37,10 +37,9 @@ export declare function thawTier(row: NodeRow, pendingWake: boolean): ThawTier;
  *  splits the saturation counts). A candidate with no `frozen_at` has not waited at all and sorts last. */
 export declare function compareThaw(a: ThawEntry, b: ThawEntry): number;
 export declare class FreezeLaneReconciler {
-    private readonly relaunchState;
     /** End a stuck-delivery episode the tick decided is over — the node's profile
      * paused, or it has no wake-worthy mail left. */
-    endEpisode(nodeId: string): void;
+    endEpisode(nodeId: string): Promise<void>;
     run(now: number, ctx: FreezeLaneContext): Promise<FrozenCounts>;
     private enact;
     /** A parked resident owed wake mail: the message-delivery reopen (resident, latch check, `revive`

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js

@@ -1 +1 @@
-var E=Object.defineProperty;var d=(o,t)=>E(o,"name",{value:t,configurable:!0});import{clearFrozen as f,getNode as h,getRow as R}from"../../../core/canvas/index.js";import{fullName as S}from"../../../core/canvas/labels.js";import{isSafeNodeId as s}from"../../../core/canvas/paths.js";import{emitEvent as c}from"../../../core/events/emit.js";import{operationIdContext as g}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as z}from"../../../core/canvas/deliveries.js";import{publishNotice as A}from"../../../core/runtime/close.js";import{transition as l}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as w}from"../../../core/runtime/start-check.js";import{commitReopenResident as k}from"../../../core/runtime/reopen.js";import{reviveNode as p}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{mapAsync as M}from"../../../core/async-array.js";const m=6,C=10*6e4,N=2e4;function D(o,t){if(t)return 2;const a=o.launched_at;if(a==null)return 1;const e=o.last_settled_at;return e==null?2:Date.parse(e)>Date.parse(a)?1:2}d(D,"thawTier");function T(o,t){const a=o.row.frozen_at,e=t.row.frozen_at;return a==null?e==null?0:1:e==null?-1:Date.parse(a)-Date.parse(e)}d(T,"compareThaw");class G{static{d(this,"FreezeLaneReconciler")}relaunchState=new Map;endEpisode(t){this.relaunchState.delete(t)}async run(t,a){const e=await M(a.candidates,async n=>({...n,tier:D(n.row,await v(n.row.node_id))}));e.sort(T);const r={stalled:0,waitingOnMail:0};for(const n of e)await g.fresh(()=>this.enact(t,n,a.fleet))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(t,a,e){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(t,a.row):(await e.applyDeadRowPolicy(r),e.has(r)?"launched":(await R(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return c({level:"error",event:"node.lifecycle.enact_failed",...s(r)?{node_id:r}:{},error:n,fields:{...s(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(t){await k(t.node_id);try{return(await p(t.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(w(a))return await this.settleRefusedStart(t.node_id,a,"reopen");throw a}}async settleRefusedStart(t,a,e){return await l(t,"release"),await f(t),c({level:"warn",event:"node.start.refused",node_id:t,error:a,fields:{enact:e}}),"skipped"}async enactThaw(t){const a=(await L(t.node_id))?.interrupted===!0;if(!a&&t.lifecycle==="resident"&&(await h(t.node_id))?.pi_session_id!=null&&!await v(t.node_id))return await l(t.node_id,"release"),await f(t.node_id),"skipped";try{return(await p(t.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(e){if(w(e))return await this.settleRefusedStart(t.node_id,e,"thaw");const r=await h(t.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await l(t.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}),await f(t.node_id)),e}}async enactIdleRelease(t,a){const e=a.node_id,r=await z(e),n=this.relaunchState.get(e),u=n===void 0||n.acknowledgedCount!==r,i=u?0:n.attempts;if(i>=m)return this.relaunchState.delete(e),c({level:"warn",event:"broker.idle_release.exhausted",...s(e)?{node_id:e}:{},fields:{...s(e)?{}:{affected_node_id:e},attempts:i,max_attempts:m}}),await l(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:i}}}),await A(e,"notice.crash",`Child crashed \u2014 ${S(a)} (${e}) failed to relaunch after ${i} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,{reason:"child-crashed",child:e}),"skipped";const _=u?void 0:n?.lastAttempt,y=Math.min(N*2**i,C);return _!==void 0&&t-_<y?"skipped":(await p(e,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":(this.relaunchState.set(e,{acknowledgedCount:r,attempts:i+1,lastAttempt:t}),c({level:"info",event:"broker.revive.idle_release",...s(e)?{node_id:e}:{},attempt:i+1,fields:{...s(e)?{}:{affected_node_id:e},attempts:i+1,max_attempts:m}}),"launched")}}export{G as FreezeLaneReconciler,T as compareThaw,D as thawTier};
+var g=Object.defineProperty;var s=(i,e)=>g(i,"name",{value:e,configurable:!0});import{clearFrozen as p,getNode as h,getRow as y}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as k}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as S}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as D}from"../../../core/canvas/deliveries.js";import{publishNotice as N}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as E}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as R}from"./remote-wake.js";import{commitReopenResident as z}from"../../../core/runtime/reopen.js";import{reviveNode as m}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as T}from"../../../core/canvas/recovery.js";import{mapAsync as L}from"../../../core/async-array.js";const f=6,C=10*6e4,b=2e4;async function I(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(I,"readEpisode");async function M(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(M,"claimAttempt");async function O(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(O,"writeEpisode");function P(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s(P,"thawTier");function U(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(U,"compareThaw");class te{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await L(a.candidates,async n=>({...n,tier:P(n.row,await v(n.row.node_id))}));t.sort(U);const r={stalled:0,waitingOnMail:0};for(const n of t)await S.fresh(()=>this.enact(e,n,a.fleet))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await y(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await z(e.node_id);try{return(await m(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(E(a)||a instanceof R)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return await u(e,"release"),await p(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await T(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await h(e.node_id))?.pi_session_id!=null&&!await v(e.node_id))return await u(e.node_id,"release"),await p(e.node_id),"skipped";try{return(await m(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(E(t)||t instanceof R)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await h(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await p(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await D(t),n=await I(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=f)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:f}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await N(t,"notice.crash",`Child crashed \u2014 ${k(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const w=l?void 0:n.lastAttempt,A=Math.min(b*2**o,C);return w!==void 0&&e-w<A||!await M(t,n.attempts,o+1,r,e)?"skipped":(await m(t,{resume:!0,capacity:"freeze"})).outcome==="frozen"?(await O(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:f}}),"launched")}}export{te as FreezeLaneReconciler,U as compareThaw,P as thawTier};

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.d.ts

@@ -0,0 +1,31 @@
+import { type Execution } from '../../../core/canvas/executions.js';
+import type { NodeMeta } from '../../../core/canvas/index.js';
+import { type EnvironmentImage } from '../recipe-image.js';
+/** A `launching` execution running past this since `started_at` is stopped: the 30-minute recipe setup limit plus boot. */
+export declare const LAUNCH_DEADLINE_MS: number;
+/** A `launching` execution with no process is given this long before it is called lost, so the starter's own call is not raced. */
+export declare const LAUNCH_LOST_GRACE_MS = 120000;
+/** The release the control plane runs, which is the release a wake names. */
+export declare const acceptedRelease: () => string;
+/** The Blaxel environment the node's profile points at, or null when it runs on a machine. Reads rows only, so it is safe before any environment is registered. */
+export declare function blaxelEnvironmentOf(profileId: string | null): Promise<string | null>;
+/** Step 2: the `funding bill` check. A refusal writes `nodes.spend_refusal` and one refused `sandbox` row per node, limit and month,
+ *  then throws `402 cap_exhausted`; no execution row is written and no runner is called. A pass clears `spend_refusal`. */
+export declare function checkBilledWake(meta: NodeMeta): Promise<void>;
+/** A recipe environment with no built image: the wake is not started, the mail stays pending, and the image-built tick wakes it. */
+export declare class EnvironmentImagePending extends Error {
+    readonly environmentId: string;
+    readonly imageError: string | null;
+    constructor(environmentId: string, imageError: string | null);
+}
+/** D95: the image a new sandbox is created from. `pending` means the environment has a recipe and no image has built. */
+export declare function imageForWake(environmentId: string): Promise<EnvironmentImage>;
+/** Step 3: a `running` Blaxel execution is a resident broker, possibly frozen. On the accepted release it is woken and nothing else
+ *  is written. On another release it is asked to stop for the release first, so the thawed broker exits and the next claim starts
+ *  a new attempt on the accepted one. `lost` means no broker answered: the execution is ended and the caller starts a new attempt. */
+export declare function wakeResident(execution: Execution): Promise<'woken' | 'lost'>;
+/** A `running` remote execution is live while its runner reports the broker process running, frozen or not. When the runner
+ *  reports it gone (or ended non-zero), that exact execution is ended; a runner that cannot tell leaves it as it is. */
+export declare function endRemoteExecutionIfGone(execution: Execution): Promise<boolean>;
+/** The recovery table for a `launching` Blaxel execution found by a claim after its starter is gone. */
+export declare function recoverLaunching(nodeId: string, now: number): Promise<void>;

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.js

@@ -0,0 +1 @@
+var p=Object.defineProperty;var a=(e,t)=>p(e,"name",{value:t,configurable:!0});import{liveExecutionForNode as m,endExecution as d,recordExecutionAwake as w,endLaunchingExecution as s,requestExecutionStop as h}from"../../../core/canvas/executions.js";import{openDb as c}from"../../../core/canvas/db.js";import{emitEvent as u}from"../../../core/events/emit.js";import{findRefusal as _,refusalError as x,refusedMark as E,writeCharge as g}from"../../../core/ledger/index.js";import{transition as f}from"../../../core/runtime/lifecycle.js";import{runnerForExecution as l}from"../../../core/runtime/runner/index.js";import{startCandidateOf as b}from"../../../core/runtime/start-check.js";import{settleSandboxBeforeWake as v}from"../ledger-sandbox.js";import{crtrVersion as k}from"../../../shared/crtr-version.js";import{resolveEnvironmentImage as I}from"../recipe-image.js";const S=45*6e4,y=12e4,N=a(()=>k(),"acceptedRelease");async function j(e){return e===null?null:(await(await c()).prepare("SELECT e.object AS object FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ? AND e.kind = 'blaxel'").get(e))?.object??null}a(j,"blaxelEnvironmentOf");async function F(e){const t=await b(e);if(t===null)return;const n={scope:"call",payer:t.payer,funding:"bill",sub:t.sub,grantee:t.grantee,provider:"blaxel"},r=await _(n),o=await c();if(r===null){await o.prepare("UPDATE nodes SET spend_refusal = NULL WHERE node_id = ? AND spend_refusal IS NOT NULL").run(e.node_id);return}throw await o.prepare("UPDATE nodes SET spend_refusal = ? WHERE node_id = ?").run(JSON.stringify({limit:r.limit.id,usd:r.usd,reset_at:r.reset_at,at:new Date().toISOString()}),e.node_id),await g({id:`refused:sandbox:${e.node_id}:${r.limit.id}:${r.month}`,kind:"sandbox",what:"wake",usd:null,payer:n.payer,funding:"bill",sub:n.sub,grantee:n.grantee,node:e.node_id,run:e.run_id??null,provider:"blaxel",refused:E(r)}),x(r)}a(F,"checkBilledWake");class M extends Error{static{a(this,"EnvironmentImagePending")}environmentId;imageError;constructor(t,n){super(`environment ${t} has no built image yet${n===null?"":` (${n})`}; the wake waits for the build.`),this.environmentId=t,this.imageError=n,this.name="EnvironmentImagePending"}}async function P(e){return await I(e)}a(P,"imageForWake");async function B(e){return e.release!==N()&&await h(e.nodeId,{afterTurn:!1,reason:"release"}),await l(e.runner).wake(e)==="no_broker"?(await d(e.executionId,{reason:"lost"}),"lost"):(await v(e.executionId,Date.now()),await w(e.executionId),"woken")}a(B,"wakeResident");async function G(e){const t=await l(e.runner).status(e);if(t.state!=="gone"&&t.state!=="failed")return!1;const n=await d(e.executionId,{reason:"lost"});return n&&u({level:"warn",event:"execution.lost",node_id:e.nodeId}),n}a(G,"endRemoteExecutionIfGone");async function J(e,t){const n=await m(e);if(n===null||n.state!=="launching"||n.runner==="process")return;const r=l(n.runner),o=t-Date.parse(n.startedAt),i=await r.status(n);if(i.state!=="unknown"){if(i.state==="running"){if(o<=S||!await s(n.executionId,{reason:"launch_timeout"}))return;await r.stop(n,{reason:"launch_timeout"}),await f(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:"the sandbox launch did not report within the launch deadline"}}}),u({level:"warn",event:"execution.launch_timeout",node_id:e});return}if(i.state==="failed"){if(!await s(n.executionId,{reason:"launch_failed"}))return;await f(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:`the sandbox start script failed: ${i.reason}`}}});return}if(o>y){if(!await s(n.executionId,{reason:"launch_lost"}))return;u({level:"warn",event:"execution.launch_lost",node_id:e})}}}a(J,"recoverLaunching");export{M as EnvironmentImagePending,S as LAUNCH_DEADLINE_MS,y as LAUNCH_LOST_GRACE_MS,N as acceptedRelease,j as blaxelEnvironmentOf,F as checkBilledWake,G as endRemoteExecutionIfGone,P as imageForWake,J as recoverLaunching,B as wakeResident};

package/dist/daemon/reconcilers/node-lifecycle/requested-stops.d.ts

@@ -0,0 +1,5 @@
+/** A stop request this old was queued by a process that is no longer finishing it: the requesting process runs its own
+ *  teardown within seconds (after its response settles). */
+export declare const ORPHANED_STOP_AGE_MS = 30000;
+/** Enact every stop whose requesting process is gone, from the execution row's `stop_*` columns. */
+export declare function enactRequestedStops(now: number): Promise<void>;

package/dist/daemon/reconcilers/node-lifecycle/requested-stops.js

@@ -0,0 +1,2 @@
+var d=Object.defineProperty;var t=(e,r)=>d(e,"name",{value:r,configurable:!0});import{endLiveExecution as s,getNode as p}from"../../../core/canvas/index.js";import{executionsWithStopRequestedBefore as i}from"../../../core/canvas/executions.js";import{openDb as f}from"../../../core/canvas/db.js";import{emitEvent as o}from"../../../core/events/emit.js";import{runnerForExecution as u}from"../../../core/runtime/runner/index.js";const l=3e4;async function m(e,r){return r==null?!1:(await(await f()).prepare(`SELECT type FROM run_events WHERE node_id = ? AND type IN ('node.turn.started', 'node.turn.completed')
+    ORDER BY sequence_number DESC LIMIT 1`).get(e))?.type==="node.turn.started"}t(m,"turnIsOpen");async function c(e){const r=await p(e.nodeId);if(r===null||e.stopAfterTurn&&await m(e.nodeId,r.run_id))return;const a=await u(e.runner).stop(e,{reason:e.stopReason??"stopped"});if(!a.confirmed){o({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:a.error});return}await s(e.nodeId,{reason:e.stopReason??"stopped"}),o({level:"info",event:"broker.stop.enacted_from_row",node_id:e.nodeId,fields:{reason:e.stopReason,after_turn:e.stopAfterTurn}})}t(c,"enact");async function S(e){for(const r of await i(new Date(e-l).toISOString()))try{await c(r)}catch(n){o({level:"error",event:"broker.stop.enact_failed",node_id:r.nodeId,error:n})}}t(S,"enactRequestedStops");export{l as ORPHANED_STOP_AGE_MS,S as enactRequestedStops};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var y=Object.defineProperty;var f=(r,t)=>y(r,"name",{value:t,configurable:!0});import{clearFrozen as p,endLiveExecution as o,getNode as P}from"../../../core/canvas/index.js";import{isSafeNodeId as k}from"../../../core/canvas/paths.js";import{recordedPidLiveness as u}from"../../../core/canvas/pid.js";import{emitEvent as c}from"../../../core/events/emit.js";import{operationIdContext as v}from"../../../core/events/operation-id.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{isProfilePaused as l}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as m}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as h}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as R,isParkedResident as z}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as I}from"../../fleet.js";import{applyParkEvent as E,hasPendingWake as M,parkEvent as g}from"../live-obligation.js";import{FreezeLaneReconciler as N}from"./freeze-lane.js";import{isInBackoff as _}from"./respawn-policy.js";import{handleTerminatingRow as b}from"./terminating.js";import{wantsExecution as C}from"./wants-execution.js";function D(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}f(D,"isTerminal");const F=12e4;function T(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<F}f(T,"isLaunchInFlight");class ${static{f(this,"NodeLifecycleReconciler")}freezeLane=new N;async run(t,e){const a=[];for(const n of e.rows)await v.fresh(async()=>{try{await this.classifyRow(t,n,e,a)}catch(i){const s=n.node_id;c({level:"error",event:"node.lifecycle.classify_failed",...k(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(D(e)){if(b(e,a.fleet.has(i)))return;if(z(e)&&!a.fleet.has(i)&&!l(e.profile_id)&&!m(i)&&!_(e,t)&&await h(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&u(e.pi_pid,e.pi_pid_identity)==="dead"&&await o(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null?await p(i):n.push({row:e,enact:"thaw"}));return}if(a.fleet.has(i)){l(e.profile_id)?a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}if(l(e.profile_id)){this.freezeLane.endEpisode(i);return}if(m(i))return;const s=e.pi_pid!=null&&u(e.pi_pid,e.pi_pid_identity)==="dead";if(s&&!a.fleet.hasProfilePauseInterruption(i)&&await o(i,{reason:"observed-dead"}),!await C(e)){await this.thawMark(e),await this.settleParked(e);return}if(_(e,t))return;if(e.status==="idle"&&R(e)&&await h(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(a.fleet.isClaimed(i)||T(e,t))return;c({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!s)return;const d=I(await P(i),(await L(i))?.interrupted===!0,await M(i));if(d==="respawn-fresh"||d==="respawn-cycle"||d==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await p(t.node_id)}async settleParked(t){if(t.status!=="idle")return;this.freezeLane.endEpisode(t.node_id);const e=await g(t.node_id,t.lifecycle);e!=="release"&&await E(t.node_id,e)}}export{F as LAUNCH_GRACE_MS,$ as NodeLifecycleReconciler,T as isLaunchInFlight};
+var k=Object.defineProperty;var l=(r,t)=>k(r,"name",{value:t,configurable:!0});import{clearFrozen as m,endLiveExecution as h,getNode as R}from"../../../core/canvas/index.js";import{isSafeNodeId as b}from"../../../core/canvas/paths.js";import{recordedPidLiveness as _}from"../../../core/canvas/pid.js";import{hasObservedExit as E,hasAbsorbedProfilePauseStop as L}from"../../../core/canvas/executions.js";import{emitEvent as y}from"../../../core/events/emit.js";import{operationIdContext as z}from"../../../core/events/operation-id.js";import{readInterruptedTurn as I}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as g,isParkedResident as w}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as M}from"../../fleet.js";import{applyParkEvent as N,hasPendingWake as F,parkEvent as x}from"../live-obligation.js";import{FreezeLaneReconciler as A}from"./freeze-lane.js";import{isInBackoff as P}from"./respawn-policy.js";import{handleTerminatingRow as C}from"./terminating.js";import{wantsExecution as D}from"./wants-execution.js";import{enactRequestedStops as S}from"./requested-stops.js";import{endRemoteExecutionIfGone as T,recoverLaunching as G,wakeResident as H}from"./remote-wake.js";import{liveExecutionForNode as U}from"../../../core/canvas/executions.js";function W(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}l(W,"isTerminal");const q=12e4;function B(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<q}l(B,"isLaunchInFlight");class pe{static{l(this,"NodeLifecycleReconciler")}freezeLane=new A;async run(t,e){const a=[];await S(t);for(const n of e.rows)await z.fresh(async()=>{try{await this.classifyRow(t,n,e,a)}catch(i){const s=n.node_id;y({level:"error",event:"node.lifecycle.classify_failed",...b(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(W(e)){if(C(e,await a.fleet.has(i)))return;if(w(e)&&!await a.fleet.has(i)&&!d(e.profile_id)&&!await v(i)&&!P(e,t)&&await p(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead"&&await h(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await m(i):n.push({row:e,enact:"thaw"}));return}if(await a.fleet.has(i)){const f=await U(i);if(f?.runner==="blaxel"&&f.state==="running"&&!d(e.profile_id)&&await p(i)){if(await H(f)==="lost")return}else f?.runner==="blaxel"&&f.state==="running"&&await T(f);if(await a.fleet.has(i)){d(e.profile_id)?await a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}}if(await G(i,t),d(e.profile_id)){await this.freezeLane.endEpisode(i);return}if(await v(i))return;const s=e.pi_pid==null&&await E(i),u=e.pi_pid==null&&!s&&await L(i),c=s||u||e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead";if(c&&!await a.fleet.hasProfilePauseInterruption(i)&&await h(i,{reason:"observed-dead"}),!await D(e)){await this.thawMark(e),await this.settleParked(e);return}if(P(e,t))return;if(e.status==="idle"&&g(e)&&await p(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!s||e.frozen_at!=null)&&!u){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(await a.fleet.isClaimed(i)||B(e,t))return;y({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!c)return;const o=M(await R(i),(await I(i))?.interrupted===!0,await F(i));if(o==="respawn-fresh"||o==="respawn-cycle"||o==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await m(t.node_id)}async settleParked(t){if(t.status!=="idle")return;await this.freezeLane.endEpisode(t.node_id);const e=await x(t.node_id,t.lifecycle);e!=="release"&&await N(t.node_id,e)}}export{q as LAUNCH_GRACE_MS,pe as NodeLifecycleReconciler,B as isLaunchInFlight};

package/dist/daemon/reconcilers/pending-review-submit.js

@@ -1 +1 @@
-var f=Object.defineProperty;var n=(t,e)=>f(t,"name",{value:e,configurable:!0});import{emitEvent as s}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{listSubmitRequestedRequests as l}from"../../core/human/requests.js";import{completeRequestedSubmit as m}from"../review/finish.js";class b{static{n(this,"PendingReviewSubmitReconciler")}async run(e,i){if(!i.lifecycle.acceptsDetachedWork())return;let o;try{o=await l()}catch(r){c.fresh(()=>{s({level:"error",event:"review.deferred_submit.scan_failed",error:r})});return}for(const r of o)i.lifecycle.registerDetached(m(r.request_id,{atCompanionTurnEnd:!1}).then(()=>{},d=>{c.fresh(()=>{s({level:"error",event:"review.deferred_submit.failed",error:d})})}))}}export{b as PendingReviewSubmitReconciler};
+var c=Object.defineProperty;var s=(i,r)=>c(i,"name",{value:r,configurable:!0});import{emitEvent as a}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{listSubmitRequestedRequests as d}from"../../core/human/requests.js";import{completeRequestedSubmit as f}from"../review/finish.js";import{withRowClaim as l}from"./row-claim.js";const u=6e4;class R{static{s(this,"PendingReviewSubmitReconciler")}async run(r,o){if(!o.lifecycle.acceptsDetachedWork())return;let n;try{n=await d()}catch(e){m.fresh(()=>{a({level:"error",event:"review.deferred_submit.scan_failed",error:e})});return}for(const e of n)o.lifecycle.registerDetached(l("human_requests",e.request_id,u,async t=>{await t.assertOwned(),await f(e.request_id,{atCompanionTurnEnd:!1})}).then(()=>{},t=>{m.fresh(()=>{a({level:"error",event:"review.deferred_submit.failed",error:t})})}))}}export{R as PendingReviewSubmitReconciler};

package/dist/daemon/reconcilers/recipe-image.d.ts

@@ -0,0 +1,49 @@
+import { type Recipe } from '../../core/runtime/runner/blaxel/image-build.js';
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+/** How long a claim holds without renewal. The holder renews at a third of it. */
+export declare const LEASE_MS: number;
+/** After a failed build the same revision is not tried again for this long. */
+export declare const RETRY_AFTER_FAILURE_MS: number;
+export type EnvironmentImage = {
+    state: 'base';
+} | {
+    state: 'image';
+    ref: string;
+    revision: number;
+} | {
+    state: 'pending';
+    error: string | null;
+};
+/** What a wake of a node in `environmentId` creates its sandbox from (D25, D95). `base`: the environment has no recipe content (never set, or cleared). `pending`: it has one and no image has ever built, so the wake must not start a runner call. */
+export declare function resolveEnvironmentImage(environmentId: string): Promise<EnvironmentImage>;
+/** Starts a supervision tick when an environment's image is built, so the nodes waiting on it wake now instead of at the next beat. Returns the unsubscribe. */
+export declare function subscribeImageBuilt(requestTick: (environmentId: string) => void): Promise<() => void>;
+export interface RecipeBuildInput {
+    environmentId: string;
+    revision: number;
+    recipe: Recipe;
+    person: string;
+}
+export interface RecipeImageDeps {
+    /** Builds and pushes the image; returns its reference. The default clones with a G-1 token and uses Blaxel's build API. */
+    build?: (input: RecipeBuildInput) => Promise<{
+        ref: string;
+    }>;
+    now?: () => number;
+}
+export declare class RecipeImageReconciler {
+    private readonly deps;
+    constructor(deps?: RecipeImageDeps);
+    /** One pass: claim every environment with an unbuilt revision and build it detached from the tick. */
+    run(nowMs: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): Promise<void>;
+    private claim;
+    private buildClaimed;
+    private recordFailure;
+}
+/** Nodes of profiles that run in `environmentId` with wake mail unseen and no execution: the ones a missing image keeps dormant. */
+export declare function nodesWaitingOnImage(environmentId: string): Promise<Array<{
+    node_id: string;
+    run_id: string;
+}>>;