@north-light/cli 0.3.580 → 0.3.582

package/dist/daemon/manage.js

@@ -1,3 +1,3 @@
-var Y=Object.defineProperty;var o=(e,t)=>Y(e,"name",{value:t,configurable:!0});import{spawn as W,execFileSync as X}from"node:child_process";import{createRequire as q}from"node:module";import{dirname as b,join as E}from"node:path";import{fileURLToPath as V,pathToFileURL as k}from"node:url";import{existsSync as B,lstatSync as Q,mkdirSync as Z,openSync as H,closeSync as U,fstatSync as j,readSync as ee}from"node:fs";import{crtrHome as g}from"../core/canvas/paths.js";import{listProcessTable as te}from"../core/canvas/pid.js";import{bindDaemonEventSource as re,eventSource as ne}from"../core/events/source.js";import{hostExecPath as O}from"../core/runtime/branded-host.js";import{isDaemonRunning as oe,readPidfile as D,isPidAlive as p}from"./pidfile.js";import{DAEMON_STARTUP_WINDOW_MS as K}from"./startup-policy.js";import{exclusiveLockOwnerPid as ae}from"../core/exclusive-lock.js";import{onDiskMigrationLockPath as ie}from"../core/canvas/paths.js";import{envNoDaemonAutostart as se}from"../shared/env.js";import{awaitJoinLockRelease as ce,joinInProgress as ue,joinInProgressError as le,joinLockHolder as de}from"../core/directory/join-lock.js";import{waitForDaemonAvailability as pe}from"../api/index.js";import{localClient as fe}from"../api/node-transport.js";const me=["CRTR_NODE_ID","CRTR_KIND","CRTR_MODE","CRTR_LIFECYCLE","CRTR_NODE_CWD","CRTR_CONTEXT_DIR","CRTR_CYCLES","CRTR_PROFILE_ID","CRTR_PARENT_NODE_ID","CRTR_FRONT_DOOR","TMUX","TMUX_PANE","CRTR_BROKER_ENGINE","CRTR_PI_BINARY","NODE_OPTIONS"];function M(){const e={...process.env};for(const t of me)delete e[t];return e}o(M,"sanitizedDaemonEnv");function he(){for(let e=b(V(import.meta.url));;e=b(e)){if(B(E(e,"bin","runtime-generation-manifest.mjs")))return e;if(b(e)===e)throw new Error(`cannot locate the crouter generation root from ${import.meta.url}`)}}o(he,"generationRoot");async function A(){const e=he(),t=k(E(e,"bin","runtime-generation-manifest.mjs")).href,{runtimeHome:r}=await import(t);try{Q(E(r(),"selected"))}catch(i){if(i.code!=="ENOENT")throw i;return["--conditions","crtr-src","--import",k(q(import.meta.url).resolve("tsx/esm")).href,E(e,"src","daemon","crtrd-cli.ts")]}const n=k(E(e,"bin","runtime-selector.mjs")).href,{tryResolveRuntime:a}=await import(n),c=a("daemon");if(!c.ok)throw new Error(`cannot resolve the selected runtime generation's daemon entry: ${c.error}`);return[c.runtime.entryPath]}o(A,"resolveDaemonLaunch");function we(e,t){return e.includes(` --canvas-home ${t} `)||e.endsWith(` --canvas-home ${t}`)}o(we,"daemonCommandDeclaresCanvasHome");function _e(){const e=te();if(e===null)return[];const t=g(),r=[];for(const{pid:n,command:a}of e)a.includes("daemon/crtrd")&&we(a,t)&&n!==process.pid&&r.push(n);return r}o(_e,"findDaemonPids");function Ee(e){const t=_e().filter(n=>n!==e);for(const n of t)try{process.kill(n,"SIGTERM")}catch{}if(t.length===0)return t;const r=Date.now()+2e3;for(;Date.now()<r&&t.some(n=>p(n));)X("sleep",["0.1"]);for(const n of t)if(p(n))try{process.kill(n,"SIGKILL")}catch{}return t}o(Ee,"sweepStrayDaemons");async function qe(){const e=process.execve;if(e===void 0)throw new Error(`this Node.js ${process.version} runtime cannot replace PID 1 with the successor daemon`);const t=O(),r=await A();return e(t,[t,...r,"--canvas-home",g()],M())}o(qe,"execDaemon");async function Ve(){const e=await A(),t=W(O(),[...e,"export-sqlite"],{stdio:["ignore","pipe","pipe"],env:M()});let r="",n="";t.stdout.setEncoding("utf8").on("data",i=>{r+=i}),t.stderr.setEncoding("utf8").on("data",i=>{n+=i});const a=await new Promise((i,l)=>{t.once("error",l),t.once("close",f=>i(f))});if(a!==0)throw new Error(n.trim()||`the export process exited with code ${a}`);const c=r.trim().split(`
-`).at(-1)??"";return JSON.parse(c)}o(Ve,"runDaemonExportSqlite");const De=3*6e4,G=2e4,I=10,N=1e3;async function P(e=N){const t=Math.min(e,N);return fe({autostart:!1,timeoutMs:t}).probeHealthz(t)}o(P,"probeDaemonHealth");function ye(e){const t=e instanceof Error?e.message:String(e);if(typeof e!="object"||e===null)return t;const r=e.code,n=e.cause,a=typeof r=="string"?r:n?.code;return typeof a=="string"&&!t.includes(a)?`${t}: ${a}`:t}o(ye,"probeErrorMessage");async function z(e=N){const t=await P(e);if(t.startup_phase==="blocked"){const r=D()??process.pid;throw new R(r,t.startup_blocked??"daemon startup is blocked")}if(!t.ok)throw new Error(t.startup_blocked??"crtrd health check reported not ready")}o(z,"probeDaemonServing");async function Qe(){try{return await z(),!0}catch{return!1}}o(Qe,"isDaemonServing");function L(e){return new Promise(t=>{setTimeout(t,e)})}o(L,"sleepMs");class $ extends Error{static{o(this,"DaemonExitedBeforeReadyError")}stderrTail;constructor(t,r){super(r===""?t:`${t}
-${r}`),this.name="DaemonExitedBeforeReadyError",this.stderrTail=r}}class R extends Error{static{o(this,"DaemonStartupBlockedError")}constructor(t,r){super(`daemon ${t} is running but normal startup is blocked: ${r}`),this.name="DaemonStartupBlockedError"}}class C extends Error{static{o(this,"DaemonNotPreparedError")}constructor(t,r){super(r===void 0?`daemon ${t} is serving but did not report the prepared startup phase`:`daemon ${t} reported startup phase ${r}, not prepared`),this.name="DaemonNotPreparedError"}}async function xe(e,t=K,r={},n="ready"){const a=r.readPidfile??D,c=r.isPidAlive??p,i=r.childExited,l=r.isDaemonServing,f=r.probeDaemonServing??(async w=>{if(l!==void 0){if(await l())return;throw new Error("crtrd health check reported not ready")}await z(w)}),m=r.probeDaemonHealth??P,y=r.sleepMs??L,x=r.now??Date.now;let s=null,u=null;return await pe({windowMs:t,pollIntervalMs:I,now:x,sleep:y,retry:o(w=>w!==u,"retry"),probe:o(async w=>{const h=a();if(h===null||!c(h)){const d=i?.();throw d!=null&&(d.signal!==null?u=new $(`daemon ${e} exited before becoming ready (by ${d.signal})`,r.childStderrTail?.()??""):d.code!==0&&(u=new $(`daemon ${e} exited before becoming ready (with exit code ${d.code??"?"})`,r.childStderrTail?.()??"")),u!==null)?u:new Error(`daemon ${e} has no live pidfile owner`)}if(n==="ready"){try{await f(w)}catch(d){throw d instanceof R&&(u=d),d}s=h;return}const _=await m(w);if(_.startup_phase==="prepared"){s=h;return}throw _.startup_phase==="blocked"?(u=new R(h,_.startup_blocked??"on-disk migration is blocked"),u):_.startup_phase==="ready"?(u=new C(h,_.startup_phase),u):new Error(`daemon ${h} is ${_.startup_phase??"not reporting a startup phase"}; waiting for prepared`)},"probe")}),s===e?null:s}o(xe,"verifyDaemonStartup");async function ge(e,t=G,r={}){const n=r.readPidfile??D,a=r.isPidAlive??p,c=r.sleepMs??L,i=r.now??Date.now,l=i()+t;for(;i()<=l;){if(!a(e)&&n()!==e)return;await c(I)}throw new Error(`daemon ${e} did not exit and release its pidfile within ${t}ms`)}o(ge,"waitForDaemonExit");const Se=2e3,Re=5e3;async function J(e,t=G,r={}){const n=r.isPidAlive??p,a=r.sleepMs??L,c=r.now??Date.now;try{process.kill(e,"SIGTERM")}catch{}try{return await ge(e,t,r),"terminated"}catch{}try{process.kill(e,"SIGKILL")}catch{}const i=c()+Se;for(;c()<=i;){if(!n(e))return"killed";await a(I)}throw new Error(`daemon ${e} survived both SIGTERM and SIGKILL`)}o(J,"stopDaemonProcess");async function Ze(){const e=D(),t=e!==null&&p(e),r=t&&await J(e)==="killed",n=Ee(null);return{stopped:t||n.length>0,...t?{pid:e}:{},strays_killed:n,force_killed:r}}o(Ze,"stopAllDaemons");function ve(e,t,r=2048){let n;try{n=H(e,"r");const{size:a}=j(n),c=Math.max(t,a-r),i=a-c;if(i<=0)return"";const l=Buffer.alloc(i);return ee(n,l,0,i,c),l.toString("utf8").trim()}catch{return""}finally{if(n!==void 0)try{U(n)}catch{}}}o(ve,"readLogTailFrom");async function Te(e={}){if(process.platform==="linux"&&B("/run/crtr/launcher.sock")&&!p(D()??-1))throw new Error("crtrd is managed by the root launcher; it cannot self-daemonize");const t=e.targetPhase??"ready",r=D();if(r!==null&&p(r))try{const s=await P();if(t==="prepared"){if(s.startup_phase==="prepared")return{started:!1,existing_pid:r,startup_phase:s.startup_phase};throw new C(r,s.startup_phase)}return s.ok?{started:!1,existing_pid:r,...s.startup_phase===void 0?{}:{startup_phase:s.startup_phase}}:{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:s.startup_blocked??"crtrd health check reported not ready",...s.startup_phase===void 0?{}:{startup_phase:s.startup_phase}}}catch(s){if(s instanceof C)throw s;return{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:ye(s)}}Z(g(),{recursive:!0});const n=await A();e.waitForJoinLock===!0&&await ce(De);const a=de();if(a!==null&&a!==process.pid&&e.ignoreJoinLock!==!0)throw le(a);const c=E(g(),"crtrd.err"),i=H(c,"a"),l=(()=>{try{return j(i).size}catch{return 0}})();ne()===void 0&&re();const f=W(O(),[...n,"--canvas-home",g(),...t==="prepared"?["--prepare"]:[]],{detached:!0,stdio:["ignore",i,i],env:M()});U(i);const m=f.pid;if(m===void 0)throw new Error("daemon spawn did not return a pid");let y=null;f.once("exit",(s,u)=>{y={code:s,signal:u}}),f.unref();let x;try{x=await xe(m,K,{childExited:o(()=>y,"childExited"),childStderrTail:o(()=>ve(c,l),"childStderrTail")},t)}catch(s){const u=ae(ie())===m;if(y===null&&!u&&!(s instanceof R))try{await J(m,Re)}catch{}throw s}return x!==null?{started:!1,existing_pid:x,startup_phase:t}:{started:!0,pid:m,startup_phase:t}}o(Te,"spawnDaemon");function be(){return se()||process.argv.includes("--no-autostart")}o(be,"autostartDisabled");const ke=5e3,Oe=6e4;let F=!1,S,v=0,T=0;function Me(e){return Math.min(ke*2**(e-1),Oe)}o(Me,"autostartBackoffMs");function Ae(){S=void 0,v=0,T=0}o(Ae,"resetDaemonAutostart");function et(){return S}o(et,"daemonAutostartStartupFailure");function tt(e={}){const t=e.isDaemonRunning??oe,r=e.spawnDaemon??Te,n=e.now??Date.now;if(!be()&&!ue()&&!F&&!(n()<T)){if(t()){Ae();return}S=void 0,F=!0,r().then(()=>{S=void 0,v=0,T=0},a=>{a instanceof $&&(S=a.message),v+=1,T=n()+Me(v)}).finally(()=>{F=!1})}}o(tt,"ensureDaemon");export{me as DAEMON_ENV_STRIP_KEYS,$ as DaemonExitedBeforeReadyError,C as DaemonNotPreparedError,R as DaemonStartupBlockedError,be as autostartDisabled,et as daemonAutostartStartupFailure,we as daemonCommandDeclaresCanvasHome,tt as ensureDaemon,qe as execDaemon,_e as findDaemonPids,Qe as isDaemonServing,P as probeDaemonHealth,z as probeDaemonServing,ye as probeErrorMessage,Ae as resetDaemonAutostart,Ve as runDaemonExportSqlite,M as sanitizedDaemonEnv,Te as spawnDaemon,Ze as stopAllDaemons,J as stopDaemonProcess,Ee as sweepStrayDaemons,xe as verifyDaemonStartup,ge as waitForDaemonExit};
+var X=Object.defineProperty;var o=(e,t)=>X(e,"name",{value:t,configurable:!0});import{spawn as B,execFileSync as q}from"node:child_process";import{createRequire as V}from"node:module";import{dirname as k,join as D}from"node:path";import{fileURLToPath as Q,pathToFileURL as I}from"node:url";import{existsSync as H,lstatSync as Z,mkdirSync as ee,openSync as U,closeSync as j,fstatSync as K,readSync as te}from"node:fs";import{crtrHome as g}from"../core/canvas/paths.js";import{listProcessTable as re}from"../core/canvas/pid.js";import{bindDaemonEventSource as ne,eventSource as oe}from"../core/events/source.js";import{hostExecPath as O}from"../core/runtime/branded-host.js";import{isDaemonRunning as ae,readPidfile as E,isPidAlive as f}from"./pidfile.js";import{DAEMON_STARTUP_WINDOW_MS as G}from"./startup-policy.js";import{exclusiveLockOwnerPid as ie}from"../core/exclusive-lock.js";import{onDiskMigrationLockPath as se}from"../core/canvas/paths.js";import{envNoDaemonAutostart as ce}from"../shared/env.js";import{awaitJoinLockRelease as ue,joinInProgress as le,joinInProgressError as de,joinLockHolder as pe}from"../core/directory/join-lock.js";import{waitForDaemonAvailability as fe}from"../api/index.js";import{localClient as me}from"../api/node-transport.js";const he=["CRTR_NODE_ID","CRTR_KIND","CRTR_MODE","CRTR_LIFECYCLE","CRTR_NODE_CWD","CRTR_CONTEXT_DIR","CRTR_CYCLES","CRTR_PROFILE_ID","CRTR_PARENT_NODE_ID","CRTR_FRONT_DOOR","TMUX","TMUX_PANE","CRTR_BROKER_ENGINE","CRTR_PI_BINARY","NODE_OPTIONS"];function M(){const e={...process.env};for(const t of he)delete e[t];return e}o(M,"sanitizedDaemonEnv");function we(){for(let e=k(Q(import.meta.url));;e=k(e)){if(H(D(e,"bin","runtime-generation-manifest.mjs")))return e;if(k(e)===e)throw new Error(`cannot locate the crouter generation root from ${import.meta.url}`)}}o(we,"generationRoot");async function A(){const e=we(),t=I(D(e,"bin","runtime-generation-manifest.mjs")).href,{runtimeHome:r}=await import(t);try{Z(D(r(),"selected"))}catch(s){if(s.code!=="ENOENT")throw s;return["--conditions","crtr-src","--import",I(V(import.meta.url).resolve("tsx/esm")).href,D(e,"src","daemon","crtrd-cli.ts")]}const n=I(D(e,"bin","runtime-selector.mjs")).href,{tryResolveRuntime:a}=await import(n),c=a("daemon");if(!c.ok)throw new Error(`cannot resolve the selected runtime generation's daemon entry: ${c.error}`);return[c.runtime.entryPath]}o(A,"resolveDaemonLaunch");function _e(e,t){return e.includes(` --canvas-home ${t} `)||e.endsWith(` --canvas-home ${t}`)}o(_e,"daemonCommandDeclaresCanvasHome");function De(){const e=re();if(e===null)return[];const t=g(),r=[];for(const{pid:n,command:a}of e)a.includes("daemon/crtrd")&&_e(a,t)&&n!==process.pid&&r.push(n);return r}o(De,"findDaemonPids");function Ee(e){const t=De().filter(n=>n!==e);for(const n of t)try{process.kill(n,"SIGTERM")}catch{}if(t.length===0)return t;const r=Date.now()+2e3;for(;Date.now()<r&&t.some(n=>f(n));)q("sleep",["0.1"]);for(const n of t)if(f(n))try{process.kill(n,"SIGKILL")}catch{}return t}o(Ee,"sweepStrayDaemons");async function Ve(){const e=process.execve;if(e===void 0)throw new Error(`this Node.js ${process.version} runtime cannot replace PID 1 with the successor daemon`);const t=O(),r=await A();return e(t,[t,...r,"--canvas-home",g()],M())}o(Ve,"execDaemon");async function R(e){const t=await A(),r=B(O(),[...t,...e],{stdio:["ignore","pipe","pipe"],env:M()});let n="",a="";r.stdout.setEncoding("utf8").on("data",u=>{n+=u}),r.stderr.setEncoding("utf8").on("data",u=>{a+=u});const c=await new Promise((u,m)=>{r.once("error",m),r.once("close",p=>u(p))});if(c!==0)throw new Error(a.trim()||`the ${e[0]} process exited with code ${c}`);const s=n.trim().split(`
+`).at(-1)??"";return JSON.parse(s)}o(R,"runDaemonEntry");function Qe(){return R(["export-sqlite"])}o(Qe,"runDaemonExportSqlite");function Ze(e,t){return R(["import-dump",e,t])}o(Ze,"runDaemonImportDump");function et(e){return R(["import-dump",e,"bodies"])}o(et,"runDaemonBodiesDump");function tt(){return R(["import-dump","","install-id"])}o(tt,"runDaemonInstallId");const xe=3*6e4,z=2e4,N=10,P=1e3;async function L(e=P){const t=Math.min(e,P);return me({autostart:!1,timeoutMs:t}).probeHealthz(t)}o(L,"probeDaemonHealth");function ye(e){const t=e instanceof Error?e.message:String(e);if(typeof e!="object"||e===null)return t;const r=e.code,n=e.cause,a=typeof r=="string"?r:n?.code;return typeof a=="string"&&!t.includes(a)?`${t}: ${a}`:t}o(ye,"probeErrorMessage");async function J(e=P){const t=await L(e);if(t.startup_phase==="blocked"){const r=E()??process.pid;throw new v(r,t.startup_blocked??"daemon startup is blocked")}if(!t.ok)throw new Error(t.startup_blocked??"crtrd health check reported not ready")}o(J,"probeDaemonServing");async function rt(){try{return await J(),!0}catch{return!1}}o(rt,"isDaemonServing");function $(e){return new Promise(t=>{setTimeout(t,e)})}o($,"sleepMs");class C extends Error{static{o(this,"DaemonExitedBeforeReadyError")}stderrTail;constructor(t,r){super(r===""?t:`${t}
+${r}`),this.name="DaemonExitedBeforeReadyError",this.stderrTail=r}}class v extends Error{static{o(this,"DaemonStartupBlockedError")}constructor(t,r){super(`daemon ${t} is running but normal startup is blocked: ${r}`),this.name="DaemonStartupBlockedError"}}class F extends Error{static{o(this,"DaemonNotPreparedError")}constructor(t,r){super(r===void 0?`daemon ${t} is serving but did not report the prepared startup phase`:`daemon ${t} reported startup phase ${r}, not prepared`),this.name="DaemonNotPreparedError"}}async function ge(e,t=G,r={},n="ready"){const a=r.readPidfile??E,c=r.isPidAlive??f,s=r.childExited,u=r.isDaemonServing,m=r.probeDaemonServing??(async w=>{if(u!==void 0){if(await u())return;throw new Error("crtrd health check reported not ready")}await J(w)}),p=r.probeDaemonHealth??L,x=r.sleepMs??$,y=r.now??Date.now;let i=null,l=null;return await fe({windowMs:t,pollIntervalMs:N,now:y,sleep:x,retry:o(w=>w!==l,"retry"),probe:o(async w=>{const h=a();if(h===null||!c(h)){const d=s?.();throw d!=null&&(d.signal!==null?l=new C(`daemon ${e} exited before becoming ready (by ${d.signal})`,r.childStderrTail?.()??""):d.code!==0&&(l=new C(`daemon ${e} exited before becoming ready (with exit code ${d.code??"?"})`,r.childStderrTail?.()??"")),l!==null)?l:new Error(`daemon ${e} has no live pidfile owner`)}if(n==="ready"){try{await m(w)}catch(d){throw d instanceof v&&(l=d),d}i=h;return}const _=await p(w);if(_.startup_phase==="prepared"){i=h;return}throw _.startup_phase==="blocked"?(l=new v(h,_.startup_blocked??"on-disk migration is blocked"),l):_.startup_phase==="ready"?(l=new F(h,_.startup_phase),l):new Error(`daemon ${h} is ${_.startup_phase??"not reporting a startup phase"}; waiting for prepared`)},"probe")}),i===e?null:i}o(ge,"verifyDaemonStartup");async function Se(e,t=z,r={}){const n=r.readPidfile??E,a=r.isPidAlive??f,c=r.sleepMs??$,s=r.now??Date.now,u=s()+t;for(;s()<=u;){if(!a(e)&&n()!==e)return;await c(N)}throw new Error(`daemon ${e} did not exit and release its pidfile within ${t}ms`)}o(Se,"waitForDaemonExit");const Re=2e3,ve=5e3;async function Y(e,t=z,r={}){const n=r.isPidAlive??f,a=r.sleepMs??$,c=r.now??Date.now;try{process.kill(e,"SIGTERM")}catch{}try{return await Se(e,t,r),"terminated"}catch{}try{process.kill(e,"SIGKILL")}catch{}const s=c()+Re;for(;c()<=s;){if(!n(e))return"killed";await a(N)}throw new Error(`daemon ${e} survived both SIGTERM and SIGKILL`)}o(Y,"stopDaemonProcess");async function nt(){const e=E(),t=e!==null&&f(e),r=t&&await Y(e)==="killed",n=Ee(null);return{stopped:t||n.length>0,...t?{pid:e}:{},strays_killed:n,force_killed:r}}o(nt,"stopAllDaemons");function Te(e,t,r=2048){let n;try{n=U(e,"r");const{size:a}=K(n),c=Math.max(t,a-r),s=a-c;if(s<=0)return"";const u=Buffer.alloc(s);return te(n,u,0,s,c),u.toString("utf8").trim()}catch{return""}finally{if(n!==void 0)try{j(n)}catch{}}}o(Te,"readLogTailFrom");async function be(e={}){if(process.platform==="linux"&&H("/run/crtr/launcher.sock")&&!f(E()??-1))throw new Error("crtrd is managed by the root launcher; it cannot self-daemonize");const t=e.targetPhase??"ready",r=E();if(r!==null&&f(r))try{const i=await L();if(t==="prepared"){if(i.startup_phase==="prepared")return{started:!1,existing_pid:r,startup_phase:i.startup_phase};throw new F(r,i.startup_phase)}return i.ok?{started:!1,existing_pid:r,...i.startup_phase===void 0?{}:{startup_phase:i.startup_phase}}:{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:i.startup_blocked??"crtrd health check reported not ready",...i.startup_phase===void 0?{}:{startup_phase:i.startup_phase}}}catch(i){if(i instanceof F)throw i;return{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:ye(i)}}ee(g(),{recursive:!0});const n=await A();e.waitForJoinLock===!0&&await ue(xe);const a=pe();if(a!==null&&a!==process.pid&&e.ignoreJoinLock!==!0)throw de(a);const c=D(g(),"crtrd.err"),s=U(c,"a"),u=(()=>{try{return K(s).size}catch{return 0}})();oe()===void 0&&ne();const m=B(O(),[...n,"--canvas-home",g(),...t==="prepared"?["--prepare"]:[]],{detached:!0,stdio:["ignore",s,s],env:M()});j(s);const p=m.pid;if(p===void 0)throw new Error("daemon spawn did not return a pid");let x=null;m.once("exit",(i,l)=>{x={code:i,signal:l}}),m.unref();let y;try{y=await ge(p,G,{childExited:o(()=>x,"childExited"),childStderrTail:o(()=>Te(c,u),"childStderrTail")},t)}catch(i){const l=ie(se())===p;if(x===null&&!l&&!(i instanceof v))try{await Y(p,ve)}catch{}throw i}return y!==null?{started:!1,existing_pid:y,startup_phase:t}:{started:!0,pid:p,startup_phase:t}}o(be,"spawnDaemon");function ke(){return ce()||process.argv.includes("--no-autostart")}o(ke,"autostartDisabled");const Ie=5e3,Oe=6e4;let W=!1,S,T=0,b=0;function Me(e){return Math.min(Ie*2**(e-1),Oe)}o(Me,"autostartBackoffMs");function Ae(){S=void 0,T=0,b=0}o(Ae,"resetDaemonAutostart");function ot(){return S}o(ot,"daemonAutostartStartupFailure");function at(e={}){const t=e.isDaemonRunning??ae,r=e.spawnDaemon??be,n=e.now??Date.now;if(!ke()&&!le()&&!W&&!(n()<b)){if(t()){Ae();return}S=void 0,W=!0,r().then(()=>{S=void 0,T=0,b=0},a=>{a instanceof C&&(S=a.message),T+=1,b=n()+Me(T)}).finally(()=>{W=!1})}}o(at,"ensureDaemon");export{he as DAEMON_ENV_STRIP_KEYS,C as DaemonExitedBeforeReadyError,F as DaemonNotPreparedError,v as DaemonStartupBlockedError,ke as autostartDisabled,ot as daemonAutostartStartupFailure,_e as daemonCommandDeclaresCanvasHome,at as ensureDaemon,Ve as execDaemon,De as findDaemonPids,rt as isDaemonServing,L as probeDaemonHealth,J as probeDaemonServing,ye as probeErrorMessage,Ae as resetDaemonAutostart,et as runDaemonBodiesDump,Qe as runDaemonExportSqlite,Ze as runDaemonImportDump,tt as runDaemonInstallId,M as sanitizedDaemonEnv,be as spawnDaemon,nt as stopAllDaemons,Y as stopDaemonProcess,Ee as sweepStrayDaemons,ge as verifyDaemonStartup,Se as waitForDaemonExit};

package/dist/daemon/reconcilers/cron-lane.js

@@ -1 +1 @@
-var i=Object.defineProperty;var t=(c,r)=>i(c,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{runThrottledLane as m}from"./throttled-lane.js";class _{static{t(this,"CronLaneReconciler")}async run(r,a){await n.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}try{await m("cron-folder-sweep","",r,6e4,6e4,async()=>{await l()})}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{_ as CronLaneReconciler};
+var c=Object.defineProperty;var t=(i,e)=>c(i,"name",{value:e,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{isDrained as m}from"../drain.js";import{runThrottledLane as d}from"./throttled-lane.js";class g{static{t(this,"CronLaneReconciler")}async run(e,a){await n.fresh(async()=>{try{await f(e,{register:t(r=>a.lifecycle.registerDetached(r),"register")})}catch(r){o({level:"error",event:"cron.lease_recovery.failed",error:r})}});try{await s()}catch(r){o({level:"error",event:"cron.execution.reap_failed",error:r})}try{await d("cron-folder-sweep","",e,6e4,6e4,async()=>{await l()})}catch(r){o({level:"error",event:"cron.folder.sweep_failed",error:r})}if(a.lifecycle.acceptsDetachedWork()&&!await m())try{await p(e,{register:t(r=>a.lifecycle.registerDetached(r),"register")})}catch(r){o({level:"error",event:"cron.run.failed",error:r})}}}export{g as CronLaneReconciler};

package/dist/daemon/reconcilers/delivery-lane.js

@@ -1,8 +1,8 @@
-var g=Object.defineProperty;var p=(e,t)=>g(e,"name",{value:t,configurable:!0});import{randomUUID as D}from"node:crypto";import{claimPushDelivery as h,duePushDeliveries as j,messageBodyForDelivery as N,pushDeliveryClaimIsCurrent as O,settleDelivery as d}from"../../core/canvas/deliveries.js";import{deliveryBackoffMs as f}from"../../core/canvas/delivery-contract.js";import{openDb as c}from"../../core/canvas/db.js";import{pushLineFor as u}from"../../core/graph/custom.js";import{postReceiverUrl as b}from"../../core/graph/receiver-url.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{runDelivery as w,DELIVERY_TIMEOUT_MS as E}from"../delivery/run-delivery.js";const v=D();async function o(e,t,a){await d(e.deliveryId,e.attempts,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:a}},Date.now())}p(o,"failed");async function T(e,t){const a=await(await c()).prepare(`SELECT o.type, o.name, c.receiver_json, c.payload_json, m.object_id, i.external_id AS app,
+var g=Object.defineProperty;var p=(e,t)=>g(e,"name",{value:t,configurable:!0});import{randomUUID as D}from"node:crypto";import{claimPushDelivery as h,duePushDeliveries as j,messageBodyForDelivery as N,pushDeliveryClaimIsCurrent as O,settleDelivery as d}from"../../core/canvas/deliveries.js";import{deliveryBackoffMs as f}from"../../core/canvas/delivery-contract.js";import{openDb as c}from"../../core/canvas/db.js";import{pushLineFor as u}from"../../core/graph/custom.js";import{postReceiverUrl as b}from"../../core/graph/receiver-url.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{runDelivery as w,DELIVERY_TIMEOUT_MS as E}from"../delivery/run-delivery.js";import{isDrained as T}from"../drain.js";const v=D();async function o(e,t,a){await d(e.deliveryId,e.attempts,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:a}},Date.now())}p(o,"failed");async function A(e,t){const a=await(await c()).prepare(`SELECT o.type, o.name, c.receiver_json, c.payload_json, m.object_id, i.external_id AS app,
     d.message_id, owner.type AS owner_type, parent.external_id AS parent_app
     FROM deliveries d JOIN objects o ON o.id = d.destination_id
     JOIN custom_objects c ON c.object = o.id JOIN messages m ON m.message_id = d.message_id
     LEFT JOIN objects owner ON owner.id = o.owner
     LEFT JOIN identities i ON i.object = o.owner AND i.scheme = 'app'
     LEFT JOIN identities parent ON parent.object = owner.owner AND parent.scheme = 'app'
-    WHERE d.delivery_id = ?`).get(e.deliveryId);if(!a||!a.receiver_json){await o(e,t,`delivery ${e.deliveryId} has no supported receiver`);return}const i=JSON.parse(a.receiver_json);if(!await O(e,t)){await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now());return}if(typeof i=="object"&&typeof i.url=="string"&&a.type!=="action"){const n=await u(await c(),e.deliveryId);if(!n){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}try{const r=await b(i.url,JSON.stringify(n),{"Content-Type":"application/json","X-Crtr-Delivery":e.deliveryId,"X-Crtr-Event":a.message_id},E);r>=200&&r<300?await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now()):r>=300&&r<400?await o(e,t,`url receiver returned HTTP ${r}; redirects are not followed \u2014 register the final URL`):r>=400&&r<500&&r!==408&&r!==429?await o(e,t,`url receiver returned HTTP ${r}`):await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:`url receiver returned HTTP ${r}`}},Date.now())}catch(r){await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:String(r)}},Date.now())}return}if(typeof i!="object"||!Array.isArray(i.argv)||typeof i.cwd!="string"){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}if(a.type==="action"){if(typeof i.name!="string"||!a.object_id){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}const n=await N(e.deliveryId);if(n===null){await o(e,t,`delivery ${e.deliveryId} has no message body`);return}const{push:r,...y}=n,I={...y,action:{name:i.name,payload:a.payload_json===null?null:JSON.parse(a.payload_json)}};await w({argv:i.argv,cwd:i.cwd,document:I,attempt:e.attempts,nodeId:a.object_id,...a.app??a.parent_app?{app:a.app??a.parent_app}:{}},async _=>{await d(e.deliveryId,e.attempts,t,_,Date.now())});return}const s=await u(await c(),e.deliveryId);if(!s){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}await w({argv:i.argv,cwd:i.cwd,document:s,attempt:e.attempts,app:a.app??a.parent_app??"app:terminal",custom:{deliveryId:e.deliveryId,messageId:a.message_id}},async n=>{await d(e.deliveryId,e.attempts,t,n,Date.now())})}p(T,"deliverExternal");class P{static{p(this,"DeliveryLaneReconciler")}async run(t,a){if(!a.lifecycle.acceptsDetachedWork())return;let i;try{i=await j(t)}catch(s){m.fresh(()=>l({level:"error",event:"delivery.scan_failed",error:s}));return}for(const s of i)try{const n=await h(s.deliveryId,t,v);if(n===null)continue;a.lifecycle.registerDetached(T(n,v).catch(r=>{m.fresh(()=>l({level:"error",event:"delivery.failed",error:r,fields:{delivery_id:n.deliveryId}}))}))}catch(n){m.fresh(()=>l({level:"error",event:"delivery.claim_failed",error:n,fields:{delivery_id:s.deliveryId}}))}}}export{P as DeliveryLaneReconciler,v as deliveryDaemonInstanceId};
+    WHERE d.delivery_id = ?`).get(e.deliveryId);if(!a||!a.receiver_json){await o(e,t,`delivery ${e.deliveryId} has no supported receiver`);return}const i=JSON.parse(a.receiver_json);if(!await O(e,t)){await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now());return}if(typeof i=="object"&&typeof i.url=="string"&&a.type!=="action"){const n=await u(await c(),e.deliveryId);if(!n){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}try{const r=await b(i.url,JSON.stringify(n),{"Content-Type":"application/json","X-Crtr-Delivery":e.deliveryId,"X-Crtr-Event":a.message_id},E);r>=200&&r<300?await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now()):r>=300&&r<400?await o(e,t,`url receiver returned HTTP ${r}; redirects are not followed \u2014 register the final URL`):r>=400&&r<500&&r!==408&&r!==429?await o(e,t,`url receiver returned HTTP ${r}`):await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:`url receiver returned HTTP ${r}`}},Date.now())}catch(r){await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:String(r)}},Date.now())}return}if(typeof i!="object"||!Array.isArray(i.argv)||typeof i.cwd!="string"){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}if(a.type==="action"){if(typeof i.name!="string"||!a.object_id){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}const n=await N(e.deliveryId);if(n===null){await o(e,t,`delivery ${e.deliveryId} has no message body`);return}const{push:r,...y}=n,I={...y,action:{name:i.name,payload:a.payload_json===null?null:JSON.parse(a.payload_json)}};await w({argv:i.argv,cwd:i.cwd,document:I,attempt:e.attempts,nodeId:a.object_id,...a.app??a.parent_app?{app:a.app??a.parent_app}:{}},async _=>{await d(e.deliveryId,e.attempts,t,_,Date.now())});return}const s=await u(await c(),e.deliveryId);if(!s){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}await w({argv:i.argv,cwd:i.cwd,document:s,attempt:e.attempts,app:a.app??a.parent_app??"app:terminal",custom:{deliveryId:e.deliveryId,messageId:a.message_id}},async n=>{await d(e.deliveryId,e.attempts,t,n,Date.now())})}p(A,"deliverExternal");class H{static{p(this,"DeliveryLaneReconciler")}async run(t,a){if(!a.lifecycle.acceptsDetachedWork()||await T())return;let i;try{i=await j(t)}catch(s){m.fresh(()=>l({level:"error",event:"delivery.scan_failed",error:s}));return}for(const s of i)try{const n=await h(s.deliveryId,t,v);if(n===null)continue;a.lifecycle.registerDetached(A(n,v).catch(r=>{m.fresh(()=>l({level:"error",event:"delivery.failed",error:r,fields:{delivery_id:n.deliveryId}}))}))}catch(n){m.fresh(()=>l({level:"error",event:"delivery.claim_failed",error:n,fields:{delivery_id:s.deliveryId}}))}}}export{H as DeliveryLaneReconciler,v as deliveryDaemonInstanceId};

package/dist/daemon/reconcilers/import-lane.d.ts

@@ -0,0 +1,9 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+export declare const IMPORT_LEASE_MS: number;
+export declare class ImportLaneReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass over the runs due now. Exposed for tests. */
+    pass(now: number): Promise<number>;
+}

package/dist/daemon/reconcilers/import-lane.js

@@ -0,0 +1 @@
+var p=Object.defineProperty;var n=(i,e)=>p(i,"name",{value:e,configurable:!0});import{openDb as m}from"../../core/canvas/db.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{processRun as f}from"../../core/import/load/run.js";import{ClaimLostError as d,withRowClaim as _}from"./row-claim.js";const R=2*6e4;class S{static{n(this,"ImportLaneReconciler")}run(e,o){if(!o.lifecycle.acceptsDetachedWork())return;const a=c.fresh(()=>this.pass(e).then(()=>{})).catch(t=>{c.fresh(()=>l({level:"error",event:"import.lane.failed",error:t}))});o.lifecycle.registerDetached(a)}async pass(e){const a=await(await m()).prepare("SELECT * FROM import_runs WHERE (state = 'loading' OR state = 'committed') AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?) ORDER BY created_at").all(new Date(e).toISOString());let t=0;for(const s of a)try{await _("import_runs",s.run_id,R,r=>f(s,r))&&(t+=1)}catch(r){if(!(r instanceof d))throw r}return t}}export{R as IMPORT_LEASE_MS,S as ImportLaneReconciler};

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js

@@ -1 +1 @@
-var g=Object.defineProperty;var s=(i,e)=>g(i,"name",{value:e,configurable:!0});import{clearFrozen as p,getNode as h,getRow as y}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as k}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as S}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as N}from"../../../core/canvas/deliveries.js";import{publishNotice as D}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as E}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as R}from"./remote-wake.js";import{asNodePerson as z}from"./node-person.js";import{commitReopenResident as T}from"../../../core/runtime/reopen.js";import{reviveNode as m}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{mapAsync as C}from"../../../core/async-array.js";const f=6,b=10*6e4,I=2e4;async function M(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(M,"readEpisode");async function O(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(O,"claimAttempt");async function P(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(P,"writeEpisode");function U(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s(U,"thawTier");function x(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(x,"compareThaw");class re{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await C(a.candidates,async n=>({...n,tier:U(n.row,await v(n.row.node_id))}));t.sort(x);const r={stalled:0,waitingOnMail:0};for(const n of t)await S.fresh(()=>z(n.row,()=>this.enact(e,n,a.fleet)))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await y(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await T(e.node_id);try{return(await m(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(E(a)||a instanceof R)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return await u(e,"release"),await p(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await L(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await h(e.node_id))?.pi_session_id!=null&&!await v(e.node_id))return await u(e.node_id,"release"),await p(e.node_id),"skipped";try{return(await m(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(E(t)||t instanceof R)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await h(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await p(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await N(t),n=await M(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=f)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:f}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await D(t,"notice.crash",`Child crashed \u2014 ${k(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const w=l?void 0:n.lastAttempt,A=Math.min(I*2**o,b);return w!==void 0&&e-w<A||!await O(t,n.attempts,o+1,r,e)?"skipped":(await m(t,{resume:!0,capacity:"freeze"})).outcome==="frozen"?(await P(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:f}}),"launched")}}export{re as FreezeLaneReconciler,x as compareThaw,U as thawTier};
+var N=Object.defineProperty;var s=(i,e)=>N(i,"name",{value:e,configurable:!0});import{clearFrozen as f,getNode as g,getRow as D}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as z}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as L}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as T}from"../../../core/canvas/deliveries.js";import{publishNotice as C}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as y}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as A}from"./remote-wake.js";import{asNodePerson as M}from"./node-person.js";import{MachineLaunchUncertainError as b,MachineOfflineError as m}from"../../../core/runtime/runner/machine/ops.js";import{commitReopenResident as I}from"../../../core/runtime/reopen.js";import{reviveNode as w}from"../../../core/runtime/revive.js";import{hasPendingWake as k}from"../live-obligation.js";import{readInterruptedTurn as O}from"../../../core/canvas/recovery.js";import{mapAsync as P}from"../../../core/async-array.js";const h=6,U=10*6e4,x=2e4;async function H(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(H,"readEpisode");async function F(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(F,"claimAttempt");async function W(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(W,"writeEpisode");function $(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s($,"thawTier");function B(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(B,"compareThaw");class le{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await P(a.candidates,async n=>({...n,tier:$(n.row,await k(n.row.node_id))}));t.sort(B);const r={stalled:0,waitingOnMail:0};for(const n of t)await L.fresh(()=>M(n.row,()=>this.enact(e,n,a.fleet)))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await D(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await I(e.node_id);try{return(await w(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(y(a)||a instanceof A||a instanceof m)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return a instanceof b||await u(e,"release"),await f(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await O(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await g(e.node_id))?.pi_session_id!=null&&!await k(e.node_id))return await u(e.node_id,"release"),await f(e.node_id),"skipped";try{return(await w(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(y(t)||t instanceof A||t instanceof m)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await g(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await f(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await T(t),n=await H(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=h)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:h}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await C(t,"notice.crash",`Child crashed \u2014 ${z(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const E=l?void 0:n.lastAttempt,S=Math.min(x*2**o,U);if(E!==void 0&&e-E<S||!await F(t,n.attempts,o+1,r,e))return"skipped";const v=s(()=>W(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"giveBack");let R;try{R=await w(t,{resume:!0,capacity:"freeze"})}catch(p){if(!(p instanceof m))throw p;return await v(),await this.settleRefusedStart(t,p,"idle-release")}return R.outcome==="frozen"?(await v(),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:h}}),"launched")}}export{le as FreezeLaneReconciler,B as compareThaw,$ as thawTier};

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.d.ts

@@ -7,6 +7,9 @@ export declare const LAUNCH_DEADLINE_MS: number;
 export declare const LAUNCH_LOST_GRACE_MS = 120000;
 /** The release the control plane runs, which is the release a wake names. */
 export declare const acceptedRelease: () => string;
+/** The machine environment the node's profile points at when this process is a cloud role (the machine is another host, reached
+ *  through the remote machine runner), else null: a Mac's own crtrd runs its machine environment's nodes as processes. */
+export declare function remoteMachineEnvironmentOf(profileId: string | null): Promise<string | null>;
 /** Step 2: the `funding bill` check. A refusal writes `nodes.spend_refusal` and one refused `sandbox` row per node, limit and month,
  *  then throws `402 cap_exhausted`; no execution row is written and no runner is called. A pass clears `spend_refusal`. */
 export declare function checkBilledWake(meta: NodeMeta): Promise<void>;

package/dist/daemon/reconcilers/node-lifecycle/remote-wake.js

@@ -1 +1 @@
-var f=Object.defineProperty;var a=(e,n)=>f(e,"name",{value:n,configurable:!0});import{liveExecutionForNode as m,endExecution as l,recordExecutionAwake as p,endLaunchingExecution as s,requestExecutionStop as w}from"../../../core/canvas/executions.js";import{openDb as h}from"../../../core/canvas/db.js";import{emitEvent as u}from"../../../core/events/emit.js";import{findRefusal as _,refusalError as x,refusedMark as g,writeCharge as E}from"../../../core/ledger/index.js";import{transition as c}from"../../../core/runtime/lifecycle.js";import{runnerForExecution as d}from"../../../core/runtime/runner/index.js";import{startCandidateOf as v}from"../../../core/runtime/start-check.js";import{settleSandboxBeforeWake as b}from"../ledger-sandbox.js";import{crtrVersion as k}from"../../../shared/crtr-version.js";import{resolveEnvironmentImage as I}from"../recipe-image.js";const y=45*6e4,S=12e4,L=a(()=>k(),"acceptedRelease");async function P(e){const n=await v(e);if(n===null)return;const t={scope:"call",payer:n.payer,funding:"bill",sub:n.sub,grantee:n.grantee,provider:"blaxel"},r=await _(t),o=await h();if(r===null){await o.prepare("UPDATE nodes SET spend_refusal = NULL WHERE node_id = ? AND spend_refusal IS NOT NULL").run(e.node_id);return}throw await o.prepare("UPDATE nodes SET spend_refusal = ? WHERE node_id = ?").run(JSON.stringify({limit:r.limit.id,usd:r.usd,reset_at:r.reset_at,at:new Date().toISOString()}),e.node_id),await E({id:`refused:sandbox:${e.node_id}:${r.limit.id}:${r.month}`,kind:"sandbox",what:"wake",usd:null,payer:t.payer,funding:"bill",sub:t.sub,grantee:t.grantee,node:e.node_id,run:e.run_id??null,provider:"blaxel",refused:g(r)}),x(r)}a(P,"checkBilledWake");class F extends Error{static{a(this,"EnvironmentImagePending")}environmentId;imageError;constructor(n,t){super(`environment ${n} has no built image yet${t===null?"":` (${t})`}; the wake waits for the build.`),this.environmentId=n,this.imageError=t,this.name="EnvironmentImagePending"}}async function M(e){return await I(e)}a(M,"imageForWake");async function B(e){return e.release!==L()&&await w(e.nodeId,{afterTurn:!1,reason:"release"}),await d(e.runner).wake(e)==="no_broker"?(await l(e.executionId,{reason:"lost"}),"lost"):(await b(e.executionId,Date.now()),await p(e.executionId),"woken")}a(B,"wakeResident");async function G(e){const n=await d(e.runner).status(e);if(n.state!=="gone"&&n.state!=="failed")return!1;const t=await l(e.executionId,{reason:"lost"});return t&&u({level:"warn",event:"execution.lost",node_id:e.nodeId}),t}a(G,"endRemoteExecutionIfGone");async function q(e,n){const t=await m(e);if(t===null||t.state!=="launching"||t.runner==="process")return;const r=d(t.runner),o=n-Date.parse(t.startedAt),i=await r.status(t);if(i.state!=="unknown"){if(i.state==="running"){if(o<=y||!await s(t.executionId,{reason:"launch_timeout"}))return;await r.stop(t,{reason:"launch_timeout"}),await c(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:"the sandbox launch did not report within the launch deadline"}}}),u({level:"warn",event:"execution.launch_timeout",node_id:e});return}if(i.state==="failed"){if(!await s(t.executionId,{reason:"launch_failed"}))return;await c(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:`the sandbox start script failed: ${i.reason}`}}});return}if(o>S){if(!await s(t.executionId,{reason:"launch_lost"}))return;u({level:"warn",event:"execution.launch_lost",node_id:e})}}}a(q,"recoverLaunching");export{F as EnvironmentImagePending,y as LAUNCH_DEADLINE_MS,S as LAUNCH_LOST_GRACE_MS,L as acceptedRelease,P as checkBilledWake,G as endRemoteExecutionIfGone,M as imageForWake,q as recoverLaunching,B as wakeResident};
+var f=Object.defineProperty;var a=(e,t)=>f(e,"name",{value:t,configurable:!0});import{liveExecutionForNode as m,endExecution as l,recordExecutionAwake as p,endLaunchingExecution as s,requestExecutionStop as w}from"../../../core/canvas/executions.js";import{openDb as h}from"../../../core/canvas/db.js";import{isCloudRole as _}from"../../../core/canvas/person-state.js";import{emitEvent as u}from"../../../core/events/emit.js";import{findRefusal as x,refusalError as E,refusedMark as g,writeCharge as v}from"../../../core/ledger/index.js";import{transition as c}from"../../../core/runtime/lifecycle.js";import{runnerForExecution as d}from"../../../core/runtime/runner/index.js";import{machineEnvironmentOfProfile as b}from"../../../core/runtime/runner/machine/environment.js";import{startCandidateOf as k}from"../../../core/runtime/start-check.js";import{settleSandboxBeforeWake as y}from"../ledger-sandbox.js";import{crtrVersion as I}from"../../../shared/crtr-version.js";import{resolveEnvironmentImage as S}from"../recipe-image.js";const L=45*6e4,A=12e4,D=a(()=>I(),"acceptedRelease");async function G(e){return e===null||!_()?null:await b(e)}a(G,"remoteMachineEnvironmentOf");async function q(e){const t=await k(e);if(t===null)return;const n={scope:"call",payer:t.payer,funding:"bill",sub:t.sub,grantee:t.grantee,provider:"blaxel"},r=await x(n),o=await h();if(r===null){await o.prepare("UPDATE nodes SET spend_refusal = NULL WHERE node_id = ? AND spend_refusal IS NOT NULL").run(e.node_id);return}throw await o.prepare("UPDATE nodes SET spend_refusal = ? WHERE node_id = ?").run(JSON.stringify({limit:r.limit.id,usd:r.usd,reset_at:r.reset_at,at:new Date().toISOString()}),e.node_id),await v({id:`refused:sandbox:${e.node_id}:${r.limit.id}:${r.month}`,kind:"sandbox",what:"wake",usd:null,payer:n.payer,funding:"bill",sub:n.sub,grantee:n.grantee,node:e.node_id,run:e.run_id??null,provider:"blaxel",refused:g(r)}),E(r)}a(q,"checkBilledWake");class J extends Error{static{a(this,"EnvironmentImagePending")}environmentId;imageError;constructor(t,n){super(`environment ${t} has no built image yet${n===null?"":` (${n})`}; the wake waits for the build.`),this.environmentId=t,this.imageError=n,this.name="EnvironmentImagePending"}}async function V(e){return await S(e)}a(V,"imageForWake");async function j(e){return e.release!==D()&&await w(e.nodeId,{afterTurn:!1,reason:"release"}),await d(e.runner).wake(e)==="no_broker"?(await l(e.executionId,{reason:"lost"}),"lost"):(await y(e.executionId,Date.now()),await p(e.executionId),"woken")}a(j,"wakeResident");async function z(e){const t=await d(e.runner).status(e);if(t.state!=="gone"&&t.state!=="failed")return!1;const n=await l(e.executionId,{reason:"lost"});return n&&u({level:"warn",event:"execution.lost",node_id:e.nodeId}),n}a(z,"endRemoteExecutionIfGone");async function K(e,t){const n=await m(e);if(n===null||n.state!=="launching"||n.runner==="process")return;const r=d(n.runner),o=t-Date.parse(n.startedAt),i=await r.status(n);if(i.state!=="unknown"){if(i.state==="running"){if(o<=L||!await s(n.executionId,{reason:"launch_timeout"}))return;await r.stop(n,{reason:"launch_timeout"}),await c(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:"the sandbox launch did not report within the launch deadline"}}}),u({level:"warn",event:"execution.launch_timeout",node_id:e});return}if(i.state==="failed"){if(!await s(n.executionId,{reason:"launch_failed"}))return;await c(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:`the sandbox start script failed: ${i.reason}`}}});return}if(o>A){if(!await s(n.executionId,{reason:"launch_lost"}))return;u({level:"warn",event:"execution.launch_lost",node_id:e})}}}a(K,"recoverLaunching");export{J as EnvironmentImagePending,L as LAUNCH_DEADLINE_MS,A as LAUNCH_LOST_GRACE_MS,D as acceptedRelease,q as checkBilledWake,z as endRemoteExecutionIfGone,V as imageForWake,K as recoverLaunching,G as remoteMachineEnvironmentOf,j as wakeResident};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var b=Object.defineProperty;var o=(r,a)=>b(r,"name",{value:a,configurable:!0});import{clearFrozen as P,getNode as x}from"../../../core/canvas/index.js";import{isSafeNodeId as z}from"../../../core/canvas/paths.js";import{recordedPidLiveness as L}from"../../../core/canvas/pid.js";import{endExecution as v,hasAbsorbedProfilePauseStop as g,hasObservedExit as N,latestExecutionForNode as M,liveExecutionForNode as c}from"../../../core/canvas/executions.js";import{emitEvent as w}from"../../../core/events/emit.js";import{operationIdContext as F}from"../../../core/events/operation-id.js";import{readInterruptedTurn as D}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as I}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as A,isParkedResident as C}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as S}from"../../fleet.js";import{applyParkEvent as T,hasPendingWake as U,parkEvent as W}from"../live-obligation.js";import{FreezeLaneReconciler as B}from"./freeze-lane.js";import{isInBackoff as E}from"./respawn-policy.js";import{asNodePerson as G,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as H}from"./terminating.js";import{wantsExecution as q}from"./wants-execution.js";import{enactRequestedStops as O}from"./requested-stops.js";import{endRemoteExecutionIfGone as j,recoverLaunching as J,wakeResident as K}from"./remote-wake.js";function Q(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}o(Q,"isTerminal");const V=12e4;function X(r,a){const e=Math.max(...[r.created,r.launched_at].map(i=>i==null?0:Date.parse(i)||0));return e>0&&a-e<V}o(X,"isLaunchInFlight");async function R(r){const a=await c(r);return a!==null&&a.state==="running"&&a.pid!=null&&L(a.pid,a.processIdentity)==="dead"?a:null}o(R,"executionWithDeadPid");class Pe{static{o(this,"NodeLifecycleReconciler")}freezeLane=new B;async run(a,e){const i=[];await O(a);for(const s of e.rows)await F.fresh(async()=>{try{await G(s,()=>this.classifyRow(a,s,e,i))}catch(t){const l=s.node_id;w({level:t instanceof k?"warn":"error",event:t instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...z(l)?{node_id:l}:{fields:{affected_node_id:l}},error:t})}});return this.freezeLane.run(a,{fleet:e.fleet,candidates:i})}async classifyRow(a,e,i,s){const t=e.node_id;if(Q(e)){if(H(e,await i.fleet.has(t)))return;if(C(e)&&!await i.fleet.has(t)&&!d(e.profile_id)&&!await I(t)&&!E(e,a)&&await p(t)){s.push({row:e,enact:"reopen"});return}if(e.pi_pid!=null){const n=await R(t);n!==null&&await v(n.executionId,{reason:"observed-dead"})}e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await P(t):s.push({row:e,enact:"thaw"}));return}if(await i.fleet.has(t)){const n=await c(t);if(n?.runner==="blaxel"&&n.state==="running"&&!d(e.profile_id)&&await p(t)){if(await K(n)==="lost")return}else n?.runner==="blaxel"&&n.state==="running"&&await j(n);if(await i.fleet.has(t)){d(e.profile_id)?await i.fleet.interruptForProfilePause(t):await this.thawMark(e);return}}if(await J(t,a),d(e.profile_id)){await this.freezeLane.endEpisode(t);return}if(await I(t))return;const l=e.pi_pid==null?await M(t):null,f=l!==null&&N(l),m=l!==null&&!f&&g(l),h=e.pi_pid!=null?await R(t):null,_=f||m||h!==null;if(_&&!await i.fleet.hasProfilePauseInterruption(t)){const n=h??l;n!==null&&await v(n.executionId,{reason:"observed-dead"});const y=await c(t);if(y!==null&&y.executionId!==n?.executionId)return}if(!await q(e)){await this.thawMark(e),await this.settleParked(e);return}if(E(e,a))return;if(e.status==="idle"&&A(e)&&await p(t)){s.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!f||e.frozen_at!=null)&&!m){if(e.respawn_not_before!=null){s.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){s.push({row:e,enact:"thaw"});return}if(await i.fleet.isClaimed(t)||X(e,a))return;w({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),s.push({row:e,enact:"thaw"});return}if(!_)return;const u=S(await x(t),(await D(t))?.interrupted===!0,await U(t));if(u==="respawn-fresh"||u==="respawn-cycle"||u==="respawn-resume"){s.push({row:e,enact:"policy"});return}await i.fleet.applyDeadRowPolicy(t),await this.thawMark(e)}async thawMark(a){a.frozen_at!=null&&await P(a.node_id)}async settleParked(a){if(a.status!=="idle")return;await this.freezeLane.endEpisode(a.node_id);const e=await W(a.node_id,a.lifecycle);e!=="release"&&await T(a.node_id,e)}}export{V as LAUNCH_GRACE_MS,Pe as NodeLifecycleReconciler,X as isLaunchInFlight};
+var b=Object.defineProperty;var o=(r,i)=>b(r,"name",{value:i,configurable:!0});import{clearFrozen as P,getNode as g}from"../../../core/canvas/index.js";import{isSafeNodeId as z}from"../../../core/canvas/paths.js";import{recordedPidLiveness as L}from"../../../core/canvas/pid.js";import{endExecution as v,hasAbsorbedProfilePauseStop as N,hasObservedExit as x,latestExecutionForNode as M,liveExecutionForNode as c}from"../../../core/canvas/executions.js";import{emitEvent as w}from"../../../core/events/emit.js";import{operationIdContext as D}from"../../../core/events/operation-id.js";import{readInterruptedTurn as F}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as I}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as A,isParkedResident as C}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as S}from"../../fleet.js";import{applyParkEvent as T,hasPendingWake as U,parkEvent as W}from"../live-obligation.js";import{FreezeLaneReconciler as B}from"./freeze-lane.js";import{isInBackoff as E}from"./respawn-policy.js";import{asNodePerson as G,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as H}from"./terminating.js";import{wantsExecution as O}from"./wants-execution.js";import{enactRequestedStops as q}from"./requested-stops.js";import{endRemoteExecutionIfGone as j,recoverLaunching as J,wakeResident as K}from"./remote-wake.js";import{isDrained as Q}from"../../drain.js";function V(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}o(V,"isTerminal");const X=12e4;function Y(r,i){const e=Math.max(...[r.created,r.launched_at].map(n=>n==null?0:Date.parse(n)||0));return e>0&&i-e<X}o(Y,"isLaunchInFlight");async function R(r){const i=await c(r);return i!==null&&i.state==="running"&&i.pid!=null&&L(i.pid,i.processIdentity)==="dead"?i:null}o(R,"executionWithDeadPid");class we{static{o(this,"NodeLifecycleReconciler")}freezeLane=new B;async run(i,e){const n=[];await q(i);for(const s of e.rows)await D.fresh(async()=>{try{await G(s,()=>this.classifyRow(i,s,e,n))}catch(t){const l=s.node_id;w({level:t instanceof k?"warn":"error",event:t instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...z(l)?{node_id:l}:{fields:{affected_node_id:l}},error:t})}});return await Q()?{stalled:0,waitingOnMail:0}:this.freezeLane.run(i,{fleet:e.fleet,candidates:n})}async classifyRow(i,e,n,s){const t=e.node_id;if(V(e)){if(H(e,await n.fleet.has(t)))return;if(C(e)&&!await n.fleet.has(t)&&!d(e.profile_id)&&!await I(t)&&!E(e,i)&&await p(t)){s.push({row:e,enact:"reopen"});return}if(e.pi_pid!=null){const a=await R(t);a!==null&&await v(a.executionId,{reason:"observed-dead"})}e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await P(t):s.push({row:e,enact:"thaw"}));return}if(await n.fleet.has(t)){const a=await c(t);if(a?.runner!==void 0&&a.runner!=="process"&&a.state==="running"&&!d(e.profile_id)&&await p(t)){if(await K(a)==="lost")return}else a?.runner!==void 0&&a.runner!=="process"&&a.state==="running"&&await j(a);if(await n.fleet.has(t)){d(e.profile_id)?await n.fleet.interruptForProfilePause(t):await this.thawMark(e);return}}if(await J(t,i),d(e.profile_id)){await this.freezeLane.endEpisode(t);return}if(await I(t))return;const l=e.pi_pid==null?await M(t):null,f=l!==null&&x(l),m=l!==null&&!f&&N(l),h=e.pi_pid!=null?await R(t):null,_=f||m||h!==null;if(_&&!await n.fleet.hasProfilePauseInterruption(t)){const a=h??l;a!==null&&await v(a.executionId,{reason:"observed-dead"});const y=await c(t);if(y!==null&&y.executionId!==a?.executionId)return}if(!await O(e)){await this.thawMark(e),await this.settleParked(e);return}if(E(e,i))return;if(e.status==="idle"&&A(e)&&await p(t)){s.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!f||e.frozen_at!=null)&&!m){if(e.respawn_not_before!=null){s.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){s.push({row:e,enact:"thaw"});return}if(await n.fleet.isClaimed(t)||Y(e,i))return;w({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),s.push({row:e,enact:"thaw"});return}if(!_)return;const u=S(await g(t),(await F(t))?.interrupted===!0,await U(t));if(u==="respawn-fresh"||u==="respawn-cycle"||u==="respawn-resume"){s.push({row:e,enact:"policy"});return}await n.fleet.applyDeadRowPolicy(t),await this.thawMark(e)}async thawMark(i){i.frozen_at!=null&&await P(i.node_id)}async settleParked(i){if(i.status!=="idle")return;await this.freezeLane.endEpisode(i.node_id);const e=await W(i.node_id,i.lifecycle);e!=="release"&&await T(i.node_id,e)}}export{X as LAUNCH_GRACE_MS,we as NodeLifecycleReconciler,Y as isLaunchInFlight};

package/dist/daemon/reconcilers/purge-due-work.d.ts

@@ -0,0 +1,13 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+import { type LanePass } from './throttled-lane.js';
+export declare const PURGE_DUE_WORK_INTERVAL_MS: number;
+/** Rows purged per pass; the rest wait for the next pass. */
+export declare const PURGE_BUDGET = 1000;
+export interface PurgeDueWorkContext {
+    lifecycle: DetachedWorkLifecycle;
+}
+export declare class PurgeDueWorkReconciler {
+    run(now: number, ctx: PurgeDueWorkContext): void;
+    /** One pass. Exposed for tests. */
+    pass(now: number, lane?: Pick<LanePass, 'holds'>): Promise<number>;
+}

package/dist/daemon/reconcilers/purge-due-work.js

@@ -0,0 +1 @@
+var a=Object.defineProperty;var c=(s,e)=>a(s,"name",{value:e,configurable:!0});import{emitEvent as n}from"../../core/events/emit.js";import{operationIdContext as i}from"../../core/events/operation-id.js";import{purgeSoftDeleted as d}from"../../core/storage/purge.js";import{runThrottledLane as p}from"./throttled-lane.js";const u=10*6e4,l=1e3,f=5*6e4;class D{static{c(this,"PurgeDueWorkReconciler")}run(e,t){if(!t.lifecycle.acceptsDetachedWork())return;const r=i.fresh(()=>p("purge-due-work","",e,u,f,async o=>{await this.pass(e,o)})).then(()=>{}).catch(o=>{i.fresh(()=>n({level:"error",event:"purge.due_work.failed",error:o}))});t.lifecycle.registerDetached(r)}async pass(e,t){const r=await d(e,l,t?.holds);return r>0&&n({level:"info",event:"purge.due_work.settled",outcome:"succeeded",fields:{purged:r}}),r}}export{l as PURGE_BUDGET,u as PURGE_DUE_WORK_INTERVAL_MS,D as PurgeDueWorkReconciler};

package/dist/daemon/reconcilers/row-claim.d.ts

@@ -3,6 +3,7 @@ declare const CLAIMABLE: {
     readonly nodes: "node_id";
     readonly structured_requests: "request_id";
     readonly human_requests: "request_id";
+    readonly import_runs: "run_id";
 };
 export type ClaimableTable = keyof typeof CLAIMABLE;
 export declare class ClaimLostError extends Error {

package/dist/daemon/reconcilers/row-claim.js

@@ -1,2 +1,2 @@
-var E=Object.defineProperty;var i=(e,r)=>E(e,"name",{value:r,configurable:!0});import{randomUUID as D}from"node:crypto";import{openDb as w}from"../../core/canvas/db.js";const d={nodes:"node_id",structured_requests:"request_id",human_requests:"request_id"};class _ extends Error{static{i(this,"ClaimLostError")}constructor(r,t){super(`row claim lost: ${r} ${t} is no longer held by this process`)}}async function L(e,r,t,p){const a=d[e],n=D(),o=await w(),u=new Date().toISOString();if((await o.prepare(`UPDATE ${e} SET claimed_by = ?, lease_expires_at = ?
-    WHERE ${a} = ? AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?)`).run(n,new Date(Date.now()+t).toISOString(),r,u)).changes!==1)return!1;let s=!1;const c=setInterval(()=>{o.prepare(`UPDATE ${e} SET lease_expires_at = ? WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ?`).run(new Date(Date.now()+t).toISOString(),r,n,new Date().toISOString()).then(l=>{l.changes!==1&&(s=!0)},()=>{s=!0})},Math.max(1,Math.floor(t/3)));c.unref();const m={async assertOwned(){if(s)throw new _(e,r);if(await(await w()).prepare(`SELECT 1 AS held FROM ${e} WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ? FOR UPDATE`).get(r,n,new Date().toISOString())===void 0)throw s=!0,new _(e,r)}};try{await p(m)}finally{clearInterval(c),await o.prepare(`UPDATE ${e} SET claimed_by = NULL, lease_expires_at = NULL WHERE ${a} = ? AND claimed_by = ?`).run(r,n)}return!0}i(L,"withRowClaim");export{_ as ClaimLostError,L as withRowClaim};
+var E=Object.defineProperty;var i=(e,r)=>E(e,"name",{value:r,configurable:!0});import{randomUUID as d}from"node:crypto";import{openDb as _}from"../../core/canvas/db.js";const D={nodes:"node_id",structured_requests:"request_id",human_requests:"request_id",import_runs:"run_id"};class w extends Error{static{i(this,"ClaimLostError")}constructor(r,t){super(`row claim lost: ${r} ${t} is no longer held by this process`)}}async function L(e,r,t,u){const a=D[e],n=d(),o=await _(),p=new Date().toISOString();if((await o.prepare(`UPDATE ${e} SET claimed_by = ?, lease_expires_at = ?
+    WHERE ${a} = ? AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?)`).run(n,new Date(Date.now()+t).toISOString(),r,p)).changes!==1)return!1;let s=!1;const c=setInterval(()=>{o.prepare(`UPDATE ${e} SET lease_expires_at = ? WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ?`).run(new Date(Date.now()+t).toISOString(),r,n,new Date().toISOString()).then(l=>{l.changes!==1&&(s=!0)},()=>{s=!0})},Math.max(1,Math.floor(t/3)));c.unref();const m={async assertOwned(){if(s)throw new w(e,r);if(await(await _()).prepare(`SELECT 1 AS held FROM ${e} WHERE ${a} = ? AND claimed_by = ? AND lease_expires_at > ? FOR UPDATE`).get(r,n,new Date().toISOString())===void 0)throw s=!0,new w(e,r)}};try{await u(m)}finally{clearInterval(c),await o.prepare(`UPDATE ${e} SET claimed_by = NULL, lease_expires_at = NULL WHERE ${a} = ? AND claimed_by = ?`).run(r,n)}return!0}i(L,"withRowClaim");export{w as ClaimLostError,L as withRowClaim};

package/dist/daemon/runs/observer.js

@@ -1,6 +1,6 @@
-var b=Object.defineProperty;var l=(t,e)=>b(t,"name",{value:e,configurable:!0});import{randomUUID as y}from"node:crypto";import{openDb as d,withCanvasWrite as C}from"../../core/canvas/db.js";import{claimReconcilerLease as F,releaseReconcilerLease as O,renewReconcilerLease as M}from"../../core/canvas/reconciler-leases.js";import{getNode as R}from"../../core/canvas/canvas.js";import{BrokerClient as x,SocketTransport as D}from"../../core/broker-client/client.js";import{liveExecutionForNode as N}from"../../core/canvas/executions.js";import{runnerForExecution as H}from"../../core/runtime/runner/index.js";import{connectWssView as W}from"../../core/runtime/view-socket.js";import{ViewLinkTransport as q}from"./view-transport.js";import{emitRunEvent as g,emitRunSettlement as A,emitRunStatus as U}from"../../core/runs/events.js";import{cancelOpenQuestions as j}from"../../core/runs/questions.js";import{emitEvent as c}from"../../core/events/emit.js";import{toRunOutcomeDTO as J}from"../api/map.js";import{readOutputResult as P}from"../../core/runtime/output-result.js";const V=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),p=new Map,h="run-observer",E=1e4,B=250;async function w(t,e,i){return await C(async s=>await s.prepare("SELECT 1 AS held FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ? FOR SHARE").get(h,t,e,new Date().toISOString())===void 0?{held:!1}:{held:!0,value:await i()})}l(w,"whileHoldingLease");class v{static{l(this,"RunObserver")}nodeId;runId;owner;client;timer;status;piPid=null;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;lastRenewAt=Date.now();released=Promise.resolve();polling;frames=Promise.resolve();constructor(e,i,s,n,r){this.nodeId=e,this.runId=i,this.owner=s,this.status=n,this.outcomeRevision=r,this.timer=setInterval(()=>{this.pollOrClose()},250),this.timer.unref(),p.set(e,this)}static async start(e,i,s){const n=await d(),r=await n.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(e,e===i?"node.settled":"child.settled"),a=await n.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(e),o=new v(e,i,s,a?JSON.parse(a.body_json).status:void 0,r?JSON.parse(r.body_json).outcome.revision:void 0);return await o.writeStream(null,null),await o.poll(),o}async pollOrClose(){try{await this.poll()}catch(e){this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:e})}}poll(){return this.polling===void 0&&(this.polling=this.pollOnce().finally(()=>{this.polling=void 0})),this.polling}async holdLease(){return Date.now()-this.lastRenewAt<E/3?!0:await M(h,this.nodeId,this.owner,E)?(this.lastRenewAt=Date.now(),!0):!1}async writeStream(e,i,s=!1){return(await(await d()).prepare(`UPDATE nodes SET ${s?"":`stream_message_seq = ${e===null?"NULL":"(SELECT event_cursor FROM nodes WHERE node_id = ?)"},`} stream_block = ?
-      WHERE node_id = ? AND EXISTS (SELECT 1 FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?)`).run(...e===null||s?[]:[this.runId],i===null?null:JSON.stringify(i),this.nodeId,h,this.nodeId,this.owner,new Date().toISOString())).changes===1}async pollOnce(){if(this.closed)return;if(!await this.holdLease()){this.close();return}const e=await R(this.nodeId);if(!e||e.run_id!==this.runId||!await T(this.runId)){this.close();return}this.piPid=e.pi_pid??null;const i=e.pi_pid===null?"dormant":e.status;if(this.status!==i){if(!(await w(this.nodeId,this.owner,()=>U(this.runId,this.nodeId,i,e.intent??null))).held){this.close();return}this.status=i}const s=await J(e);if(s&&s.revision!==this.outcomeRevision&&Date.now()-Date.parse(s.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const a=await(await d()).prepare("SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const o of a)await I(o.node_id,this.runId),await p.get(o.node_id)?.poll()}this.outcomeRevision=s.revision;const n=this.nodeId===this.runId&&s.kind==="result"?await P(this.nodeId):void 0;if(!(await w(this.nodeId,this.owner,()=>this.nodeId===this.runId?A(this.runId,this.nodeId,"node.settled",{outcome:s,usage:JSON.parse(e.usage_json??V),...n===void 0?{}:{result:n}}):A(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:s}))).held){this.close();return}if(e.grantee&&e.grantee!=="app:terminal"){const a=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};j(a,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(o=>c({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:o}))}}e.pi_pid===null?this.close():await this.connect()}async settlePoll(){await this.polling?.catch(()=>{}),await this.frames,await this.released}async viewAdmitted(){const e=await N(this.nodeId);return!(e?.runner==="blaxel"&&e.idleAt!==null)}async dropView(){const e=this.client;e&&(this.client=void 0,await this.frames,await this.flushDeltas(),this.discardDeltas(),this.openAssistant=!1,this.block=null,await this.writeStream(null,null),e.close())}async connect(){if(this.closed)return;if(!await this.viewAdmitted()){await this.dropView();return}if(this.client)return;const e=await Y(this.nodeId),i=await this.viewAdmitted();if(this.client||this.closed||!i){e.destroy();return}const s=new x(this.nodeId,{transport:e});this.client=s,s.on("connect",()=>s.send({type:"hello",role:"observer",client_id:y(),attends:!1}));const n=l(async r=>{if(!(this.closed||this.client!==s)){if(r.type==="message_start"&&r.message.role==="assistant"){if(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",!await this.writeStream("run-cursor",null)){this.close();return}this.assistantStartSequence=Number((await(await d()).prepare("SELECT stream_message_seq FROM nodes WHERE node_id = ?").get(this.nodeId)).stream_message_seq)}if(r.type==="message_update"&&r.message.role==="assistant"&&this.openAssistant){const a=r.message.content;if(!Array.isArray(a))return;const o=a.at(-1);if(o&&typeof o=="object"&&"type"in o&&(o.type==="toolCall"?this.block={kind:"tool_call",name:"name"in o&&typeof o.name=="string"?o.name:""}:(o.type==="text"||o.type==="thinking")&&(this.block={kind:o.type})),!await this.writeStream(null,this.block,!0)){this.close();return}const f=a.filter(u=>u?.type==="thinking").map(u=>u.thinking).join(""),_=a.filter(u=>u?.type==="text").map(u=>u.text).join(""),k=f.startsWith(this.thinking)?f.slice(this.thinking.length):f,L=_.startsWith(this.output)?_.slice(this.output.length):_;this.thinking=f,this.output=_,this.pendingThinking+=k,this.pendingOutput+=L,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>{this.flushDeltas().catch(u=>c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:u}))},B),this.flushTimer.unref())}r.type==="message_end"&&r.message.role==="assistant"&&(await this.flushDeltas(),this.openAssistant=!1,this.block=null,await this.writeStream(null,null))}},"handleFrame");s.on("frame",r=>{this.frames=this.frames.then(()=>n(r)).catch(a=>{this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:a})})}),s.on("error",()=>{}),s.on("close",()=>{this.client===s&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null,this.released=this.released.then(async()=>{await this.writeStream(null,null)}).catch(()=>{}))}),s.connect()}async flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const e=this.pendingThinking,i=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!e&&!i)return;(await w(this.nodeId,this.owner,async()=>{if(await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}e&&await g(this.runId,this.nodeId,"node.thinking.delta",{delta:e}),i&&await g(this.runId,this.nodeId,"node.output_text.delta",{delta:i})})).held||this.close()}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){if(this.closed)return;this.closed=!0,this.discardDeltas(),clearInterval(this.timer),p.delete(this.nodeId),this.released=this.released.then(async()=>{await this.writeStream(null,null),await O(h,this.nodeId,this.owner)}).then(()=>{}).catch(i=>{c({level:"error",event:"daemon.runs.observer_release_failed",node_id:this.nodeId,error:i})});const e=this.client;this.client=void 0,e?.close()}}async function Y(t){const e=await N(t);if(e===null)return new D(t,{recordsViewerFaults:!1});const i=await H(e.runner).viewerAddress(e);return i.kind==="unix"?new D(t,{recordsViewerFaults:!1}):new q(t,()=>W(i.url,i.headers))}l(Y,"viewTransport");async function ue(t){const e=await(await d()).prepare("SELECT stream_message_seq FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(t);return e?.stream_message_seq==null?null:Number(e.stream_message_seq)}l(ue,"openAssistantMessageStartSequence");async function ce(t){const e=await(await d()).prepare("SELECT stream_message_seq, stream_block FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(t);return e?.stream_message_seq==null||e.stream_block===null?null:JSON.parse(e.stream_block)}l(ce,"openAssistantBlock");async function he(){const t=[...p.values()];for(const e of t)e.close();await Promise.all(t.map(e=>e.settlePoll())),await Promise.all([...m.values()].map(e=>e.catch(()=>{})))}l(he,"closeRunObserversForTests");async function T(t){return await(await d()).prepare("SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(t,t)!==void 0}l(T,"runRootExists");const m=new Map;async function I(t,e){const i=m.get(t);if(i)return i;if(p.has(t))return;const s=$(t,e).finally(()=>m.delete(t));return m.set(t,s),s}l(I,"ensureRunObserver");async function $(t,e){if(!await T(e))return;const i=y();if(await F(h,t,i,E)!==void 0)try{if(t!==e){const s=await R(t);if(!await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(e,t)&&s&&Date.now()-Date.parse(s.created)<720*60*6e4&&!(await w(t,i,()=>g(e,t,"child.started",{child_node_id:t,kind:s.kind}))).held)return}await v.start(t,e,i)}catch(s){throw await O(h,t,i).catch(()=>{}),s}}l($,"startRunObserver");async function pe(){const t=new Date(Date.now()-2592e6).toISOString(),e=await(await d()).prepare(`SELECT n.node_id, n.run_id, n.pi_pid, COALESCE(n.outcome_revision, 1) AS revision, e.body_json
+var M=Object.defineProperty;var l=(t,e)=>M(t,"name",{value:e,configurable:!0});import{randomUUID as y}from"node:crypto";import{openDb as d,withCanvasWrite as C}from"../../core/canvas/db.js";import{claimReconcilerLease as F,releaseReconcilerLease as O,renewReconcilerLease as x}from"../../core/canvas/reconciler-leases.js";import{getNode as R}from"../../core/canvas/canvas.js";import{BrokerClient as H,SocketTransport as D}from"../../core/broker-client/client.js";import{liveExecutionForNode as N}from"../../core/canvas/executions.js";import{runnerForExecution as W}from"../../core/runtime/runner/index.js";import{connectWssView as q}from"../../core/runtime/view-socket.js";import{openMachineView as U}from"../machine-link/forward.js";import{ViewLinkTransport as A}from"./view-transport.js";import{emitRunEvent as g,emitRunSettlement as T,emitRunStatus as j}from"../../core/runs/events.js";import{cancelOpenQuestions as J}from"../../core/runs/questions.js";import{emitEvent as c}from"../../core/events/emit.js";import{toRunOutcomeDTO as P}from"../api/map.js";import{readOutputResult as V}from"../../core/runtime/output-result.js";const B=JSON.stringify({input_tokens:0,output_tokens:0,cache_read_tokens:0,cache_write_tokens:0,cost_usd:0}),p=new Map,h="run-observer",E=1e4,Y=250;async function m(t,e,i){return await C(async n=>await n.prepare("SELECT 1 AS held FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ? FOR SHARE").get(h,t,e,new Date().toISOString())===void 0?{held:!1}:{held:!0,value:await i()})}l(m,"whileHoldingLease");class v{static{l(this,"RunObserver")}nodeId;runId;owner;client;timer;status;piPid=null;outcomeRevision;openAssistant=!1;assistantStartSequence=0;block=null;thinking="";output="";pendingThinking="";pendingOutput="";flushTimer;closed=!1;lastRenewAt=Date.now();released=Promise.resolve();polling;frames=Promise.resolve();constructor(e,i,n,s,r){this.nodeId=e,this.runId=i,this.owner=n,this.status=s,this.outcomeRevision=r,this.timer=setInterval(()=>{this.pollOrClose()},250),this.timer.unref(),p.set(e,this)}static async start(e,i,n){const s=await d(),r=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = ? ORDER BY sequence_number DESC LIMIT 1").get(e,e===i?"node.settled":"child.settled"),a=await s.prepare("SELECT body_json FROM run_events WHERE node_id = ? AND type = 'node.status.changed' ORDER BY sequence_number DESC LIMIT 1").get(e),o=new v(e,i,n,a?JSON.parse(a.body_json).status:void 0,r?JSON.parse(r.body_json).outcome.revision:void 0);return await o.writeStream(null,null),await o.poll(),o}async pollOrClose(){try{await this.poll()}catch(e){this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:e})}}poll(){return this.polling===void 0&&(this.polling=this.pollOnce().finally(()=>{this.polling=void 0})),this.polling}async holdLease(){return Date.now()-this.lastRenewAt<E/3?!0:await x(h,this.nodeId,this.owner,E)?(this.lastRenewAt=Date.now(),!0):!1}async writeStream(e,i,n=!1){return(await(await d()).prepare(`UPDATE nodes SET ${n?"":`stream_message_seq = ${e===null?"NULL":"(SELECT event_cursor FROM nodes WHERE node_id = ?)"},`} stream_block = ?
+      WHERE node_id = ? AND EXISTS (SELECT 1 FROM reconciler_leases WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?)`).run(...e===null||n?[]:[this.runId],i===null?null:JSON.stringify(i),this.nodeId,h,this.nodeId,this.owner,new Date().toISOString())).changes===1}async pollOnce(){if(this.closed)return;if(!await this.holdLease()){this.close();return}const e=await R(this.nodeId);if(!e||e.run_id!==this.runId||!await I(this.runId)){this.close();return}this.piPid=e.pi_pid??null;const i=e.pi_pid===null?"dormant":e.status;if(this.status!==i){if(!(await m(this.nodeId,this.owner,()=>j(this.runId,this.nodeId,i,e.intent??null))).held){this.close();return}this.status=i}const n=await P(e);if(n&&n.revision!==this.outcomeRevision&&Date.now()-Date.parse(n.settled_at)<720*60*6e4){if(this.nodeId===this.runId){const a=await(await d()).prepare("SELECT node_id FROM nodes WHERE run_id = ? AND node_id != ? ORDER BY created DESC").all(this.runId,this.runId);for(const o of a)await k(o.node_id,this.runId),await p.get(o.node_id)?.poll()}this.outcomeRevision=n.revision;const s=this.nodeId===this.runId&&n.kind==="result"?await V(this.nodeId):void 0;if(!(await m(this.nodeId,this.owner,()=>this.nodeId===this.runId?T(this.runId,this.nodeId,"node.settled",{outcome:n,usage:JSON.parse(e.usage_json??B),...s===void 0?{}:{result:s}}):T(this.runId,this.nodeId,"child.settled",{child_node_id:this.nodeId,outcome:n}))).held){this.close();return}if(e.grantee&&e.grantee!=="app:terminal"){const a=this.nodeId===this.runId?{run_id:this.runId}:{node_id:this.nodeId};J(a,this.nodeId===this.runId?"the run settled":"the asking node was closed").catch(o=>c({level:"error",event:"daemon.runs.question_cancel_failed",node_id:this.nodeId,error:o}))}}e.pi_pid===null?this.close():await this.connect()}async settlePoll(){await this.polling?.catch(()=>{}),await this.frames,await this.released}async viewAdmitted(){const e=await N(this.nodeId);return!(e?.runner==="blaxel"&&e.idleAt!==null)}async dropView(){const e=this.client;e&&(this.client=void 0,await this.frames,await this.flushDeltas(),this.discardDeltas(),this.openAssistant=!1,this.block=null,await this.writeStream(null,null),e.close())}async connect(){if(this.closed)return;if(!await this.viewAdmitted()){await this.dropView();return}if(this.client)return;const e=await $(this.nodeId),i=await this.viewAdmitted();if(this.client||this.closed||!i){e.destroy();return}const n=new H(this.nodeId,{transport:e});this.client=n,n.on("connect",()=>n.send({type:"hello",role:"observer",client_id:y(),attends:!1}));const s=l(async r=>{if(!(this.closed||this.client!==n)){if(r.type==="message_start"&&r.message.role==="assistant"){if(this.discardDeltas(),this.openAssistant=!0,this.block=null,this.thinking="",this.output="",!await this.writeStream("run-cursor",null)){this.close();return}this.assistantStartSequence=Number((await(await d()).prepare("SELECT stream_message_seq FROM nodes WHERE node_id = ?").get(this.nodeId)).stream_message_seq)}if(r.type==="message_update"&&r.message.role==="assistant"&&this.openAssistant){const a=r.message.content;if(!Array.isArray(a))return;const o=a.at(-1);if(o&&typeof o=="object"&&"type"in o&&(o.type==="toolCall"?this.block={kind:"tool_call",name:"name"in o&&typeof o.name=="string"?o.name:""}:(o.type==="text"||o.type==="thinking")&&(this.block={kind:o.type})),!await this.writeStream(null,this.block,!0)){this.close();return}const f=a.filter(u=>u?.type==="thinking").map(u=>u.thinking).join(""),_=a.filter(u=>u?.type==="text").map(u=>u.text).join(""),L=f.startsWith(this.thinking)?f.slice(this.thinking.length):f,b=_.startsWith(this.output)?_.slice(this.output.length):_;this.thinking=f,this.output=_,this.pendingThinking+=L,this.pendingOutput+=b,(this.pendingThinking||this.pendingOutput)&&!this.flushTimer&&(this.flushTimer=setTimeout(()=>{this.flushDeltas().catch(u=>c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:u}))},Y),this.flushTimer.unref())}r.type==="message_end"&&r.message.role==="assistant"&&(await this.flushDeltas(),this.openAssistant=!1,this.block=null,await this.writeStream(null,null))}},"handleFrame");n.on("frame",r=>{this.frames=this.frames.then(()=>s(r)).catch(a=>{this.close(),c({level:"error",event:"daemon.runs.observer_failed",node_id:this.nodeId,error:a})})}),n.on("error",()=>{}),n.on("close",()=>{this.client===n&&(this.client=void 0,this.discardDeltas(),this.openAssistant=!1,this.block=null,this.released=this.released.then(async()=>{await this.writeStream(null,null)}).catch(()=>{}))}),n.connect()}async flushDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0;const e=this.pendingThinking,i=this.pendingOutput;if(this.pendingThinking="",this.pendingOutput="",!this.openAssistant||!e&&!i)return;(await m(this.nodeId,this.owner,async()=>{if(await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND sequence_number >= ? AND type IN ('node.output_text.done', 'node.thinking.done', 'usage') LIMIT 1").get(this.runId,this.nodeId,this.assistantStartSequence)){this.openAssistant=!1;return}e&&await g(this.runId,this.nodeId,"node.thinking.delta",{delta:e}),i&&await g(this.runId,this.nodeId,"node.output_text.delta",{delta:i})})).held||this.close()}discardDeltas(){this.flushTimer&&clearTimeout(this.flushTimer),this.flushTimer=void 0,this.pendingThinking="",this.pendingOutput=""}close(){if(this.closed)return;this.closed=!0,this.discardDeltas(),clearInterval(this.timer),p.delete(this.nodeId),this.released=this.released.then(async()=>{await this.writeStream(null,null),await O(h,this.nodeId,this.owner)}).then(()=>{}).catch(i=>{c({level:"error",event:"daemon.runs.observer_release_failed",node_id:this.nodeId,error:i})});const e=this.client;this.client=void 0,e?.close()}}async function $(t){const e=await N(t);if(e===null)return new D(t,{recordsViewerFaults:!1});const i=await W(e.runner).viewerAddress(e);return i.kind==="machine"?new A(t,()=>U(i.environmentId,t)):i.kind==="unix"?new D(t,{recordsViewerFaults:!1}):new A(t,()=>q(i.url,i.headers))}l($,"viewTransport");async function he(t){const e=await(await d()).prepare("SELECT stream_message_seq FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(t);return e?.stream_message_seq==null?null:Number(e.stream_message_seq)}l(he,"openAssistantMessageStartSequence");async function pe(t){const e=await(await d()).prepare("SELECT stream_message_seq, stream_block FROM node_rows WHERE node_id = ? AND pi_pid IS NOT NULL").get(t);return e?.stream_message_seq==null||e.stream_block===null?null:JSON.parse(e.stream_block)}l(pe,"openAssistantBlock");async function fe(){const t=[...p.values()];for(const e of t)e.close();await Promise.all(t.map(e=>e.settlePoll())),await Promise.all([...w.values()].map(e=>e.catch(()=>{})))}l(fe,"closeRunObserversForTests");async function I(t){return await(await d()).prepare("SELECT 1 FROM nodes WHERE node_id = ? AND run_id = ? AND event_cursor IS NOT NULL").get(t,t)!==void 0}l(I,"runRootExists");const w=new Map;async function k(t,e){const i=w.get(t);if(i)return i;if(p.has(t))return;const n=G(t,e).finally(()=>w.delete(t));return w.set(t,n),n}l(k,"ensureRunObserver");async function G(t,e){if(!await I(e))return;const i=y();if(await F(h,t,i,E)!==void 0)try{if(t!==e){const n=await R(t);if(!await(await d()).prepare("SELECT 1 FROM run_events WHERE run_id = ? AND node_id = ? AND type = 'child.started' LIMIT 1").get(e,t)&&n&&Date.now()-Date.parse(n.created)<720*60*6e4&&!(await m(t,i,()=>g(e,t,"child.started",{child_node_id:t,kind:n.kind}))).held)return}await v.start(t,e,i)}catch(n){throw await O(h,t,i).catch(()=>{}),n}}l(G,"startRunObserver");async function _e(){const t=new Date(Date.now()-2592e6).toISOString(),e=await(await d()).prepare(`SELECT n.node_id, n.run_id, n.pi_pid, COALESCE(n.outcome_revision, 1) AS revision, e.body_json
     FROM node_rows n
     LEFT JOIN run_events e ON n.pi_pid IS NULL AND e.run_id = n.run_id AND e.node_id = n.node_id
       AND e.type = CASE WHEN n.node_id = n.run_id THEN 'node.settled' ELSE 'child.settled' END
-    WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ? AND n.outcome_reason != 'parked'))`).all(t),i=new Map;for(const n of e){const r=i.get(n.node_id)??{node_id:n.node_id,run_id:n.run_id,reported:!1};n.body_json!==null&&JSON.parse(n.body_json).outcome?.revision===n.revision&&(r.reported=!0),i.set(n.node_id,r)}const s=[...i.values()].filter(n=>!n.reported);for(const n of s)try{await I(n.node_id,n.run_id),S.delete(n.node_id)}catch(r){G(n.node_id,r)}}l(pe,"pollRunObservers");const S=new Map;function G(t,e){const i=e instanceof Error?e.message:String(e);S.get(t)!==i&&(S.set(t,i),c({level:"error",event:"daemon.runs.observer_failed",node_id:t,error:e}))}l(G,"reportObserverFailure");export{he as closeRunObserversForTests,I as ensureRunObserver,ce as openAssistantBlock,ue as openAssistantMessageStartSequence,pe as pollRunObservers};
+    WHERE n.run_id IS NOT NULL AND (n.pi_pid IS NOT NULL OR (n.outcome_at IS NOT NULL AND n.outcome_at > ? AND n.outcome_reason != 'parked'))`).all(t),i=new Map;for(const s of e){const r=i.get(s.node_id)??{node_id:s.node_id,run_id:s.run_id,reported:!1};s.body_json!==null&&JSON.parse(s.body_json).outcome?.revision===s.revision&&(r.reported=!0),i.set(s.node_id,r)}const n=[...i.values()].filter(s=>!s.reported);for(const s of n)try{await k(s.node_id,s.run_id),S.delete(s.node_id)}catch(r){Q(s.node_id,r)}}l(_e,"pollRunObservers");const S=new Map;function Q(t,e){const i=e instanceof Error?e.message:String(e);S.get(t)!==i&&(S.set(t,i),c({level:"error",event:"daemon.runs.observer_failed",node_id:t,error:e}))}l(Q,"reportObserverFailure");export{fe as closeRunObserversForTests,k as ensureRunObserver,pe as openAssistantBlock,he as openAssistantMessageStartSequence,_e as pollRunObservers};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.580",
+  "version": "0.3.582",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -145,7 +145,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.580",
+    "@north-light/identity": "0.3.582",
     "@earendil-works/chord": "1.1.0",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.580",
+  "version": "0.3.582",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.580",
+  "version": "0.3.582",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -28,7 +28,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.580",
+        "@north-light/identity": "0.3.582",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -11956,17 +11956,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -11974,21 +11974,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.580",
-        "@north-light/identity": "0.3.580"
+        "@north-light/api": "0.3.582",
+        "@north-light/identity": "0.3.582"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.580",
+      "version": "0.3.582",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.580",
-        "@north-light/identity": "0.3.580",
-        "@north-light/plugin": "0.3.580",
+        "@north-light/api": "0.3.582",
+        "@north-light/identity": "0.3.582",
+        "@north-light/plugin": "0.3.582",
         "jose": "^6.2.1"
       },
       "bin": {