@north-light/cli 0.3.561 → 0.3.563

package/dist/daemon/crtrd.js

@@ -1,17 +1,18 @@
-var Ae=Object.defineProperty;var i=(n,s)=>Ae(n,"name",{value:s,configurable:!0});import{startPeerOutbox as Me}from"./peer/outbox.js";import{reportMachineStatus as Ne}from"../core/peer/directory.js";import{createPeerRelay as Fe}from"./peer/relay-service.js";import{writeFileSync as Ge,readFileSync as Oe,rmSync as Be,mkdirSync as se,existsSync as k,chmodSync as He}from"node:fs";import{createServer as ce}from"node:http";import{dirname as je}from"node:path";import{envCrtrdTcp as qe,envDocumentPort as ze}from"../shared/env.js";import{serveCanvasConversationStore as We}from"../core/canvas/sessions.js";import{pollRunObservers as Ue}from"./runs/observer.js";import{randomBytes as de}from"node:crypto";import{apiSocketPath as Ke,crtrHome as Je}from"../core/canvas/paths.js";import{activeAppUsers as Ve}from"../core/spaces/users.js";import{ensureAppSpace as Qe,ensureUserFiles as Ye}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as C}from"../launcher/client.js";import{installLauncherEventSink as Xe}from"./launcher-events.js";import{initializeEmptyLayout as Ze,layoutNeedsMigration as er,socketDir as le}from"../core/layout.js";import{migrateLooseKeys as rr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as tr}from"../core/profiles/relocate.js";import{getNode as nr,listNodesForSupervision as pe,listParkedResidentsWithWakeMail as or,setFrozen as me}from"../core/canvas/index.js";import{createApiServer as ir}from"./api/server.js";import{createDocumentListener as ar}from"./api/document-listener.js";import{documentSecret as sr}from"./api/document-secret.js";import{createAppListener as cr}from"./api/app-listener.js";import{registerMachineEnvironment as dr}from"../core/environments.js";import{resolvePerson as lr,retireLocalPerson as pr,setPersonId as mr}from"../core/canvas/person.js";import{backfillNodeApps as fr,runPermissionsBoot as ur}from"../core/permissions/boot.js";import{onRunEventAppended as wr}from"../core/runs/events.js";import{runNotices as hr}from"./run-notices.js";import{createGrantSync as vr,GRANT_SYNC_INTERVAL_S as gr}from"../core/grants/sync.js";import{runRemovalCascades as yr}from"../core/grants/remove.js";import{sweepSandboxes as Sr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as _r}from"./cron/run-socket.js";import{hasSyncedGrants as br}from"../core/grants/mirror.js";import{readConfig as fe}from"../core/config.js";import{reconcileUndeliveredTickets as kr}from"./human/sweep.js";import{resetHandedOffDeliveries as Pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Er}from"../core/canvas/daemon-actions.js";import{sweepReviews as Rr}from"./review/sweep.js";import{pidfilePath as K}from"./pidfile.js";import{rendererWarning as Ir}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as xr}from"./supervise-cadence.js";import{createResumeDetector as Dr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Tr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Cr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Lr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as $r}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as Ar}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as Mr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as Nr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as Fr}from"./reconcilers/action-lane.js";import{endLiveExecution as Gr,migrateLegacyExecutionIdentities as Or}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as Br}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as Hr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as jr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as qr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as zr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Wr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Ur}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Kr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as ue,capturePidCommand as Jr,captureTeardownSnapshot as we,isPidAlive as Vr,isAnyPidAlive as he,killProcessTreePids as ve}from"../core/canvas/pid.js";import{bindFleet as Qr,boundFleet as Yr}from"../core/runtime/fleet.js";import{bindDaemonControl as Xr,clearDaemonControl as Zr}from"./control.js";import{execDaemon as et,spawnDaemon as rt}from"./manage.js";import{reconcileAndMigrateBoot as tt}from"../core/canvas/boot.js";import{resolveInstallId as nt}from"../core/canvas/install-id.js";import{setInstallId as ot}from"../core/runtime/nodes.js";import{claimDaemonOwnership as ge,daemonOwnershipClaimed as it,releaseDaemonOwnership as ye}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as at,closeAllDbs as J,openDb as Se,withCanvasWrite as st}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as ct}from"../core/canvas/import-sqlite.js";import{ensurePostgres as dt,ensurePostgresHealthy as lt,stopPostgres as pt,withPostgresStartLock as mt}from"./postgres/index.js";import{brokerSignalBus as V}from"./broker-signals/bus.js";import{brokerSignalStreams as ft}from"./api/handlers/broker-signals.js";import{reconcilePackages as ut}from"../core/graph/package-reconcile.js";import{startRepoSync as wt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as ht,reapStaleSpares as vt}from"../core/runtime/warm-pool.js";import{DaemonFleet as gt}from"./fleet.js";import{BOOT_MIGRATIONS as yt}from"./boot-migrations.js";import{emitEvent as m}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as St}from"../core/events/source.js";import{activateCanvasImport as _t,MigrationBlockedError as bt,runForcedCanvasImport as kt}from"../migrations/activation.js";import{convertLegacyProviderStore as Pt,convertLegacyScopedStore as Et}from"../core/scoped-state/migrate.js";import{userScopeRoot as Rt}from"../core/scope.js";import{secretsDir as It}from"../core/layout.js";function Q(n,s){return n.includes(` --canvas-home ${s} `)}i(Q,"brokerCommandDeclaresCanvasHome");function Y(n){return/ --epoch (\S+) /.exec(n)?.[1]??null}i(Y,"brokerCommandEpoch");const xt=5e3,_e=150;async function Dt(n){const s=ue();if(s===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const l=Je(),w=[];for(const[a,u]of s.processes){const y=u.command;if(u.zombie||!y.includes("broker-cli.js")||!Q(y,l)||Y(y)===n)continue;const P=y.trim().split(/\s+/).at(-1)??"";P===""||P.endsWith("broker-cli.js")||w.push({pid:a,nodeId:P})}let d=0;const t=[];for(const a of w){const u=Jr(a.pid);if(u===null||!u.includes("broker-cli.js")||!Q(u,l)||Y(u)===n||u.trim().split(/\s+/).at(-1)!==a.nodeId)continue;const y=we(a.pid,null);if(!y.reused){try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{continue}d++,t.push({pid:a.pid,snap:y})}}const g=Date.now()+xt;for(;Date.now()<g&&t.some(a=>he(a.snap.tree.length>0?a.snap.tree:[a.pid]));)await new Promise(a=>setTimeout(a,_e));let h=0;for(const a of t){const u=a.snap.tree.length>0?a.snap.tree:[a.pid];he(u)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):ve(u,"SIGKILL",a.snap.identities),h++)}h>0&&await new Promise(a=>setTimeout(a,_e));const f=ue();if(f===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[a,u]of f.processes){const y=u.command;u.zombie||!y.includes("broker-cli.js")||Q(y,l)&&Y(y)!==n&&c.push(a)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);m({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:w.length,sigtermed:d,sigkilled:h}})}i(Dt,"reapPriorEpochBrokers");const Tt=5e3;async function Ct(n){const s=Array.from(n.entries());if(s.length===0)return;for(const[t]of s){const g=(await nr(t))?.status;(g==="active"||g==="idle")&&await me(t,new Date().toISOString())}const l=s.map(([,t])=>({pid:t.pid,snap:we(t.pid,null),exited:t.handle.exited}));for(const t of l)if(!t.snap.reused)try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):process.kill(t.pid,"SIGTERM")}catch{}const w=new Promise(t=>{setTimeout(t,Tt).unref()});await Promise.allSettled(l.map(t=>Promise.race([t.exited.then(()=>{}),w])));const d=[];for(const t of l)Vr(t.pid)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):ve(t.snap.tree.length>0?t.snap.tree:[t.pid],"SIGKILL",t.snap.identities),d.push(t));await Promise.all(d.map(t=>t.exited));for(const[t]of s)await Gr(t,{reason:"daemon-teardown"});m({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:l.length,sigkilled:d.length}})}i(Ct,"teardownFleet");function be(){se(je(K()),{recursive:!0}),Ge(K(),String(process.pid),"utf8")}i(be,"writePidfile");function ke(n){return typeof n=="object"&&n!==null&&n.code==="ENOENT"}i(ke,"isMissingPathError");function Pe(n){const s=K();let l;try{l=Oe(s,"utf8")}catch(w){if(ke(w))return;throw w}if(l.trim()!==String(n))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${n})`);try{Be(s)}catch(w){if(ke(w))return;throw w}}i(Pe,"removeOwnedPidfileMirror");const Lt={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(n=>{n.catch(()=>{})},"registerDetached")},H=new Cr;Tr(n=>H.handleFatalFault(n,!1));const $t=new Lr,At=new $r,Mt=new Ar,Nt=new Mr,Ft=new Nr,Gt=new Fr,Ot=new Hr,Bt=new jr,Ht=new qr,jt=new zr,qt=new Wr,zt=new Ur;async function Wt(n=Date.now(),s=Lt){const l=Yr(),w=await H.runFleetPreamble(l);let d,t;try{const h=await pe();d=h.filter(c=>c.status==="active"||c.status==="idle"),t=h.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const f=new Set(t.map(c=>c.node_id));t.push(...(await or()).filter(c=>!f.has(c.node_id)))}catch(h){m({level:"error",event:"canvas.node_list.failed",error:h});return}await H.run(n,{rows:d,fleet:l,lifecycle:s}),await $t.run(n,{rows:d});const g=await At.run(n,{rows:[...d,...t],fleet:l});await Mt.run(n),await zt.run(n),await Nt.run(n,{lifecycle:s}),await Ft.run(n,{lifecycle:s}),await Gt.run(n,{lifecycle:s}),await Ot.run(n,{lifecycle:s}),await Bt.run(n,{lifecycle:s,fleet:l}),Ht.run(n,{lifecycle:s}),await Ue(),qt.run(n,{fleet:l,lifecycle:s}),await jt.run(n,{rows:d}),H.warnIfSaturated({fleet:l,lifecycle:s,thresholds:w,frozen:g})}i(Wt,"superviseTick");async function Ut(n){se(le(),{recursive:!0,mode:493}),process.platform==="linux"&&He(le(),493);const s=Ke();if(k(s))throw new Error(`layout refusal cannot bind occupied socket: ${s}`);const l=ce((f,c)=>{if(f.method==="GET"&&f.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((f,c)=>{l.once("error",c),l.listen(s,()=>{l.off("error",c),f()})});let w;const d=n.tcp??qe();if(d){const f=/^(.*):(\d+)$/.exec(d);if(!f||Number(f[2])>65535)throw new Error(`invalid refusal TCP address: ${d}`);const c=ce(l.listeners("request")[0]);await new Promise((a,u)=>{c.once("error",u),c.listen(Number(f[2]),f[1],()=>{c.off("error",u),a()})}),w=c}be();let t;const g=i(()=>(t??=(async()=>{l.closeAllConnections(),w?.closeAllConnections(),await Promise.all([l,w].filter(f=>f!==void 0).map(f=>new Promise((c,a)=>f.close(u=>u?a(u):c())))),Pe(process.pid),process.off("SIGINT",h),process.off("SIGTERM",h)})(),t),"cleanup"),h=i(()=>{g().then(()=>process.exit(0),f=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(f)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",h),process.on("SIGTERM",h),g}i(Ut,"serveLayoutRefusal");async function Kt(n={}){if(er())return Ut(n);Ze();const s=i(r=>{process.stderr.write(`[crtrd] ${r}
-`)},"logPostgres"),l=!!process.env.CRTR_TEST_PG_URL,{database:w,ownership:d}=l?{database:{url:await at()},ownership:await ge(de(16).toString("hex"))}:await mt(async()=>({database:await dt({log:s,startLockHeld:!0}),ownership:await ge(de(16).toString("hex"))}));if(!d.claimed){if(d.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${d.incumbentPid??"?"})
-`);else{const r=[d.incumbentPid!==void 0?`pid ${d.incumbentPid}`:void 0,d.verdict!==void 0?`verdict ${d.verdict}`:void 0].filter(S=>S!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${d.reason}${e}
-`),d.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(d.reason==="own_identity_indeterminate"||d.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const t=d.claim;let g=null,h=null,f=null,c="";try{await Dt(t.token),Et({kind:"user",root:Rt()});for(const e of tr())process.stderr.write(`[crtrd] ${e}
-`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:S}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:o}=await import("node:os");S("user",p=>{p.paths??={},p.paths.user_files??=o()})}}await Pt(It());for(const e of rr())process.stderr.write(`[crtrd] ${e}
-`);await Se(),await ct(w.url,s),await V.listen(w.url);const r=await st(async e=>(await fr(e),await ut(e)));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
-`);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
-`);for(const e of r.quarantined)process.stderr.write(`[crtrd] plugin "${e.name}" has a damaged ${e.file} (${e.reason}); ${e.movedTo!==void 0?`moved to ${e.movedTo}`:"left in place and skipped"}
-`);for(const e of r.kinds.refused)process.stderr.write(`[crtrd] plugin "${e.plugin}" kind ${e.kind} not registered: ${e.message}
-`);for(const e of r.kinds.invalid)process.stderr.write(`[crtrd] plugin "${e.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${e.error}
-`);c=await nt(),ot(c),g=await lr(),g!==null&&(mr(g.id),h=await pr(),f=await ur()),process.platform==="linux"&&k(x)&&(Xe(),await C({operation:"users.sync",users:await Ve()})),await Br(t.token)}catch(r){await V.close();try{const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");await J()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let a=!0,u,y=Promise.resolve(),P=!1,N,D;const F=new Set;let j=!1,X=!1,E,G,q,Z=!1,O,L,$,A,z;const Ee=Dr();let R,v={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},T="initial startup activation",Re=null;function Ie(){return v.phase==="ready"?null:{phase:v.phase,message:v.message,blockerPaths:v.blockerPaths}}i(Ie,"startupBlock");function b(r){F.add(r),r.finally(()=>{F.delete(r)}).catch(()=>{})}i(b,"registerDetached");const ee={acceptsDetachedWork:i(()=>a,"acceptsDetachedWork"),registerDetached:b};await wt(ee,Se);async function xe(){try{a=!1,ft.close(),await V.close(),N!==void 0&&(clearInterval(N),N=void 0),$!==void 0&&(clearInterval($),$=void 0),z?.(),z=void 0;let r;do r=y,await r;while(r!==y);for(;F.size>0;)await Promise.allSettled([...F]);if(D!==void 0&&(await Ct(D),await D.exitsSettled(),D.dispose()),q!==void 0&&await q.close(),G!==void 0&&!Z&&(Z=!0,await G.close()),A!==void 0){const e=A;A=void 0,await e.stop()}if(L!==void 0){const e=L;L=void 0,await e.close()}if(O!==void 0){const e=O;O=void 0,await e.close()}if(j||(Pe(process.pid),j=!0),j&&!X){const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");X=!0,await J()}}finally{Zr(),process.off("SIGINT",ne),process.off("SIGTERM",oe),process.off("exit",ie)}}i(xe,"cleanupAttempt");const B=1e3;let re=!1;function De(){return re?{graceMs:B}:(re=!0,_.fresh(()=>{m({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:B}})}),setTimeout(()=>{Te()},B),{graceMs:B})}i(De,"requestHandover");async function Te(){try{await I()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
-${e}
-`),process.exit(1)}try{process.platform==="linux"&&k(x)&&(await C({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await et();const r=await rt({waitForJoinLock:!0});_.fresh(()=>{m({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
-`),process.exit(1)}process.exit(0)}i(Te,"performHandover");function I(){if(E!==void 0)return E;const r=xe();return E=r,r.then(()=>{E===r&&(E=void 0)},()=>{E===r&&(E=void 0)}),r}i(I,"requestCleanup");async function te(r){try{await I(),await pt({claimed:i(async()=>{try{return await it()}finally{await J()}},"claimed")}),process.exit(0)}catch(e){process.exitCode=1;const S=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
-${S}
-`),process.exit(1)}}i(te,"handleSignal");function ne(){te("SIGINT")}i(ne,"onSigint");function oe(){te("SIGTERM")}i(oe,"onSigterm");function ie(){I().catch(()=>{})}i(ie,"onExit");async function W(r){a&&(y=y.then(r).then(()=>{},e=>{_.fresh(()=>{m({level:"error",event:"daemon.lane_job.failed",error:e})})}))}i(W,"enqueue");async function ae(){!a||P||(Ee.check()&&R&&(m({level:"info",event:"daemon.resume.detected"}),b(R().catch(()=>{}))),P=!0,await W(async()=>{if(P=!1,!!a)try{l||await lt({log:s}),await Wt(Date.now(),ee)}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(ae,"enqueueTick");let M=null;async function Ce(){if(M!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${M instanceof Error?M.message:String(M)}`};if(v.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(v.phase!=="prepared"&&v.phase!=="recovering")throw{status:409,code:"not_prepared",message:v.message};const r=v.phase==="prepared";try{return await U(),v={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw M=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}i(Ce,"admitStartup");async function Le(r){if(T!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${T})`};T="repair";try{const e=await kt({dryRun:r.dryRun});if(r.dryRun||e.pending||v.phase!=="blocked")return{result:e,resumed:!1};if(n.prepare)return v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await U(),v={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(S){const o=S instanceof Error?S.message:String(S);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:o}}}finally{T=null}}i(Le,"repairMigrations");async function $e(){u=n.intervalMs??xr;const r=t.token;D=new gt({epoch:r,enqueue:W}),Qr(D),v={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{m({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:u}})});const e=c;_.fresh(()=>{m({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})});const S=await dr(e);if(_.fresh(()=>{m({level:"info",event:"daemon.environment.registered",fields:{environment_id:S.environmentId,created:S.created}})}),_.fresh(()=>{let o;try{o=Ir()}catch{return}o!==null&&m({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:o.slice(0,2048)}})}),await _.fresh(async()=>{await Rr(),await kr(),await Pr(Date.now()),await Er(Date.now())}),await _.fresh(async()=>{const{reconcile:o,migrated:p}=await tt(Or);o.changed&&m({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:o.cleared.length,same_boot_proven:o.sameBootProven}}),p.length>0&&m({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:p.length}})}),await _.fresh(async()=>{for(const o of await pe())o.status!=="active"||o.pi_pid!=null||o.frozen_at!=null||o.respawn_not_before!=null||(await me(o.node_id,new Date().toISOString()),m({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:o.node_id}))}),await _.fresh(async()=>{await Kr();for(const o of yt){const p=o.apply();p.changed&&m({level:"info",event:"daemon.boot_migration.applied",fields:{migration:o.id,...p.summary}})}}),b(_r()),R){const o=i(()=>{b(R().catch(()=>{})),b(Ne().then(p=>A?.observe(p)).then(()=>{},p=>{m({level:"warn",event:"daemon.peer.status_failed",error:p})}))},"refresh");b(yr().catch(()=>{})),b(Sr().then(({completion:p})=>p).catch(p=>{m({level:"error",event:"daemon.grants.sweep_failed",error:p})})),o(),$=setInterval(o,gr*1e3),$.unref(),z=Me({joined:!0,register:b})}await W(async()=>{await vt(),await ht()}),N=setInterval(()=>{ae().catch(o=>m({level:"error",event:"daemon.tick.failed",error:o}))},u),await ae()}i($e,"completeStartup");function U(){return Re??=$e().catch(r=>{throw v={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(U,"resumeStartup");try{process.on("SIGINT",ne),process.on("SIGTERM",oe),process.on("exit",ie),St(),g!==null&&m({level:"info",event:"daemon.person.resolved",fields:{person_id:g.id,source:g.source}}),f!==null&&m({level:"info",event:"daemon.permissions.boot",fields:{...f}}),h!==null&&m({level:"info",event:"daemon.person.retired",fields:{person_id:h.person,joined_from:h.joinedFrom,retired:h.retired,columns:h.columns}}),O=await We();const r=t.token;Xr({epoch:r,requestHandover:De,registerDetached:b,startupBlock:Ie,admitStartup:Ce,repairMigrations:Le}),be(),G=ir({tcp:n.tcp}),await G.ready;const e=ze()??fe("user").document_port,S=ar(e,sr());try{await S.ready,q=S}catch(p){m({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:p instanceof Error?p.message:String(p)}})}const o=fe("user").runtime;o?.issuer&&(R=vr(o.issuer,b),o.unsandboxed===!0&&(A=Fe({syncGrants:R})),wr(p=>hr().appended(p))),o?.unsandboxed===!0&&o.listener!==void 0&&m({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),o!==void 0&&o.listener!==void 0&&o.unsandboxed!==!0&&(L=await cr(o,async()=>await br(o.issuer.replace(/\/$/,"")),R),await L.ready);try{await _t()}catch(p){if(T=null,p instanceof bt)return v={phase:"blocked",message:p.message,blockerPaths:p.blockerPaths},I;throw p}process.platform==="linux"&&k(x)&&(Ye(),await Qe("app:terminal"));try{n.prepare?v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await U(),v={phase:"ready",message:"",blockerPaths:[]})}finally{T=null}return I}catch(r){try{await I()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}i(Kt,"runDaemon");var xo=Kt;export{Q as brokerCommandDeclaresCanvasHome,Y as brokerCommandEpoch,xo as default,Kt as runDaemon,Wt as superviseTick,Ct as teardownFleet};
+var er=Object.defineProperty;var i=(e,l)=>er(e,"name",{value:l,configurable:!0});import{applyBootSettings as rr}from"./boot-settings.js";import{startPeerOutbox as tr}from"./peer/outbox.js";import{reportMachineStatus as nr}from"../core/peer/directory.js";import{createPeerRelay as or}from"./peer/relay-service.js";import{writeFileSync as ir,readFileSync as ar,rmSync as sr,mkdirSync as ye,existsSync as _,unlinkSync as be,chmodSync as _e}from"node:fs";import{createServer as J}from"node:http";import{dirname as cr}from"node:path";import{envCrtrdTcp as ke,envDocumentPort as lr}from"../shared/env.js";import{serveCanvasConversationStore as dr}from"../core/canvas/sessions.js";import{pollRunObservers as pr}from"./runs/observer.js";import{randomBytes as Re}from"node:crypto";import{apiSocketPath as Pe,crtrHome as mr}from"../core/canvas/paths.js";import{activeAppUsers as fr}from"../core/spaces/users.js";import{ensureAppSpace as ur,ensureUserFiles as wr}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as L}from"../launcher/client.js";import{installLauncherEventSink as hr}from"./launcher-events.js";import{initializeEmptyLayout as vr,layoutNeedsMigration as gr,socketDir as Ee}from"../core/layout.js";import{migrateLooseKeys as Sr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as yr}from"../core/profiles/relocate.js";import{getNode as br,listNodesForSupervision as Ie,listParkedResidentsWithWakeMail as _r,setFrozen as xe}from"../core/canvas/index.js";import{createApiServer as kr,closeServer as Te}from"./api/server.js";import{Router as Rr}from"./api/router.js";import{healthRoutes as Pr}from"./api/handlers/health.js";import{createDocumentListener as Er}from"./api/document-listener.js";import{documentSecret as Ir}from"./api/document-secret.js";import{createAppListener as xr}from"./api/app-listener.js";import{registerMachineEnvironment as Tr}from"../core/environments.js";import{registerBlaxelRunnerFromEnv as Dr}from"../core/secrets/blaxel-register.js";import{markCloudRole as Cr,resolvePerson as Lr,retireLocalPerson as $r,setPersonId as Ar}from"../core/canvas/person.js";import{backfillNodeApps as Nr,runPermissionsBoot as Mr}from"../core/permissions/boot.js";import{startRunNotices as Gr}from"./run-notices.js";import{startGrantEffects as Or}from"./reconcilers/grant-effects.js";import{createGrantSync as Fr,GRANT_SYNC_INTERVAL_S as Br}from"../core/grants/sync.js";import{sweepSandboxes as jr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as zr}from"./cron/run-socket.js";import{hasSyncedGrants as Hr}from"../core/grants/mirror.js";import{readConfig as se}from"../core/config.js";import{reconcileUndeliveredTickets as qr}from"./human/sweep.js";import{resetHandedOffDeliveries as Ur}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Wr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Jr}from"./review/sweep.js";import{pidfilePath as ce}from"./pidfile.js";import{rendererWarning as Kr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Vr}from"./supervise-cadence.js";import{createResumeDetector as Qr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Yr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Xr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Zr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as et}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as rt}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as tt}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as nt}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as ot}from"./reconcilers/action-lane.js";import{endLiveExecution as it,migrateLegacyExecutionIdentities as at}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as st}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as ct}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as lt}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as dt}from"./reconcilers/storage-due-work.js";import{LedgerSandboxReconciler as pt}from"./reconcilers/ledger-sandbox.js";import{LedgerStorageReconciler as mt}from"./reconcilers/ledger-storage.js";import{LedgerR2Reconciler as ft}from"./reconcilers/ledger-r2.js";import{BashDeadlineReconciler as ut}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as wt}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as ht}from"./reconcilers/structured-request-lane.js";import{RecipeImageReconciler as vt,subscribeImageBuilt as gt}from"./reconcilers/recipe-image.js";import{reconcileStructuredRequestSidecars as St}from"./structured-requests/boot.js";import{captureLivenessSnapshot as De,capturePidCommand as yt,captureTeardownSnapshot as Ce,isPidAlive as bt,isAnyPidAlive as Le,killProcessTreePids as $e}from"../core/canvas/pid.js";import{bindFleet as _t,boundFleet as kt}from"../core/runtime/fleet.js";import{bindDaemonControl as Rt,clearDaemonControl as Pt}from"./control.js";import{execDaemon as Et,spawnDaemon as It}from"./manage.js";import{reconcileAndMigrateBoot as xt}from"../core/canvas/boot.js";import{resolveInstallId as Tt}from"../core/canvas/install-id.js";import{setInstallId as Dt}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Ct,daemonOwnershipApplies as Lt,daemonOwnershipClaimed as $t,releaseDaemonOwnership as Ae}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as At,closeAllDbs as le,openDb as Ne,withCanvasWrite as Nt}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as Mt}from"../core/canvas/import-sqlite.js";import{ensurePostgres as Gt,ensurePostgresHealthy as Ot,stopPostgres as Ft,withPostgresStartLock as Bt}from"./postgres/index.js";import{brokerSignalBus as F}from"./broker-signals/bus.js";import{brokerSignalStreams as jt}from"./api/handlers/broker-signals.js";import{setRequestsStartRunObservers as zt}from"./api/handlers/broker-ops.js";import{reconcilePackages as Ht}from"../core/graph/package-reconcile.js";import{startRepoSync as qt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Ut,reapStaleSpares as Wt}from"../core/runtime/warm-pool.js";import{DaemonFleet as Jt}from"./fleet.js";import{copyNodeFilesToRows as Kt}from"../core/runtime/first-boot-rows.js";import{BOOT_MIGRATIONS as Vt}from"./boot-migrations.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as b}from"../core/events/operation-id.js";import{bindDaemonEventSource as Qt}from"../core/events/source.js";import{activateCanvasImport as Yt,MigrationBlockedError as Xt,runForcedCanvasImport as Zt}from"../migrations/activation.js";import{adoptInstallPerson as en,convertLegacyScopedStore as rn,personStoresCopied as tn}from"../core/scoped-state/migrate.js";import{readLegacyUserSnapshot as nn}from"../core/scoped-state/legacy-profiles.js";import{heldPersons as Me,installPreludeSnapshot as on}from"../core/scoped-state/person-store.js";import{loadPersonSettings as Ge}from"../core/scoped-state/person-rows.js";import{userScopeRoot as Oe}from"../core/scope.js";function de(e,l){return e.includes(` --canvas-home ${l} `)}i(de,"brokerCommandDeclaresCanvasHome");function pe(e){return/ --epoch (\S+) /.exec(e)?.[1]??null}i(pe,"brokerCommandEpoch");const an=5e3,Fe=150;async function sn(e){const l=De();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const f=mr(),m=[];for(const[n,d]of l.processes){const y=d.command;if(d.zombie||!y.includes("broker-cli.js")||!de(y,f)||pe(y)===e)continue;const k=y.trim().split(/\s+/).at(-1)??"";k===""||k.endsWith("broker-cli.js")||m.push({pid:n,nodeId:k})}let v=0;const a=[];for(const n of m){const d=yt(n.pid);if(d===null||!d.includes("broker-cli.js")||!de(d,f)||pe(d)===e||d.trim().split(/\s+/).at(-1)!==n.nodeId)continue;const y=Ce(n.pid,null);if(!y.reused){try{process.platform==="linux"&&_(x)?await L({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{continue}v++,a.push({pid:n.pid,snap:y})}}const w=Date.now()+an;for(;Date.now()<w&&a.some(n=>Le(n.snap.tree.length>0?n.snap.tree:[n.pid]));)await new Promise(n=>setTimeout(n,Fe));let g=0;for(const n of a){const d=n.snap.tree.length>0?n.snap.tree:[n.pid];Le(d)&&(process.platform==="linux"&&_(x)?await L({operation:"stop",pid:n.pid}):$e(d,"SIGKILL",n.snap.identities),g++)}g>0&&await new Promise(n=>setTimeout(n,Fe));const p=De();if(p===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[n,d]of p.processes){const y=d.command;d.zombie||!y.includes("broker-cli.js")||de(y,f)&&pe(y)!==e&&c.push(n)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:v,sigkilled:g}})}i(sn,"reapPriorEpochBrokers");const cn=5e3;async function ln(e){const l=Array.from(e.entries());if(l.length===0)return;for(const[a]of l){const w=(await br(a))?.status;(w==="active"||w==="idle")&&await xe(a,new Date().toISOString())}const f=l.map(([,a])=>({pid:a.pid,snap:Ce(a.pid,null),exited:a.handle.exited}));for(const a of f)if(!a.snap.reused)try{process.platform==="linux"&&_(x)?await L({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{}const m=new Promise(a=>{setTimeout(a,cn).unref()});await Promise.allSettled(f.map(a=>Promise.race([a.exited.then(()=>{}),m])));const v=[];for(const a of f)bt(a.pid)&&(process.platform==="linux"&&_(x)?await L({operation:"stop",pid:a.pid}):$e(a.snap.tree.length>0?a.snap.tree:[a.pid],"SIGKILL",a.snap.identities),v.push(a));await Promise.all(v.map(a=>a.exited));for(const[a]of l)await it(a,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:f.length,sigkilled:v.length}})}i(ln,"teardownFleet");function Be(){ye(cr(ce()),{recursive:!0}),ir(ce(),String(process.pid),"utf8")}i(Be,"writePidfile");function je(e){return typeof e=="object"&&e!==null&&e.code==="ENOENT"}i(je,"isMissingPathError");function ze(e){const l=ce();let f;try{f=ar(l,"utf8")}catch(m){if(je(m))return;throw m}if(f.trim()!==String(e))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${e})`);try{sr(l)}catch(m){if(je(m))return;throw m}}i(ze,"removeOwnedPidfileMirror");const dn={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(e=>{e.catch(()=>{})},"registerDetached")},K=new Xr;Yr(e=>K.handleFatalFault(e,!1));const pn=new Zr,mn=new et,fn=new rt,un=new tt,wn=new nt,hn=new ot,vn=new ct,gn=new lt,Sn=new dt,yn=new pt,bn=new mt,_n=new ft,kn=new ut,Rn=new wt,Pn=new ht,En=new vt;async function In(e=Date.now(),l=dn){const f=kt(),m=await K.runFleetPreamble(f);let v,a;try{const g=await Ie();v=g.filter(c=>c.status==="active"||c.status==="idle"),a=g.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const p=new Set(a.map(c=>c.node_id));a.push(...(await _r()).filter(c=>!p.has(c.node_id)))}catch(g){u({level:"error",event:"canvas.node_list.failed",error:g});return}await K.run(e,{rows:v,fleet:f,lifecycle:l}),await pn.run(e,{rows:v});const w=await mn.run(e,{rows:[...v,...a],fleet:f});await fn.run(e),await Pn.run(e),await En.run(e,{lifecycle:l}),await un.run(e,{lifecycle:l}),await wn.run(e,{lifecycle:l}),await hn.run(e,{lifecycle:l}),await vn.run(e,{lifecycle:l}),await gn.run(e,{lifecycle:l,fleet:f}),Sn.run(e,{lifecycle:l}),yn.run(e,{lifecycle:l}),bn.run(e,{lifecycle:l}),_n.run(e,{lifecycle:l}),await pr(),Rn.run(e,{fleet:f,lifecycle:l}),await kn.run(e,{rows:v}),K.warnIfSaturated({fleet:f,lifecycle:l,thresholds:m,frozen:w})}i(In,"superviseTick");function V(e){return e!=="api"}i(V,"roleRunsLanes");function $(e){return e!=="reconcile"}i($,"roleServesRequests");const xn=66e4;function Tn(e){const l=new Rr().registerAll(Pr(()=>null).filter(n=>n.pattern==="/healthz")),f=i((n,d)=>{const y=(n.url??"/").split("?")[0];if((n.method??"GET").toUpperCase()==="GET"&&y==="/healthz"){l.handle(n,d,null,{class:"person"});return}d.writeHead(404,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"not_found",message:"this process serves only /healthz",origin:"runtime"}}))},"handler"),m=Pe();_(m)&&be(m);const v=J(f),a=new Promise((n,d)=>{v.once("error",d),v.listen(m,()=>{v.off("error",d),_e(m,384),n()})});let w,g=null;const p=e.tcp??ke()??se("user").api.tcp;if(p!==void 0&&p!==""){const n=/^(.*):(\d+)$/.exec(p.trim());if(n===null||Number(n[2])>65535)throw new Error(`invalid reconcile probe TCP address: ${p}`);const d=J(f);d.on("error",y=>u({event:"api.server.failed",level:"error",error:y})),d.listen(Number(n[2]),n[1]===""?"0.0.0.0":n[1],()=>{g=`${n[1]===""?"0.0.0.0":n[1]}:${n[2]}`}),w=d}let c=!1;return{ready:a,tcpAddress:i(()=>g,"tcpAddress"),async close(n={}){if(!c){c=!0,await Promise.all([Te(v,new Set,n.graceMs),w===void 0?Promise.resolve():Te(w,new Set,n.graceMs)]);try{_(m)&&be(m)}catch{}}}}}i(Tn,"createProbeServer");const Dn="daemon is draining for shutdown and admits no new work";async function Cn(e){ye(Ee(),{recursive:!0,mode:493}),process.platform==="linux"&&_e(Ee(),493);const l=Pe();if(_(l))throw new Error(`layout refusal cannot bind occupied socket: ${l}`);const f=J((p,c)=>{if(p.method==="GET"&&p.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((p,c)=>{f.once("error",c),f.listen(l,()=>{f.off("error",c),p()})});let m;const v=e.tcp??ke();if(v){const p=/^(.*):(\d+)$/.exec(v);if(!p||Number(p[2])>65535)throw new Error(`invalid refusal TCP address: ${v}`);const c=J(f.listeners("request")[0]);await new Promise((n,d)=>{c.once("error",d),c.listen(Number(p[2]),p[1],()=>{c.off("error",d),n()})}),m=c}Be();let a;const w=i(()=>(a??=(async()=>{f.closeAllConnections(),m?.closeAllConnections(),await Promise.all([f,m].filter(p=>p!==void 0).map(p=>new Promise((c,n)=>p.close(d=>d?n(d):c())))),ze(process.pid),process.off("SIGINT",g),process.off("SIGTERM",g)})(),a),"cleanup"),g=i(()=>{w().then(()=>process.exit(0),p=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(p)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",g),process.on("SIGTERM",g),w}i(Cn,"serveLayoutRefusal");async function Ln(e={}){if(gr())return Cn(e);vr();const l=i(r=>{process.stderr.write(`[crtrd] ${r}
+`)},"logPostgres"),f=!!process.env.CRTR_TEST_PG_URL,m=Lt(e.role);zt(V(e.role));const v=i(async()=>m?await Ct(Re(16).toString("hex")):{claimed:!0,claim:{pid:process.pid,pid_identity:"",token:Re(16).toString("hex")}},"claimOwnership"),{database:a,ownership:w}=f?{database:{url:await At()},ownership:await v()}:await Bt(async()=>({database:await Gt({log:l,startLockHeld:!0}),ownership:await v()}));if(!w.claimed){if(w.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${w.incumbentPid??"?"})
+`);else{const r=[w.incumbentPid!==void 0?`pid ${w.incumbentPid}`:void 0,w.verdict!==void 0?`verdict ${w.verdict}`:void 0].filter(h=>h!==void 0),o=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${w.reason}${o}
+`),w.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(w.reason==="own_identity_indeterminate"||w.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const g=w.claim;let p=null,c=null,n=null,d,y="";try{m&&await sn(g.token);for(const t of yr())process.stderr.write(`[crtrd] ${t}
+`);for(const t of await Sr())process.stderr.write(`[crtrd] ${t}
+`);await Ne(),await Mt(a.url,l);const r=e.role===void 0&&!await tn();let o=null;r&&(rn({kind:"user",root:Oe()}),o=nn(Oe())),e.role===void 0&&on(o??{profiles:[],defaults:{},settings:{},state:{}}),await F.listen(a.url);const h=i(t=>{Ge(t).catch(O=>u({level:"error",event:"daemon.settings.reload_failed",error:O}))},"reloadPerson");d=await F.subscribe(t=>{Me().includes(t)&&h(t)},{kinds:["settings"],onListenStart:i(()=>{for(const t of Me())h(t)},"onListenStart")}),e.role!==void 0&&Cr();const s=await Nt(async t=>(await Nr(t),await Ht(t)));for(const t of s.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${t.message}
+`);for(const t of s.invalid)process.stderr.write(`[crtrd] package file is not a document: ${t.path}: ${t.error}
+`);for(const t of s.quarantined)process.stderr.write(`[crtrd] plugin "${t.name}" has a damaged ${t.file} (${t.reason}); ${t.movedTo!==void 0?`moved to ${t.movedTo}`:"left in place and skipped"}
+`);for(const t of s.kinds.refused)process.stderr.write(`[crtrd] plugin "${t.plugin}" kind ${t.kind} not registered: ${t.message}
+`);for(const t of s.kinds.invalid)process.stderr.write(`[crtrd] plugin "${t.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${t.error}
+`);if(y=await Tt(),Dt(y),e.role===void 0&&(p=await Lr(),p!==null)){if(Ar(p.id),await en(p.id),process.platform==="darwin"){const{readConfig:t,updateConfig:O}=await import("../core/config.js");if(t("user").paths?.user_files===void 0){const{homedir:Ze}=await import("node:os");await O("user",Se=>{Se.paths??={},Se.paths.user_files??=Ze()})}}c=await $r(),c.joinedFrom!==null&&await Ge(c.person),await rr(),n=await Mr()}if(process.platform==="linux"&&_(x)&&(hr(),await L({operation:"users.sync",users:await fr()})),m){await st(g.token);const t=await Kt();Object.values(t).some(O=>O>0)&&process.stderr.write(`[crtrd] node files copied into rows: ${JSON.stringify(t)}
+`)}}catch(r){await F.close();try{if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}await le()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}let k=!0,Q,B=Promise.resolve(),Y=!1,j,X,Z,T;const z=new Set;let ee=!1,me=!1,P,D,re,H=!1,q,A,N,M,te,ne,oe;const He=Qr();let E,S={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},C="initial startup activation",qe=null;function Ue(){return S.phase==="ready"?null:{phase:S.phase,message:S.message,blockerPaths:S.blockerPaths}}i(Ue,"startupBlock");function R(r){z.add(r),r.finally(()=>{z.delete(r)}).catch(()=>{})}i(R,"registerDetached");const fe={acceptsDetachedWork:i(()=>k,"acceptsDetachedWork"),registerDetached:R};V(e.role)&&await qt(fe,Ne);async function We(){try{k=!1,jt.close(),X?.(),X=void 0,Z?.(),d?.(),Z=void 0,ne?.(),ne=void 0,await F.close(),j!==void 0&&(clearInterval(j),j=void 0),N!==void 0&&(clearInterval(N),N=void 0),oe?.(),oe=void 0;let r;do r=B,await r;while(r!==B);for(;z.size>0;)await Promise.allSettled([...z]);if(T!==void 0&&(m&&await ln(T),await T.exitsSettled(),T.dispose()),re!==void 0&&await re.close(),D!==void 0&&!H&&(H=!0,await D.close()),te?.(),te=void 0,M!==void 0){const o=M;M=void 0,await o.stop()}if(A!==void 0){const o=A;A=void 0,await o.close()}if(q!==void 0){const o=q;q=void 0,await o.close()}if(ee||(ze(process.pid),ee=!0),ee&&!me){if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}me=!0,await le()}}finally{Pt(),process.off("SIGINT",he),process.off("SIGTERM",ve),process.off("exit",ge)}}i(We,"cleanupAttempt");const U=1e3;let ue=!1;function Je(){return ue?{graceMs:U}:(ue=!0,b.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:U}})}),setTimeout(()=>{Ke()},U),{graceMs:U})}i(Je,"requestHandover");async function Ke(){try{await I()}catch(r){const o=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+${o}
+`),process.exit(1)}try{process.platform==="linux"&&_(x)&&(await L({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Et();const r=await It({waitForJoinLock:!0});b.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){b.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const o=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${o}
+`),process.exit(1)}process.exit(0)}i(Ke,"performHandover");function I(){if(P!==void 0)return P;const r=We();return P=r,r.then(()=>{P===r&&(P=void 0)},()=>{P===r&&(P=void 0)}),r}i(I,"requestCleanup");async function Ve(){S={phase:"initializing",message:Dn,blockerPaths:[]},D!==void 0&&!H&&(H=!0,await D.close({graceMs:xn}))}i(Ve,"drainApi");async function we(r){try{r==="SIGTERM"&&e.role==="api"&&await Ve(),await I(),await Ft({claimed:i(async()=>{try{return m&&await $t()}finally{await le()}},"claimed")}),process.exit(0)}catch(o){process.exitCode=1;const h=o instanceof Error?o.stack??`${o.name}: ${o.message}`:String(o);process.stderr.write(`[crtrd] ${r} cleanup failed:
+${h}
+`),process.exit(1)}}i(we,"handleSignal");function he(){we("SIGINT")}i(he,"onSigint");function ve(){we("SIGTERM")}i(ve,"onSigterm");function ge(){I().catch(()=>{})}i(ge,"onExit");async function ie(r){k&&(B=B.then(r).then(()=>{},o=>{b.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:o})})}))}i(ie,"enqueue");async function W(){!k||Y||(He.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),R(E().catch(()=>{}))),Y=!0,await ie(async()=>{if(Y=!1,!!k)try{f||await Ot({log:l}),await In(Date.now(),fe)}catch(r){b.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(W,"enqueueTick");let G=null;async function Qe(){if(G!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${G instanceof Error?G.message:String(G)}`};if(S.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(S.phase!=="prepared"&&S.phase!=="recovering")throw{status:409,code:"not_prepared",message:S.message};const r=S.phase==="prepared";try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(o){throw G=o,{status:500,code:"admission_failed",message:`daemon admission failed: ${o instanceof Error?o.message:String(o)}`}}}i(Qe,"admitStartup");async function Ye(r){if(C!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${C})`};C="repair";try{const o=await Zt({dryRun:r.dryRun});if(r.dryRun||o.pending||S.phase!=="blocked")return{result:o,resumed:!1};if(e.prepare)return S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:o,resumed:!1};try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},{result:o,resumed:!0}}catch(h){const s=h instanceof Error?h.message:String(h);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:o,resumed:!1,resumeError:s}}}finally{C=null}}i(Ye,"repairMigrations");async function Xe(){Q=e.intervalMs??Vr;const r=g.token;T=new Jt({epoch:r,enqueue:ie}),_t(T),S={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},b.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:Q}})});const o=y;b.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:o}})});const h=await Tr(o);if(b.fresh(()=>{u({level:"info",event:"daemon.environment.registered",fields:{environment_id:h.environmentId,created:h.created}})}),b.fresh(()=>{let s;try{s=Kr()}catch{return}s!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:s.slice(0,2048)}})}),e.role!==void 0&&Dr(),!V(e.role)){S={phase:"ready",message:"",blockerPaths:[]};return}if(await b.fresh(async()=>{await Jr(),await qr(),await Ur(Date.now()),await Wr(Date.now())}),await b.fresh(async()=>{const{reconcile:s,migrated:t}=await xt(at);s.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:s.cleared.length,same_boot_proven:s.sameBootProven}}),t.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:t.length}})}),await b.fresh(async()=>{for(const s of await Ie())s.status!=="active"||s.pi_pid!=null||s.frozen_at!=null||s.respawn_not_before!=null||(await xe(s.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:s.node_id}))}),await b.fresh(async()=>{await St();for(const s of Vt){const t=s.apply();t.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:s.id,...t.summary}})}}),R(zr()),E){const s=i(()=>{R(E().catch(()=>{})),R(nr().then(t=>M?.observe(t)).then(()=>{},t=>{u({level:"warn",event:"daemon.peer.status_failed",error:t})}))},"refresh");R(jr().then(({completion:t})=>t).catch(t=>{u({level:"error",event:"daemon.grants.sweep_failed",error:t})})),s(),N=setInterval(s,Br*1e3),N.unref(),oe=tr({joined:!0,register:R})}await ie(async()=>{await Wt(),await Ut()}),j=setInterval(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},Q),X=await gt(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))}),Z=await F.subscribe(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},{kinds:["spend_limits"]}),await W()}i(Xe,"completeStartup");function ae(){return qe??=Xe().catch(r=>{throw S={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(ae,"resumeStartup");try{process.on("SIGINT",he),process.on("SIGTERM",ve),process.on("exit",ge),Qt(),p!==null&&u({level:"info",event:"daemon.person.resolved",fields:{person_id:p.id,source:p.source}}),n!==null&&u({level:"info",event:"daemon.permissions.boot",fields:{...n}}),c!==null&&u({level:"info",event:"daemon.person.retired",fields:{person_id:c.person,joined_from:c.joinedFrom,retired:c.retired,columns:c.columns}}),$(e.role)&&(q=await dr());const r=g.token;Rt({epoch:r,requestHandover:Je,registerDetached:R,startupBlock:Ue,admitStartup:Qe,repairMigrations:Ye}),Be(),D=$(e.role)?kr({tcp:e.tcp}):Tn(e),await D.ready;const o=lr()??se("user").document_port;if($(e.role)){const s=Er(o,Ir());try{await s.ready,re=s}catch(t){u({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:o,error:t instanceof Error?t.message:String(t)}})}}const h=se("user").runtime;if(h?.issuer&&(E=Fr(h.issuer,R),h.unsandboxed===!0&&(M=or({syncGrants:E}))),$(e.role)&&(te=await Gr()),ne=await Or({serves:$(e.role),runsLanes:V(e.role)}),h?.unsandboxed===!0&&h.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),$(e.role)&&h!==void 0&&h.listener!==void 0&&h.unsandboxed!==!0&&(A=await xr(h,async()=>await Hr(h.issuer.replace(/\/$/,"")),E),await A.ready),e.role===void 0)try{await Yt()}catch(s){if(C=null,s instanceof Xt)return S={phase:"blocked",message:s.message,blockerPaths:s.blockerPaths},I;throw s}process.platform==="linux"&&_(x)&&(wr(),await ur("app:terminal"));try{e.prepare?S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await ae(),S={phase:"ready",message:"",blockerPaths:[]})}finally{C=null}return I}catch(r){try{await I()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}}i(Ln,"runDaemon");var ki=Ln;export{xn as API_DRAIN_GRACE_MS,de as brokerCommandDeclaresCanvasHome,pe as brokerCommandEpoch,ki as default,V as roleRunsLanes,$ as roleServesRequests,Ln as runDaemon,In as superviseTick,ln as teardownFleet};

package/dist/daemon/delivery/run-delivery.d.ts

@@ -31,4 +31,4 @@ export interface DeliveryJob {
 }
 /** Spawn and settle one already-claimed delivery. `settle` writes the outcome
  *  to whichever durable row owns this attempt. */
-export declare function runDelivery(job: DeliveryJob, settle: (settlement: DeliverySettlement) => void): Promise<void>;
+export declare function runDelivery(job: DeliveryJob, settle: (settlement: DeliverySettlement) => void | Promise<void>): Promise<void>;

package/dist/daemon/delivery/run-delivery.js

@@ -1 +1 @@
-var C=Object.defineProperty;var l=(e,t)=>C(e,"name",{value:t,configurable:!0});import{spawn as T}from"node:child_process";import{envHomeOverride as _}from"../../shared/env.js";import{deliveryBackoffMs as g}from"../../core/canvas/delivery-contract.js";import{killProcessGroup as x}from"../../core/canvas/pid.js";import{buildOperationalEnvBase as y}from"../../core/runtime/spawn-env.js";import{getNode as R}from"../../core/canvas/canvas.js";import{appProcessSandboxed as A,launchAppProcess as O}from"../../core/spaces/process.js";const I=12e4,h=2e3,E=4*1024;function w(e,t,r){return e.spawnError!==void 0?{state:"pending",nextAttemptAt:r+g(t),failure:m("spawn_error",{message:e.spawnError.message,stderr:e.stderr})}:e.timedOut?{state:"pending",nextAttemptAt:r+g(t),failure:m("timeout",{...e.signal===null?{}:{signal:e.signal},stderr:e.stderr})}:e.signal!==null?{state:"pending",nextAttemptAt:r+g(t),failure:m("signal",{signal:e.signal,stderr:e.stderr})}:e.exitCode===0?{state:"accepted"}:e.exitCode===78?{state:"permanent_failed",failure:m("exit",{exitCode:78,stderr:e.stderr})}:{state:"pending",nextAttemptAt:r+g(t),failure:m("exit",{exitCode:e.exitCode??void 0,stderr:e.stderr})}}l(w,"deliverySettlementForOutcome");function m(e,t){return{kind:e,...t}}l(m,"failure");function D(){let e=Buffer.alloc(0);return{take(t){e=t.length>=E?t.subarray(t.length-E):Buffer.concat([e,t]).subarray(Math.max(0,e.length+t.length-E))},decode(){return e.toString("utf8")}}}l(D,"stderrTailCapture");function S(e,t,r){if(r)return JSON.stringify(e);const d=e,n=d.delivery;return JSON.stringify({...d,delivery:{...n,attempt:t}})}l(S,"documentForAttempt");function L(e,t){const r=y({targetCwd:e,targetProfileId:null}),d=_();return d!==void 0&&d!==""&&(r.CRTR_HOME=d),{...r,...t}}l(L,"deliveryEnv");async function H(e,t){const r=D();let d;try{d=S(e.document,e.attempt,e.custom!==void 0)}catch(s){t(w({exitCode:null,signal:null,timedOut:!1,spawnError:s,stderr:r.decode()},e.attempt,Date.now()));return}if(process.platform==="linux"){let s;try{if(!A())throw new Error("root launcher is required for delivery");const u=e.nodeId===null||e.nodeId===void 0?null:await R(e.nodeId),f=e.app??u?.grantee??(e.nodeId==null?"app:terminal":void 0);if(!f)throw new Error(`delivery origin ${e.nodeId} was deleted without a stored app`);const o=await O({...u?{nodeId:e.nodeId}:{},app:f,command:e.argv,cwd:e.cwd,env:{...y({targetCwd:e.cwd,targetProfileId:null,skipSpawnAllow:!0}),...e.custom?{CRTR_DELIVERY_ID:e.custom.deliveryId,CRTR_CAUSE_EVENT:e.custom.messageId}:{}},stdin:d,stdoutBytes:0,stderrBytes:E,stderrTail:!0});let a,p=!1;try{a=setTimeout(()=>{p=!0,o.stop().catch(i=>{console.error(`delivery process ${o.pid} could not be stopped:`,i)})},I),a.unref?.();const c=await o.completion;s={exitCode:c.exitCode,signal:c.signal,timedOut:p,stderr:c.stderr}}finally{a!==void 0&&clearTimeout(a)}}catch(u){s={exitCode:null,signal:null,timedOut:!1,spawnError:u,stderr:""}}t(w(s,e.attempt,Date.now()));return}let n;try{n=T(e.argv[0],e.argv.slice(1),{cwd:e.cwd,env:L(e.cwd,e.custom?{CRTR_DELIVERY_ID:e.custom.deliveryId,CRTR_CAUSE_EVENT:e.custom.messageId}:void 0),detached:!0,stdio:["pipe","pipe","pipe"]})}catch(s){t(w({exitCode:null,signal:null,timedOut:!1,spawnError:s,stderr:r.decode()},e.attempt,Date.now()));return}await new Promise(s=>{let u=!1,f=!1,o,a;const p=l(i=>{u||(u=!0,a!==void 0&&clearTimeout(a),!i.timedOut&&o!==void 0&&clearTimeout(o),n.stdin?.destroy(),n.stdout?.destroy(),n.stderr?.destroy(),t(w(i,e.attempt,Date.now())),s())},"finish");n.stdout?.on("data",()=>{}),n.stderr?.on("data",i=>r.take(i)),n.stdin?.on("error",()=>{}),n.on("error",i=>{p({exitCode:null,signal:null,timedOut:!1,spawnError:i,stderr:r.decode()})}),n.on("exit",(i,v)=>{p({exitCode:i,signal:v,timedOut:f,stderr:r.decode()})});const c=l(()=>{n.pid!==void 0&&(x(n.pid),o=setTimeout(()=>x(n.pid,"SIGKILL"),h),o.unref?.())},"killWithEscalation");a=setTimeout(()=>{f=!0,c()},I),a.unref?.(),n.stdin?.end(d,"utf8")})}l(H,"runDelivery");export{E as DELIVERY_STDERR_TAIL_BYTES,I as DELIVERY_TIMEOUT_MS,w as deliverySettlementForOutcome,H as runDelivery};
+var T=Object.defineProperty;var a=(e,t)=>T(e,"name",{value:t,configurable:!0});import{spawn as _}from"node:child_process";import{envHomeOverride as R}from"../../shared/env.js";import{deliveryBackoffMs as g}from"../../core/canvas/delivery-contract.js";import{killProcessGroup as y}from"../../core/canvas/pid.js";import{buildOperationalEnvBase as v}from"../../core/runtime/spawn-env.js";import{getNode as h}from"../../core/canvas/canvas.js";import{appProcessSandboxed as A,launchAppProcess as O}from"../../core/spaces/process.js";const I=12e4,D=2e3,w=4*1024;function E(e,t,r){return e.spawnError!==void 0?{state:"pending",nextAttemptAt:r+g(t),failure:m("spawn_error",{message:e.spawnError.message,stderr:e.stderr})}:e.timedOut?{state:"pending",nextAttemptAt:r+g(t),failure:m("timeout",{...e.signal===null?{}:{signal:e.signal},stderr:e.stderr})}:e.signal!==null?{state:"pending",nextAttemptAt:r+g(t),failure:m("signal",{signal:e.signal,stderr:e.stderr})}:e.exitCode===0?{state:"accepted"}:e.exitCode===78?{state:"permanent_failed",failure:m("exit",{exitCode:78,stderr:e.stderr})}:{state:"pending",nextAttemptAt:r+g(t),failure:m("exit",{exitCode:e.exitCode??void 0,stderr:e.stderr})}}a(E,"deliverySettlementForOutcome");function m(e,t){return{kind:e,...t}}a(m,"failure");function S(){let e=Buffer.alloc(0);return{take(t){e=t.length>=w?t.subarray(t.length-w):Buffer.concat([e,t]).subarray(Math.max(0,e.length+t.length-w))},decode(){return e.toString("utf8")}}}a(S,"stderrTailCapture");function L(e,t,r){if(r)return JSON.stringify(e);const i=e,n=i.delivery;return JSON.stringify({...i,delivery:{...n,attempt:t}})}a(L,"documentForAttempt");function B(e,t){const r=v({targetCwd:e,targetProfileId:null}),i=R();return i!==void 0&&i!==""&&(r.CRTR_HOME=i),{...r,...t}}a(B,"deliveryEnv");async function U(e,t){const r=S();let i;try{i=L(e.document,e.attempt,e.custom!==void 0)}catch(d){await t(E({exitCode:null,signal:null,timedOut:!1,spawnError:d,stderr:r.decode()},e.attempt,Date.now()));return}if(process.platform==="linux"){let d;try{if(!A())throw new Error("root launcher is required for delivery");const u=e.nodeId===null||e.nodeId===void 0?null:await h(e.nodeId),c=e.app??u?.grantee??(e.nodeId==null?"app:terminal":void 0);if(!c)throw new Error(`delivery origin ${e.nodeId} was deleted without a stored app`);const f=await O({...u?{nodeId:e.nodeId}:{},app:c,command:e.argv,cwd:e.cwd,env:{...v({targetCwd:e.cwd,targetProfileId:null,skipSpawnAllow:!0}),...e.custom?{CRTR_DELIVERY_ID:e.custom.deliveryId,CRTR_CAUSE_EVENT:e.custom.messageId}:{}},stdin:i,stdoutBytes:0,stderrBytes:w,stderrTail:!0});let s,o=!1;try{s=setTimeout(()=>{o=!0,f.stop().catch(x=>{console.error(`delivery process ${f.pid} could not be stopped:`,x)})},I),s.unref?.();const p=await f.completion;d={exitCode:p.exitCode,signal:p.signal,timedOut:o,stderr:p.stderr}}finally{s!==void 0&&clearTimeout(s)}}catch(u){d={exitCode:null,signal:null,timedOut:!1,spawnError:u,stderr:""}}await t(E(d,e.attempt,Date.now()));return}let n;try{n=_(e.argv[0],e.argv.slice(1),{cwd:e.cwd,env:B(e.cwd,e.custom?{CRTR_DELIVERY_ID:e.custom.deliveryId,CRTR_CAUSE_EVENT:e.custom.messageId}:void 0),detached:!0,stdio:["pipe","pipe","pipe"]})}catch(d){await t(E({exitCode:null,signal:null,timedOut:!1,spawnError:d,stderr:r.decode()},e.attempt,Date.now()));return}await new Promise((d,u)=>{let c=!1,f=!1,s,o;const p=a(l=>{c||(c=!0,o!==void 0&&clearTimeout(o),!l.timedOut&&s!==void 0&&clearTimeout(s),n.stdin?.destroy(),n.stdout?.destroy(),n.stderr?.destroy(),Promise.resolve(t(E(l,e.attempt,Date.now()))).then(d,u))},"finish");n.stdout?.on("data",()=>{}),n.stderr?.on("data",l=>r.take(l)),n.stdin?.on("error",()=>{}),n.on("error",l=>{p({exitCode:null,signal:null,timedOut:!1,spawnError:l,stderr:r.decode()})}),n.on("exit",(l,C)=>{p({exitCode:l,signal:C,timedOut:f,stderr:r.decode()})});const x=a(()=>{n.pid!==void 0&&(y(n.pid),s=setTimeout(()=>y(n.pid,"SIGKILL"),D),s.unref?.())},"killWithEscalation");o=setTimeout(()=>{f=!0,x()},I),o.unref?.(),n.stdin?.end(i,"utf8")})}a(U,"runDelivery");export{w as DELIVERY_STDERR_TAIL_BYTES,I as DELIVERY_TIMEOUT_MS,E as deliverySettlementForOutcome,U as runDelivery};

package/dist/daemon/due-sources.d.ts

@@ -1,5 +1,5 @@
 /** Every lane `superviseTick` runs, named as in crtrd.ts. Keep in tick order. */
-export declare const TICK_LANES: readonly ["brokerSupervision", "controllerDeath", "nodeLifecycle", "nodeDeadline", "structuredRequests", "cronLane", "deliveryLane", "actionLane", "pendingReviewSubmit", "storageMaintenance", "storageDueWork", "pollRunObservers", "faultRetryBackstop", "bashDeadline"];
+export declare const TICK_LANES: readonly ["brokerSupervision", "controllerDeath", "nodeLifecycle", "nodeDeadline", "structuredRequests", "recipeImage", "cronLane", "deliveryLane", "actionLane", "pendingReviewSubmit", "storageMaintenance", "storageDueWork", "pollRunObservers", "faultRetryBackstop", "bashDeadline", "ledgerSandbox", "ledgerStorage", "ledgerR2"];
 export interface DueSource {
     /** Epoch ms of the earliest timed work this lane owes; `now` when work is due already; null when none. */
     nextDue(now: number): Promise<number | null>;

package/dist/daemon/due-sources.js

@@ -1 +1 @@
-var f=Object.defineProperty;var r=(e,n)=>f(e,"name",{value:n,configurable:!0});const d=["brokerSupervision","controllerDeath","nodeLifecycle","nodeDeadline","structuredRequests","cronLane","deliveryLane","actionLane","pendingReviewSubmit","storageMaintenance","storageDueWork","pollRunObservers","faultRetryBackstop","bashDeadline"],u=new Map,c=new Map;function p(e,n){u.set(e,n)}r(p,"registerDueSource");function x(e,n){c.set(e,n)}r(x,"declareBusyOnly");function D(){return[...u.keys()]}r(D,"dueSourceLanes");function b(){return[...c.keys()]}r(b,"busyOnlyLanes");async function L(e,n=()=>{}){let o=null,s=!1;for(const[a,i]of u){let t;try{t=await i.nextDue(e)}catch(l){n(a,l),t=e}t===null||!Number.isFinite(t)||(t<=e&&(s=!0),(o===null||t<o)&&(o=t))}return{dueNow:s,nextDueMs:o}}r(L,"collectDue");export{d as TICK_LANES,b as busyOnlyLanes,L as collectDue,x as declareBusyOnly,D as dueSourceLanes,p as registerDueSource};
+var d=Object.defineProperty;var r=(e,n)=>d(e,"name",{value:n,configurable:!0});const p=["brokerSupervision","controllerDeath","nodeLifecycle","nodeDeadline","structuredRequests","recipeImage","cronLane","deliveryLane","actionLane","pendingReviewSubmit","storageMaintenance","storageDueWork","pollRunObservers","faultRetryBackstop","bashDeadline","ledgerSandbox","ledgerStorage","ledgerR2"],u=new Map,a=new Map;function y(e,n){u.set(e,n)}r(y,"registerDueSource");function x(e,n){a.set(e,n)}r(x,"declareBusyOnly");function g(){return[...u.keys()]}r(g,"dueSourceLanes");function b(){return[...a.keys()]}r(b,"busyOnlyLanes");async function D(e,n=()=>{}){let o=null,s=!1;for(const[c,l]of u){let t;try{t=await l.nextDue(e)}catch(i){n(c,i),t=e}t===null||!Number.isFinite(t)||(t<=e&&(s=!0),(o===null||t<o)&&(o=t))}return{dueNow:s,nextDueMs:o}}r(D,"collectDue");export{p as TICK_LANES,b as busyOnlyLanes,D as collectDue,x as declareBusyOnly,g as dueSourceLanes,y as registerDueSource};

package/dist/daemon/due.js

@@ -1,11 +1,12 @@
-var D=Object.defineProperty;var a=(n,e)=>D(n,"name",{value:e,configurable:!0});import{openDb as f}from"../core/canvas/db.js";import{isPidPresent as R}from"../core/canvas/pid.js";import{isProfilePaused as p}from"../core/profiles/manifest.js";import{hasUnseenWakeMail as E}from"../core/canvas/deliveries.js";import{listPendingProviderRetries as T}from"../core/canvas/recovery.js";import{listSubmitRequestedRequests as x}from"../core/human/requests.js";import{storageConfig as I}from"../core/storage/client.js";import{emitEvent as A}from"../core/events/emit.js";import{wantsExecution as L}from"./reconcilers/node-lifecycle/wants-execution.js";import{dueAtMs as O}from"./reconcilers/fault-retry-backstop.js";import{UNUSED_UPLOAD_TTL_MS as y}from"./reconcilers/storage-due-work.js";import{busyOnlyLanes as M,collectDue as b,declareBusyOnly as l,dueSourceLanes as g,registerDueSource as r}from"./due-sources.js";import{filterAsync as h}from"../core/async-array.js";const c=a(async(n,...e)=>await(await f()).prepare(n).all(...e),"rows"),s=a(async n=>await(await f()).prepare(n).get(),"first"),o=a(n=>{if(n==null)return null;const e=Date.parse(n);return Number.isFinite(e)?e:null},"iso"),_=a(n=>{let e=null;for(const t of n)t!==null&&(e===null||t<e)&&(e=t);return e},"earliest"),N="node_id NOT IN (SELECT node_id FROM warm_pool)";let m=!1;async function S(){if(m)return;m=!0,l("brokerSupervision","acts only on live broker processes, which already make busy true"),l("controllerDeath","acts only on live controller processes"),l("pollRunObservers","follows live pids only"),l("bashDeadline","acts only on running bash jobs, which already make busy true"),l("storageMaintenance","retention and placement housekeeping: never time-critical, and it must not wake or hold a sleeping runtime"),r("nodeLifecycle",{nextDue:a(async()=>_((await h(await c(`SELECT node_id, status, intent, profile_id, respawn_not_before FROM node_rows
+var D=Object.defineProperty;var a=(n,e)=>D(n,"name",{value:e,configurable:!0});import{openDb as E}from"../core/canvas/db.js";import{isPidPresent as R}from"../core/canvas/pid.js";import{isProfilePaused as p}from"../core/profiles/manifest.js";import{hasUnseenWakeMail as f}from"../core/canvas/deliveries.js";import{listPendingProviderRetries as x}from"../core/canvas/recovery.js";import{listSubmitRequestedRequests as g}from"../core/human/requests.js";import{storageConfig as T}from"../core/storage/client.js";import{emitEvent as A}from"../core/events/emit.js";import{wantsExecution as m}from"./reconcilers/node-lifecycle/wants-execution.js";import{dueAtMs as I}from"./reconcilers/fault-retry-backstop.js";import{UNUSED_UPLOAD_TTL_MS as O}from"./reconcilers/storage-due-work.js";import{RECIPE_HAS_CONTENT_SQL as y}from"./api/environment-of.js";import{busyOnlyLanes as M,collectDue as b,declareBusyOnly as u,dueSourceLanes as h,registerDueSource as i}from"./due-sources.js";import{filterAsync as v}from"../core/async-array.js";const _=a(async(n,...e)=>await(await E()).prepare(n).all(...e),"rows"),s=a(async n=>await(await E()).prepare(n).get(),"first"),o=a(n=>{if(n==null)return null;const e=Date.parse(n);return Number.isFinite(e)?e:null},"iso"),c=a(n=>{let e=null;for(const t of n)t!==null&&(e===null||t<e)&&(e=t);return e},"earliest"),N="node_id NOT IN (SELECT node_id FROM warm_pool)";let L=!1;async function S(){if(L)return;L=!0,u("brokerSupervision","acts only on live broker processes, which already make busy true"),u("controllerDeath","acts only on live controller processes"),u("pollRunObservers","follows live pids only"),u("bashDeadline","acts only on running bash jobs, which already make busy true"),u("storageMaintenance","retention and placement housekeeping: never time-critical, and it must not wake or hold a sleeping runtime"),u("ledgerSandbox","bills awake time of executions that are busy or just exited; a sleeping runtime owes nothing until it wakes, and the lane must not wake it"),u("ledgerStorage","monthly storage billing: never time-critical, and it must not wake or hold a sleeping runtime"),u("ledgerR2","moves closed months to R2: housekeeping that must not wake or hold a sleeping runtime"),i("nodeLifecycle",{nextDue:a(async()=>c((await v(await _(`SELECT node_id, status, intent, profile_id, respawn_not_before FROM node_rows
         WHERE status IN ('active', 'idle') AND kind != 'human' AND ${N}
-          AND pi_pid IS NULL AND respawn_not_before IS NOT NULL`),async e=>!p(e.profile_id)&&await L({node_id:e.node_id,status:e.status,intent:e.intent}))).map(e=>o(e.respawn_not_before))),"nextDue")}),r("nodeDeadline",{nextDue:a(async()=>_((await c(`SELECT deadline_at, profile_id FROM node_rows
-        WHERE deadline_at IS NOT NULL AND outcome_at IS NULL AND status IN ('active', 'idle') AND kind != 'human'`)).filter(e=>!p(e.profile_id)).map(e=>o(e.deadline_at))),"nextDue")}),r("structuredRequests",{nextDue:a(async e=>await s("SELECT 1 FROM structured_requests WHERE state != 'pending' AND effects_done_at IS NULL LIMIT 1")!==void 0?e:(await s("SELECT MIN(deadline_at) AS at FROM structured_requests WHERE state = 'pending' AND deadline_at IS NOT NULL"))?.at??null,"nextDue")}),r("cronLane",{nextDue:a(async()=>{const e=(await c(`SELECT fire_at, profile FROM crons
+          AND pi_pid IS NULL AND respawn_not_before IS NOT NULL`),async e=>!p(e.profile_id)&&await m({node_id:e.node_id,status:e.status,intent:e.intent}))).map(e=>o(e.respawn_not_before))),"nextDue")}),i("nodeDeadline",{nextDue:a(async()=>c((await _(`SELECT deadline_at, profile_id FROM node_rows
+        WHERE deadline_at IS NOT NULL AND outcome_at IS NULL AND status IN ('active', 'idle') AND kind != 'human'`)).filter(e=>!p(e.profile_id)).map(e=>o(e.deadline_at))),"nextDue")}),i("structuredRequests",{nextDue:a(async e=>await s("SELECT 1 FROM structured_requests WHERE state != 'pending' AND effects_done_at IS NULL LIMIT 1")!==void 0?e:(await s("SELECT MIN(deadline_at) AS at FROM structured_requests WHERE state = 'pending' AND deadline_at IS NOT NULL"))?.at??null,"nextDue")}),i("recipeImage",{nextDue:a(async e=>c((await _(`SELECT image_building_at FROM environments
+        WHERE kind = 'blaxel' AND recipe_revision > COALESCE(image_revision, 0) AND ${y}`)).map(t=>t.image_building_at===null?e:Math.max(e,o(t.image_building_at)??e))),"nextDue")}),i("cronLane",{nextDue:a(async()=>{const e=(await _(`SELECT fire_at, profile FROM crons
           WHERE state = 'active' AND run_state = 'idle'
-            AND NOT (held = 1 AND recur IS NULL AND expires_at IS NULL)`)).filter(i=>!p(i.profile)).map(i=>o(i.fire_at)),t=(await c("SELECT expires_at FROM crons WHERE expires_at IS NOT NULL AND run_state = 'idle'")).map(i=>o(i.expires_at));return _([...e,...t])},"nextDue")});const n=a(async e=>async()=>(await s(`SELECT MIN(next_attempt_at) AS at FROM ${e} WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at IS NOT NULL`))?.at??null,"attempts");r("deliveryLane",{nextDue:await n("deliveries")}),r("actionLane",{nextDue:await n("daemon_actions")}),r("pendingReviewSubmit",{nextDue:a(async e=>(await x()).length>0?e:null,"nextDue")}),r("storageDueWork",{nextDue:a(async e=>{if(I()===null)return null;if(await s("SELECT 1 FROM uploads WHERE delete_pending_since IS NOT NULL LIMIT 1")!==void 0||await s("SELECT 1 FROM shares WHERE status = 'off' AND object_deleted_at IS NULL LIMIT 1")!==void 0)return e;const t=(await s("SELECT MIN(expires_at) AS at FROM shares WHERE status = 'on' AND expires_at IS NOT NULL"))?.at,i=(await s(`SELECT MIN(created_at) AS at FROM uploads u WHERE delete_pending_since IS NULL
-          AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`))?.at,u=o(i);return _([o(t),u===null?null:u+y])},"nextDue")}),r("faultRetryBackstop",{nextDue:a(async()=>_((await T()).map(e=>O(e))),"nextDue")})}a(S,"installDueSources");const w=12e4;async function U(n){const e=await c(`SELECT n.node_id, n.status, n.intent, n.profile_id, n.pi_pid, n.frozen_at, n.respawn_not_before, n.created, n.launched_at,
+            AND NOT (held = 1 AND recur IS NULL AND expires_at IS NULL)`)).filter(r=>!p(r.profile)).map(r=>o(r.fire_at)),t=(await _("SELECT expires_at FROM crons WHERE expires_at IS NOT NULL AND run_state = 'idle'")).map(r=>o(r.expires_at));return c([...e,...t])},"nextDue")});const n=a(async e=>async()=>(await s(`SELECT MIN(next_attempt_at) AS at FROM ${e} WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at IS NOT NULL`))?.at??null,"attempts");i("deliveryLane",{nextDue:await n("deliveries")}),i("actionLane",{nextDue:await n("daemon_actions")}),i("pendingReviewSubmit",{nextDue:a(async e=>(await g()).length>0?e:null,"nextDue")}),i("storageDueWork",{nextDue:a(async e=>{if(T()===null)return null;if(await s("SELECT 1 FROM uploads WHERE delete_pending_since IS NOT NULL LIMIT 1")!==void 0||await s("SELECT 1 FROM shares WHERE status = 'off' AND object_deleted_at IS NULL LIMIT 1")!==void 0)return e;const t=(await s("SELECT MIN(expires_at) AS at FROM shares WHERE status = 'on' AND expires_at IS NOT NULL"))?.at,r=(await s(`SELECT MIN(created_at) AS at FROM uploads u WHERE delete_pending_since IS NULL
+          AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`))?.at,l=o(r);return c([o(t),l===null?null:l+O])},"nextDue")}),i("faultRetryBackstop",{nextDue:a(async()=>c((await x()).map(e=>I(e))),"nextDue")})}a(S,"installDueSources");const w=12e4;async function C(n){const e=await _(`SELECT n.node_id, n.status, n.intent, n.profile_id, n.pi_pid, n.frozen_at, n.respawn_not_before, n.created, n.launched_at,
             n.turn_open_at, n.turn_closed_at,
             (SELECT x.started_at FROM executions x WHERE x.node_id = n.node_id AND x.state != 'exited') AS exec_started_at
        FROM node_rows n
-      WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${N}`);for(const t of e){if(p(t.profile_id)||!await L({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&R(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=o(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<w||await E(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null||t.status==="active")return!0;const u=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>o(d)??0));if(u>0&&n-u<w||t.status==="idle"&&await E(t.node_id))return!0}return!1}a(U,"nodesOweWork");async function X(n=Date.now()){await S();const{dueNow:e,nextDueMs:t}=await b(n,(u,d)=>{A({level:"warn",event:"daemon.due.source_failed",error:d,fields:{lane:u}})});return{busy:e||await U(n)||await s("SELECT 1 FROM crons WHERE run_state = 'running' LIMIT 1")!==void 0||await s("SELECT 1 FROM jobs j JOIN objects o ON o.id = j.object WHERE j.status = 'running' AND o.deleted_at IS NULL LIMIT 1")!==void 0,next_due_at:t===null?null:new Date(t).toISOString()}}a(X,"dueReport");async function K(){return await S(),[...g(),...M()]}a(K,"coveredLanes");export{w as START_GRACE_MS,K as coveredLanes,X as dueReport,S as installDueSources};
+      WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${N}`);for(const t of e){if(p(t.profile_id)||!await m({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&R(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=o(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<w||await f(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null||t.status==="active")return!0;const l=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>o(d)??0));if(l>0&&n-l<w||t.status==="idle"&&await f(t.node_id))return!0}return!1}a(C,"nodesOweWork");async function K(n=Date.now()){await S();const{dueNow:e,nextDueMs:t}=await b(n,(l,d)=>{A({level:"warn",event:"daemon.due.source_failed",error:d,fields:{lane:l}})});return{busy:e||await C(n)||await s("SELECT 1 FROM crons WHERE run_state = 'running' LIMIT 1")!==void 0||await s("SELECT 1 FROM jobs j JOIN objects o ON o.id = j.object WHERE j.status = 'running' AND o.deleted_at IS NULL LIMIT 1")!==void 0,next_due_at:t===null?null:new Date(t).toISOString()}}a(K,"dueReport");async function V(){return await S(),[...h(),...M()]}a(V,"coveredLanes");export{w as START_GRACE_MS,V as coveredLanes,K as dueReport,S as installDueSources};

package/dist/daemon/fleet.d.ts

@@ -55,16 +55,16 @@ export declare class DaemonFleet implements FleetRegistry {
     constructor(opts: DaemonFleetOpts);
     epoch(): string;
     register(nodeId: string, handle: HostHandle): void;
-    has(nodeId: string): boolean;
-    isClaimed(nodeId: string): boolean;
+    has(nodeId: string): Promise<boolean>;
+    isClaimed(nodeId: string): Promise<boolean>;
     get(nodeId: string): FleetEntry | undefined;
     size(): number;
-    occupancy(): number;
-    reserveSlot(nodeId: string): boolean;
+    occupancy(): Promise<number>;
+    reserveSlot(nodeId: string): Promise<boolean>;
     releaseReservation(nodeId: string): void;
     entries(): Iterable<[string, FleetEntry]>;
-    interruptForProfilePause(nodeId: string, pid?: number | undefined): void;
-    hasProfilePauseInterruption(nodeId: string): boolean;
+    interruptForProfilePause(nodeId: string, pid?: number | undefined): Promise<void>;
+    hasProfilePauseInterruption(nodeId: string): Promise<boolean>;
     forget(nodeId: string): void;
     exitsSettled(): Promise<void>;
     onChildExit(nodeId: string, status: FleetExitStatus, executionId?: string): Promise<void>;

package/dist/daemon/fleet.js

@@ -1,21 +1,21 @@
-var g=Object.defineProperty;var l=(i,e)=>g(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as x}from"../core/runtime/close.js";import{readFault as E,readInterruptedTurn as P,readTransportPoisonVerdict as N}from"../core/canvas/recovery.js";import{hasPendingWake as S}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as C,MAX_RESPAWN_ATTEMPTS as F,OFFLINE_RESPAWN_RETRY_MS as M,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{isStartRefusal as D}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as L}from"../core/runtime/fleet.js";import{isRecoveryHeld as W}from"../core/runtime/recovery-hold.js";async function _(i,e){const t=await E(i.node_id),r=t===null?"":`The broker reported a ${t.kind} fault on ${t.link} (${t.op}) before it exited: ${t.message}
+var P=Object.defineProperty;var l=(i,e)=>P(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as R}from"../core/canvas/index.js";import{revokeExitedNodeToken as T}from"../core/canvas/node-tokens.js";import{PROFILE_PAUSE_STOP_REASON as _,absorbProfilePauseStop as N,endExecution as y,hasLaunchingExecution as S,hasProfilePauseStop as F,hasRunningRemoteExecution as b,liveExecutionForNode as g,liveExecutionNodeIds as C,markProfilePauseStop as k,recordExecutionExit as O,takeObservedExit as M}from"../core/canvas/executions.js";import{isProfilePaused as z}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as A,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as L}from"../core/runtime/close.js";import{readFault as B,readInterruptedTurn as D,readTransportPoisonVerdict as W}from"../core/canvas/recovery.js";import{hasPendingWake as q}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as G,MAX_RESPAWN_ATTEMPTS as H,OFFLINE_RESPAWN_RETRY_MS as Y,recordRespawnPolicy as U,respawnBackoffMs as V}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as j}from"../core/runtime/connectivity.js";import{runnerForExecution as K}from"../core/runtime/runner/index.js";import{reviveNode as X}from"../core/runtime/revive.js";import{isStartRefusal as J}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as Q}from"../core/runtime/fleet.js";import{isRecoveryHeld as Z}from"../core/runtime/recovery-hold.js";async function $(i,e){const r=await B(i.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
 
 `,s=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${i.name}\` (${i.kind}) never started.
 
 Its pi vehicle exited before the session came up (no pi_session_id was ever recorded), so the node produced no output. ${s}
 
-`+r+`Inspect canonical event diagnostics with \`crtr sys logs --node ${i.node_id}\`.
+`+t+`Inspect canonical event diagnostics with \`crtr sys logs --node ${i.node_id}\`.
 
-If the work still needs doing, re-spawn it.`}l(_,"bootFailureBody");async function y(i,e){await m(i.node_id,await _(i,e),{from:i.node_id})}l(y,"surfaceBootFailure");function b(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
+If the work still needs doing, re-spawn it.`}l($,"bootFailureBody");async function E(i,e){await m(i.node_id,await $(i,e),{from:i.node_id})}l(E,"surfaceBootFailure");function x(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
 
-Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function q(i){const e=await N(i.node_id),t=e===null?"":`
+Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(x,"outOfMemoryBody");async function I(i){const e=await W(i.node_id),r=e===null?"":`
 
-Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,r=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
+Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
 
-Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+t+`
+Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
 
-Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(q,"surfaceCrashLoop");async function G(i,e){const t=e instanceof Error?e.message:String(e),r=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
+Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(I,"surfaceCrashLoop");async function ee(i,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
 
-The daemon refused this recovery before starting a broker: ${t}
+The daemon refused this recovery before starting a broker: ${r}
 
-The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(G,"surfaceRecoveryFailure");function H(i){return i?.cycle_pending!==!0}l(H,"retryResumeMode");function Y(i,e,t=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":R(i)?"dormant":i.lifecycle==="resident"&&!e&&!t&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?H(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(Y,"classifyDeadNode");const V=3e4;class ue{static{l(this,"DaemonFleet")}#o;#l;#i;#c;#e=new Map;#t=new Set;#n=new Set;#r=new Map;#u=Number.NEGATIVE_INFINITY;constructor(e){this.#o=e.epoch,this.#l=e.enqueue,this.#i=e.now??Date.now,this.#c=e.probeOnline??(()=>A())}epoch(){return this.#o}register(e,t){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(t.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#r.delete(e),this.#e.set(e,{pid:t.pid,handle:t})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#t.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#t.size}reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:t}=L();if(this.occupancy()<t)return this.#t.add(e),!0;const r=this.#i();return r-this.#u>=V&&(this.#u=r,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:t}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,t=this.#e.get(e)?.pid){if(t!==void 0){this.#n.add(e);try{process.kill(t,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#n.has(e)}forget(e){this.#e.delete(e),this.#t.delete(e),this.#r.delete(e)}#s=new Set;async exitsSettled(){for(;this.#s.size>0;)await Promise.allSettled([...this.#s])}async onChildExit(e,t,r){const s=this.#f(e,t,r);this.#s.add(s);try{await s}finally{this.#s.delete(s)}}async#f(e,t,r){r&&await $(r),this.#e.delete(e),this.#l(async()=>{const s=await w(e),a=s?.launched_at==null?Number.NaN:Date.parse(s.launched_at),n=Number.isFinite(a)?Math.max(0,this.#i()-a):null,d=s?.respawn_failures??0;s?.status==="done"||s?.status==="canceled"||s?.status==="dead"?this.#r.delete(e):this.#r.set(e,t),this.#h(e,t,n,d)})}async deliverExit(e,t){await this.onChildExit(e,t)}async applyDeadRowPolicy(e){const t=this.#r.get(e);try{await this.#d(e,t)}finally{this.#r.get(e)===t&&this.#r.delete(e)}}async terminalizeIfStillNonTerminal(e,t,r,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#a(e,n,{event:t,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>G(n,r),"surface")}),!0)}dispose(){}#h(e,t,r,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:t.code,signal:t.signal,uptime_ms:r,consecutive_failures:s}})}async#d(e,t){return this.#p(e,t)}async#p(e,t){const r=await w(e);if(r!==null&&T(r.profile_id))return await h(e),"forget";if(this.#n.delete(e))return await h(e),"forget";if(W(e))return"dormant";const s=(await P(e))?.interrupted===!0;if(t?.oomKilled===!0&&r!==null&&r.status!=="done"&&r.status!=="canceled"&&r.status!=="dead")return await h(e),await this.#a(e,r,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(r)},wake:`Child out of memory \u2014 ${p(r)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(r),{from:e})},"surface")}),"forget";const a=Y(r,s,await S(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#a(e,r,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await _(r,t)},wake:`Child crashed \u2014 ${p(r)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(r,t),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#m(e,r,a,s);break}return a}async#m(e,t,r,s){const a=await k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#c()){await C(e,n,this.#i()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,retry_in_ms:M,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await O(a,this.#i()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=F};if(f.exhausted){const c=`Child crashed \u2014 ${p(t)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#a(e,t,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...t.pi_session_id!=null?{surface:l(()=>q(t),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:r==="respawn-resume",wakeReason:r==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(D(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:r}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#a(e,t,r){u({level:"error",event:r.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:r.reason,outcome:{detail:r.detail}});try{await(r.surface??(()=>y(t)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await x(e,"notice.crash",r.wake,{reason:"child-crashed",child:e})}}export{ue as DaemonFleet,Y as classifyDeadNode,H as retryResumeMode,y as surfaceBootFailure};
+The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(ee,"surfaceRecoveryFailure");function te(i){return i?.cycle_pending!==!0}l(te,"retryResumeMode");function re(i,e,r=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":A(i)?"dormant":i.lifecycle==="resident"&&!e&&!r&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?te(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(re,"classifyDeadNode");const ie=3e4;class Ee{static{l(this,"DaemonFleet")}#a;#n;#r;#o;#e=new Map;#t=new Set;#l=Number.NEGATIVE_INFINITY;constructor(e){this.#a=e.epoch,this.#n=e.enqueue,this.#r=e.now??Date.now,this.#o=e.probeOnline??(()=>j())}epoch(){return this.#a}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}async has(e){return this.#e.has(e)||await b(e)}async isClaimed(e){return this.#e.has(e)||this.#t.has(e)||await b(e)||await S(e,this.#a)}get(e){return this.#e.get(e)}size(){return this.#e.size}async#c(){const e=await C();for(const r of this.#e.keys())e.add(r);for(const r of this.#t)e.add(r);return e}async occupancy(){return(await this.#c()).size}async reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=Q(),t=await this.#c();if(this.#t.has(e)||this.#e.has(e))return!0;if(t.delete(e),t.size<r)return this.#t.has(e)||this.#t.add(e),!0;const s=this.#r();return s-this.#l>=ie&&(this.#l=s,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:t.size,hard_cap:r}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}async interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r===void 0){const t=await g(e);if(t===null||t.runner==="process"||t.state!=="running")return;await k(e),(await K(t.runner).stop(t,{reason:_})).confirmed&&await y(t.executionId,{reason:_});return}await k(e);try{process.kill(r,"SIGTERM")}catch{}}async hasProfilePauseInterruption(e){return await F(e)}forget(e){this.#e.delete(e),this.#t.delete(e)}#i=new Set;async exitsSettled(){for(;this.#i.size>0;)await Promise.allSettled([...this.#i])}async onChildExit(e,r,t){const s=this.#u(e,r,t);this.#i.add(s);try{await s}finally{this.#i.delete(s)}}async#u(e,r,t){const s=t??(await g(e))?.executionId;t&&await T(t),this.#e.delete(e),s!==void 0&&(await O(s,r),await y(s,{reason:"exited"})),this.#n(async()=>{const a=await w(e),n=a?.launched_at==null?Number.NaN:Date.parse(a.launched_at),d=Number.isFinite(n)?Math.max(0,this.#r()-n):null,f=a?.respawn_failures??0;this.#f(e,r,d,f)})}async deliverExit(e,r){await this.onChildExit(e,r)}async applyDeadRowPolicy(e){await this.#h(e,await M(e))}async terminalizeIfStillNonTerminal(e,r,t,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#s(e,n,{event:r,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>ee(n,t),"surface")}),!0)}dispose(){}#f(e,r,t,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:s}})}async#h(e,r){return this.#d(e,r)}async#d(e,r){const t=await w(e);if(t!==null&&z(t.profile_id))return await h(e),"forget";if(await N(e))return await h(e),"forget";if(await Z(e))return"dormant";const s=(await D(e))?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return await h(e),await this.#s(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:x(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,x(t),{from:e})},"surface")}),"forget";const a=re(t,s,await q(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#s(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await $(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>E(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#p(e,t,a,s);break}return a}async#p(e,r,t,s){const a=await R(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#o()){await G(e,n,this.#r()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:Y,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await U(a,this.#r()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=H};if(f.exhausted){const c=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#s(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...r.pi_session_id!=null?{surface:l(()=>I(r),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:V(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await X(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(J(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#s(e,r,t){u({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>E(r)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await L(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{Ee as DaemonFleet,re as classifyDeadNode,te as retryResumeMode,E as surfaceBootFailure};

package/dist/daemon/park-activity.d.ts

@@ -1,4 +1,5 @@
 import type { NodeMeta } from '../core/canvas/types.js';
+import type { ViewerProbe } from '../core/runtime/placement.js';
 import type { PendingPark } from './park-pending.js';
 /** A main-session write after parking was armed is durable evidence of live
  * input even when the turn has since settled and the viewer detached. */
@@ -6,4 +7,4 @@ export declare function mainSessionActivitySince(sessionFile: string | null | un
 /** One verdict shared by normal completion and grace expiry. A broker records
  * accepted live input against the process-local marker before it drives Pi, so
  * the verdict cannot race a viewer, live delivery, or durable wake. */
-export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark): Promise<boolean>;
+export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark, viewer?: ViewerProbe): Promise<boolean>;

package/dist/daemon/park-activity.js

@@ -1 +1 @@
-var n=Object.defineProperty;var e=(r,t)=>n(r,"name",{value:t,configurable:!0});import{conversationIdFromPath as o,conversationStore as f,ConversationNotFoundError as c}from"../core/conversation-store/index.js";import{emitEvent as s}from"../core/events/emit.js";import{hasWakeMailSince as u}from"../core/canvas/deliveries.js";import{readLiveTurn as m}from"../core/canvas/recovery.js";import{parkEvent as l}from"./reconcilers/live-obligation.js";async function v(r,t){if(r==null)return!1;const a=o(r);if(a===null)return!1;try{return(await f().header(a)).mtime>=t}catch(i){return i instanceof c?!1:(s({level:"warn",event:"broker.park.session_activity_scan_failed",error:i}),!0)}}e(v,"mainSessionActivitySince");async function _(r,t,a){return!a.activity&&(await m(r))?.busy!==!0&&!await v(t.pi_session_file,a.startedAt)&&!await u(r,a.startedAt)&&await l(r,t.lifecycle)==="park"}e(_,"shouldParkAfterSummary");export{v as mainSessionActivitySince,_ as shouldParkAfterSummary};
+var n=Object.defineProperty;var i=(r,t)=>n(r,"name",{value:t,configurable:!0});import{conversationIdFromPath as o,conversationStore as f,ConversationNotFoundError as c}from"../core/conversation-store/index.js";import{emitEvent as s}from"../core/events/emit.js";import{hasWakeMailSince as u}from"../core/canvas/deliveries.js";import{readLiveTurn as m}from"../core/canvas/recovery.js";import{parkEvent as l}from"./reconcilers/live-obligation.js";async function v(r,t){if(r==null)return!1;const a=o(r);if(a===null)return!1;try{return(await f().header(a)).mtime>=t}catch(e){return e instanceof c?!1:(s({level:"warn",event:"broker.park.session_activity_scan_failed",error:e}),!0)}}i(v,"mainSessionActivitySince");async function _(r,t,a,e){return!a.activity&&(await m(r))?.busy!==!0&&!await v(t.pi_session_file,a.startedAt)&&!await u(r,a.startedAt)&&await l(r,t.lifecycle,e)==="park"}i(_,"shouldParkAfterSummary");export{v as mainSessionActivitySince,_ as shouldParkAfterSummary};

package/dist/daemon/park-pending.d.ts

@@ -1,18 +1,25 @@
+import { type CanvasDb } from '../core/canvas/db.js';
 export type PendingPark = {
     startedAt: number;
     activity: boolean;
 };
 /** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
-export declare function beginPendingPark(nodeId: string, now: number): void;
-/** Record main-engine input accepted while the isolated turn is in flight. */
-export declare function recordPendingParkActivity(nodeId: string): boolean;
-export declare function isPendingPark(nodeId: string): boolean;
-/** When the pending park was armed, or undefined when none is armed. */
-export declare function pendingParkStartedAt(nodeId: string): number | undefined;
-/** Delete-and-report the pending marker. Both the supervision tick and the
- * settle handler run on the daemon thread with no await between their check and
- * this call, so exactly one of them ever enacts a given park. */
-export declare function consumePendingPark(nodeId: string): PendingPark | undefined;
-/** Every node with a parking turn in flight — the reconciler's sweep drops the
- *  ones whose broker has left the fleet. */
-export declare function pendingParkIds(): string[];
+export declare function beginPendingPark(nodeId: string, now: number): Promise<void>;
+/** Record main-engine input accepted while the isolated turn is in flight. False when no park is armed. */
+export declare function recordPendingParkActivity(nodeId: string): Promise<boolean>;
+export declare function isPendingPark(nodeId: string): Promise<boolean>;
+/** When the pending park was armed (epoch ms), or undefined when none is armed. */
+export declare function pendingParkStartedAt(nodeId: string): Promise<number | undefined>;
+/** Clear-and-report the pending marker with one conditional update. Exactly one caller, in any process, gets the prior values back. */
+export declare function consumePendingPark(nodeId: string): Promise<PendingPark | undefined>;
+/** Every node with a parking turn in flight — the supervision sweep drops the ones whose broker has left the fleet. */
+export declare function pendingParkIds(): Promise<string[]>;
+/** The same clear inside the caller's transaction, for the execution leaving `running`. */
+export declare function clearPendingParkIn(db: CanvasDb, nodeId: string): Promise<void>;
+export type SupervisionMark = 'unattended_since' | 'stall_noticed_at' | 'wedge_noticed_at';
+/** The mark's time (epoch ms), or undefined when unset. */
+export declare function readSupervisionMark(nodeId: string, mark: SupervisionMark): Promise<number | undefined>;
+/** Set the mark to `at` (epoch ms). */
+export declare function writeSupervisionMark(nodeId: string, mark: SupervisionMark, at: number): Promise<void>;
+/** Clear the mark; a tick that finds it already clear takes no write. */
+export declare function clearSupervisionMark(nodeId: string, mark: SupervisionMark): Promise<void>;

package/dist/daemon/park-pending.js

@@ -1 +1,3 @@
-var i=Object.defineProperty;var n=(e,t)=>i(e,"name",{value:t,configurable:!0});const r=new Map;function o(e,t){r.set(e,{startedAt:t,activity:!1})}n(o,"beginPendingPark");function u(e){const t=r.get(e);return t===void 0?!1:(t.activity=!0,!0)}n(u,"recordPendingParkActivity");function d(e){return r.has(e)}n(d,"isPendingPark");function c(e){return r.get(e)?.startedAt}n(c,"pendingParkStartedAt");function f(e){const t=r.get(e);return t!==void 0&&r.delete(e),t}n(f,"consumePendingPark");function s(){return Array.from(r.keys())}n(s,"pendingParkIds");export{o as beginPendingPark,f as consumePendingPark,d as isPendingPark,s as pendingParkIds,c as pendingParkStartedAt,u as recordPendingParkActivity};
+var p=Object.defineProperty;var r=(e,a)=>p(e,"name",{value:a,configurable:!0});import{openDb as i,withCanvasWrite as n}from"../core/canvas/db.js";async function c(e,a){await n(async t=>{await t.prepare("UPDATE nodes SET park_requested_at = ?, park_activity = 0 WHERE node_id = ?").run(new Date(a).toISOString(),e)})}r(c,"beginPendingPark");async function E(e){return await n(async a=>Number((await a.prepare("UPDATE nodes SET park_activity = 1 WHERE node_id = ? AND park_requested_at IS NOT NULL").run(e)).changes)>0)}r(E,"recordPendingParkActivity");async function k(e){return await o(e)!==void 0}r(k,"isPendingPark");async function o(e){const a=await(await i()).prepare("SELECT park_requested_at FROM nodes WHERE node_id = ?").get(e);return a?.park_requested_at?Date.parse(a.park_requested_at):void 0}r(o,"pendingParkStartedAt");async function w(e){return await n(async a=>{const t=await a.prepare(`WITH prior AS (SELECT park_requested_at, park_activity FROM nodes WHERE node_id = ? AND park_requested_at IS NOT NULL FOR UPDATE)
+      UPDATE nodes n SET park_requested_at = NULL, park_activity = 0 FROM prior WHERE n.node_id = ?
+      RETURNING prior.park_requested_at AS park_requested_at, prior.park_activity AS park_activity`).get(e,e);return t===void 0?void 0:{startedAt:Date.parse(t.park_requested_at),activity:Number(t.park_activity)===1}})}r(w,"consumePendingPark");async function y(){return(await(await i()).prepare("SELECT node_id FROM nodes WHERE park_requested_at IS NOT NULL").all()).map(e=>e.node_id)}r(y,"pendingParkIds");async function S(e,a){await e.prepare("UPDATE nodes SET park_requested_at = NULL, park_activity = 0, unattended_since = NULL, stall_noticed_at = NULL, wedge_noticed_at = NULL WHERE node_id = ?").run(a)}r(S,"clearPendingParkIn");async function s(e,a){const t=await(await i()).prepare(`SELECT ${a} AS at FROM nodes WHERE node_id = ?`).get(e);return t?.at?Date.parse(t.at):void 0}r(s,"readSupervisionMark");async function L(e,a,t){await n(async d=>{await d.prepare(`UPDATE nodes SET ${a} = ? WHERE node_id = ?`).run(new Date(t).toISOString(),e)})}r(L,"writeSupervisionMark");async function T(e,a){await s(e,a)!==void 0&&await n(async t=>{await t.prepare(`UPDATE nodes SET ${a} = NULL WHERE node_id = ?`).run(e)})}r(T,"clearSupervisionMark");export{c as beginPendingPark,S as clearPendingParkIn,T as clearSupervisionMark,w as consumePendingPark,k as isPendingPark,y as pendingParkIds,o as pendingParkStartedAt,s as readSupervisionMark,E as recordPendingParkActivity,L as writeSupervisionMark};

package/dist/daemon/profile-delete.js

@@ -1 +1 @@
-var q=Object.defineProperty;var d=(e,t)=>q(e,"name",{value:t,configurable:!0});import{existsSync as w,readFileSync as E,readdirSync as g,rmSync as p}from"node:fs";import{join as u}from"node:path";import{cronIdsForProfileDeletion as R,cronIdsForProfileDetach as C,deleteCronsForProfile as T,deleteNodes as O,getNode as m,getRow as W,globalizeCronsForProfile as A,listNodesByProfile as M,nodesRoot as B,reviewDir as $}from"../core/canvas/index.js";import{withCanvasWrite as v}from"../core/canvas/db.js";import{deleteProfileWithOwned as _}from"../core/graph/deletion.js";import{usage as f}from"../core/errors.js";import{emitEvent as z}from"../core/events/emit.js";import{clearDefaultProfileId as k,defaultProfileDirs as L}from"../core/profiles/default-binding.js";import{reserveProfileDeletion as J}from"../core/profiles/deletion-reservation.js";import{deleteProfile as D,loadProfileManifest as G}from"../core/profiles/manifest.js";import{deleteReviewsByOriginNodes as H,reviewIdsByOriginNodes as K}from"../core/review/store.js";import{boundFleet as U}from"../core/runtime/fleet.js";import{headlessBrokerHost as b}from"../core/runtime/host.js";import{transition as Q}from"../core/runtime/lifecycle.js";import{tearDownNode as V}from"../core/runtime/placement.js";import{reshapeNode as X}from"../core/runtime/promote.js";import{reviveNode as Y}from"../core/runtime/revive.js";import{filterAsync as Z,flatMapAsync as ee}from"../core/async-array.js";const P=1e3,N=new Set(["page.json","page.tsx","page.html","page.js","review.json","response.json","progress.json","claim.json","reply-route.json","feedback-comments.json","feedback.json",".ticket-lock"]),h=new Map;function te(e){if(typeof e!="object"||e===null||Array.isArray(e))throw f("profile deletion requires a JSON request body",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});return e}d(te,"requestObject");function re(e){const t=te(e);if(t.force!==!0)throw f("profile deletion requires --force",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});if(t.detach!==void 0&&typeof t.detach!="boolean")throw f("detach must be a boolean",{error:"invalid_detach",message:"detach must be a boolean",next:"Use --detach as a presence-only flag, or omit it."});const r=t.caller_node_id;if(r!=null&&(typeof r!="string"||r===""))throw f("caller_node_id must be a nonempty string or null");return{force:!0,...t.detach===!0?{detach:!0}:{},...r===void 0?{}:{caller_node_id:r}}}d(re,"validateRequest");function j(e){if(!w(e))return null;try{const t=JSON.parse(E(e,"utf8"));return typeof t.source?.nodeId=="string"&&t.source.nodeId!==""?t.source.nodeId:null}catch{return null}}d(j,"manifestSourceNodeId");async function oe(e){const t=new Set(e),r=B();if(t.size===0||!w(r))return[];const o=[];for(const n of g(r,{withFileTypes:!0})){if(!n.isDirectory())continue;const s=u(r,n.name);[j(u(s,"page.json")),j(u(s,"review.json"))].some(a=>a!==null&&t.has(a))&&o.push({dir:s,wholeDirectory:t.has(n.name)||await W(n.name)===null})}return o.sort((n,s)=>n.dir.localeCompare(s.dir))}d(oe,"ticketCleanupForNodes");function ne(e){if(N.has(e))return!0;for(const t of N)if(e.startsWith(`${t}.`)&&(e.endsWith(".tmp")||e.endsWith(".lock")))return!0;return!1}d(ne,"isTicketOwnedResidue");function ie(e){let t=0;for(const r of e)if(w(r.dir)){if(r.wholeDirectory){p(r.dir,{recursive:!0,force:!0}),t+=1;continue}for(const o of g(r.dir,{withFileTypes:!0}))ne(o.name)&&p(u(r.dir,o.name),{recursive:!0,force:!0});t+=1}return t}d(ie,"cleanTickets");function S(e){const t=e.filter(o=>o.managed_worktree?.state==="open").map(o=>({node_id:o.node_id,path:o.managed_worktree.path}));if(t.length===0)return null;const r=`profile deletion is blocked by ${t.length} open managed worktree${t.length===1?"":"s"}`;return f(r,{error:"open_managed_worktree",message:r,nodes:t,next:`Close ${t.map(({node_id:o})=>o).join(", ")} with \`crtr node worktree close <node>\`, then retry.`})}d(S,"openWorktreeError");async function se(e,t){const r=await M(e.profileId),o=S(r);if(o!==null)throw o;const n=r.map(({node_id:c})=>c),s=t.detach===!0?"detached":"deleted";return{entry:e,mode:s,nodes:r,nodeIds:n,tickets:s==="deleted"?await oe(n):[],cronIds:s==="deleted"?await R(e.profileId,n):await C(e.profileId),reviewIds:s==="deleted"?await K(n):[],defaultDirs:L(e.profileId)}}d(se,"prepareDeletion");async function x(e){const t=await ee(e.nodeIds,async o=>{const n=await m(o);return n===null?[]:[n]}),r=S(t);if(r!==null)throw r}d(x,"assertDeletionPlanPreconditions");function y(e){return{profile_id:e.entry.profileId,name:e.entry.manifest.name,mode:e.mode,nodes:e.nodeIds.length,crons:e.cronIds.length,tickets:e.tickets.length,reviews:e.reviewIds.length,default_pins:e.defaultDirs.length}}d(y,"resultForPlan");async function de(e){await x(e);const t=await Z(e.nodeIds,async i=>await m(i)!==null),r=U(),o=t.flatMap(i=>{const l=r.get(i);return l===void 0?[]:[{nodeId:i,exited:l.handle.exited}]}),n=[];for(const{nodeId:i}of o){const l=await m(i);(l?.status==="active"||l?.status==="idle")&&(await Q(i,"release"),n.push(i)),b.teardown(i)}await Promise.all(o.map(({exited:i})=>i));const s=[];for(const i of t)await m(i)!==null&&(await X(i,{profileId:null}),s.push(i));const c=await A(e.entry.profileId),a=k(e.entry.profileId);await v(async i=>await _(i,e.entry.profileId)),D(e.entry.profileId);const I=new Set(s);for(const i of n)I.has(i)&&await Y(i,{resume:!0,recovery:!0,capacity:"freeze"});return{...y(e),nodes:s.length,crons:c.length,default_pins:a.length}}d(de,"detachProfile");async function ce(e){await x(e);const t=await T(e.entry.profileId,e.nodeIds);for(const c of e.nodeIds)b.teardown(c),await V(c);const r=await H(e.nodeIds);for(const c of r)p($(c),{recursive:!0,force:!0});const o=ie(e.tickets),n=await O(e.nodeIds),s=k(e.entry.profileId);return await v(async c=>await _(c,e.entry.profileId)),D(e.entry.profileId),{...y(e),nodes:n.length,crons:t.length,tickets:o,reviews:r.length,default_pins:s.length}}d(ce,"hardDeleteProfile");async function F(e){return e.mode==="detached"?de(e):await ce(e)}d(F,"executePlan");function ae(e,t){const r={...y(e),accepted:!0,grace_ms:P};return h.set(e.entry.profileId,{result:r}),setTimeout(()=>{(async()=>{try{await F(e)}catch(o){z({level:"error",event:"profile.delete.deferred_failed",error:o instanceof Error?o:new Error(String(o)),fields:{profile_id:e.entry.profileId,mode:e.mode}})}finally{h.delete(e.entry.profileId),t()}})()},P),r}d(ae,"scheduleSelfDeletion");async function xe(e,t){const r=re(t),o=G(e),n=h.get(o.profileId);if(n!==void 0)return{status:202,body:n.result};const s=J(o.profileId);let c=!1;try{const a=await se(o,r);if(r.caller_node_id!=null&&a.nodeIds.includes(r.caller_node_id)){const i=ae(a,s);return c=!0,{status:202,body:i}}return{status:200,body:await F(a)}}finally{c||s()}}d(xe,"requestProfileDeletion");export{xe as requestProfileDeletion};
+var q=Object.defineProperty;var d=(e,t)=>q(e,"name",{value:t,configurable:!0});import{existsSync as w,readFileSync as E,readdirSync as g,rmSync as p}from"node:fs";import{join as u}from"node:path";import{cronIdsForProfileDeletion as R,cronIdsForProfileDetach as C,deleteCronsForProfile as T,deleteNodes as O,getNode as m,getRow as W,globalizeCronsForProfile as A,listNodesByProfile as M,nodesRoot as B,reviewDir as $}from"../core/canvas/index.js";import{withCanvasWrite as v}from"../core/canvas/db.js";import{deleteProfileWithOwned as _}from"../core/graph/deletion.js";import{usage as f}from"../core/errors.js";import{emitEvent as z}from"../core/events/emit.js";import{clearDefaultProfileId as k,defaultProfileDirs as L}from"../core/profiles/default-binding.js";import{reserveProfileDeletion as J}from"../core/profiles/deletion-reservation.js";import{deleteProfile as D,loadProfileManifest as G}from"../core/profiles/manifest.js";import{deleteReviewsByOriginNodes as H,reviewIdsByOriginNodes as K}from"../core/review/store.js";import{boundFleet as U}from"../core/runtime/fleet.js";import{headlessBrokerHost as b}from"../core/runtime/host.js";import{transition as Q}from"../core/runtime/lifecycle.js";import{tearDownNode as V}from"../core/runtime/placement.js";import{reshapeNode as X}from"../core/runtime/promote.js";import{reviveNode as Y}from"../core/runtime/revive.js";import{filterAsync as Z,flatMapAsync as ee}from"../core/async-array.js";const P=1e3,N=new Set(["page.json","page.tsx","page.html","page.js","review.json","response.json","progress.json","claim.json","reply-route.json","feedback-comments.json","feedback.json",".ticket-lock"]),h=new Map;function te(e){if(typeof e!="object"||e===null||Array.isArray(e))throw f("profile deletion requires a JSON request body",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});return e}d(te,"requestObject");function re(e){const t=te(e);if(t.force!==!0)throw f("profile deletion requires --force",{error:"force_required",message:"profile deletion requires --force",next:"Re-run `crtr profile delete <profile> --force` after reviewing its effects."});if(t.detach!==void 0&&typeof t.detach!="boolean")throw f("detach must be a boolean",{error:"invalid_detach",message:"detach must be a boolean",next:"Use --detach as a presence-only flag, or omit it."});const r=t.caller_node_id;if(r!=null&&(typeof r!="string"||r===""))throw f("caller_node_id must be a nonempty string or null");return{force:!0,...t.detach===!0?{detach:!0}:{},...r===void 0?{}:{caller_node_id:r}}}d(re,"validateRequest");function j(e){if(!w(e))return null;try{const t=JSON.parse(E(e,"utf8"));return typeof t.source?.nodeId=="string"&&t.source.nodeId!==""?t.source.nodeId:null}catch{return null}}d(j,"manifestSourceNodeId");async function oe(e){const t=new Set(e),r=B();if(t.size===0||!w(r))return[];const o=[];for(const n of g(r,{withFileTypes:!0})){if(!n.isDirectory())continue;const s=u(r,n.name);[j(u(s,"page.json")),j(u(s,"review.json"))].some(c=>c!==null&&t.has(c))&&o.push({dir:s,wholeDirectory:t.has(n.name)||await W(n.name)===null})}return o.sort((n,s)=>n.dir.localeCompare(s.dir))}d(oe,"ticketCleanupForNodes");function ne(e){if(N.has(e))return!0;for(const t of N)if(e.startsWith(`${t}.`)&&(e.endsWith(".tmp")||e.endsWith(".lock")))return!0;return!1}d(ne,"isTicketOwnedResidue");function ie(e){let t=0;for(const r of e)if(w(r.dir)){if(r.wholeDirectory){p(r.dir,{recursive:!0,force:!0}),t+=1;continue}for(const o of g(r.dir,{withFileTypes:!0}))ne(o.name)&&p(u(r.dir,o.name),{recursive:!0,force:!0});t+=1}return t}d(ie,"cleanTickets");function S(e){const t=e.filter(o=>o.managed_worktree?.state==="open").map(o=>({node_id:o.node_id,path:o.managed_worktree.path}));if(t.length===0)return null;const r=`profile deletion is blocked by ${t.length} open managed worktree${t.length===1?"":"s"}`;return f(r,{error:"open_managed_worktree",message:r,nodes:t,next:`Close ${t.map(({node_id:o})=>o).join(", ")} with \`crtr node worktree close <node>\`, then retry.`})}d(S,"openWorktreeError");async function se(e,t){const r=await M(e.profileId),o=S(r);if(o!==null)throw o;const n=r.map(({node_id:a})=>a),s=t.detach===!0?"detached":"deleted";return{entry:e,mode:s,nodes:r,nodeIds:n,tickets:s==="deleted"?await oe(n):[],cronIds:s==="deleted"?await R(e.profileId,n):await C(e.profileId),reviewIds:s==="deleted"?await K(n):[],defaultDirs:L(e.profileId)}}d(se,"prepareDeletion");async function x(e){const t=await ee(e.nodeIds,async o=>{const n=await m(o);return n===null?[]:[n]}),r=S(t);if(r!==null)throw r}d(x,"assertDeletionPlanPreconditions");function y(e){return{profile_id:e.entry.profileId,name:e.entry.manifest.name,mode:e.mode,nodes:e.nodeIds.length,crons:e.cronIds.length,tickets:e.tickets.length,reviews:e.reviewIds.length,default_pins:e.defaultDirs.length}}d(y,"resultForPlan");async function de(e){await x(e);const t=await Z(e.nodeIds,async i=>await m(i)!==null),r=U(),o=t.flatMap(i=>{const l=r.get(i);return l===void 0?[]:[{nodeId:i,exited:l.handle.exited}]}),n=[];for(const{nodeId:i}of o){const l=await m(i);(l?.status==="active"||l?.status==="idle")&&(await Q(i,"release"),n.push(i)),b.teardown(i)}await Promise.all(o.map(({exited:i})=>i));const s=[];for(const i of t)await m(i)!==null&&(await X(i,{profileId:null}),s.push(i));const a=await A(e.entry.profileId),c=await k(e.entry.profileId);await v(async i=>await _(i,e.entry.profileId)),await D(e.entry.profileId);const I=new Set(s);for(const i of n)I.has(i)&&await Y(i,{resume:!0,recovery:!0,capacity:"freeze"});return{...y(e),nodes:s.length,crons:a.length,default_pins:c.length}}d(de,"detachProfile");async function ae(e){await x(e);const t=await T(e.entry.profileId,e.nodeIds);for(const a of e.nodeIds)b.teardown(a),await V(a);const r=await H(e.nodeIds);for(const a of r)p($(a),{recursive:!0,force:!0});const o=ie(e.tickets),n=await O(e.nodeIds),s=await k(e.entry.profileId);return await v(async a=>await _(a,e.entry.profileId)),await D(e.entry.profileId),{...y(e),nodes:n.length,crons:t.length,tickets:o,reviews:r.length,default_pins:s.length}}d(ae,"hardDeleteProfile");async function F(e){return e.mode==="detached"?de(e):await ae(e)}d(F,"executePlan");function ce(e,t){const r={...y(e),accepted:!0,grace_ms:P};return h.set(e.entry.profileId,{result:r}),setTimeout(()=>{(async()=>{try{await F(e)}catch(o){z({level:"error",event:"profile.delete.deferred_failed",error:o instanceof Error?o:new Error(String(o)),fields:{profile_id:e.entry.profileId,mode:e.mode}})}finally{h.delete(e.entry.profileId),t()}})()},P),r}d(ce,"scheduleSelfDeletion");async function xe(e,t){const r=re(t),o=G(e),n=h.get(o.profileId);if(n!==void 0)return{status:202,body:n.result};const s=J(o.profileId);let a=!1;try{const c=await se(o,r);if(r.caller_node_id!=null&&c.nodeIds.includes(r.caller_node_id)){const i=ce(c,s);return a=!0,{status:202,body:i}}return{status:200,body:await F(c)}}finally{a||s()}}d(xe,"requestProfileDeletion");export{xe as requestProfileDeletion};

package/dist/daemon/reconcilers/bash-deadline.d.ts

@@ -17,25 +17,30 @@ export interface BashDeadlineContext {
  * a job the broker backgrounded while exiting (SIGTERM/SIGHUP), which could not
  * record its own job object. */
 export declare class BashDeadlineReconciler {
-    private lastScanAt;
     /** An exit written before this daemon started is history, not a job to adopt. */
     private readonly startedAt;
+    /** Bytes of each running job's log already in the blob store. */
+    private readonly uploadedBytes;
     run(now: number, ctx: BashDeadlineContext): Promise<void>;
+    private scan;
     /** Tell the owner about a live background job that has shown no activity (see the supervisor's
      *  activity sampler) for 15 minutes, again at 30 and 60, then hourly. Any activity restarts the
      *  schedule; a job marked as an expected wait is skipped. Messages only: the job is never stopped. */
     private warnSilent;
-    /** A running job whose supervisor wrote job.exit has exited: `finishJob` publishes `exited`
-     *  once (the row leaves `running`). A job being stopped (job.stop) is ended by its killer. */
+    /** Mirror what each live node's job supervisors wrote beside the jobs (process identity, activity,
+     *  exit) into the job rows, upload the output written since, and report a process confirmed gone
+     *  without an exit record as vanished. The supervisor files live on this host's disk; a broker on
+     *  another host reports the same things through the `report` route. */
+    private mirrorSupervisors;
+    private mirror;
+    /** Put the job's log in the blob store as 64 KiB chunks under its output key; chunks that were already
+     *  full are kept, the last one is rewritten as it grows. */
+    private uploadOutput;
+    /** End every job whose exit the supervisor reported: `finishJob` publishes `exited` once (the row leaves `running`). */
     private publishExits;
     /** Give every backgrounded job dir (job.bg) that has no job object one; the next tick's
-     *  `publishExits` then ends it like any other job. */
+     *  mirror then reports its process and exit like any other job's. */
     private adoptUnrecorded;
-    /** A running job whose process is confirmed gone without job.exit (killed outright, so its
-     *  supervisor never wrote one) is ended once as killed, with a null exit code, through
-     *  `finishJob`. job.exit is then written so the file-based listing and stop guard stop counting
-     *  the job. A job.exit that appears by the re-check is a real exit: `publishExits` reports it.
-     *  A job being stopped (job.stop) is ended by its killer. */
-    private settleVanished;
+    /** Cancel a job whose deadline passed: signal its process (a sandbox job's reporter does that once the row ends), then end its row as killed by the deadline. */
     private cancel;
 }