@north-light/cli 0.3.551 → 0.3.552

package/dist/daemon/crtrd.js

@@ -1,17 +1,17 @@
-var Ae=Object.defineProperty;var i=(n,s)=>Ae(n,"name",{value:s,configurable:!0});import{startPeerOutbox as Me}from"./peer/outbox.js";import{reportMachineStatus as Ne}from"../core/peer/directory.js";import{createPeerRelay as Fe}from"./peer/relay-service.js";import{writeFileSync as Ge,readFileSync as Oe,rmSync as Be,mkdirSync as se,existsSync as k,chmodSync as He}from"node:fs";import{createServer as ce}from"node:http";import{dirname as je}from"node:path";import{envCrtrdTcp as qe,envDocumentPort as ze}from"../shared/env.js";import{serveCanvasConversationStore as We}from"../core/canvas/sessions.js";import{pollRunObservers as Ue}from"./runs/observer.js";import{randomBytes as de}from"node:crypto";import{apiSocketPath as Ke,crtrHome as Je}from"../core/canvas/paths.js";import{activeAppUsers as Ve}from"../core/spaces/users.js";import{ensureAppSpace as Qe,ensureUserFiles as Ye}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as C}from"../launcher/client.js";import{installLauncherEventSink as Xe}from"./launcher-events.js";import{initializeEmptyLayout as Ze,layoutNeedsMigration as er,socketDir as le}from"../core/layout.js";import{migrateLooseKeys as rr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as tr}from"../core/profiles/relocate.js";import{getNode as nr,listNodesForSupervision as pe,listParkedResidentsWithWakeMail as or,setFrozen as me}from"../core/canvas/index.js";import{createApiServer as ir}from"./api/server.js";import{createDocumentListener as ar}from"./api/document-listener.js";import{documentSecret as sr}from"./api/document-secret.js";import{createAppListener as cr}from"./api/app-listener.js";import{registerMachineEnvironment as dr}from"../core/environments.js";import{resolvePerson as lr,retireLocalPerson as pr,setPersonId as mr}from"../core/canvas/person.js";import{backfillNodeApps as fr,runPermissionsBoot as ur}from"../core/permissions/boot.js";import{onRunEventAppended as wr}from"../core/runs/events.js";import{runNotices as hr}from"./run-notices.js";import{createGrantSync as vr,GRANT_SYNC_INTERVAL_S as gr}from"../core/grants/sync.js";import{runRemovalCascades as yr}from"../core/grants/remove.js";import{sweepSandboxes as Sr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as _r}from"./cron/run-socket.js";import{hasSyncedGrants as br}from"../core/grants/mirror.js";import{readConfig as fe}from"../core/config.js";import{reconcileUndeliveredTickets as kr}from"./human/sweep.js";import{resetHandedOffDeliveries as Pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Er}from"../core/canvas/daemon-actions.js";import{sweepReviews as Rr}from"./review/sweep.js";import{pidfilePath as K}from"./pidfile.js";import{rendererWarning as Ir}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as xr}from"./supervise-cadence.js";import{createResumeDetector as Dr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Tr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Cr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Lr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as $r}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as Ar}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as Mr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as Nr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as Fr}from"./reconcilers/action-lane.js";import{endLiveExecution as Gr,migrateLegacyExecutionIdentities as Or}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as Br}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as Hr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as jr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as qr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as zr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Wr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Ur}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Kr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as ue,capturePidCommand as Jr,captureTeardownSnapshot as we,isPidAlive as Vr,isAnyPidAlive as he,killProcessTreePids as ve}from"../core/canvas/pid.js";import{bindFleet as Qr,boundFleet as Yr}from"../core/runtime/fleet.js";import{bindDaemonControl as Xr,clearDaemonControl as Zr}from"./control.js";import{execDaemon as et,spawnDaemon as rt}from"./manage.js";import{reconcileAndMigrateBoot as tt}from"../core/canvas/boot.js";import{resolveInstallId as nt}from"../core/canvas/install-id.js";import{setInstallId as ot}from"../core/runtime/nodes.js";import{claimDaemonOwnership as ge,daemonOwnershipClaimed as it,releaseDaemonOwnership as ye}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as at,closeAllDbs as J,openDb as Se,withCanvasWrite as st}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as ct}from"../core/canvas/import-sqlite.js";import{ensurePostgres as dt,ensurePostgresHealthy as lt,stopPostgres as pt,withPostgresStartLock as mt}from"./postgres/index.js";import{brokerSignalBus as V}from"./broker-signals/bus.js";import{brokerSignalStreams as ft}from"./api/handlers/broker-signals.js";import{reconcilePackages as ut}from"../core/graph/package-reconcile.js";import{startRepoSync as wt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as ht,reapStaleSpares as vt}from"../core/runtime/warm-pool.js";import{DaemonFleet as gt}from"./fleet.js";import{BOOT_MIGRATIONS as yt}from"./boot-migrations.js";import{emitEvent as m}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as St}from"../core/events/source.js";import{activateCanvasImport as _t,MigrationBlockedError as bt,runForcedCanvasImport as kt}from"../migrations/activation.js";import{convertLegacyProviderStore as Pt,convertLegacyScopedStore as Et}from"../core/scoped-state/migrate.js";import{userScopeRoot as Rt}from"../core/scope.js";import{secretsDir as It}from"../core/layout.js";function Q(n,s){return n.includes(` --canvas-home ${s} `)}i(Q,"brokerCommandDeclaresCanvasHome");function Y(n){return/ --epoch (\S+) /.exec(n)?.[1]??null}i(Y,"brokerCommandEpoch");const xt=5e3,_e=150;async function Dt(n){const s=ue();if(s===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const l=Je(),w=[];for(const[a,u]of s.processes){const y=u.command;if(u.zombie||!y.includes("broker-cli.js")||!Q(y,l)||Y(y)===n)continue;const P=y.trim().split(/\s+/).at(-1)??"";P===""||P.endsWith("broker-cli.js")||w.push({pid:a,nodeId:P})}let d=0;const t=[];for(const a of w){const u=Jr(a.pid);if(u===null||!u.includes("broker-cli.js")||!Q(u,l)||Y(u)===n||u.trim().split(/\s+/).at(-1)!==a.nodeId)continue;const y=we(a.pid,null);if(!y.reused){try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{continue}d++,t.push({pid:a.pid,snap:y})}}const g=Date.now()+xt;for(;Date.now()<g&&t.some(a=>he(a.snap.tree.length>0?a.snap.tree:[a.pid]));)await new Promise(a=>setTimeout(a,_e));let h=0;for(const a of t){const u=a.snap.tree.length>0?a.snap.tree:[a.pid];he(u)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):ve(u,"SIGKILL",a.snap.identities),h++)}h>0&&await new Promise(a=>setTimeout(a,_e));const f=ue();if(f===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[a,u]of f.processes){const y=u.command;u.zombie||!y.includes("broker-cli.js")||Q(y,l)&&Y(y)!==n&&c.push(a)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);m({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:w.length,sigtermed:d,sigkilled:h}})}i(Dt,"reapPriorEpochBrokers");const Tt=5e3;async function Ct(n){const s=Array.from(n.entries());if(s.length===0)return;for(const[t]of s){const g=(await nr(t))?.status;(g==="active"||g==="idle")&&await me(t,new Date().toISOString())}const l=s.map(([,t])=>({pid:t.pid,snap:we(t.pid,null),exited:t.handle.exited}));for(const t of l)if(!t.snap.reused)try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):process.kill(t.pid,"SIGTERM")}catch{}const w=new Promise(t=>{setTimeout(t,Tt).unref()});await Promise.allSettled(l.map(t=>Promise.race([t.exited.then(()=>{}),w])));const d=[];for(const t of l)Vr(t.pid)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):ve(t.snap.tree.length>0?t.snap.tree:[t.pid],"SIGKILL",t.snap.identities),d.push(t));await Promise.all(d.map(t=>t.exited));for(const[t]of s)await Gr(t,{reason:"daemon-teardown"});m({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:l.length,sigkilled:d.length}})}i(Ct,"teardownFleet");function be(){se(je(K()),{recursive:!0}),Ge(K(),String(process.pid),"utf8")}i(be,"writePidfile");function ke(n){return typeof n=="object"&&n!==null&&n.code==="ENOENT"}i(ke,"isMissingPathError");function Pe(n){const s=K();let l;try{l=Oe(s,"utf8")}catch(w){if(ke(w))return;throw w}if(l.trim()!==String(n))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${n})`);try{Be(s)}catch(w){if(ke(w))return;throw w}}i(Pe,"removeOwnedPidfileMirror");const Lt={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(n=>{n.catch(()=>{})},"registerDetached")},H=new Cr;Tr(n=>H.handleFatalFault(n,!1));const $t=new Lr,At=new $r,Mt=new Ar,Nt=new Mr,Ft=new Nr,Gt=new Fr,Ot=new Hr,Bt=new jr,Ht=new qr,jt=new zr,qt=new Wr,zt=new Ur;async function Wt(n=Date.now(),s=Lt){const l=Yr(),w=await H.runFleetPreamble(l);let d,t;try{const h=await pe();d=h.filter(c=>c.status==="active"||c.status==="idle"),t=h.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const f=new Set(t.map(c=>c.node_id));t.push(...(await or()).filter(c=>!f.has(c.node_id)))}catch(h){m({level:"error",event:"canvas.node_list.failed",error:h});return}await H.run(n,{rows:d,fleet:l,lifecycle:s}),await $t.run(n,{rows:d});const g=await At.run(n,{rows:[...d,...t],fleet:l});await Mt.run(n),await zt.run(n),await Nt.run(n,{lifecycle:s}),await Ft.run(n,{lifecycle:s}),await Gt.run(n,{lifecycle:s}),await Ot.run(n,{lifecycle:s}),await Bt.run(n,{lifecycle:s,fleet:l}),Ht.run(n,{lifecycle:s}),await Ue(),qt.run(n,{fleet:l,lifecycle:s}),await jt.run(n,{rows:d}),H.warnIfSaturated({fleet:l,lifecycle:s,thresholds:w,frozen:g})}i(Wt,"superviseTick");async function Ut(n){se(le(),{recursive:!0,mode:493}),process.platform==="linux"&&He(le(),493);const s=Ke();if(k(s))throw new Error(`layout refusal cannot bind occupied socket: ${s}`);const l=ce((f,c)=>{if(f.method==="GET"&&f.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((f,c)=>{l.once("error",c),l.listen(s,()=>{l.off("error",c),f()})});let w;const d=n.tcp??qe();if(d){const f=/^(.*):(\d+)$/.exec(d);if(!f||Number(f[2])>65535)throw new Error(`invalid refusal TCP address: ${d}`);const c=ce(l.listeners("request")[0]);await new Promise((a,u)=>{c.once("error",u),c.listen(Number(f[2]),f[1],()=>{c.off("error",u),a()})}),w=c}be();let t;const g=i(()=>(t??=(async()=>{l.closeAllConnections(),w?.closeAllConnections(),await Promise.all([l,w].filter(f=>f!==void 0).map(f=>new Promise((c,a)=>f.close(u=>u?a(u):c())))),Pe(process.pid),process.off("SIGINT",h),process.off("SIGTERM",h)})(),t),"cleanup"),h=i(()=>{g().then(()=>process.exit(0),f=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(f)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",h),process.on("SIGTERM",h),g}i(Ut,"serveLayoutRefusal");async function Kt(n={}){if(er())return Ut(n);Ze();const s=i(r=>{process.stderr.write(`[crtrd] ${r}
-`)},"logPostgres"),l=!!process.env.CRTR_TEST_PG_URL,{database:w,ownership:d}=l?{database:{url:await at()},ownership:await ge(de(16).toString("hex"))}:await mt(async()=>({database:await dt({log:s,startLockHeld:!0}),ownership:await ge(de(16).toString("hex"))}));if(!d.claimed){if(d.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${d.incumbentPid??"?"})
+var Ae=Object.defineProperty;var i=(n,s)=>Ae(n,"name",{value:s,configurable:!0});import{backfillLedger as Me}from"../core/ledger/backfill.js";import{startPeerOutbox as Ne}from"./peer/outbox.js";import{reportMachineStatus as Fe}from"../core/peer/directory.js";import{createPeerRelay as Ge}from"./peer/relay-service.js";import{writeFileSync as Oe,readFileSync as Be,rmSync as He,mkdirSync as se,existsSync as k,chmodSync as je}from"node:fs";import{createServer as ce}from"node:http";import{dirname as qe}from"node:path";import{envCrtrdTcp as ze,envDocumentPort as We}from"../shared/env.js";import{serveCanvasConversationStore as Ue}from"../core/canvas/sessions.js";import{pollRunObservers as Ke}from"./runs/observer.js";import{randomBytes as de}from"node:crypto";import{apiSocketPath as Je,crtrHome as Ve}from"../core/canvas/paths.js";import{activeAppUsers as Qe}from"../core/spaces/users.js";import{ensureAppSpace as Ye,ensureUserFiles as Xe}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as C}from"../launcher/client.js";import{installLauncherEventSink as Ze}from"./launcher-events.js";import{initializeEmptyLayout as er,layoutNeedsMigration as rr,socketDir as le}from"../core/layout.js";import{migrateLooseKeys as tr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as nr}from"../core/profiles/relocate.js";import{getNode as or,listNodesForSupervision as pe,listParkedResidentsWithWakeMail as ir,setFrozen as me}from"../core/canvas/index.js";import{createApiServer as ar}from"./api/server.js";import{createDocumentListener as sr}from"./api/document-listener.js";import{documentSecret as cr}from"./api/document-secret.js";import{createAppListener as dr}from"./api/app-listener.js";import{registerMachineEnvironment as lr}from"../core/environments.js";import{resolvePerson as pr,retireLocalPerson as mr,setPersonId as fr}from"../core/canvas/person.js";import{backfillNodeApps as ur,runPermissionsBoot as wr}from"../core/permissions/boot.js";import{onRunEventAppended as hr}from"../core/runs/events.js";import{runNotices as vr}from"./run-notices.js";import{createGrantSync as gr,GRANT_SYNC_INTERVAL_S as yr}from"../core/grants/sync.js";import{runRemovalCascades as Sr}from"../core/grants/remove.js";import{sweepSandboxes as _r}from"../core/grants/on-sync.js";import{rebuildCronExecutions as br}from"./cron/run-socket.js";import{hasSyncedGrants as kr}from"../core/grants/mirror.js";import{readConfig as fe}from"../core/config.js";import{reconcileUndeliveredTickets as Pr}from"./human/sweep.js";import{resetHandedOffDeliveries as Er}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Rr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Ir}from"./review/sweep.js";import{pidfilePath as K}from"./pidfile.js";import{rendererWarning as xr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Dr}from"./supervise-cadence.js";import{createResumeDetector as Tr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Cr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Lr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as $r}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as Ar}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as Mr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as Nr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as Fr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as Gr}from"./reconcilers/action-lane.js";import{endLiveExecution as Or,migrateLegacyExecutionIdentities as Br}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as Hr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as jr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as qr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as zr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Wr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Ur}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Kr}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Jr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as ue,capturePidCommand as Vr,captureTeardownSnapshot as we,isPidAlive as Qr,isAnyPidAlive as he,killProcessTreePids as ve}from"../core/canvas/pid.js";import{bindFleet as Yr,boundFleet as Xr}from"../core/runtime/fleet.js";import{bindDaemonControl as Zr,clearDaemonControl as et}from"./control.js";import{execDaemon as rt,spawnDaemon as tt}from"./manage.js";import{reconcileAndMigrateBoot as nt}from"../core/canvas/boot.js";import{resolveInstallId as ot}from"../core/canvas/install-id.js";import{setInstallId as it}from"../core/runtime/nodes.js";import{claimDaemonOwnership as ge,daemonOwnershipClaimed as at,releaseDaemonOwnership as ye}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as st,closeAllDbs as J,openDb as Se,withCanvasWrite as ct}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as dt}from"../core/canvas/import-sqlite.js";import{ensurePostgres as lt,ensurePostgresHealthy as pt,stopPostgres as mt,withPostgresStartLock as ft}from"./postgres/index.js";import{brokerSignalBus as V}from"./broker-signals/bus.js";import{brokerSignalStreams as ut}from"./api/handlers/broker-signals.js";import{reconcilePackages as wt}from"../core/graph/package-reconcile.js";import{startRepoSync as ht}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as vt,reapStaleSpares as gt}from"../core/runtime/warm-pool.js";import{DaemonFleet as yt}from"./fleet.js";import{BOOT_MIGRATIONS as St}from"./boot-migrations.js";import{emitEvent as m}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as _t}from"../core/events/source.js";import{activateCanvasImport as bt,MigrationBlockedError as kt,runForcedCanvasImport as Pt}from"../migrations/activation.js";import{convertLegacyProviderStore as Et,convertLegacyScopedStore as Rt}from"../core/scoped-state/migrate.js";import{userScopeRoot as It}from"../core/scope.js";import{secretsDir as xt}from"../core/layout.js";function Q(n,s){return n.includes(` --canvas-home ${s} `)}i(Q,"brokerCommandDeclaresCanvasHome");function Y(n){return/ --epoch (\S+) /.exec(n)?.[1]??null}i(Y,"brokerCommandEpoch");const Dt=5e3,_e=150;async function Tt(n){const s=ue();if(s===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const l=Ve(),w=[];for(const[a,u]of s.processes){const y=u.command;if(u.zombie||!y.includes("broker-cli.js")||!Q(y,l)||Y(y)===n)continue;const P=y.trim().split(/\s+/).at(-1)??"";P===""||P.endsWith("broker-cli.js")||w.push({pid:a,nodeId:P})}let d=0;const t=[];for(const a of w){const u=Vr(a.pid);if(u===null||!u.includes("broker-cli.js")||!Q(u,l)||Y(u)===n||u.trim().split(/\s+/).at(-1)!==a.nodeId)continue;const y=we(a.pid,null);if(!y.reused){try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{continue}d++,t.push({pid:a.pid,snap:y})}}const v=Date.now()+Dt;for(;Date.now()<v&&t.some(a=>he(a.snap.tree.length>0?a.snap.tree:[a.pid]));)await new Promise(a=>setTimeout(a,_e));let h=0;for(const a of t){const u=a.snap.tree.length>0?a.snap.tree:[a.pid];he(u)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:a.pid}):ve(u,"SIGKILL",a.snap.identities),h++)}h>0&&await new Promise(a=>setTimeout(a,_e));const f=ue();if(f===null)throw m({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[a,u]of f.processes){const y=u.command;u.zombie||!y.includes("broker-cli.js")||Q(y,l)&&Y(y)!==n&&c.push(a)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);m({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:w.length,sigtermed:d,sigkilled:h}})}i(Tt,"reapPriorEpochBrokers");const Ct=5e3;async function Lt(n){const s=Array.from(n.entries());if(s.length===0)return;for(const[t]of s){const v=(await or(t))?.status;(v==="active"||v==="idle")&&await me(t,new Date().toISOString())}const l=s.map(([,t])=>({pid:t.pid,snap:we(t.pid,null),exited:t.handle.exited}));for(const t of l)if(!t.snap.reused)try{process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):process.kill(t.pid,"SIGTERM")}catch{}const w=new Promise(t=>{setTimeout(t,Ct).unref()});await Promise.allSettled(l.map(t=>Promise.race([t.exited.then(()=>{}),w])));const d=[];for(const t of l)Qr(t.pid)&&(process.platform==="linux"&&k(x)?await C({operation:"stop",pid:t.pid}):ve(t.snap.tree.length>0?t.snap.tree:[t.pid],"SIGKILL",t.snap.identities),d.push(t));await Promise.all(d.map(t=>t.exited));for(const[t]of s)await Or(t,{reason:"daemon-teardown"});m({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:l.length,sigkilled:d.length}})}i(Lt,"teardownFleet");function be(){se(qe(K()),{recursive:!0}),Oe(K(),String(process.pid),"utf8")}i(be,"writePidfile");function ke(n){return typeof n=="object"&&n!==null&&n.code==="ENOENT"}i(ke,"isMissingPathError");function Pe(n){const s=K();let l;try{l=Be(s,"utf8")}catch(w){if(ke(w))return;throw w}if(l.trim()!==String(n))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${n})`);try{He(s)}catch(w){if(ke(w))return;throw w}}i(Pe,"removeOwnedPidfileMirror");const $t={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(n=>{n.catch(()=>{})},"registerDetached")},H=new Lr;Cr(n=>H.handleFatalFault(n,!1));const At=new $r,Mt=new Ar,Nt=new Mr,Ft=new Nr,Gt=new Fr,Ot=new Gr,Bt=new jr,Ht=new qr,jt=new zr,qt=new Wr,zt=new Ur,Wt=new Kr;async function Ut(n=Date.now(),s=$t){const l=Xr(),w=await H.runFleetPreamble(l);let d,t;try{const h=await pe();d=h.filter(c=>c.status==="active"||c.status==="idle"),t=h.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const f=new Set(t.map(c=>c.node_id));t.push(...(await ir()).filter(c=>!f.has(c.node_id)))}catch(h){m({level:"error",event:"canvas.node_list.failed",error:h});return}await H.run(n,{rows:d,fleet:l,lifecycle:s}),await At.run(n,{rows:d});const v=await Mt.run(n,{rows:[...d,...t],fleet:l});await Nt.run(n),await Wt.run(n),await Ft.run(n,{lifecycle:s}),await Gt.run(n,{lifecycle:s}),await Ot.run(n,{lifecycle:s}),await Bt.run(n,{lifecycle:s}),await Ht.run(n,{lifecycle:s,fleet:l}),jt.run(n,{lifecycle:s}),await Ke(),zt.run(n,{fleet:l,lifecycle:s}),await qt.run(n,{rows:d}),H.warnIfSaturated({fleet:l,lifecycle:s,thresholds:w,frozen:v})}i(Ut,"superviseTick");async function Kt(n){se(le(),{recursive:!0,mode:493}),process.platform==="linux"&&je(le(),493);const s=Je();if(k(s))throw new Error(`layout refusal cannot bind occupied socket: ${s}`);const l=ce((f,c)=>{if(f.method==="GET"&&f.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((f,c)=>{l.once("error",c),l.listen(s,()=>{l.off("error",c),f()})});let w;const d=n.tcp??ze();if(d){const f=/^(.*):(\d+)$/.exec(d);if(!f||Number(f[2])>65535)throw new Error(`invalid refusal TCP address: ${d}`);const c=ce(l.listeners("request")[0]);await new Promise((a,u)=>{c.once("error",u),c.listen(Number(f[2]),f[1],()=>{c.off("error",u),a()})}),w=c}be();let t;const v=i(()=>(t??=(async()=>{l.closeAllConnections(),w?.closeAllConnections(),await Promise.all([l,w].filter(f=>f!==void 0).map(f=>new Promise((c,a)=>f.close(u=>u?a(u):c())))),Pe(process.pid),process.off("SIGINT",h),process.off("SIGTERM",h)})(),t),"cleanup"),h=i(()=>{v().then(()=>process.exit(0),f=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(f)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",h),process.on("SIGTERM",h),v}i(Kt,"serveLayoutRefusal");async function Jt(n={}){if(rr())return Kt(n);er();const s=i(r=>{process.stderr.write(`[crtrd] ${r}
+`)},"logPostgres"),l=!!process.env.CRTR_TEST_PG_URL,{database:w,ownership:d}=l?{database:{url:await st()},ownership:await ge(de(16).toString("hex"))}:await ft(async()=>({database:await lt({log:s,startLockHeld:!0}),ownership:await ge(de(16).toString("hex"))}));if(!d.claimed){if(d.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${d.incumbentPid??"?"})
 `);else{const r=[d.incumbentPid!==void 0?`pid ${d.incumbentPid}`:void 0,d.verdict!==void 0?`verdict ${d.verdict}`:void 0].filter(S=>S!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${d.reason}${e}
-`),d.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(d.reason==="own_identity_indeterminate"||d.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const t=d.claim;let g=null,h=null,f=null,c="";try{await Dt(t.token),Et({kind:"user",root:Rt()});for(const e of tr())process.stderr.write(`[crtrd] ${e}
-`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:S}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:o}=await import("node:os");S("user",p=>{p.paths??={},p.paths.user_files??=o()})}}await Pt(It());for(const e of rr())process.stderr.write(`[crtrd] ${e}
-`);await Se(),await ct(w.url,s),await V.listen(w.url);const r=await st(async e=>(await fr(e),await ut(e)));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
+`),d.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(d.reason==="own_identity_indeterminate"||d.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const t=d.claim;let v=null,h=null,f=null,c="";try{await Tt(t.token),Rt({kind:"user",root:It()});for(const e of nr())process.stderr.write(`[crtrd] ${e}
+`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:S}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:o}=await import("node:os");S("user",p=>{p.paths??={},p.paths.user_files??=o()})}}await Et(xt());for(const e of tr())process.stderr.write(`[crtrd] ${e}
+`);await Se(),await dt(w.url,s),await V.listen(w.url);const r=await ct(async e=>(await ur(e),await wt(e)));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
 `);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
 `);for(const e of r.quarantined)process.stderr.write(`[crtrd] plugin "${e.name}" has a damaged ${e.file} (${e.reason}); ${e.movedTo!==void 0?`moved to ${e.movedTo}`:"left in place and skipped"}
 `);for(const e of r.kinds.refused)process.stderr.write(`[crtrd] plugin "${e.plugin}" kind ${e.kind} not registered: ${e.message}
 `);for(const e of r.kinds.invalid)process.stderr.write(`[crtrd] plugin "${e.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${e.error}
-`);c=await nt(),ot(c),g=await lr(),g!==null&&(mr(g.id),h=await pr(),f=await ur()),process.platform==="linux"&&k(x)&&(Xe(),await C({operation:"users.sync",users:await Ve()})),await Br(t.token)}catch(r){await V.close();try{const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");await J()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let a=!0,u,y=Promise.resolve(),P=!1,N,D;const F=new Set;let j=!1,X=!1,E,G,q,Z=!1,O,L,$,A,z;const Ee=Dr();let R,v={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},T="initial startup activation",Re=null;function Ie(){return v.phase==="ready"?null:{phase:v.phase,message:v.message,blockerPaths:v.blockerPaths}}i(Ie,"startupBlock");function b(r){F.add(r),r.finally(()=>{F.delete(r)}).catch(()=>{})}i(b,"registerDetached");const ee={acceptsDetachedWork:i(()=>a,"acceptsDetachedWork"),registerDetached:b};await wt(ee,Se);async function xe(){try{a=!1,ft.close(),await V.close(),N!==void 0&&(clearInterval(N),N=void 0),$!==void 0&&(clearInterval($),$=void 0),z?.(),z=void 0;let r;do r=y,await r;while(r!==y);for(;F.size>0;)await Promise.allSettled([...F]);if(D!==void 0&&(await Ct(D),await D.exitsSettled(),D.dispose()),q!==void 0&&await q.close(),G!==void 0&&!Z&&(Z=!0,await G.close()),A!==void 0){const e=A;A=void 0,await e.stop()}if(L!==void 0){const e=L;L=void 0,await e.close()}if(O!==void 0){const e=O;O=void 0,await e.close()}if(j||(Pe(process.pid),j=!0),j&&!X){const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");X=!0,await J()}}finally{Zr(),process.off("SIGINT",ne),process.off("SIGTERM",oe),process.off("exit",ie)}}i(xe,"cleanupAttempt");const B=1e3;let re=!1;function De(){return re?{graceMs:B}:(re=!0,_.fresh(()=>{m({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:B}})}),setTimeout(()=>{Te()},B),{graceMs:B})}i(De,"requestHandover");async function Te(){try{await I()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+`);c=await ot(),it(c),v=await pr(),v!==null&&(fr(v.id),h=await mr(),f=await wr(),await Me(v.id)),process.platform==="linux"&&k(x)&&(Ze(),await C({operation:"users.sync",users:await Qe()})),await Hr(t.token)}catch(r){await V.close();try{const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");await J()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let a=!0,u,y=Promise.resolve(),P=!1,N,D;const F=new Set;let j=!1,X=!1,E,G,q,Z=!1,O,L,$,A,z;const Ee=Tr();let R,g={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},T="initial startup activation",Re=null;function Ie(){return g.phase==="ready"?null:{phase:g.phase,message:g.message,blockerPaths:g.blockerPaths}}i(Ie,"startupBlock");function b(r){F.add(r),r.finally(()=>{F.delete(r)}).catch(()=>{})}i(b,"registerDetached");const ee={acceptsDetachedWork:i(()=>a,"acceptsDetachedWork"),registerDetached:b};await ht(ee,Se);async function xe(){try{a=!1,ut.close(),await V.close(),N!==void 0&&(clearInterval(N),N=void 0),$!==void 0&&(clearInterval($),$=void 0),z?.(),z=void 0;let r;do r=y,await r;while(r!==y);for(;F.size>0;)await Promise.allSettled([...F]);if(D!==void 0&&(await Lt(D),await D.exitsSettled(),D.dispose()),q!==void 0&&await q.close(),G!==void 0&&!Z&&(Z=!0,await G.close()),A!==void 0){const e=A;A=void 0,await e.stop()}if(L!==void 0){const e=L;L=void 0,await e.close()}if(O!==void 0){const e=O;O=void 0,await e.close()}if(j||(Pe(process.pid),j=!0),j&&!X){const{released:e}=await ye(t);if(!e)throw new Error("daemon ownership claim was not released");X=!0,await J()}}finally{et(),process.off("SIGINT",ne),process.off("SIGTERM",oe),process.off("exit",ie)}}i(xe,"cleanupAttempt");const B=1e3;let re=!1;function De(){return re?{graceMs:B}:(re=!0,_.fresh(()=>{m({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:B}})}),setTimeout(()=>{Te()},B),{graceMs:B})}i(De,"requestHandover");async function Te(){try{await I()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
 ${e}
-`),process.exit(1)}try{process.platform==="linux"&&k(x)&&(await C({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await et();const r=await rt({waitForJoinLock:!0});_.fresh(()=>{m({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
-`),process.exit(1)}process.exit(0)}i(Te,"performHandover");function I(){if(E!==void 0)return E;const r=xe();return E=r,r.then(()=>{E===r&&(E=void 0)},()=>{E===r&&(E=void 0)}),r}i(I,"requestCleanup");async function te(r){try{await I(),await pt({claimed:i(async()=>{try{return await it()}finally{await J()}},"claimed")}),process.exit(0)}catch(e){process.exitCode=1;const S=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
+`),process.exit(1)}try{process.platform==="linux"&&k(x)&&(await C({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await rt();const r=await tt({waitForJoinLock:!0});_.fresh(()=>{m({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
+`),process.exit(1)}process.exit(0)}i(Te,"performHandover");function I(){if(E!==void 0)return E;const r=xe();return E=r,r.then(()=>{E===r&&(E=void 0)},()=>{E===r&&(E=void 0)}),r}i(I,"requestCleanup");async function te(r){try{await I(),await mt({claimed:i(async()=>{try{return await at()}finally{await J()}},"claimed")}),process.exit(0)}catch(e){process.exitCode=1;const S=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
 ${S}
-`),process.exit(1)}}i(te,"handleSignal");function ne(){te("SIGINT")}i(ne,"onSigint");function oe(){te("SIGTERM")}i(oe,"onSigterm");function ie(){I().catch(()=>{})}i(ie,"onExit");async function W(r){a&&(y=y.then(r).then(()=>{},e=>{_.fresh(()=>{m({level:"error",event:"daemon.lane_job.failed",error:e})})}))}i(W,"enqueue");async function ae(){!a||P||(Ee.check()&&R&&(m({level:"info",event:"daemon.resume.detected"}),b(R().catch(()=>{}))),P=!0,await W(async()=>{if(P=!1,!!a)try{l||await lt({log:s}),await Wt(Date.now(),ee)}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(ae,"enqueueTick");let M=null;async function Ce(){if(M!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${M instanceof Error?M.message:String(M)}`};if(v.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(v.phase!=="prepared"&&v.phase!=="recovering")throw{status:409,code:"not_prepared",message:v.message};const r=v.phase==="prepared";try{return await U(),v={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw M=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}i(Ce,"admitStartup");async function Le(r){if(T!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${T})`};T="repair";try{const e=await kt({dryRun:r.dryRun});if(r.dryRun||e.pending||v.phase!=="blocked")return{result:e,resumed:!1};if(n.prepare)return v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await U(),v={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(S){const o=S instanceof Error?S.message:String(S);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:o}}}finally{T=null}}i(Le,"repairMigrations");async function $e(){u=n.intervalMs??xr;const r=t.token;D=new gt({epoch:r,enqueue:W}),Qr(D),v={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{m({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:u}})});const e=c;_.fresh(()=>{m({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})});const S=await dr(e);if(_.fresh(()=>{m({level:"info",event:"daemon.environment.registered",fields:{environment_id:S.environmentId,created:S.created}})}),_.fresh(()=>{let o;try{o=Ir()}catch{return}o!==null&&m({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:o.slice(0,2048)}})}),await _.fresh(async()=>{await Rr(),await kr(),await Pr(Date.now()),await Er(Date.now())}),await _.fresh(async()=>{const{reconcile:o,migrated:p}=await tt(Or);o.changed&&m({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:o.cleared.length,same_boot_proven:o.sameBootProven}}),p.length>0&&m({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:p.length}})}),await _.fresh(async()=>{for(const o of await pe())o.status!=="active"||o.pi_pid!=null||o.frozen_at!=null||o.respawn_not_before!=null||(await me(o.node_id,new Date().toISOString()),m({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:o.node_id}))}),await _.fresh(async()=>{await Kr();for(const o of yt){const p=o.apply();p.changed&&m({level:"info",event:"daemon.boot_migration.applied",fields:{migration:o.id,...p.summary}})}}),b(_r()),R){const o=i(()=>{b(R().catch(()=>{})),b(Ne().then(p=>A?.observe(p)).then(()=>{},p=>{m({level:"warn",event:"daemon.peer.status_failed",error:p})}))},"refresh");b(yr().catch(()=>{})),b(Sr().then(({completion:p})=>p).catch(p=>{m({level:"error",event:"daemon.grants.sweep_failed",error:p})})),o(),$=setInterval(o,gr*1e3),$.unref(),z=Me({joined:!0,register:b})}await W(async()=>{await vt(),await ht()}),N=setInterval(()=>{ae().catch(o=>m({level:"error",event:"daemon.tick.failed",error:o}))},u),await ae()}i($e,"completeStartup");function U(){return Re??=$e().catch(r=>{throw v={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(U,"resumeStartup");try{process.on("SIGINT",ne),process.on("SIGTERM",oe),process.on("exit",ie),St(),g!==null&&m({level:"info",event:"daemon.person.resolved",fields:{person_id:g.id,source:g.source}}),f!==null&&m({level:"info",event:"daemon.permissions.boot",fields:{...f}}),h!==null&&m({level:"info",event:"daemon.person.retired",fields:{person_id:h.person,joined_from:h.joinedFrom,retired:h.retired,columns:h.columns}}),O=await We();const r=t.token;Xr({epoch:r,requestHandover:De,registerDetached:b,startupBlock:Ie,admitStartup:Ce,repairMigrations:Le}),be(),G=ir({tcp:n.tcp}),await G.ready;const e=ze()??fe("user").document_port,S=ar(e,sr());try{await S.ready,q=S}catch(p){m({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:p instanceof Error?p.message:String(p)}})}const o=fe("user").runtime;o?.issuer&&(R=vr(o.issuer,b),o.unsandboxed===!0&&(A=Fe({syncGrants:R})),wr(p=>hr().appended(p))),o?.unsandboxed===!0&&o.listener!==void 0&&m({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),o!==void 0&&o.listener!==void 0&&o.unsandboxed!==!0&&(L=await cr(o,async()=>await br(o.issuer.replace(/\/$/,"")),R),await L.ready);try{await _t()}catch(p){if(T=null,p instanceof bt)return v={phase:"blocked",message:p.message,blockerPaths:p.blockerPaths},I;throw p}process.platform==="linux"&&k(x)&&(Ye(),await Qe("app:terminal"));try{n.prepare?v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await U(),v={phase:"ready",message:"",blockerPaths:[]})}finally{T=null}return I}catch(r){try{await I()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}i(Kt,"runDaemon");var xo=Kt;export{Q as brokerCommandDeclaresCanvasHome,Y as brokerCommandEpoch,xo as default,Kt as runDaemon,Wt as superviseTick,Ct as teardownFleet};
+`),process.exit(1)}}i(te,"handleSignal");function ne(){te("SIGINT")}i(ne,"onSigint");function oe(){te("SIGTERM")}i(oe,"onSigterm");function ie(){I().catch(()=>{})}i(ie,"onExit");async function W(r){a&&(y=y.then(r).then(()=>{},e=>{_.fresh(()=>{m({level:"error",event:"daemon.lane_job.failed",error:e})})}))}i(W,"enqueue");async function ae(){!a||P||(Ee.check()&&R&&(m({level:"info",event:"daemon.resume.detected"}),b(R().catch(()=>{}))),P=!0,await W(async()=>{if(P=!1,!!a)try{l||await pt({log:s}),await Ut(Date.now(),ee)}catch(r){_.fresh(()=>{m({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(ae,"enqueueTick");let M=null;async function Ce(){if(M!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${M instanceof Error?M.message:String(M)}`};if(g.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(g.phase!=="prepared"&&g.phase!=="recovering")throw{status:409,code:"not_prepared",message:g.message};const r=g.phase==="prepared";try{return await U(),g={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw M=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}i(Ce,"admitStartup");async function Le(r){if(T!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${T})`};T="repair";try{const e=await Pt({dryRun:r.dryRun});if(r.dryRun||e.pending||g.phase!=="blocked")return{result:e,resumed:!1};if(n.prepare)return g={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await U(),g={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(S){const o=S instanceof Error?S.message:String(S);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:o}}}finally{T=null}}i(Le,"repairMigrations");async function $e(){u=n.intervalMs??Dr;const r=t.token;D=new yt({epoch:r,enqueue:W}),Yr(D),g={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{m({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:u}})});const e=c;_.fresh(()=>{m({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})});const S=await lr(e);if(_.fresh(()=>{m({level:"info",event:"daemon.environment.registered",fields:{environment_id:S.environmentId,created:S.created}})}),_.fresh(()=>{let o;try{o=xr()}catch{return}o!==null&&m({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:o.slice(0,2048)}})}),await _.fresh(async()=>{await Ir(),await Pr(),await Er(Date.now()),await Rr(Date.now())}),await _.fresh(async()=>{const{reconcile:o,migrated:p}=await nt(Br);o.changed&&m({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:o.cleared.length,same_boot_proven:o.sameBootProven}}),p.length>0&&m({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:p.length}})}),await _.fresh(async()=>{for(const o of await pe())o.status!=="active"||o.pi_pid!=null||o.frozen_at!=null||o.respawn_not_before!=null||(await me(o.node_id,new Date().toISOString()),m({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:o.node_id}))}),await _.fresh(async()=>{await Jr();for(const o of St){const p=o.apply();p.changed&&m({level:"info",event:"daemon.boot_migration.applied",fields:{migration:o.id,...p.summary}})}}),b(br()),R){const o=i(()=>{b(R().catch(()=>{})),b(Fe().then(p=>A?.observe(p)).then(()=>{},p=>{m({level:"warn",event:"daemon.peer.status_failed",error:p})}))},"refresh");b(Sr().catch(()=>{})),b(_r().then(({completion:p})=>p).catch(p=>{m({level:"error",event:"daemon.grants.sweep_failed",error:p})})),o(),$=setInterval(o,yr*1e3),$.unref(),z=Ne({joined:!0,register:b})}await W(async()=>{await gt(),await vt()}),N=setInterval(()=>{ae().catch(o=>m({level:"error",event:"daemon.tick.failed",error:o}))},u),await ae()}i($e,"completeStartup");function U(){return Re??=$e().catch(r=>{throw g={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(U,"resumeStartup");try{process.on("SIGINT",ne),process.on("SIGTERM",oe),process.on("exit",ie),_t(),v!==null&&m({level:"info",event:"daemon.person.resolved",fields:{person_id:v.id,source:v.source}}),f!==null&&m({level:"info",event:"daemon.permissions.boot",fields:{...f}}),h!==null&&m({level:"info",event:"daemon.person.retired",fields:{person_id:h.person,joined_from:h.joinedFrom,retired:h.retired,columns:h.columns}}),O=await Ue();const r=t.token;Zr({epoch:r,requestHandover:De,registerDetached:b,startupBlock:Ie,admitStartup:Ce,repairMigrations:Le}),be(),G=ar({tcp:n.tcp}),await G.ready;const e=We()??fe("user").document_port,S=sr(e,cr());try{await S.ready,q=S}catch(p){m({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:p instanceof Error?p.message:String(p)}})}const o=fe("user").runtime;o?.issuer&&(R=gr(o.issuer,b),o.unsandboxed===!0&&(A=Ge({syncGrants:R})),hr(p=>vr().appended(p))),o?.unsandboxed===!0&&o.listener!==void 0&&m({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),o!==void 0&&o.listener!==void 0&&o.unsandboxed!==!0&&(L=await dr(o,async()=>await kr(o.issuer.replace(/\/$/,"")),R),await L.ready);try{await bt()}catch(p){if(T=null,p instanceof kt)return g={phase:"blocked",message:p.message,blockerPaths:p.blockerPaths},I;throw p}process.platform==="linux"&&k(x)&&(Xe(),await Ye("app:terminal"));try{n.prepare?g={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await U(),g={phase:"ready",message:"",blockerPaths:[]})}finally{T=null}return I}catch(r){try{await I()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}i(Jt,"runDaemon");var To=Jt;export{Q as brokerCommandDeclaresCanvasHome,Y as brokerCommandEpoch,To as default,Jt as runDaemon,Ut as superviseTick,Lt as teardownFleet};

package/dist/daemon/fleet.js

@@ -1,4 +1,4 @@
-var g=Object.defineProperty;var l=(i,e)=>g(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as x,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as E}from"../core/runtime/close.js";import{readFault as P,readInterruptedTurn as R,readTransportPoisonVerdict as N}from"../core/canvas/recovery.js";import{hasPendingWake as C}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as F,MAX_RESPAWN_ATTEMPTS as S,OFFLINE_RESPAWN_RETRY_MS as M,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as c}from"../core/events/emit.js";import{brokerThresholdsForDaemon as D}from"../core/runtime/fleet.js";import{isRecoveryHeld as L}from"../core/runtime/recovery-hold.js";async function _(i,e){const t=await P(i.node_id),r=t===null?"":`The broker reported a ${t.kind} fault on ${t.link} (${t.op}) before it exited: ${t.message}
+var g=Object.defineProperty;var l=(i,e)=>g(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as k}from"../core/canvas/index.js";import{revokeExitedNodeToken as $}from"../core/canvas/node-tokens.js";import{isProfilePaused as T}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as R,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as x}from"../core/runtime/close.js";import{readFault as E,readInterruptedTurn as P,readTransportPoisonVerdict as N}from"../core/canvas/recovery.js";import{hasPendingWake as S}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as C,MAX_RESPAWN_ATTEMPTS as F,OFFLINE_RESPAWN_RETRY_MS as M,recordRespawnPolicy as O,respawnBackoffMs as z}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as A}from"../core/runtime/connectivity.js";import{reviveNode as B}from"../core/runtime/revive.js";import{isStartRefusal as D}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as L}from"../core/runtime/fleet.js";import{isRecoveryHeld as W}from"../core/runtime/recovery-hold.js";async function _(i,e){const t=await E(i.node_id),r=t===null?"":`The broker reported a ${t.kind} fault on ${t.link} (${t.op}) before it exited: ${t.message}
 
 `,s=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${i.name}\` (${i.kind}) never started.
 
@@ -8,14 +8,14 @@ Its pi vehicle exited before the session came up (no pi_session_id was ever reco
 
 If the work still needs doing, re-spawn it.`}l(_,"bootFailureBody");async function y(i,e){await m(i.node_id,await _(i,e),{from:i.node_id})}l(y,"surfaceBootFailure");function b(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
 
-Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function W(i){const e=await N(i.node_id),t=e===null?"":`
+Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(b,"outOfMemoryBody");async function q(i){const e=await N(i.node_id),t=e===null?"":`
 
 Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,r=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
 
 Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+t+`
 
-Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(W,"surfaceCrashLoop");async function q(i,e){const t=e instanceof Error?e.message:String(e),r=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
+Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(q,"surfaceCrashLoop");async function G(i,e){const t=e instanceof Error?e.message:String(e),r=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
 
 The daemon refused this recovery before starting a broker: ${t}
 
-The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(q,"surfaceRecoveryFailure");function G(i){return i?.cycle_pending!==!0}l(G,"retryResumeMode");function H(i,e,t=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":x(i)?"dormant":i.lifecycle==="resident"&&!e&&!t&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?G(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(H,"classifyDeadNode");const Y=3e4;class le{static{l(this,"DaemonFleet")}#o;#l;#i;#c;#e=new Map;#t=new Set;#n=new Set;#r=new Map;#u=Number.NEGATIVE_INFINITY;constructor(e){this.#o=e.epoch,this.#l=e.enqueue,this.#i=e.now??Date.now,this.#c=e.probeOnline??(()=>A())}epoch(){return this.#o}register(e,t){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(t.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#r.delete(e),this.#e.set(e,{pid:t.pid,handle:t})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#t.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#t.size}reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:t}=D();if(this.occupancy()<t)return this.#t.add(e),!0;const r=this.#i();return r-this.#u>=Y&&(this.#u=r,c({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:t}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,t=this.#e.get(e)?.pid){if(t!==void 0){this.#n.add(e);try{process.kill(t,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#n.has(e)}forget(e){this.#e.delete(e),this.#t.delete(e),this.#r.delete(e)}#s=new Set;async exitsSettled(){for(;this.#s.size>0;)await Promise.allSettled([...this.#s])}async onChildExit(e,t,r){const s=this.#f(e,t,r);this.#s.add(s);try{await s}finally{this.#s.delete(s)}}async#f(e,t,r){r&&await $(r),this.#e.delete(e),this.#l(async()=>{const s=await w(e),a=s?.launched_at==null?Number.NaN:Date.parse(s.launched_at),n=Number.isFinite(a)?Math.max(0,this.#i()-a):null,d=s?.respawn_failures??0;s?.status==="done"||s?.status==="canceled"||s?.status==="dead"?this.#r.delete(e):this.#r.set(e,t),this.#h(e,t,n,d)})}async deliverExit(e,t){await this.onChildExit(e,t)}async applyDeadRowPolicy(e){const t=this.#r.get(e);try{await this.#d(e,t)}finally{this.#r.get(e)===t&&this.#r.delete(e)}}async terminalizeIfStillNonTerminal(e,t,r,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#a(e,n,{event:t,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>q(n,r),"surface")}),!0)}dispose(){}#h(e,t,r,s){c({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:t.code,signal:t.signal,uptime_ms:r,consecutive_failures:s}})}async#d(e,t){return this.#p(e,t)}async#p(e,t){const r=await w(e);if(r!==null&&T(r.profile_id))return await h(e),"forget";if(this.#n.delete(e))return await h(e),"forget";if(L(e))return"dormant";const s=(await R(e))?.interrupted===!0;if(t?.oomKilled===!0&&r!==null&&r.status!=="done"&&r.status!=="canceled"&&r.status!=="dead")return await h(e),await this.#a(e,r,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(r)},wake:`Child out of memory \u2014 ${p(r)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(r),{from:e})},"surface")}),"forget";const a=H(r,s,await C(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),c({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#a(e,r,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await _(r,t)},wake:`Child crashed \u2014 ${p(r)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(r,t),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#m(e,r,a,s);break}return a}async#m(e,t,r,s){const a=await k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#c()){await F(e,n,this.#i()),c({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,retry_in_ms:M,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await O(a,this.#i()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=S};if(f.exhausted){const u=`Child crashed \u2014 ${p(t)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#a(e,t,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:u,respawn_failures:f.failures},wake:u,...t.pi_session_id!=null?{surface:l(()=>W(t),"surface")}:{}});return}if(d&&f.notBefore!==null){c({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}c({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:r==="respawn-resume",wakeReason:r==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(u){c({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:u,fields:{...o(e)?{}:{affected_node_id:e},action:r}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",u,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:u instanceof Error?u.message:String(u)})}}async#a(e,t,r){c({level:"error",event:r.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:r.reason,outcome:{detail:r.detail}});try{await(r.surface??(()=>y(t)))()}catch(s){c({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await E(e,"notice.crash",r.wake,{reason:"child-crashed",child:e})}}export{le as DaemonFleet,H as classifyDeadNode,G as retryResumeMode,y as surfaceBootFailure};
+The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,r,{from:i.node_id})}l(G,"surfaceRecoveryFailure");function H(i){return i?.cycle_pending!==!0}l(H,"retryResumeMode");function Y(i,e,t=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":R(i)?"dormant":i.lifecycle==="resident"&&!e&&!t&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?H(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(Y,"classifyDeadNode");const V=3e4;class ue{static{l(this,"DaemonFleet")}#o;#l;#i;#c;#e=new Map;#t=new Set;#n=new Set;#r=new Map;#u=Number.NEGATIVE_INFINITY;constructor(e){this.#o=e.epoch,this.#l=e.enqueue,this.#i=e.now??Date.now,this.#c=e.probeOnline??(()=>A())}epoch(){return this.#o}register(e,t){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(t.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#r.delete(e),this.#e.set(e,{pid:t.pid,handle:t})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#t.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#t.size}reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:t}=L();if(this.occupancy()<t)return this.#t.add(e),!0;const r=this.#i();return r-this.#u>=V&&(this.#u=r,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:t}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,t=this.#e.get(e)?.pid){if(t!==void 0){this.#n.add(e);try{process.kill(t,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#n.has(e)}forget(e){this.#e.delete(e),this.#t.delete(e),this.#r.delete(e)}#s=new Set;async exitsSettled(){for(;this.#s.size>0;)await Promise.allSettled([...this.#s])}async onChildExit(e,t,r){const s=this.#f(e,t,r);this.#s.add(s);try{await s}finally{this.#s.delete(s)}}async#f(e,t,r){r&&await $(r),this.#e.delete(e),this.#l(async()=>{const s=await w(e),a=s?.launched_at==null?Number.NaN:Date.parse(s.launched_at),n=Number.isFinite(a)?Math.max(0,this.#i()-a):null,d=s?.respawn_failures??0;s?.status==="done"||s?.status==="canceled"||s?.status==="dead"?this.#r.delete(e):this.#r.set(e,t),this.#h(e,t,n,d)})}async deliverExit(e,t){await this.onChildExit(e,t)}async applyDeadRowPolicy(e){const t=this.#r.get(e);try{await this.#d(e,t)}finally{this.#r.get(e)===t&&this.#r.delete(e)}}async terminalizeIfStillNonTerminal(e,t,r,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#a(e,n,{event:t,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>G(n,r),"surface")}),!0)}dispose(){}#h(e,t,r,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:t.code,signal:t.signal,uptime_ms:r,consecutive_failures:s}})}async#d(e,t){return this.#p(e,t)}async#p(e,t){const r=await w(e);if(r!==null&&T(r.profile_id))return await h(e),"forget";if(this.#n.delete(e))return await h(e),"forget";if(W(e))return"dormant";const s=(await P(e))?.interrupted===!0;if(t?.oomKilled===!0&&r!==null&&r.status!=="done"&&r.status!=="canceled"&&r.status!=="dead")return await h(e),await this.#a(e,r,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:b(r)},wake:`Child out of memory \u2014 ${p(r)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,b(r),{from:e})},"surface")}),"forget";const a=Y(r,s,await S(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#a(e,r,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await _(r,t)},wake:`Child crashed \u2014 ${p(r)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(r,t),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#m(e,r,a,s);break}return a}async#m(e,t,r,s){const a=await k(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#c()){await C(e,n,this.#i()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,retry_in_ms:M,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await O(a,this.#i()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=F};if(f.exhausted){const c=`Child crashed \u2014 ${p(t)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#a(e,t,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...t.pi_session_id!=null?{surface:l(()=>q(t),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,backoff_ms:z(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:r,consecutive_failures:f.failures}});try{await B(e,{capacity:"freeze",resume:r==="respawn-resume",wakeReason:r==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(D(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:r}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#a(e,t,r){u({level:"error",event:r.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:r.reason,outcome:{detail:r.detail}});try{await(r.surface??(()=>y(t)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await x(e,"notice.crash",r.wake,{reason:"child-crashed",child:e})}}export{ue as DaemonFleet,Y as classifyDeadNode,H as retryResumeMode,y as surfaceBootFailure};

package/dist/daemon/reconcilers/action-lane.js

@@ -1 +1 @@
-var p=Object.defineProperty;var o=(n,t)=>p(n,"name",{value:t,configurable:!0});import{rmSync as h}from"node:fs";import{claimDaemonAction as g,dueDaemonActions as w,settleDaemonAction as y}from"../../core/canvas/daemon-actions.js";import{getNode as _}from"../../core/canvas/canvas.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{createPageRequest as I,getRequest as f}from"../../core/human/requests.js";import{synthesizedPageDocument as v,writeSynthesizedPage as k}from"../../core/human/page-synth.js";import{preparePage as $}from"../../core/human/page-authoring.js";import{resolvePageComponents as b}from"../../core/config.js";import{spawnChild as q}from"../../core/runtime/spawn.js";import{runDelivery as D}from"../delivery/run-delivery.js";import{deliveryDaemonInstanceId as m}from"./delivery-lane.js";import{cronBirthPlacement as P}from"../cron/armed-context.js";import{checkCronBirth as N}from"../cron/authority.js";async function s(n,t,e){await y(n.actionId,n.attempts,t,e,Date.now())}o(s,"settle");async function u(n,t,e){await s(n,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:e}})}o(u,"failed");async function S(n,t,e){const i=await _(e.nodeId);if(i!==null){if(i.frozen_at!==null||i.pi_session_id!==null||i.status==="idle"||i.status==="done"||i.status==="canceled"){await s(n,t,{state:"acknowledged"});return}await u(n,t,i.status==="dead"?`cron spawn ${e.nodeId} recorded a failed birth`:`cron spawn ${e.nodeId} found an incomplete birth`);return}const r=e.principal;if(r===void 0){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:e.nodeId,fields:{cron_id:e.cronId??null,action_id:n.actionId,reason:"grant_missing",stage:"delivery"}})),await s(n,t,{state:"permanent_failed",failure:{kind:"grant_refused",message:"grant_missing: the queued birth carries no cron principal"}});return}const a=await N(r);if(!a.allowed){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:e.nodeId,fields:{cron_id:r.cron_id,action_id:n.actionId,grant_id:r.grant_id,reason:a.reason,stage:"delivery"}})),await s(n,t,{state:"permanent_failed",failure:{kind:"grant_refused",message:`${a.reason}: ${a.message}`}});return}try{await q({nodeId:e.nodeId,kind:e.kind,cwd:e.cwd,prompt:e.prompt,...await P(r,e.nodeId),readiness:"launch",profile:e.profile,isolated:e.isolated===!0,wakeOrigin:e.wakeOrigin}),await s(n,t,{state:"acknowledged"})}catch(d){await u(n,t,`cron spawn ${e.nodeId} failed: ${d instanceof Error?d.message:String(d)}`),c.fresh(()=>l({level:"error",event:"delivery.cron_spawn_failed",node_id:e.nodeId,error:d,fields:{action_id:n.actionId}}))}}o(S,"deliverSpawn");function z(n,t){const e=k(v(n.title,n.summary,t,!1));try{return $({sourceFile:e.path,source:{sessionName:n.sessionName,...n.profileName===null?{}:{profileName:n.profileName}},productKinds:b(),delivery:{placement:"inline",inbox:!0,reply:!1}})}finally{h(e.dir,{recursive:!0,force:!0})}}o(z,"preparedCronPage");async function A(n,t,e){if(await f(e.requestId)!==null){await s(n,t,{state:"acknowledged"});return}const i=n.body,r=typeof i?.stdout=="string"?i.stdout:null;if(r===null){await u(n,t,`cron human ${e.requestId} has no frozen stdout`);return}try{await I({request_id:e.requestId,prepared:z(e,r)}),await s(n,t,{state:"acknowledged"})}catch(a){await u(n,t,`cron human ${e.requestId} failed: ${a instanceof Error?a.message:String(a)}`)}}o(A,"deliverHuman");async function C(n,t){const e=n.adapter;if(n.body===null){await u(n,t,`daemon action ${n.actionId} has no body`);return}if(e.adapter==="subprocess"){let i=e.node_id;if(i===void 0){const r=n.idempotencyKey;if(!r.startsWith("human-action:"))throw new Error(`daemon action ${n.actionId} has no originating app`);const a=await f(r.slice(13));if(!a)throw new Error(`daemon action ${n.actionId} has no originating human request`);i=a.requester_node_id}await D({argv:e.argv,cwd:e.cwd,document:n.body,attempt:n.attempts,nodeId:i,...e.app?{app:e.app}:{}},async r=>await s(n,t,r));return}if(e.adapter==="cron_spawn"){await S(n,t,e);return}if(e.adapter==="cron_human"){await A(n,t,e);return}await u(n,t,`daemon action ${n.actionId} has unknown adapter ${String(e.adapter)}`)}o(C,"deliverAction");class T{static{o(this,"ActionLaneReconciler")}async run(t,e){if(!e.lifecycle.acceptsDetachedWork())return;let i;try{i=await w(t)}catch(r){c.fresh(()=>l({level:"error",event:"daemon_action.scan_failed",error:r}));return}for(const r of i)try{const a=await g(r.actionId,t,m);if(a===null)continue;e.lifecycle.registerDetached(C(a,m).catch(d=>{c.fresh(()=>l({level:"error",event:"daemon_action.failed",error:d,fields:{action_id:a.actionId}}))}))}catch(a){c.fresh(()=>l({level:"error",event:"daemon_action.claim_failed",error:a,fields:{action_id:r.actionId}}))}}}export{T as ActionLaneReconciler};
+var p=Object.defineProperty;var d=(e,t)=>p(e,"name",{value:t,configurable:!0});import{rmSync as g}from"node:fs";import{claimDaemonAction as h,dueDaemonActions as w,settleDaemonAction as _}from"../../core/canvas/daemon-actions.js";import{getNode as y}from"../../core/canvas/canvas.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{createPageRequest as I,getRequest as u}from"../../core/human/requests.js";import{synthesizedPageDocument as v,writeSynthesizedPage as k}from"../../core/human/page-synth.js";import{preparePage as $}from"../../core/human/page-authoring.js";import{resolvePageComponents as b}from"../../core/config.js";import{spawnChild as q}from"../../core/runtime/spawn.js";import{isStartRefusal as D}from"../../core/runtime/start-check.js";import{runDelivery as P}from"../delivery/run-delivery.js";import{deliveryDaemonInstanceId as m}from"./delivery-lane.js";import{cronBirthPlacement as S}from"../cron/armed-context.js";import{checkCronBirth as N}from"../cron/authority.js";async function o(e,t,n){await _(e.actionId,e.attempts,t,n,Date.now())}d(o,"settle");async function f(e,t,n){await o(e,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:n}})}d(f,"failed");async function z(e,t,n){const i=await y(n.nodeId);if(i!==null){if(i.frozen_at!==null||i.pi_session_id!==null||i.status==="idle"||i.status==="done"||i.status==="canceled"){await o(e,t,{state:"acknowledged"});return}await f(e,t,i.status==="dead"?`cron spawn ${n.nodeId} recorded a failed birth`:`cron spawn ${n.nodeId} found an incomplete birth`);return}const r=n.principal;if(r===void 0){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:n.nodeId,fields:{cron_id:n.cronId??null,action_id:e.actionId,reason:"grant_missing",stage:"delivery"}})),await o(e,t,{state:"permanent_failed",failure:{kind:"grant_refused",message:"grant_missing: the queued birth carries no cron principal"}});return}const a=await N(r);if(!a.allowed){c.fresh(()=>l({level:"warn",event:"cron.birth.refused",node_id:n.nodeId,fields:{cron_id:r.cron_id,action_id:e.actionId,grant_id:r.grant_id,reason:a.reason,stage:"delivery"}})),await o(e,t,{state:"permanent_failed",failure:{kind:"grant_refused",message:`${a.reason}: ${a.message}`}});return}try{await q({nodeId:n.nodeId,kind:n.kind,cwd:n.cwd,prompt:n.prompt,...await S(r,n.nodeId),readiness:"launch",profile:n.profile,isolated:n.isolated===!0,wakeOrigin:n.wakeOrigin}),await o(e,t,{state:"acknowledged"})}catch(s){if(D(s)){await o(e,t,{state:"permanent_failed",failure:{kind:"grant_refused",message:s.message}});return}await f(e,t,`cron spawn ${n.nodeId} failed: ${s instanceof Error?s.message:String(s)}`),c.fresh(()=>l({level:"error",event:"delivery.cron_spawn_failed",node_id:n.nodeId,error:s,fields:{action_id:e.actionId}}))}}d(z,"deliverSpawn");function A(e,t){const n=k(v(e.title,e.summary,t,!1));try{return $({sourceFile:n.path,source:{sessionName:e.sessionName,...e.profileName===null?{}:{profileName:e.profileName}},productKinds:b(),delivery:{placement:"inline",inbox:!0,reply:!1}})}finally{g(n.dir,{recursive:!0,force:!0})}}d(A,"preparedCronPage");async function C(e,t,n){if(await u(n.requestId)!==null){await o(e,t,{state:"acknowledged"});return}const i=e.body,r=typeof i?.stdout=="string"?i.stdout:null;if(r===null){await f(e,t,`cron human ${n.requestId} has no frozen stdout`);return}try{await I({request_id:n.requestId,prepared:A(n,r)}),await o(e,t,{state:"acknowledged"})}catch(a){await f(e,t,`cron human ${n.requestId} failed: ${a instanceof Error?a.message:String(a)}`)}}d(C,"deliverHuman");async function E(e,t){const n=e.adapter;if(e.body===null){await f(e,t,`daemon action ${e.actionId} has no body`);return}if(n.adapter==="subprocess"){let i=n.node_id;if(i===void 0){const r=e.idempotencyKey;if(!r.startsWith("human-action:"))throw new Error(`daemon action ${e.actionId} has no originating app`);const a=await u(r.slice(13));if(!a)throw new Error(`daemon action ${e.actionId} has no originating human request`);i=a.requester_node_id}await P({argv:n.argv,cwd:n.cwd,document:e.body,attempt:e.attempts,nodeId:i,...n.app?{app:n.app}:{}},async r=>await o(e,t,r));return}if(n.adapter==="cron_spawn"){await z(e,t,n);return}if(n.adapter==="cron_human"){await C(e,t,n);return}await f(e,t,`daemon action ${e.actionId} has unknown adapter ${String(n.adapter)}`)}d(E,"deliverAction");class V{static{d(this,"ActionLaneReconciler")}async run(t,n){if(!n.lifecycle.acceptsDetachedWork())return;let i;try{i=await w(t)}catch(r){c.fresh(()=>l({level:"error",event:"daemon_action.scan_failed",error:r}));return}for(const r of i)try{const a=await h(r.actionId,t,m);if(a===null)continue;n.lifecycle.registerDetached(E(a,m).catch(s=>{c.fresh(()=>l({level:"error",event:"daemon_action.failed",error:s,fields:{action_id:a.actionId}}))}))}catch(a){c.fresh(()=>l({level:"error",event:"daemon_action.claim_failed",error:a,fields:{action_id:r.actionId}}))}}}export{V as ActionLaneReconciler};

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts

@@ -47,6 +47,8 @@ export declare class FreezeLaneReconciler {
      *  transition), then the one launcher. A capacity freeze leaves the row `active` with its frozen mark,
      *  and a lost launch leaves it a stranded launch the tick relaunches, so the mail is never dropped. */
     private enactReopen;
+    /** A start refused by a limit leaves the node dormant (released, no freeze mark, nothing launched) and records the limit. */
+    private settleRefusedStart;
     private enactThaw;
     private enactIdleRelease;
 }

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js

@@ -1 +1 @@
-var E=Object.defineProperty;var d=(o,t)=>E(o,"name",{value:t,configurable:!0});import{clearFrozen as _,getNode as h,getRow as y}from"../../../core/canvas/index.js";import{fullName as g}from"../../../core/canvas/labels.js";import{isSafeNodeId as s}from"../../../core/canvas/paths.js";import{emitEvent as l}from"../../../core/events/emit.js";import{operationIdContext as z}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as R}from"../../../core/canvas/deliveries.js";import{publishNotice as A}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{commitReopenResident as S}from"../../../core/runtime/reopen.js";import{reviveNode as f}from"../../../core/runtime/revive.js";import{hasPendingWake as w}from"../live-obligation.js";import{readInterruptedTurn as k}from"../../../core/canvas/recovery.js";import{mapAsync as L}from"../../../core/async-array.js";const m=6,M=10*6e4,C=2e4;function I(o,t){if(t)return 2;const a=o.launched_at;if(a==null)return 1;const e=o.last_settled_at;return e==null?2:Date.parse(e)>Date.parse(a)?1:2}d(I,"thawTier");function N(o,t){const a=o.row.frozen_at,e=t.row.frozen_at;return a==null?e==null?0:1:e==null?-1:Date.parse(a)-Date.parse(e)}d(N,"compareThaw");class j{static{d(this,"FreezeLaneReconciler")}relaunchState=new Map;endEpisode(t){this.relaunchState.delete(t)}async run(t,a){const e=await L(a.candidates,async r=>({...r,tier:I(r.row,await w(r.row.node_id))}));e.sort(N);const n={stalled:0,waitingOnMail:0};for(const r of e)await z.fresh(()=>this.enact(t,r,a.fleet))==="frozen"&&(r.tier===1?n.stalled+=1:n.waitingOnMail+=1);return n}async enact(t,a,e){const n=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(t,a.row):(await e.applyDeadRowPolicy(n),e.has(n)?"launched":(await y(n))?.frozen_at!=null?"frozen":"skipped")}catch(r){return l({level:"error",event:"node.lifecycle.enact_failed",...s(n)?{node_id:n}:{},error:r,fields:{...s(n)?{}:{affected_node_id:n},enact:a.enact}}),"skipped"}}async enactReopen(t){return await S(t.node_id),(await f(t.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}async enactThaw(t){const a=(await k(t.node_id))?.interrupted===!0;if(!a&&t.lifecycle==="resident"&&(await h(t.node_id))?.pi_session_id!=null&&!await w(t.node_id))return await u(t.node_id,"release"),await _(t.node_id),"skipped";try{return(await f(t.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(e){const n=await h(t.node_id);throw n!==null&&(n.status==="active"||n.status==="idle")&&(await u(t.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}),await _(t.node_id)),e}}async enactIdleRelease(t,a){const e=a.node_id,n=await R(e),r=this.relaunchState.get(e),c=r===void 0||r.acknowledgedCount!==n,i=c?0:r.attempts;if(i>=m)return this.relaunchState.delete(e),l({level:"warn",event:"broker.idle_release.exhausted",...s(e)?{node_id:e}:{},fields:{...s(e)?{}:{affected_node_id:e},attempts:i,max_attempts:m}}),await u(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:i}}}),await A(e,"notice.crash",`Child crashed \u2014 ${g(a)} (${e}) failed to relaunch after ${i} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,{reason:"child-crashed",child:e}),"skipped";const p=c?void 0:r?.lastAttempt,v=Math.min(C*2**i,M);return p!==void 0&&t-p<v?"skipped":(await f(e,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":(this.relaunchState.set(e,{acknowledgedCount:n,attempts:i+1,lastAttempt:t}),l({level:"info",event:"broker.revive.idle_release",...s(e)?{node_id:e}:{},attempt:i+1,fields:{...s(e)?{}:{affected_node_id:e},attempts:i+1,max_attempts:m}}),"launched")}}export{j as FreezeLaneReconciler,N as compareThaw,I as thawTier};
+var E=Object.defineProperty;var d=(o,t)=>E(o,"name",{value:t,configurable:!0});import{clearFrozen as f,getNode as h,getRow as R}from"../../../core/canvas/index.js";import{fullName as S}from"../../../core/canvas/labels.js";import{isSafeNodeId as s}from"../../../core/canvas/paths.js";import{emitEvent as c}from"../../../core/events/emit.js";import{operationIdContext as g}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as z}from"../../../core/canvas/deliveries.js";import{publishNotice as A}from"../../../core/runtime/close.js";import{transition as l}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as w}from"../../../core/runtime/start-check.js";import{commitReopenResident as k}from"../../../core/runtime/reopen.js";import{reviveNode as p}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{mapAsync as M}from"../../../core/async-array.js";const m=6,C=10*6e4,N=2e4;function D(o,t){if(t)return 2;const a=o.launched_at;if(a==null)return 1;const e=o.last_settled_at;return e==null?2:Date.parse(e)>Date.parse(a)?1:2}d(D,"thawTier");function T(o,t){const a=o.row.frozen_at,e=t.row.frozen_at;return a==null?e==null?0:1:e==null?-1:Date.parse(a)-Date.parse(e)}d(T,"compareThaw");class G{static{d(this,"FreezeLaneReconciler")}relaunchState=new Map;endEpisode(t){this.relaunchState.delete(t)}async run(t,a){const e=await M(a.candidates,async n=>({...n,tier:D(n.row,await v(n.row.node_id))}));e.sort(T);const r={stalled:0,waitingOnMail:0};for(const n of e)await g.fresh(()=>this.enact(t,n,a.fleet))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(t,a,e){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(t,a.row):(await e.applyDeadRowPolicy(r),e.has(r)?"launched":(await R(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return c({level:"error",event:"node.lifecycle.enact_failed",...s(r)?{node_id:r}:{},error:n,fields:{...s(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(t){await k(t.node_id);try{return(await p(t.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(w(a))return await this.settleRefusedStart(t.node_id,a,"reopen");throw a}}async settleRefusedStart(t,a,e){return await l(t,"release"),await f(t),c({level:"warn",event:"node.start.refused",node_id:t,error:a,fields:{enact:e}}),"skipped"}async enactThaw(t){const a=(await L(t.node_id))?.interrupted===!0;if(!a&&t.lifecycle==="resident"&&(await h(t.node_id))?.pi_session_id!=null&&!await v(t.node_id))return await l(t.node_id,"release"),await f(t.node_id),"skipped";try{return(await p(t.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(e){if(w(e))return await this.settleRefusedStart(t.node_id,e,"thaw");const r=await h(t.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await l(t.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}),await f(t.node_id)),e}}async enactIdleRelease(t,a){const e=a.node_id,r=await z(e),n=this.relaunchState.get(e),u=n===void 0||n.acknowledgedCount!==r,i=u?0:n.attempts;if(i>=m)return this.relaunchState.delete(e),c({level:"warn",event:"broker.idle_release.exhausted",...s(e)?{node_id:e}:{},fields:{...s(e)?{}:{affected_node_id:e},attempts:i,max_attempts:m}}),await l(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:i}}}),await A(e,"notice.crash",`Child crashed \u2014 ${S(a)} (${e}) failed to relaunch after ${i} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,{reason:"child-crashed",child:e}),"skipped";const _=u?void 0:n?.lastAttempt,y=Math.min(N*2**i,C);return _!==void 0&&t-_<y?"skipped":(await p(e,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":(this.relaunchState.set(e,{acknowledgedCount:r,attempts:i+1,lastAttempt:t}),c({level:"info",event:"broker.revive.idle_release",...s(e)?{node_id:e}:{},attempt:i+1,fields:{...s(e)?{}:{affected_node_id:e},attempts:i+1,max_attempts:m}}),"launched")}}export{G as FreezeLaneReconciler,T as compareThaw,D as thawTier};

package/dist/types.d.ts

@@ -402,6 +402,10 @@ export interface ScopeConfig {
     paths?: {
         user_files?: string;
     };
+    /** The money ledger. `take_pct` (0-100, default 0) is Northlight's take on each priced model or tool charge; at 0 no take rows are written. */
+    ledger?: {
+        take_pct?: number;
+    };
     /** Conversation store configuration for the daemon. */
     store?: {
         root?: string;

package/docs/sdk/release-notes.md

@@ -7,6 +7,16 @@ description: Behavior changes in @north-light/sdk that an app must act on, newes
 
 Changes to `@north-light/sdk` that can change what an existing app sees. Newest first.
 
+## 0.3.552
+
+One ledger records every charge. Additive, except that the per-app cap now counts more.
+
+- **Charges.** `client.charges.list({ month?, under?, after?, limit? })` (`GET /v1/charges`) returns a page of the month's charge rows on this runtime, in time order; while `next` is not null, pass it back as `after` for the rest (`limit` is 1 to 10,000, default 1,000). Each row has: payer, funding, `caused_by`, `request` and the vendor's `vendor_ref`. `under` takes `run:<id>`, `message:<id>`, `request:<id>`, `cron_run:<id>` or a charge id, and lists the rows of every month unless `month` is also given; the answer carries a `month` only when you filtered. An app sees only the rows it is the grantee of. `usage.get` is deprecated in favour of it and keeps its shape.
+- **The per-app cap counts priced tool calls.** The cap a user sets on an app (`spend_cap` in `usage.get`) used to count only model calls on the person's subscription or key. It now counts every charge the app makes on the person, including priced tool calls, so an app can reach its cap sooner than before.
+- **`cap_exhausted` details.** The `402` carries `details.limit` (`id`, `payer`, `filters`, `usd_per_month`, `source`), `details.usd` and `details.reset_at`. `userAction` is `raise_cap` for an app's cap on the person, `clear_limit` for a limit the person set, and absent for an operator limit. `details.spend_cap` and `details.usd_equivalent` are still sent, with the same numbers as `details.limit.usd_per_month` and `details.usd`, but are deprecated: read the new keys. `RateLimitError.capExhausted` returns them typed (`CapExhaustedDetails`). A new run, node or job can now be refused with it too, when a limit covers every charge of its payer, such as a `$0` stop or the app's cap.
+- **Spend limits.** `client.spendLimits.list | set | remove` (`/v1/spend-limits`) reads and writes dollar limits, for the person's own client only; an app is refused `403 owner_only`, and `spendLimits` is not on `AppClient`. These are not `/v1/limits`, which stays the per-app memory and disk cap. `crtr sys spend show | chain | limit` is the command line for all of it.
+- **`Crtr-Caused-By`.** `runs.start({ causedBy })` and the client option `causedBy` send one reference (`message:<id>`, `request:<id>` or `cron_run:<id>`) that the runtime stores on the run's charges. A runtime older than this release ignores it.
+
 ## 0.3.550
 
 An app registers a permission schema for its own object types and decides who may read each object with relations. Additive, except that `friends` leaves the privacy levels and nodes move under their profile.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.551",
+  "version": "0.3.552",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -140,7 +140,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.551",
+    "@north-light/identity": "0.3.552",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.551",
+  "version": "0.3.552",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.551",
+  "version": "0.3.552",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -24,7 +24,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.551",
+        "@north-light/identity": "0.3.552",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12455,17 +12455,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12473,21 +12473,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.551",
-        "@north-light/identity": "0.3.551"
+        "@north-light/api": "0.3.552",
+        "@north-light/identity": "0.3.552"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.551",
+      "version": "0.3.552",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.551",
-        "@north-light/identity": "0.3.551",
-        "@north-light/plugin": "0.3.551",
+        "@north-light/api": "0.3.552",
+        "@north-light/identity": "0.3.552",
+        "@north-light/plugin": "0.3.552",
         "jose": "^6.2.1"
       },
       "bin": {

package/dist/core/usage/ledger.d.ts

@@ -1,49 +0,0 @@
-import { type CanvasDb } from '../canvas/db.js';
-import type { SpendUsageDTO } from '../../api/dto/usage.js';
-/** The person's own agent. Its grant carries the directory's default cap, but the
- *  cap bounds what an app spends of the person's subscription, never the person's
- *  own work, so `app:terminal` is never stopped. */
-export declare const UNCAPPED_GRANTEE = "app:terminal";
-export type Payer = 'user' | 'app';
-/** The UTC calendar month `YYYY-MM` of `at`. */
-export declare function usageMonth(at?: Date): string;
-/** 00:00Z on the first day of the UTC month after `month`. */
-export declare function resetAt(month: string): string;
-export interface UsageRecord {
-    grantee: string;
-    payer: Payer;
-    /** `model` for a completion, `provider:<id>` for a provider receipt. */
-    source: string;
-    /** The model that answered; `''` for a provider receipt. */
-    served_model: string;
-    input_tokens: number;
-    output_tokens: number;
-    cached_tokens: number;
-    /** List-price cost; null when the served model has no list price (it adds nothing to the total). */
-    usd_equivalent: number | null;
-}
-/** Add one completion's or receipt's usage to its grantee's month, inside the caller's write. */
-export declare function addUsage(db: CanvasDb, record: UsageRecord, at?: Date): Promise<void>;
-/** Record one model completion for `grantee`. */
-export declare function recordModelUsage(grantee: string, payer: Payer, servedModel: string, usage: {
-    input: number;
-    output: number;
-    cacheRead: number;
-}, usdEquivalent: number | null): Promise<void>;
-/** The grantee's user-paid month-to-date model spend; provider receipts are not in it. */
-export declare function monthSpend(grantee: string, month?: string): Promise<number>;
-/** Who a user-paid call bills, and the cap that bounds it (`null` = uncapped). */
-export interface SpendTarget {
-    grantee: string;
-    spend_cap: number | null;
-}
-/** The cap that applies to `target`: none for the person's own agent or a grant with no cap. */
-export declare function effectiveCap(target: SpendTarget): number | null;
-/** Refuse a user-paid call once the grantee's month spend reached its cap (SPD-2).
- *  The check runs before the call; one response may overshoot the cap (no
- *  reserve-then-settle), and the next call is refused. */
-export declare function assertUnderCap(target: SpendTarget | undefined, at?: Date): Promise<void>;
-/** `usage.get` for one grantee and month. */
-export declare function usageFor(grantee: string, spendCap: number | null, month?: string): Promise<SpendUsageDTO>;
-/** A `month` query value: `YYYY-MM` with a real month, years 1000-9999; absent means the current month. */
-export declare function parseUsageMonth(value: string | null): string;

package/dist/core/usage/ledger.js

@@ -1,10 +0,0 @@
-var f=Object.defineProperty;var a=(e,t)=>f(e,"name",{value:t,configurable:!0});import{withCanvasWrite as h,openDb as d}from"../canvas/db.js";import{RunError as _}from"../runs/operations.js";const q="app:terminal";function r(e=new Date){return e.toISOString().slice(0,7)}a(r,"usageMonth");function l(e){const[t,n]=e.split("-").map(Number);return new Date(Date.UTC(t,n,1)).toISOString()}a(l,"resetAt");async function w(e,t,n=new Date){const u=r(n);await e.prepare(`INSERT INTO usage (grantee, month, payer, source, served_model, input_tokens, output_tokens, cached_tokens, usd_equivalent)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(grantee, month, payer, source, served_model) DO UPDATE SET
-      input_tokens = usage.input_tokens + excluded.input_tokens,
-      output_tokens = usage.output_tokens + excluded.output_tokens,
-      cached_tokens = usage.cached_tokens + excluded.cached_tokens,
-      usd_equivalent = CASE WHEN usage.usd_equivalent IS NULL AND excluded.usd_equivalent IS NULL THEN NULL
-        ELSE COALESCE(usage.usd_equivalent, 0) + COALESCE(excluded.usd_equivalent, 0) END`).run(t.grantee,u,t.payer,t.source,t.served_model,t.input_tokens,t.output_tokens,t.cached_tokens,t.usd_equivalent),t.payer==="user"&&t.source==="model"&&t.usd_equivalent!==null&&t.usd_equivalent>0&&await e.prepare(`INSERT INTO grantee_usage (grantee, month, usd_equivalent) VALUES (?, ?, ?)
-      ON CONFLICT(grantee, month) DO UPDATE SET usd_equivalent = grantee_usage.usd_equivalent + excluded.usd_equivalent`).run(t.grantee,u,t.usd_equivalent)}a(w,"addUsage");async function y(e,t,n,u,s){await h(async p=>await w(p,{grantee:e,payer:t,source:"model",served_model:n,input_tokens:u.input,output_tokens:u.output,cached_tokens:u.cacheRead,usd_equivalent:s}))}a(y,"recordModelUsage");async function c(e,t=r()){return(await(await d()).prepare("SELECT usd_equivalent FROM grantee_usage WHERE grantee = ? AND month = ?").get(e,t))?.usd_equivalent??0}a(c,"monthSpend");function m(e){return e.grantee===q?null:e.spend_cap}a(m,"effectiveCap");async function x(e,t=new Date){if(!e)return;const n=m(e);if(n===null)return;const u=r(t),s=await c(e.grantee,u);if(!(s<n))throw new _(402,"cap_exhausted",`${e.grantee} reached its monthly usage limit`,void 0,{spend_cap:n,usd_equivalent:s,reset_at:l(u)})}a(x,"assertUnderCap");async function C(e,t,n=r()){const u=await(await d()).prepare(`SELECT payer, source, served_model, input_tokens, output_tokens, usd_equivalent
-    FROM usage WHERE grantee = ? AND month = ? ORDER BY served_model`).all(e,n),s=a(i=>u.filter(o=>o.payer===i&&o.source==="model").map(({served_model:o,input_tokens:E,output_tokens:v,usd_equivalent:g})=>({served_model:o,input_tokens:E,output_tokens:v,usd_equivalent:g})),"byModel"),p=u.filter(i=>i.payer==="app").reduce((i,o)=>i+(o.usd_equivalent??0),0);return{grantee:e,month:n,usd_equivalent:await c(e,n),by_model:s("user"),app_paid:{usd_equivalent:p,by_model:s("app")},spend_cap:m({grantee:e,spend_cap:t}),reset_at:l(n)}}a(C,"usageFor");function D(e){if(e===null)return r();if(!/^[1-9]\d{3}-(0[1-9]|1[0-2])$/.test(e))throw new _(400,"invalid_request","month must be YYYY-MM","month");return e}a(D,"parseUsageMonth");export{q as UNCAPPED_GRANTEE,w as addUsage,x as assertUnderCap,m as effectiveCap,c as monthSpend,D as parseUsageMonth,y as recordModelUsage,l as resetAt,C as usageFor,r as usageMonth};