@north-light/cli 0.3.566 → 0.3.567

package/dist/daemon/crtrd.js

@@ -1,18 +1,18 @@
-var er=Object.defineProperty;var i=(e,l)=>er(e,"name",{value:l,configurable:!0});import{applyBootSettings as rr}from"./boot-settings.js";import{startPeerOutbox as tr}from"./peer/outbox.js";import{reportMachineStatus as nr}from"../core/peer/directory.js";import{createPeerRelay as or}from"./peer/relay-service.js";import{writeFileSync as ir,readFileSync as ar,rmSync as sr,mkdirSync as ye,existsSync as _,unlinkSync as be,chmodSync as _e}from"node:fs";import{createServer as J}from"node:http";import{dirname as cr}from"node:path";import{envCrtrdTcp as ke,envDocumentPort as lr}from"../shared/env.js";import{serveCanvasConversationStore as dr}from"../core/canvas/sessions.js";import{pollRunObservers as pr}from"./runs/observer.js";import{randomBytes as Re}from"node:crypto";import{apiSocketPath as Pe,crtrHome as mr}from"../core/canvas/paths.js";import{activeAppUsers as fr}from"../core/spaces/users.js";import{ensureAppSpace as ur,ensureUserFiles as wr}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as L}from"../launcher/client.js";import{installLauncherEventSink as hr}from"./launcher-events.js";import{initializeEmptyLayout as vr,layoutNeedsMigration as gr,socketDir as Ee}from"../core/layout.js";import{migrateLooseKeys as Sr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as yr}from"../core/profiles/relocate.js";import{getNode as br,listNodesForSupervision as Ie,listParkedResidentsWithWakeMail as _r,setFrozen as xe}from"../core/canvas/index.js";import{createApiServer as kr,closeServer as Te}from"./api/server.js";import{Router as Rr}from"./api/router.js";import{healthRoutes as Pr}from"./api/handlers/health.js";import{createDocumentListener as Er}from"./api/document-listener.js";import{documentSecret as Ir}from"./api/document-secret.js";import{createAppListener as xr}from"./api/app-listener.js";import{registerMachineEnvironment as Tr}from"../core/environments.js";import{registerBlaxelRunnerFromEnv as Dr}from"../core/secrets/blaxel-register.js";import{markCloudRole as Cr,resolvePerson as Lr,retireLocalPerson as $r,setPersonId as Ar}from"../core/canvas/person.js";import{backfillNodeApps as Nr,runPermissionsBoot as Mr}from"../core/permissions/boot.js";import{startRunNotices as Gr}from"./run-notices.js";import{startGrantEffects as Or}from"./reconcilers/grant-effects.js";import{createGrantSync as Fr,GRANT_SYNC_INTERVAL_S as Br}from"../core/grants/sync.js";import{sweepSandboxes as jr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as zr}from"./cron/run-socket.js";import{hasSyncedGrants as Hr}from"../core/grants/mirror.js";import{readConfig as se}from"../core/config.js";import{reconcileUndeliveredTickets as qr}from"./human/sweep.js";import{resetHandedOffDeliveries as Ur}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Wr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Jr}from"./review/sweep.js";import{pidfilePath as ce}from"./pidfile.js";import{rendererWarning as Kr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Vr}from"./supervise-cadence.js";import{createResumeDetector as Qr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Yr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Xr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Zr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as et}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as rt}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as tt}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as nt}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as ot}from"./reconcilers/action-lane.js";import{endLiveExecution as it,migrateLegacyExecutionIdentities as at}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as st}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as ct}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as lt}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as dt}from"./reconcilers/storage-due-work.js";import{LedgerSandboxReconciler as pt}from"./reconcilers/ledger-sandbox.js";import{LedgerStorageReconciler as mt}from"./reconcilers/ledger-storage.js";import{LedgerR2Reconciler as ft}from"./reconcilers/ledger-r2.js";import{BashDeadlineReconciler as ut}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as wt}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as ht}from"./reconcilers/structured-request-lane.js";import{RecipeImageReconciler as vt,subscribeImageBuilt as gt}from"./reconcilers/recipe-image.js";import{reconcileStructuredRequestSidecars as St}from"./structured-requests/boot.js";import{captureLivenessSnapshot as De,capturePidCommand as yt,captureTeardownSnapshot as Ce,isPidAlive as bt,isAnyPidAlive as Le,killProcessTreePids as $e}from"../core/canvas/pid.js";import{bindFleet as _t,boundFleet as kt}from"../core/runtime/fleet.js";import{bindDaemonControl as Rt,clearDaemonControl as Pt}from"./control.js";import{execDaemon as Et,spawnDaemon as It}from"./manage.js";import{reconcileAndMigrateBoot as xt}from"../core/canvas/boot.js";import{resolveInstallId as Tt}from"../core/canvas/install-id.js";import{setInstallId as Dt}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Ct,daemonOwnershipApplies as Lt,daemonOwnershipClaimed as $t,releaseDaemonOwnership as Ae}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as At,closeAllDbs as le,openDb as Ne,withCanvasWrite as Nt}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as Mt}from"../core/canvas/import-sqlite.js";import{ensurePostgres as Gt,ensurePostgresHealthy as Ot,stopPostgres as Ft,withPostgresStartLock as Bt}from"./postgres/index.js";import{brokerSignalBus as F}from"./broker-signals/bus.js";import{brokerSignalStreams as jt}from"./api/handlers/broker-signals.js";import{setRequestsStartRunObservers as zt}from"./api/handlers/broker-ops.js";import{reconcilePackages as Ht}from"../core/graph/package-reconcile.js";import{startRepoSync as qt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Ut,reapStaleSpares as Wt}from"../core/runtime/warm-pool.js";import{DaemonFleet as Jt}from"./fleet.js";import{copyNodeFilesToRows as Kt}from"../core/runtime/first-boot-rows.js";import{BOOT_MIGRATIONS as Vt}from"./boot-migrations.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as b}from"../core/events/operation-id.js";import{bindDaemonEventSource as Qt}from"../core/events/source.js";import{activateCanvasImport as Yt,MigrationBlockedError as Xt,runForcedCanvasImport as Zt}from"../migrations/activation.js";import{adoptInstallPerson as en,convertLegacyScopedStore as rn,personStoresCopied as tn}from"../core/scoped-state/migrate.js";import{readLegacyUserSnapshot as nn}from"../core/scoped-state/legacy-profiles.js";import{heldPersons as Me,installPreludeSnapshot as on}from"../core/scoped-state/person-store.js";import{loadPersonSettings as Ge}from"../core/scoped-state/person-rows.js";import{userScopeRoot as Oe}from"../core/scope.js";function de(e,l){return e.includes(` --canvas-home ${l} `)}i(de,"brokerCommandDeclaresCanvasHome");function pe(e){return/ --epoch (\S+) /.exec(e)?.[1]??null}i(pe,"brokerCommandEpoch");const an=5e3,Fe=150;async function sn(e){const l=De();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const f=mr(),m=[];for(const[n,d]of l.processes){const y=d.command;if(d.zombie||!y.includes("broker-cli.js")||!de(y,f)||pe(y)===e)continue;const k=y.trim().split(/\s+/).at(-1)??"";k===""||k.endsWith("broker-cli.js")||m.push({pid:n,nodeId:k})}let v=0;const a=[];for(const n of m){const d=yt(n.pid);if(d===null||!d.includes("broker-cli.js")||!de(d,f)||pe(d)===e||d.trim().split(/\s+/).at(-1)!==n.nodeId)continue;const y=Ce(n.pid,null);if(!y.reused){try{process.platform==="linux"&&_(x)?await L({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{continue}v++,a.push({pid:n.pid,snap:y})}}const w=Date.now()+an;for(;Date.now()<w&&a.some(n=>Le(n.snap.tree.length>0?n.snap.tree:[n.pid]));)await new Promise(n=>setTimeout(n,Fe));let g=0;for(const n of a){const d=n.snap.tree.length>0?n.snap.tree:[n.pid];Le(d)&&(process.platform==="linux"&&_(x)?await L({operation:"stop",pid:n.pid}):$e(d,"SIGKILL",n.snap.identities),g++)}g>0&&await new Promise(n=>setTimeout(n,Fe));const p=De();if(p===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[n,d]of p.processes){const y=d.command;d.zombie||!y.includes("broker-cli.js")||de(y,f)&&pe(y)!==e&&c.push(n)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:v,sigkilled:g}})}i(sn,"reapPriorEpochBrokers");const cn=5e3;async function ln(e){const l=Array.from(e.entries());if(l.length===0)return;for(const[a]of l){const w=(await br(a))?.status;(w==="active"||w==="idle")&&await xe(a,new Date().toISOString())}const f=l.map(([,a])=>({pid:a.pid,snap:Ce(a.pid,null),exited:a.handle.exited}));for(const a of f)if(!a.snap.reused)try{process.platform==="linux"&&_(x)?await L({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{}const m=new Promise(a=>{setTimeout(a,cn).unref()});await Promise.allSettled(f.map(a=>Promise.race([a.exited.then(()=>{}),m])));const v=[];for(const a of f)bt(a.pid)&&(process.platform==="linux"&&_(x)?await L({operation:"stop",pid:a.pid}):$e(a.snap.tree.length>0?a.snap.tree:[a.pid],"SIGKILL",a.snap.identities),v.push(a));await Promise.all(v.map(a=>a.exited));for(const[a]of l)await it(a,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:f.length,sigkilled:v.length}})}i(ln,"teardownFleet");function Be(){ye(cr(ce()),{recursive:!0}),ir(ce(),String(process.pid),"utf8")}i(Be,"writePidfile");function je(e){return typeof e=="object"&&e!==null&&e.code==="ENOENT"}i(je,"isMissingPathError");function ze(e){const l=ce();let f;try{f=ar(l,"utf8")}catch(m){if(je(m))return;throw m}if(f.trim()!==String(e))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${e})`);try{sr(l)}catch(m){if(je(m))return;throw m}}i(ze,"removeOwnedPidfileMirror");const dn={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(e=>{e.catch(()=>{})},"registerDetached")},K=new Xr;Yr(e=>K.handleFatalFault(e,!1));const pn=new Zr,mn=new et,fn=new rt,un=new tt,wn=new nt,hn=new ot,vn=new ct,gn=new lt,Sn=new dt,yn=new pt,bn=new mt,_n=new ft,kn=new ut,Rn=new wt,Pn=new ht,En=new vt;async function In(e=Date.now(),l=dn){const f=kt(),m=await K.runFleetPreamble(f);let v,a;try{const g=await Ie();v=g.filter(c=>c.status==="active"||c.status==="idle"),a=g.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const p=new Set(a.map(c=>c.node_id));a.push(...(await _r()).filter(c=>!p.has(c.node_id)))}catch(g){u({level:"error",event:"canvas.node_list.failed",error:g});return}await K.run(e,{rows:v,fleet:f,lifecycle:l}),await pn.run(e,{rows:v});const w=await mn.run(e,{rows:[...v,...a],fleet:f});await fn.run(e),await Pn.run(e),await En.run(e,{lifecycle:l}),await un.run(e,{lifecycle:l}),await wn.run(e,{lifecycle:l}),await hn.run(e,{lifecycle:l}),await vn.run(e,{lifecycle:l}),await gn.run(e,{lifecycle:l,fleet:f}),Sn.run(e,{lifecycle:l}),yn.run(e,{lifecycle:l}),bn.run(e,{lifecycle:l}),_n.run(e,{lifecycle:l}),await pr(),Rn.run(e,{fleet:f,lifecycle:l}),await kn.run(e,{rows:v}),K.warnIfSaturated({fleet:f,lifecycle:l,thresholds:m,frozen:w})}i(In,"superviseTick");function V(e){return e!=="api"}i(V,"roleRunsLanes");function $(e){return e!=="reconcile"}i($,"roleServesRequests");const xn=66e4;function Tn(e){const l=new Rr().registerAll(Pr(()=>null).filter(n=>n.pattern==="/healthz")),f=i((n,d)=>{const y=(n.url??"/").split("?")[0];if((n.method??"GET").toUpperCase()==="GET"&&y==="/healthz"){l.handle(n,d,null,{class:"person"});return}d.writeHead(404,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"not_found",message:"this process serves only /healthz",origin:"runtime"}}))},"handler"),m=Pe();_(m)&&be(m);const v=J(f),a=new Promise((n,d)=>{v.once("error",d),v.listen(m,()=>{v.off("error",d),_e(m,384),n()})});let w,g=null;const p=e.tcp??ke()??se("user").api.tcp;if(p!==void 0&&p!==""){const n=/^(.*):(\d+)$/.exec(p.trim());if(n===null||Number(n[2])>65535)throw new Error(`invalid reconcile probe TCP address: ${p}`);const d=J(f);d.on("error",y=>u({event:"api.server.failed",level:"error",error:y})),d.listen(Number(n[2]),n[1]===""?"0.0.0.0":n[1],()=>{g=`${n[1]===""?"0.0.0.0":n[1]}:${n[2]}`}),w=d}let c=!1;return{ready:a,tcpAddress:i(()=>g,"tcpAddress"),async close(n={}){if(!c){c=!0,await Promise.all([Te(v,new Set,n.graceMs),w===void 0?Promise.resolve():Te(w,new Set,n.graceMs)]);try{_(m)&&be(m)}catch{}}}}}i(Tn,"createProbeServer");const Dn="daemon is draining for shutdown and admits no new work";async function Cn(e){ye(Ee(),{recursive:!0,mode:493}),process.platform==="linux"&&_e(Ee(),493);const l=Pe();if(_(l))throw new Error(`layout refusal cannot bind occupied socket: ${l}`);const f=J((p,c)=>{if(p.method==="GET"&&p.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((p,c)=>{f.once("error",c),f.listen(l,()=>{f.off("error",c),p()})});let m;const v=e.tcp??ke();if(v){const p=/^(.*):(\d+)$/.exec(v);if(!p||Number(p[2])>65535)throw new Error(`invalid refusal TCP address: ${v}`);const c=J(f.listeners("request")[0]);await new Promise((n,d)=>{c.once("error",d),c.listen(Number(p[2]),p[1],()=>{c.off("error",d),n()})}),m=c}Be();let a;const w=i(()=>(a??=(async()=>{f.closeAllConnections(),m?.closeAllConnections(),await Promise.all([f,m].filter(p=>p!==void 0).map(p=>new Promise((c,n)=>p.close(d=>d?n(d):c())))),ze(process.pid),process.off("SIGINT",g),process.off("SIGTERM",g)})(),a),"cleanup"),g=i(()=>{w().then(()=>process.exit(0),p=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(p)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",g),process.on("SIGTERM",g),w}i(Cn,"serveLayoutRefusal");async function Ln(e={}){if(gr())return Cn(e);vr();const l=i(r=>{process.stderr.write(`[crtrd] ${r}
-`)},"logPostgres"),f=!!process.env.CRTR_TEST_PG_URL,m=Lt(e.role);zt(V(e.role));const v=i(async()=>m?await Ct(Re(16).toString("hex")):{claimed:!0,claim:{pid:process.pid,pid_identity:"",token:Re(16).toString("hex")}},"claimOwnership"),{database:a,ownership:w}=f?{database:{url:await At()},ownership:await v()}:await Bt(async()=>({database:await Gt({log:l,startLockHeld:!0}),ownership:await v()}));if(!w.claimed){if(w.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${w.incumbentPid??"?"})
+var er=Object.defineProperty;var i=(e,l)=>er(e,"name",{value:l,configurable:!0});import{applyBootSettings as rr}from"./boot-settings.js";import{startPeerOutbox as tr}from"./peer/outbox.js";import{reportMachineStatus as nr}from"../core/peer/directory.js";import{createPeerRelay as or}from"./peer/relay-service.js";import{writeFileSync as ir,readFileSync as ar,rmSync as sr,mkdirSync as ye,existsSync as b,unlinkSync as _e,chmodSync as be}from"node:fs";import{createServer as J}from"node:http";import{dirname as cr}from"node:path";import{envCrtrdTcp as ke,envDocumentPort as lr}from"../shared/env.js";import{serveCanvasConversationStore as dr}from"../core/canvas/sessions.js";import{pollRunObservers as pr}from"./runs/observer.js";import{randomBytes as Re}from"node:crypto";import{apiSocketPath as Pe,crtrHome as mr}from"../core/canvas/paths.js";import{activeAppUsers as fr}from"../core/spaces/users.js";import{ensureAppSpace as ur,ensureUserFiles as wr}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as L}from"../launcher/client.js";import{installLauncherEventSink as hr}from"./launcher-events.js";import{initializeEmptyLayout as vr,layoutNeedsMigration as gr,socketDir as Ee}from"../core/layout.js";import{migrateLooseKeys as Sr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as yr}from"../core/profiles/relocate.js";import{getNode as _r,listNodesForSupervision as Ie,listParkedResidentsWithWakeMail as br,setFrozen as xe}from"../core/canvas/index.js";import{createApiServer as kr,closeServer as Te}from"./api/server.js";import{Router as Rr}from"./api/router.js";import{healthRoutes as Pr}from"./api/handlers/health.js";import{createDocumentListener as Er}from"./api/document-listener.js";import{documentSecret as Ir}from"./api/document-secret.js";import{createAppListener as xr}from"./api/app-listener.js";import{registerMachineEnvironment as Tr}from"../core/environments.js";import{registerBlaxelRunnerFromEnv as Dr}from"../core/secrets/blaxel-register.js";import{markCloudRole as Cr,resolvePerson as Lr,retireLocalPerson as $r,setPersonId as Ar}from"../core/canvas/person.js";import{backfillNodeApps as Nr,runPermissionsBoot as Mr}from"../core/permissions/boot.js";import{startRunNotices as Gr}from"./run-notices.js";import{startGrantEffects as Or}from"./reconcilers/grant-effects.js";import{createGrantSync as Fr,GRANT_SYNC_INTERVAL_S as Br}from"../core/grants/sync.js";import{sweepSandboxes as jr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as zr}from"./cron/run-socket.js";import{hasSyncedGrants as Hr}from"../core/grants/mirror.js";import{readConfig as se}from"../core/config.js";import{reconcileUndeliveredTickets as Ur}from"./human/sweep.js";import{resetHandedOffDeliveries as qr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Wr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Jr}from"./review/sweep.js";import{pidfilePath as ce}from"./pidfile.js";import{rendererWarning as Kr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Vr}from"./supervise-cadence.js";import{createResumeDetector as Yr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Qr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Xr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Zr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as et}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as rt}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as tt}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as nt}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as ot}from"./reconcilers/action-lane.js";import{endLiveExecution as it,migrateLegacyExecutionIdentities as at}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as st}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as ct}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as lt}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as dt}from"./reconcilers/storage-due-work.js";import{assertLedgerEnv as pt}from"../core/ledger/rates.js";import{LedgerSandboxReconciler as mt}from"./reconcilers/ledger-sandbox.js";import{LedgerStorageReconciler as ft}from"./reconcilers/ledger-storage.js";import{LedgerR2Reconciler as ut}from"./reconcilers/ledger-r2.js";import{BashDeadlineReconciler as wt}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as ht}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as vt}from"./reconcilers/structured-request-lane.js";import{RecipeImageReconciler as gt,subscribeImageBuilt as St}from"./reconcilers/recipe-image.js";import{reconcileStructuredRequestSidecars as yt}from"./structured-requests/boot.js";import{captureLivenessSnapshot as De,capturePidCommand as _t,captureTeardownSnapshot as Ce,isPidAlive as bt,isAnyPidAlive as Le,killProcessTreePids as $e}from"../core/canvas/pid.js";import{bindFleet as kt,boundFleet as Rt}from"../core/runtime/fleet.js";import{bindDaemonControl as Pt,clearDaemonControl as Et}from"./control.js";import{execDaemon as It,spawnDaemon as xt}from"./manage.js";import{reconcileAndMigrateBoot as Tt}from"../core/canvas/boot.js";import{resolveInstallId as Dt}from"../core/canvas/install-id.js";import{setInstallId as Ct}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Lt,daemonOwnershipApplies as $t,daemonOwnershipClaimed as At,releaseDaemonOwnership as Ae}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as Nt,closeAllDbs as le,openDb as Ne,withCanvasWrite as Mt}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as Gt}from"../core/canvas/import-sqlite.js";import{ensurePostgres as Ot,ensurePostgresHealthy as Ft,stopPostgres as Bt,withPostgresStartLock as jt}from"./postgres/index.js";import{brokerSignalBus as F}from"./broker-signals/bus.js";import{brokerSignalStreams as zt}from"./api/handlers/broker-signals.js";import{setRequestsStartRunObservers as Ht}from"./api/handlers/broker-ops.js";import{reconcilePackages as Ut}from"../core/graph/package-reconcile.js";import{startRepoSync as qt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Wt,reapStaleSpares as Jt}from"../core/runtime/warm-pool.js";import{DaemonFleet as Kt}from"./fleet.js";import{copyNodeFilesToRows as Vt}from"../core/runtime/first-boot-rows.js";import{BOOT_MIGRATIONS as Yt}from"./boot-migrations.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as Qt}from"../core/events/source.js";import{activateCanvasImport as Xt,MigrationBlockedError as Zt,runForcedCanvasImport as en}from"../migrations/activation.js";import{adoptInstallPerson as rn,convertLegacyScopedStore as tn,personStoresCopied as nn}from"../core/scoped-state/migrate.js";import{readLegacyUserSnapshot as on}from"../core/scoped-state/legacy-profiles.js";import{heldPersons as Me,installPreludeSnapshot as an}from"../core/scoped-state/person-store.js";import{loadPersonSettings as Ge}from"../core/scoped-state/person-rows.js";import{userScopeRoot as Oe}from"../core/scope.js";function de(e,l){return e.includes(` --canvas-home ${l} `)}i(de,"brokerCommandDeclaresCanvasHome");function pe(e){return/ --epoch (\S+) /.exec(e)?.[1]??null}i(pe,"brokerCommandEpoch");const sn=5e3,Fe=150;async function cn(e){const l=De();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const f=mr(),m=[];for(const[n,d]of l.processes){const y=d.command;if(d.zombie||!y.includes("broker-cli.js")||!de(y,f)||pe(y)===e)continue;const k=y.trim().split(/\s+/).at(-1)??"";k===""||k.endsWith("broker-cli.js")||m.push({pid:n,nodeId:k})}let v=0;const a=[];for(const n of m){const d=_t(n.pid);if(d===null||!d.includes("broker-cli.js")||!de(d,f)||pe(d)===e||d.trim().split(/\s+/).at(-1)!==n.nodeId)continue;const y=Ce(n.pid,null);if(!y.reused){try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{continue}v++,a.push({pid:n.pid,snap:y})}}const w=Date.now()+sn;for(;Date.now()<w&&a.some(n=>Le(n.snap.tree.length>0?n.snap.tree:[n.pid]));)await new Promise(n=>setTimeout(n,Fe));let g=0;for(const n of a){const d=n.snap.tree.length>0?n.snap.tree:[n.pid];Le(d)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):$e(d,"SIGKILL",n.snap.identities),g++)}g>0&&await new Promise(n=>setTimeout(n,Fe));const p=De();if(p===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[n,d]of p.processes){const y=d.command;d.zombie||!y.includes("broker-cli.js")||de(y,f)&&pe(y)!==e&&c.push(n)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:v,sigkilled:g}})}i(cn,"reapPriorEpochBrokers");const ln=5e3;async function dn(e){const l=Array.from(e.entries());if(l.length===0)return;for(const[a]of l){const w=(await _r(a))?.status;(w==="active"||w==="idle")&&await xe(a,new Date().toISOString())}const f=l.map(([,a])=>({pid:a.pid,snap:Ce(a.pid,null),exited:a.handle.exited}));for(const a of f)if(!a.snap.reused)try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{}const m=new Promise(a=>{setTimeout(a,ln).unref()});await Promise.allSettled(f.map(a=>Promise.race([a.exited.then(()=>{}),m])));const v=[];for(const a of f)bt(a.pid)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):$e(a.snap.tree.length>0?a.snap.tree:[a.pid],"SIGKILL",a.snap.identities),v.push(a));await Promise.all(v.map(a=>a.exited));for(const[a]of l)await it(a,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:f.length,sigkilled:v.length}})}i(dn,"teardownFleet");function Be(){ye(cr(ce()),{recursive:!0}),ir(ce(),String(process.pid),"utf8")}i(Be,"writePidfile");function je(e){return typeof e=="object"&&e!==null&&e.code==="ENOENT"}i(je,"isMissingPathError");function ze(e){const l=ce();let f;try{f=ar(l,"utf8")}catch(m){if(je(m))return;throw m}if(f.trim()!==String(e))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${e})`);try{sr(l)}catch(m){if(je(m))return;throw m}}i(ze,"removeOwnedPidfileMirror");const pn={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(e=>{e.catch(()=>{})},"registerDetached")},K=new Xr;Qr(e=>K.handleFatalFault(e,!1));const mn=new Zr,fn=new et,un=new rt,wn=new tt,hn=new nt,vn=new ot,gn=new ct,Sn=new lt,yn=new dt,_n=new mt,bn=new ft,kn=new ut,Rn=new wt,Pn=new ht,En=new vt,In=new gt;async function xn(e=Date.now(),l=pn){const f=Rt(),m=await K.runFleetPreamble(f);let v,a;try{const g=await Ie();v=g.filter(c=>c.status==="active"||c.status==="idle"),a=g.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const p=new Set(a.map(c=>c.node_id));a.push(...(await br()).filter(c=>!p.has(c.node_id)))}catch(g){u({level:"error",event:"canvas.node_list.failed",error:g});return}await K.run(e,{rows:v,fleet:f,lifecycle:l}),await mn.run(e,{rows:v});const w=await fn.run(e,{rows:[...v,...a],fleet:f});await un.run(e),await En.run(e),await In.run(e,{lifecycle:l}),await wn.run(e,{lifecycle:l}),await hn.run(e,{lifecycle:l}),await vn.run(e,{lifecycle:l}),await gn.run(e,{lifecycle:l}),await Sn.run(e,{lifecycle:l,fleet:f}),yn.run(e,{lifecycle:l}),_n.run(e,{lifecycle:l}),bn.run(e,{lifecycle:l}),kn.run(e,{lifecycle:l}),await pr(),Pn.run(e,{fleet:f,lifecycle:l}),await Rn.run(e,{rows:v}),K.warnIfSaturated({fleet:f,lifecycle:l,thresholds:m,frozen:w})}i(xn,"superviseTick");function V(e){return e!=="api"}i(V,"roleRunsLanes");function $(e){return e!=="reconcile"}i($,"roleServesRequests");const Tn=66e4;function Dn(e){const l=new Rr().registerAll(Pr(()=>null).filter(n=>n.pattern==="/healthz")),f=i((n,d)=>{const y=(n.url??"/").split("?")[0];if((n.method??"GET").toUpperCase()==="GET"&&y==="/healthz"){l.handle(n,d,null,{class:"person"});return}d.writeHead(404,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"not_found",message:"this process serves only /healthz",origin:"runtime"}}))},"handler"),m=Pe();b(m)&&_e(m);const v=J(f),a=new Promise((n,d)=>{v.once("error",d),v.listen(m,()=>{v.off("error",d),be(m,384),n()})});let w,g=null;const p=e.tcp??ke()??se("user").api.tcp;if(p!==void 0&&p!==""){const n=/^(.*):(\d+)$/.exec(p.trim());if(n===null||Number(n[2])>65535)throw new Error(`invalid reconcile probe TCP address: ${p}`);const d=J(f);d.on("error",y=>u({event:"api.server.failed",level:"error",error:y})),d.listen(Number(n[2]),n[1]===""?"0.0.0.0":n[1],()=>{g=`${n[1]===""?"0.0.0.0":n[1]}:${n[2]}`}),w=d}let c=!1;return{ready:a,tcpAddress:i(()=>g,"tcpAddress"),async close(n={}){if(!c){c=!0,await Promise.all([Te(v,new Set,n.graceMs),w===void 0?Promise.resolve():Te(w,new Set,n.graceMs)]);try{b(m)&&_e(m)}catch{}}}}}i(Dn,"createProbeServer");const Cn="daemon is draining for shutdown and admits no new work";async function Ln(e){ye(Ee(),{recursive:!0,mode:493}),process.platform==="linux"&&be(Ee(),493);const l=Pe();if(b(l))throw new Error(`layout refusal cannot bind occupied socket: ${l}`);const f=J((p,c)=>{if(p.method==="GET"&&p.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((p,c)=>{f.once("error",c),f.listen(l,()=>{f.off("error",c),p()})});let m;const v=e.tcp??ke();if(v){const p=/^(.*):(\d+)$/.exec(v);if(!p||Number(p[2])>65535)throw new Error(`invalid refusal TCP address: ${v}`);const c=J(f.listeners("request")[0]);await new Promise((n,d)=>{c.once("error",d),c.listen(Number(p[2]),p[1],()=>{c.off("error",d),n()})}),m=c}Be();let a;const w=i(()=>(a??=(async()=>{f.closeAllConnections(),m?.closeAllConnections(),await Promise.all([f,m].filter(p=>p!==void 0).map(p=>new Promise((c,n)=>p.close(d=>d?n(d):c())))),ze(process.pid),process.off("SIGINT",g),process.off("SIGTERM",g)})(),a),"cleanup"),g=i(()=>{w().then(()=>process.exit(0),p=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(p)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",g),process.on("SIGTERM",g),w}i(Ln,"serveLayoutRefusal");async function $n(e={}){if(gr())return Ln(e);vr();const l=i(r=>{process.stderr.write(`[crtrd] ${r}
+`)},"logPostgres"),f=!!process.env.CRTR_TEST_PG_URL,m=$t(e.role);Ht(V(e.role));const v=i(async()=>m?await Lt(Re(16).toString("hex")):{claimed:!0,claim:{pid:process.pid,pid_identity:"",token:Re(16).toString("hex")}},"claimOwnership"),{database:a,ownership:w}=f?{database:{url:await Nt()},ownership:await v()}:await jt(async()=>({database:await Ot({log:l,startLockHeld:!0}),ownership:await v()}));if(!w.claimed){if(w.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${w.incumbentPid??"?"})
 `);else{const r=[w.incumbentPid!==void 0?`pid ${w.incumbentPid}`:void 0,w.verdict!==void 0?`verdict ${w.verdict}`:void 0].filter(h=>h!==void 0),o=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${w.reason}${o}
-`),w.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(w.reason==="own_identity_indeterminate"||w.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const g=w.claim;let p=null,c=null,n=null,d,y="";try{m&&await sn(g.token);for(const t of yr())process.stderr.write(`[crtrd] ${t}
+`),w.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(w.reason==="own_identity_indeterminate"||w.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const g=w.claim;let p=null,c=null,n=null,d,y="";try{m&&await cn(g.token);for(const t of yr())process.stderr.write(`[crtrd] ${t}
 `);for(const t of await Sr())process.stderr.write(`[crtrd] ${t}
-`);await Ne(),await Mt(a.url,l);const r=e.role===void 0&&!await tn();let o=null;r&&(rn({kind:"user",root:Oe()}),o=nn(Oe())),e.role===void 0&&on(o??{profiles:[],defaults:{},settings:{},state:{}}),await F.listen(a.url);const h=i(t=>{Ge(t).catch(O=>u({level:"error",event:"daemon.settings.reload_failed",error:O}))},"reloadPerson");d=await F.subscribe(t=>{Me().includes(t)&&h(t)},{kinds:["settings"],onListenStart:i(()=>{for(const t of Me())h(t)},"onListenStart")}),e.role!==void 0&&Cr();const s=await Nt(async t=>(await Nr(t),await Ht(t)));for(const t of s.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${t.message}
+`);await Ne(),await Gt(a.url,l);const r=e.role===void 0&&!await nn();let o=null;r&&(tn({kind:"user",root:Oe()}),o=on(Oe())),e.role===void 0&&an(o??{profiles:[],defaults:{},settings:{},state:{}}),await F.listen(a.url);const h=i(t=>{Ge(t).catch(O=>u({level:"error",event:"daemon.settings.reload_failed",error:O}))},"reloadPerson");d=await F.subscribe(t=>{Me().includes(t)&&h(t)},{kinds:["settings"],onListenStart:i(()=>{for(const t of Me())h(t)},"onListenStart")}),e.role!==void 0&&Cr();const s=await Mt(async t=>(await Nr(t),await Ut(t)));for(const t of s.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${t.message}
 `);for(const t of s.invalid)process.stderr.write(`[crtrd] package file is not a document: ${t.path}: ${t.error}
 `);for(const t of s.quarantined)process.stderr.write(`[crtrd] plugin "${t.name}" has a damaged ${t.file} (${t.reason}); ${t.movedTo!==void 0?`moved to ${t.movedTo}`:"left in place and skipped"}
 `);for(const t of s.kinds.refused)process.stderr.write(`[crtrd] plugin "${t.plugin}" kind ${t.kind} not registered: ${t.message}
 `);for(const t of s.kinds.invalid)process.stderr.write(`[crtrd] plugin "${t.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${t.error}
-`);if(y=await Tt(),Dt(y),e.role===void 0&&(p=await Lr(),p!==null)){if(Ar(p.id),await en(p.id),process.platform==="darwin"){const{readConfig:t,updateConfig:O}=await import("../core/config.js");if(t("user").paths?.user_files===void 0){const{homedir:Ze}=await import("node:os");await O("user",Se=>{Se.paths??={},Se.paths.user_files??=Ze()})}}c=await $r(),c.joinedFrom!==null&&await Ge(c.person),await rr(),n=await Mr()}if(process.platform==="linux"&&_(x)&&(hr(),await L({operation:"users.sync",users:await fr()})),m){await st(g.token);const t=await Kt();Object.values(t).some(O=>O>0)&&process.stderr.write(`[crtrd] node files copied into rows: ${JSON.stringify(t)}
-`)}}catch(r){await F.close();try{if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}await le()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}let k=!0,Q,B=Promise.resolve(),Y=!1,j,X,Z,T;const z=new Set;let ee=!1,me=!1,P,D,re,H=!1,q,A,N,M,te,ne,oe;const He=Qr();let E,S={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},C="initial startup activation",qe=null;function Ue(){return S.phase==="ready"?null:{phase:S.phase,message:S.message,blockerPaths:S.blockerPaths}}i(Ue,"startupBlock");function R(r){z.add(r),r.finally(()=>{z.delete(r)}).catch(()=>{})}i(R,"registerDetached");const fe={acceptsDetachedWork:i(()=>k,"acceptsDetachedWork"),registerDetached:R};V(e.role)&&await qt(fe,Ne);async function We(){try{k=!1,jt.close(),X?.(),X=void 0,Z?.(),d?.(),Z=void 0,ne?.(),ne=void 0,await F.close(),j!==void 0&&(clearInterval(j),j=void 0),N!==void 0&&(clearInterval(N),N=void 0),oe?.(),oe=void 0;let r;do r=B,await r;while(r!==B);for(;z.size>0;)await Promise.allSettled([...z]);if(T!==void 0&&(m&&await ln(T),await T.exitsSettled(),T.dispose()),re!==void 0&&await re.close(),D!==void 0&&!H&&(H=!0,await D.close()),te?.(),te=void 0,M!==void 0){const o=M;M=void 0,await o.stop()}if(A!==void 0){const o=A;A=void 0,await o.close()}if(q!==void 0){const o=q;q=void 0,await o.close()}if(ee||(ze(process.pid),ee=!0),ee&&!me){if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}me=!0,await le()}}finally{Pt(),process.off("SIGINT",he),process.off("SIGTERM",ve),process.off("exit",ge)}}i(We,"cleanupAttempt");const U=1e3;let ue=!1;function Je(){return ue?{graceMs:U}:(ue=!0,b.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:U}})}),setTimeout(()=>{Ke()},U),{graceMs:U})}i(Je,"requestHandover");async function Ke(){try{await I()}catch(r){const o=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+`);if(y=await Dt(),Ct(y),e.role===void 0&&(p=await Lr(),p!==null)){if(Ar(p.id),await rn(p.id),process.platform==="darwin"){const{readConfig:t,updateConfig:O}=await import("../core/config.js");if(t("user").paths?.user_files===void 0){const{homedir:Ze}=await import("node:os");await O("user",Se=>{Se.paths??={},Se.paths.user_files??=Ze()})}}c=await $r(),c.joinedFrom!==null&&await Ge(c.person),await rr(),n=await Mr()}if(process.platform==="linux"&&b(x)&&(hr(),await L({operation:"users.sync",users:await fr()})),m){await st(g.token);const t=await Vt();Object.values(t).some(O=>O>0)&&process.stderr.write(`[crtrd] node files copied into rows: ${JSON.stringify(t)}
+`)}}catch(r){await F.close();try{if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}await le()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}let k=!0,Y,B=Promise.resolve(),Q=!1,j,X,Z,T;const z=new Set;let ee=!1,me=!1,P,D,re,H=!1,U,A,N,M,te,ne,oe;const He=Yr();let E,S={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},C="initial startup activation",Ue=null;function qe(){return S.phase==="ready"?null:{phase:S.phase,message:S.message,blockerPaths:S.blockerPaths}}i(qe,"startupBlock");function R(r){z.add(r),r.finally(()=>{z.delete(r)}).catch(()=>{})}i(R,"registerDetached");const fe={acceptsDetachedWork:i(()=>k,"acceptsDetachedWork"),registerDetached:R};V(e.role)&&await qt(fe,Ne);async function We(){try{k=!1,zt.close(),X?.(),X=void 0,Z?.(),d?.(),Z=void 0,ne?.(),ne=void 0,await F.close(),j!==void 0&&(clearInterval(j),j=void 0),N!==void 0&&(clearInterval(N),N=void 0),oe?.(),oe=void 0;let r;do r=B,await r;while(r!==B);for(;z.size>0;)await Promise.allSettled([...z]);if(T!==void 0&&(m&&await dn(T),await T.exitsSettled(),T.dispose()),re!==void 0&&await re.close(),D!==void 0&&!H&&(H=!0,await D.close()),te?.(),te=void 0,M!==void 0){const o=M;M=void 0,await o.stop()}if(A!==void 0){const o=A;A=void 0,await o.close()}if(U!==void 0){const o=U;U=void 0,await o.close()}if(ee||(ze(process.pid),ee=!0),ee&&!me){if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}me=!0,await le()}}finally{Et(),process.off("SIGINT",he),process.off("SIGTERM",ve),process.off("exit",ge)}}i(We,"cleanupAttempt");const q=1e3;let ue=!1;function Je(){return ue?{graceMs:q}:(ue=!0,_.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:q}})}),setTimeout(()=>{Ke()},q),{graceMs:q})}i(Je,"requestHandover");async function Ke(){try{await I()}catch(r){const o=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
 ${o}
-`),process.exit(1)}try{process.platform==="linux"&&_(x)&&(await L({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Et();const r=await It({waitForJoinLock:!0});b.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){b.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const o=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${o}
-`),process.exit(1)}process.exit(0)}i(Ke,"performHandover");function I(){if(P!==void 0)return P;const r=We();return P=r,r.then(()=>{P===r&&(P=void 0)},()=>{P===r&&(P=void 0)}),r}i(I,"requestCleanup");async function Ve(){S={phase:"initializing",message:Dn,blockerPaths:[]},D!==void 0&&!H&&(H=!0,await D.close({graceMs:xn}))}i(Ve,"drainApi");async function we(r){try{r==="SIGTERM"&&e.role==="api"&&await Ve(),await I(),await Ft({claimed:i(async()=>{try{return m&&await $t()}finally{await le()}},"claimed")}),process.exit(0)}catch(o){process.exitCode=1;const h=o instanceof Error?o.stack??`${o.name}: ${o.message}`:String(o);process.stderr.write(`[crtrd] ${r} cleanup failed:
+`),process.exit(1)}try{process.platform==="linux"&&b(x)&&(await L({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await It();const r=await xt({waitForJoinLock:!0});_.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const o=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${o}
+`),process.exit(1)}process.exit(0)}i(Ke,"performHandover");function I(){if(P!==void 0)return P;const r=We();return P=r,r.then(()=>{P===r&&(P=void 0)},()=>{P===r&&(P=void 0)}),r}i(I,"requestCleanup");async function Ve(){S={phase:"initializing",message:Cn,blockerPaths:[]},D!==void 0&&!H&&(H=!0,await D.close({graceMs:Tn}))}i(Ve,"drainApi");async function we(r){try{r==="SIGTERM"&&e.role==="api"&&await Ve(),await I(),await Bt({claimed:i(async()=>{try{return m&&await At()}finally{await le()}},"claimed")}),process.exit(0)}catch(o){process.exitCode=1;const h=o instanceof Error?o.stack??`${o.name}: ${o.message}`:String(o);process.stderr.write(`[crtrd] ${r} cleanup failed:
 ${h}
-`),process.exit(1)}}i(we,"handleSignal");function he(){we("SIGINT")}i(he,"onSigint");function ve(){we("SIGTERM")}i(ve,"onSigterm");function ge(){I().catch(()=>{})}i(ge,"onExit");async function ie(r){k&&(B=B.then(r).then(()=>{},o=>{b.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:o})})}))}i(ie,"enqueue");async function W(){!k||Y||(He.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),R(E().catch(()=>{}))),Y=!0,await ie(async()=>{if(Y=!1,!!k)try{f||await Ot({log:l}),await In(Date.now(),fe)}catch(r){b.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(W,"enqueueTick");let G=null;async function Qe(){if(G!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${G instanceof Error?G.message:String(G)}`};if(S.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(S.phase!=="prepared"&&S.phase!=="recovering")throw{status:409,code:"not_prepared",message:S.message};const r=S.phase==="prepared";try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(o){throw G=o,{status:500,code:"admission_failed",message:`daemon admission failed: ${o instanceof Error?o.message:String(o)}`}}}i(Qe,"admitStartup");async function Ye(r){if(C!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${C})`};C="repair";try{const o=await Zt({dryRun:r.dryRun});if(r.dryRun||o.pending||S.phase!=="blocked")return{result:o,resumed:!1};if(e.prepare)return S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:o,resumed:!1};try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},{result:o,resumed:!0}}catch(h){const s=h instanceof Error?h.message:String(h);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:o,resumed:!1,resumeError:s}}}finally{C=null}}i(Ye,"repairMigrations");async function Xe(){Q=e.intervalMs??Vr;const r=g.token;T=new Jt({epoch:r,enqueue:ie}),_t(T),S={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},b.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:Q}})});const o=y;b.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:o}})});const h=await Tr(o);if(b.fresh(()=>{u({level:"info",event:"daemon.environment.registered",fields:{environment_id:h.environmentId,created:h.created}})}),b.fresh(()=>{let s;try{s=Kr()}catch{return}s!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:s.slice(0,2048)}})}),e.role!==void 0&&Dr(),!V(e.role)){S={phase:"ready",message:"",blockerPaths:[]};return}if(await b.fresh(async()=>{await Jr(),await qr(),await Ur(Date.now()),await Wr(Date.now())}),await b.fresh(async()=>{const{reconcile:s,migrated:t}=await xt(at);s.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:s.cleared.length,same_boot_proven:s.sameBootProven}}),t.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:t.length}})}),await b.fresh(async()=>{for(const s of await Ie())s.status!=="active"||s.pi_pid!=null||s.frozen_at!=null||s.respawn_not_before!=null||(await xe(s.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:s.node_id}))}),await b.fresh(async()=>{await St();for(const s of Vt){const t=s.apply();t.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:s.id,...t.summary}})}}),R(zr()),E){const s=i(()=>{R(E().catch(()=>{})),R(nr().then(t=>M?.observe(t)).then(()=>{},t=>{u({level:"warn",event:"daemon.peer.status_failed",error:t})}))},"refresh");R(jr().then(({completion:t})=>t).catch(t=>{u({level:"error",event:"daemon.grants.sweep_failed",error:t})})),s(),N=setInterval(s,Br*1e3),N.unref(),oe=tr({joined:!0,register:R})}await ie(async()=>{await Wt(),await Ut()}),j=setInterval(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},Q),X=await gt(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))}),Z=await F.subscribe(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},{kinds:["spend_limits"]}),await W()}i(Xe,"completeStartup");function ae(){return qe??=Xe().catch(r=>{throw S={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(ae,"resumeStartup");try{process.on("SIGINT",he),process.on("SIGTERM",ve),process.on("exit",ge),Qt(),p!==null&&u({level:"info",event:"daemon.person.resolved",fields:{person_id:p.id,source:p.source}}),n!==null&&u({level:"info",event:"daemon.permissions.boot",fields:{...n}}),c!==null&&u({level:"info",event:"daemon.person.retired",fields:{person_id:c.person,joined_from:c.joinedFrom,retired:c.retired,columns:c.columns}}),$(e.role)&&(q=await dr());const r=g.token;Rt({epoch:r,requestHandover:Je,registerDetached:R,startupBlock:Ue,admitStartup:Qe,repairMigrations:Ye}),Be(),D=$(e.role)?kr({tcp:e.tcp}):Tn(e),await D.ready;const o=lr()??se("user").document_port;if($(e.role)){const s=Er(o,Ir());try{await s.ready,re=s}catch(t){u({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:o,error:t instanceof Error?t.message:String(t)}})}}const h=se("user").runtime;if(h?.issuer&&(E=Fr(h.issuer,R),h.unsandboxed===!0&&(M=or({syncGrants:E}))),$(e.role)&&(te=await Gr()),ne=await Or({serves:$(e.role),runsLanes:V(e.role)}),h?.unsandboxed===!0&&h.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),$(e.role)&&h!==void 0&&h.listener!==void 0&&h.unsandboxed!==!0&&(A=await xr(h,async()=>await Hr(h.issuer.replace(/\/$/,"")),E),await A.ready),e.role===void 0)try{await Yt()}catch(s){if(C=null,s instanceof Xt)return S={phase:"blocked",message:s.message,blockerPaths:s.blockerPaths},I;throw s}process.platform==="linux"&&_(x)&&(wr(),await ur("app:terminal"));try{e.prepare?S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await ae(),S={phase:"ready",message:"",blockerPaths:[]})}finally{C=null}return I}catch(r){try{await I()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}}i(Ln,"runDaemon");var ki=Ln;export{xn as API_DRAIN_GRACE_MS,de as brokerCommandDeclaresCanvasHome,pe as brokerCommandEpoch,ki as default,V as roleRunsLanes,$ as roleServesRequests,Ln as runDaemon,In as superviseTick,ln as teardownFleet};
+`),process.exit(1)}}i(we,"handleSignal");function he(){we("SIGINT")}i(he,"onSigint");function ve(){we("SIGTERM")}i(ve,"onSigterm");function ge(){I().catch(()=>{})}i(ge,"onExit");async function ie(r){k&&(B=B.then(r).then(()=>{},o=>{_.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:o})})}))}i(ie,"enqueue");async function W(){!k||Q||(He.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),R(E().catch(()=>{}))),Q=!0,await ie(async()=>{if(Q=!1,!!k)try{f||await Ft({log:l}),await xn(Date.now(),fe)}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(W,"enqueueTick");let G=null;async function Ye(){if(G!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${G instanceof Error?G.message:String(G)}`};if(S.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(S.phase!=="prepared"&&S.phase!=="recovering")throw{status:409,code:"not_prepared",message:S.message};const r=S.phase==="prepared";try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(o){throw G=o,{status:500,code:"admission_failed",message:`daemon admission failed: ${o instanceof Error?o.message:String(o)}`}}}i(Ye,"admitStartup");async function Qe(r){if(C!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${C})`};C="repair";try{const o=await en({dryRun:r.dryRun});if(r.dryRun||o.pending||S.phase!=="blocked")return{result:o,resumed:!1};if(e.prepare)return S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:o,resumed:!1};try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},{result:o,resumed:!0}}catch(h){const s=h instanceof Error?h.message:String(h);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:o,resumed:!1,resumeError:s}}}finally{C=null}}i(Qe,"repairMigrations");async function Xe(){Y=e.intervalMs??Vr;const r=g.token;T=new Kt({epoch:r,enqueue:ie}),kt(T),S={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:Y}})});const o=y;_.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:o}})});const h=await Tr(o);if(_.fresh(()=>{u({level:"info",event:"daemon.environment.registered",fields:{environment_id:h.environmentId,created:h.created}})}),_.fresh(()=>{let s;try{s=Kr()}catch{return}s!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:s.slice(0,2048)}})}),pt(),e.role!==void 0&&!process.env.CRTR_DIRECTORY_URL)throw new Error("CRTR_DIRECTORY_URL must be set on a cloud role (--role api|reconcile): it names the directory this deployment mints tokens against");if(e.role!==void 0&&Dr(),!V(e.role)){S={phase:"ready",message:"",blockerPaths:[]};return}if(await _.fresh(async()=>{await Jr(),await Ur(),await qr(Date.now()),await Wr(Date.now())}),await _.fresh(async()=>{const{reconcile:s,migrated:t}=await Tt(at);s.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:s.cleared.length,same_boot_proven:s.sameBootProven}}),t.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:t.length}})}),await _.fresh(async()=>{for(const s of await Ie())s.status!=="active"||s.pi_pid!=null||s.frozen_at!=null||s.respawn_not_before!=null||(await xe(s.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:s.node_id}))}),await _.fresh(async()=>{await yt();for(const s of Yt){const t=s.apply();t.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:s.id,...t.summary}})}}),R(zr()),E){const s=i(()=>{R(E().catch(()=>{})),R(nr().then(t=>M?.observe(t)).then(()=>{},t=>{u({level:"warn",event:"daemon.peer.status_failed",error:t})}))},"refresh");R(jr().then(({completion:t})=>t).catch(t=>{u({level:"error",event:"daemon.grants.sweep_failed",error:t})})),s(),N=setInterval(s,Br*1e3),N.unref(),oe=tr({joined:!0,register:R})}await ie(async()=>{await Jt(),await Wt()}),j=setInterval(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},Y),X=await St(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))}),Z=await F.subscribe(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},{kinds:["spend_limits"]}),await W()}i(Xe,"completeStartup");function ae(){return Ue??=Xe().catch(r=>{throw S={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(ae,"resumeStartup");try{process.on("SIGINT",he),process.on("SIGTERM",ve),process.on("exit",ge),Qt(),p!==null&&u({level:"info",event:"daemon.person.resolved",fields:{person_id:p.id,source:p.source}}),n!==null&&u({level:"info",event:"daemon.permissions.boot",fields:{...n}}),c!==null&&u({level:"info",event:"daemon.person.retired",fields:{person_id:c.person,joined_from:c.joinedFrom,retired:c.retired,columns:c.columns}}),$(e.role)&&(U=await dr());const r=g.token;Pt({epoch:r,requestHandover:Je,registerDetached:R,startupBlock:qe,admitStartup:Ye,repairMigrations:Qe}),Be(),D=$(e.role)?kr({tcp:e.tcp}):Dn(e),await D.ready;const o=lr()??se("user").document_port;if($(e.role)){const s=Er(o,Ir());try{await s.ready,re=s}catch(t){u({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:o,error:t instanceof Error?t.message:String(t)}})}}const h=se("user").runtime;if(h?.issuer&&(E=Fr(h.issuer,R),h.unsandboxed===!0&&(M=or({syncGrants:E}))),$(e.role)&&(te=await Gr()),ne=await Or({serves:$(e.role),runsLanes:V(e.role)}),h?.unsandboxed===!0&&h.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),$(e.role)&&h!==void 0&&h.listener!==void 0&&h.unsandboxed!==!0&&(A=await xr(h,async()=>await Hr(h.issuer.replace(/\/$/,"")),E),await A.ready),e.role===void 0)try{await Xt()}catch(s){if(C=null,s instanceof Zt)return S={phase:"blocked",message:s.message,blockerPaths:s.blockerPaths},I;throw s}process.platform==="linux"&&b(x)&&(wr(),await ur("app:terminal"));try{e.prepare?S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await ae(),S={phase:"ready",message:"",blockerPaths:[]})}finally{C=null}return I}catch(r){try{await I()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}}i($n,"runDaemon");var Pi=$n;export{Tn as API_DRAIN_GRACE_MS,de as brokerCommandDeclaresCanvasHome,pe as brokerCommandEpoch,Pi as default,V as roleRunsLanes,$ as roleServesRequests,$n as runDaemon,xn as superviseTick,dn as teardownFleet};

package/dist/daemon/peer/relay-service.js

@@ -1 +1 @@
-var U=Object.defineProperty;var a=(e,c)=>U(e,"name",{value:c,configurable:!0});import{JwksCache as E}from"@north-light/identity";import{randomUUID as N}from"node:crypto";import{readConfig as k}from"../../core/config.js";import{hasSyncedGrants as O}from"../../core/grants/mirror.js";import{emitEvent as _}from"../../core/events/emit.js";import{directoryIssuer as b,machineUrl as j,ownMachineId as R}from"../../core/peer/directory.js";import{RunError as m}from"../../core/runs/operations.js";import{createResumeDetector as G}from"../resume-detector.js";import{applySyncNotice as H,NoticeLedger as M}from"../api/sync-notice.js";import{runtimeOwner as J,servePeer as T}from"./ingress.js";import{startLocalIngress as D}from"./local-ingress.js";import{startRelayClient as L}from"./relay-client.js";const $="/v1/runtime/sync-notice",z=5e3;async function A(e,c,f){if(e.headersSent){e.destroy();return}e.writeHead(c,{"content-type":"application/json; charset=utf-8"}),e.end(JSON.stringify(f))}a(A,"answerJson");function ae(e){const c=e.unsandboxed??(()=>k("user").runtime?.unsandboxed===!0),f=e.issuer??b;let s;const S=new M;async function d(){const t=s;s=void 0,t!==void 0&&(clearInterval(t.resume),await t.client.stop(),await t.ingress.close())}a(d,"stopStarted");async function C(t,r){const i=new E(r,fetch,e.jwksUrl?e.jwksUrl():k("user").runtime?.jwksUrl),u={issuer:r,jwks:i,audience:j,owner:J,self:R,ready:a(async()=>await O(r.replace(/\/$/,"")),"ready"),syncGrants:e.syncGrants,...e.ingress},w=await D(a(async(o,l)=>{const x=(o.url??"").split("?",1)[0];if(o.method==="POST"&&x===$){o.resume();try{const n=await u.audience();if(n===null||u.ready&&!await u.ready())throw new m(503,"runtime_starting","runtime_starting");await H(await S.accept(o.headers.authorization,i,r,n),e.syncGrants,N()),l.writeHead(204),l.end()}catch(n){const I=n instanceof m?n.status:500;await A(l,I,{error:{code:n instanceof m?n.code:"server_failure",message:n instanceof m?n.message:"sync notice failed"}})}return}await T(o,l,u)},"serve"),e.socketPath);let y;try{y=L({issuer:r,relayOrigin:t,socketPath:w.socketPath,...e.client})}catch(o){throw await w.close(),o}const P=G(),v=setInterval(()=>{P.check()&&(_({level:"info",event:"daemon.peer.relay_resume"}),y.reconnectNow())},e.resumeCheckMs??z);v.unref(),s={relay:t,ingress:w,client:y,resume:v,jwks:i},_({level:"info",event:"daemon.peer.relay_started",fields:{relay:t}})}a(C,"begin");let h=Promise.resolve(),p=!1;const g=a(t=>{const r=h.then(t);return h=r.catch(()=>{}),r},"serial");return{observe(t){return g(async()=>{if(p)return;const r=f();if(t===null)return;if(t.reach?.kind!=="relay"||!c()||r===null){await d();return}const i=t.reach.relay.replace(/\/$/,"");s?.relay!==i&&(await d(),await C(i,r))})},running:a(()=>s!==void 0,"running"),stop(){return p=!0,g(d)}}}a(ae,"createPeerRelay");export{z as RESUME_CHECK_MS,$ as SYNC_NOTICE_PATH,ae as createPeerRelay};
+var U=Object.defineProperty;var a=(e,c)=>U(e,"name",{value:c,configurable:!0});import{JwksCache as E}from"@north-light/identity";import{randomUUID as N}from"node:crypto";import{readConfig as k}from"../../core/config.js";import{hasSyncedGrants as O}from"../../core/grants/mirror.js";import{emitEvent as _}from"../../core/events/emit.js";import{machineUrl as b,ownMachineId as j}from"../../core/peer/directory.js";import{directoryIssuer as R}from"../../core/directory/issuer.js";import{RunError as m}from"../../core/runs/operations.js";import{createResumeDetector as G}from"../resume-detector.js";import{applySyncNotice as H,NoticeLedger as M}from"../api/sync-notice.js";import{runtimeOwner as J,servePeer as T}from"./ingress.js";import{startLocalIngress as D}from"./local-ingress.js";import{startRelayClient as L}from"./relay-client.js";const $="/v1/runtime/sync-notice",z=5e3;async function A(e,c,f){if(e.headersSent){e.destroy();return}e.writeHead(c,{"content-type":"application/json; charset=utf-8"}),e.end(JSON.stringify(f))}a(A,"answerJson");function oe(e){const c=e.unsandboxed??(()=>k("user").runtime?.unsandboxed===!0),f=e.issuer??R;let s;const S=new M;async function d(){const t=s;s=void 0,t!==void 0&&(clearInterval(t.resume),await t.client.stop(),await t.ingress.close())}a(d,"stopStarted");async function C(t,r){const i=new E(r,fetch,e.jwksUrl?e.jwksUrl():k("user").runtime?.jwksUrl),u={issuer:r,jwks:i,audience:b,owner:J,self:j,ready:a(async()=>await O(r.replace(/\/$/,"")),"ready"),syncGrants:e.syncGrants,...e.ingress},w=await D(a(async(o,l)=>{const x=(o.url??"").split("?",1)[0];if(o.method==="POST"&&x===$){o.resume();try{const n=await u.audience();if(n===null||u.ready&&!await u.ready())throw new m(503,"runtime_starting","runtime_starting");await H(await S.accept(o.headers.authorization,i,r,n),e.syncGrants,N()),l.writeHead(204),l.end()}catch(n){const I=n instanceof m?n.status:500;await A(l,I,{error:{code:n instanceof m?n.code:"server_failure",message:n instanceof m?n.message:"sync notice failed"}})}return}await T(o,l,u)},"serve"),e.socketPath);let y;try{y=L({issuer:r,relayOrigin:t,socketPath:w.socketPath,...e.client})}catch(o){throw await w.close(),o}const P=G(),v=setInterval(()=>{P.check()&&(_({level:"info",event:"daemon.peer.relay_resume"}),y.reconnectNow())},e.resumeCheckMs??z);v.unref(),s={relay:t,ingress:w,client:y,resume:v,jwks:i},_({level:"info",event:"daemon.peer.relay_started",fields:{relay:t}})}a(C,"begin");let h=Promise.resolve(),p=!1;const g=a(t=>{const r=h.then(t);return h=r.catch(()=>{}),r},"serial");return{observe(t){return g(async()=>{if(p)return;const r=f();if(t===null)return;if(t.reach?.kind!=="relay"||!c()||r===null){await d();return}const i=t.reach.relay.replace(/\/$/,"");s?.relay!==i&&(await d(),await C(i,r))})},running:a(()=>s!==void 0,"running"),stop(){return p=!0,g(d)}}}a(oe,"createPeerRelay");export{z as RESUME_CHECK_MS,$ as SYNC_NOTICE_PATH,oe as createPeerRelay};

package/dist/daemon/readiness.d.ts

@@ -1,5 +1,6 @@
 import type { ReadyDTO } from '../api/dto/health.js';
-/** True on a joined runtime whose issuer snapshot has never committed (a committed empty snapshot counts as committed). */
+/** True on a joined Mac whose issuer snapshot has never committed (a committed empty snapshot counts as committed). A cloud role
+ *  never waits on a snapshot: its grant authority is the shared grant rows the directory writes directly, always installed. */
 export declare function grantSnapshotMissing(): Promise<boolean>;
 export interface ReadinessInputs {
     /** App listener only: the JWKS mirror has loaded. */

package/dist/daemon/readiness.js

@@ -1 +1 @@
-var a=Object.defineProperty;var e=(s,n)=>a(s,"name",{value:n,configurable:!0});import{readConfig as o}from"../core/config.js";import{grantsSource as i,hasSyncedGrants as r}from"../core/grants/mirror.js";import{boundDaemonControl as u}from"./control.js";function t(){const s=o("user").runtime?.issuer;return s?s.replace(/\/$/,""):null}e(t,"joinedIssuer");async function g(){const s=t();return s!==null&&!await r(s)}e(g,"grantSnapshotMissing");async function l(){const s=t();return s===null?await i()==="local":await r(s)}e(l,"grantsInstalled");async function p(s={}){const n=u()?.startupBlock()??null;return n!==null?{ready:!1,reason:"startup_pending",message:`daemon startup is ${n.phase}`}:s.jwksReady===!1?{ready:!1,reason:"jwks_pending",message:"the directory key set has not loaded"}:s.grantsReady?.()??await l()?{ready:!0}:t()===null?{ready:!1,reason:"grant_snapshot_missing",message:"the local grant is not installed yet"}:{ready:!1,reason:"grant_snapshot_missing",message:"the first grant snapshot from the directory has not committed"}}e(p,"runtimeReadiness");export{g as grantSnapshotMissing,p as runtimeReadiness};
+var o=Object.defineProperty;var t=(e,s)=>o(e,"name",{value:s,configurable:!0});import{isCloudRole as a}from"../core/canvas/person-state.js";import{directoryIssuer as r}from"../core/directory/issuer.js";import{grantsSource as i,hasSyncedGrants as n}from"../core/grants/mirror.js";import{boundDaemonControl as l}from"./control.js";async function p(){if(a())return!1;const e=r();return e!==null&&!await n(e)}t(p,"grantSnapshotMissing");async function u(){if(a())return!0;const e=r();return e===null?await i()==="local":await n(e)}t(u,"grantsInstalled");async function y(e={}){const s=l()?.startupBlock()??null;return s!==null?{ready:!1,reason:"startup_pending",message:`daemon startup is ${s.phase}`}:e.jwksReady===!1?{ready:!1,reason:"jwks_pending",message:"the directory key set has not loaded"}:e.grantsReady?.()??await u()?{ready:!0}:r()===null?{ready:!1,reason:"grant_snapshot_missing",message:"the local grant is not installed yet"}:{ready:!1,reason:"grant_snapshot_missing",message:"the first grant snapshot from the directory has not committed"}}t(y,"runtimeReadiness");export{p as grantSnapshotMissing,y as runtimeReadiness};

package/dist/daemon/reconcilers/ledger-r2.js

@@ -1,7 +1,7 @@
-var g=Object.defineProperty;var u=(d,e)=>g(d,"name",{value:e,configurable:!0});import{createHash as _}from"node:crypto";import{zstdCompressSync as S}from"node:zlib";import{blobStore as A}from"../../core/blobs/index.js";import{openDb as E,withCanvasWrite as N}from"../../core/canvas/db.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{usageMonth as D}from"../../core/ledger/months.js";import{sandboxUnbilledIn as L}from"./ledger-sandbox.js";import{storageSettledThrough as w}from"./ledger-storage.js";import{runThrottledLane as y}from"./throttled-lane.js";const T=60*6e4,v=2880*6e4,C=25,M=10*6e4,I=u((d,e,t)=>`ledger/${d}/${e}.${t}.jsonl.zst`,"chargeChunkKey"),p="(billed_usd IS NULL OR billed_usd = 0 OR invoice_id IS NOT NULL)";class P{static{u(this,"LedgerR2Reconciler")}blobs;constructor(e=A){this.blobs=e}run(e,t){if(!t.lifecycle.acceptsDetachedWork())return;const a=R.fresh(()=>y("ledger-r2","",e,T,M,async r=>{await this.pass(e,r)})).then(()=>{}).catch(r=>{R.fresh(()=>f({level:"error",event:"ledger.r2.failed",error:r}))});t.lifecycle.registerDetached(a)}async pass(e,t,a){const r={moved:0,deferred:0};if(O("user").ledger?.r2_move!==!0)return r;const i=await w();if(i===null)return r;const o=D(new Date(e-v)),s=await(await E()).prepare(`SELECT c.month, c.payer FROM charges c
+var g=Object.defineProperty;var p=(d,e)=>g(d,"name",{value:e,configurable:!0});import{createHash as _}from"node:crypto";import{zstdCompressSync as S}from"node:zlib";import{blobStore as A}from"../../core/blobs/index.js";import{openDb as E,withCanvasWrite as N}from"../../core/canvas/db.js";import{r2MoveEnabled as O}from"../../core/ledger/rates.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{usageMonth as D}from"../../core/ledger/months.js";import{sandboxUnbilledIn as L}from"./ledger-sandbox.js";import{storageSettledThrough as w}from"./ledger-storage.js";import{runThrottledLane as y}from"./throttled-lane.js";const T=60*6e4,v=2880*6e4,M=25,C=10*6e4,I=p((d,e,t)=>`ledger/${d}/${e}.${t}.jsonl.zst`,"chargeChunkKey"),u="(billed_usd IS NULL OR billed_usd = 0 OR invoice_id IS NOT NULL)";class P{static{p(this,"LedgerR2Reconciler")}blobs;constructor(e=A){this.blobs=e}run(e,t){if(!t.lifecycle.acceptsDetachedWork())return;const a=R.fresh(()=>y("ledger-r2","",e,T,C,async r=>{await this.pass(e,r)})).then(()=>{}).catch(r=>{R.fresh(()=>f({level:"error",event:"ledger.r2.failed",error:r}))});t.lifecycle.registerDetached(a)}async pass(e,t,a){const r={moved:0,deferred:0};if(!O())return r;const i=await w();if(i===null)return r;const o=D(new Date(e-v)),s=await(await E()).prepare(`SELECT c.month, c.payer FROM charges c
       WHERE c.month < ? AND (?::text IS NULL OR c.month <= ?) AND NOT EXISTS (SELECT 1 FROM charge_chunks k WHERE k.month = c.month AND k.payer = c.payer)
-      GROUP BY c.month, c.payer HAVING bool_and(${p.replaceAll("billed_usd","c.billed_usd").replaceAll("invoice_id","c.invoice_id")})
-      ORDER BY c.month, c.payer LIMIT ?`).all(o,i??null,i??null,C),l=await this.blobs();for(const{month:n,payer:h}of s){if(t&&!await t.holds())break;if(await L(n,e)){r.deferred+=1;continue}await this.move(l,n,h,a)?r.moved+=1:r.deferred+=1}return r.moved>0&&f({level:"info",event:"ledger.r2.moved",outcome:"succeeded",fields:{...r}}),r}async move(e,t,a,r){const o=await(await E()).prepare("SELECT id, to_jsonb(charges)::text AS body FROM charges WHERE month = ? AND payer = ? ORDER BY id").all(t,a);if(o.length===0||!await this.movable(t,a))return!1;const c=o.map(n=>n.body).join(`
+      GROUP BY c.month, c.payer HAVING bool_and(${u.replaceAll("billed_usd","c.billed_usd").replaceAll("invoice_id","c.invoice_id")})
+      ORDER BY c.month, c.payer LIMIT ?`).all(o,i??null,i??null,M),l=await this.blobs();for(const{month:n,payer:h}of s){if(t&&!await t.holds())break;if(await L(n,e)){r.deferred+=1;continue}await this.move(l,n,h,a)?r.moved+=1:r.deferred+=1}return r.moved>0&&f({level:"info",event:"ledger.r2.moved",outcome:"succeeded",fields:{...r}}),r}async move(e,t,a,r){const o=await(await E()).prepare("SELECT id, to_jsonb(charges)::text AS body FROM charges WHERE month = ? AND payer = ? ORDER BY id").all(t,a);if(o.length===0||!await this.movable(t,a))return!1;const c=o.map(n=>n.body).join(`
 `)+`
-`,s=I(t,a,_("sha256").update(c).digest("hex").slice(0,16));await e.put(s,S(Buffer.from(c)));const l=await this.commit(t,a,s,o,r);return!l&&await(await E()).prepare("SELECT 1 FROM charge_chunks WHERE blob_key = ?").get(s)===void 0&&await e.delete(s),l}async commit(e,t,a,r,i){return await i?.(),await N(async o=>{const c=await o.prepare("SELECT id FROM charges WHERE month = ? AND payer = ? ORDER BY id FOR UPDATE").all(e,t),s=await o.prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND ${p}`).get(e,t);if(c.length!==r.length||Number(s.n)!==r.length||c.some((m,b)=>m.id!==r[b].id)||(await o.prepare("SELECT string_agg(to_jsonb(c)::text, E'\\n' ORDER BY c.id) AS body FROM charges c WHERE month = ? AND payer = ?").get(e,t)).body!==r.map(m=>m.body).join(`
-`))return!1;const n=await o.prepare("SELECT sum(usd) AS usd, sum(billed_usd) AS billed, min(at) AS first_at, max(at) AS last_at FROM charges WHERE month = ? AND payer = ?").get(e,t);return(await o.prepare('INSERT INTO charge_chunks (month, payer, blob_key, "rows", usd, billed_usd, first_at, last_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (month, payer) DO NOTHING').run(e,t,a,r.length,n.usd,n.billed,n.first_at,n.last_at)).changes===0?!1:(await o.prepare("DELETE FROM charges WHERE month = ? AND payer = ?").run(e,t),!0)})}async movable(e,t){const a=await(await E()).prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND NOT ${p}`).get(e,t);return Number(a.n)===0}}export{C as LEDGER_R2_BUDGET,v as LEDGER_R2_GRACE_MS,T as LEDGER_R2_INTERVAL_MS,P as LedgerR2Reconciler,I as chargeChunkKey};
+`,s=I(t,a,_("sha256").update(c).digest("hex").slice(0,16));await e.put(s,S(Buffer.from(c)));const l=await this.commit(t,a,s,o,r);return!l&&await(await E()).prepare("SELECT 1 FROM charge_chunks WHERE blob_key = ?").get(s)===void 0&&await e.delete(s),l}async commit(e,t,a,r,i){return await i?.(),await N(async o=>{const c=await o.prepare("SELECT id FROM charges WHERE month = ? AND payer = ? ORDER BY id FOR UPDATE").all(e,t),s=await o.prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND ${u}`).get(e,t);if(c.length!==r.length||Number(s.n)!==r.length||c.some((m,b)=>m.id!==r[b].id)||(await o.prepare("SELECT string_agg(to_jsonb(c)::text, E'\\n' ORDER BY c.id) AS body FROM charges c WHERE month = ? AND payer = ?").get(e,t)).body!==r.map(m=>m.body).join(`
+`))return!1;const n=await o.prepare("SELECT sum(usd) AS usd, sum(billed_usd) AS billed, min(at) AS first_at, max(at) AS last_at FROM charges WHERE month = ? AND payer = ?").get(e,t);return(await o.prepare('INSERT INTO charge_chunks (month, payer, blob_key, "rows", usd, billed_usd, first_at, last_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (month, payer) DO NOTHING').run(e,t,a,r.length,n.usd,n.billed,n.first_at,n.last_at)).changes===0?!1:(await o.prepare("DELETE FROM charges WHERE month = ? AND payer = ?").run(e,t),!0)})}async movable(e,t){const a=await(await E()).prepare(`SELECT count(*) AS n FROM charges WHERE month = ? AND payer = ? AND NOT ${u}`).get(e,t);return Number(a.n)===0}}export{M as LEDGER_R2_BUDGET,v as LEDGER_R2_GRACE_MS,T as LEDGER_R2_INTERVAL_MS,P as LedgerR2Reconciler,I as chargeChunkKey};

package/dist/daemon/reconcilers/ledger-sandbox.d.ts

@@ -1,7 +1,6 @@
 import type { DetachedWorkLifecycle } from './broker-supervision.js';
 import { type LanePass } from './throttled-lane.js';
 export declare const LEDGER_SANDBOX_INTERVAL_MS = 60000;
-export declare const DEFAULT_STANDBY_S = 30;
 /** Executions billed per pass; the rest wait for the next pass. */
 export declare const LEDGER_SANDBOX_BUDGET = 200;
 interface Rates {

package/dist/daemon/reconcilers/ledger-sandbox.js

@@ -1,9 +1,9 @@
-var W=Object.defineProperty;var d=(r,e)=>W(r,"name",{value:e,configurable:!0});import{openDb as x,withCanvasWrite as F}from"../../core/canvas/db.js";import{getNode as A}from"../../core/canvas/canvas.js";import{readConfig as z}from"../../core/config.js";import{emitEvent as E}from"../../core/events/emit.js";import{operationIdContext as T}from"../../core/events/operation-id.js";import{billingAccountOf as $}from"../../core/ledger/account.js";import{writeInfraCharge as B}from"../../core/ledger/infra-allowance.js";import{resetAt as G}from"../../core/ledger/months.js";import{startCandidateOf as P}from"../../core/runtime/start-check.js";import{runThrottledLane as J}from"./throttled-lane.js";const Y=6e4,V=5*6e4,b=36e5,X=30,y=200;function N(){const r=z("user").ledger;return{usdPerGbHour:r?.sandbox_usd_per_gb_hour??null,standbyMs:(r?.sandbox_standby_s??X)*1e3}}d(N,"sandboxRates");function j(r,e,n){const t=r.exited_at??r.idle_at;if(t===null)return{end:e,closed:!1};const a=Date.parse(t)+n;return a<=e?{end:a,closed:!0}:{end:e,closed:!1}}d(j,"periodEnd");const K=d(r=>{const e=new Date(r);return Date.UTC(e.getUTCFullYear(),e.getUTCMonth()+1,1)},"monthEndAfter");function R(r,e){const n=[];for(let t=r;t<e;){const a=Math.min(e,K(t));n.push([t,a]),t=a}return n}d(R,"withinMonths");function k(r,e,n){const t=[];let a=r;for(;e-a>=b;)t.push(...R(a,a+b)),a+=b;return n&&e>a&&t.push(...R(a,e)),t}d(k,"chunksOf");const Z=d(r=>new Date(r-1),"chunkAccruedAt"),q=d((r,e)=>`sandbox:${r}:${Math.floor(e/1e3)}`,"sandboxChargeId"),S="ledger-sandbox-unsettled",M=d((r,e)=>`${r}@${Date.parse(e)}`,"unsettledKey");class Q{static{d(this,"LedgerSandboxReconciler")}run(e,n){if(!n.lifecycle.acceptsDetachedWork())return;const t=T.fresh(()=>J("ledger-sandbox","",e,Y,V,async a=>{await this.pass(e,a)})).then(()=>{}).catch(a=>{T.fresh(()=>E({level:"error",event:"ledger.sandbox.failed",error:a}))});n.lifecycle.registerDetached(t)}async pass(e,n){const t={executions:0,rows:0,failed:0},{usdPerGbHour:a,standbyMs:l}=N();if(a===null)return t;const i=await x(),_=await i.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+var W=Object.defineProperty;var d=(r,e)=>W(r,"name",{value:e,configurable:!0});import{openDb as x,withCanvasWrite as F}from"../../core/canvas/db.js";import{getNode as A}from"../../core/canvas/canvas.js";import{LEDGER_ENV as z,sandboxStandbyS as G,sandboxUsdPerGbHour as $,warnRateUnset as P}from"../../core/ledger/rates.js";import{emitEvent as E}from"../../core/events/emit.js";import{operationIdContext as R}from"../../core/events/operation-id.js";import{billingAccountOf as B}from"../../core/ledger/account.js";import{writeInfraCharge as J}from"../../core/ledger/infra-allowance.js";import{resetAt as V}from"../../core/ledger/months.js";import{startCandidateOf as Y}from"../../core/runtime/start-check.js";import{runThrottledLane as X}from"./throttled-lane.js";const j=6e4,K=5*6e4,g=36e5,T=200;function N(){return{usdPerGbHour:$(),standbyMs:G()*1e3}}d(N,"sandboxRates");function Z(r,e,n){const t=r.exited_at??r.idle_at;if(t===null)return{end:e,closed:!1};const a=Date.parse(t)+n;return a<=e?{end:a,closed:!0}:{end:e,closed:!1}}d(Z,"periodEnd");const q=d(r=>{const e=new Date(r);return Date.UTC(e.getUTCFullYear(),e.getUTCMonth()+1,1)},"monthEndAfter");function y(r,e){const n=[];for(let t=r;t<e;){const a=Math.min(e,q(t));n.push([t,a]),t=a}return n}d(y,"withinMonths");function k(r,e,n){const t=[];let a=r;for(;e-a>=g;)t.push(...y(a,a+g)),a+=g;return n&&e>a&&t.push(...y(a,e)),t}d(k,"chunksOf");const Q=d(r=>new Date(r-1),"chunkAccruedAt"),ee=d((r,e)=>`sandbox:${r}:${Math.floor(e/1e3)}`,"sandboxChargeId"),S="ledger-sandbox-unsettled",M=d((r,e)=>`${r}@${Date.parse(e)}`,"unsettledKey");class te{static{d(this,"LedgerSandboxReconciler")}run(e,n){if(!n.lifecycle.acceptsDetachedWork())return;const t=R.fresh(()=>X("ledger-sandbox","",e,j,K,async a=>{await this.pass(e,a)})).then(()=>{}).catch(a=>{R.fresh(()=>E({level:"error",event:"ledger.sandbox.failed",error:a}))});n.lifecycle.registerDetached(t)}async pass(e,n){const t={executions:0,rows:0,failed:0},{usdPerGbHour:a,standbyMs:l}=N();if(a===null)return P("ledger-sandbox",z.sandboxUsdPerGbHour),t;const i=await x(),_=await i.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
       WHERE runner = 'blaxel' AND awake_since IS NOT NULL
         AND (state <> 'exited' OR charged_through IS NULL OR charged_through::timestamptz < exited_at::timestamptz + make_interval(secs => ?::float8 / 1000))
-      ORDER BY execution_id LIMIT ?`).all(l,y);for(const s of _){if(n&&!await n.holds())break;try{const o=await this.bill(s.execution_id,e,a,l);o>0&&(t.executions+=1),t.rows+=o}catch(o){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:o,fields:{execution_id:s.execution_id}})}}const f=await i.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? ORDER BY key LIMIT ?").all(S,y);for(const{cursor:s}of f){const o=JSON.parse(s);if(n&&!await n.holds())break;try{const c=await this.bill(o.execution_id,e,a,l,!1,o);c>0&&(t.executions+=1),t.rows+=c}catch(c){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:c,fields:{execution_id:o.execution_id}})}}return(t.rows>0||t.failed>0)&&E({level:"info",event:"ledger.sandbox.settled",outcome:"succeeded",fields:{...t}}),t}async bill(e,n,t,a,l=!1,i){const _=await x(),f=await _.prepare("SELECT node_id FROM executions WHERE execution_id = ?").get(e);if(f===void 0)return 0;const s=await A(f.node_id);if(s===null)throw new Error(`execution ${e} names node ${f.node_id}, which has no row`);const o=await P(s);if(o===null)throw new Error(`node ${s.node_id} has no person yet, so its sandbox hours have no payer`);const c=await _.prepare("SELECT e.memory_mb AS mb FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ?").get(s.profile_id??null);if(c===void 0)throw new Error(`node ${s.node_id} has no environment, so its sandbox size is unknown`);const p=Number(c.mb),{account:C,billable:v}=await $(_,o.payer),H=(s.run_id?await A(s.run_id):null)?.caused_by??s.caused_by??null;return await F(async D=>{const u=await D.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
-        WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL FOR UPDATE`).get(e);if(i===void 0&&u===void 0||i===void 0&&l&&u.idle_at===null)return 0;let g,h;if(i!==void 0)g=Math.max(Date.parse(i.awake_since),i.charged_through===null?0:Date.parse(i.charged_through)),h=k(g,Date.parse(i.idle_at)+a,!0);else{const{end:m,closed:w}=l?{end:Math.min(n,Date.parse(u.idle_at)+a),closed:!0}:j(u,n,a);g=Math.max(Date.parse(u.awake_since),u.charged_through===null?0:Date.parse(u.charged_through)),h=k(g,m,w)}let L=0;for(const[m,w]of h){const I=Math.round((w-m)/1e3),O=p/1024*((w-m)/b),U=O*t;(await B({id:q(e,m),at:Z(w),kind:"sandbox",what:`${p} MB`,units:{gb_hours:O,seconds:I,memory_mb:p},usd:U,payer:o.payer,funding:"bill",sub:o.sub,grantee:o.grantee,node:s.node_id,run:s.run_id??null,provider:"blaxel",causedBy:H,account:C},v)).written&&(L+=1)}return i!==void 0?await D.prepare("DELETE FROM reconciler_leases WHERE lane = ? AND key = ?").run(S,M(e,i.awake_since)):h.length>0&&await D.prepare("UPDATE executions SET charged_through = ? WHERE execution_id = ?").run(new Date(h[h.length-1][1]).toISOString(),e),L})}}async function pe(r,e){const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return!1;const a=await x(),l=new Date(`${r}-01T00:00:00Z`).toISOString(),i=G(r);return await a.prepare(`SELECT 1 AS found FROM executions e,
+      ORDER BY execution_id LIMIT ?`).all(l,T);for(const s of _){if(n&&!await n.holds())break;try{const o=await this.bill(s.execution_id,e,a,l);o>0&&(t.executions+=1),t.rows+=o}catch(o){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:o,fields:{execution_id:s.execution_id}})}}const f=await i.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ? ORDER BY key LIMIT ?").all(S,T);for(const{cursor:s}of f){const o=JSON.parse(s);if(n&&!await n.holds())break;try{const c=await this.bill(o.execution_id,e,a,l,!1,o);c>0&&(t.executions+=1),t.rows+=c}catch(c){t.failed+=1,E({level:"warn",event:"ledger.sandbox.execution_failed",error:c,fields:{execution_id:o.execution_id}})}}return(t.rows>0||t.failed>0)&&E({level:"info",event:"ledger.sandbox.settled",outcome:"succeeded",fields:{...t}}),t}async bill(e,n,t,a,l=!1,i){const _=await x(),f=await _.prepare("SELECT node_id FROM executions WHERE execution_id = ?").get(e);if(f===void 0)return 0;const s=await A(f.node_id);if(s===null)throw new Error(`execution ${e} names node ${f.node_id}, which has no row`);const o=await Y(s);if(o===null)throw new Error(`node ${s.node_id} has no person yet, so its sandbox hours have no payer`);const c=await _.prepare("SELECT e.memory_mb AS mb FROM profiles p JOIN environments e ON e.object = p.environment_id WHERE p.id = ?").get(s.profile_id??null);if(c===void 0)throw new Error(`node ${s.node_id} has no environment, so its sandbox size is unknown`);const p=Number(c.mb),{account:C,billable:v}=await B(_,o.payer),H=(s.run_id?await A(s.run_id):null)?.caused_by??s.caused_by??null;return await F(async D=>{const u=await D.prepare(`SELECT execution_id, node_id, awake_since, idle_at, exited_at, charged_through FROM executions
+        WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL FOR UPDATE`).get(e);if(i===void 0&&u===void 0||i===void 0&&l&&u.idle_at===null)return 0;let b,h;if(i!==void 0)b=Math.max(Date.parse(i.awake_since),i.charged_through===null?0:Date.parse(i.charged_through)),h=k(b,Date.parse(i.idle_at)+a,!0);else{const{end:m,closed:w}=l?{end:Math.min(n,Date.parse(u.idle_at)+a),closed:!0}:Z(u,n,a);b=Math.max(Date.parse(u.awake_since),u.charged_through===null?0:Date.parse(u.charged_through)),h=k(b,m,w)}let L=0;for(const[m,w]of h){const U=Math.round((w-m)/1e3),O=p/1024*((w-m)/g),I=O*t;(await J({id:ee(e,m),at:Q(w),kind:"sandbox",what:`${p} MB`,units:{gb_hours:O,seconds:U,memory_mb:p},usd:I,payer:o.payer,funding:"bill",sub:o.sub,grantee:o.grantee,node:s.node_id,run:s.run_id??null,provider:"blaxel",causedBy:H,account:C},v)).written&&(L+=1)}return i!==void 0?await D.prepare("DELETE FROM reconciler_leases WHERE lane = ? AND key = ?").run(S,M(e,i.awake_since)):h.length>0&&await D.prepare("UPDATE executions SET charged_through = ? WHERE execution_id = ?").run(new Date(h[h.length-1][1]).toISOString(),e),L})}}async function Ee(r,e){const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return!1;const a=await x(),l=new Date(`${r}-01T00:00:00Z`).toISOString(),i=V(r);return await a.prepare(`SELECT 1 AS found FROM executions e,
       LATERAL (SELECT greatest(e.awake_since::timestamptz, COALESCE(e.charged_through::timestamptz, '-infinity'::timestamptz)) AS from_at,
                       CASE WHEN COALESCE(e.exited_at, e.idle_at) IS NULL THEN ?::timestamptz
                            ELSE least(COALESCE(e.exited_at, e.idle_at)::timestamptz + make_interval(secs => ?::float8 / 1000), ?::timestamptz) END AS end_at) p
-    WHERE e.runner = 'blaxel' AND e.awake_since IS NOT NULL AND p.from_at < p.end_at AND p.from_at < ?::timestamptz AND p.end_at > ?::timestamptz LIMIT 1`).get(new Date(e).toISOString(),t,new Date(e).toISOString(),i,l)!==void 0?!0:(await a.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ?").all(S)).some(({cursor:s})=>{const o=JSON.parse(s),c=Math.max(Date.parse(o.awake_since),o.charged_through===null?0:Date.parse(o.charged_through)),p=Date.parse(o.idle_at)+t;return c<p&&c<Date.parse(i)&&p>Date.parse(l)})}d(pe,"sandboxUnbilledIn");async function he(r,e=Date.now()){try{const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return;await new Q().bill(r,e,n,t,!0)}catch(n){E({level:"warn",event:"ledger.sandbox.settle_failed",error:n,fields:{execution_id:r}}),await ee(r)}}d(he,"settleSandboxBeforeWake");async function ee(r){try{const e=await x(),n=await e.prepare("SELECT execution_id, awake_since, idle_at, charged_through FROM executions WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL AND idle_at IS NOT NULL").get(r);if(n===void 0)return;await e.prepare("INSERT INTO reconciler_leases (lane, key, owner, expires_at, cursor) VALUES (?, ?, ?, ?, ?) ON CONFLICT (lane, key) DO NOTHING").run(S,M(r,n.awake_since),"settle",new Date(0).toISOString(),JSON.stringify(n))}catch(e){E({level:"error",event:"ledger.sandbox.unsettled_lost",error:e,fields:{execution_id:r}})}}d(ee,"keepUnsettled");export{X as DEFAULT_STANDBY_S,y as LEDGER_SANDBOX_BUDGET,Y as LEDGER_SANDBOX_INTERVAL_MS,Q as LedgerSandboxReconciler,Z as chunkAccruedAt,k as chunksOf,q as sandboxChargeId,N as sandboxRates,pe as sandboxUnbilledIn,he as settleSandboxBeforeWake};
+    WHERE e.runner = 'blaxel' AND e.awake_since IS NOT NULL AND p.from_at < p.end_at AND p.from_at < ?::timestamptz AND p.end_at > ?::timestamptz LIMIT 1`).get(new Date(e).toISOString(),t,new Date(e).toISOString(),i,l)!==void 0?!0:(await a.prepare("SELECT cursor FROM reconciler_leases WHERE lane = ?").all(S)).some(({cursor:s})=>{const o=JSON.parse(s),c=Math.max(Date.parse(o.awake_since),o.charged_through===null?0:Date.parse(o.charged_through)),p=Date.parse(o.idle_at)+t;return c<p&&c<Date.parse(i)&&p>Date.parse(l)})}d(Ee,"sandboxUnbilledIn");async function me(r,e=Date.now()){try{const{usdPerGbHour:n,standbyMs:t}=N();if(n===null)return;await new te().bill(r,e,n,t,!0)}catch(n){E({level:"warn",event:"ledger.sandbox.settle_failed",error:n,fields:{execution_id:r}}),await re(r)}}d(me,"settleSandboxBeforeWake");async function re(r){try{const e=await x(),n=await e.prepare("SELECT execution_id, awake_since, idle_at, charged_through FROM executions WHERE execution_id = ? AND runner = 'blaxel' AND awake_since IS NOT NULL AND idle_at IS NOT NULL").get(r);if(n===void 0)return;await e.prepare("INSERT INTO reconciler_leases (lane, key, owner, expires_at, cursor) VALUES (?, ?, ?, ?, ?) ON CONFLICT (lane, key) DO NOTHING").run(S,M(r,n.awake_since),"settle",new Date(0).toISOString(),JSON.stringify(n))}catch(e){E({level:"error",event:"ledger.sandbox.unsettled_lost",error:e,fields:{execution_id:r}})}}d(re,"keepUnsettled");export{T as LEDGER_SANDBOX_BUDGET,j as LEDGER_SANDBOX_INTERVAL_MS,te as LedgerSandboxReconciler,Q as chunkAccruedAt,k as chunksOf,ee as sandboxChargeId,N as sandboxRates,Ee as sandboxUnbilledIn,me as settleSandboxBeforeWake};

package/dist/daemon/reconcilers/ledger-storage.js

@@ -1,5 +1,5 @@
-var M=Object.defineProperty;var p=(s,e)=>M(s,"name",{value:e,configurable:!0});import{openDb as N}from"../../core/canvas/db.js";import{getNode as D}from"../../core/canvas/canvas.js";import{readConfig as O}from"../../core/config.js";import{emitEvent as y}from"../../core/events/emit.js";import{operationIdContext as S}from"../../core/events/operation-id.js";import{billingAccountOf as T}from"../../core/ledger/account.js";import{writeInfraCharge as v}from"../../core/ledger/infra-allowance.js";import{usageMonth as w}from"../../core/ledger/months.js";import{startCandidateOf as I}from"../../core/runtime/start-check.js";import{runThrottledLane as L}from"./throttled-lane.js";const A=60*6e4,C=10*6e4,G=1024**3;async function Q(){const s=O("user").ledger;if((s?.storage_usd_per_gb_month??null)===null&&(s?.sandbox_disk_usd_per_gb_month??null)===null)return;const e=await(await N()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = 'ledger-storage' AND key = ''").get(),r=e?.cursor==null?null:JSON.parse(e.cursor).state;return r===null?null:r.slice(0,7)}p(Q,"storageSettledThrough");function U(s){const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r-2,1)))}p(U,"previousMonth");const x=p(s=>{const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r,1)))},"nextMonth"),F=p(s=>{const[e,r]=s.split("-").map(Number);return new Date(Date.UTC(e,r,1)).toISOString()},"endOf");class X{static{p(this,"LedgerStorageReconciler")}run(e,r){if(!r.lifecycle.acceptsDetachedWork())return;const u=S.fresh(()=>L("ledger-storage","",e,A,C,async o=>{await this.pass(e,o)})).then(()=>{}).catch(o=>{S.fresh(()=>y({level:"error",event:"ledger.storage.failed",error:o}))});r.lifecycle.registerDetached(u)}async pass(e,r){const u=U(w(new Date(e))),o={month:null,rows:0,failed:0},_=O("user").ledger,f={r2:_?.storage_usd_per_gb_month??null,disk:_?.sandbox_disk_usd_per_gb_month??null};if(f.r2===null&&f.disk===null)return o;const i=r?.state?.slice(0,7)??null,l=[];if(i===null)l.push(u);else for(let n=x(i);n<=u;n=x(n))l.push(n);for(const n of l){if(o.month=n,!await this.settleMonth(n,f,r,o))break;await r?.save(n)}return(o.rows>0||o.failed>0)&&y({level:"info",event:"ledger.storage.settled",outcome:"succeeded",fields:{...o}}),o}async settleMonth(e,r,u,o){const{r2:_,disk:f}=r,i=F(e),l=await N(),n=new Map,g=p(t=>{let a=n.get(t);return a||n.set(t,a={bytes:0,diskGb:0}),a},"entry");for(const t of await l.prepare("SELECT split_part(blob_key, '/', 2) AS node, sum(bytes) AS bytes FROM conversation_parts WHERE created < ? GROUP BY 1").all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node, sum(u.size) AS bytes FROM (
+var v=Object.defineProperty;var b=(s,e)=>v(s,"name",{value:e,configurable:!0});import{openDb as N}from"../../core/canvas/db.js";import{getNode as G}from"../../core/canvas/canvas.js";import{LEDGER_ENV as O,sandboxDiskUsdPerGbMonth as S,storageUsdPerGbMonth as M,warnRateUnset as R}from"../../core/ledger/rates.js";import{emitEvent as y}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{billingAccountOf as I}from"../../core/ledger/account.js";import{writeInfraCharge as L}from"../../core/ledger/infra-allowance.js";import{usageMonth as w}from"../../core/ledger/months.js";import{startCandidateOf as A}from"../../core/runtime/start-check.js";import{runThrottledLane as C}from"./throttled-lane.js";const U=60*6e4,P=10*6e4,F=1024**3;async function ee(){if(M()===null&&S()===null)return;const s=await(await N()).prepare("SELECT cursor FROM reconciler_leases WHERE lane = 'ledger-storage' AND key = ''").get(),e=s?.cursor==null?null:JSON.parse(s.cursor).state;return e===null?null:e.slice(0,7)}b(ee,"storageSettledThrough");function J(s){const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r-2,1)))}b(J,"previousMonth");const k=b(s=>{const[e,r]=s.split("-").map(Number);return w(new Date(Date.UTC(e,r,1)))},"nextMonth"),W=b(s=>{const[e,r]=s.split("-").map(Number);return new Date(Date.UTC(e,r,1)).toISOString()},"endOf");class te{static{b(this,"LedgerStorageReconciler")}run(e,r){if(!r.lifecycle.acceptsDetachedWork())return;const d=D.fresh(()=>C("ledger-storage","",e,U,P,async o=>{await this.pass(e,o)})).then(()=>{}).catch(o=>{D.fresh(()=>y({level:"error",event:"ledger.storage.failed",error:o}))});r.lifecycle.registerDetached(d)}async pass(e,r){const d=J(w(new Date(e))),o={month:null,rows:0,failed:0},c={r2:M(),disk:S()};if(c.r2===null&&R("ledger-storage",O.storageUsdPerGbMonth),c.disk===null&&R("ledger-storage",O.sandboxDiskUsdPerGbMonth),c.r2===null&&c.disk===null)return o;const m=r?.state?.slice(0,7)??null,i=[];if(m===null)i.push(d);else for(let n=k(m);n<=d;n=k(n))i.push(n);for(const n of i){if(o.month=n,!await this.settleMonth(n,c,r,o))break;await r?.save(n)}return(o.rows>0||o.failed>0)&&y({level:"info",event:"ledger.storage.settled",outcome:"succeeded",fields:{...o}}),o}async settleMonth(e,r,d,o){const{r2:c,disk:m}=r,i=W(e),n=await N(),g=new Map,h=b(t=>{let a=g.get(t);return a||g.set(t,a={bytes:0,diskGb:0}),a},"entry");for(const t of await n.prepare("SELECT split_part(blob_key, '/', 2) AS node, sum(bytes) AS bytes FROM conversation_parts WHERE created < ? GROUP BY 1").all(i))h(t.node).bytes+=Number(t.bytes);for(const t of await n.prepare(`SELECT x.node, sum(u.size) AS bytes FROM (
         SELECT DISTINCT upload_id, split_part(blob_key, '/', 2) AS node FROM upload_uses WHERE blob_key LIKE 'attachments/%') x
-      JOIN uploads u ON u.upload_id = x.upload_id WHERE u.created_at < ? GROUP BY x.node`).all(i))g(t.node).bytes+=Number(t.bytes);for(const t of await l.prepare(`SELECT x.node_id AS node, sum(e.memory_mb) AS mb FROM (
+      JOIN uploads u ON u.upload_id = x.upload_id WHERE u.created_at < ? GROUP BY x.node`).all(i))h(t.node).bytes+=Number(t.bytes);for(const t of await n.prepare(`SELECT x.node_id AS node, sum(e.memory_mb) AS mb FROM (
         SELECT DISTINCT node_id, sandbox_name FROM executions WHERE runner = 'blaxel' AND sandbox_name IS NOT NULL AND started_at < ?) x
-      JOIN nodes n ON n.node_id = x.node_id JOIN objects o ON o.id = n.node_id AND (o.deleted_at IS NULL OR o.deleted_at >= ?) JOIN profiles p ON p.id = n.profile_id JOIN environments e ON e.object = p.environment_id GROUP BY x.node_id`).all(i,i))g(t.node).diskGb+=Number(t.mb)/2/1024;const h=new Map;for(const[t,a]of n){const b=await D(t),c=b===null?null:await I(b);if(c===null)continue;const m=h.get(c.payer)??{sub:c.sub,grantee:c.grantee,bytes:0,diskGb:0,node:t};m.bytes+=a.bytes,m.diskGb+=a.diskGb,h.set(c.payer,m)}const R=new Date(Date.parse(i)-1);let E=0;for(const[t,a]of h){if(u&&!await u.holds())return!1;try{const{account:b,billable:c}=await T(l,t),m=[{id:`storage:${t}:${e}:r2`,provider:"r2",gbMonths:a.bytes/G,rate:_},{id:`storage:${t}:${e}:blaxel`,provider:"blaxel",gbMonths:a.diskGb,rate:f}];for(const d of m){if(d.rate===null||d.gbMonths<=0)continue;const k=d.gbMonths*d.rate;(await v({id:d.id,at:R,kind:"storage",what:d.provider==="r2"?"object storage":"sandbox disk",units:{gb_months:d.gbMonths},usd:k,payer:t,funding:"bill",sub:a.sub,grantee:a.grantee,provider:d.provider,causedBy:null,account:b},c)).written&&(o.rows+=1)}}catch(b){o.failed+=1,E+=1,y({level:"warn",event:"ledger.storage.payer_failed",error:b,fields:{payer:t,month:e}})}}return E===0}}export{A as LEDGER_STORAGE_INTERVAL_MS,X as LedgerStorageReconciler,Q as storageSettledThrough};
+      JOIN nodes n ON n.node_id = x.node_id JOIN objects o ON o.id = n.node_id AND (o.deleted_at IS NULL OR o.deleted_at >= ?) JOIN profiles p ON p.id = n.profile_id JOIN environments e ON e.object = p.environment_id GROUP BY x.node_id`).all(i,i))h(t.node).diskGb+=Number(t.mb)/2/1024;const _=new Map;for(const[t,a]of g){const p=await G(t),u=p===null?null:await A(p);if(u===null)continue;const f=_.get(u.payer)??{sub:u.sub,grantee:u.grantee,bytes:0,diskGb:0,node:t};f.bytes+=a.bytes,f.diskGb+=a.diskGb,_.set(u.payer,f)}const x=new Date(Date.parse(i)-1);let E=0;for(const[t,a]of _){if(d&&!await d.holds())return!1;try{const{account:p,billable:u}=await I(n,t),f=[{id:`storage:${t}:${e}:r2`,provider:"r2",gbMonths:a.bytes/F,rate:c},{id:`storage:${t}:${e}:blaxel`,provider:"blaxel",gbMonths:a.diskGb,rate:m}];for(const l of f){if(l.rate===null||l.gbMonths<=0)continue;const T=l.gbMonths*l.rate;(await L({id:l.id,at:x,kind:"storage",what:l.provider==="r2"?"object storage":"sandbox disk",units:{gb_months:l.gbMonths},usd:T,payer:t,funding:"bill",sub:a.sub,grantee:a.grantee,provider:l.provider,causedBy:null,account:p},u)).written&&(o.rows+=1)}}catch(p){o.failed+=1,E+=1,y({level:"warn",event:"ledger.storage.payer_failed",error:p,fields:{payer:t,month:e}})}}return E===0}}export{U as LEDGER_STORAGE_INTERVAL_MS,te as LedgerStorageReconciler,ee as storageSettledThrough};

package/dist/types.d.ts

@@ -402,22 +402,6 @@ export interface ScopeConfig {
     paths?: {
         user_files?: string;
     };
-    /** The money ledger. `take_pct` (0-100, default 0) is Northlight's take on each priced model or tool charge; at 0 no take rows are written. */
-    ledger?: {
-        take_pct?: number;
-        /** USD per GB-hour of Blaxel sandbox awake time; the `ledger-sandbox` lane writes no rows while it is unset. */
-        sandbox_usd_per_gb_hour?: number;
-        /** Seconds a sandbox stays awake after its broker goes idle or exits (default 30). */
-        sandbox_standby_s?: number;
-        /** USD per GB-month of R2 storage; the `ledger-storage` lane writes no `r2` rows while it is unset. */
-        storage_usd_per_gb_month?: number;
-        /** USD per GB-month of Blaxel sandbox disk; the `ledger-storage` lane writes no `blaxel` rows while it is unset. */
-        sandbox_disk_usd_per_gb_month?: number;
-        /** USD of sandbox and storage a billing account uses free each UTC month (default 50); `billed_usd` on those rows is the part above it. */
-        infra_free_usd_per_month?: number;
-        /** The `ledger-r2` lane moves closed months into the blob store only while this is true (off until 5b is accepted). */
-        r2_move?: boolean;
-    };
     /** Conversation store configuration for the daemon. */
     store?: {
         root?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.566",
+  "version": "0.3.567",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -144,7 +144,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.566",
+    "@north-light/identity": "0.3.567",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.566",
+  "version": "0.3.567",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.566",
+  "version": "0.3.567",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -24,7 +24,7 @@
         "@milkdown/preset-gfm": "^7.22.3",
         "@milkdown/prose": "^7.22.3",
         "@milkdown/utils": "^7.22.3",
-        "@north-light/identity": "0.3.566",
+        "@north-light/identity": "0.3.567",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12455,17 +12455,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12473,21 +12473,21 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.566",
-        "@north-light/identity": "0.3.566"
+        "@north-light/api": "0.3.567",
+        "@north-light/identity": "0.3.567"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.566",
+      "version": "0.3.567",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.566",
-        "@north-light/identity": "0.3.566",
-        "@north-light/plugin": "0.3.566",
+        "@north-light/api": "0.3.567",
+        "@north-light/identity": "0.3.567",
+        "@north-light/plugin": "0.3.567",
         "jose": "^6.2.1"
       },
       "bin": {