@north-light/cli 0.3.464 → 0.3.465
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon/crtrd.js +11 -11
- package/dist/daemon/due.js +2 -2
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +1 -1
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +3 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -1
- package/package.json +2 -2
- package/packages/crouter-identity/package.json +1 -1
- package/runtime.lock.json +12 -12
package/dist/daemon/crtrd.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
var _e=Object.defineProperty;var s=(t,n)=>_e(t,"name",{value:n,configurable:!0});import{startPeerOutbox as ke}from"./peer/outbox.js";import{reportMachineStatus as Ee}from"../core/peer/directory.js";import{createPeerRelay as Re}from"./peer/relay-service.js";import{writeFileSync as xe,readFileSync as Pe,rmSync as Ie,mkdirSync as Z,existsSync as k,chmodSync as Te}from"node:fs";import{createServer as ee}from"node:http";import{dirname as De}from"node:path";import{envCrtrdTcp as Ce}from"../shared/env.js";import{serveCanvasConversationStore as Le}from"../core/canvas/sessions.js";import{pollRunObservers as $e}from"./runs/observer.js";import{randomBytes as Ae}from"node:crypto";import{apiSocketPath as Ne,crtrHome as Me}from"../core/canvas/paths.js";import{activeAppUsers as Oe}from"../core/spaces/users.js";import{ensureAppSpace as Ge,ensureUserFiles as Fe}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as I}from"../launcher/client.js";import{installLauncherEventSink as He}from"./launcher-events.js";import{initializeEmptyLayout as Be,layoutNeedsMigration as je,socketDir as re}from"../core/layout.js";import{migrateLooseKeys as qe}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as ze}from"../core/profiles/relocate.js";import{getNode as We,listNodesForSupervision as Ue,setFrozen as Ke}from"../core/canvas/index.js";import{createApiServer as Je}from"./api/server.js";import{createAppListener as Ve}from"./api/app-listener.js";import{installLocalTerminalGrant as Qe}from"../core/grants/local.js";import{onRunEventAppended as Ye}from"../core/runs/events.js";import{runNotices as Xe}from"./run-notices.js";import{createGrantSync as Ze,GRANT_SYNC_INTERVAL_S as er}from"../core/grants/sync.js";import{runRemovalCascades as rr}from"../core/grants/remove.js";import{sweepSandboxes as tr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as nr}from"./cron/run-socket.js";import{hasSyncedGrants as or}from"../core/grants/mirror.js";import{readConfig as ir}from"../core/config.js";import{reconcileUndeliveredTickets as sr}from"./human/sweep.js";import{resetHandedOffDeliveries as ar}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as cr}from"../core/canvas/daemon-actions.js";import{sweepReviews as dr}from"./review/sweep.js";import{pidfilePath as F}from"./pidfile.js";import{rendererWarning as lr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as pr}from"./supervise-cadence.js";import{createResumeDetector as mr}from"./resume-detector.js";import{BrokerSupervisionReconciler as fr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as ur}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as hr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as vr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as wr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as gr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as yr}from"./reconcilers/action-lane.js";import{endLiveExecution as Sr,migrateLegacyExecutionIdentities as br}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as _r}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as kr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Er}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Rr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as xr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Pr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Ir}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Tr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as te,capturePidCommand as Dr,captureTeardownSnapshot as ne,isPidAlive as Cr,isAnyPidAlive as oe,killProcessTreePids as ie}from"../core/canvas/pid.js";import{bindFleet as Lr,boundFleet as $r}from"../core/runtime/fleet.js";import{bindDaemonControl as Ar,clearDaemonControl as Nr}from"./control.js";import{execDaemon as Mr,spawnDaemon as Or}from"./manage.js";import{reconcileAndMigrateBoot as Gr}from"../core/canvas/boot.js";import{resolveInstallId as Fr}from"../core/canvas/install-id.js";import{setInstallId as Hr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Br,releaseDaemonOwnership as se}from"../core/canvas/daemon-owner.js";import{migrate as jr,openDb as ae,withCanvasWrite as qr,bindDaemonCanvasOpenHook as zr,clearDaemonCanvasOpenHook as ce}from"../core/canvas/db.js";import{brokerSignalBus as H}from"./broker-signals/bus.js";import{brokerSignalStreams as Wr}from"./api/handlers/broker-signals.js";import{reconcilePackageDocuments as Ur}from"../core/graph/package-docs.js";import{startRepoSync as Kr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Jr,reapStaleSpares as Vr}from"../core/runtime/warm-pool.js";import{DaemonFleet as Qr}from"./fleet.js";import{BOOT_MIGRATIONS as Yr}from"./boot-migrations.js";import{applyJoinedOwner as Xr}from"./joined-owner.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as b}from"../core/events/operation-id.js";import{bindDaemonEventSource as Zr}from"../core/events/source.js";import{activateCanvasImport as et,MigrationBlockedError as rt,runForcedCanvasImport as tt}from"../migrations/activation.js";import{convertLegacyProviderStore as nt,convertLegacyScopedStore as ot}from"../core/scoped-state/migrate.js";import{userScopeRoot as it}from"../core/scope.js";import{secretsDir as st}from"../core/layout.js";function B(t,n){return t.includes(` --canvas-home ${n} `)}s(B,"brokerCommandDeclaresCanvasHome");function j(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}s(j,"brokerCommandEpoch");const at=5e3,de=150;async function ct(t){const n=te();if(n===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const a=Me(),m=[];for(const[i,p]of n.processes){const y=p.command;if(p.zombie||!y.includes("broker-cli.js")||!B(y,a)||j(y)===t)continue;const S=y.trim().split(/\s+/).at(-1)??"";S===""||S.endsWith("broker-cli.js")||m.push({pid:i,nodeId:S})}let f=0;const o=[];for(const i of m){const p=Dr(i.pid);if(p===null||!p.includes("broker-cli.js")||!B(p,a)||j(p)===t||p.trim().split(/\s+/).at(-1)!==i.nodeId)continue;const y=ne(i.pid,null);if(!y.reused){try{process.platform==="linux"&&k(x)?await I({operation:"stop",pid:i.pid}):process.kill(i.pid,"SIGTERM")}catch{continue}f++,o.push({pid:i.pid,snap:y})}}const g=Date.now()+at;for(;Date.now()<g&&o.some(i=>oe(i.snap.tree.length>0?i.snap.tree:[i.pid]));)await new Promise(i=>setTimeout(i,de));let v=0;for(const i of o){const p=i.snap.tree.length>0?i.snap.tree:[i.pid];oe(p)&&(process.platform==="linux"&&k(x)?await I({operation:"stop",pid:i.pid}):ie(p,"SIGKILL",i.snap.identities),v++)}v>0&&await new Promise(i=>setTimeout(i,de));const d=te();if(d===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const l=[];for(const[i,p]of d.processes){const y=p.command;p.zombie||!y.includes("broker-cli.js")||B(y,a)&&j(y)!==t&&l.push(i)}if(l.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${l.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:f,sigkilled:v}})}s(ct,"reapPriorEpochBrokers");const dt=5e3;async function lt(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=We(o)?.status;(g==="active"||g==="idle")&&Ke(o,new Date().toISOString())}const a=n.map(([,o])=>({pid:o.pid,snap:ne(o.pid,null),exited:o.handle.exited}));for(const o of a)if(!o.snap.reused)try{process.platform==="linux"&&k(x)?await I({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const m=new Promise(o=>{setTimeout(o,dt).unref()});await Promise.allSettled(a.map(o=>Promise.race([o.exited.then(()=>{}),m])));const f=[];for(const o of a)Cr(o.pid)&&(process.platform==="linux"&&k(x)?await I({operation:"stop",pid:o.pid}):ie(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),f.push(o));await Promise.all(f.map(o=>o.exited));for(const[o]of n)Sr(o,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:a.length,sigkilled:f.length}})}s(lt,"teardownFleet");function le(){Z(De(F()),{recursive:!0}),xe(F(),String(process.pid),"utf8")}s(le,"writePidfile");function pe(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}s(pe,"isMissingPathError");function me(t){const n=F();let a;try{a=Pe(n,"utf8")}catch(m){if(pe(m))return;throw m}if(a.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{Ie(n)}catch(m){if(pe(m))return;throw m}}s(me,"removeOwnedPidfileMirror");const pt={acceptsDetachedWork:s(()=>!0,"acceptsDetachedWork"),registerDetached:s(t=>{t.catch(()=>{})},"registerDetached")},q=new fr,mt=new ur,ft=new hr,ut=new vr,ht=new wr,vt=new gr,wt=new yr,gt=new kr,yt=new Er,St=new Rr,bt=new xr,_t=new Pr,kt=new Ir;async function Et(t=Date.now(),n=pt){const a=$r(),m=await q.runFleetPreamble(a);let f,o;try{const v=Ue();f=v.filter(d=>d.status==="active"||d.status==="idle"),o=v.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(v){u({level:"error",event:"canvas.node_list.failed",error:v});return}await q.run(t,{rows:f,fleet:a,lifecycle:n}),mt.run(t,{rows:f});const g=await ft.run(t,{rows:[...f,...o],fleet:a});ut.run(t),kt.run(t),ht.run(t,{lifecycle:n}),vt.run(t,{lifecycle:n}),wt.run(t,{lifecycle:n}),gt.run(t,{lifecycle:n}),yt.run(t,{lifecycle:n,fleet:a}),St.run(t,{lifecycle:n}),$e(),_t.run(t,{fleet:a,lifecycle:n}),await bt.run(t,{rows:f}),q.warnIfSaturated({fleet:a,lifecycle:n,thresholds:m,frozen:g})}s(Et,"superviseTick");async function Rt(t){Z(re(),{recursive:!0,mode:493}),process.platform==="linux"&&Te(re(),493);const n=Ne();if(k(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const a=ee((d,l)=>{if(d.method==="GET"&&d.url==="/healthz"){l.writeHead(200,{"content-type":"application/json; charset=utf-8"}),l.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}l.writeHead(503,{"content-type":"application/json; charset=utf-8"}),l.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,l)=>{a.once("error",l),a.listen(n,()=>{a.off("error",l),d()})});let m;const f=t.tcp??Ce();if(f){const d=/^(.*):(\d+)$/.exec(f);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${f}`);const l=ee(a.listeners("request")[0]);await new Promise((i,p)=>{l.once("error",p),l.listen(Number(d[2]),d[1],()=>{l.off("error",p),i()})}),m=l}le();let o;const g=s(()=>(o??=(async()=>{a.closeAllConnections(),m?.closeAllConnections(),await Promise.all([a,m].filter(d=>d!==void 0).map(d=>new Promise((l,i)=>d.close(p=>p?i(p):l())))),me(process.pid),process.off("SIGINT",v),process.off("SIGTERM",v)})(),o),"cleanup"),v=s(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
|
|
2
|
-
`),process.exit(1)})},"onSignal");return process.on("SIGINT",v),process.on("SIGTERM",v),g}
|
|
3
|
-
`);else{const r=[n.incumbentPid!==void 0?`pid ${n.incumbentPid}`:void 0,n.verdict!==void 0?`verdict ${n.verdict}`:void 0].filter(
|
|
4
|
-
`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const
|
|
5
|
-
`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:
|
|
6
|
-
`);jr(
|
|
1
|
+
var Ee=Object.defineProperty;var a=(t,n)=>Ee(t,"name",{value:n,configurable:!0});import{startPeerOutbox as Re}from"./peer/outbox.js";import{reportMachineStatus as xe}from"../core/peer/directory.js";import{createPeerRelay as Pe}from"./peer/relay-service.js";import{writeFileSync as Ie,readFileSync as De,rmSync as Te,mkdirSync as Z,existsSync as k,chmodSync as Ce}from"node:fs";import{createServer as ee}from"node:http";import{dirname as Le}from"node:path";import{envCrtrdTcp as $e}from"../shared/env.js";import{serveCanvasConversationStore as Ae}from"../core/canvas/sessions.js";import{pollRunObservers as Ne}from"./runs/observer.js";import{randomBytes as Me}from"node:crypto";import{apiSocketPath as Oe,crtrHome as Ge}from"../core/canvas/paths.js";import{activeAppUsers as Fe}from"../core/spaces/users.js";import{ensureAppSpace as He,ensureUserFiles as Be}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as I}from"../launcher/client.js";import{installLauncherEventSink as je}from"./launcher-events.js";import{initializeEmptyLayout as qe,layoutNeedsMigration as ze,socketDir as re}from"../core/layout.js";import{migrateLooseKeys as We}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as Ue}from"../core/profiles/relocate.js";import{getNode as Ke,listNodesForSupervision as te,setFrozen as ne}from"../core/canvas/index.js";import{createApiServer as Je}from"./api/server.js";import{createAppListener as Ve}from"./api/app-listener.js";import{installLocalTerminalGrant as Qe}from"../core/grants/local.js";import{onRunEventAppended as Ye}from"../core/runs/events.js";import{runNotices as Xe}from"./run-notices.js";import{createGrantSync as Ze,GRANT_SYNC_INTERVAL_S as er}from"../core/grants/sync.js";import{runRemovalCascades as rr}from"../core/grants/remove.js";import{sweepSandboxes as tr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as nr}from"./cron/run-socket.js";import{hasSyncedGrants as or}from"../core/grants/mirror.js";import{readConfig as ir}from"../core/config.js";import{reconcileUndeliveredTickets as sr}from"./human/sweep.js";import{resetHandedOffDeliveries as ar}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as cr}from"../core/canvas/daemon-actions.js";import{sweepReviews as dr}from"./review/sweep.js";import{pidfilePath as F}from"./pidfile.js";import{rendererWarning as lr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as pr}from"./supervise-cadence.js";import{createResumeDetector as mr}from"./resume-detector.js";import{BrokerSupervisionReconciler as fr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as ur}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as hr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as vr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as wr}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as gr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as yr}from"./reconcilers/action-lane.js";import{endLiveExecution as Sr,migrateLegacyExecutionIdentities as _r}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as br}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as kr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Er}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Rr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as xr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Pr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Ir}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Dr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as oe,capturePidCommand as Tr,captureTeardownSnapshot as ie,isPidAlive as Cr,isAnyPidAlive as se,killProcessTreePids as ae}from"../core/canvas/pid.js";import{bindFleet as Lr,boundFleet as $r}from"../core/runtime/fleet.js";import{bindDaemonControl as Ar,clearDaemonControl as Nr}from"./control.js";import{execDaemon as Mr,spawnDaemon as Or}from"./manage.js";import{reconcileAndMigrateBoot as Gr}from"../core/canvas/boot.js";import{resolveInstallId as Fr}from"../core/canvas/install-id.js";import{setInstallId as Hr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Br,releaseDaemonOwnership as ce}from"../core/canvas/daemon-owner.js";import{migrate as jr,openDb as de,withCanvasWrite as qr,bindDaemonCanvasOpenHook as zr,clearDaemonCanvasOpenHook as le}from"../core/canvas/db.js";import{brokerSignalBus as H}from"./broker-signals/bus.js";import{brokerSignalStreams as Wr}from"./api/handlers/broker-signals.js";import{reconcilePackageDocuments as Ur}from"../core/graph/package-docs.js";import{startRepoSync as Kr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Jr,reapStaleSpares as Vr}from"../core/runtime/warm-pool.js";import{DaemonFleet as Qr}from"./fleet.js";import{BOOT_MIGRATIONS as Yr}from"./boot-migrations.js";import{applyJoinedOwner as Xr}from"./joined-owner.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as S}from"../core/events/operation-id.js";import{bindDaemonEventSource as Zr}from"../core/events/source.js";import{activateCanvasImport as et,MigrationBlockedError as rt,runForcedCanvasImport as tt}from"../migrations/activation.js";import{convertLegacyProviderStore as nt,convertLegacyScopedStore as ot}from"../core/scoped-state/migrate.js";import{userScopeRoot as it}from"../core/scope.js";import{secretsDir as st}from"../core/layout.js";function B(t,n){return t.includes(` --canvas-home ${n} `)}a(B,"brokerCommandDeclaresCanvasHome");function j(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}a(j,"brokerCommandEpoch");const at=5e3,pe=150;async function ct(t){const n=oe();if(n===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const c=Ge(),m=[];for(const[s,p]of n.processes){const y=p.command;if(p.zombie||!y.includes("broker-cli.js")||!B(y,c)||j(y)===t)continue;const _=y.trim().split(/\s+/).at(-1)??"";_===""||_.endsWith("broker-cli.js")||m.push({pid:s,nodeId:_})}let f=0;const o=[];for(const s of m){const p=Tr(s.pid);if(p===null||!p.includes("broker-cli.js")||!B(p,c)||j(p)===t||p.trim().split(/\s+/).at(-1)!==s.nodeId)continue;const y=ie(s.pid,null);if(!y.reused){try{process.platform==="linux"&&k(x)?await I({operation:"stop",pid:s.pid}):process.kill(s.pid,"SIGTERM")}catch{continue}f++,o.push({pid:s.pid,snap:y})}}const g=Date.now()+at;for(;Date.now()<g&&o.some(s=>se(s.snap.tree.length>0?s.snap.tree:[s.pid]));)await new Promise(s=>setTimeout(s,pe));let v=0;for(const s of o){const p=s.snap.tree.length>0?s.snap.tree:[s.pid];se(p)&&(process.platform==="linux"&&k(x)?await I({operation:"stop",pid:s.pid}):ae(p,"SIGKILL",s.snap.identities),v++)}v>0&&await new Promise(s=>setTimeout(s,pe));const d=oe();if(d===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const l=[];for(const[s,p]of d.processes){const y=p.command;p.zombie||!y.includes("broker-cli.js")||B(y,c)&&j(y)!==t&&l.push(s)}if(l.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${l.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:f,sigkilled:v}})}a(ct,"reapPriorEpochBrokers");const dt=5e3;async function lt(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=Ke(o)?.status;(g==="active"||g==="idle")&&ne(o,new Date().toISOString())}const c=n.map(([,o])=>({pid:o.pid,snap:ie(o.pid,null),exited:o.handle.exited}));for(const o of c)if(!o.snap.reused)try{process.platform==="linux"&&k(x)?await I({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const m=new Promise(o=>{setTimeout(o,dt).unref()});await Promise.allSettled(c.map(o=>Promise.race([o.exited.then(()=>{}),m])));const f=[];for(const o of c)Cr(o.pid)&&(process.platform==="linux"&&k(x)?await I({operation:"stop",pid:o.pid}):ae(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),f.push(o));await Promise.all(f.map(o=>o.exited));for(const[o]of n)Sr(o,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:c.length,sigkilled:f.length}})}a(lt,"teardownFleet");function me(){Z(Le(F()),{recursive:!0}),Ie(F(),String(process.pid),"utf8")}a(me,"writePidfile");function fe(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}a(fe,"isMissingPathError");function ue(t){const n=F();let c;try{c=De(n,"utf8")}catch(m){if(fe(m))return;throw m}if(c.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{Te(n)}catch(m){if(fe(m))return;throw m}}a(ue,"removeOwnedPidfileMirror");const pt={acceptsDetachedWork:a(()=>!0,"acceptsDetachedWork"),registerDetached:a(t=>{t.catch(()=>{})},"registerDetached")},q=new fr,mt=new ur,ft=new hr,ut=new vr,ht=new wr,vt=new gr,wt=new yr,gt=new kr,yt=new Er,St=new Rr,_t=new xr,bt=new Pr,kt=new Ir;async function Et(t=Date.now(),n=pt){const c=$r(),m=await q.runFleetPreamble(c);let f,o;try{const v=te();f=v.filter(d=>d.status==="active"||d.status==="idle"),o=v.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(v){u({level:"error",event:"canvas.node_list.failed",error:v});return}await q.run(t,{rows:f,fleet:c,lifecycle:n}),mt.run(t,{rows:f});const g=await ft.run(t,{rows:[...f,...o],fleet:c});ut.run(t),kt.run(t),ht.run(t,{lifecycle:n}),vt.run(t,{lifecycle:n}),wt.run(t,{lifecycle:n}),gt.run(t,{lifecycle:n}),yt.run(t,{lifecycle:n,fleet:c}),St.run(t,{lifecycle:n}),Ne(),bt.run(t,{fleet:c,lifecycle:n}),await _t.run(t,{rows:f}),q.warnIfSaturated({fleet:c,lifecycle:n,thresholds:m,frozen:g})}a(Et,"superviseTick");async function Rt(t){Z(re(),{recursive:!0,mode:493}),process.platform==="linux"&&Ce(re(),493);const n=Oe();if(k(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const c=ee((d,l)=>{if(d.method==="GET"&&d.url==="/healthz"){l.writeHead(200,{"content-type":"application/json; charset=utf-8"}),l.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}l.writeHead(503,{"content-type":"application/json; charset=utf-8"}),l.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,l)=>{c.once("error",l),c.listen(n,()=>{c.off("error",l),d()})});let m;const f=t.tcp??$e();if(f){const d=/^(.*):(\d+)$/.exec(f);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${f}`);const l=ee(c.listeners("request")[0]);await new Promise((s,p)=>{l.once("error",p),l.listen(Number(d[2]),d[1],()=>{l.off("error",p),s()})}),m=l}me();let o;const g=a(()=>(o??=(async()=>{c.closeAllConnections(),m?.closeAllConnections(),await Promise.all([c,m].filter(d=>d!==void 0).map(d=>new Promise((l,s)=>d.close(p=>p?s(p):l())))),ue(process.pid),process.off("SIGINT",v),process.off("SIGTERM",v)})(),o),"cleanup"),v=a(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
|
|
2
|
+
`),process.exit(1)})},"onSignal");return process.on("SIGINT",v),process.on("SIGTERM",v),g}a(Rt,"serveLayoutRefusal");async function xt(t={}){if(ze())return Rt(t);qe();const n=Br(Me(16).toString("hex"));if(!n.claimed){if(n.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${n.incumbentPid??"?"})
|
|
3
|
+
`);else{const r=[n.incumbentPid!==void 0?`pid ${n.incumbentPid}`:void 0,n.verdict!==void 0?`verdict ${n.verdict}`:void 0].filter(i=>i!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${n.reason}${e}
|
|
4
|
+
`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const c=n.claim;let m=null;try{await ct(c.token),ot({kind:"user",root:it()});for(const e of Ue())process.stderr.write(`[crtrd] ${e}
|
|
5
|
+
`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:i}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:w}=await import("node:os");i("user",X=>{X.paths??={},X.paths.user_files??=w()})}}await nt(st());for(const e of We())process.stderr.write(`[crtrd] ${e}
|
|
6
|
+
`);jr(de()),zr(e=>H.install(e));const r=qr(e=>Ur(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
|
|
7
7
|
`);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
|
|
8
8
|
`);for(const e of r.quarantined)process.stderr.write(`[crtrd] plugin "${e.name}" has a damaged ${e.file} (${e.reason}); ${e.movedTo!==void 0?`moved to ${e.movedTo}`:"left in place and skipped"}
|
|
9
|
-
`);m=Xr(),process.platform==="linux"&&k(x)&&(
|
|
9
|
+
`);m=Xr(),process.platform==="linux"&&k(x)&&(je(),await I({operation:"users.sync",users:Fe()})),br(c.token)}catch(r){le(),H.close();try{const{released:e}=ce(c);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let f=!0,o,g=Promise.resolve(),v=!1,d,l;const s=new Set;let p=!1,y=!1,_,$,z=!1,A,D,T,C,M;const he=mr();let E,h={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},P="initial startup activation",ve=null;function we(){return h.phase==="ready"?null:{phase:h.phase,message:h.message,blockerPaths:h.blockerPaths}}a(we,"startupBlock");function b(r){s.add(r),r.finally(()=>{s.delete(r)}).catch(()=>{})}a(b,"registerDetached");const W={acceptsDetachedWork:a(()=>f,"acceptsDetachedWork"),registerDetached:b};Kr(W,de);async function ge(){try{f=!1,Wr.close(),le(),H.close(),d!==void 0&&(clearInterval(d),d=void 0),T!==void 0&&(clearInterval(T),T=void 0),M?.(),M=void 0;let r;do r=g,await r;while(r!==g);for(;s.size>0;)await Promise.allSettled([...s]);if(l!==void 0&&(await lt(l),l.dispose()),$!==void 0&&!z&&(z=!0,await $.close()),C!==void 0){const e=C;C=void 0,await e.stop()}if(D!==void 0){const e=D;D=void 0,await e.close()}if(A!==void 0){const e=A;A=void 0,await e.close()}if(p||(ue(process.pid),p=!0),p&&!y){const{released:e}=ce(c);if(!e)throw new Error("daemon ownership claim was not released");y=!0}}finally{Nr(),process.off("SIGINT",J),process.off("SIGTERM",V),process.off("exit",Q)}}a(ge,"cleanupAttempt");const N=1e3;let U=!1;function ye(){return U?{graceMs:N}:(U=!0,S.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:N}})}),setTimeout(()=>{Se()},N),{graceMs:N})}a(ye,"requestHandover");async function Se(){try{await R()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
|
|
10
10
|
${e}
|
|
11
|
-
`),process.exit(1)}try{process.platform==="linux"&&k(x)&&(await I({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Mr();const r=await Or({waitForJoinLock:!0});
|
|
12
|
-
`),process.exit(1)}process.exit(0)}
|
|
13
|
-
${
|
|
14
|
-
`),process.exit(1)}}
|
|
11
|
+
`),process.exit(1)}try{process.platform==="linux"&&k(x)&&(await I({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Mr();const r=await Or({waitForJoinLock:!0});S.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){S.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
|
|
12
|
+
`),process.exit(1)}process.exit(0)}a(Se,"performHandover");function R(){if(_!==void 0)return _;const r=ge();return _=r,r.then(()=>{_===r&&(_=void 0)},()=>{_===r&&(_=void 0)}),r}a(R,"requestCleanup");async function K(r){try{await R(),process.exit(0)}catch(e){process.exitCode=1;const i=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
|
|
13
|
+
${i}
|
|
14
|
+
`),process.exit(1)}}a(K,"handleSignal");function J(){K("SIGINT")}a(J,"onSigint");function V(){K("SIGTERM")}a(V,"onSigterm");function Q(){R().catch(()=>{})}a(Q,"onExit");function O(r){f&&(g=g.then(r).then(()=>{},e=>{S.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:e})})}))}a(O,"enqueue");function Y(){!f||v||(he.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),b(E().catch(()=>{}))),v=!0,O(async()=>{if(v=!1,!!f)try{await Et(Date.now(),W)}catch(r){S.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}a(Y,"enqueueTick");let L=null;async function _e(){if(L!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${L instanceof Error?L.message:String(L)}`};if(h.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(h.phase!=="prepared"&&h.phase!=="recovering")throw{status:409,code:"not_prepared",message:h.message};const r=h.phase==="prepared";try{return await G(),h={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw L=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}a(_e,"admitStartup");async function be(r){if(P!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${P})`};P="repair";try{const e=await tt({dryRun:r.dryRun});if(r.dryRun||e.pending||h.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await G(),h={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(i){const w=i instanceof Error?i.message:String(i);return setTimeout(()=>{R().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:w}}}finally{P=null}}a(be,"repairMigrations");async function ke(){o=t.intervalMs??pr;const r=c.token;l=new Qr({epoch:r,enqueue:O}),Lr(l),h={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},S.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:o}})});const e=Fr();if(Hr(e),S.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),S.fresh(()=>{let i;try{i=lr()}catch{return}i!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:i.slice(0,2048)}})}),await S.fresh(async()=>{await dr(),await sr(),ar(Date.now()),cr(Date.now())}),S.fresh(()=>{const{reconcile:i,migrated:w}=Gr(_r);i.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:i.cleared.length,same_boot_proven:i.sameBootProven}}),w.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:w.length}})}),S.fresh(()=>{for(const i of te())i.status!=="active"||i.pi_pid!=null||i.frozen_at!=null||i.respawn_not_before!=null||(ne(i.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:i.node_id}))}),S.fresh(()=>{Dr();for(const i of Yr){const w=i.apply();w.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:i.id,...w.summary}})}}),b(nr()),E){const i=a(()=>{b(E().catch(()=>{})),b(xe().then(w=>C?.observe(w)).then(()=>{},w=>{u({level:"warn",event:"daemon.peer.status_failed",error:w})}))},"refresh");b(rr().catch(()=>{})),b(tr().then(({completion:w})=>w).catch(w=>{u({level:"error",event:"daemon.grants.sweep_failed",error:w})})),i(),T=setInterval(i,er*1e3),T.unref(),M=Re({joined:!0,register:b})}O(()=>{Vr(),Jr()}),d=setInterval(Y,o),Y()}a(ke,"completeStartup");function G(){return ve??=ke().catch(r=>{throw h={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}a(G,"resumeStartup");try{process.on("SIGINT",J),process.on("SIGTERM",V),process.on("exit",Q),Zr(),m!==null&&u({level:"info",event:"daemon.joined_owner.applied",fields:{owner:m.owner,rows:m.rows}}),A=await Ae();const r=c.token;Ar({epoch:r,requestHandover:ye,registerDetached:b,startupBlock:we,admitStartup:_e,repairMigrations:be}),me(),$=Je({tcp:t.tcp}),await $.ready;const e=ir("user").runtime;e?.issuer?(E=Ze(e.issuer,b),e.unsandboxed===!0&&(C=Pe({syncGrants:E})),Ye(i=>Xe().appended(i))):Qe(),e?.unsandboxed===!0&&e.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),e!==void 0&&e.listener!==void 0&&e.unsandboxed!==!0&&(D=Ve(e,()=>or(e.issuer.replace(/\/$/,"")),E),await D.ready);try{await et()}catch(i){if(P=null,i instanceof rt)return h={phase:"blocked",message:i.message,blockerPaths:i.blockerPaths},R;throw i}process.platform==="linux"&&k(x)&&(Be(),await He("app:terminal"));try{t.prepare?h={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await G(),h={phase:"ready",message:"",blockerPaths:[]})}finally{P=null}return R}catch(r){try{await R()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}a(xt,"runDaemon");var eo=xt;export{B as brokerCommandDeclaresCanvasHome,j as brokerCommandEpoch,eo as default,xt as runDaemon,Et as superviseTick,lt as teardownFleet};
|
package/dist/daemon/due.js
CHANGED
|
@@ -4,8 +4,8 @@ var T=Object.defineProperty;var r=(n,e)=>T(n,"name",{value:e,configurable:!0});i
|
|
|
4
4
|
WHERE deadline_at IS NOT NULL AND outcome_at IS NULL AND status IN ('active', 'idle') AND kind != 'human'`).filter(e=>!p(e.profile_id)).map(e=>s(e.deadline_at))),"nextDue")}),o("structuredRequests",{nextDue:r(e=>a("SELECT 1 FROM structured_requests WHERE state != 'pending' AND effects_done_at IS NULL LIMIT 1")!==void 0?e:a("SELECT MIN(deadline_at) AS at FROM structured_requests WHERE state = 'pending' AND deadline_at IS NOT NULL")?.at??null,"nextDue")}),o("cronLane",{nextDue:r(()=>{const e=_(`SELECT fire_at, profile FROM crons
|
|
5
5
|
WHERE state = 'active' AND run_state = 'idle'
|
|
6
6
|
AND NOT (held = 1 AND recur IS NULL AND expires_at IS NULL)`).filter(i=>!p(i.profile)).map(i=>s(i.fire_at)),t=_("SELECT expires_at FROM crons WHERE expires_at IS NOT NULL AND run_state = 'idle'").map(i=>s(i.expires_at));return c([...e,...t])},"nextDue")});const n=r(e=>()=>a(`SELECT MIN(next_attempt_at) AS at FROM ${e} WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at IS NOT NULL`)?.at??null,"attempts");o("deliveryLane",{nextDue:n("deliveries")}),o("actionLane",{nextDue:n("daemon_actions")}),o("pendingReviewSubmit",{nextDue:r(e=>O().length>0?e:null,"nextDue")}),o("storageDueWork",{nextDue:r(e=>{if(A()===null)return null;if(a("SELECT 1 FROM uploads WHERE delete_pending_since IS NOT NULL LIMIT 1")!==void 0||a("SELECT 1 FROM shares WHERE status = 'off' AND object_deleted_at IS NULL LIMIT 1")!==void 0)return e;const t=a("SELECT MIN(expires_at) AS at FROM shares WHERE status = 'on' AND expires_at IS NOT NULL")?.at,i=a(`SELECT MIN(created_at) AS at FROM uploads u WHERE delete_pending_since IS NULL
|
|
7
|
-
AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`)?.at,u=s(i);return c([s(t),u===null?null:u+h])},"nextDue")}),o("faultRetryBackstop",{nextDue:r(()=>c(I().map(e=>b(e))),"nextDue")})}r(D,"installDueSources");const R=12e4;function
|
|
7
|
+
AND NOT EXISTS (SELECT 1 FROM upload_uses x WHERE x.upload_id = u.upload_id)`)?.at,u=s(i);return c([s(t),u===null?null:u+h])},"nextDue")}),o("faultRetryBackstop",{nextDue:r(()=>c(I().map(e=>b(e))),"nextDue")})}r(D,"installDueSources");const R=12e4;function v(n){const e=_(`SELECT n.node_id, n.status, n.intent, n.profile_id, n.pi_pid, n.frozen_at, n.respawn_not_before, n.created, n.launched_at,
|
|
8
8
|
n.turn_open_at, n.turn_closed_at,
|
|
9
9
|
(SELECT x.started_at FROM executions x WHERE x.node_id = n.node_id AND x.state != 'exited') AS exec_started_at
|
|
10
10
|
FROM node_rows n
|
|
11
|
-
WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${m}`);for(const t of e){if(p(t.profile_id)||!N({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&x(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=s(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<R||L(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null)return!0;const u=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>s(d)??0));if(u>0&&n-u<R||t.status==="idle"&&L(t.node_id))return!0}return!1}r(
|
|
11
|
+
WHERE n.status IN ('active', 'idle') AND n.kind != 'human' AND n.${m}`);for(const t of e){if(p(t.profile_id)||!N({node_id:t.node_id,status:t.status,intent:t.intent}))continue;if(t.pi_pid!==null&&x(t.pi_pid)){if(t.turn_open_at!==null&&t.turn_closed_at===null)return!0;const d=s(t.exec_started_at);if(t.turn_open_at===null&&d!==null&&n-d<R||L(t.node_id))return!0;continue}if(t.respawn_not_before!==null)continue;if(t.pi_pid!==null||t.frozen_at!==null||t.status==="active")return!0;const u=Math.max(...[t.created,t.launched_at,t.exec_started_at].map(d=>s(d)??0));if(u>0&&n-u<R||t.status==="idle"&&L(t.node_id))return!0}return!1}r(v,"nodesOweWork");function X(n=Date.now()){D();const{dueNow:e,nextDueMs:t}=w(n,(u,d)=>{M({level:"warn",event:"daemon.due.source_failed",error:d,fields:{lane:u}})});return{busy:e||v(n)||a("SELECT 1 FROM crons WHERE run_state = 'running' LIMIT 1")!==void 0||a("SELECT 1 FROM jobs j JOIN objects o ON o.rowid = j.object WHERE j.status = 'running' AND o.deleted_at IS NULL LIMIT 1")!==void 0,next_due_at:t===null?null:new Date(t).toISOString()}}r(X,"dueReport");function K(){return D(),[...U(),...g()]}r(K,"coveredLanes");export{R as START_GRACE_MS,K as coveredLanes,X as dueReport,D as installDueSources};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
var
|
|
1
|
+
var E=Object.defineProperty;var d=(i,t)=>E(i,"name",{value:t,configurable:!0});import{clearFrozen as m,getNode as _,getRow as g}from"../../../core/canvas/index.js";import{fullName as y}from"../../../core/canvas/labels.js";import{isSafeNodeId as s}from"../../../core/canvas/paths.js";import{emitEvent as l}from"../../../core/events/emit.js";import{operationIdContext as A}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as S}from"../../../core/canvas/deliveries.js";import{publishNotice as k}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{reviveNode as h}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as z}from"../../../core/canvas/recovery.js";const f=6,R=10*6e4,L=2e4;function M(i,t){if(t)return 2;const r=i.launched_at;if(r==null)return 1;const e=i.last_settled_at;return e==null?2:Date.parse(e)>Date.parse(r)?1:2}d(M,"thawTier");function C(i,t){if(i.tier!==t.tier)return i.tier-t.tier;const r=i.row.frozen_at,e=t.row.frozen_at;return r==null?e==null?0:1:e==null?-1:Date.parse(r)-Date.parse(e)}d(C,"compareThaw");class B{static{d(this,"FreezeLaneReconciler")}relaunchState=new Map;endEpisode(t){this.relaunchState.delete(t)}async run(t,r){const e=r.candidates.map(a=>({...a,tier:M(a.row,v(a.row.node_id))}));e.sort(C);const n={stalled:0,waitingOnMail:0};for(const a of e)await A.fresh(()=>this.enact(t,a,r.fleet))==="frozen"&&(a.tier===1?n.stalled+=1:n.waitingOnMail+=1);return n}async enact(t,r,e){const n=r.row.node_id;try{return r.enact==="thaw"?await this.enactThaw(r.row):r.enact==="idle-release"?await this.enactIdleRelease(t,r.row):(await e.applyDeadRowPolicy(n),e.has(n)?"launched":g(n)?.frozen_at!=null?"frozen":"skipped")}catch(a){return l({level:"error",event:"node.lifecycle.enact_failed",...s(n)?{node_id:n}:{},error:a,fields:{...s(n)?{}:{affected_node_id:n},enact:r.enact}}),"skipped"}}async enactThaw(t){if(t.lifecycle==="resident"&&_(t.node_id)?.pi_session_id!=null&&!v(t.node_id))return u(t.node_id,"release"),m(t.node_id),"skipped";const r=z(t.node_id)?.interrupted===!0;try{return(await h(t.node_id,{resume:!0,capacity:"freeze",recovery:!0,...r?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(e){const n=_(t.node_id);throw n!==null&&(n.status==="active"||n.status==="idle")&&(u(t.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:e instanceof Error?e.message:String(e)}}}),m(t.node_id)),e}}async enactIdleRelease(t,r){const e=r.node_id,n=S(e),a=this.relaunchState.get(e),c=a===void 0||a.acknowledgedCount!==n,o=c?0:a.attempts;if(o>=f)return this.relaunchState.delete(e),l({level:"warn",event:"broker.idle_release.exhausted",...s(e)?{node_id:e}:{},fields:{...s(e)?{}:{affected_node_id:e},attempts:o,max_attempts:f}}),u(e,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),k(e,"notice.crash",`Child crashed \u2014 ${y(r)} (${e}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,{reason:"child-crashed",child:e}),"skipped";const p=c?void 0:a?.lastAttempt,w=Math.min(L*2**o,R);return p!==void 0&&t-p<w?"skipped":(await h(e,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":(this.relaunchState.set(e,{acknowledgedCount:n,attempts:o+1,lastAttempt:t}),l({level:"info",event:"broker.revive.idle_release",...s(e)?{node_id:e}:{},attempt:o+1,fields:{...s(e)?{}:{affected_node_id:e},attempts:o+1,max_attempts:f}}),"launched")}}export{B as FreezeLaneReconciler,C as compareThaw,M as thawTier};
|
|
@@ -7,6 +7,9 @@ export interface NodeLifecycleContext {
|
|
|
7
7
|
rows: readonly NodeRow[];
|
|
8
8
|
fleet: FleetRegistry;
|
|
9
9
|
}
|
|
10
|
+
/** How long an active row with no engine may be a birth in flight rather than stranded. */
|
|
11
|
+
export declare const LAUNCH_GRACE_MS = 120000;
|
|
12
|
+
export declare function isLaunchInFlight(row: Pick<NodeRow, 'created' | 'launched_at'>, now: number): boolean;
|
|
10
13
|
export declare class NodeLifecycleReconciler {
|
|
11
14
|
private readonly freezeLane;
|
|
12
15
|
run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
var
|
|
1
|
+
var c=Object.defineProperty;var d=(r,i)=>c(r,"name",{value:i,configurable:!0});import{clearFrozen as m,endLiveExecution as l,getNode as h}from"../../../core/canvas/index.js";import{isSafeNodeId as _}from"../../../core/canvas/paths.js";import{recordedPidLiveness as o}from"../../../core/canvas/pid.js";import{emitEvent as p}from"../../../core/events/emit.js";import{operationIdContext as y}from"../../../core/events/operation-id.js";import{readInterruptedTurn as P}from"../../../core/canvas/recovery.js";import{isProfilePaused as u}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as k}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as L}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as z}from"../../fleet.js";import{applyParkEvent as I,hasPendingWake as R,parkEvent as E}from"../live-obligation.js";import{FreezeLaneReconciler as M}from"./freeze-lane.js";import{isInBackoff as g}from"./respawn-policy.js";import{handleTerminatingRow as N}from"./terminating.js";import{wantsExecution as b}from"./wants-execution.js";function C(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}d(C,"isTerminal");const D=12e4;function F(r,i){const e=Math.max(...[r.created,r.launched_at].map(n=>n==null?0:Date.parse(n)||0));return e>0&&i-e<D}d(F,"isLaunchInFlight");class Y{static{d(this,"NodeLifecycleReconciler")}freezeLane=new M;async run(i,e){const n=[];for(const a of e.rows)await y.fresh(async()=>{try{await this.classifyRow(i,a,e,n)}catch(t){const s=a.node_id;p({level:"error",event:"node.lifecycle.classify_failed",..._(s)?{node_id:s}:{fields:{affected_node_id:s}},error:t})}});return this.freezeLane.run(i,{fleet:e.fleet,candidates:n})}async classifyRow(i,e,n,a){const t=e.node_id;if(C(e)){if(N(e,n.fleet.has(t)))return;e.pi_pid!=null&&o(e.pi_pid,e.pi_pid_identity)==="dead"&&l(t,{reason:"observed-dead"}),e.frozen_at!=null&&a.push({row:e,enact:"thaw"});return}if(n.fleet.has(t)){u(e.profile_id)?n.fleet.interruptForProfilePause(t):this.thawMark(e);return}if(u(e.profile_id)){this.freezeLane.endEpisode(t);return}if(v(t))return;const s=e.pi_pid!=null&&o(e.pi_pid,e.pi_pid_identity)==="dead";if(s&&!n.fleet.hasProfilePauseInterruption(t)&&l(t,{reason:"observed-dead"}),!b(e)){this.thawMark(e),this.settleParked(e);return}if(g(e,i))return;if(e.status==="idle"&&L(e)&&k(t)){a.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null){if(e.respawn_not_before!=null){a.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){a.push({row:e,enact:"thaw"});return}if(n.fleet.isClaimed(t)||F(e,i))return;p({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),a.push({row:e,enact:"thaw"});return}if(!s)return;const f=z(h(t),P(t)?.interrupted===!0,R(t));if(f==="respawn-fresh"||f==="respawn-cycle"||f==="respawn-resume"){a.push({row:e,enact:"policy"});return}await n.fleet.applyDeadRowPolicy(t),this.thawMark(e)}thawMark(i){i.frozen_at!=null&&m(i.node_id)}settleParked(i){if(i.status!=="idle")return;this.freezeLane.endEpisode(i.node_id);const e=E(i.node_id,i.lifecycle);e!=="release"&&I(i.node_id,e)}}export{D as LAUNCH_GRACE_MS,Y as NodeLifecycleReconciler,F as isLaunchInFlight};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.465",
|
|
4
4
|
"description": "crtr — agent runtime with memory, plugins, and marketplaces",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -137,7 +137,7 @@
|
|
|
137
137
|
},
|
|
138
138
|
"license": "UNLICENSED",
|
|
139
139
|
"dependencies": {
|
|
140
|
-
"@north-light/identity": "0.3.
|
|
140
|
+
"@north-light/identity": "0.3.465",
|
|
141
141
|
"@earendil-works/pi-agent-core": "1.1.0",
|
|
142
142
|
"@earendil-works/pi-ai": "1.1.0",
|
|
143
143
|
"@earendil-works/pi-coding-agent": "1.1.0",
|
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.465",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/cli",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.465",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "UNLICENSED",
|
|
12
12
|
"workspaces": [
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"@earendil-works/pi-ai": "1.1.0",
|
|
19
19
|
"@earendil-works/pi-coding-agent": "1.1.0",
|
|
20
20
|
"@earendil-works/pi-tui": "1.1.0",
|
|
21
|
-
"@north-light/identity": "0.3.
|
|
21
|
+
"@north-light/identity": "0.3.465",
|
|
22
22
|
"cron-parser": "^5.6.0",
|
|
23
23
|
"esbuild": "^0.27.7",
|
|
24
24
|
"jose": "^6.2.12",
|
|
@@ -12095,17 +12095,17 @@
|
|
|
12095
12095
|
},
|
|
12096
12096
|
"packages/crouter-api": {
|
|
12097
12097
|
"name": "@north-light/api",
|
|
12098
|
-
"version": "0.3.
|
|
12098
|
+
"version": "0.3.465",
|
|
12099
12099
|
"license": "UNLICENSED"
|
|
12100
12100
|
},
|
|
12101
12101
|
"packages/crouter-env-docker": {
|
|
12102
12102
|
"name": "@north-light/env-docker",
|
|
12103
|
-
"version": "0.3.
|
|
12103
|
+
"version": "0.3.465",
|
|
12104
12104
|
"license": "UNLICENSED"
|
|
12105
12105
|
},
|
|
12106
12106
|
"packages/crouter-identity": {
|
|
12107
12107
|
"name": "@north-light/identity",
|
|
12108
|
-
"version": "0.3.
|
|
12108
|
+
"version": "0.3.465",
|
|
12109
12109
|
"license": "UNLICENSED",
|
|
12110
12110
|
"dependencies": {
|
|
12111
12111
|
"jose": "^6.2.1"
|
|
@@ -12113,20 +12113,20 @@
|
|
|
12113
12113
|
},
|
|
12114
12114
|
"packages/crouter-plugin": {
|
|
12115
12115
|
"name": "@north-light/plugin",
|
|
12116
|
-
"version": "0.3.
|
|
12116
|
+
"version": "0.3.465",
|
|
12117
12117
|
"license": "UNLICENSED",
|
|
12118
12118
|
"dependencies": {
|
|
12119
|
-
"@north-light/api": "0.3.
|
|
12120
|
-
"@north-light/identity": "0.3.
|
|
12119
|
+
"@north-light/api": "0.3.465",
|
|
12120
|
+
"@north-light/identity": "0.3.465"
|
|
12121
12121
|
}
|
|
12122
12122
|
},
|
|
12123
12123
|
"packages/crouter-sdk": {
|
|
12124
12124
|
"name": "@north-light/sdk",
|
|
12125
|
-
"version": "0.3.
|
|
12125
|
+
"version": "0.3.465",
|
|
12126
12126
|
"license": "UNLICENSED",
|
|
12127
12127
|
"dependencies": {
|
|
12128
|
-
"@north-light/api": "0.3.
|
|
12129
|
-
"@north-light/identity": "0.3.
|
|
12128
|
+
"@north-light/api": "0.3.465",
|
|
12129
|
+
"@north-light/identity": "0.3.465",
|
|
12130
12130
|
"jose": "^6.2.1"
|
|
12131
12131
|
},
|
|
12132
12132
|
"bin": {
|