@north-light/cli 0.3.575 → 0.3.577

package/dist/daemon/api/handlers/providers.js

@@ -1 +1 @@
-var I=Object.defineProperty;var s=(e,r)=>I(e,"name",{value:r,configurable:!0});import{randomUUID as R}from"node:crypto";import{cloudPersonScope as $}from"../../../core/runs/operations.js";import{directoryIssuer as O}from"../../../core/directory/issuer.js";import{activeGrantForApp as q,grantFor as G,localGrants as _,providerIdsFromGrants as N}from"../../../core/grants/mirror.js";import{getNode as T}from"../../../core/canvas/canvas.js";import{scopeAllowed as F}from"../../../core/scopes.js";import{ProviderError as f,providerErrorBody as x,scopeMissing as j}from"../../../core/providers/errors.js";import{callProvider as k,providerTools as C}from"../../../core/providers/call.js";import{providerInstallState as B}from"../../../core/providers/bundles.js";import{providerRootClaims as D,resolveProviderRef as M}from"../../../core/providers/roots.js";import{listInstalledPlugins as U}from"../../../core/resolver.js";import{listAudit as z,writeProviderAudit as L}from"../../../core/providers/audit.js";import{emitEvent as g}from"../../../core/events/emit.js";import{resolvedScopes as y}from"../router.js";const _e=new Set(["provider.tools","provider.call","audit.list"]);async function V(e){if(e.class==="forwarded"){if(e.actor.kind==="person")return{class:"person",sub:null,grantee:null,client_id:null,resolved:null,nodeScopes:null,run_id:null,node_id:null,origin:{node:null,run:null,machine:e.via}};const r=await G(e.actor.app,e.sub);if(!r)throw new f("grant_revoked",`app ${e.actor.app} has no active grant`);return{class:"app",sub:e.sub,grantee:e.actor.app,client_id:null,resolved:r.resolved,nodeScopes:e.actor.node_scopes,run_id:null,node_id:null,origin:{node:e.actor.node,run:e.actor.place?.run??null,machine:e.via}}}if(e.class==="machine")throw new f("permission_error","this operation is not available to the caller");if(e.class==="person"&&e.sub!==void 0)throw new f("permission_error","provider calls are made by the install's person, a node or an app");if(e.class==="person")return{class:"person",sub:null,grantee:null,client_id:null,resolved:null,nodeScopes:null,run_id:null,node_id:null};if(e.class==="app"){const r=await q(e.app);if(!r)throw new f("grant_revoked",`app ${e.app} has no active grant`);return{class:"app",sub:r.sub,grantee:e.app,client_id:null,resolved:r.resolved,nodeScopes:e.scopes,run_id:null,node_id:null}}return e.class==="cron"?{class:"app",sub:e.sub,grantee:e.app,client_id:null,resolved:await y(e),nodeScopes:e.scopes,run_id:e.origin_run,node_id:null}:{class:"broker",sub:e.sub,grantee:e.app,client_id:e.client_id,resolved:await y(e),nodeScopes:e.scopes,run_id:e.run??(await T(e.node))?.run_id??null,node_id:e.node}}s(V,"callerFromRequest");function ge(e){return{class:"app",sub:e.sub,grantee:e.grantee,client_id:e.client_id,resolved:e.resolved,nodeScopes:null,run_id:null,node_id:null}}s(ge,"callerFromApp");const Z=/^([A-Za-z0-9_-]{1,64}):([A-Za-z0-9_-]{1,64})$/;function W(e){const r=Z.exec(e);return r!==null&&r[1]!=="crtr"}s(W,"isProviderScope");const S=new WeakMap;function J(e){let r=S.get(e);return r===void 0&&(r=new Set(e),S.set(e,r)),r}s(J,"heldSet");function A(e,r){return e===null?!0:W(r)?J(e).has(r):F(e,r)}s(A,"holds");function P(e,r){return e.class==="person"?!0:A(e.resolved??[],r)&&A(e.nodeScopes,r)}s(P,"callerAllows");function ye(e,r,t){const o=`${r}:${t}`;if(!P(e,o))throw j(o)}s(ye,"requireGroup");function X(){return!!O()}s(X,"providerOperationsEnabled");const H=/^[A-Za-z0-9_-]{1,64}$/;function m(e,r){return new f("invalid_request",e,{param:r})}s(m,"invalid");function E(e){if(typeof e!="string"||!H.test(e)||e==="crtr")throw m("provider must be a provider id","provider");return e}s(E,"parseProvider");function K(e){if(!e||typeof e!="object"||Array.isArray(e))throw new f("invalid_request","a JSON object is required");const r=e,t=E(r.provider),o=r.tool;if(typeof o!="string"||!o.trim())throw m("tool must be a non-empty string","tool");const i=r.arguments??{};if(!i||typeof i!="object"||Array.isArray(i))throw m("arguments must be an object","arguments");const d=r.version;if(d!==void 0&&(typeof d!="number"||!Number.isInteger(d)||d<1))throw m("version must be an integer >= 1","version");const a=r.approval;if(a!==void 0&&(typeof a!="string"||!a))throw m("approval must be a question id","approval");const l=r.machine;if(l!==void 0&&(typeof l!="string"||!l.trim()))throw m("machine must be a machine id","machine");return{provider:t,tool:o.trim(),arguments:i,...d===void 0?{}:{version:d},...a===void 0?{}:{approval:a},...l===void 0?{}:{machine:l.trim()}}}s(K,"parseCallInput");async function Q(e,r=new URLSearchParams){const t=s((u,v,w)=>{const c=r.get(u);if(c===null)return v;if(!/^\d+$/.test(c)||Number(c)>w||Number(c)<1)throw m(`invalid ${u}`,u);return Number(c)},"numeric"),o=r.has("after")?t("after",0,Number.MAX_SAFE_INTEGER):void 0,i=t("limit",100,1e3),d=Object.fromEntries(["operation","outcome","run_id","since","until"].filter(u=>r.has(u)).map(u=>[u,r.get(u)])),a=$(e.sub),l=a===void 0?{}:{sub:a};return{status:200,body:await z(e.class==="person"?null:e.grantee,{after:o,limit:i,...l,...d})}}s(Q,"auditList");async function Y(e,r,t){let o=r;const i=performance.now(),d=U("user"),{skipped:a}=D(d),l=new Set(d.filter(n=>n.enabled&&n.manifest.provider?.id===n.name&&!a.has(n.name)).map(n=>n.name)),u=N(await _()).filter(n=>!a.has(n)),v=[...new Set([...l,...u])].sort();if(o!==void 0){const n=M(o);if(n===null)throw new f("not_found",`provider "${o}" is not available`);o=n}if(o!==void 0&&!v.includes(o))throw new f("not_found",`provider "${o}" is not available`);const w=o===void 0?v:[o],c=e.class==="person"?(await _()).flatMap(n=>n.status==="active"?n.resolved_scopes.map(({scope:p})=>p):[]):e.resolved??[],h=await Promise.all(w.map(async n=>{if(l.has(n))return C(e,n);if(!c.some(b=>b.startsWith(`${n}:`)&&P(e,b)))return null;const p=B(n);return{provider:n,display_name:n,status:p.status,...p.error?{error:p.error}:{},groups:[],commands:[]}})).then(n=>n.filter(p=>p!==null));if(g({level:"debug",event:"api.provider.tools",duration_ms:performance.now()-i,fields:{caller:e.class,provider:o??null,providers:h.length}}),o!==void 0){if(h.length===0)throw new f("not_found",`provider "${o}" is not available`);return{status:200,body:h[0]}}return{status:200,body:{providers:h}}}s(Y,"providerTools");async function ee(e,r,t){return k(e,r,t)}s(ee,"providerCall");async function re(e,r,t,o){if(!X())throw new f("not_found","provider operations need a directory-connected runtime");if(e==="audit.list")return await Q(r,t.query);if(e==="provider.tools")return Y(r,t.params.provider===void 0?void 0:E(t.params.provider),o);if(e==="provider.call"){let i;try{i=K(t.body)}catch(d){const a=t.body&&typeof t.body=="object"&&!Array.isArray(t.body)?t.body:{};throw await L({request_id:o.requestId,caller:r,provider:typeof a.provider=="string"?a.provider:"",tool:typeof a.tool=="string"?a.tool:"",outcome:`refused:${d instanceof f?d.code:"invalid_request"}`}),d}return ee(r,i,o)}throw new f("not_found","unknown operation")}s(re,"handleProviderOperation");function oe(e){const r=new AbortController,t=s(()=>{e.writableFinished||r.abort()},"abort");return e.on("close",t),{signal:r.signal,release:s(()=>e.off("close",t),"release")}}s(oe,"abortOnClose");function te(e,r,t,o){const i=x(e,r,t,o??void 0);return i.status>=500&&!(e instanceof f)&&g({event:"api.provider.failed",level:"error",error:e instanceof Error?e:new Error(String(e)),fields:{request_id:r}}),i}s(te,"providerFailure");const Se=[{method:"GET",pattern:"/v1/audit",name:"audit.list"},{method:"GET",pattern:"/v1/providers/tools",name:"provider.tools"},{method:"GET",pattern:"/v1/providers/:provider/tools",name:"provider.tools"},{method:"POST",pattern:"/v1/provider/call",name:"provider.call"}].map(({method:e,pattern:r,name:t})=>({method:e,pattern:r,handler:s(async o=>{const i=R(),{signal:d,release:a}=oe(o.res);let l=null;try{return l=await V(o.principal),{...await re(t,l,{params:o.params,body:o.body,query:new URL(o.req.url??"/","http://runtime.local").searchParams},{requestId:i,signal:d,...o.causedBy===void 0?{}:{causedBy:o.causedBy}}),headers:{"Request-Id":i}}}catch(u){return te(u,i,!1,l?.run_id)}finally{a()}},"handler")}));export{_e as PROVIDER_OPERATIONS,oe as abortOnClose,P as callerAllows,ge as callerFromApp,V as callerFromRequest,re as handleProviderOperation,K as parseCallInput,E as parseProvider,te as providerFailure,X as providerOperationsEnabled,Se as providerRoutes,ye as requireGroup};
+var q=Object.defineProperty;var s=(e,r)=>q(e,"name",{value:r,configurable:!0});import{randomUUID as G}from"node:crypto";import{cloudPersonScope as N}from"../../../core/runs/operations.js";import{directoryIssuer as T}from"../../../core/directory/issuer.js";import{activeGrantForApp as F,allGrants as x,grantFor as C,localGrants as j,providerIdsFromGrants as y}from"../../../core/grants/mirror.js";import{getNode as k}from"../../../core/canvas/canvas.js";import{scopeAllowed as B}from"../../../core/scopes.js";import{ProviderError as f,providerErrorBody as D,scopeMissing as M}from"../../../core/providers/errors.js";import{callProvider as U,providerTools as z}from"../../../core/providers/call.js";import{providerInstallState as L}from"../../../core/providers/bundles.js";import{installedProviderPlugins as V,providerRootClaims as Z,resolveProviderRef as W}from"../../../core/providers/roots.js";import{isCloudRole as h}from"../../../core/canvas/person-state.js";import{listAudit as J,writeProviderAudit as X}from"../../../core/providers/audit.js";import{emitEvent as S}from"../../../core/events/emit.js";import{resolvedScopes as P}from"../router.js";const Ae=new Set(["provider.tools","provider.call","audit.list"]);async function H(e){if(e.class==="forwarded"){if(e.actor.kind==="person")return{class:"person",sub:null,grantee:null,client_id:null,resolved:null,nodeScopes:null,run_id:null,node_id:null,origin:{node:null,run:null,machine:e.via}};const r=await C(e.actor.app,e.sub);if(!r)throw new f("grant_revoked",`app ${e.actor.app} has no active grant`);return{class:"app",sub:e.sub,grantee:e.actor.app,client_id:null,resolved:r.resolved,nodeScopes:e.actor.node_scopes,run_id:null,node_id:null,origin:{node:e.actor.node,run:e.actor.place?.run??null,machine:e.via}}}if(e.class==="machine")throw new f("permission_error","this operation is not available to the caller");if(e.class==="person"&&e.sub!==void 0)throw new f("permission_error","provider calls are made by the install's person, a node or an app");if(e.class==="person")return{class:"person",sub:null,grantee:null,client_id:null,resolved:null,nodeScopes:null,run_id:null,node_id:null};if(e.class==="app"){const r=await F(e.app);if(!r)throw new f("grant_revoked",`app ${e.app} has no active grant`);return{class:"app",sub:r.sub,grantee:e.app,client_id:null,resolved:r.resolved,nodeScopes:e.scopes,run_id:null,node_id:null}}return e.class==="cron"?{class:"app",sub:e.sub,grantee:e.app,client_id:null,resolved:await P(e),nodeScopes:e.scopes,run_id:e.origin_run,node_id:null}:{class:"broker",sub:e.sub,grantee:e.app,client_id:e.client_id,resolved:await P(e),nodeScopes:e.scopes,run_id:e.run??(await k(e.node))?.run_id??null,node_id:e.node}}s(H,"callerFromRequest");function Ee(e){return{class:"app",sub:e.sub,grantee:e.grantee,client_id:e.client_id,resolved:e.resolved,nodeScopes:null,run_id:null,node_id:null}}s(Ee,"callerFromApp");const K=/^([A-Za-z0-9_-]{1,64}):([A-Za-z0-9_-]{1,64})$/;function Q(e){const r=K.exec(e);return r!==null&&r[1]!=="crtr"}s(Q,"isProviderScope");const A=new WeakMap;function Y(e){let r=A.get(e);return r===void 0&&(r=new Set(e),A.set(e,r)),r}s(Y,"heldSet");function E(e,r){return e===null?!0:Q(r)?Y(e).has(r):B(e,r)}s(E,"holds");function R(e,r){return e.class==="person"?!0:E(e.resolved??[],r)&&E(e.nodeScopes,r)}s(R,"callerAllows");function Re(e,r,n){const o=`${r}:${n}`;if(!R(e,o))throw M(o)}s(Re,"requireGroup");function ee(){return!!T()}s(ee,"providerOperationsEnabled");const re=/^[A-Za-z0-9_-]{1,64}$/;function m(e,r){return new f("invalid_request",e,{param:r})}s(m,"invalid");function I(e){if(typeof e!="string"||!re.test(e)||e==="crtr")throw m("provider must be a provider id","provider");return e}s(I,"parseProvider");function oe(e){if(!e||typeof e!="object"||Array.isArray(e))throw new f("invalid_request","a JSON object is required");const r=e,n=I(r.provider),o=r.tool;if(typeof o!="string"||!o.trim())throw m("tool must be a non-empty string","tool");const i=r.arguments??{};if(!i||typeof i!="object"||Array.isArray(i))throw m("arguments must be an object","arguments");const a=r.version;if(a!==void 0&&(typeof a!="number"||!Number.isInteger(a)||a<1))throw m("version must be an integer >= 1","version");const d=r.approval;if(d!==void 0&&(typeof d!="string"||!d))throw m("approval must be a question id","approval");const u=r.machine;if(u!==void 0&&(typeof u!="string"||!u.trim()))throw m("machine must be a machine id","machine");return{provider:n,tool:o.trim(),arguments:i,...a===void 0?{}:{version:a},...d===void 0?{}:{approval:d},...u===void 0?{}:{machine:u.trim()}}}s(oe,"parseCallInput");async function te(e,r=new URLSearchParams){const n=s((l,v,w)=>{const p=r.get(l);if(p===null)return v;if(!/^\d+$/.test(p)||Number(p)>w||Number(p)<1)throw m(`invalid ${l}`,l);return Number(p)},"numeric"),o=r.has("after")?n("after",0,Number.MAX_SAFE_INTEGER):void 0,i=n("limit",100,1e3),a=Object.fromEntries(["operation","outcome","run_id","since","until"].filter(l=>r.has(l)).map(l=>[l,r.get(l)])),d=N(e.sub),u=d===void 0?{}:{sub:d};return{status:200,body:await J(e.class==="person"?null:e.grantee,{after:o,limit:i,...u,...a})}}s(te,"auditList");async function ne(e,r,n){let o=r;const i=performance.now(),a=V(),{skipped:d}=Z(a),u=h()?await x():await j(),l=h()?new Set(y(u.filter(t=>t.sub===e.sub))):null,v=s(t=>!d.has(t)&&(l===null||l.has(t)),"shown"),w=new Set(a.filter(t=>t.enabled&&t.manifest.provider?.id===t.name&&v(t.name)).map(t=>t.name)),p=y(u).filter(v),_=[...new Set([...w,...p])].sort();if(o!==void 0){const t=W(o);if(t===null)throw new f("not_found",`provider "${o}" is not available`);o=t}if(o!==void 0&&!_.includes(o))throw new f("not_found",`provider "${o}" is not available`);const $=o===void 0?_:[o],O=e.class==="person"?u.flatMap(t=>t.status==="active"&&(!h()||t.sub===e.sub)?t.resolved_scopes.map(({scope:c})=>c):[]):e.resolved??[],b=await Promise.all($.map(async t=>{if(w.has(t))return z(e,t);if(!O.some(g=>g.startsWith(`${t}:`)&&R(e,g)))return null;const c=L(t);return c===void 0?null:{provider:t,display_name:t,status:c.status,...c.error?{error:c.error}:{},groups:[],commands:[]}})).then(t=>t.filter(c=>c!==null));if(S({level:"debug",event:"api.provider.tools",duration_ms:performance.now()-i,fields:{caller:e.class,provider:o??null,providers:b.length}}),o!==void 0){if(b.length===0)throw new f("not_found",`provider "${o}" is not available`);return{status:200,body:b[0]}}return{status:200,body:{providers:b}}}s(ne,"providerTools");async function se(e,r,n){return U(e,r,n)}s(se,"providerCall");async function ie(e,r,n,o){if(!ee())throw new f("not_found","provider operations need a directory-connected runtime");if(e==="audit.list")return await te(r,n.query);if(e==="provider.tools")return ne(r,n.params.provider===void 0?void 0:I(n.params.provider),o);if(e==="provider.call"){let i;try{i=oe(n.body)}catch(a){const d=n.body&&typeof n.body=="object"&&!Array.isArray(n.body)?n.body:{};throw await X({request_id:o.requestId,caller:r,provider:typeof d.provider=="string"?d.provider:"",tool:typeof d.tool=="string"?d.tool:"",outcome:`refused:${a instanceof f?a.code:"invalid_request"}`}),a}return se(r,i,o)}throw new f("not_found","unknown operation")}s(ie,"handleProviderOperation");function ae(e){const r=new AbortController,n=s(()=>{e.writableFinished||r.abort()},"abort");return e.on("close",n),{signal:r.signal,release:s(()=>e.off("close",n),"release")}}s(ae,"abortOnClose");function de(e,r,n,o){const i=D(e,r,n,o??void 0);return i.status>=500&&!(e instanceof f)&&S({event:"api.provider.failed",level:"error",error:e instanceof Error?e:new Error(String(e)),fields:{request_id:r}}),i}s(de,"providerFailure");const Ie=[{method:"GET",pattern:"/v1/audit",name:"audit.list"},{method:"GET",pattern:"/v1/providers/tools",name:"provider.tools"},{method:"GET",pattern:"/v1/providers/:provider/tools",name:"provider.tools"},{method:"POST",pattern:"/v1/provider/call",name:"provider.call"}].map(({method:e,pattern:r,name:n})=>({method:e,pattern:r,handler:s(async o=>{const i=G(),{signal:a,release:d}=ae(o.res);let u=null;try{return u=await H(o.principal),{...await ie(n,u,{params:o.params,body:o.body,query:new URL(o.req.url??"/","http://runtime.local").searchParams},{requestId:i,signal:a,...o.causedBy===void 0?{}:{causedBy:o.causedBy}}),headers:{"Request-Id":i}}}catch(l){return de(l,i,!1,u?.run_id)}finally{d()}},"handler")}));export{Ae as PROVIDER_OPERATIONS,ae as abortOnClose,R as callerAllows,Ee as callerFromApp,H as callerFromRequest,ie as handleProviderOperation,oe as parseCallInput,I as parseProvider,de as providerFailure,ee as providerOperationsEnabled,Ie as providerRoutes,Re as requireGroup};

package/dist/daemon/crtrd.js

@@ -1,18 +1,18 @@
-var er=Object.defineProperty;var i=(e,l)=>er(e,"name",{value:l,configurable:!0});import{applyBootSettings as rr}from"./boot-settings.js";import{startPeerOutbox as tr}from"./peer/outbox.js";import{reportMachineStatus as nr}from"../core/peer/directory.js";import{createPeerRelay as or}from"./peer/relay-service.js";import{writeFileSync as ir,readFileSync as ar,rmSync as sr,mkdirSync as ye,existsSync as b,unlinkSync as _e,chmodSync as be}from"node:fs";import{createServer as J}from"node:http";import{dirname as cr}from"node:path";import{envCrtrdTcp as ke,envDocumentPort as lr}from"../shared/env.js";import{serveCanvasConversationStore as dr}from"../core/canvas/sessions.js";import{pollRunObservers as pr}from"./runs/observer.js";import{randomBytes as Re}from"node:crypto";import{apiSocketPath as Pe,crtrHome as mr}from"../core/canvas/paths.js";import{activeAppUsers as fr}from"../core/spaces/users.js";import{ensureAppSpace as ur,ensureUserFiles as wr}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as L}from"../launcher/client.js";import{installLauncherEventSink as hr}from"./launcher-events.js";import{initializeEmptyLayout as vr,layoutNeedsMigration as gr,socketDir as Ee}from"../core/layout.js";import{migrateLooseKeys as Sr}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as yr}from"../core/profiles/relocate.js";import{getNode as _r,listNodesForSupervision as Ie,listParkedResidentsWithWakeMail as br,setFrozen as xe}from"../core/canvas/index.js";import{createApiServer as kr,closeServer as Te}from"./api/server.js";import{Router as Rr}from"./api/router.js";import{healthRoutes as Pr}from"./api/handlers/health.js";import{createDocumentListener as Er}from"./api/document-listener.js";import{documentSecret as Ir}from"./api/document-secret.js";import{createAppListener as xr}from"./api/app-listener.js";import{registerMachineEnvironment as Tr}from"../core/environments.js";import{registerBlaxelRunnerFromEnv as Dr}from"../core/secrets/blaxel-register.js";import{markCloudRole as Cr,resolvePerson as Lr,retireLocalPerson as $r,setPersonId as Ar}from"../core/canvas/person.js";import{backfillNodeApps as Nr,runPermissionsBoot as Mr}from"../core/permissions/boot.js";import{startRunNotices as Gr}from"./run-notices.js";import{startGrantEffects as Or}from"./reconcilers/grant-effects.js";import{createGrantSync as Fr,GRANT_SYNC_INTERVAL_S as Br}from"../core/grants/sync.js";import{sweepSandboxes as jr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as zr}from"./cron/run-socket.js";import{hasSyncedGrants as Hr}from"../core/grants/mirror.js";import{readConfig as se}from"../core/config.js";import{reconcileUndeliveredTickets as Ur}from"./human/sweep.js";import{resetHandedOffDeliveries as qr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Wr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Jr}from"./review/sweep.js";import{pidfilePath as ce}from"./pidfile.js";import{rendererWarning as Kr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Vr}from"./supervise-cadence.js";import{createResumeDetector as Yr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Qr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as Xr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as Zr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as et}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as rt}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as tt}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as nt}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as ot}from"./reconcilers/action-lane.js";import{endLiveExecution as it,migrateLegacyExecutionIdentities as at}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as st}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as ct}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as lt}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as dt}from"./reconcilers/storage-due-work.js";import{assertLedgerEnv as pt}from"../core/ledger/rates.js";import{LedgerSandboxReconciler as mt}from"./reconcilers/ledger-sandbox.js";import{LedgerStorageReconciler as ft}from"./reconcilers/ledger-storage.js";import{LedgerR2Reconciler as ut}from"./reconcilers/ledger-r2.js";import{BashDeadlineReconciler as wt}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as ht}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as vt}from"./reconcilers/structured-request-lane.js";import{RecipeImageReconciler as gt,subscribeImageBuilt as St}from"./reconcilers/recipe-image.js";import{reconcileStructuredRequestSidecars as yt}from"./structured-requests/boot.js";import{captureLivenessSnapshot as De,capturePidCommand as _t,captureTeardownSnapshot as Ce,isPidAlive as bt,isAnyPidAlive as Le,killProcessTreePids as $e}from"../core/canvas/pid.js";import{bindFleet as kt,boundFleet as Rt}from"../core/runtime/fleet.js";import{bindDaemonControl as Pt,clearDaemonControl as Et}from"./control.js";import{execDaemon as It,spawnDaemon as xt}from"./manage.js";import{reconcileAndMigrateBoot as Tt}from"../core/canvas/boot.js";import{resolveInstallId as Dt}from"../core/canvas/install-id.js";import{setInstallId as Ct}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Lt,daemonOwnershipApplies as $t,daemonOwnershipClaimed as At,releaseDaemonOwnership as Ae}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as Nt,closeAllDbs as le,openDb as Ne,withCanvasWrite as Mt}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as Gt}from"../core/canvas/import-sqlite.js";import{ensurePostgres as Ot,ensurePostgresHealthy as Ft,stopPostgres as Bt,withPostgresStartLock as jt}from"./postgres/index.js";import{brokerSignalBus as F}from"./broker-signals/bus.js";import{brokerSignalStreams as zt}from"./api/handlers/broker-signals.js";import{setRequestsStartRunObservers as Ht}from"./api/handlers/broker-ops.js";import{reconcilePackages as Ut}from"../core/graph/package-reconcile.js";import{startRepoSync as qt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Wt,reapStaleSpares as Jt}from"../core/runtime/warm-pool.js";import{DaemonFleet as Kt}from"./fleet.js";import{copyNodeFilesToRows as Vt}from"../core/runtime/first-boot-rows.js";import{BOOT_MIGRATIONS as Yt}from"./boot-migrations.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as Qt}from"../core/events/source.js";import{activateCanvasImport as Xt,MigrationBlockedError as Zt,runForcedCanvasImport as en}from"../migrations/activation.js";import{adoptInstallPerson as rn,convertLegacyScopedStore as tn,personStoresCopied as nn}from"../core/scoped-state/migrate.js";import{readLegacyUserSnapshot as on}from"../core/scoped-state/legacy-profiles.js";import{heldPersons as Me,installPreludeSnapshot as an}from"../core/scoped-state/person-store.js";import{loadPersonSettings as Ge}from"../core/scoped-state/person-rows.js";import{userScopeRoot as Oe}from"../core/scope.js";function de(e,l){return e.includes(` --canvas-home ${l} `)}i(de,"brokerCommandDeclaresCanvasHome");function pe(e){return/ --epoch (\S+) /.exec(e)?.[1]??null}i(pe,"brokerCommandEpoch");const sn=5e3,Fe=150;async function cn(e){const l=De();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const f=mr(),m=[];for(const[n,d]of l.processes){const y=d.command;if(d.zombie||!y.includes("broker-cli.js")||!de(y,f)||pe(y)===e)continue;const k=y.trim().split(/\s+/).at(-1)??"";k===""||k.endsWith("broker-cli.js")||m.push({pid:n,nodeId:k})}let v=0;const a=[];for(const n of m){const d=_t(n.pid);if(d===null||!d.includes("broker-cli.js")||!de(d,f)||pe(d)===e||d.trim().split(/\s+/).at(-1)!==n.nodeId)continue;const y=Ce(n.pid,null);if(!y.reused){try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{continue}v++,a.push({pid:n.pid,snap:y})}}const w=Date.now()+sn;for(;Date.now()<w&&a.some(n=>Le(n.snap.tree.length>0?n.snap.tree:[n.pid]));)await new Promise(n=>setTimeout(n,Fe));let g=0;for(const n of a){const d=n.snap.tree.length>0?n.snap.tree:[n.pid];Le(d)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):$e(d,"SIGKILL",n.snap.identities),g++)}g>0&&await new Promise(n=>setTimeout(n,Fe));const p=De();if(p===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[n,d]of p.processes){const y=d.command;d.zombie||!y.includes("broker-cli.js")||de(y,f)&&pe(y)!==e&&c.push(n)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:v,sigkilled:g}})}i(cn,"reapPriorEpochBrokers");const ln=5e3;async function dn(e){const l=Array.from(e.entries());if(l.length===0)return;for(const[a]of l){const w=(await _r(a))?.status;(w==="active"||w==="idle")&&await xe(a,new Date().toISOString())}const f=l.map(([,a])=>({pid:a.pid,snap:Ce(a.pid,null),exited:a.handle.exited}));for(const a of f)if(!a.snap.reused)try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{}const m=new Promise(a=>{setTimeout(a,ln).unref()});await Promise.allSettled(f.map(a=>Promise.race([a.exited.then(()=>{}),m])));const v=[];for(const a of f)bt(a.pid)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):$e(a.snap.tree.length>0?a.snap.tree:[a.pid],"SIGKILL",a.snap.identities),v.push(a));await Promise.all(v.map(a=>a.exited));for(const[a]of l)await it(a,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:f.length,sigkilled:v.length}})}i(dn,"teardownFleet");function Be(){ye(cr(ce()),{recursive:!0}),ir(ce(),String(process.pid),"utf8")}i(Be,"writePidfile");function je(e){return typeof e=="object"&&e!==null&&e.code==="ENOENT"}i(je,"isMissingPathError");function ze(e){const l=ce();let f;try{f=ar(l,"utf8")}catch(m){if(je(m))return;throw m}if(f.trim()!==String(e))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${e})`);try{sr(l)}catch(m){if(je(m))return;throw m}}i(ze,"removeOwnedPidfileMirror");const pn={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(e=>{e.catch(()=>{})},"registerDetached")},K=new Xr;Qr(e=>K.handleFatalFault(e,!1));const mn=new Zr,fn=new et,un=new rt,wn=new tt,hn=new nt,vn=new ot,gn=new ct,Sn=new lt,yn=new dt,_n=new mt,bn=new ft,kn=new ut,Rn=new wt,Pn=new ht,En=new vt,In=new gt;async function xn(e=Date.now(),l=pn){const f=Rt(),m=await K.runFleetPreamble(f);let v,a;try{const g=await Ie();v=g.filter(c=>c.status==="active"||c.status==="idle"),a=g.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const p=new Set(a.map(c=>c.node_id));a.push(...(await br()).filter(c=>!p.has(c.node_id)))}catch(g){u({level:"error",event:"canvas.node_list.failed",error:g});return}await K.run(e,{rows:v,fleet:f,lifecycle:l}),await mn.run(e,{rows:v});const w=await fn.run(e,{rows:[...v,...a],fleet:f});await un.run(e),await En.run(e),await In.run(e,{lifecycle:l}),await wn.run(e,{lifecycle:l}),await hn.run(e,{lifecycle:l}),await vn.run(e,{lifecycle:l}),await gn.run(e,{lifecycle:l}),await Sn.run(e,{lifecycle:l,fleet:f}),yn.run(e,{lifecycle:l}),_n.run(e,{lifecycle:l}),bn.run(e,{lifecycle:l}),kn.run(e,{lifecycle:l}),await pr(),Pn.run(e,{fleet:f,lifecycle:l}),await Rn.run(e,{rows:v}),K.warnIfSaturated({fleet:f,lifecycle:l,thresholds:m,frozen:w})}i(xn,"superviseTick");function V(e){return e!=="api"}i(V,"roleRunsLanes");function $(e){return e!=="reconcile"}i($,"roleServesRequests");const Tn=66e4;function Dn(e){const l=new Rr().registerAll(Pr(()=>null).filter(n=>n.pattern==="/healthz")),f=i((n,d)=>{const y=(n.url??"/").split("?")[0];if((n.method??"GET").toUpperCase()==="GET"&&y==="/healthz"){l.handle(n,d,null,{class:"person"});return}d.writeHead(404,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"not_found",message:"this process serves only /healthz",origin:"runtime"}}))},"handler"),m=Pe();b(m)&&_e(m);const v=J(f),a=new Promise((n,d)=>{v.once("error",d),v.listen(m,()=>{v.off("error",d),be(m,384),n()})});let w,g=null;const p=e.tcp??ke()??se("user").api.tcp;if(p!==void 0&&p!==""){const n=/^(.*):(\d+)$/.exec(p.trim());if(n===null||Number(n[2])>65535)throw new Error(`invalid reconcile probe TCP address: ${p}`);const d=J(f);d.on("error",y=>u({event:"api.server.failed",level:"error",error:y})),d.listen(Number(n[2]),n[1]===""?"0.0.0.0":n[1],()=>{g=`${n[1]===""?"0.0.0.0":n[1]}:${n[2]}`}),w=d}let c=!1;return{ready:a,tcpAddress:i(()=>g,"tcpAddress"),async close(n={}){if(!c){c=!0,await Promise.all([Te(v,new Set,n.graceMs),w===void 0?Promise.resolve():Te(w,new Set,n.graceMs)]);try{b(m)&&_e(m)}catch{}}}}}i(Dn,"createProbeServer");const Cn="daemon is draining for shutdown and admits no new work";async function Ln(e){ye(Ee(),{recursive:!0,mode:493}),process.platform==="linux"&&be(Ee(),493);const l=Pe();if(b(l))throw new Error(`layout refusal cannot bind occupied socket: ${l}`);const f=J((p,c)=>{if(p.method==="GET"&&p.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((p,c)=>{f.once("error",c),f.listen(l,()=>{f.off("error",c),p()})});let m;const v=e.tcp??ke();if(v){const p=/^(.*):(\d+)$/.exec(v);if(!p||Number(p[2])>65535)throw new Error(`invalid refusal TCP address: ${v}`);const c=J(f.listeners("request")[0]);await new Promise((n,d)=>{c.once("error",d),c.listen(Number(p[2]),p[1],()=>{c.off("error",d),n()})}),m=c}Be();let a;const w=i(()=>(a??=(async()=>{f.closeAllConnections(),m?.closeAllConnections(),await Promise.all([f,m].filter(p=>p!==void 0).map(p=>new Promise((c,n)=>p.close(d=>d?n(d):c())))),ze(process.pid),process.off("SIGINT",g),process.off("SIGTERM",g)})(),a),"cleanup"),g=i(()=>{w().then(()=>process.exit(0),p=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(p)}
-`),process.exit(1)})},"onSignal");return process.on("SIGINT",g),process.on("SIGTERM",g),w}i(Ln,"serveLayoutRefusal");async function $n(e={}){if(gr())return Ln(e);vr();const l=i(r=>{process.stderr.write(`[crtrd] ${r}
-`)},"logPostgres"),f=!!process.env.CRTR_TEST_PG_URL,m=$t(e.role);Ht(V(e.role));const v=i(async()=>m?await Lt(Re(16).toString("hex")):{claimed:!0,claim:{pid:process.pid,pid_identity:"",token:Re(16).toString("hex")}},"claimOwnership"),{database:a,ownership:w}=f?{database:{url:await Nt()},ownership:await v()}:await jt(async()=>({database:await Ot({log:l,startLockHeld:!0}),ownership:await v()}));if(!w.claimed){if(w.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${w.incumbentPid??"?"})
+var er=Object.defineProperty;var i=(e,l)=>er(e,"name",{value:l,configurable:!0});import{applyBootSettings as rr}from"./boot-settings.js";import{startPeerOutbox as tr}from"./peer/outbox.js";import{reportMachineStatus as nr}from"../core/peer/directory.js";import{createPeerRelay as or}from"./peer/relay-service.js";import{writeFileSync as ir,readFileSync as ar,rmSync as sr,mkdirSync as ye,existsSync as b,unlinkSync as _e,chmodSync as be}from"node:fs";import{createServer as J}from"node:http";import{dirname as cr}from"node:path";import{envCrtrdTcp as ke,envDocumentPort as lr}from"../shared/env.js";import{openCanvasConversationStore as dr,serveCanvasConversationStore as pr}from"../core/canvas/sessions.js";import{installConversationStore as mr}from"../core/conversation-store/index.js";import{pollRunObservers as fr}from"./runs/observer.js";import{randomBytes as Re}from"node:crypto";import{apiSocketPath as Pe,crtrHome as ur}from"../core/canvas/paths.js";import{activeAppUsers as wr}from"../core/spaces/users.js";import{ensureAppSpace as hr,ensureUserFiles as vr}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as L}from"../launcher/client.js";import{installLauncherEventSink as gr}from"./launcher-events.js";import{initializeEmptyLayout as Sr,layoutNeedsMigration as yr,socketDir as Ee}from"../core/layout.js";import{migrateLooseKeys as _r}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as br}from"../core/profiles/relocate.js";import{getNode as kr,listNodesForSupervision as Ie,listParkedResidentsWithWakeMail as Rr,setFrozen as xe}from"../core/canvas/index.js";import{createApiServer as Pr,closeServer as Te}from"./api/server.js";import{Router as Er}from"./api/router.js";import{healthRoutes as Ir}from"./api/handlers/health.js";import{createDocumentListener as xr}from"./api/document-listener.js";import{documentSecret as Tr}from"./api/document-secret.js";import{createAppListener as Cr}from"./api/app-listener.js";import{registerMachineEnvironment as Dr}from"../core/environments.js";import{registerBlaxelRunnerFromEnv as Lr}from"../core/secrets/blaxel-register.js";import{markCloudRole as $r,resolvePerson as Ar,retireLocalPerson as Nr,setPersonId as Mr}from"../core/canvas/person.js";import{backfillNodeApps as Gr,runPermissionsBoot as Or}from"../core/permissions/boot.js";import{startRunNotices as Fr}from"./run-notices.js";import{startGrantEffects as Br}from"./reconcilers/grant-effects.js";import{createGrantSync as jr,GRANT_SYNC_INTERVAL_S as zr}from"../core/grants/sync.js";import{sweepSandboxes as Hr}from"../core/grants/on-sync.js";import{rebuildCronExecutions as Ur}from"./cron/run-socket.js";import{hasSyncedGrants as qr}from"../core/grants/mirror.js";import{readConfig as se}from"../core/config.js";import{reconcileUndeliveredTickets as Wr}from"./human/sweep.js";import{resetHandedOffDeliveries as Jr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as Kr}from"../core/canvas/daemon-actions.js";import{sweepReviews as Vr}from"./review/sweep.js";import{pidfilePath as ce}from"./pidfile.js";import{rendererWarning as Yr}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as Qr}from"./supervise-cadence.js";import{createResumeDetector as Xr}from"./resume-detector.js";import{setFatalFaultAnnouncer as Zr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as et}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as rt}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as tt}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as nt}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as ot}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as it}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as at}from"./reconcilers/action-lane.js";import{endLiveExecution as st,migrateLegacyExecutionIdentities as ct}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as lt}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as dt}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as pt}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as mt}from"./reconcilers/storage-due-work.js";import{assertLedgerEnv as ft}from"../core/ledger/rates.js";import{LedgerSandboxReconciler as ut}from"./reconcilers/ledger-sandbox.js";import{LedgerStorageReconciler as wt}from"./reconcilers/ledger-storage.js";import{LedgerR2Reconciler as ht}from"./reconcilers/ledger-r2.js";import{BashDeadlineReconciler as vt}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as gt}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as St}from"./reconcilers/structured-request-lane.js";import{RecipeImageReconciler as yt,subscribeImageBuilt as _t}from"./reconcilers/recipe-image.js";import{reconcileStructuredRequestSidecars as bt}from"./structured-requests/boot.js";import{captureLivenessSnapshot as Ce,capturePidCommand as kt,captureTeardownSnapshot as De,isPidAlive as Rt,isAnyPidAlive as Le,killProcessTreePids as $e}from"../core/canvas/pid.js";import{bindFleet as Pt,boundFleet as Et}from"../core/runtime/fleet.js";import{bindDaemonControl as It,clearDaemonControl as xt}from"./control.js";import{execDaemon as Tt,spawnDaemon as Ct}from"./manage.js";import{reconcileAndMigrateBoot as Dt}from"../core/canvas/boot.js";import{resolveInstallId as Lt}from"../core/canvas/install-id.js";import{setInstallId as $t}from"../core/runtime/nodes.js";import{claimDaemonOwnership as At,daemonOwnershipApplies as Nt,daemonOwnershipClaimed as Mt,releaseDaemonOwnership as Ae}from"../core/canvas/daemon-owner.js";import{canvasDatabaseUrl as Gt,closeAllDbs as le,openDb as Ne,withCanvasWrite as Ot}from"../core/canvas/db.js";import{importSqliteCanvasIfPresent as Ft}from"../core/canvas/import-sqlite.js";import{ensurePostgres as Bt,ensurePostgresHealthy as jt,stopPostgres as zt,withPostgresStartLock as Ht}from"./postgres/index.js";import{brokerSignalBus as F}from"./broker-signals/bus.js";import{brokerSignalStreams as Ut}from"./api/handlers/broker-signals.js";import{setRequestsStartRunObservers as qt}from"./api/handlers/broker-ops.js";import{reconcilePackages as Wt}from"../core/graph/package-reconcile.js";import{startRepoSync as Jt}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Kt,reapStaleSpares as Vt}from"../core/runtime/warm-pool.js";import{DaemonFleet as Yt}from"./fleet.js";import{copyNodeFilesToRows as Qt}from"../core/runtime/first-boot-rows.js";import{BOOT_MIGRATIONS as Xt}from"./boot-migrations.js";import{emitEvent as u}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as Zt}from"../core/events/source.js";import{activateCanvasImport as en,MigrationBlockedError as rn,runForcedCanvasImport as tn}from"../migrations/activation.js";import{adoptInstallPerson as nn,convertLegacyScopedStore as on,personStoresCopied as an}from"../core/scoped-state/migrate.js";import{readLegacyUserSnapshot as sn}from"../core/scoped-state/legacy-profiles.js";import{heldPersons as Me,installPreludeSnapshot as cn}from"../core/scoped-state/person-store.js";import{loadPersonSettings as Ge}from"../core/scoped-state/person-rows.js";import{userScopeRoot as Oe}from"../core/scope.js";function de(e,l){return e.includes(` --canvas-home ${l} `)}i(de,"brokerCommandDeclaresCanvasHome");function pe(e){return/ --epoch (\S+) /.exec(e)?.[1]??null}i(pe,"brokerCommandEpoch");const ln=5e3,Fe=150;async function dn(e){const l=Ce();if(l===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const f=ur(),m=[];for(const[n,d]of l.processes){const y=d.command;if(d.zombie||!y.includes("broker-cli.js")||!de(y,f)||pe(y)===e)continue;const k=y.trim().split(/\s+/).at(-1)??"";k===""||k.endsWith("broker-cli.js")||m.push({pid:n,nodeId:k})}let v=0;const a=[];for(const n of m){const d=kt(n.pid);if(d===null||!d.includes("broker-cli.js")||!de(d,f)||pe(d)===e||d.trim().split(/\s+/).at(-1)!==n.nodeId)continue;const y=De(n.pid,null);if(!y.reused){try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):process.kill(n.pid,"SIGTERM")}catch{continue}v++,a.push({pid:n.pid,snap:y})}}const w=Date.now()+ln;for(;Date.now()<w&&a.some(n=>Le(n.snap.tree.length>0?n.snap.tree:[n.pid]));)await new Promise(n=>setTimeout(n,Fe));let g=0;for(const n of a){const d=n.snap.tree.length>0?n.snap.tree:[n.pid];Le(d)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:n.pid}):$e(d,"SIGKILL",n.snap.identities),g++)}g>0&&await new Promise(n=>setTimeout(n,Fe));const p=Ce();if(p===null)throw u({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const c=[];for(const[n,d]of p.processes){const y=d.command;d.zombie||!y.includes("broker-cli.js")||de(y,f)&&pe(y)!==e&&c.push(n)}if(c.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${c.join(", ")}`);u({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:m.length,sigtermed:v,sigkilled:g}})}i(dn,"reapPriorEpochBrokers");const pn=5e3;async function mn(e){const l=Array.from(e.entries());if(l.length===0)return;for(const[a]of l){const w=(await kr(a))?.status;(w==="active"||w==="idle")&&await xe(a,new Date().toISOString())}const f=l.map(([,a])=>({pid:a.pid,snap:De(a.pid,null),exited:a.handle.exited}));for(const a of f)if(!a.snap.reused)try{process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):process.kill(a.pid,"SIGTERM")}catch{}const m=new Promise(a=>{setTimeout(a,pn).unref()});await Promise.allSettled(f.map(a=>Promise.race([a.exited.then(()=>{}),m])));const v=[];for(const a of f)Rt(a.pid)&&(process.platform==="linux"&&b(x)?await L({operation:"stop",pid:a.pid}):$e(a.snap.tree.length>0?a.snap.tree:[a.pid],"SIGKILL",a.snap.identities),v.push(a));await Promise.all(v.map(a=>a.exited));for(const[a]of l)await st(a,{reason:"daemon-teardown"});u({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:f.length,sigkilled:v.length}})}i(mn,"teardownFleet");function Be(){ye(cr(ce()),{recursive:!0}),ir(ce(),String(process.pid),"utf8")}i(Be,"writePidfile");function je(e){return typeof e=="object"&&e!==null&&e.code==="ENOENT"}i(je,"isMissingPathError");function ze(e){const l=ce();let f;try{f=ar(l,"utf8")}catch(m){if(je(m))return;throw m}if(f.trim()!==String(e))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${e})`);try{sr(l)}catch(m){if(je(m))return;throw m}}i(ze,"removeOwnedPidfileMirror");const fn={acceptsDetachedWork:i(()=>!0,"acceptsDetachedWork"),registerDetached:i(e=>{e.catch(()=>{})},"registerDetached")},K=new et;Zr(e=>K.handleFatalFault(e,!1));const un=new rt,wn=new tt,hn=new nt,vn=new ot,gn=new it,Sn=new at,yn=new dt,_n=new pt,bn=new mt,kn=new ut,Rn=new wt,Pn=new ht,En=new vt,In=new gt,xn=new St,Tn=new yt;async function Cn(e=Date.now(),l=fn){const f=Et(),m=await K.runFleetPreamble(f);let v,a;try{const g=await Ie();v=g.filter(c=>c.status==="active"||c.status==="idle"),a=g.filter(c=>c.status==="done"||c.status==="canceled"||c.status==="dead");const p=new Set(a.map(c=>c.node_id));a.push(...(await Rr()).filter(c=>!p.has(c.node_id)))}catch(g){u({level:"error",event:"canvas.node_list.failed",error:g});return}await K.run(e,{rows:v,fleet:f,lifecycle:l}),await un.run(e,{rows:v});const w=await wn.run(e,{rows:[...v,...a],fleet:f});await hn.run(e),await xn.run(e),await Tn.run(e,{lifecycle:l}),await vn.run(e,{lifecycle:l}),await gn.run(e,{lifecycle:l}),await Sn.run(e,{lifecycle:l}),await yn.run(e,{lifecycle:l}),await _n.run(e,{lifecycle:l,fleet:f}),bn.run(e,{lifecycle:l}),kn.run(e,{lifecycle:l}),Rn.run(e,{lifecycle:l}),Pn.run(e,{lifecycle:l}),await fr(),In.run(e,{fleet:f,lifecycle:l}),await En.run(e,{rows:v}),K.warnIfSaturated({fleet:f,lifecycle:l,thresholds:m,frozen:w})}i(Cn,"superviseTick");function V(e){return e!=="api"}i(V,"roleRunsLanes");function $(e){return e!=="reconcile"}i($,"roleServesRequests");const Dn=66e4;function Ln(e){const l=new Er().registerAll(Ir(()=>null).filter(n=>n.pattern==="/healthz")),f=i((n,d)=>{const y=(n.url??"/").split("?")[0];if((n.method??"GET").toUpperCase()==="GET"&&y==="/healthz"){l.handle(n,d,null,{class:"person"});return}d.writeHead(404,{"content-type":"application/json; charset=utf-8"}),d.end(JSON.stringify({error:{code:"not_found",message:"this process serves only /healthz",origin:"runtime"}}))},"handler"),m=Pe();b(m)&&_e(m);const v=J(f),a=new Promise((n,d)=>{v.once("error",d),v.listen(m,()=>{v.off("error",d),be(m,384),n()})});let w,g=null;const p=e.tcp??ke()??se("user").api.tcp;if(p!==void 0&&p!==""){const n=/^(.*):(\d+)$/.exec(p.trim());if(n===null||Number(n[2])>65535)throw new Error(`invalid reconcile probe TCP address: ${p}`);const d=J(f);d.on("error",y=>u({event:"api.server.failed",level:"error",error:y})),d.listen(Number(n[2]),n[1]===""?"0.0.0.0":n[1],()=>{g=`${n[1]===""?"0.0.0.0":n[1]}:${n[2]}`}),w=d}let c=!1;return{ready:a,tcpAddress:i(()=>g,"tcpAddress"),async close(n={}){if(!c){c=!0,await Promise.all([Te(v,new Set,n.graceMs),w===void 0?Promise.resolve():Te(w,new Set,n.graceMs)]);try{b(m)&&_e(m)}catch{}}}}}i(Ln,"createProbeServer");const $n="daemon is draining for shutdown and admits no new work";async function An(e){ye(Ee(),{recursive:!0,mode:493}),process.platform==="linux"&&be(Ee(),493);const l=Pe();if(b(l))throw new Error(`layout refusal cannot bind occupied socket: ${l}`);const f=J((p,c)=>{if(p.method==="GET"&&p.url==="/healthz"){c.writeHead(200,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}c.writeHead(503,{"content-type":"application/json; charset=utf-8"}),c.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((p,c)=>{f.once("error",c),f.listen(l,()=>{f.off("error",c),p()})});let m;const v=e.tcp??ke();if(v){const p=/^(.*):(\d+)$/.exec(v);if(!p||Number(p[2])>65535)throw new Error(`invalid refusal TCP address: ${v}`);const c=J(f.listeners("request")[0]);await new Promise((n,d)=>{c.once("error",d),c.listen(Number(p[2]),p[1],()=>{c.off("error",d),n()})}),m=c}Be();let a;const w=i(()=>(a??=(async()=>{f.closeAllConnections(),m?.closeAllConnections(),await Promise.all([f,m].filter(p=>p!==void 0).map(p=>new Promise((c,n)=>p.close(d=>d?n(d):c())))),ze(process.pid),process.off("SIGINT",g),process.off("SIGTERM",g)})(),a),"cleanup"),g=i(()=>{w().then(()=>process.exit(0),p=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(p)}
+`),process.exit(1)})},"onSignal");return process.on("SIGINT",g),process.on("SIGTERM",g),w}i(An,"serveLayoutRefusal");async function Nn(e={}){if(yr())return An(e);Sr();const l=i(r=>{process.stderr.write(`[crtrd] ${r}
+`)},"logPostgres"),f=!!process.env.CRTR_TEST_PG_URL,m=Nt(e.role);qt(V(e.role));const v=i(async()=>m?await At(Re(16).toString("hex")):{claimed:!0,claim:{pid:process.pid,pid_identity:"",token:Re(16).toString("hex")}},"claimOwnership"),{database:a,ownership:w}=f?{database:{url:await Gt()},ownership:await v()}:await Ht(async()=>({database:await Bt({log:l,startLockHeld:!0}),ownership:await v()}));if(!w.claimed){if(w.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${w.incumbentPid??"?"})
 `);else{const r=[w.incumbentPid!==void 0?`pid ${w.incumbentPid}`:void 0,w.verdict!==void 0?`verdict ${w.verdict}`:void 0].filter(h=>h!==void 0),o=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${w.reason}${o}
-`),w.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(w.reason==="own_identity_indeterminate"||w.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const g=w.claim;let p=null,c=null,n=null,d,y="";try{m&&await cn(g.token);for(const t of yr())process.stderr.write(`[crtrd] ${t}
-`);for(const t of await Sr())process.stderr.write(`[crtrd] ${t}
-`);await Ne(),await Gt(a.url,l);const r=e.role===void 0&&!await nn();let o=null;r&&(tn({kind:"user",root:Oe()}),o=on(Oe())),e.role===void 0&&an(o??{profiles:[],defaults:{},settings:{},state:{}}),await F.listen(a.url);const h=i(t=>{Ge(t).catch(O=>u({level:"error",event:"daemon.settings.reload_failed",error:O}))},"reloadPerson");d=await F.subscribe(t=>{Me().includes(t)&&h(t)},{kinds:["settings"],onListenStart:i(()=>{for(const t of Me())h(t)},"onListenStart")}),e.role!==void 0&&Cr();const s=await Mt(async t=>(await Nr(t),await Ut(t)));for(const t of s.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${t.message}
+`),w.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(w.reason==="own_identity_indeterminate"||w.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const g=w.claim;let p=null,c=null,n=null,d,y="";try{m&&await dn(g.token);for(const t of br())process.stderr.write(`[crtrd] ${t}
+`);for(const t of await _r())process.stderr.write(`[crtrd] ${t}
+`);await Ne(),await Ft(a.url,l);const r=e.role===void 0&&!await an();let o=null;r&&(on({kind:"user",root:Oe()}),o=sn(Oe())),e.role===void 0&&cn(o??{profiles:[],defaults:{},settings:{},state:{}}),await F.listen(a.url);const h=i(t=>{Ge(t).catch(O=>u({level:"error",event:"daemon.settings.reload_failed",error:O}))},"reloadPerson");d=await F.subscribe(t=>{Me().includes(t)&&h(t)},{kinds:["settings"],onListenStart:i(()=>{for(const t of Me())h(t)},"onListenStart")}),e.role!==void 0&&$r();const s=await Ot(async t=>(await Gr(t),await Wt(t)));for(const t of s.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${t.message}
 `);for(const t of s.invalid)process.stderr.write(`[crtrd] package file is not a document: ${t.path}: ${t.error}
 `);for(const t of s.quarantined)process.stderr.write(`[crtrd] plugin "${t.name}" has a damaged ${t.file} (${t.reason}); ${t.movedTo!==void 0?`moved to ${t.movedTo}`:"left in place and skipped"}
 `);for(const t of s.kinds.refused)process.stderr.write(`[crtrd] plugin "${t.plugin}" kind ${t.kind} not registered: ${t.message}
 `);for(const t of s.kinds.invalid)process.stderr.write(`[crtrd] plugin "${t.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${t.error}
-`);if(y=await Dt(),Ct(y),e.role===void 0&&(p=await Lr(),p!==null)){if(Ar(p.id),await rn(p.id),process.platform==="darwin"){const{readConfig:t,updateConfig:O}=await import("../core/config.js");if(t("user").paths?.user_files===void 0){const{homedir:Ze}=await import("node:os");await O("user",Se=>{Se.paths??={},Se.paths.user_files??=Ze()})}}c=await $r(),c.joinedFrom!==null&&await Ge(c.person),await rr(),n=await Mr()}if(process.platform==="linux"&&b(x)&&(hr(),await L({operation:"users.sync",users:await fr()})),m){await st(g.token);const t=await Vt();Object.values(t).some(O=>O>0)&&process.stderr.write(`[crtrd] node files copied into rows: ${JSON.stringify(t)}
-`)}}catch(r){await F.close();try{if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}await le()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}let k=!0,Y,B=Promise.resolve(),Q=!1,j,X,Z,T;const z=new Set;let ee=!1,me=!1,P,D,re,H=!1,U,A,N,M,te,ne,oe;const He=Yr();let E,S={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},C="initial startup activation",Ue=null;function qe(){return S.phase==="ready"?null:{phase:S.phase,message:S.message,blockerPaths:S.blockerPaths}}i(qe,"startupBlock");function R(r){z.add(r),r.finally(()=>{z.delete(r)}).catch(()=>{})}i(R,"registerDetached");const fe={acceptsDetachedWork:i(()=>k,"acceptsDetachedWork"),registerDetached:R};V(e.role)&&await qt(fe,Ne);async function We(){try{k=!1,zt.close(),X?.(),X=void 0,Z?.(),d?.(),Z=void 0,ne?.(),ne=void 0,await F.close(),j!==void 0&&(clearInterval(j),j=void 0),N!==void 0&&(clearInterval(N),N=void 0),oe?.(),oe=void 0;let r;do r=B,await r;while(r!==B);for(;z.size>0;)await Promise.allSettled([...z]);if(T!==void 0&&(m&&await dn(T),await T.exitsSettled(),T.dispose()),re!==void 0&&await re.close(),D!==void 0&&!H&&(H=!0,await D.close()),te?.(),te=void 0,M!==void 0){const o=M;M=void 0,await o.stop()}if(A!==void 0){const o=A;A=void 0,await o.close()}if(U!==void 0){const o=U;U=void 0,await o.close()}if(ee||(ze(process.pid),ee=!0),ee&&!me){if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}me=!0,await le()}}finally{Et(),process.off("SIGINT",he),process.off("SIGTERM",ve),process.off("exit",ge)}}i(We,"cleanupAttempt");const q=1e3;let ue=!1;function Je(){return ue?{graceMs:q}:(ue=!0,_.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:q}})}),setTimeout(()=>{Ke()},q),{graceMs:q})}i(Je,"requestHandover");async function Ke(){try{await I()}catch(r){const o=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
+`);if(y=await Lt(),$t(y),e.role===void 0&&(p=await Ar(),p!==null)){if(Mr(p.id),await nn(p.id),process.platform==="darwin"){const{readConfig:t,updateConfig:O}=await import("../core/config.js");if(t("user").paths?.user_files===void 0){const{homedir:Ze}=await import("node:os");await O("user",Se=>{Se.paths??={},Se.paths.user_files??=Ze()})}}c=await Nr(),c.joinedFrom!==null&&await Ge(c.person),await rr(),n=await Or()}if(process.platform==="linux"&&b(x)&&(gr(),await L({operation:"users.sync",users:await wr()})),m){await lt(g.token);const t=await Qt();Object.values(t).some(O=>O>0)&&process.stderr.write(`[crtrd] node files copied into rows: ${JSON.stringify(t)}
+`)}}catch(r){await F.close();try{if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}await le()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}let k=!0,Y,B=Promise.resolve(),Q=!1,j,X,Z,T;const z=new Set;let ee=!1,me=!1,P,C,re,H=!1,U,A,N,M,te,ne,oe;const He=Xr();let E,S={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},D="initial startup activation",Ue=null;function qe(){return S.phase==="ready"?null:{phase:S.phase,message:S.message,blockerPaths:S.blockerPaths}}i(qe,"startupBlock");function R(r){z.add(r),r.finally(()=>{z.delete(r)}).catch(()=>{})}i(R,"registerDetached");const fe={acceptsDetachedWork:i(()=>k,"acceptsDetachedWork"),registerDetached:R};V(e.role)&&await Jt(fe,Ne);async function We(){try{k=!1,Ut.close(),X?.(),X=void 0,Z?.(),d?.(),Z=void 0,ne?.(),ne=void 0,await F.close(),j!==void 0&&(clearInterval(j),j=void 0),N!==void 0&&(clearInterval(N),N=void 0),oe?.(),oe=void 0;let r;do r=B,await r;while(r!==B);for(;z.size>0;)await Promise.allSettled([...z]);if(T!==void 0&&(m&&await mn(T),await T.exitsSettled(),T.dispose()),re!==void 0&&await re.close(),C!==void 0&&!H&&(H=!0,await C.close()),te?.(),te=void 0,M!==void 0){const o=M;M=void 0,await o.stop()}if(A!==void 0){const o=A;A=void 0,await o.close()}if(U!==void 0){const o=U;U=void 0,await o.close()}if(ee||(ze(process.pid),ee=!0),ee&&!me){if(m){const{released:o}=await Ae(g);if(!o)throw new Error("daemon ownership claim was not released")}me=!0,await le()}}finally{xt(),process.off("SIGINT",he),process.off("SIGTERM",ve),process.off("exit",ge)}}i(We,"cleanupAttempt");const q=1e3;let ue=!1;function Je(){return ue?{graceMs:q}:(ue=!0,_.fresh(()=>{u({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:q}})}),setTimeout(()=>{Ke()},q),{graceMs:q})}i(Je,"requestHandover");async function Ke(){try{await I()}catch(r){const o=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
 ${o}
-`),process.exit(1)}try{process.platform==="linux"&&b(x)&&(await L({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await It();const r=await xt({waitForJoinLock:!0});_.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const o=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${o}
-`),process.exit(1)}process.exit(0)}i(Ke,"performHandover");function I(){if(P!==void 0)return P;const r=We();return P=r,r.then(()=>{P===r&&(P=void 0)},()=>{P===r&&(P=void 0)}),r}i(I,"requestCleanup");async function Ve(){S={phase:"initializing",message:Cn,blockerPaths:[]},D!==void 0&&!H&&(H=!0,await D.close({graceMs:Tn}))}i(Ve,"drainApi");async function we(r){try{r==="SIGTERM"&&e.role==="api"&&await Ve(),await I(),await Bt({claimed:i(async()=>{try{return m&&await At()}finally{await le()}},"claimed")}),process.exit(0)}catch(o){process.exitCode=1;const h=o instanceof Error?o.stack??`${o.name}: ${o.message}`:String(o);process.stderr.write(`[crtrd] ${r} cleanup failed:
+`),process.exit(1)}try{process.platform==="linux"&&b(x)&&(await L({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await Tt();const r=await Ct({waitForJoinLock:!0});_.fresh(()=>{u({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.handover.successor_failed",error:r})});const o=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${o}
+`),process.exit(1)}process.exit(0)}i(Ke,"performHandover");function I(){if(P!==void 0)return P;const r=We();return P=r,r.then(()=>{P===r&&(P=void 0)},()=>{P===r&&(P=void 0)}),r}i(I,"requestCleanup");async function Ve(){S={phase:"initializing",message:$n,blockerPaths:[]},C!==void 0&&!H&&(H=!0,await C.close({graceMs:Dn}))}i(Ve,"drainApi");async function we(r){try{r==="SIGTERM"&&e.role==="api"&&await Ve(),await I(),await zt({claimed:i(async()=>{try{return m&&await Mt()}finally{await le()}},"claimed")}),process.exit(0)}catch(o){process.exitCode=1;const h=o instanceof Error?o.stack??`${o.name}: ${o.message}`:String(o);process.stderr.write(`[crtrd] ${r} cleanup failed:
 ${h}
-`),process.exit(1)}}i(we,"handleSignal");function he(){we("SIGINT")}i(he,"onSigint");function ve(){we("SIGTERM")}i(ve,"onSigterm");function ge(){I().catch(()=>{})}i(ge,"onExit");async function ie(r){k&&(B=B.then(r).then(()=>{},o=>{_.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:o})})}))}i(ie,"enqueue");async function W(){!k||Q||(He.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),R(E().catch(()=>{}))),Q=!0,await ie(async()=>{if(Q=!1,!!k)try{f||await Ft({log:l}),await xn(Date.now(),fe)}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(W,"enqueueTick");let G=null;async function Ye(){if(G!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${G instanceof Error?G.message:String(G)}`};if(S.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(S.phase!=="prepared"&&S.phase!=="recovering")throw{status:409,code:"not_prepared",message:S.message};const r=S.phase==="prepared";try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(o){throw G=o,{status:500,code:"admission_failed",message:`daemon admission failed: ${o instanceof Error?o.message:String(o)}`}}}i(Ye,"admitStartup");async function Qe(r){if(C!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${C})`};C="repair";try{const o=await en({dryRun:r.dryRun});if(r.dryRun||o.pending||S.phase!=="blocked")return{result:o,resumed:!1};if(e.prepare)return S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:o,resumed:!1};try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},{result:o,resumed:!0}}catch(h){const s=h instanceof Error?h.message:String(h);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:o,resumed:!1,resumeError:s}}}finally{C=null}}i(Qe,"repairMigrations");async function Xe(){Y=e.intervalMs??Vr;const r=g.token;T=new Kt({epoch:r,enqueue:ie}),kt(T),S={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:Y}})});const o=y;_.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:o}})});const h=await Tr(o);if(_.fresh(()=>{u({level:"info",event:"daemon.environment.registered",fields:{environment_id:h.environmentId,created:h.created}})}),_.fresh(()=>{let s;try{s=Kr()}catch{return}s!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:s.slice(0,2048)}})}),pt(),e.role!==void 0&&!process.env.CRTR_DIRECTORY_URL)throw new Error("CRTR_DIRECTORY_URL must be set on a cloud role (--role api|reconcile): it names the directory this deployment mints tokens against");if(e.role!==void 0&&Dr(),!V(e.role)){S={phase:"ready",message:"",blockerPaths:[]};return}if(await _.fresh(async()=>{await Jr(),await Ur(),await qr(Date.now()),await Wr(Date.now())}),await _.fresh(async()=>{const{reconcile:s,migrated:t}=await Tt(at);s.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:s.cleared.length,same_boot_proven:s.sameBootProven}}),t.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:t.length}})}),await _.fresh(async()=>{for(const s of await Ie())s.status!=="active"||s.pi_pid!=null||s.frozen_at!=null||s.respawn_not_before!=null||(await xe(s.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:s.node_id}))}),await _.fresh(async()=>{await yt();for(const s of Yt){const t=await s.apply();t.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:s.id,...t.summary}})}}),R(zr()),E){const s=i(()=>{R(E().catch(()=>{})),R(nr().then(t=>M?.observe(t)).then(()=>{},t=>{u({level:"warn",event:"daemon.peer.status_failed",error:t})}))},"refresh");R(jr().then(({completion:t})=>t).catch(t=>{u({level:"error",event:"daemon.grants.sweep_failed",error:t})})),s(),N=setInterval(s,Br*1e3),N.unref(),oe=tr({joined:!0,register:R})}await ie(async()=>{await Jt(),await Wt()}),j=setInterval(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},Y),X=await St(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))}),Z=await F.subscribe(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},{kinds:["spend_limits"]}),await W()}i(Xe,"completeStartup");function ae(){return Ue??=Xe().catch(r=>{throw S={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(ae,"resumeStartup");try{process.on("SIGINT",he),process.on("SIGTERM",ve),process.on("exit",ge),Qt(),p!==null&&u({level:"info",event:"daemon.person.resolved",fields:{person_id:p.id,source:p.source}}),n!==null&&u({level:"info",event:"daemon.permissions.boot",fields:{...n}}),c!==null&&u({level:"info",event:"daemon.person.retired",fields:{person_id:c.person,joined_from:c.joinedFrom,retired:c.retired,columns:c.columns}}),$(e.role)&&(U=await dr());const r=g.token;Pt({epoch:r,requestHandover:Je,registerDetached:R,startupBlock:qe,admitStartup:Ye,repairMigrations:Qe}),Be(),D=$(e.role)?kr({tcp:e.tcp}):Dn(e),await D.ready;const o=lr()??se("user").document_port;if($(e.role)){const s=Er(o,Ir());try{await s.ready,re=s}catch(t){u({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:o,error:t instanceof Error?t.message:String(t)}})}}const h=se("user").runtime;if(h?.issuer&&(E=Fr(h.issuer,R),h.unsandboxed===!0&&(M=or({syncGrants:E}))),$(e.role)&&(te=await Gr()),ne=await Or({serves:$(e.role),runsLanes:V(e.role)}),h?.unsandboxed===!0&&h.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),$(e.role)&&h!==void 0&&h.listener!==void 0&&h.unsandboxed!==!0&&(A=await xr(h,async()=>await Hr(h.issuer.replace(/\/$/,"")),E),await A.ready),e.role===void 0)try{await Xt()}catch(s){if(C=null,s instanceof Zt)return S={phase:"blocked",message:s.message,blockerPaths:s.blockerPaths},I;throw s}process.platform==="linux"&&b(x)&&(wr(),await ur("app:terminal"));try{e.prepare?S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await ae(),S={phase:"ready",message:"",blockerPaths:[]})}finally{C=null}return I}catch(r){try{await I()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}}i($n,"runDaemon");var Pi=$n;export{Tn as API_DRAIN_GRACE_MS,de as brokerCommandDeclaresCanvasHome,pe as brokerCommandEpoch,Pi as default,V as roleRunsLanes,$ as roleServesRequests,$n as runDaemon,xn as superviseTick,dn as teardownFleet};
+`),process.exit(1)}}i(we,"handleSignal");function he(){we("SIGINT")}i(he,"onSigint");function ve(){we("SIGTERM")}i(ve,"onSigterm");function ge(){I().catch(()=>{})}i(ge,"onExit");async function ie(r){k&&(B=B.then(r).then(()=>{},o=>{_.fresh(()=>{u({level:"error",event:"daemon.lane_job.failed",error:o})})}))}i(ie,"enqueue");async function W(){!k||Q||(He.check()&&E&&(u({level:"info",event:"daemon.resume.detected"}),R(E().catch(()=>{}))),Q=!0,await ie(async()=>{if(Q=!1,!!k)try{f||await jt({log:l}),await Cn(Date.now(),fe)}catch(r){_.fresh(()=>{u({level:"error",event:"daemon.tick.failed",error:r})})}}))}i(W,"enqueueTick");let G=null;async function Ye(){if(G!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${G instanceof Error?G.message:String(G)}`};if(S.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(S.phase!=="prepared"&&S.phase!=="recovering")throw{status:409,code:"not_prepared",message:S.message};const r=S.phase==="prepared";try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(o){throw G=o,{status:500,code:"admission_failed",message:`daemon admission failed: ${o instanceof Error?o.message:String(o)}`}}}i(Ye,"admitStartup");async function Qe(r){if(D!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${D})`};D="repair";try{const o=await tn({dryRun:r.dryRun});if(r.dryRun||o.pending||S.phase!=="blocked")return{result:o,resumed:!1};if(e.prepare)return S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:o,resumed:!1};try{return await ae(),S={phase:"ready",message:"",blockerPaths:[]},{result:o,resumed:!0}}catch(h){const s=h instanceof Error?h.message:String(h);return setTimeout(()=>{I().catch(()=>{})},1e3),{result:o,resumed:!1,resumeError:s}}}finally{D=null}}i(Qe,"repairMigrations");async function Xe(){Y=e.intervalMs??Qr;const r=g.token;T=new Yt({epoch:r,enqueue:ie}),Pt(T),S={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{u({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:Y}})});const o=y;_.fresh(()=>{u({level:"info",event:"daemon.install_id.resolved",fields:{install_id:o}})});const h=await Dr(o);if(_.fresh(()=>{u({level:"info",event:"daemon.environment.registered",fields:{environment_id:h.environmentId,created:h.created}})}),_.fresh(()=>{let s;try{s=Yr()}catch{return}s!==null&&u({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:s.slice(0,2048)}})}),ft(),e.role!==void 0&&!process.env.CRTR_DIRECTORY_URL)throw new Error("CRTR_DIRECTORY_URL must be set on a cloud role (--role api|reconcile): it names the directory this deployment mints tokens against");if(e.role!==void 0&&Lr(),!V(e.role)){S={phase:"ready",message:"",blockerPaths:[]};return}if(await _.fresh(async()=>{await Vr(),await Wr(),await Jr(Date.now()),await Kr(Date.now())}),await _.fresh(async()=>{const{reconcile:s,migrated:t}=await Dt(ct);s.changed&&u({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:s.cleared.length,same_boot_proven:s.sameBootProven}}),t.length>0&&u({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:t.length}})}),await _.fresh(async()=>{for(const s of await Ie())s.status!=="active"||s.pi_pid!=null||s.frozen_at!=null||s.respawn_not_before!=null||(await xe(s.node_id,new Date().toISOString()),u({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:s.node_id}))}),await _.fresh(async()=>{await bt();for(const s of Xt){const t=await s.apply();t.changed&&u({level:"info",event:"daemon.boot_migration.applied",fields:{migration:s.id,...t.summary}})}}),R(Ur()),E){const s=i(()=>{R(E().catch(()=>{})),R(nr().then(t=>M?.observe(t)).then(()=>{},t=>{u({level:"warn",event:"daemon.peer.status_failed",error:t})}))},"refresh");R(Hr().then(({completion:t})=>t).catch(t=>{u({level:"error",event:"daemon.grants.sweep_failed",error:t})})),s(),N=setInterval(s,zr*1e3),N.unref(),oe=tr({joined:!0,register:R})}await ie(async()=>{await Vt(),await Kt()}),j=setInterval(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},Y),X=await _t(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))}),Z=await F.subscribe(()=>{W().catch(s=>u({level:"error",event:"daemon.tick.failed",error:s}))},{kinds:["spend_limits"]}),await W()}i(Xe,"completeStartup");function ae(){return Ue??=Xe().catch(r=>{throw S={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}i(ae,"resumeStartup");try{process.on("SIGINT",he),process.on("SIGTERM",ve),process.on("exit",ge),Zt(),p!==null&&u({level:"info",event:"daemon.person.resolved",fields:{person_id:p.id,source:p.source}}),n!==null&&u({level:"info",event:"daemon.permissions.boot",fields:{...n}}),c!==null&&u({level:"info",event:"daemon.person.retired",fields:{person_id:c.person,joined_from:c.joinedFrom,retired:c.retired,columns:c.columns}}),$(e.role)?U=await pr():mr(dr());const r=g.token;It({epoch:r,requestHandover:Je,registerDetached:R,startupBlock:qe,admitStartup:Ye,repairMigrations:Qe}),Be(),C=$(e.role)?Pr({tcp:e.tcp}):Ln(e),await C.ready;const o=lr()??se("user").document_port;if($(e.role)){const s=xr(o,Tr());try{await s.ready,re=s}catch(t){u({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:o,error:t instanceof Error?t.message:String(t)}})}}const h=se("user").runtime;if(h?.issuer&&(E=jr(h.issuer,R),h.unsandboxed===!0&&(M=or({syncGrants:E}))),$(e.role)&&(te=await Fr()),ne=await Br({serves:$(e.role),runsLanes:V(e.role)}),h?.unsandboxed===!0&&h.listener!==void 0&&u({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),$(e.role)&&h!==void 0&&h.listener!==void 0&&h.unsandboxed!==!0&&(A=await Cr(h,async()=>await qr(h.issuer.replace(/\/$/,"")),E),await A.ready),e.role===void 0)try{await en()}catch(s){if(D=null,s instanceof rn)return S={phase:"blocked",message:s.message,blockerPaths:s.blockerPaths},I;throw s}process.platform==="linux"&&b(x)&&(vr(),await hr("app:terminal"));try{e.prepare?S={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await ae(),S={phase:"ready",message:"",blockerPaths:[]})}finally{D=null}return I}catch(r){try{await I()}catch(o){throw new AggregateError([r,o],"daemon startup failed and cleanup also failed")}throw r}}i(Nn,"runDaemon");var xi=Nn;export{Dn as API_DRAIN_GRACE_MS,de as brokerCommandDeclaresCanvasHome,pe as brokerCommandEpoch,xi as default,V as roleRunsLanes,$ as roleServesRequests,Nn as runDaemon,Cn as superviseTick,mn as teardownFleet};

package/dist/daemon/peer/forward.js

@@ -1 +1 @@
-var $=Object.defineProperty;var i=(e,n)=>$(e,"name",{value:n,configurable:!0});import{ApiError as m}from"../../api/errors.js";import{openDb as g}from"../../core/canvas/db.js";import{getById as A,nodeObject as D}from"../../core/graph/objects.js";import{envelopeFor as j}from"../../core/peer/envelope.js";import{isUnreachable as w}from"../../core/peer/client.js";import{machineReach as E}from"../../core/peer/directory.js";import{defaultHomeDeps as O,homeOf as S,peersToAsk as F,relocate as N}from"../../core/peer/home.js";import{ensureStub as y,rehomeStub as q,stubOf as C}from"../../core/peer/stubs.js";import{peerCall as M}from"../../core/peer/client.js";const T={...O,call:i((e,n,o,t)=>M(e,n,{...o===void 0?{}:{envelope:o},...t===void 0?{}:{signal:t}}),"call"),envelope:i(async e=>await j(e),"envelope"),label:i(async e=>{try{const n=await E(e);return{id:e,name:n?.name??e}}catch{return{id:e,name:e}}},"label")},_={current:T},v=i(e=>e.class==="person"||e.class==="broker"||e.class==="cron","originates"),W=i(e=>e?.error?.details?.deleted===!0,"isDeletedAnswer"),k=i(e=>{const n=e?.error;return typeof n=="object"&&n!==null&&typeof n.code=="string"?n.code:null},"errorCode");function L(e,n){const o=n?.error;return new m(e,o?.code??"error",o?.message??`the home machine answered ${e}`,o?.details)}i(L,"relayError");function Q(e,n){return new m(503,"runtime_unreachable",`No machine of yours answers for ${e} (unreachable: ${n.join(", ")}).`,{id:e,unreachable:n})}i(Q,"unknownError");const X=i((e,n)=>e!==null&&typeof e=="object"&&!Array.isArray(e)?{...e,machine:n}:e,"withMachine");async function Y(e,n,o){const t=_.current,a=await g(),c=await A(a,n),u=c===null?null:await C(a,c),p=e.principal;if(!v(p)){if(u!==null)throw new m(404,"not_here",`${n} is not homed on this machine (it is a stub of ${u.home_machine})`,{id:n,home_machine:u.home_machine});return{kind:"local"}}if(c!==null&&u===null)return{kind:"local"};const h={...t,call:i((r,d)=>t.call(r,d),"call")};let s=await S(a,n,{hint:e.query.get("machine"),deps:h});for(let r=0;r<2;r++){if(s.where==="nowhere"||s.where==="here"){if(u!==null)throw new m(404,"not_found",`${n} is no longer on any machine of yours`,{id:n});return{kind:"local"}}if(s.where==="unknown")return{kind:"unknown",unreachable:s.unreachable};const d=s.machine,l=await t.call(d,o,await t.envelope(p)),b=await t.label(d);if(w(l))return{kind:"unknown",unreachable:[`${b.name} (${l.unreachable.reason})`]};const x=s.source!=="locate";if(l.status===404&&r===0&&!W(l.body)&&(k(l.body)==="not_here"||x&&k(l.body)==="not_found")){const f=await N(a,n,d,h);if(f.where==="machine"){s=f;continue}if(f.where==="unknown")return{kind:"unknown",unreachable:f.unreachable};if(k(l.body)==="not_found")return{kind:"remote",status:l.status,body:l.body,machine:b};if(u!==null)throw new m(404,"not_found",`${n} is no longer on any machine of yours`,{id:n});return{kind:"local"}}return{kind:"remote",status:l.status,body:l.body,machine:b}}return{kind:"unknown",unreachable:[]}}i(Y,"forwardVerb");async function Z(e,n,o){const t=_.current,a=e.principal;if(!v(a))return{kind:"local"};const c=await F(t);if("where"in c)return{kind:"unknown",unreachable:c.unreachable};if(c.length===0)return{kind:"local"};const u=await t.envelope(a),p=await Promise.all(c.map(async r=>({m:r,out:await t.call(r.machine_id,o,u)}))),h=p.find(r=>!w(r.out)&&r.out.status<400);if(h){const r=n.method==="GET"?h.out:await t.call(h.m.machine_id,n,u),d=await t.label(h.m.machine_id);return w(r)?{kind:"unknown",unreachable:[`${d.name} (${r.unreachable.reason})`]}:{kind:"remote",status:r.status,body:r.body,machine:d}}const s=p.filter(r=>w(r.out)||r.out.status!==404).map(r=>r.m.name||r.m.machine_id);return s.length>0?{kind:"unknown",unreachable:s}:{kind:"local"}}i(Z,"forwardNamed");function B(e){if(e.actor.kind==="person")return{sub:e.sub,act:{sub:e.via}};const{app:n,node:o,node_scopes:t,place:a}=e.actor;return{sub:e.sub,act:{sub:e.via,act:{sub:n,node:o,node_scopes:t,place:a}}}}i(B,"envelopeOfForwarded");function ee(e){const n={};for(const[o,t]of e.query)o!=="machine"&&(n[o]=t);return n}i(ee,"queryWithoutMachine");async function ne(e,n){const o=n.principal;return o.class!=="forwarded"||o.actor.kind!=="app"||o.actor.node===null?null:await y(e,o.actor.node,"node",o.via,B(o))}i(ne,"callerStub");async function oe(e,n){return n.principal.class==="broker"?await D(e,n.principal.node):null}i(oe,"callerNode");async function re(e,n,o,t,a){await q(e,n,t);const c=a?.owner,u=c&&c.type==="app"?{id:"",type:"app",scheme:"app",external_id:c.external_id??(c.handle.startsWith("app:")?c.handle:`app:${c.handle}`),handle:c.handle}:null;return await y(e,n,o,t,void 0,a===void 0?void 0:{id:n,type:o,kind:a.kind??null,name:a.name??n,...a.preview===void 0?{}:{preview:a.preview},owner:u,...a.kind_decl===void 0?{}:{kind_decl:a.kind_decl},...a.privacy===void 0?{}:{privacy:a.privacy}})}i(re,"remoteStub");export{oe as callerNode,ne as callerStub,T as defaultForwardDeps,B as envelopeOfForwarded,Z as forwardNamed,Y as forwardVerb,_ as forwarding,v as originates,ee as queryWithoutMachine,L as relayError,re as remoteStub,Q as unknownError,X as withMachine};
+var $=Object.defineProperty;var i=(e,n)=>$(e,"name",{value:n,configurable:!0});import{ApiError as p}from"../../api/errors.js";import{openDb as g}from"../../core/canvas/db.js";import{getById as E,nodeObject as A}from"../../core/graph/objects.js";import{envelopeFor as D,EnvelopeError as j}from"../../core/peer/envelope.js";import{isUnreachable as f}from"../../core/peer/client.js";import{machineReach as O}from"../../core/peer/directory.js";import{defaultHomeDeps as S,homeOf as q,peersToAsk as F,relocate as N}from"../../core/peer/home.js";import{ensureStub as y,rehomeStub as C,stubOf as M}from"../../core/peer/stubs.js";import{peerCall as T}from"../../core/peer/client.js";const W={...S,call:i((e,n,r,t)=>T(e,n,{...r===void 0?{}:{envelope:r},...t===void 0?{}:{signal:t}}),"call"),envelope:i(async e=>{try{return await D(e)}catch(n){throw n instanceof j?new p(400,"invalid_request",n.message):n}},"envelope"),label:i(async e=>{try{const n=await O(e);return{id:e,name:n?.name??e}}catch{return{id:e,name:e}}},"label")},_={current:W},v=i(e=>e.class==="person"||e.class==="broker"||e.class==="cron","originates"),B=i(e=>e?.error?.details?.deleted===!0,"isDeletedAnswer"),k=i(e=>{const n=e?.error;return typeof n=="object"&&n!==null&&typeof n.code=="string"?n.code:null},"errorCode");function Q(e,n){const r=n?.error;return new p(e,r?.code??"error",r?.message??`the home machine answered ${e}`,r?.details)}i(Q,"relayError");function X(e,n){return new p(503,"runtime_unreachable",`No machine of yours answers for ${e} (unreachable: ${n.join(", ")}).`,{id:e,unreachable:n})}i(X,"unknownError");const Y=i((e,n)=>e!==null&&typeof e=="object"&&!Array.isArray(e)?{...e,machine:n}:e,"withMachine");async function Z(e,n,r){const t=_.current,a=await g(),c=await E(a,n),u=c===null?null:await M(a,c),m=e.principal;if(!v(m)){if(u!==null)throw new p(404,"not_here",`${n} is not homed on this machine (it is a stub of ${u.home_machine})`,{id:n,home_machine:u.home_machine});return{kind:"local"}}if(c!==null&&u===null)return{kind:"local"};const h={...t,call:i((o,d)=>t.call(o,d),"call")};let s=await q(a,n,{hint:e.query.get("machine"),deps:h});for(let o=0;o<2;o++){if(s.where==="nowhere"||s.where==="here"){if(u!==null)throw new p(404,"not_found",`${n} is no longer on any machine of yours`,{id:n});return{kind:"local"}}if(s.where==="unknown")return{kind:"unknown",unreachable:s.unreachable};const d=s.machine,l=await t.call(d,r,await t.envelope(m)),b=await t.label(d);if(f(l))return{kind:"unknown",unreachable:[`${b.name} (${l.unreachable.reason})`]};const x=s.source!=="locate";if(l.status===404&&o===0&&!B(l.body)&&(k(l.body)==="not_here"||x&&k(l.body)==="not_found")){const w=await N(a,n,d,h);if(w.where==="machine"){s=w;continue}if(w.where==="unknown")return{kind:"unknown",unreachable:w.unreachable};if(k(l.body)==="not_found")return{kind:"remote",status:l.status,body:l.body,machine:b};if(u!==null)throw new p(404,"not_found",`${n} is no longer on any machine of yours`,{id:n});return{kind:"local"}}return{kind:"remote",status:l.status,body:l.body,machine:b}}return{kind:"unknown",unreachable:[]}}i(Z,"forwardVerb");async function ee(e,n,r){const t=_.current,a=e.principal;if(!v(a))return{kind:"local"};const c=await F(t);if("where"in c)return{kind:"unknown",unreachable:c.unreachable};if(c.length===0)return{kind:"local"};const u=await t.envelope(a),m=await Promise.all(c.map(async o=>({m:o,out:await t.call(o.machine_id,r,u)}))),h=m.find(o=>!f(o.out)&&o.out.status<400);if(h){const o=n.method==="GET"?h.out:await t.call(h.m.machine_id,n,u),d=await t.label(h.m.machine_id);return f(o)?{kind:"unknown",unreachable:[`${d.name} (${o.unreachable.reason})`]}:{kind:"remote",status:o.status,body:o.body,machine:d}}const s=m.filter(o=>f(o.out)||o.out.status!==404).map(o=>o.m.name||o.m.machine_id);return s.length>0?{kind:"unknown",unreachable:s}:{kind:"local"}}i(ee,"forwardNamed");function G(e){if(e.actor.kind==="person")return{sub:e.sub,act:{sub:e.via}};const{app:n,node:r,node_scopes:t,place:a}=e.actor;return{sub:e.sub,act:{sub:e.via,act:{sub:n,node:r,node_scopes:t,place:a}}}}i(G,"envelopeOfForwarded");function ne(e){const n={};for(const[r,t]of e.query)r!=="machine"&&(n[r]=t);return n}i(ne,"queryWithoutMachine");async function re(e,n){const r=n.principal;return r.class!=="forwarded"||r.actor.kind!=="app"||r.actor.node===null?null:await y(e,r.actor.node,"node",r.via,G(r))}i(re,"callerStub");async function oe(e,n){return n.principal.class==="broker"?await A(e,n.principal.node):null}i(oe,"callerNode");async function te(e,n,r,t,a){await C(e,n,t);const c=a?.owner,u=c&&c.type==="app"?{id:"",type:"app",scheme:"app",external_id:c.external_id??(c.handle.startsWith("app:")?c.handle:`app:${c.handle}`),handle:c.handle}:null;return await y(e,n,r,t,void 0,a===void 0?void 0:{id:n,type:r,kind:a.kind??null,name:a.name??n,...a.preview===void 0?{}:{preview:a.preview},owner:u,...a.kind_decl===void 0?{}:{kind_decl:a.kind_decl},...a.privacy===void 0?{}:{privacy:a.privacy}})}i(te,"remoteStub");export{oe as callerNode,re as callerStub,W as defaultForwardDeps,G as envelopeOfForwarded,ee as forwardNamed,Z as forwardVerb,_ as forwarding,v as originates,ne as queryWithoutMachine,Q as relayError,te as remoteStub,X as unknownError,Y as withMachine};

package/dist/daemon/reconcilers/broker-supervision.js

@@ -1,4 +1,4 @@
-var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,parkEvent as Ie}from"./live-obligation.js";import{readGoal as xe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Fe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Oe,envTestUnattendedParkMs as Ce,parkSummaryGraceMs as De,envUnattendedParkMs as We}from"../../shared/env.js";const R=ee(Z);async function Be(a){return!await ie(a)&&((await xe(a))?.trim()??"")!==""}m(Be,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Ge(a){return(await F(a))?.busy===!0}m(Ge,"liveBusy");async function It(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(It,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,Le=2,qe=5*6e4;function Ue(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ue,"settlesOnFatalProviderFault");function Ye(){const a=We();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Oe()?Ce()??e:e}m(Ye,"unattendedParkMsForDaemon");function H(){return De()}m(H,"parkSummaryGraceMsForDaemon");function K(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(K,"yieldStallVerdict");function He(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>Le?"pending":"wedged"}m(He,"wedgeVerdict");function V(a){const e=new Map,t=new Map;for(const r of a.split(`
-`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(V,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=V(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function Ve(a,e){const{byPid:t,childrenOf:r}=V(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(Ve,"treeHasDescendants");class xt{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=Ye();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Ge(e)||await Be(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const u=await Ie(e,t.lifecycle),w=u!=="cancel"&&Fe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<w)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<qe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
-`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}K(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(K(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=Ve(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,k=s?.hasDescendants??null;if(He({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:k})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const v=x(p),y=Math.round((d??I)/6e4),b=k??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
-`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ue(i,typeof s?.code=="string"?s.code:void 0),k=u||w,v=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${v}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&k&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&k&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{xt as BrokerSupervisionReconciler,Le as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ue as settlesOnFatalProviderFault,Ke as sumTreeCpu,Ve as treeHasDescendants,Ye as unattendedParkMsForDaemon,He as wedgeVerdict,K as yieldStallVerdict};
+var j=Object.defineProperty;var m=(a,e)=>j(a,"name",{value:e,configurable:!0});import{existsSync as J}from"node:fs";import{LAUNCHER_SOCKET as Q}from"../../launcher/spec.js";import{launcherRequest as X}from"../../launcher/client.js";import{execFile as Z}from"node:child_process";import{promisify as ee}from"node:util";import{getNode as P,isWarmSpare as te,subscribersOf as ne}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{fullName as x}from"../../core/canvas/labels.js";import{recordedPidLiveness as B}from"../../core/canvas/pid.js";import{transition as re}from"../../core/runtime/lifecycle.js";import{hasAdmittedTurn as ie,listUnnotifiedFatalFaultNodes as ae,markFaultNotified as se,readFault as G,readLiveTurn as F,recordFault as oe}from"../../core/canvas/recovery.js";import{errorClassFromFault as le}from"../../core/fault-classifier.js";import{nextFaultRetry as de}from"../../core/runtime/fault-recovery.js";import{publishNotice as L}from"../../core/runtime/close.js";import{openDb as O,withCanvasWrite as ce}from"../../core/canvas/db.js";import{getById as ue}from"../../core/graph/objects.js";import{watcherIdsOf as fe}from"../../core/graph/watches.js";import{headlessBrokerHost as pe}from"../../core/runtime/host.js";import{endExecution as me,liveExecutionForNode as C,requestExecutionStop as _e}from"../../core/canvas/executions.js";import{runnerForExecution as he}from"../../core/runtime/runner/index.js";import{emitEvent as f}from"../../core/events/emit.js";import{operationIdContext as D}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as we,resetStaleExecutionMail as ke}from"../../core/canvas/deliveries.js";import{runningJobs as ge}from"../../core/graph/jobs.js";import{focusOf as ve}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as ye}from"../api/viewer-presence.js";import{deliverLive as be}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as Se}from"../../core/runtime/broker-request.js";import{formatCard as Te,PARK_SUMMARY_PROMPT as Pe}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as $e}from"../../core/runtime/fleet.js";import{readConfig as Me}from"../../core/config.js";import{beginPendingPark as Ae,clearSupervisionMark as h,consumePendingPark as $,pendingParkIds as Ee,pendingParkStartedAt as Ne,readSupervisionMark as M,writeSupervisionMark as A}from"../park-pending.js";import{shouldParkAfterSummary as Re}from"../park-activity.js";import{applyParkEvent as E,noticeUnfinishedIdle as Ie,parkVerdict as xe}from"./live-obligation.js";import{readGoal as Fe}from"../../core/runtime/kickoff.js";import{memoryDeniedRecently as Oe}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Ce,envTestUnattendedParkMs as De,parkSummaryGraceMs as We,envUnattendedParkMs as Be}from"../../shared/env.js";const R=ee(Z);async function Ge(a){return!await ie(a)&&((await Fe(a))?.trim()??"")!==""}m(Ge,"awaitsBirthTurn");async function q(a){const e=await C(a);return e!==null&&e.runner!=="process"&&e.state==="running"?e:null}m(q,"remoteExecutionOf");async function Le(a){return(await F(a))?.busy===!0}m(Le,"liveBusy");async function xt(a){const e=(await F(a))?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}m(xt,"liveTurnProgressMs");function U(a,e){return e===null?null:`${a}:${e}`}m(U,"incarnationOf");async function S(a){const e=await F(a.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=a.launched_at===null||a.launched_at===void 0?Number.NaN:Date.parse(a.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}m(S,"busySinceForIncarnation");const W=3*6e4,Y=2e4,I=20*6e4,qe=2,Ue=5*6e4;function Ye(a,e){return a===null||a.parent!=null?!1:a.lifecycle==="terminal"?!0:e==="cap_exhausted"&&a.run_id===a.node_id}m(Ye,"settlesOnFatalProviderFault");function He(){const a=Be();if(a!==void 0)return a;const e=Me("user").lifecycle.unattendedParkMs;return Ce()?De()??e:e}m(He,"unattendedParkMsForDaemon");function H(){return We()}m(H,"parkSummaryGraceMsForDaemon");function V(a,e,t,r){return a!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}m(V,"yieldStallVerdict");function Ve(a){return!a.busy||a.quietForMs===null||a.quietForMs<I?"leave":a.cpuPercent===null||a.hasDescendants!==!1&&a.cpuPercent>qe?"pending":"wedged"}m(Ve,"wedgeVerdict");function K(a){const e=new Map,t=new Map;for(const r of a.split(`
+`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const o=Number(n[0]),l=Number(n[1]),d=Number(n[2]);if(!Number.isFinite(o)||!Number.isFinite(l)||!Number.isFinite(d))continue;e.set(o,d);const s=t.get(l);s===void 0?t.set(l,[o]):s.push(o)}return{byPid:e,childrenOf:t}}m(K,"parsePsTree");function Ke(a,e){const{byPid:t,childrenOf:r}=K(a);if(!t.has(e))return null;let i=0;const n=[e],o=new Set;for(;n.length>0;){const l=n.pop();if(o.has(l))continue;o.add(l),i+=t.get(l)??0;const d=r.get(l);d!==void 0&&n.push(...d)}return i}m(Ke,"sumTreeCpu");function ze(a,e){const{byPid:t,childrenOf:r}=K(a);return t.has(e)?(r.get(e)??[]).length>0:null}m(ze,"treeHasDescendants");class Ft{static{m(this,"BrokerSupervisionReconciler")}yieldTermedAt=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&J(Q)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return X({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return $e()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=D.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=He();for(const i of await ae())await D.fresh(async()=>{try{await this.handleFatalFault(i)}catch(n){f({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,o=t.fleet.get(n),l=o===void 0?await q(n):null;if(o===void 0&&l===null||i.status!=="active"&&i.status!=="idle"){await h(n,"unattended_since");continue}await D.fresh(async()=>{try{o!==void 0?(await this.handleYieldStall(i,o,t.fleet,e),await this.handleWedgeDetection(i,o,t.fleet,e)):(await this.handleRemoteYieldStall(i,l,e),await this.handleRemoteWedge(i,l,e)),await this.handleUnattendedParking(n,await P(n),o?.pid??l.pid,l,e,t,r)}catch(d){await h(n,"unattended_since"),f({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:d})}})}for(const i of await Ee())await t.fleet.has(i)||await $(i);await ke(e)}async parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||(await ge(await O(),e)).length>0||await ve(e)!==null||await ye(e))return!0;const n=await G(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||de(n,i)!==null)?!0:await we(e)}async handleUnattendedParking(e,t,r,i,n,o,l){if(await te(e)){await h(e,"unattended_since"),await $(e);return}if(t===null){await h(e,"unattended_since"),await $(e);return}const d=await this.parkResetCondition(e,t,r,n),s=await Ne(e);if(s!==void 0){if(n-s<H())return;const k=await $(e);if(k===void 0||(await h(e,"unattended_since"),!await Re(e,t,k)))return;f({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:H()}}),await E(e,"park"),this.teardownBroker(e,i);return}if(d||await Le(e)||await Ge(e)){await h(e,"unattended_since");return}const p=await M(e,"unattended_since");if(p===void 0){await A(e,"unattended_since",n);return}const{event:u,heldOnlyBySubscriber:w}=await xe(e,t.lifecycle),v=u!=="cancel"&&Oe(t.grantee??"app:terminal",n)?0:l;if(!(n-p<v)){if(await h(e,"unattended_since"),u==="release"){await E(e,u),w&&await Ie(e),this.teardownBroker(e,i);return}if(u==="cancel"){await E(e,u),this.teardownBroker(e,i);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){await E(e,"park"),this.teardownBroker(e,i);return}o.lifecycle.acceptsDetachedWork()&&(await Ae(e,n),o.lifecycle.registerDetached(this.requestParkSummary(e,i)))}}async requestParkSummary(e,t){try{const r=await be(e,Te("park",{},Pe),{origin:{type:"runtime",kind:"park"}});f({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:r}})}catch(r){if(await $(e)===void 0)return;if(r instanceof Se){await E(e,"park"),this.teardownBroker(e,t);return}f({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Ue)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:o}=t,l=r.stalled+r.waitingOnMail;f({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:o,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const d=l===0?"":` \u2014 ${l} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,s=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${o})${d} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const p=await R("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof p.stdout!="string")return;await Promise.allSettled(p.stdout.split(`
+`).map(async u=>{if(u!=="")try{await R("tmux",["display-message","-C","-d","10000","-c",u,s],{encoding:"utf8",timeout:2e3})}catch{try{await R("tmux",["display-message","-c",u,s],{encoding:"utf8",timeout:2e3})}catch(w){f({level:"error",event:"broker.capacity.tmux_display_failed",error:w,fields:{client:u}})}}}))}catch(p){f({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:p})}}async teardownBroker(e,t){return t===null?await pe.teardown(e):await this.stopRemote(t,"teardown")}async stopRemote(e,t){await _e(e.nodeId,{afterTurn:!1,reason:t});const r=await he(e.runner).stop(e,{reason:t});return r.confirmed?await me(e.executionId,{reason:t}):f({level:"error",event:"broker.teardown.unconfirmed",node_id:e.nodeId,error:r.error}),r}async handleRemoteYieldStall(e,t,r){const i=e.node_id;if(e.intent!=="refresh"||await S(e)!==null){await h(i,"stall_noticed_at");return}const n=await M(i,"stall_noticed_at");if(n===void 0){await A(i,"stall_noticed_at",r);return}V(!0,e.intent,!1,r-n)==="kill"&&(await C(i))?.executionId===t.executionId&&(f({level:"warn",event:"broker.yield_stall.remote_stop",...c(i)?{node_id:i}:{},fields:{...c(i)?{}:{affected_node_id:i},execution_id:t.executionId,elapsed_ms:r-n,grace_ms:W}}),await this.stopRemote(t,"yield_stall"))}async handleRemoteWedge(e,t,r){const i=e.node_id,n=await S(e);if(n===null){await h(i,"wedge_noticed_at");return}const o=r-n;if(o<I)return;const l=await P(i);if(l===null||(await C(i))?.executionId!==t.executionId||await S(l)!==n||await M(i,"wedge_noticed_at")===n)return;await A(i,"wedge_noticed_at",n);const d=Math.round(o/6e4);await this.reportWedge(i,{quietForMs:o,cpuPercent:null,hasDescendants:null,kickAttempted:!1,kicked:!1,kickSuppressionReason:"remote_no_process_evidence",pid:t.pid,label:`Child wedged \u2014 ${x(l)} (${i}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${d}+ minutes. The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`})}async handleYieldStall(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||await S(e)!==null||o===null){await h(n,"stall_noticed_at"),o!==null&&this.yieldTermedAt.delete(o);return}const l=await M(n,"stall_noticed_at");if(l===void 0){await A(n,"stall_noticed_at",i);return}const d={since:l,termedAt:this.yieldTermedAt.get(o)};if(V(!0,e.intent,!1,i-d.since)!=="kill")return;const s=await P(n);if(!(r.get(n)!==t||s?.pi_pid!==t.pid||s.pi_pid_identity!==e.pi_pid_identity||B(t.pid,e.pi_pid_identity)!=="alive"))if(d.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;this.yieldTermedAt.set(o,i),d.termedAt=i,f({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.since,grace_ms:W}})}else i-d.termedAt>=Y&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&f({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-d.termedAt,grace_ms:Y,stall_elapsed_ms:i-d.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return f({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await R("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Ke(t,e),i=ze(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,o=U(t.pid,e.pi_pid_identity),l=await S(e);if(l===null||o===null){await h(n,"wedge_noticed_at");return}const d=i-l,s=d>=I?await this.sampleTree(t.pid):null,p=await P(n),u=p===null?null:await S(p);if(r.get(n)!==t||p===null||p.pi_pid!==t.pid||p.pi_pid_identity!==e.pi_pid_identity||u!==l)return;const w=s?.cpuPercent??null,v=s?.hasDescendants??null;if(Ve({busy:!0,quietForMs:d,cpuPercent:w,hasDescendants:v})!=="wedged"||await M(n,"wedge_noticed_at")===l)return;await A(n,"wedge_noticed_at",l);const k=x(p),y=Math.round((d??I)/6e4),b=v??!0,T=!b&&B(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let N=!1,_=!1,g;if(b)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(T!==null)g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{N=!0;try{await this.signal(t.pid,"SIGTERM"),_=!0,g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(z){f({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:z,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),g=`Child wedged \u2014 ${k} (${n}) is alive and mid-turn but has shown NO engine progress for ${y}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}await this.reportWedge(n,{quietForMs:d,cpuPercent:w,hasDescendants:b,kickAttempted:N,kicked:_,kickSuppressionReason:T,pid:t.pid,label:g})}async reportWedge(e,t){f({level:"error",event:"broker.wedge.detected",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},pid:t.pid,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,has_descendants:t.hasDescendants,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}}});try{await oe(e,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:t.label})}catch(r){f({level:"error",event:"broker.wedge.fault_record_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:r})}await L(e,"notice.crash",t.label,{reason:"child-wedged",child:e,quiet_ms:t.quietForMs,cpu_percent:t.cpuPercent,kick_attempted:t.kickAttempted,kicked:t.kicked,...t.kickSuppressionReason===null?{}:{kick_suppression_reason:t.kickSuppressionReason}})}async handleFatalFault(e,t=!0){const r=await G(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=await P(e),n=await q(e),o=i!==null?x(i):e,l=r.kind==="auth",d=(r.message.split(`
+`)[0]??"").slice(0,200),s=r.providerError,p=s===void 0?"":` Provider: ${s.provider}/${s.model}.`+(s.status===void 0?"":` HTTP status: ${s.status}.`)+(s.errno===void 0?"":` Errno: ${s.errno}.`)+(s.code===void 0?"":` Code: ${s.code}.`)+(s.body===void 0?"":` Body: ${s.body.slice(0,200)}.`)+(s.ratelimit===void 0?"":` Rate limit: ${s.ratelimit.scope??"account"}${s.ratelimit.claim===void 0?"":` (${s.ratelimit.claim})`}${s.ratelimit.model===void 0?"":` on ${s.ratelimit.model}`}.`),u=r.kind==="context-overflow",w=Ye(i,typeof s?.code=="string"?s.code:void 0),v=u||w,k=u?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${d}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(w?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${d}.${p}`,y=u?"failed":"hit a fatal provider fault",b=new Set((await ne(e)).map(_=>_.node_id)),T=await fe(await O(),await ue(await O(),e)),N=[{relation:"Child",recipients:T.filter(_=>b.has(_))},{relation:"Watched node",recipients:T.filter(_=>!b.has(_))}];f({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:u,terminal_provider_fault:w}}),await ce(async()=>{for(const{relation:_,recipients:g}of N)g.length!==0&&await L(e,"notice.crash",`${_} ${y} \u2014 ${o} (${e}) ${k}`,{reason:u?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:d,...s===void 0?{}:{provider_error:s}},"normal",null,g);await se(r.faultId),t&&v&&await re(e,"crash",{reason:u?"context_overflow":s?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:le(r).class}}})}),t&&v&&this.teardownBroker(e,n).then(_=>{_.confirmed||f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_.error??new Error("Broker teardown not confirmed")})},_=>f({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:_}))}}export{Ft as BrokerSupervisionReconciler,qe as WEDGE_CPU_MAX_PERCENT,I as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,H as parkSummaryGraceMsForDaemon,Ye as settlesOnFatalProviderFault,Ke as sumTreeCpu,ze as treeHasDescendants,He as unattendedParkMsForDaemon,Ve as wedgeVerdict,V as yieldStallVerdict};

package/dist/daemon/reconcilers/live-obligation.d.ts

@@ -50,6 +50,13 @@ export declare function hasHumanViewer(nodeId: string, viewer?: ViewerProbe): Pr
  */
 export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
 export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<ParkEvent>;
+/** `parkEvent` plus whether a live subscriber awaiting the report is the ONLY thing holding a
+ *  terminal node's `release`: no viewer, no wake source, no unread mail, no fatal fault. Such a
+ *  node stopped unfinished with nothing that will wake it, and its subscriber knows nothing. */
+export declare function parkVerdict(nodeId: string, lifecycle: Lifecycle, viewer?: ViewerProbe): Promise<{
+    event: ParkEvent;
+    heldOnlyBySubscriber: boolean;
+}>;
 /** Apply an unattended decision and notify subscribers of automatic completion.
  *  This is the DEGRADED enactment: no summary turn ran, so subscribers get the
  *  doctrine notice instead of the node's own report. The live clock's good path
@@ -64,3 +71,7 @@ export declare function parkEvent(nodeId: string, lifecycle: Lifecycle, viewer?:
  *  resident throughout: its conversation ended without a final report, its state
  *  lives in whatever report it did leave, and a message reopens it. */
 export declare function applyParkEvent(nodeId: string, event: ParkEvent): Promise<void>;
+/** Tell a terminal node's subscribers it went idle unfinished: it pushed no final, holds no wait
+ *  that will wake it, and only their subscription kept it from being canceled. The cause of the
+ *  stop is not known here, so the notice claims none. */
+export declare function noticeUnfinishedIdle(nodeId: string): Promise<void>;

package/dist/daemon/reconcilers/live-obligation.js

@@ -1 +1 @@
-var l=Object.defineProperty;var r=(t,a)=>l(t,"name",{value:a,configurable:!0});import{hasActiveLiveSubscription as f,hasLiveSubscriber as p,hasPendingCronWake as w}from"../../core/canvas/index.js";import{fullName as m}from"../../core/canvas/labels.js";import{hasLiveControllerWait as h,hasLiveObjectWait as k}from"../../core/canvas/waits.js";import{newestReportName as y}from"../../core/feed/reports.js";import{countTickets as b}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as n}from"../../core/canvas/deliveries.js";import{readOutputRequest as g}from"../../core/runtime/output-result.js";import{focusOf as v}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as x}from"../api/viewer-presence.js";import{publishNotice as L}from"../../core/runtime/close.js";import{readFault as W}from"../../core/canvas/recovery.js";import{transition as c}from"../../core/runtime/lifecycle.js";async function s(t){return await f(t)||await h(t)||await k(t)||await w(t)||await b(t)>0||await g(t)!==null}r(s,"hasLiveObligation");async function B(t){return await s(t)||await n(t)?!0:await o(t)}r(B,"hasPendingWake");async function F(t){return async a=>await t.isClaimed(a)||await s(a)||await o(a)}r(F,"retentionHold");async function o(t,a){return(a!==void 0?a.live!==null:await v(t)!==null)||await x(t)}r(o,"hasHumanViewer");async function M(t,a,e){if(await o(t,e)||await s(t))return"release";const i=await W(t);return i?.link==="pi\u2192provider"&&i.retry.disposition==="fatal"?"release":a==="resident"?await n(t)?"release":"park":await n(t)||await p(t)?"release":"cancel"}r(M,"parkEvent");async function T(t,a){const e=a==="cancel"?await c(t,a,{reason:"stranded"}):await c(t,a);if(a!=="park")return;const i=await y(t),u=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await L(t,"notice.close",`Resident concluded \u2014 ${m(e)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${u} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}r(T,"applyParkEvent");export{T as applyParkEvent,o as hasHumanViewer,s as hasLiveObligation,B as hasPendingWake,M as parkEvent,F as retentionHold};
+var w=Object.defineProperty;var a=(t,e)=>w(t,"name",{value:e,configurable:!0});import{getNode as h,hasActiveLiveSubscription as m,hasLiveSubscriber as y,hasPendingCronWake as g}from"../../core/canvas/index.js";import{fullName as l}from"../../core/canvas/labels.js";import{hasLiveControllerWait as k,hasLiveObjectWait as b}from"../../core/canvas/waits.js";import{newestReportName as v}from"../../core/feed/reports.js";import{countTickets as x}from"../../core/canvas/attention.js";import{hasUnseenWakeMail as s}from"../../core/canvas/deliveries.js";import{readOutputRequest as $}from"../../core/runtime/output-result.js";import{focusOf as O}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as d}from"../api/viewer-presence.js";import{publishNotice as u}from"../../core/runtime/close.js";import{readFault as L}from"../../core/canvas/recovery.js";import{transition as f}from"../../core/runtime/lifecycle.js";async function o(t){return await m(t)||await k(t)||await b(t)||await g(t)||await x(t)>0||await $(t)!==null}a(o,"hasLiveObligation");async function F(t){return await o(t)||await s(t)?!0:await c(t)}a(F,"hasPendingWake");async function M(t){return async e=>await t.isClaimed(e)||await o(e)||await c(e)}a(M,"retentionHold");async function c(t,e){return(e!==void 0?e.live!==null:await O(t)!==null)||await d(t)}a(c,"hasHumanViewer");async function T(t,e,r){return(await W(t,e,r)).event}a(T,"parkEvent");async function W(t,e,r){const i=a(p=>({event:p,heldOnlyBySubscriber:!1}),"held");if(await c(t,r)||await o(t))return i("release");const n=await L(t);return n?.link==="pi\u2192provider"&&n.retry.disposition==="fatal"?i("release"):e==="resident"?i(await s(t)?"release":"park"):await s(t)?i("release"):await y(t)?{event:"release",heldOnlyBySubscriber:!0}:i("cancel")}a(W,"parkVerdict");async function z(t,e){const r=e==="cancel"?await f(t,e,{reason:"stranded"}):await f(t,e);if(e!=="park")return;const i=await v(t),n=i===null?`It left no report behind; \`crtr node inspect show ${t}\` shows where it stands.`:`Its latest report is where it left things: \`crtr canvas read ${i}\`.`;await u(t,"notice.close",`Resident concluded \u2014 ${l(r)} (${t}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${n} A message to it reopens it on a fresh cycle.`,{reason:"child-auto-done",child:t},"deferred")}a(z,"applyParkEvent");async function D(t){const e=await h(t);e!==null&&await u(t,"notice.crash",`Child went idle unfinished \u2014 ${l(e)} (${t}) stopped without pushing a final report and has no wait that will wake it. Its engine was released; the cause is not recorded. \`crtr node inspect show ${t}\` shows where it stands; a message to it revives it.`,{reason:"child-idle-unfinished",child:t},"urgent")}a(D,"noticeUnfinishedIdle");export{z as applyParkEvent,c as hasHumanViewer,o as hasLiveObligation,F as hasPendingWake,D as noticeUnfinishedIdle,T as parkEvent,W as parkVerdict,M as retentionHold};

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -1 +1 @@
-var E=Object.defineProperty;var l=(r,t)=>E(r,"name",{value:t,configurable:!0});import{clearFrozen as m,endLiveExecution as h,getNode as R}from"../../../core/canvas/index.js";import{isSafeNodeId as b}from"../../../core/canvas/paths.js";import{recordedPidLiveness as _}from"../../../core/canvas/pid.js";import{hasObservedExit as L,hasAbsorbedProfilePauseStop as z}from"../../../core/canvas/executions.js";import{emitEvent as y}from"../../../core/events/emit.js";import{operationIdContext as I}from"../../../core/events/operation-id.js";import{readInterruptedTurn as g}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as v}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as w,isParkedResident as N}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as M}from"../../fleet.js";import{applyParkEvent as F,hasPendingWake as x,parkEvent as A}from"../live-obligation.js";import{FreezeLaneReconciler as C}from"./freeze-lane.js";import{isInBackoff as P}from"./respawn-policy.js";import{asNodePerson as D,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as S}from"./terminating.js";import{wantsExecution as T}from"./wants-execution.js";import{enactRequestedStops as U}from"./requested-stops.js";import{endRemoteExecutionIfGone as G,recoverLaunching as H,wakeResident as W}from"./remote-wake.js";import{liveExecutionForNode as q}from"../../../core/canvas/executions.js";function B(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}l(B,"isTerminal");const O=12e4;function j(r,t){const e=Math.max(...[r.created,r.launched_at].map(a=>a==null?0:Date.parse(a)||0));return e>0&&t-e<O}l(j,"isLaunchInFlight");class me{static{l(this,"NodeLifecycleReconciler")}freezeLane=new C;async run(t,e){const a=[];await U(t);for(const n of e.rows)await I.fresh(async()=>{try{await D(n,()=>this.classifyRow(t,n,e,a))}catch(i){const s=n.node_id;y({level:i instanceof k?"warn":"error",event:i instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...b(s)?{node_id:s}:{fields:{affected_node_id:s}},error:i})}});return this.freezeLane.run(t,{fleet:e.fleet,candidates:a})}async classifyRow(t,e,a,n){const i=e.node_id;if(B(e)){if(S(e,await a.fleet.has(i)))return;if(N(e)&&!await a.fleet.has(i)&&!d(e.profile_id)&&!await v(i)&&!P(e,t)&&await p(i)){n.push({row:e,enact:"reopen"});return}e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead"&&await h(i,{reason:"observed-dead"}),e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await m(i):n.push({row:e,enact:"thaw"}));return}if(await a.fleet.has(i)){const f=await q(i);if(f?.runner==="blaxel"&&f.state==="running"&&!d(e.profile_id)&&await p(i)){if(await W(f)==="lost")return}else f?.runner==="blaxel"&&f.state==="running"&&await G(f);if(await a.fleet.has(i)){d(e.profile_id)?await a.fleet.interruptForProfilePause(i):await this.thawMark(e);return}}if(await H(i,t),d(e.profile_id)){await this.freezeLane.endEpisode(i);return}if(await v(i))return;const s=e.pi_pid==null&&await L(i),u=e.pi_pid==null&&!s&&await z(i),c=s||u||e.pi_pid!=null&&_(e.pi_pid,e.pi_pid_identity)==="dead";if(c&&!await a.fleet.hasProfilePauseInterruption(i)&&await h(i,{reason:"observed-dead"}),!await T(e)){await this.thawMark(e),await this.settleParked(e);return}if(P(e,t))return;if(e.status==="idle"&&w(e)&&await p(i)){n.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!s||e.frozen_at!=null)&&!u){if(e.respawn_not_before!=null){n.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){n.push({row:e,enact:"thaw"});return}if(await a.fleet.isClaimed(i)||j(e,t))return;y({level:"warn",event:"node.lifecycle.stranded_launch",node_id:i}),n.push({row:e,enact:"thaw"});return}if(!c)return;const o=M(await R(i),(await g(i))?.interrupted===!0,await x(i));if(o==="respawn-fresh"||o==="respawn-cycle"||o==="respawn-resume"){n.push({row:e,enact:"policy"});return}await a.fleet.applyDeadRowPolicy(i),await this.thawMark(e)}async thawMark(t){t.frozen_at!=null&&await m(t.node_id)}async settleParked(t){if(t.status!=="idle")return;await this.freezeLane.endEpisode(t.node_id);const e=await A(t.node_id,t.lifecycle);e!=="release"&&await F(t.node_id,e)}}export{O as LAUNCH_GRACE_MS,me as NodeLifecycleReconciler,j as isLaunchInFlight};
+var b=Object.defineProperty;var o=(r,a)=>b(r,"name",{value:a,configurable:!0});import{clearFrozen as P,getNode as x}from"../../../core/canvas/index.js";import{isSafeNodeId as z}from"../../../core/canvas/paths.js";import{recordedPidLiveness as L}from"../../../core/canvas/pid.js";import{endExecution as v,hasAbsorbedProfilePauseStop as g,hasObservedExit as N,latestExecutionForNode as M,liveExecutionForNode as c}from"../../../core/canvas/executions.js";import{emitEvent as w}from"../../../core/events/emit.js";import{operationIdContext as F}from"../../../core/events/operation-id.js";import{readInterruptedTurn as D}from"../../../core/canvas/recovery.js";import{isProfilePaused as d}from"../../../core/profiles/manifest.js";import{isRecoveryHeld as I}from"../../../core/runtime/recovery-hold.js";import{hasUnseenWakeMail as p}from"../../../core/canvas/deliveries.js";import{hasIdleReleaseIntent as A,isParkedResident as C}from"../../../core/runtime/lifecycle.js";import{classifyDeadNode as S}from"../../fleet.js";import{applyParkEvent as T,hasPendingWake as U,parkEvent as W}from"../live-obligation.js";import{FreezeLaneReconciler as B}from"./freeze-lane.js";import{isInBackoff as E}from"./respawn-policy.js";import{asNodePerson as G,NodePersonUnresolvedError as k}from"./node-person.js";import{handleTerminatingRow as H}from"./terminating.js";import{wantsExecution as q}from"./wants-execution.js";import{enactRequestedStops as O}from"./requested-stops.js";import{endRemoteExecutionIfGone as j,recoverLaunching as J,wakeResident as K}from"./remote-wake.js";function Q(r){return r.status==="done"||r.status==="canceled"||r.status==="dead"}o(Q,"isTerminal");const V=12e4;function X(r,a){const e=Math.max(...[r.created,r.launched_at].map(i=>i==null?0:Date.parse(i)||0));return e>0&&a-e<V}o(X,"isLaunchInFlight");async function R(r){const a=await c(r);return a!==null&&a.state==="running"&&a.pid!=null&&L(a.pid,a.processIdentity)==="dead"?a:null}o(R,"executionWithDeadPid");class Pe{static{o(this,"NodeLifecycleReconciler")}freezeLane=new B;async run(a,e){const i=[];await O(a);for(const s of e.rows)await F.fresh(async()=>{try{await G(s,()=>this.classifyRow(a,s,e,i))}catch(t){const l=s.node_id;w({level:t instanceof k?"warn":"error",event:t instanceof k?"node.lifecycle.person_unresolved":"node.lifecycle.classify_failed",...z(l)?{node_id:l}:{fields:{affected_node_id:l}},error:t})}});return this.freezeLane.run(a,{fleet:e.fleet,candidates:i})}async classifyRow(a,e,i,s){const t=e.node_id;if(Q(e)){if(H(e,await i.fleet.has(t)))return;if(C(e)&&!await i.fleet.has(t)&&!d(e.profile_id)&&!await I(t)&&!E(e,a)&&await p(t)){s.push({row:e,enact:"reopen"});return}if(e.pi_pid!=null){const n=await R(t);n!==null&&await v(n.executionId,{reason:"observed-dead"})}e.frozen_at!=null&&(e.final_report!=null||e.status==="dead"?await P(t):s.push({row:e,enact:"thaw"}));return}if(await i.fleet.has(t)){const n=await c(t);if(n?.runner==="blaxel"&&n.state==="running"&&!d(e.profile_id)&&await p(t)){if(await K(n)==="lost")return}else n?.runner==="blaxel"&&n.state==="running"&&await j(n);if(await i.fleet.has(t)){d(e.profile_id)?await i.fleet.interruptForProfilePause(t):await this.thawMark(e);return}}if(await J(t,a),d(e.profile_id)){await this.freezeLane.endEpisode(t);return}if(await I(t))return;const l=e.pi_pid==null?await M(t):null,f=l!==null&&N(l),m=l!==null&&!f&&g(l),h=e.pi_pid!=null?await R(t):null,_=f||m||h!==null;if(_&&!await i.fleet.hasProfilePauseInterruption(t)){const n=h??l;n!==null&&await v(n.executionId,{reason:"observed-dead"});const y=await c(t);if(y!==null&&y.executionId!==n?.executionId)return}if(!await q(e)){await this.thawMark(e),await this.settleParked(e);return}if(E(e,a))return;if(e.status==="idle"&&A(e)&&await p(t)){s.push({row:e,enact:"idle-release"});return}if(e.pi_pid==null&&(!f||e.frozen_at!=null)&&!m){if(e.respawn_not_before!=null){s.push({row:e,enact:"policy"});return}if(e.frozen_at!=null){s.push({row:e,enact:"thaw"});return}if(await i.fleet.isClaimed(t)||X(e,a))return;w({level:"warn",event:"node.lifecycle.stranded_launch",node_id:t}),s.push({row:e,enact:"thaw"});return}if(!_)return;const u=S(await x(t),(await D(t))?.interrupted===!0,await U(t));if(u==="respawn-fresh"||u==="respawn-cycle"||u==="respawn-resume"){s.push({row:e,enact:"policy"});return}await i.fleet.applyDeadRowPolicy(t),await this.thawMark(e)}async thawMark(a){a.frozen_at!=null&&await P(a.node_id)}async settleParked(a){if(a.status!=="idle")return;await this.freezeLane.endEpisode(a.node_id);const e=await W(a.node_id,a.lifecycle);e!=="release"&&await T(a.node_id,e)}}export{V as LAUNCH_GRACE_MS,Pe as NodeLifecycleReconciler,X as isLaunchInFlight};