@north-light/cli 0.3.579 → 0.3.581

package/dist/daemon/fleet.js

@@ -1,4 +1,4 @@
-var P=Object.defineProperty;var l=(i,e)=>P(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as R}from"../core/canvas/index.js";import{revokeExitedNodeToken as T}from"../core/canvas/node-tokens.js";import{PROFILE_PAUSE_STOP_REASON as _,absorbProfilePauseStop as N,endExecution as y,hasLaunchingExecution as S,hasProfilePauseStop as F,hasRunningRemoteExecution as b,liveExecutionForNode as g,liveExecutionNodeIds as C,markProfilePauseStop as k,recordExecutionExit as O,takeObservedExit as M}from"../core/canvas/executions.js";import{isProfilePaused as z}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as A,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as L}from"../core/runtime/close.js";import{readFault as B,readInterruptedTurn as D,readTransportPoisonVerdict as W}from"../core/canvas/recovery.js";import{hasPendingWake as q}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as G,MAX_RESPAWN_ATTEMPTS as H,OFFLINE_RESPAWN_RETRY_MS as Y,recordRespawnPolicy as U,respawnBackoffMs as V}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as j}from"../core/runtime/connectivity.js";import{runnerForExecution as K}from"../core/runtime/runner/index.js";import{reviveNode as X}from"../core/runtime/revive.js";import{isStartRefusal as J}from"../core/runtime/start-check.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as Q}from"../core/runtime/fleet.js";import{isRecoveryHeld as Z}from"../core/runtime/recovery-hold.js";async function $(i,e){const r=await B(i.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
+var P=Object.defineProperty;var l=(i,e)=>P(i,"name",{value:e,configurable:!0});import{getNode as w,getRow as R}from"../core/canvas/index.js";import{revokeExitedNodeToken as T}from"../core/canvas/node-tokens.js";import{PROFILE_PAUSE_STOP_REASON as _,absorbProfilePauseStop as N,endExecution as y,hasLaunchingExecution as S,hasProfilePauseStop as F,hasRunningRemoteExecution as b,liveExecutionForNode as g,liveExecutionNodeIds as C,markProfilePauseStop as k,recordExecutionExit as O,takeObservedExit as M}from"../core/canvas/executions.js";import{isProfilePaused as z}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as o}from"../core/canvas/paths.js";import{hasIdleReleaseIntent as A,transition as v}from"../core/runtime/lifecycle.js";import{publishNotice as L}from"../core/runtime/close.js";import{readFault as B,readInterruptedTurn as D,readTransportPoisonVerdict as W}from"../core/canvas/recovery.js";import{hasPendingWake as q}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as G,MAX_RESPAWN_ATTEMPTS as H,OFFLINE_RESPAWN_RETRY_MS as Y,recordRespawnPolicy as U,respawnBackoffMs as V}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as j}from"../core/runtime/connectivity.js";import{runnerForExecution as K}from"../core/runtime/runner/index.js";import{reviveNode as X}from"../core/runtime/revive.js";import{isStartRefusal as J}from"../core/runtime/start-check.js";import{MachineOfflineError as Q}from"../core/runtime/runner/machine/ops.js";import{pushUrgent as m}from"../core/feed/feed.js";import{emitEvent as u}from"../core/events/emit.js";import{brokerThresholdsForDaemon as Z}from"../core/runtime/fleet.js";import{isRecoveryHeld as I}from"../core/runtime/recovery-hold.js";async function E(i,e){const r=await B(i.node_id),t=r===null?"":`The broker reported a ${r.kind} fault on ${r.link} (${r.op}) before it exited: ${r.message}
 
 `,s=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${i.name}\` (${i.kind}) never started.
 
@@ -6,16 +6,16 @@ Its pi vehicle exited before the session came up (no pi_session_id was ever reco
 
 `+t+`Inspect canonical event diagnostics with \`crtr sys logs --node ${i.node_id}\`.
 
-If the work still needs doing, re-spawn it.`}l($,"bootFailureBody");async function E(i,e){await m(i.node_id,await $(i,e),{from:i.node_id})}l(E,"surfaceBootFailure");function x(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
+If the work still needs doing, re-spawn it.`}l(E,"bootFailureBody");async function $(i,e){await m(i.node_id,await E(i,e),{from:i.node_id})}l($,"surfaceBootFailure");function x(i){return`\u26A0 Out of memory \u2014 \`${i.name}\` (${i.kind}) was killed because its app's processes together passed the app's memory limit.
 
-Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(x,"outOfMemoryBody");async function I(i){const e=await W(i.node_id),r=e===null?"":`
+Every process of the run was stopped. The daemon does not restart it, because it would reach the same limit. Reduce what the run holds in memory, or ask the person to raise the app's limit (\`crtr sys limits set --app ${i.grantee??"app:terminal"} --memory <size>\`), then revive the node.`}l(x,"outOfMemoryBody");async function ee(i){const e=await W(i.node_id),r=e===null?"":`
 
 Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${i.name}\` (${i.kind}) kept dying without settling a turn.
 
 Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
 
-Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(I,"surfaceCrashLoop");async function ee(i,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
+Fix the crash cause, then revive the node if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(ee,"surfaceCrashLoop");async function te(i,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${i.name}\` (${i.kind}) could not be relaunched.
 
 The daemon refused this recovery before starting a broker: ${r}
 
-The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(ee,"surfaceRecoveryFailure");function te(i){return i?.cycle_pending!==!0}l(te,"retryResumeMode");function re(i,e,r=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":A(i)?"dormant":i.lifecycle==="resident"&&!e&&!r&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?te(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(re,"classifyDeadNode");const ie=3e4;class Ee{static{l(this,"DaemonFleet")}#a;#n;#r;#o;#e=new Map;#t=new Set;#l=Number.NEGATIVE_INFINITY;constructor(e){this.#a=e.epoch,this.#n=e.enqueue,this.#r=e.now??Date.now,this.#o=e.probeOnline??(()=>j())}epoch(){return this.#a}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}async has(e){return this.#e.has(e)||await b(e)}async isClaimed(e){return this.#e.has(e)||this.#t.has(e)||await b(e)||await S(e,this.#a)}get(e){return this.#e.get(e)}size(){return this.#e.size}async#c(){const e=await C();for(const r of this.#e.keys())e.add(r);for(const r of this.#t)e.add(r);return e}async occupancy(){return(await this.#c()).size}async reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=Q(),t=await this.#c();if(this.#t.has(e)||this.#e.has(e))return!0;if(t.delete(e),t.size<r)return this.#t.has(e)||this.#t.add(e),!0;const s=this.#r();return s-this.#l>=ie&&(this.#l=s,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:t.size,hard_cap:r}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}async interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r===void 0){const t=await g(e);if(t===null||t.runner==="process"||t.state!=="running")return;await k(e),(await K(t.runner).stop(t,{reason:_})).confirmed&&await y(t.executionId,{reason:_});return}await k(e);try{process.kill(r,"SIGTERM")}catch{}}async hasProfilePauseInterruption(e){return await F(e)}forget(e){this.#e.delete(e),this.#t.delete(e)}#i=new Set;async exitsSettled(){for(;this.#i.size>0;)await Promise.allSettled([...this.#i])}async onChildExit(e,r,t){const s=this.#u(e,r,t);this.#i.add(s);try{await s}finally{this.#i.delete(s)}}async#u(e,r,t){const s=t??(await g(e))?.executionId;t&&await T(t),this.#e.delete(e),s!==void 0&&(await O(s,r),await y(s,{reason:"exited"})),this.#n(async()=>{const a=await w(e),n=a?.launched_at==null?Number.NaN:Date.parse(a.launched_at),d=Number.isFinite(n)?Math.max(0,this.#r()-n):null,f=a?.respawn_failures??0;this.#f(e,r,d,f)})}async deliverExit(e,r){await this.onChildExit(e,r)}async applyDeadRowPolicy(e){await this.#h(e,await M(e))}async terminalizeIfStillNonTerminal(e,r,t,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#s(e,n,{event:r,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>ee(n,t),"surface")}),!0)}dispose(){}#f(e,r,t,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:s}})}async#h(e,r){return this.#d(e,r)}async#d(e,r){const t=await w(e);if(t!==null&&z(t.profile_id))return await h(e),"forget";if(await N(e))return await h(e),"forget";if(await Z(e))return"dormant";const s=(await D(e))?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return await h(e),await this.#s(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:x(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,x(t),{from:e})},"surface")}),"forget";const a=re(t,s,await q(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#s(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await $(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>E(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#p(e,t,a,s);break}return a}async#p(e,r,t,s){const a=await R(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#o()){await G(e,n,this.#r()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:Y,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await U(a,this.#r()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=H};if(f.exhausted){const c=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#s(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...r.pi_session_id!=null?{surface:l(()=>I(r),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:V(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await X(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(J(c)){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#s(e,r,t){u({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>E(r)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await L(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{Ee as DaemonFleet,re as classifyDeadNode,te as retryResumeMode,E as surfaceBootFailure};
+The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await m(i.node_id,t,{from:i.node_id})}l(te,"surfaceRecoveryFailure");function re(i){return i?.cycle_pending!==!0}l(re,"retryResumeMode");function ie(i,e,r=!0){return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?"forget":i.intent==="refresh"?"respawn-fresh":A(i)?"dormant":i.lifecycle==="resident"&&!e&&!r&&i.pi_session_id!=null?"release-idle":i.pi_session_id!=null?re(i)?"respawn-resume":"respawn-cycle":e||i.fork_from!=null?"respawn-fresh":"boot-failure"}l(ie,"classifyDeadNode");const se=3e4;class Pe{static{l(this,"DaemonFleet")}#a;#n;#r;#o;#e=new Map;#t=new Set;#l=Number.NEGATIVE_INFINITY;constructor(e){this.#a=e.epoch,this.#n=e.enqueue,this.#r=e.now??Date.now,this.#o=e.probeOnline??(()=>j())}epoch(){return this.#a}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}async has(e){return this.#e.has(e)||await b(e)}async isClaimed(e){return this.#e.has(e)||this.#t.has(e)||await b(e)||await S(e,this.#a)}get(e){return this.#e.get(e)}size(){return this.#e.size}async#c(){const e=await C();for(const r of this.#e.keys())e.add(r);for(const r of this.#t)e.add(r);return e}async occupancy(){return(await this.#c()).size}async reserveSlot(e){if(this.#t.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=Z(),t=await this.#c();if(this.#t.has(e)||this.#e.has(e))return!0;if(t.delete(e),t.size<r)return this.#t.has(e)||this.#t.add(e),!0;const s=this.#r();return s-this.#l>=se&&(this.#l=s,u({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:t.size,hard_cap:r}})),!1}releaseReservation(e){this.#t.delete(e)}entries(){return this.#e.entries()}async interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r===void 0){const t=await g(e);if(t===null||t.runner==="process"||t.state!=="running")return;await k(e),(await K(t.runner).stop(t,{reason:_})).confirmed&&await y(t.executionId,{reason:_});return}await k(e);try{process.kill(r,"SIGTERM")}catch{}}async hasProfilePauseInterruption(e){return await F(e)}forget(e){this.#e.delete(e),this.#t.delete(e)}#i=new Set;async exitsSettled(){for(;this.#i.size>0;)await Promise.allSettled([...this.#i])}async onChildExit(e,r,t){const s=this.#u(e,r,t);this.#i.add(s);try{await s}finally{this.#i.delete(s)}}async#u(e,r,t){const s=t??(await g(e))?.executionId;t&&await T(t),this.#e.delete(e),s!==void 0&&(await O(s,r),await y(s,{reason:"exited"})),this.#n(async()=>{const a=await w(e),n=a?.launched_at==null?Number.NaN:Date.parse(a.launched_at),d=Number.isFinite(n)?Math.max(0,this.#r()-n):null,f=a?.respawn_failures??0;this.#f(e,r,d,f)})}async deliverExit(e,r){await this.onChildExit(e,r)}async applyDeadRowPolicy(e){await this.#h(e,await M(e))}async terminalizeIfStillNonTerminal(e,r,t,s,a){const n=await w(e);return n===null||n.status==="done"||n.status==="canceled"||n.status==="dead"?!1:(await this.#s(e,n,{event:r,reason:s,detail:a,wake:`Child crashed \u2014 ${p(n)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>te(n,t),"surface")}),!0)}dispose(){}#f(e,r,t,s){u({level:"info",event:"broker.exit.observed",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:s}})}async#h(e,r){return this.#d(e,r)}async#d(e,r){const t=await w(e);if(t!==null&&z(t.profile_id))return await h(e),"forget";if(await N(e))return await h(e),"forget";if(await I(e))return"dormant";const s=(await D(e))?.interrupted===!0;if(r?.oomKilled===!0&&t!==null&&t.status!=="done"&&t.status!=="canceled"&&t.status!=="dead")return await h(e),await this.#s(e,t,{event:"broker.out_of_memory",reason:"out_of_memory",detail:{schema:"crtr.node-outcome-detail/v1",message:x(t)},wake:`Child out of memory \u2014 ${p(t)} (${e}) used more memory than its app is allowed and is now dead. It is not restarted automatically. Revive it with \`crtr node lifecycle revive ${e}\` once the work needs less memory or the cap is raised (\`crtr sys limits set\`).`,surface:l(async()=>{await m(e,x(t),{from:e})},"surface")}),"forget";const a=ie(t,s,await q(e));switch(a){case"forget":await h(e);break;case"dormant":await h(e);break;case"release-idle":await h(e),await v(e,"release"),u({level:"info",event:"broker.recovery.released",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#s(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:await E(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>$(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#p(e,t,a,s);break}return a}async#p(e,r,t,s){const a=await R(e);if(a===null)return;const n=a.respawn_failures??0;if(!await this.#o()){await G(e,n,this.#r()),u({level:"info",event:"broker.respawn.offline_deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,retry_in_ms:Y,consecutive_failures:n}});return}const d=a.respawn_not_before==null,f=d?await U(a,this.#r()):{failures:n,notBefore:a.respawn_not_before,exhausted:n>=H};if(f.exhausted){const c=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#s(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:c,respawn_failures:f.failures},wake:c,...r.pi_session_id!=null?{surface:l(()=>ee(r),"surface")}:{}});return}if(d&&f.notBefore!==null){u({level:"info",event:"broker.respawn.deferred",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,backoff_ms:V(f.failures),consecutive_failures:f.failures}});return}u({level:"info",event:"broker.respawn.started",...o(e)?{node_id:e}:{},fields:{...o(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await X(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&s?"runtime-restart-abort":void 0,recovery:!0})}catch(c){if(J(c)||c instanceof Q){await h(e),await v(e,"release"),u({level:"warn",event:"node.start.refused",node_id:e,error:c,fields:{enact:"respawn"}});return}u({level:"error",event:"broker.respawn.failed",...o(e)?{node_id:e}:{},error:c,fields:{...o(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",c,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:c instanceof Error?c.message:String(c)})}}async#s(e,r,t){u({level:"error",event:t.event,...o(e)?{node_id:e}:{fields:{affected_node_id:e}}}),await v(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>$(r)))()}catch(s){u({level:"error",event:"broker.boot_failure_surface.failed",...o(e)?{node_id:e}:{fields:{affected_node_id:e}},error:s})}await L(e,"notice.crash",t.wake,{reason:"child-crashed",child:e})}}export{Pe as DaemonFleet,ie as classifyDeadNode,re as retryResumeMode,$ as surfaceBootFailure};

package/dist/daemon/import-dump-run.d.ts

@@ -0,0 +1 @@
+export declare function runImportDump(out: string | undefined, kind: string | undefined): Promise<number>;

package/dist/daemon/import-dump-run.js

@@ -0,0 +1,5 @@
+var a=Object.defineProperty;var s=(r,t)=>a(r,"name",{value:t,configurable:!0});import{closeAllDbs as n,openDb as d}from"../core/canvas/db.js";import{chmodSync as m,writeFileSync as p}from"node:fs";import{join as c}from"node:path";import{dumpBodiesPre as f,dumpRows as l}from"../core/import/export/dump.js";import{STORE_FILE as w}from"../core/import/export/cloud.js";import{readStores as u}from"../core/import/export/stores.js";async function v(r,t){try{if(t==="install-id"){const o=await(await d()).prepare("SELECT value FROM canvas_meta WHERE key = 'install_id'").get();if(o===void 0)throw new Error("this canvas database has no install id");return process.stdout.write(`${JSON.stringify({installId:o.value})}
+`),0}if(t==="bodies"&&r!==void 0&&r!=="")return process.stdout.write(`${JSON.stringify(await f(r))}
+`),0;if(r===void 0||t!=="machine"&&t!=="hosted")throw new Error("usage: import-dump <out dir> machine|hosted|bodies|install-id");const e=await l(r,t),i=c(r,w);return p(i,JSON.stringify(await u()),{mode:384}),m(i,384),process.stdout.write(`${JSON.stringify(e)}
+`),0}catch(e){return process.stderr.write(`${e.message}
+`),1}finally{await n()}}s(v,"runImportDump");export{v as runImportDump};

package/dist/daemon/machine-daemon.d.ts

@@ -0,0 +1,7 @@
+import type { HealthDTO } from '../api/index.js';
+/** `/healthz` of the machine role. The daemon is up and serving whatever the link does, so `ok` stays true (a start waiting for it must not fail on an offline Mac);
+ *  the link is its own fact, and the phase reads `ready` only while it is up. No brokers are reconciled here: this role holds no database. */
+export declare function machineHealth(linkConnected: boolean): HealthDTO;
+/** Start the machine role and return its teardown. Throws when `runtime/machine.json` or the credential is missing: a machine role
+ *  with nothing to link to must not look like a running daemon. */
+export declare function runMachineDaemon(): Promise<() => Promise<void>>;

package/dist/daemon/machine-daemon.js

@@ -0,0 +1 @@
+var v=Object.defineProperty;var c=(e,r)=>v(e,"name",{value:r,configurable:!0});import{chmodSync as S,existsSync as w,mkdirSync as p,unlinkSync as y,writeFileSync as k,rmSync as f}from"node:fs";import{createServer as g}from"node:http";import{dirname as I}from"node:path";import{apiSocketPath as E}from"../core/canvas/paths.js";import{bindDaemonEventSource as M}from"../core/events/source.js";import{emitEvent as x}from"../core/events/emit.js";import{createMachineExecutor as G}from"../core/runtime/runner/machine/executor.js";import{socketDir as P}from"../core/layout.js";import{isDaemonRunning as T,pidfilePath as m,readPidfile as _}from"./pidfile.js";import{cancelMachineCron as b,runMachineCron as C}from"./cron/machine-exec.js";import{startMachineLink as D}from"./machine-link/client.js";import{readMachineConfig as U,readMachineCredential as h}from"./machine-link/config.js";function N(e){return{ok:!0,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,link_connected:e,startup_phase:e?"ready":"recovering"}}c(N,"machineHealth");async function W(){const e=U();if(e===void 0)throw new Error("crtrd --role machine needs runtime/machine.json: this install has not been imported as a machine");if(h(),T())throw new Error("a crtrd is already running for this home");M(),p(P(),{recursive:!0,mode:493});const r=E();w(r)&&y(r);let a;const t=g((o,i)=>{const l=(o.method??"GET")==="GET"&&(o.url??"/").split("?")[0]==="/healthz";i.writeHead(l?200:404,{"content-type":"application/json; charset=utf-8"}),i.end(JSON.stringify(l?N(a?.connected()===!0):{error:{code:"not_found",message:"a machine-role crtrd serves only /healthz",origin:"runtime"}}))});await new Promise((o,i)=>{t.once("error",i),t.listen(r,()=>{t.off("error",i),S(r,384),o()})}),p(I(m()),{recursive:!0}),k(m(),String(process.pid),"utf8");const n=G({apiUrl:e.apiUrl,environmentId:e.environmentId});a=D({apiUrl:e.apiUrl,environmentId:e.environmentId,credential:h,handlers:{start:n.start,wake:n.wake,stop:n.stop,status:n.status,delete:n.delete,viewer:n.viewer,"cron.run":C,"cron.cancel":b}}),x({level:"info",event:"daemon.machine.started",fields:{environment:e.environmentId}});let u;const d=c(()=>u??=(async()=>{a?.stop(),await new Promise(o=>{t.close(()=>o()),t.closeAllConnections()}),_()===process.pid&&(f(r,{force:!0}),f(m(),{force:!0})),process.off("SIGINT",s),process.off("SIGTERM",s)})(),"teardown"),s=c(()=>{d().then(()=>process.exit(0),()=>process.exit(1))},"onSignal");return process.on("SIGINT",s),process.on("SIGTERM",s),d}c(W,"runMachineDaemon");export{N as machineHealth,W as runMachineDaemon};

package/dist/daemon/machine-link/client.d.ts

@@ -0,0 +1,29 @@
+import type { Socket } from 'node:net';
+import type { MachineCronCancelArgs, MachineCronCancelResult, MachineCronRunArgs, MachineCronRunResult, MachineOp } from '../../core/runtime/runner/machine/ops.js';
+export declare const LINK_BACKOFF_MIN_MS = 1000;
+export declare const LINK_BACKOFF_MAX_MS = 60000;
+/** Bytes written to a view.sock that its peer has not yet accepted, above which the channel is closed. */
+export declare const VIEW_WRITE_LIMIT_BYTES: number;
+/** One function per op; a machine-role crtrd injects the executor and the cron runner. */
+export type MachineHandlers = {
+    [O in MachineOp]?: (args: never, signal: AbortSignal) => Promise<unknown>;
+} & {
+    'cron.run'?: (args: MachineCronRunArgs) => Promise<MachineCronRunResult>;
+    'cron.cancel'?: (args: MachineCronCancelArgs) => Promise<MachineCronCancelResult>;
+};
+export interface MachineLinkOptions {
+    apiUrl: string;
+    environmentId: string;
+    credential: () => string;
+    handlers: MachineHandlers;
+    /** Opens a node's view.sock; default `connectViewSocket`. */
+    openView?: (node: string) => Socket;
+    backoffMinMs?: number;
+    backoffMaxMs?: number;
+}
+export interface MachineLink {
+    stop(): void;
+    connected(): boolean;
+}
+export declare function linkUrl(apiUrl: string, environmentId: string): string;
+export declare function startMachineLink(options: MachineLinkOptions): MachineLink;

package/dist/daemon/machine-link/client.js

@@ -0,0 +1 @@
+var b=Object.defineProperty;var c=(o,d)=>b(o,"name",{value:d,configurable:!0});import{WebSocket as w}from"ws";import{emitEvent as E}from"../../core/events/emit.js";import{connectViewSocket as A}from"../../core/runtime/view-socket.js";import{CLOSE_REVOKED as O,LINK_MAX_PAYLOAD_BYTES as T,parseFrame as x}from"./frames.js";const L=1e3,N=6e4,B=4*1024*1024,C=1024*1024,V=8*1024*1024;function K(o,d){const l=new URL(`/v1/machines/${encodeURIComponent(d)}/link`,o);return l.protocol=l.protocol==="https:"?"wss:":l.protocol==="http:"?"ws:":l.protocol,l.toString()}c(K,"linkUrl");function Y(o){const d=o.backoffMinMs??L,l=o.backoffMaxMs??N,S=o.openView??A;let v=!1,f,s=d,p;const k=c((_,t)=>{E({level:"error",event:_,fields:{environment:o.environmentId},error:t})},"fault");let m;const y=c(()=>{if(v)return;if(m!==void 0){if(o.credential()===m){p=setTimeout(y,s),s=Math.min(s*2,l);return}m=void 0}const _=o.credential(),t=new w(K(o.apiUrl,o.environmentId),{headers:{authorization:`Bearer ${_}`},maxPayload:T});f=t;const i=new Map,h=new Map,a=c(e=>{t.readyState===w.OPEN&&t.send(JSON.stringify(e))},"send"),M=c(async e=>{const n=o.handlers[e.op];if(n===void 0){a({type:"result",call_id:e.call_id,ok:!1,error:`this machine does not execute ${e.op}`});return}const r=new AbortController;t.once("close",()=>r.abort(new Error("the machine link closed")));try{a({type:"result",call_id:e.call_id,ok:!0,value:await n(e.args,r.signal)??null})}catch(u){a({type:"result",call_id:e.call_id,ok:!1,error:u instanceof Error?u.message:String(u)})}},"runCall"),g=c(e=>{const n=S(e.node);i.set(e.channel,n),n.once("connect",()=>a({type:"view.ready",channel:e.channel})),n.on("data",r=>{t.readyState===w.OPEN&&(t.send(JSON.stringify({type:"view.data",channel:e.channel,bytes:r.toString("base64")}),()=>{t.bufferedAmount<C&&n.resume()}),t.bufferedAmount>B&&n.pause())}),n.once("error",r=>{i.delete(e.channel)&&a({type:"view.close",channel:e.channel,error:r.message})}),n.once("close",()=>{i.delete(e.channel)&&a({type:"view.close",channel:e.channel})})},"openChannel"),I=c((e,n)=>{const r=i.get(e);if(r===void 0||r.write(n))return;const u=(h.get(e)??0)+n.length;if(u>V){i.delete(e),h.delete(e),r.destroy(),a({type:"view.close",channel:e,error:"the broker is not draining the viewer channel"});return}h.has(e)||r.once("drain",()=>h.delete(e)),h.set(e,u)},"writeToChannel");t.on("open",()=>{s=d}),t.on("message",e=>{const n=x(e);if(n!==null)try{switch(n.type){case"call":M(n);return;case"view.open":g(n);return;case"view.data":I(n.channel,Buffer.from(n.bytes,"base64"));return;case"view.close":{const r=i.get(n.channel);i.delete(n.channel),r?.destroy();return}default:return}}catch(r){k("daemon.machine_link.frame_failed",r)}}),t.on("error",e=>k("daemon.machine_link.client_error",e)),t.on("close",e=>{for(const n of i.values())n.destroy();i.clear(),f===t&&(f=void 0),!v&&(e===O&&(E({level:"error",event:"daemon.machine_link.revoked",fields:{environment:o.environmentId}}),m=_),p=setTimeout(y,s),s=Math.min(s*2,l))})},"connect");return y(),{stop(){v=!0,p!==void 0&&clearTimeout(p),f?.close()},connected:c(()=>f?.readyState===w.OPEN,"connected")}}c(Y,"startMachineLink");export{N as LINK_BACKOFF_MAX_MS,L as LINK_BACKOFF_MIN_MS,V as VIEW_WRITE_LIMIT_BYTES,K as linkUrl,Y as startMachineLink};

package/dist/daemon/machine-link/config.d.ts

@@ -0,0 +1,14 @@
+/** `directoryUrl` is the directory the person signs in against (`crtr sys signin`). */
+export interface MachineConfig {
+    apiUrl: string;
+    environmentId: string;
+    runId: string;
+    directoryUrl: string;
+}
+export declare const machineConfigPath: () => string;
+export declare const machineCredentialPath: () => string;
+/** The machine config, or undefined when this install is not an imported machine. A present but malformed file throws. */
+export declare function readMachineConfig(): MachineConfig | undefined;
+export declare function writeMachineConfig(config: MachineConfig): void;
+export declare function readMachineCredential(): string;
+export declare function writeMachineCredential(secret: string): void;

package/dist/daemon/machine-link/config.js

@@ -0,0 +1,3 @@
+var p=Object.defineProperty;var e=(n,r)=>p(n,"name",{value:r,configurable:!0});import{chmodSync as u,existsSync as i,mkdirSync as o,readFileSync as c,writeFileSync as a}from"node:fs";import{dirname as d,join as m}from"node:path";import{runtimeRoot as s,secretsDir as h}from"../../core/layout.js";const t=e(()=>m(s(),"machine.json"),"machineConfigPath"),f=e(()=>m(h(),"machine-credential"),"machineCredentialPath");function I(){const n=t();if(!i(n))return;const r=JSON.parse(c(n,"utf8"));if(typeof r.apiUrl!="string"||typeof r.environmentId!="string"||typeof r.runId!="string"||typeof r.directoryUrl!="string")throw new Error(`${n} must hold {apiUrl, environmentId, runId, directoryUrl}`);return{apiUrl:r.apiUrl,environmentId:r.environmentId,runId:r.runId,directoryUrl:r.directoryUrl}}e(I,"readMachineConfig");function U(n){o(d(t()),{recursive:!0}),a(t(),`${JSON.stringify(n,null,2)}
+`)}e(U,"writeMachineConfig");function x(){const n=f();if(!i(n))throw new Error(`no machine credential at ${n}`);return c(n,"utf8").trim()}e(x,"readMachineCredential");function S(n){const r=f();o(d(r),{recursive:!0}),a(r,`${n}
+`,{mode:384}),u(r,384)}e(S,"writeMachineCredential");export{t as machineConfigPath,f as machineCredentialPath,I as readMachineConfig,x as readMachineCredential,U as writeMachineConfig,S as writeMachineCredential};

package/dist/daemon/machine-link/credential.d.ts

@@ -0,0 +1,7 @@
+/** True when the environment has an unrevoked credential. */
+export declare function hasMachineCredential(environmentId: string): Promise<boolean>;
+/** Mint (or replace) the environment's credential and return the secret. */
+export declare function mintMachineCredential(environmentId: string): Promise<string>;
+export declare function revokeMachineCredential(environmentId: string): Promise<void>;
+/** True when `token` is the environment's current, unrevoked credential. */
+export declare function verifyMachineCredential(environmentId: string, token: string): Promise<boolean>;

package/dist/daemon/machine-link/credential.js

@@ -0,0 +1,2 @@
+var i=Object.defineProperty;var t=(e,n)=>i(e,"name",{value:n,configurable:!0});import{createHash as o,randomBytes as s}from"node:crypto";import{openDb as a}from"../../core/canvas/db.js";const r=t(e=>o("sha256").update(e).digest("hex"),"hash");async function E(e){return await(await a()).prepare("SELECT 1 AS ok FROM machine_credentials WHERE environment = ? AND revoked_at IS NULL").get(e)!==void 0}t(E,"hasMachineCredential");async function _(e){const n=`mc_${s(32).toString("base64url")}`;return await(await a()).prepare(`INSERT INTO machine_credentials (environment, token_hash, issued_at, revoked_at) VALUES (?, ?, ?, NULL)
+    ON CONFLICT (environment) DO UPDATE SET token_hash = EXCLUDED.token_hash, issued_at = EXCLUDED.issued_at, revoked_at = NULL`).run(e,r(n),new Date().toISOString()),n}t(_,"mintMachineCredential");async function m(e){await(await a()).prepare("UPDATE machine_credentials SET revoked_at = ? WHERE environment = ? AND revoked_at IS NULL").run(new Date().toISOString(),e)}t(m,"revokeMachineCredential");async function p(e,n){return await(await a()).prepare("SELECT 1 AS ok FROM machine_credentials WHERE environment = ? AND token_hash = ? AND revoked_at IS NULL").get(e,r(n))!==void 0}t(p,"verifyMachineCredential");export{E as hasMachineCredential,_ as mintMachineCredential,m as revokeMachineCredential,p as verifyMachineCredential};

package/dist/daemon/machine-link/forward.d.ts

@@ -0,0 +1,16 @@
+import type { IncomingMessage } from 'node:http';
+import type { Duplex } from 'node:stream';
+import type { BrokerViewLink } from '../../core/runtime/view-socket.js';
+import { type MachineLinkRefusal } from './holder.js';
+/** Unsent bytes on a viewer WebSocket above which the channel is closed (a viewer that cannot keep up is dropped, never buffered without bound). */
+export declare const VIEW_BUFFER_LIMIT_BYTES: number;
+/** A viewer channel to a node on a machine: over the link this process holds, or forwarded to the API process that does. */
+export declare function openMachineView(environmentId: string, nodeId: string): BrokerViewLink;
+/** One readiness check: the Mac connected the node's view.sock within `timeoutMs`. */
+export declare function probeMachineView(environmentId: string, nodeId: string, timeoutMs: number): Promise<boolean>;
+/** Wait for the node's broker on the Mac to accept a viewer, retrying while its view.sock refuses; false on the deadline or when the launch exits. */
+export declare function waitForMachineView(environmentId: string, nodeId: string, exited: Promise<unknown> | undefined, waitMs: number): Promise<boolean>;
+/** The environment id of a forwarded viewer upgrade path, or null for any other path. */
+export declare function machineViewEnvironment(req: IncomingMessage): string | null;
+/** The holder's side: authenticate the forwarding API process, open a channel on the link this process holds and pipe it. Returns a refusal for the caller to write, or null once upgraded. */
+export declare function handleMachineViewUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer, environmentId: string): MachineLinkRefusal | null;

package/dist/daemon/machine-link/forward.js

@@ -0,0 +1 @@
+var x=Object.defineProperty;var a=(s,e)=>x(s,"name",{value:e,configurable:!0});import{EventEmitter as b}from"node:events";import{createHmac as A,timingSafeEqual as I}from"node:crypto";import{WebSocket as l,WebSocketServer as P}from"ws";import{openDb as R}from"../../core/canvas/db.js";import{loadSecretsKey as k}from"../../core/secrets/key.js";import{LINK_MAX_PAYLOAD_BYTES as _,parseFrame as v}from"./frames.js";import{apiAddress as B,heldLinkOf as g,MachineViewLink as E,UNROUTABLE_PREFIX as O}from"./holder.js";const w=8*1024*1024,T=1013,D=/^\/v1\/machines\/([A-Za-z0-9_-]+)\/view$/,L=new P({noServer:!0,maxPayload:_}),S=a(s=>A("sha256",k().key).update(`machine-view:${s}`).digest("hex"),"forwardToken");function $(s,e){const r=g(s);return r!==null?new E(r,e):new q(s,e)}a($,"openMachineView");class q extends b{static{a(this,"ForwardedViewLink")}environmentId;nodeId;ws;ended=!1;failed=!1;queued=[];queuedBytes=0;constructor(e,r){super(),this.environmentId=e,this.nodeId=r,this.dial().catch(t=>this.finish(t instanceof Error?t.message:String(t)))}get destroyed(){return this.ended}write(e){return this.ended?!1:this.ws?.readyState!==l.OPEN?(this.queuedBytes+=e.length,this.queuedBytes>w?(this.finish("machine view: the holder did not accept the channel before its queue filled"),!1):(this.queued.push(e),!0)):this.ws.bufferedAmount>w?(this.finish("machine view: the holder is not draining the channel"),!1):(this.sendData(e),!0)}destroy(){return this.ended||(this.ws?.readyState===l.OPEN&&this.ws.send(JSON.stringify({type:"view.close",channel:""})),this.finish(void 0)),this}sendData(e){this.ws?.send(JSON.stringify({type:"view.data",channel:"",bytes:Buffer.from(e).toString("base64")}))}async dial(){const e=await(await R()).prepare("SELECT api_address, lease_expires_at FROM machine_links WHERE environment = ?").get(this.environmentId);if(e===void 0||Date.parse(e.lease_expires_at)<=Date.now())throw new Error(`machine_offline: machine ${this.environmentId} has no live link`);if(e.api_address.startsWith(O))throw new Error(`machine_unreachable: the API process holding the link wrote ${e.api_address}; set CRTR_API_ADDRESS or listen on TCP`);if(e.api_address===B())throw new Error(`machine_offline: the link row names this API process (${e.api_address}) but it holds no link`);if(this.ended)return;const r=new URL(`/v1/machines/${encodeURIComponent(this.environmentId)}/view`,e.api_address.includes("://")?e.api_address:`ws://${e.api_address}`);r.protocol=r.protocol==="https:"?"wss:":r.protocol==="http:"?"ws:":r.protocol,r.searchParams.set("node",this.nodeId);const t=new l(r.toString(),{headers:{authorization:`Bearer ${S(this.environmentId)}`},maxPayload:_});this.ws=t,t.on("open",()=>{for(const n of this.queued.splice(0))this.sendData(n)}),t.on("unexpected-response",(n,i)=>{i.resume(),this.finish(i.statusCode===409?"machine_offline: the holder no longer holds the link":`machine view refused by ${e.api_address}: ${i.statusCode}`)}),t.on("message",n=>{const i=v(n);i!==null&&(i.type==="view.ready"?this.emit("connect"):i.type==="view.data"?this.emit("data",Buffer.from(i.bytes,"base64")):i.type==="view.close"&&this.finish(i.error))}),t.on("error",n=>this.finish(n.message)),t.on("close",()=>this.finish("machine view forward closed"))}finish(e){this.ended||(this.ended=!0,this.ws?.close(),e!==void 0&&!this.failed&&(this.failed=!0,this.emit("error",Object.assign(new Error(e),{code:"ECONNRESET"}))),this.emit("close",this.failed))}}function M(s,e,r){return new Promise(t=>{const n=$(s,e),i=setTimeout(()=>{t(!1),n.destroy()},r);n.on("connect",()=>{clearTimeout(i),t(!0),n.destroy()}),n.on("error",()=>{}),n.on("close",()=>{clearTimeout(i),t(!1)})})}a(M,"probeMachineView");async function J(s,e,r,t){const n=Date.now()+t;let i=!1;for(r?.then(()=>{i=!0});Date.now()<=n&&!i;){if(await M(s,e,Math.max(1,Math.min(2e3,n-Date.now()))))return!0;await new Promise(c=>setTimeout(c,250))}return!1}a(J,"waitForMachineView");function Y(s){const e=D.exec(new URL(s.url??"/","http://runtime.local").pathname);return e===null?null:e[1]}a(Y,"machineViewEnvironment");function K(s,e,r,t){const n=s.headers.authorization,i=/^Bearer (\S+)$/.exec(Array.isArray(n)?n[0]??"":n??"")?.[1]??"",c=Buffer.from(S(t)),p=Buffer.from(i);if(p.length!==c.length||!I(p,c))return{status:401,text:"Unauthorized",code:"unauthorized",message:"machine view forward credential refused"};const f=new URL(s.url??"/","http://runtime.local").searchParams.get("node");if(f===null||f==="")return{status:400,text:"Bad Request",code:"bad_request",message:"node is required"};const y=g(t);return y===null?{status:409,text:"Conflict",code:"machine_offline",message:"this API process holds no link for the machine"}:(L.handleUpgrade(s,e,r,d=>{const o=new E(y,f),u=a(h=>{d.readyState===l.OPEN&&d.send(JSON.stringify(h))},"send");o.on("connect",()=>u({type:"view.ready",channel:""})),o.on("data",h=>{if(d.bufferedAmount>w){o.destroy(),d.close(T,"viewer too slow");return}u({type:"view.data",channel:"",bytes:h.toString("base64")})}),o.on("error",h=>u({type:"view.close",channel:"",error:h.message})),o.on("close",()=>d.close()),d.on("message",h=>{const m=v(h);m?.type==="view.data"?o.write(Buffer.from(m.bytes,"base64")):m?.type==="view.close"&&o.destroy()}),d.on("close",()=>o.destroy()),d.on("error",()=>o.destroy())}),null)}a(K,"handleMachineViewUpgrade");export{w as VIEW_BUFFER_LIMIT_BYTES,K as handleMachineViewUpgrade,Y as machineViewEnvironment,$ as openMachineView,M as probeMachineView,J as waitForMachineView};

package/dist/daemon/machine-link/frames.d.ts

@@ -0,0 +1,49 @@
+import type { MachineOp } from '../../core/runtime/runner/machine/ops.js';
+export type MachineFrame = 
+/** holder → Mac: run `op`. */
+{
+    type: 'call';
+    call_id: string;
+    op: MachineOp;
+    args: unknown;
+}
+/** Mac → holder: the call's outcome. */
+ | {
+    type: 'result';
+    call_id: string;
+    ok: true;
+    value: unknown;
+} | {
+    type: 'result';
+    call_id: string;
+    ok: false;
+    error: string;
+}
+/** holder → Mac: open a viewer channel to `node`'s view.sock. */
+ | {
+    type: 'view.open';
+    channel: string;
+    node: string;
+}
+/** Mac → holder: the channel's view.sock accepted the connection. */
+ | {
+    type: 'view.ready';
+    channel: string;
+}
+/** Either way: base64 bytes of the view.sock stream. */
+ | {
+    type: 'view.data';
+    channel: string;
+    bytes: string;
+}
+/** Either way: the channel ended; `error` when it failed (the Mac could not reach the view.sock). */
+ | {
+    type: 'view.close';
+    channel: string;
+    error?: string;
+};
+export declare const LINK_MAX_PAYLOAD_BYTES: number;
+/** WebSocket close codes of the holder: a newer link took the environment's row; the credential is revoked (the Mac does not reconnect with it). */
+export declare const CLOSE_SUPERSEDED = 4001;
+export declare const CLOSE_REVOKED = 4003;
+export declare function parseFrame(data: Buffer | ArrayBuffer | Buffer[]): MachineFrame | null;

package/dist/daemon/machine-link/frames.js

@@ -0,0 +1 @@
+var o=Object.defineProperty;var l=(r,t)=>o(r,"name",{value:t,configurable:!0});const a=33554432,f=4001,i=4003;function p(r){const t=Array.isArray(r)?Buffer.concat(r):Buffer.isBuffer(r)?r:Buffer.from(new Uint8Array(r));try{const e=JSON.parse(t.toString("utf8"));if(typeof e!="object"||e===null)return null;const n=l((...u)=>u.every(c=>typeof e[c]=="string"),"strings");switch(e.type){case"call":return n("call_id","op")?e:null;case"result":return n("call_id")&&(e.ok===!0||e.ok===!1&&n("error"))?e:null;case"view.open":return n("channel","node")?e:null;case"view.ready":case"view.close":return n("channel")?e:null;case"view.data":return n("channel","bytes")?e:null;default:return null}}catch{return null}}l(p,"parseFrame");export{i as CLOSE_REVOKED,f as CLOSE_SUPERSEDED,a as LINK_MAX_PAYLOAD_BYTES,p as parseFrame};

package/dist/daemon/machine-link/holder.d.ts

@@ -0,0 +1,73 @@
+import { EventEmitter } from 'node:events';
+import type { IncomingMessage } from 'node:http';
+import type { Duplex } from 'node:stream';
+import { WebSocket } from 'ws';
+import type { BrokerViewLink } from '../../core/runtime/view-socket.js';
+import { type MachineFrame } from './frames.js';
+export declare const LINK_LEASE_MS = 45000;
+export declare const LINK_RENEW_MS = 15000;
+export declare const LINK_PING_MS = 30000;
+/** Prefix of the address an API process writes when nothing tells it where other API processes can reach it; the viewer forward refuses it. */
+export declare const UNROUTABLE_PREFIX = "unroutable:";
+/** The TCP listener's bound `host:port`, used as this process's address when `CRTR_API_ADDRESS` is not set. */
+export declare const setApiAddress: (address: string) => void;
+/** The address other API processes find this one at, written to `machine_links.api_address`: `CRTR_API_ADDRESS`, else the listener's bound address, else an unroutable marker. */
+export declare const apiAddress: () => string;
+export declare class HeldLink extends EventEmitter {
+    readonly environmentId: string;
+    readonly generation: number;
+    readonly ws: WebSocket;
+    private readonly token;
+    private readonly timers;
+    private unsubscribe;
+    private claiming;
+    private closed;
+    private alive;
+    readonly channels: Map<string, MachineViewLink>;
+    constructor(environmentId: string, generation: number, ws: WebSocket, token: string);
+    start(): void;
+    private fault;
+    send(frame: MachineFrame): boolean;
+    /** Renew the row, my sent calls' leases and `last_seen_at`; a lost generation or a revoked credential closes the link. */
+    renew(): Promise<void>;
+    /** The credential this link authenticated with is still the environment's current one: rotation or revocation evicts the link. */
+    private credentialValid;
+    /** Claim this environment's pending calls (fenced by this link's generation) and forward them. */
+    claim(): Promise<void>;
+    /** `sent` is whether the call was written to the link: a call that was sent may have run, one that was not ran nowhere. */
+    private failCall;
+    /** This link's generation is still the one in the environment's `machine_links` row. */
+    private isCurrentGeneration;
+    /** Give a claimed, unsent call back to `pending`. */
+    private release;
+    /** Settle a call this link holds; a call another holder owns or one already settled is left as it is. A result settles only
+     *  while this link is the current generation (`requireCurrent`); a failure for a dropped link settles whatever the row now says. */
+    private settle;
+    private onMessage;
+    private teardown;
+}
+/** A viewer channel to one node's view.sock on the Mac, as the relay's broker-side link. */
+export declare class MachineViewLink extends EventEmitter implements BrokerViewLink {
+    private readonly link;
+    private ended;
+    private failed;
+    private readonly channel;
+    constructor(link: HeldLink, nodeId: string);
+    get destroyed(): boolean;
+    write(line: string | Buffer): boolean;
+    destroy(): this;
+    /** The channel is over: `error` names a failure (the relay records a fault), none is a clean end. */
+    finish(error: string | undefined): void;
+}
+/** The link this process holds for `environmentId`, or null (no link, or another API process holds it). */
+export declare const heldLinkOf: (environmentId: string) => HeldLink | null;
+/** The environment id of a machine-link upgrade path, or null for any other path. */
+export declare function machineLinkEnvironment(req: IncomingMessage): string | null;
+export interface MachineLinkRefusal {
+    status: number;
+    text: string;
+    code: string;
+    message: string;
+}
+/** Authenticate the machine credential and take over the environment's link. Returns a refusal for the caller to write, or null once upgraded. */
+export declare function handleMachineLinkUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer, environmentId: string): Promise<MachineLinkRefusal | null>;

package/dist/daemon/machine-link/holder.js

@@ -0,0 +1,9 @@
+var N=Object.defineProperty;var s=(i,e)=>N(i,"name",{value:e,configurable:!0});import{EventEmitter as u}from"node:events";import{hostname as k}from"node:os";import{randomUUID as R}from"node:crypto";import{WebSocket as T,WebSocketServer as I}from"ws";import{openDb as r}from"../../core/canvas/db.js";import{emitEvent as _}from"../../core/events/emit.js";import{brokerSignalBus as L}from"../broker-signals/bus.js";import{verifyMachineCredential as E}from"./credential.js";import{CLOSE_REVOKED as y,CLOSE_SUPERSEDED as p,LINK_MAX_PAYLOAD_BYTES as O,parseFrame as b}from"./frames.js";const U=45e3,C=15e3,x=3e4,M=1e3,w=10,P=new I({noServer:!0,maxPayload:O}),c=new Map,H="unroutable:";let g;const q=s(i=>{g=i},"setApiAddress"),W=s(()=>process.env.CRTR_API_ADDRESS??g??`${H}${k()}:${process.pid}`,"apiAddress"),m=s(()=>new Date(Date.now()+U).toISOString(),"lease");class F extends u{static{s(this,"HeldLink")}environmentId;generation;ws;token;timers=[];unsubscribe;claiming=!1;closed=!1;alive=!0;channels=new Map;constructor(e,t,n,a){super(),this.environmentId=e,this.generation=t,this.ws=n,this.token=a}start(){const{ws:e}=this;e.on("message",t=>{this.onMessage(t).catch(n=>this.fault("daemon.machine_link.frame_failed",n))}),e.on("pong",()=>{this.alive=!0}),e.on("close",()=>{this.teardown().catch(t=>this.fault("daemon.machine_link.teardown_failed",t))}),e.on("error",()=>{}),this.timers.push(setInterval(()=>{this.renew().catch(t=>this.fault("daemon.machine_link.renew_failed",t))},C)),this.timers.push(setInterval(()=>{if(!this.alive){e.terminate();return}this.alive=!1;try{e.ping()}catch{}},x)),this.timers.push(setInterval(()=>{this.claim().catch(t=>this.fault("daemon.machine_link.claim_failed",t))},M)),L.subscribe(t=>{t===this.environmentId&&this.claim().catch(n=>this.fault("daemon.machine_link.claim_failed",n))},{kinds:["machine"],onListenStart:s(()=>{this.claim().catch(t=>this.fault("daemon.machine_link.claim_failed",t))},"onListenStart")}).then(t=>{this.closed?t():this.unsubscribe=t})}fault(e,t){_({level:"error",event:e,fields:{environment:this.environmentId},error:t})}send(e){return this.ws.readyState!==T.OPEN?!1:(this.ws.send(JSON.stringify(e)),!0)}async renew(){if(this.closed)return;const e=await r(),t=m();if((await e.prepare("UPDATE machine_links SET lease_expires_at = ? WHERE environment = ? AND generation = ?").run(t,this.environmentId,this.generation)).changes===0){this.ws.close(p,"superseded");return}await e.prepare("UPDATE machine_calls SET lease_expires_at = ? WHERE environment = ? AND state = 'sent' AND claimed_generation = ?").run(t,this.environmentId,this.generation),await e.prepare("UPDATE environments SET last_seen_at = ? WHERE object = ?").run(new Date().toISOString(),this.environmentId),await this.credentialValid()||this.ws.close(y,"revoked")}async credentialValid(){return await E(this.environmentId,this.token)}async claim(){if(!(this.closed||this.claiming)){this.claiming=!0;try{for(;;){const e=await(await r()).prepare(`UPDATE machine_calls SET state = 'sent', claimed_generation = ?, lease_expires_at = ?
+          WHERE call_id IN (SELECT call_id FROM machine_calls WHERE environment = ? AND state = 'pending'
+            AND EXISTS (SELECT 1 FROM machine_links WHERE environment = ? AND generation = ?)
+            ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT ${w})
+          RETURNING call_id, op, args_json`).all(this.generation,m(),this.environmentId,this.environmentId,this.generation),t=!this.closed&&await this.isCurrentGeneration();for(const n of e)t?this.send({type:"call",call_id:n.call_id,op:n.op,args:JSON.parse(n.args_json)})||await this.failCall(n.call_id,"machine_offline","link closed before the call was sent",!1):await this.release(n.call_id);if(e.length<w)return}}finally{this.claiming=!1}}}async failCall(e,t,n,a){await this.settle(e,"failed",{code:t,message:n,sent:a},!1)}async isCurrentGeneration(){return await(await r()).prepare("SELECT 1 AS live FROM machine_links WHERE environment = ? AND generation = ?").get(this.environmentId,this.generation)!==void 0}async release(e){await(await r()).prepare("UPDATE machine_calls SET state = 'pending', claimed_generation = NULL, lease_expires_at = NULL WHERE call_id = ? AND state = 'sent' AND claimed_generation = ?").run(e,this.generation)}async settle(e,t,n,a){await B(e,t,n,this.environmentId,this.generation,a)}async onMessage(e){const t=b(e);if(t!==null)switch(t.type){case"result":t.ok?await this.settle(t.call_id,"done",t.value??null,!0):await this.settle(t.call_id,"failed",{code:"machine_call_failed",message:t.error},!0);return;case"view.ready":this.channels.get(t.channel)?.emit("connect");return;case"view.data":this.channels.get(t.channel)?.emit("data",Buffer.from(t.bytes,"base64"));return;case"view.close":this.channels.get(t.channel)?.finish(t.error);return;default:return}}async teardown(){if(this.closed)return;this.closed=!0;for(const n of this.timers)clearInterval(n);this.unsubscribe?.(),c.get(this.environmentId)===this&&c.delete(this.environmentId);for(const n of[...this.channels.values()])n.finish("machine link closed");const e=await r(),t=await e.prepare("SELECT call_id FROM machine_calls WHERE environment = ? AND state = 'sent' AND claimed_generation = ?").all(this.environmentId,this.generation);for(const{call_id:n}of t)await this.failCall(n,"machine_offline","link dropped before the result",!0);await e.prepare("DELETE FROM machine_links WHERE environment = ? AND generation = ?").run(this.environmentId,this.generation)}}async function B(i,e,t,n,a,l){const o=await r();(await o.prepare(`UPDATE machine_calls SET state = ?, result_json = ?, finished_at = ?
+    WHERE call_id = ? AND environment = ? AND claimed_generation = ? AND state = 'sent'
+    ${l?"AND EXISTS (SELECT 1 FROM machine_links WHERE environment = ? AND generation = ?)":""}`).run(e,JSON.stringify(t),new Date().toISOString(),i,n,a,...l?[n,a]:[])).changes>0&&await o.prepare("SELECT pg_notify(?, ?)").get("crtr_signal",`machine_call:${i}`)}s(B,"withNotify");class ee extends u{static{s(this,"MachineViewLink")}link;ended=!1;failed=!1;channel=R();constructor(e,t){super(),this.link=e,e.channels.set(this.channel,this),e.send({type:"view.open",channel:this.channel,node:t})||queueMicrotask(()=>this.finish("machine link closed"))}get destroyed(){return this.ended}write(e){return!this.ended&&this.link.send({type:"view.data",channel:this.channel,bytes:Buffer.from(e).toString("base64")})}destroy(){return this.ended||(this.link.send({type:"view.close",channel:this.channel}),this.finish(void 0)),this}finish(e){this.ended||(this.ended=!0,this.link.channels.delete(this.channel),e!==void 0&&!this.failed&&(this.failed=!0,this.emit("error",Object.assign(new Error(e),{code:"ECONNRESET"}))),this.emit("close",this.failed))}}const te=s(i=>c.get(i)??null,"heldLinkOf"),K=/^\/v1\/machines\/([A-Za-z0-9_-]+)\/link$/;function ne(i){const e=K.exec(new URL(i.url??"/","http://runtime.local").pathname);return e===null?null:e[1]}s(ne,"machineLinkEnvironment");async function ie(i,e,t,n){const a=i.headers.authorization,l=/^Bearer (\S+)$/.exec(Array.isArray(a)?a[0]??"":a??"")?.[1];if(l===void 0||!await E(n,l))return{status:401,text:"Unauthorized",code:"unauthorized",message:"machine credential refused"};const o=new Date().toISOString(),h=await r(),v=await h.prepare(`INSERT INTO machine_links (environment, generation, api_address, connected_at, lease_expires_at) VALUES (?, 1, ?, ?, ?)
+    ON CONFLICT (environment) DO UPDATE SET generation = machine_links.generation + 1, api_address = EXCLUDED.api_address, connected_at = EXCLUDED.connected_at, lease_expires_at = EXCLUDED.lease_expires_at
+    RETURNING generation`).get(n,W(),o,m());await h.prepare("UPDATE environments SET last_seen_at = ? WHERE object = ?").run(o,n);const f=Number(v.generation);return e.destroyed?(await h.prepare("DELETE FROM machine_links WHERE environment = ? AND generation = ?").run(n,f),null):(P.handleUpgrade(i,e,t,S=>{const d=new F(n,f,S,l),D=c.get(n);c.set(n,d),D?.ws.close(p,"superseded"),d.start(),d.claim().catch(A=>_({level:"error",event:"daemon.machine_link.claim_failed",fields:{environment:n},error:A}))}),null)}s(ie,"handleMachineLinkUpgrade");export{F as HeldLink,U as LINK_LEASE_MS,x as LINK_PING_MS,C as LINK_RENEW_MS,ee as MachineViewLink,H as UNROUTABLE_PREFIX,W as apiAddress,ie as handleMachineLinkUpgrade,te as heldLinkOf,ne as machineLinkEnvironment,q as setApiAddress};

package/dist/daemon/manage.d.ts

@@ -1,4 +1,5 @@
 import { type HealthDTO, type StartupPhaseDTO } from '../api/index.js';
+import type { DumpResult } from '../core/import/export/dump.js';
 /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
  *  overwhelmingly done from inside an agent node's own bash tool (`crtr sys
  *  daemon stop && start`), which inherits that node's FULL env — its identity
@@ -46,10 +47,25 @@ export interface DaemonExportSqliteResult {
     ms: number;
     userVersion: number;
 }
-/** Run the selected generation's `crtrd-cli export-sqlite` and wait for it. The export loads canvas database code, which
- *  a CLI process may not, so it runs in the daemon entry the way the daemon itself is launched. Throws with the child's
- *  stderr when it fails. */
+export interface DaemonExportSqliteResult {
+    path: string;
+    rows: number;
+    ms: number;
+    userVersion: number;
+}
+/** Run the selected generation's `crtrd-cli export-sqlite` and wait for it. */
 export declare function runDaemonExportSqlite(): Promise<DaemonExportSqliteResult>;
+/** Run the selected generation's `crtrd-cli import-dump`: the row files of the running canvas into `out`/rows. */
+export declare function runDaemonImportDump(out: string, kind: 'machine' | 'hosted'): Promise<DumpResult>;
+/** Run `crtrd-cli import-dump <out> bodies`: every `bodies` row into `out`/rows/bodies (the pre-pause dump; the later `runDaemonImportDump` adds the rows written since). */
+export declare function runDaemonBodiesDump(out: string): Promise<{
+    parts: string[];
+    count: number;
+}>;
+/** Run `crtrd-cli import-dump <out> install-id`: this install's id, read from the running canvas. */
+export declare function runDaemonInstallId(): Promise<{
+    installId: string;
+}>;
 export interface SpawnDaemonOptions {
     /** The phase that proves a daemon launched by this call is available. */
     targetPhase?: 'ready' | 'prepared';

package/dist/daemon/manage.js

@@ -1,3 +1,3 @@
-var Y=Object.defineProperty;var o=(e,t)=>Y(e,"name",{value:t,configurable:!0});import{spawn as W,execFileSync as X}from"node:child_process";import{createRequire as q}from"node:module";import{dirname as b,join as E}from"node:path";import{fileURLToPath as V,pathToFileURL as k}from"node:url";import{existsSync as B,lstatSync as Q,mkdirSync as Z,openSync as H,closeSync as U,fstatSync as j,readSync as ee}from"node:fs";import{crtrHome as g}from"../core/canvas/paths.js";import{listProcessTable as te}from"../core/canvas/pid.js";import{bindDaemonEventSource as re,eventSource as ne}from"../core/events/source.js";import{hostExecPath as O}from"../core/runtime/branded-host.js";import{isDaemonRunning as oe,readPidfile as D,isPidAlive as p}from"./pidfile.js";import{DAEMON_STARTUP_WINDOW_MS as K}from"./startup-policy.js";import{exclusiveLockOwnerPid as ae}from"../core/exclusive-lock.js";import{onDiskMigrationLockPath as ie}from"../core/canvas/paths.js";import{envNoDaemonAutostart as se}from"../shared/env.js";import{awaitJoinLockRelease as ce,joinInProgress as ue,joinInProgressError as le,joinLockHolder as de}from"../core/directory/join-lock.js";import{waitForDaemonAvailability as pe}from"../api/index.js";import{localClient as fe}from"../api/node-transport.js";const me=["CRTR_NODE_ID","CRTR_KIND","CRTR_MODE","CRTR_LIFECYCLE","CRTR_NODE_CWD","CRTR_CONTEXT_DIR","CRTR_CYCLES","CRTR_PROFILE_ID","CRTR_PARENT_NODE_ID","CRTR_FRONT_DOOR","TMUX","TMUX_PANE","CRTR_BROKER_ENGINE","CRTR_PI_BINARY","NODE_OPTIONS"];function M(){const e={...process.env};for(const t of me)delete e[t];return e}o(M,"sanitizedDaemonEnv");function he(){for(let e=b(V(import.meta.url));;e=b(e)){if(B(E(e,"bin","runtime-generation-manifest.mjs")))return e;if(b(e)===e)throw new Error(`cannot locate the crouter generation root from ${import.meta.url}`)}}o(he,"generationRoot");async function A(){const e=he(),t=k(E(e,"bin","runtime-generation-manifest.mjs")).href,{runtimeHome:r}=await import(t);try{Q(E(r(),"selected"))}catch(i){if(i.code!=="ENOENT")throw i;return["--conditions","crtr-src","--import",k(q(import.meta.url).resolve("tsx/esm")).href,E(e,"src","daemon","crtrd-cli.ts")]}const n=k(E(e,"bin","runtime-selector.mjs")).href,{tryResolveRuntime:a}=await import(n),c=a("daemon");if(!c.ok)throw new Error(`cannot resolve the selected runtime generation's daemon entry: ${c.error}`);return[c.runtime.entryPath]}o(A,"resolveDaemonLaunch");function we(e,t){return e.includes(` --canvas-home ${t} `)||e.endsWith(` --canvas-home ${t}`)}o(we,"daemonCommandDeclaresCanvasHome");function _e(){const e=te();if(e===null)return[];const t=g(),r=[];for(const{pid:n,command:a}of e)a.includes("daemon/crtrd")&&we(a,t)&&n!==process.pid&&r.push(n);return r}o(_e,"findDaemonPids");function Ee(e){const t=_e().filter(n=>n!==e);for(const n of t)try{process.kill(n,"SIGTERM")}catch{}if(t.length===0)return t;const r=Date.now()+2e3;for(;Date.now()<r&&t.some(n=>p(n));)X("sleep",["0.1"]);for(const n of t)if(p(n))try{process.kill(n,"SIGKILL")}catch{}return t}o(Ee,"sweepStrayDaemons");async function qe(){const e=process.execve;if(e===void 0)throw new Error(`this Node.js ${process.version} runtime cannot replace PID 1 with the successor daemon`);const t=O(),r=await A();return e(t,[t,...r,"--canvas-home",g()],M())}o(qe,"execDaemon");async function Ve(){const e=await A(),t=W(O(),[...e,"export-sqlite"],{stdio:["ignore","pipe","pipe"],env:M()});let r="",n="";t.stdout.setEncoding("utf8").on("data",i=>{r+=i}),t.stderr.setEncoding("utf8").on("data",i=>{n+=i});const a=await new Promise((i,l)=>{t.once("error",l),t.once("close",f=>i(f))});if(a!==0)throw new Error(n.trim()||`the export process exited with code ${a}`);const c=r.trim().split(`
-`).at(-1)??"";return JSON.parse(c)}o(Ve,"runDaemonExportSqlite");const De=3*6e4,G=2e4,I=10,N=1e3;async function P(e=N){const t=Math.min(e,N);return fe({autostart:!1,timeoutMs:t}).probeHealthz(t)}o(P,"probeDaemonHealth");function ye(e){const t=e instanceof Error?e.message:String(e);if(typeof e!="object"||e===null)return t;const r=e.code,n=e.cause,a=typeof r=="string"?r:n?.code;return typeof a=="string"&&!t.includes(a)?`${t}: ${a}`:t}o(ye,"probeErrorMessage");async function z(e=N){const t=await P(e);if(t.startup_phase==="blocked"){const r=D()??process.pid;throw new R(r,t.startup_blocked??"daemon startup is blocked")}if(!t.ok)throw new Error(t.startup_blocked??"crtrd health check reported not ready")}o(z,"probeDaemonServing");async function Qe(){try{return await z(),!0}catch{return!1}}o(Qe,"isDaemonServing");function L(e){return new Promise(t=>{setTimeout(t,e)})}o(L,"sleepMs");class $ extends Error{static{o(this,"DaemonExitedBeforeReadyError")}stderrTail;constructor(t,r){super(r===""?t:`${t}
-${r}`),this.name="DaemonExitedBeforeReadyError",this.stderrTail=r}}class R extends Error{static{o(this,"DaemonStartupBlockedError")}constructor(t,r){super(`daemon ${t} is running but normal startup is blocked: ${r}`),this.name="DaemonStartupBlockedError"}}class C extends Error{static{o(this,"DaemonNotPreparedError")}constructor(t,r){super(r===void 0?`daemon ${t} is serving but did not report the prepared startup phase`:`daemon ${t} reported startup phase ${r}, not prepared`),this.name="DaemonNotPreparedError"}}async function xe(e,t=K,r={},n="ready"){const a=r.readPidfile??D,c=r.isPidAlive??p,i=r.childExited,l=r.isDaemonServing,f=r.probeDaemonServing??(async w=>{if(l!==void 0){if(await l())return;throw new Error("crtrd health check reported not ready")}await z(w)}),m=r.probeDaemonHealth??P,y=r.sleepMs??L,x=r.now??Date.now;let s=null,u=null;return await pe({windowMs:t,pollIntervalMs:I,now:x,sleep:y,retry:o(w=>w!==u,"retry"),probe:o(async w=>{const h=a();if(h===null||!c(h)){const d=i?.();throw d!=null&&(d.signal!==null?u=new $(`daemon ${e} exited before becoming ready (by ${d.signal})`,r.childStderrTail?.()??""):d.code!==0&&(u=new $(`daemon ${e} exited before becoming ready (with exit code ${d.code??"?"})`,r.childStderrTail?.()??"")),u!==null)?u:new Error(`daemon ${e} has no live pidfile owner`)}if(n==="ready"){try{await f(w)}catch(d){throw d instanceof R&&(u=d),d}s=h;return}const _=await m(w);if(_.startup_phase==="prepared"){s=h;return}throw _.startup_phase==="blocked"?(u=new R(h,_.startup_blocked??"on-disk migration is blocked"),u):_.startup_phase==="ready"?(u=new C(h,_.startup_phase),u):new Error(`daemon ${h} is ${_.startup_phase??"not reporting a startup phase"}; waiting for prepared`)},"probe")}),s===e?null:s}o(xe,"verifyDaemonStartup");async function ge(e,t=G,r={}){const n=r.readPidfile??D,a=r.isPidAlive??p,c=r.sleepMs??L,i=r.now??Date.now,l=i()+t;for(;i()<=l;){if(!a(e)&&n()!==e)return;await c(I)}throw new Error(`daemon ${e} did not exit and release its pidfile within ${t}ms`)}o(ge,"waitForDaemonExit");const Se=2e3,Re=5e3;async function J(e,t=G,r={}){const n=r.isPidAlive??p,a=r.sleepMs??L,c=r.now??Date.now;try{process.kill(e,"SIGTERM")}catch{}try{return await ge(e,t,r),"terminated"}catch{}try{process.kill(e,"SIGKILL")}catch{}const i=c()+Se;for(;c()<=i;){if(!n(e))return"killed";await a(I)}throw new Error(`daemon ${e} survived both SIGTERM and SIGKILL`)}o(J,"stopDaemonProcess");async function Ze(){const e=D(),t=e!==null&&p(e),r=t&&await J(e)==="killed",n=Ee(null);return{stopped:t||n.length>0,...t?{pid:e}:{},strays_killed:n,force_killed:r}}o(Ze,"stopAllDaemons");function ve(e,t,r=2048){let n;try{n=H(e,"r");const{size:a}=j(n),c=Math.max(t,a-r),i=a-c;if(i<=0)return"";const l=Buffer.alloc(i);return ee(n,l,0,i,c),l.toString("utf8").trim()}catch{return""}finally{if(n!==void 0)try{U(n)}catch{}}}o(ve,"readLogTailFrom");async function Te(e={}){if(process.platform==="linux"&&B("/run/crtr/launcher.sock")&&!p(D()??-1))throw new Error("crtrd is managed by the root launcher; it cannot self-daemonize");const t=e.targetPhase??"ready",r=D();if(r!==null&&p(r))try{const s=await P();if(t==="prepared"){if(s.startup_phase==="prepared")return{started:!1,existing_pid:r,startup_phase:s.startup_phase};throw new C(r,s.startup_phase)}return s.ok?{started:!1,existing_pid:r,...s.startup_phase===void 0?{}:{startup_phase:s.startup_phase}}:{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:s.startup_blocked??"crtrd health check reported not ready",...s.startup_phase===void 0?{}:{startup_phase:s.startup_phase}}}catch(s){if(s instanceof C)throw s;return{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:ye(s)}}Z(g(),{recursive:!0});const n=await A();e.waitForJoinLock===!0&&await ce(De);const a=de();if(a!==null&&a!==process.pid&&e.ignoreJoinLock!==!0)throw le(a);const c=E(g(),"crtrd.err"),i=H(c,"a"),l=(()=>{try{return j(i).size}catch{return 0}})();ne()===void 0&&re();const f=W(O(),[...n,"--canvas-home",g(),...t==="prepared"?["--prepare"]:[]],{detached:!0,stdio:["ignore",i,i],env:M()});U(i);const m=f.pid;if(m===void 0)throw new Error("daemon spawn did not return a pid");let y=null;f.once("exit",(s,u)=>{y={code:s,signal:u}}),f.unref();let x;try{x=await xe(m,K,{childExited:o(()=>y,"childExited"),childStderrTail:o(()=>ve(c,l),"childStderrTail")},t)}catch(s){const u=ae(ie())===m;if(y===null&&!u&&!(s instanceof R))try{await J(m,Re)}catch{}throw s}return x!==null?{started:!1,existing_pid:x,startup_phase:t}:{started:!0,pid:m,startup_phase:t}}o(Te,"spawnDaemon");function be(){return se()||process.argv.includes("--no-autostart")}o(be,"autostartDisabled");const ke=5e3,Oe=6e4;let F=!1,S,v=0,T=0;function Me(e){return Math.min(ke*2**(e-1),Oe)}o(Me,"autostartBackoffMs");function Ae(){S=void 0,v=0,T=0}o(Ae,"resetDaemonAutostart");function et(){return S}o(et,"daemonAutostartStartupFailure");function tt(e={}){const t=e.isDaemonRunning??oe,r=e.spawnDaemon??Te,n=e.now??Date.now;if(!be()&&!ue()&&!F&&!(n()<T)){if(t()){Ae();return}S=void 0,F=!0,r().then(()=>{S=void 0,v=0,T=0},a=>{a instanceof $&&(S=a.message),v+=1,T=n()+Me(v)}).finally(()=>{F=!1})}}o(tt,"ensureDaemon");export{me as DAEMON_ENV_STRIP_KEYS,$ as DaemonExitedBeforeReadyError,C as DaemonNotPreparedError,R as DaemonStartupBlockedError,be as autostartDisabled,et as daemonAutostartStartupFailure,we as daemonCommandDeclaresCanvasHome,tt as ensureDaemon,qe as execDaemon,_e as findDaemonPids,Qe as isDaemonServing,P as probeDaemonHealth,z as probeDaemonServing,ye as probeErrorMessage,Ae as resetDaemonAutostart,Ve as runDaemonExportSqlite,M as sanitizedDaemonEnv,Te as spawnDaemon,Ze as stopAllDaemons,J as stopDaemonProcess,Ee as sweepStrayDaemons,xe as verifyDaemonStartup,ge as waitForDaemonExit};
+var X=Object.defineProperty;var o=(e,t)=>X(e,"name",{value:t,configurable:!0});import{spawn as B,execFileSync as q}from"node:child_process";import{createRequire as V}from"node:module";import{dirname as k,join as D}from"node:path";import{fileURLToPath as Q,pathToFileURL as I}from"node:url";import{existsSync as H,lstatSync as Z,mkdirSync as ee,openSync as U,closeSync as j,fstatSync as K,readSync as te}from"node:fs";import{crtrHome as g}from"../core/canvas/paths.js";import{listProcessTable as re}from"../core/canvas/pid.js";import{bindDaemonEventSource as ne,eventSource as oe}from"../core/events/source.js";import{hostExecPath as O}from"../core/runtime/branded-host.js";import{isDaemonRunning as ae,readPidfile as E,isPidAlive as f}from"./pidfile.js";import{DAEMON_STARTUP_WINDOW_MS as G}from"./startup-policy.js";import{exclusiveLockOwnerPid as ie}from"../core/exclusive-lock.js";import{onDiskMigrationLockPath as se}from"../core/canvas/paths.js";import{envNoDaemonAutostart as ce}from"../shared/env.js";import{awaitJoinLockRelease as ue,joinInProgress as le,joinInProgressError as de,joinLockHolder as pe}from"../core/directory/join-lock.js";import{waitForDaemonAvailability as fe}from"../api/index.js";import{localClient as me}from"../api/node-transport.js";const he=["CRTR_NODE_ID","CRTR_KIND","CRTR_MODE","CRTR_LIFECYCLE","CRTR_NODE_CWD","CRTR_CONTEXT_DIR","CRTR_CYCLES","CRTR_PROFILE_ID","CRTR_PARENT_NODE_ID","CRTR_FRONT_DOOR","TMUX","TMUX_PANE","CRTR_BROKER_ENGINE","CRTR_PI_BINARY","NODE_OPTIONS"];function M(){const e={...process.env};for(const t of he)delete e[t];return e}o(M,"sanitizedDaemonEnv");function we(){for(let e=k(Q(import.meta.url));;e=k(e)){if(H(D(e,"bin","runtime-generation-manifest.mjs")))return e;if(k(e)===e)throw new Error(`cannot locate the crouter generation root from ${import.meta.url}`)}}o(we,"generationRoot");async function A(){const e=we(),t=I(D(e,"bin","runtime-generation-manifest.mjs")).href,{runtimeHome:r}=await import(t);try{Z(D(r(),"selected"))}catch(s){if(s.code!=="ENOENT")throw s;return["--conditions","crtr-src","--import",I(V(import.meta.url).resolve("tsx/esm")).href,D(e,"src","daemon","crtrd-cli.ts")]}const n=I(D(e,"bin","runtime-selector.mjs")).href,{tryResolveRuntime:a}=await import(n),c=a("daemon");if(!c.ok)throw new Error(`cannot resolve the selected runtime generation's daemon entry: ${c.error}`);return[c.runtime.entryPath]}o(A,"resolveDaemonLaunch");function _e(e,t){return e.includes(` --canvas-home ${t} `)||e.endsWith(` --canvas-home ${t}`)}o(_e,"daemonCommandDeclaresCanvasHome");function De(){const e=re();if(e===null)return[];const t=g(),r=[];for(const{pid:n,command:a}of e)a.includes("daemon/crtrd")&&_e(a,t)&&n!==process.pid&&r.push(n);return r}o(De,"findDaemonPids");function Ee(e){const t=De().filter(n=>n!==e);for(const n of t)try{process.kill(n,"SIGTERM")}catch{}if(t.length===0)return t;const r=Date.now()+2e3;for(;Date.now()<r&&t.some(n=>f(n));)q("sleep",["0.1"]);for(const n of t)if(f(n))try{process.kill(n,"SIGKILL")}catch{}return t}o(Ee,"sweepStrayDaemons");async function Ve(){const e=process.execve;if(e===void 0)throw new Error(`this Node.js ${process.version} runtime cannot replace PID 1 with the successor daemon`);const t=O(),r=await A();return e(t,[t,...r,"--canvas-home",g()],M())}o(Ve,"execDaemon");async function R(e){const t=await A(),r=B(O(),[...t,...e],{stdio:["ignore","pipe","pipe"],env:M()});let n="",a="";r.stdout.setEncoding("utf8").on("data",u=>{n+=u}),r.stderr.setEncoding("utf8").on("data",u=>{a+=u});const c=await new Promise((u,m)=>{r.once("error",m),r.once("close",p=>u(p))});if(c!==0)throw new Error(a.trim()||`the ${e[0]} process exited with code ${c}`);const s=n.trim().split(`
+`).at(-1)??"";return JSON.parse(s)}o(R,"runDaemonEntry");function Qe(){return R(["export-sqlite"])}o(Qe,"runDaemonExportSqlite");function Ze(e,t){return R(["import-dump",e,t])}o(Ze,"runDaemonImportDump");function et(e){return R(["import-dump",e,"bodies"])}o(et,"runDaemonBodiesDump");function tt(){return R(["import-dump","","install-id"])}o(tt,"runDaemonInstallId");const xe=3*6e4,z=2e4,N=10,P=1e3;async function L(e=P){const t=Math.min(e,P);return me({autostart:!1,timeoutMs:t}).probeHealthz(t)}o(L,"probeDaemonHealth");function ye(e){const t=e instanceof Error?e.message:String(e);if(typeof e!="object"||e===null)return t;const r=e.code,n=e.cause,a=typeof r=="string"?r:n?.code;return typeof a=="string"&&!t.includes(a)?`${t}: ${a}`:t}o(ye,"probeErrorMessage");async function J(e=P){const t=await L(e);if(t.startup_phase==="blocked"){const r=E()??process.pid;throw new v(r,t.startup_blocked??"daemon startup is blocked")}if(!t.ok)throw new Error(t.startup_blocked??"crtrd health check reported not ready")}o(J,"probeDaemonServing");async function rt(){try{return await J(),!0}catch{return!1}}o(rt,"isDaemonServing");function $(e){return new Promise(t=>{setTimeout(t,e)})}o($,"sleepMs");class C extends Error{static{o(this,"DaemonExitedBeforeReadyError")}stderrTail;constructor(t,r){super(r===""?t:`${t}
+${r}`),this.name="DaemonExitedBeforeReadyError",this.stderrTail=r}}class v extends Error{static{o(this,"DaemonStartupBlockedError")}constructor(t,r){super(`daemon ${t} is running but normal startup is blocked: ${r}`),this.name="DaemonStartupBlockedError"}}class F extends Error{static{o(this,"DaemonNotPreparedError")}constructor(t,r){super(r===void 0?`daemon ${t} is serving but did not report the prepared startup phase`:`daemon ${t} reported startup phase ${r}, not prepared`),this.name="DaemonNotPreparedError"}}async function ge(e,t=G,r={},n="ready"){const a=r.readPidfile??E,c=r.isPidAlive??f,s=r.childExited,u=r.isDaemonServing,m=r.probeDaemonServing??(async w=>{if(u!==void 0){if(await u())return;throw new Error("crtrd health check reported not ready")}await J(w)}),p=r.probeDaemonHealth??L,x=r.sleepMs??$,y=r.now??Date.now;let i=null,l=null;return await fe({windowMs:t,pollIntervalMs:N,now:y,sleep:x,retry:o(w=>w!==l,"retry"),probe:o(async w=>{const h=a();if(h===null||!c(h)){const d=s?.();throw d!=null&&(d.signal!==null?l=new C(`daemon ${e} exited before becoming ready (by ${d.signal})`,r.childStderrTail?.()??""):d.code!==0&&(l=new C(`daemon ${e} exited before becoming ready (with exit code ${d.code??"?"})`,r.childStderrTail?.()??"")),l!==null)?l:new Error(`daemon ${e} has no live pidfile owner`)}if(n==="ready"){try{await m(w)}catch(d){throw d instanceof v&&(l=d),d}i=h;return}const _=await p(w);if(_.startup_phase==="prepared"){i=h;return}throw _.startup_phase==="blocked"?(l=new v(h,_.startup_blocked??"on-disk migration is blocked"),l):_.startup_phase==="ready"?(l=new F(h,_.startup_phase),l):new Error(`daemon ${h} is ${_.startup_phase??"not reporting a startup phase"}; waiting for prepared`)},"probe")}),i===e?null:i}o(ge,"verifyDaemonStartup");async function Se(e,t=z,r={}){const n=r.readPidfile??E,a=r.isPidAlive??f,c=r.sleepMs??$,s=r.now??Date.now,u=s()+t;for(;s()<=u;){if(!a(e)&&n()!==e)return;await c(N)}throw new Error(`daemon ${e} did not exit and release its pidfile within ${t}ms`)}o(Se,"waitForDaemonExit");const Re=2e3,ve=5e3;async function Y(e,t=z,r={}){const n=r.isPidAlive??f,a=r.sleepMs??$,c=r.now??Date.now;try{process.kill(e,"SIGTERM")}catch{}try{return await Se(e,t,r),"terminated"}catch{}try{process.kill(e,"SIGKILL")}catch{}const s=c()+Re;for(;c()<=s;){if(!n(e))return"killed";await a(N)}throw new Error(`daemon ${e} survived both SIGTERM and SIGKILL`)}o(Y,"stopDaemonProcess");async function nt(){const e=E(),t=e!==null&&f(e),r=t&&await Y(e)==="killed",n=Ee(null);return{stopped:t||n.length>0,...t?{pid:e}:{},strays_killed:n,force_killed:r}}o(nt,"stopAllDaemons");function Te(e,t,r=2048){let n;try{n=U(e,"r");const{size:a}=K(n),c=Math.max(t,a-r),s=a-c;if(s<=0)return"";const u=Buffer.alloc(s);return te(n,u,0,s,c),u.toString("utf8").trim()}catch{return""}finally{if(n!==void 0)try{j(n)}catch{}}}o(Te,"readLogTailFrom");async function be(e={}){if(process.platform==="linux"&&H("/run/crtr/launcher.sock")&&!f(E()??-1))throw new Error("crtrd is managed by the root launcher; it cannot self-daemonize");const t=e.targetPhase??"ready",r=E();if(r!==null&&f(r))try{const i=await L();if(t==="prepared"){if(i.startup_phase==="prepared")return{started:!1,existing_pid:r,startup_phase:i.startup_phase};throw new F(r,i.startup_phase)}return i.ok?{started:!1,existing_pid:r,...i.startup_phase===void 0?{}:{startup_phase:i.startup_phase}}:{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:i.startup_blocked??"crtrd health check reported not ready",...i.startup_phase===void 0?{}:{startup_phase:i.startup_phase}}}catch(i){if(i instanceof F)throw i;return{started:!1,existing_pid:r,running:!0,serving:!1,probe_error:ye(i)}}ee(g(),{recursive:!0});const n=await A();e.waitForJoinLock===!0&&await ue(xe);const a=pe();if(a!==null&&a!==process.pid&&e.ignoreJoinLock!==!0)throw de(a);const c=D(g(),"crtrd.err"),s=U(c,"a"),u=(()=>{try{return K(s).size}catch{return 0}})();oe()===void 0&&ne();const m=B(O(),[...n,"--canvas-home",g(),...t==="prepared"?["--prepare"]:[]],{detached:!0,stdio:["ignore",s,s],env:M()});j(s);const p=m.pid;if(p===void 0)throw new Error("daemon spawn did not return a pid");let x=null;m.once("exit",(i,l)=>{x={code:i,signal:l}}),m.unref();let y;try{y=await ge(p,G,{childExited:o(()=>x,"childExited"),childStderrTail:o(()=>Te(c,u),"childStderrTail")},t)}catch(i){const l=ie(se())===p;if(x===null&&!l&&!(i instanceof v))try{await Y(p,ve)}catch{}throw i}return y!==null?{started:!1,existing_pid:y,startup_phase:t}:{started:!0,pid:p,startup_phase:t}}o(be,"spawnDaemon");function ke(){return ce()||process.argv.includes("--no-autostart")}o(ke,"autostartDisabled");const Ie=5e3,Oe=6e4;let W=!1,S,T=0,b=0;function Me(e){return Math.min(Ie*2**(e-1),Oe)}o(Me,"autostartBackoffMs");function Ae(){S=void 0,T=0,b=0}o(Ae,"resetDaemonAutostart");function ot(){return S}o(ot,"daemonAutostartStartupFailure");function at(e={}){const t=e.isDaemonRunning??ae,r=e.spawnDaemon??be,n=e.now??Date.now;if(!ke()&&!le()&&!W&&!(n()<b)){if(t()){Ae();return}S=void 0,W=!0,r().then(()=>{S=void 0,T=0,b=0},a=>{a instanceof C&&(S=a.message),T+=1,b=n()+Me(T)}).finally(()=>{W=!1})}}o(at,"ensureDaemon");export{he as DAEMON_ENV_STRIP_KEYS,C as DaemonExitedBeforeReadyError,F as DaemonNotPreparedError,v as DaemonStartupBlockedError,ke as autostartDisabled,ot as daemonAutostartStartupFailure,_e as daemonCommandDeclaresCanvasHome,at as ensureDaemon,Ve as execDaemon,De as findDaemonPids,rt as isDaemonServing,L as probeDaemonHealth,J as probeDaemonServing,ye as probeErrorMessage,Ae as resetDaemonAutostart,et as runDaemonBodiesDump,Qe as runDaemonExportSqlite,Ze as runDaemonImportDump,tt as runDaemonInstallId,M as sanitizedDaemonEnv,be as spawnDaemon,nt as stopAllDaemons,Y as stopDaemonProcess,Ee as sweepStrayDaemons,ge as verifyDaemonStartup,Se as waitForDaemonExit};

package/dist/daemon/reconcilers/cron-lane.js

@@ -1 +1 @@
-var i=Object.defineProperty;var t=(c,r)=>i(c,"name",{value:r,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{runThrottledLane as m}from"./throttled-lane.js";class _{static{t(this,"CronLaneReconciler")}async run(r,a){await n.fresh(async()=>{try{await f(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.lease_recovery.failed",error:e})}});try{await s()}catch(e){o({level:"error",event:"cron.execution.reap_failed",error:e})}try{await m("cron-folder-sweep","",r,6e4,6e4,async()=>{await l()})}catch(e){o({level:"error",event:"cron.folder.sweep_failed",error:e})}if(a.lifecycle.acceptsDetachedWork())try{await p(r,{register:t(e=>a.lifecycle.registerDetached(e),"register")})}catch(e){o({level:"error",event:"cron.run.failed",error:e})}}}export{_ as CronLaneReconciler};
+var c=Object.defineProperty;var t=(i,e)=>c(i,"name",{value:e,configurable:!0});import{emitEvent as o}from"../../core/events/emit.js";import{operationIdContext as n}from"../../core/events/operation-id.js";import{sweepOrphanCronFolders as l}from"../cron/folders.js";import{reapRebuiltCronExecutions as s}from"../cron/run-socket.js";import{recoverStaleCronLeases as f,runDueCrons as p}from"../cron-run.js";import{isDrained as m}from"../drain.js";import{runThrottledLane as d}from"./throttled-lane.js";class g{static{t(this,"CronLaneReconciler")}async run(e,a){await n.fresh(async()=>{try{await f(e,{register:t(r=>a.lifecycle.registerDetached(r),"register")})}catch(r){o({level:"error",event:"cron.lease_recovery.failed",error:r})}});try{await s()}catch(r){o({level:"error",event:"cron.execution.reap_failed",error:r})}try{await d("cron-folder-sweep","",e,6e4,6e4,async()=>{await l()})}catch(r){o({level:"error",event:"cron.folder.sweep_failed",error:r})}if(a.lifecycle.acceptsDetachedWork()&&!await m())try{await p(e,{register:t(r=>a.lifecycle.registerDetached(r),"register")})}catch(r){o({level:"error",event:"cron.run.failed",error:r})}}}export{g as CronLaneReconciler};

package/dist/daemon/reconcilers/delivery-lane.js

@@ -1,8 +1,8 @@
-var g=Object.defineProperty;var p=(e,t)=>g(e,"name",{value:t,configurable:!0});import{randomUUID as D}from"node:crypto";import{claimPushDelivery as h,duePushDeliveries as j,messageBodyForDelivery as N,pushDeliveryClaimIsCurrent as O,settleDelivery as d}from"../../core/canvas/deliveries.js";import{deliveryBackoffMs as f}from"../../core/canvas/delivery-contract.js";import{openDb as c}from"../../core/canvas/db.js";import{pushLineFor as u}from"../../core/graph/custom.js";import{postReceiverUrl as b}from"../../core/graph/receiver-url.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{runDelivery as w,DELIVERY_TIMEOUT_MS as E}from"../delivery/run-delivery.js";const v=D();async function o(e,t,a){await d(e.deliveryId,e.attempts,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:a}},Date.now())}p(o,"failed");async function T(e,t){const a=await(await c()).prepare(`SELECT o.type, o.name, c.receiver_json, c.payload_json, m.object_id, i.external_id AS app,
+var g=Object.defineProperty;var p=(e,t)=>g(e,"name",{value:t,configurable:!0});import{randomUUID as D}from"node:crypto";import{claimPushDelivery as h,duePushDeliveries as j,messageBodyForDelivery as N,pushDeliveryClaimIsCurrent as O,settleDelivery as d}from"../../core/canvas/deliveries.js";import{deliveryBackoffMs as f}from"../../core/canvas/delivery-contract.js";import{openDb as c}from"../../core/canvas/db.js";import{pushLineFor as u}from"../../core/graph/custom.js";import{postReceiverUrl as b}from"../../core/graph/receiver-url.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as m}from"../../core/events/operation-id.js";import{runDelivery as w,DELIVERY_TIMEOUT_MS as E}from"../delivery/run-delivery.js";import{isDrained as T}from"../drain.js";const v=D();async function o(e,t,a){await d(e.deliveryId,e.attempts,t,{state:"permanent_failed",failure:{kind:"spawn_error",message:a}},Date.now())}p(o,"failed");async function A(e,t){const a=await(await c()).prepare(`SELECT o.type, o.name, c.receiver_json, c.payload_json, m.object_id, i.external_id AS app,
     d.message_id, owner.type AS owner_type, parent.external_id AS parent_app
     FROM deliveries d JOIN objects o ON o.id = d.destination_id
     JOIN custom_objects c ON c.object = o.id JOIN messages m ON m.message_id = d.message_id
     LEFT JOIN objects owner ON owner.id = o.owner
     LEFT JOIN identities i ON i.object = o.owner AND i.scheme = 'app'
     LEFT JOIN identities parent ON parent.object = owner.owner AND parent.scheme = 'app'
-    WHERE d.delivery_id = ?`).get(e.deliveryId);if(!a||!a.receiver_json){await o(e,t,`delivery ${e.deliveryId} has no supported receiver`);return}const i=JSON.parse(a.receiver_json);if(!await O(e,t)){await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now());return}if(typeof i=="object"&&typeof i.url=="string"&&a.type!=="action"){const n=await u(await c(),e.deliveryId);if(!n){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}try{const r=await b(i.url,JSON.stringify(n),{"Content-Type":"application/json","X-Crtr-Delivery":e.deliveryId,"X-Crtr-Event":a.message_id},E);r>=200&&r<300?await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now()):r>=300&&r<400?await o(e,t,`url receiver returned HTTP ${r}; redirects are not followed \u2014 register the final URL`):r>=400&&r<500&&r!==408&&r!==429?await o(e,t,`url receiver returned HTTP ${r}`):await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:`url receiver returned HTTP ${r}`}},Date.now())}catch(r){await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:String(r)}},Date.now())}return}if(typeof i!="object"||!Array.isArray(i.argv)||typeof i.cwd!="string"){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}if(a.type==="action"){if(typeof i.name!="string"||!a.object_id){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}const n=await N(e.deliveryId);if(n===null){await o(e,t,`delivery ${e.deliveryId} has no message body`);return}const{push:r,...y}=n,I={...y,action:{name:i.name,payload:a.payload_json===null?null:JSON.parse(a.payload_json)}};await w({argv:i.argv,cwd:i.cwd,document:I,attempt:e.attempts,nodeId:a.object_id,...a.app??a.parent_app?{app:a.app??a.parent_app}:{}},async _=>{await d(e.deliveryId,e.attempts,t,_,Date.now())});return}const s=await u(await c(),e.deliveryId);if(!s){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}await w({argv:i.argv,cwd:i.cwd,document:s,attempt:e.attempts,app:a.app??a.parent_app??"app:terminal",custom:{deliveryId:e.deliveryId,messageId:a.message_id}},async n=>{await d(e.deliveryId,e.attempts,t,n,Date.now())})}p(T,"deliverExternal");class P{static{p(this,"DeliveryLaneReconciler")}async run(t,a){if(!a.lifecycle.acceptsDetachedWork())return;let i;try{i=await j(t)}catch(s){m.fresh(()=>l({level:"error",event:"delivery.scan_failed",error:s}));return}for(const s of i)try{const n=await h(s.deliveryId,t,v);if(n===null)continue;a.lifecycle.registerDetached(T(n,v).catch(r=>{m.fresh(()=>l({level:"error",event:"delivery.failed",error:r,fields:{delivery_id:n.deliveryId}}))}))}catch(n){m.fresh(()=>l({level:"error",event:"delivery.claim_failed",error:n,fields:{delivery_id:s.deliveryId}}))}}}export{P as DeliveryLaneReconciler,v as deliveryDaemonInstanceId};
+    WHERE d.delivery_id = ?`).get(e.deliveryId);if(!a||!a.receiver_json){await o(e,t,`delivery ${e.deliveryId} has no supported receiver`);return}const i=JSON.parse(a.receiver_json);if(!await O(e,t)){await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now());return}if(typeof i=="object"&&typeof i.url=="string"&&a.type!=="action"){const n=await u(await c(),e.deliveryId);if(!n){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}try{const r=await b(i.url,JSON.stringify(n),{"Content-Type":"application/json","X-Crtr-Delivery":e.deliveryId,"X-Crtr-Event":a.message_id},E);r>=200&&r<300?await d(e.deliveryId,e.attempts,t,{state:"acknowledged"},Date.now()):r>=300&&r<400?await o(e,t,`url receiver returned HTTP ${r}; redirects are not followed \u2014 register the final URL`):r>=400&&r<500&&r!==408&&r!==429?await o(e,t,`url receiver returned HTTP ${r}`):await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:`url receiver returned HTTP ${r}`}},Date.now())}catch(r){await d(e.deliveryId,e.attempts,t,{state:"pending",nextAttemptAt:Date.now()+f(e.attempts),failure:{kind:"spawn_error",message:String(r)}},Date.now())}return}if(typeof i!="object"||!Array.isArray(i.argv)||typeof i.cwd!="string"){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}if(a.type==="action"){if(typeof i.name!="string"||!a.object_id){await o(e,t,`delivery ${e.deliveryId} has an invalid argv receiver`);return}const n=await N(e.deliveryId);if(n===null){await o(e,t,`delivery ${e.deliveryId} has no message body`);return}const{push:r,...y}=n,I={...y,action:{name:i.name,payload:a.payload_json===null?null:JSON.parse(a.payload_json)}};await w({argv:i.argv,cwd:i.cwd,document:I,attempt:e.attempts,nodeId:a.object_id,...a.app??a.parent_app?{app:a.app??a.parent_app}:{}},async _=>{await d(e.deliveryId,e.attempts,t,_,Date.now())});return}const s=await u(await c(),e.deliveryId);if(!s){await o(e,t,`delivery ${e.deliveryId} has no event document`);return}await w({argv:i.argv,cwd:i.cwd,document:s,attempt:e.attempts,app:a.app??a.parent_app??"app:terminal",custom:{deliveryId:e.deliveryId,messageId:a.message_id}},async n=>{await d(e.deliveryId,e.attempts,t,n,Date.now())})}p(A,"deliverExternal");class H{static{p(this,"DeliveryLaneReconciler")}async run(t,a){if(!a.lifecycle.acceptsDetachedWork()||await T())return;let i;try{i=await j(t)}catch(s){m.fresh(()=>l({level:"error",event:"delivery.scan_failed",error:s}));return}for(const s of i)try{const n=await h(s.deliveryId,t,v);if(n===null)continue;a.lifecycle.registerDetached(A(n,v).catch(r=>{m.fresh(()=>l({level:"error",event:"delivery.failed",error:r,fields:{delivery_id:n.deliveryId}}))}))}catch(n){m.fresh(()=>l({level:"error",event:"delivery.claim_failed",error:n,fields:{delivery_id:s.deliveryId}}))}}}export{H as DeliveryLaneReconciler,v as deliveryDaemonInstanceId};

package/dist/daemon/reconcilers/import-lane.d.ts

@@ -0,0 +1,9 @@
+import type { DetachedWorkLifecycle } from './broker-supervision.js';
+export declare const IMPORT_LEASE_MS: number;
+export declare class ImportLaneReconciler {
+    run(now: number, ctx: {
+        lifecycle: DetachedWorkLifecycle;
+    }): void;
+    /** One pass over the runs due now. Exposed for tests. */
+    pass(now: number): Promise<number>;
+}

package/dist/daemon/reconcilers/import-lane.js

@@ -0,0 +1 @@
+var p=Object.defineProperty;var n=(i,e)=>p(i,"name",{value:e,configurable:!0});import{openDb as m}from"../../core/canvas/db.js";import{emitEvent as l}from"../../core/events/emit.js";import{operationIdContext as c}from"../../core/events/operation-id.js";import{processRun as f}from"../../core/import/load/run.js";import{ClaimLostError as d,withRowClaim as _}from"./row-claim.js";const R=2*6e4;class S{static{n(this,"ImportLaneReconciler")}run(e,o){if(!o.lifecycle.acceptsDetachedWork())return;const a=c.fresh(()=>this.pass(e).then(()=>{})).catch(t=>{c.fresh(()=>l({level:"error",event:"import.lane.failed",error:t}))});o.lifecycle.registerDetached(a)}async pass(e){const a=await(await m()).prepare("SELECT * FROM import_runs WHERE (state = 'loading' OR state = 'committed') AND (claimed_by IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ?) ORDER BY created_at").all(new Date(e).toISOString());let t=0;for(const s of a)try{await _("import_runs",s.run_id,R,r=>f(s,r))&&(t+=1)}catch(r){if(!(r instanceof d))throw r}return t}}export{R as IMPORT_LEASE_MS,S as ImportLaneReconciler};

package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js

@@ -1 +1 @@
-var g=Object.defineProperty;var s=(i,e)=>g(i,"name",{value:e,configurable:!0});import{clearFrozen as p,getNode as h,getRow as y}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as k}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as S}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as N}from"../../../core/canvas/deliveries.js";import{publishNotice as D}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as E}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as R}from"./remote-wake.js";import{asNodePerson as z}from"./node-person.js";import{commitReopenResident as T}from"../../../core/runtime/reopen.js";import{reviveNode as m}from"../../../core/runtime/revive.js";import{hasPendingWake as v}from"../live-obligation.js";import{readInterruptedTurn as L}from"../../../core/canvas/recovery.js";import{mapAsync as C}from"../../../core/async-array.js";const f=6,b=10*6e4,I=2e4;async function M(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(M,"readEpisode");async function O(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(O,"claimAttempt");async function P(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(P,"writeEpisode");function U(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s(U,"thawTier");function x(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(x,"compareThaw");class re{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await C(a.candidates,async n=>({...n,tier:U(n.row,await v(n.row.node_id))}));t.sort(x);const r={stalled:0,waitingOnMail:0};for(const n of t)await S.fresh(()=>z(n.row,()=>this.enact(e,n,a.fleet)))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await y(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await T(e.node_id);try{return(await m(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(E(a)||a instanceof R)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return await u(e,"release"),await p(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await L(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await h(e.node_id))?.pi_session_id!=null&&!await v(e.node_id))return await u(e.node_id,"release"),await p(e.node_id),"skipped";try{return(await m(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(E(t)||t instanceof R)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await h(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await p(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await N(t),n=await M(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=f)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:f}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await D(t,"notice.crash",`Child crashed \u2014 ${k(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const w=l?void 0:n.lastAttempt,A=Math.min(I*2**o,b);return w!==void 0&&e-w<A||!await O(t,n.attempts,o+1,r,e)?"skipped":(await m(t,{resume:!0,capacity:"freeze"})).outcome==="frozen"?(await P(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:f}}),"launched")}}export{re as FreezeLaneReconciler,x as compareThaw,U as thawTier};
+var N=Object.defineProperty;var s=(i,e)=>N(i,"name",{value:e,configurable:!0});import{clearFrozen as f,getNode as g,getRow as D}from"../../../core/canvas/index.js";import{openDb as c}from"../../../core/canvas/db.js";import{fullName as z}from"../../../core/canvas/labels.js";import{isSafeNodeId as d}from"../../../core/canvas/paths.js";import{emitEvent as _}from"../../../core/events/emit.js";import{operationIdContext as L}from"../../../core/events/operation-id.js";import{acknowledgedNodeMailCount as T}from"../../../core/canvas/deliveries.js";import{publishNotice as C}from"../../../core/runtime/close.js";import{transition as u}from"../../../core/runtime/lifecycle.js";import{isStartRefusal as y}from"../../../core/runtime/start-check.js";import{EnvironmentImagePending as A}from"./remote-wake.js";import{asNodePerson as M}from"./node-person.js";import{MachineLaunchUncertainError as b,MachineOfflineError as m}from"../../../core/runtime/runner/machine/ops.js";import{commitReopenResident as I}from"../../../core/runtime/reopen.js";import{reviveNode as w}from"../../../core/runtime/revive.js";import{hasPendingWake as k}from"../live-obligation.js";import{readInterruptedTurn as O}from"../../../core/canvas/recovery.js";import{mapAsync as P}from"../../../core/async-array.js";const h=6,U=10*6e4,x=2e4;async function H(i){const e=await(await c()).prepare("SELECT idle_release_attempts, idle_release_last_attempt_at, idle_release_acked_count FROM nodes WHERE node_id = ?").get(i);return{attempts:Number(e?.idle_release_attempts??0),acknowledgedCount:Number(e?.idle_release_acked_count??0),lastAttempt:e?.idle_release_last_attempt_at==null?void 0:Date.parse(e.idle_release_last_attempt_at)}}s(H,"readEpisode");async function F(i,e,a,t,r){const n=await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ? AND idle_release_attempts = ?").run(a,new Date(r).toISOString(),t,i,e);return Number(n.changes)===1}s(F,"claimAttempt");async function W(i,e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = ?, idle_release_last_attempt_at = ?, idle_release_acked_count = ? WHERE node_id = ?").run(e.attempts,e.lastAttempt===void 0?null:new Date(e.lastAttempt).toISOString(),e.acknowledgedCount,i)}s(W,"writeEpisode");function $(i,e){if(e)return 2;const a=i.launched_at;if(a==null)return 1;const t=i.last_settled_at;return t==null?2:Date.parse(t)>Date.parse(a)?1:2}s($,"thawTier");function B(i,e){const a=i.row.frozen_at,t=e.row.frozen_at;return a==null?t==null?0:1:t==null?-1:Date.parse(a)-Date.parse(t)}s(B,"compareThaw");class le{static{s(this,"FreezeLaneReconciler")}async endEpisode(e){await(await c()).prepare("UPDATE nodes SET idle_release_attempts = 0, idle_release_last_attempt_at = NULL, idle_release_acked_count = 0 WHERE node_id = ? AND (idle_release_attempts <> 0 OR idle_release_last_attempt_at IS NOT NULL OR idle_release_acked_count <> 0)").run(e)}async run(e,a){const t=await P(a.candidates,async n=>({...n,tier:$(n.row,await k(n.row.node_id))}));t.sort(B);const r={stalled:0,waitingOnMail:0};for(const n of t)await L.fresh(()=>M(n.row,()=>this.enact(e,n,a.fleet)))==="frozen"&&(n.tier===1?r.stalled+=1:r.waitingOnMail+=1);return r}async enact(e,a,t){const r=a.row.node_id;try{return a.enact==="thaw"?await this.enactThaw(a.row):a.enact==="reopen"?await this.enactReopen(a.row):a.enact==="idle-release"?await this.enactIdleRelease(e,a.row):(await t.applyDeadRowPolicy(r),await t.has(r)?"launched":(await D(r))?.frozen_at!=null?"frozen":"skipped")}catch(n){return _({level:"error",event:"node.lifecycle.enact_failed",...d(r)?{node_id:r}:{},error:n,fields:{...d(r)?{}:{affected_node_id:r},enact:a.enact}}),"skipped"}}async enactReopen(e){await I(e.node_id);try{return(await w(e.node_id,{resume:!0,capacity:"freeze"})).outcome==="frozen"?"frozen":"launched"}catch(a){if(y(a)||a instanceof A||a instanceof m)return await this.settleRefusedStart(e.node_id,a,"reopen");throw a}}async settleRefusedStart(e,a,t){return a instanceof b||await u(e,"release"),await f(e),_({level:"warn",event:"node.start.refused",node_id:e,error:a,fields:{enact:t}}),"skipped"}async enactThaw(e){const a=(await O(e.node_id))?.interrupted===!0;if(!a&&e.lifecycle==="resident"&&(await g(e.node_id))?.pi_session_id!=null&&!await k(e.node_id))return await u(e.node_id,"release"),await f(e.node_id),"skipped";try{return(await w(e.node_id,{resume:!0,capacity:"freeze",recovery:!0,...a?{wakeReason:"runtime-restart-abort"}:{}})).outcome==="frozen"?"frozen":"launched"}catch(t){if(y(t)||t instanceof A||t instanceof m)return await this.settleRefusedStart(e.node_id,t,"thaw");const r=await g(e.node_id);throw r!==null&&(r.status==="active"||r.status==="idle")&&(await u(e.node_id,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:t instanceof Error?t.message:String(t)}}}),await f(e.node_id)),t}}async enactIdleRelease(e,a){const t=a.node_id,r=await T(t),n=await H(t),l=n.attempts===0||n.acknowledgedCount!==r,o=l?0:n.attempts;if(o>=h)return await this.endEpisode(t),_({level:"warn",event:"broker.idle_release.exhausted",...d(t)?{node_id:t}:{},fields:{...d(t)?{}:{affected_node_id:t},attempts:o,max_attempts:h}}),await u(t,"crash",{reason:"launch_failed",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",respawn_failures:o}}}),await C(t,"notice.crash",`Child crashed \u2014 ${z(a)} (${t}) failed to relaunch after ${o} attempts while trying to deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${t}\`.`,{reason:"child-crashed",child:t}),"skipped";const E=l?void 0:n.lastAttempt,S=Math.min(x*2**o,U);if(E!==void 0&&e-E<S||!await F(t,n.attempts,o+1,r,e))return"skipped";const v=s(()=>W(t,{attempts:l?0:o,lastAttempt:l?void 0:n.lastAttempt,acknowledgedCount:r}),"giveBack");let R;try{R=await w(t,{resume:!0,capacity:"freeze"})}catch(p){if(!(p instanceof m))throw p;return await v(),await this.settleRefusedStart(t,p,"idle-release")}return R.outcome==="frozen"?(await v(),"frozen"):(_({level:"info",event:"broker.revive.idle_release",...d(t)?{node_id:t}:{},attempt:o+1,fields:{...d(t)?{}:{affected_node_id:t},attempts:o+1,max_attempts:h}}),"launched")}}export{le as FreezeLaneReconciler,B as compareThaw,$ as thawTier};