@lensmcp/cluster 1.25.24 → 1.25.26

package/executors/gateway/runtime/lifecycle.js

@@ -1,3 +1,3 @@
-"use strict";var Ue=Object.defineProperty;var v=(t,a)=>Ue(t,"name",{value:a,configurable:!0});var Ve=Object.defineProperty,s=v((t,a)=>Ve(t,"name",{value:a,configurable:!0}),"s");Object.defineProperty(exports,"__esModule",{value:!0}),exports.scaleTarget=scaleTarget,exports.isLoopbackHostname=isLoopbackHostname,exports.isOwnRunnerCommand=isOwnRunnerCommand,exports.isRunnerOf=isRunnerOf,exports.isTestSourcePath=isTestSourcePath,exports.sourceScanPass=sourceScanPass,exports.sourceSetSnapshot=sourceSetSnapshot,exports.sourceSetSignature=sourceSetSignature,exports.diffSourceSets=diffSourceSets,exports.sourceChangeWindowMs=sourceChangeWindowMs,exports.sourceScopeDirs=sourceScopeDirs,exports.podRecycleReason=podRecycleReason,exports.lensFrontendStale=lensFrontendStale,exports.lensWatcherVerdict=lensWatcherVerdict,exports.podWatcherVerdict=podWatcherVerdict,exports.lensChildWedged=lensChildWedged,exports.isPortConflictExit=isPortConflictExit,exports.podChangeClock=podChangeClock,exports.createServiceLayer=createServiceLayer,exports.chooseStalePodKillTarget=chooseStalePodKillTarget;const tslib_1=require("tslib"),node_child_process_1=require("node:child_process"),node_crypto_1=require("node:crypto"),fs=tslib_1.__importStar(require("node:fs")),http=tslib_1.__importStar(require("node:http")),net=tslib_1.__importStar(require("node:net")),path=tslib_1.__importStar(require("node:path")),node_util_1=require("node:util"),discovery_1=require("./discovery"),pod_sock_1=require("../../../pod-sock"),scope_1=require("./scope"),pod_env_1=require("./pod-env"),pod_probe_1=require("./pod-probe"),service_keys_1=require("./service-keys"),workspace_log_1=require("./workspace-log"),own_log_1=require("./own-log"),inspector_secret_1=require("../../../inspector-secret"),pod_output_1=require("./pod-output"),serve_launch_1=require("./serve-launch"),shared_build_1=require("./shared-build"),warm_start_1=require("./warm-start"),protocol_1=require("../../../shared-build/protocol"),types_1=require("./types"),exit_reason_1=require("../../../exit-reason"),handoff_retire_1=require("../../../handoff-retire"),process_name_1=require("../../../process-name");function eastWestEnv(t,a){const c={},d=t.serviceKeys[(0,service_keys_1.serviceKeyId)(a.wsKey,a.project)];if(d&&(c.LENSMCP_SERVICE_KEY=d),t.https){const f=require("../../../basic-ssl").caCertPath();fs.existsSync(f)&&(c.NODE_EXTRA_CA_CERTS=f)}return c}v(eastWestEnv,"eastWestEnv"),s(eastWestEnv,"eastWestEnv");const sleep=s(t=>new Promise(a=>setTimeout(a,t)),"sleep"),SRC_EXCLUDE=new Set(["node_modules","dist","tmp","coverage","build","out","target"]),SRC_EXT=new Set([".ts",".tsx",".js",".jsx",".mjs",".cjs",".css",".scss",".sass",".less",".json",".html",".htm",".vue",".svelte",".mdx"]),SCAN_FILE_CAP=1e5;function fnv1a(t){let a=2166136261;for(let c=0;c<t.length;c+=1)a^=t.charCodeAt(c),a=Math.imul(a,16777619);return a>>>0}v(fnv1a,"fnv1a"),s(fnv1a,"fnv1a");const execFileP=(0,node_util_1.promisify)(node_child_process_1.execFile);async function gitIgnoredPaths(t){try{const{stdout:a}=await execFileP("git",["ls-files","--others","--directory","--ignored","--exclude-standard","-z"],{cwd:t,maxBuffer:33554432,timeout:1e4,windowsHide:!0});return new Set(a.split("\0").filter(Boolean))}catch{return new Set}}v(gitIgnoredPaths,"gitIgnoredPaths"),s(gitIgnoredPaths,"gitIgnoredPaths");function scaleTarget(t,a=fs.existsSync){if(t.decl.adminUrl)return{url:t.decl.adminUrl};const c=path.join(t.pool.dir,"parent.sock");return a(c)?{socketPath:c}:t.decl.port?{url:`http://localhost:${t.decl.port}`}:void 0}v(scaleTarget,"scaleTarget"),s(scaleTarget,"scaleTarget");function isLoopbackHostname(t){const a=t.toLowerCase().replace(/^\[(.*)\]$/,"$1").replace(/\.$/,"");return a==="localhost"||a.endsWith(".localhost")?!0:net.isIPv4(a)?a.split(".")[0]==="127":net.isIPv6(a)&&new URL(`http://[${a}]/`).hostname==="[::1]"}v(isLoopbackHostname,"isLoopbackHostname"),s(isLoopbackHostname,"isLoopbackHostname");function postScale(t,a,c){let d;if("socketPath"in t)d=http.request({socketPath:t.socketPath,path:"/webpack/scale",method:"POST",headers:{"content-type":"application/json"}},f=>f.resume());else{const f=t.url.startsWith("https"),T=f?require("node:https"):http,E={method:"POST"};f&&isLoopbackHostname(new URL(t.url).hostname)&&(E.rejectUnauthorized=!1),c&&(E.lookup=c),d=T.request(`${t.url}/webpack/scale`,E,R=>R.resume())}return d.on("error",()=>{}),d.end(JSON.stringify({pods:a})),d}v(postScale,"postScale"),s(postScale,"postScale");function isOwnRunnerCommand(t,a){return(0,serve_launch_1.isServeRunnerCommand)(t)?(0,serve_launch_1.isServeRunnerCommandOf)(t,a):t.includes("/node_modules/nx/")?t.includes(`${a.replace(/\/+$/,"")}/node_modules/nx/`):!0}v(isOwnRunnerCommand,"isOwnRunnerCommand"),s(isOwnRunnerCommand,"isOwnRunnerCommand");function isRunnerOf(t,a,c){const d=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return!new RegExp(`(^|\\s)run ${d}(:|\\s|$)`).test(t)||!(0,serve_launch_1.isServeRunnerCommand)(t)&&!/\bnx(\.js)?\b/.test(t)?!1:isOwnRunnerCommand(t,c)}v(isRunnerOf,"isRunnerOf"),s(isRunnerOf,"isRunnerOf");const TEST_FILE_RE=/\.(spec|test|e2e|e2e-spec|stories)\.[cm]?[jt]sx?$/,TEST_DIR_SEGMENTS=new Set(["__tests__","__mocks__","__snapshots__"]);function isTestSourcePath(t){const a=t.split("/"),c=a[a.length-1]??"";if(TEST_FILE_RE.test(c))return!0;for(let d=0;d<a.length-1;d+=1)if(TEST_DIR_SEGMENTS.has(a[d]))return!0;return!1}v(isTestSourcePath,"isTestSourcePath"),s(isTestSourcePath,"isTestSourcePath");function sourceScanPass(t,a,c){const d=c?.ignoreTests??!0;let f;const T=new Map,E=`tree:${path.resolve(t)}`,R=s(w=>path.relative(t,w).split(path.sep).join("/"),"rel"),A=s(async(w,y)=>{const k=await(f??=gitIgnoredPaths(t)),L=[],O=new Set;let D=!1;const V=s(async(Q,S)=>{if(L.length>=SCAN_FILE_CAP){D=!0;return}let N;try{N=await fs.promises.readdir(Q,{withFileTypes:!0})}catch{return}S&&O.add(R(Q));for(const X of N){if(L.length>=SCAN_FILE_CAP){D=!0;return}const F=X.name,Z=path.join(Q,F);if(X.isDirectory()){if(!S||F.startsWith(".")||SRC_EXCLUDE.has(F)||a?.has(F)||F.endsWith("-devserver")||k.has(`${R(Z)}/`))continue;await V(Z,!0)}else if(X.isFile()){const ee=F.lastIndexOf(".");if(ee<=0||!SRC_EXT.has(F.slice(ee)))continue;const H=R(Z);if(k.has(H)||d&&isTestSourcePath(H))continue;L.push(H)}}},"walk");return await V(w,y),{paths:L,entered:O,truncated:D}},"walkDir"),I=s((w,y)=>{let k=T.get(w);return k||(k=y(),T.set(w,k)),k},"memo"),ne=s(async()=>{const w=T.get(E);if(!w)return;const y=await w;return y.truncated?void 0:y},"wideWalk"),J=new Set,ae=s(()=>I(`files:${path.resolve(t)}`,async()=>{const w=await ne();return w?{paths:w.paths.filter(y=>!y.includes("/")),entered:J,truncated:!1}:A(t,!1)}),"rootFiles"),ce=s(w=>I(`tree:${path.resolve(w)}`,async()=>{const y=await ne(),k=R(w);if(y?.entered.has(k)){const L=`${k}/`;return{paths:k===""?y.paths:y.paths.filter(O=>O.startsWith(L)),entered:J,truncated:!1}}return A(w,!0)}),"subtree"),B=s(w=>{const y=new Set;let k=0,L=w.some(D=>D.truncated);const O=s(D=>y.has(D)?!0:y.size>=SCAN_FILE_CAP?(L=!0,!1):(y.add(D),k^=fnv1a(D),!0),"add");for(const D of w)if(!D.paths.every(O))break;return{signature:`${y.size}:${(k>>>0).toString(36)}`,paths:y,truncated:L}},"compose");return{async snapshot(w){if(!w||w.length===0)return B([await I(E,()=>A(t,!0))]);const y=[await ae()];for(const k of w)y.push(await ce(k));return B(y)}}}v(sourceScanPass,"sourceScanPass"),s(sourceScanPass,"sourceScanPass");async function sourceSetSnapshot(t,a,c,d){return sourceScanPass(t,c,d).snapshot(a)}v(sourceSetSnapshot,"sourceSetSnapshot"),s(sourceSetSnapshot,"sourceSetSnapshot");async function sourceSetSignature(t,a,c,d){return(await sourceSetSnapshot(t,a,c,d)).signature}v(sourceSetSignature,"sourceSetSignature"),s(sourceSetSignature,"sourceSetSignature");const SOURCE_DIFF_PATH_CAP=128;function diffSourceSets(t,a,c,d){if(t.paths.size===a.paths.size){let R=!0;for(const A of t.paths)if(!a.paths.has(A)){R=!1;break}if(R)return null}const f=[],T=[];let E=t.truncated||a.truncated;for(const R of a.paths)t.paths.has(R)||(f.length+T.length<SOURCE_DIFF_PATH_CAP?f.push(R):E=!0);for(const R of t.paths)a.paths.has(R)||(f.length+T.length<SOURCE_DIFF_PATH_CAP?T.push(R):E=!0);return{changedAt:c,added:f,removed:T,truncated:E,...d!==void 0?{windowMs:d}:{}}}v(diffSourceSets,"diffSourceSets"),s(diffSourceSets,"diffSourceSets");function sourceChangeWindowMs(t,a,c,d){const f=a&&a>0?t-a:c;return Math.max(f,c)+d}v(sourceChangeWindowMs,"sourceChangeWindowMs"),s(sourceChangeWindowMs,"sourceChangeWindowMs");function sourceScopeDirs(t,a,c,d,f){const T=s(A=>{const I=path.relative(t,A).split(path.sep).filter(Boolean);return I.length===0||I[0].startsWith("..")?"":I[0]},"bucketOf")(a);if(!T)return[];const E=new Set(c.filter(Boolean)),R=new Set([T]);for(const A of d)A.startsWith(".")||SRC_EXCLUDE.has(A)||f?.has(A)||A.endsWith("-devserver")||E.has(A)||R.add(A);return[...R].sort().map(A=>path.join(t,A))}v(sourceScopeDirs,"sourceScopeDirs"),s(sourceScopeDirs,"sourceScopeDirs");function podRecycleReason(t,a,c,d){return!t.proc||t.state!=="up"||t.spawnAt==null||t.inflight>0||(t.activeWork??0)>0||a-t.lastUsed<d.settleMs||t.lastStaleRecycleAt!=null&&a-t.lastStaleRecycleAt<d.cooldownMs?null:c>0&&c-t.spawnAt>d.graceMs?"stale-source-set":d.maxAgeMs>0&&a-t.spawnAt>d.maxAgeMs?"max-age":null}v(podRecycleReason,"podRecycleReason"),s(podRecycleReason,"podRecycleReason");function lensFrontendStale(t,a,c,d,f){return!(d-a<f.cooldownMs||c<=0||c-t<=f.graceMs||d-c<f.settleMs)}v(lensFrontendStale,"lensFrontendStale"),s(lensFrontendStale,"lensFrontendStale");function lensWatcherVerdict(t,a,c,d=[]){if(!t)return"unknown";if("schemaVersion"in t&&t.schemaVersion>=2&&d.length>0){const f=a-c,T=new Set(t.recentEvents.filter(E=>E.at>=f).map(E=>path.resolve(E.path)));return d.every(E=>T.has(path.resolve(E)))?"heard":"missed"}return t.lastFsSetEventAt>=a-c?"heard":"missed"}v(lensWatcherVerdict,"lensWatcherVerdict"),s(lensWatcherVerdict,"lensWatcherVerdict");function podWatcherVerdict(t,a,c){if(!t)return"unknown";if(t.degraded)return"degraded";const d=a-c;return t.lastWatchRunAt>=d||t.lastBuildAt>=d?"heard":"silent"}v(podWatcherVerdict,"podWatcherVerdict"),s(podWatcherVerdict,"podWatcherVerdict");function lensChildWedged(t,a,c,d){const f=t.hasListened?d.graceMs:Math.max(d.graceMs,d.neverListenedGraceMs);return c-t.spawnedAt<f||c-a<d.cooldownMs||t.probeFails<Math.max(2,d.failureThreshold)?null:"wedged-no-listener"}v(lensChildWedged,"lensChildWedged"),s(lensChildWedged,"lensChildWedged");function isPortConflictExit(t,a){return t.signal||t.code===null||t.code===0||t.ranMs>a.fastFailMs?!1:t.portHeld}v(isPortConflictExit,"isPortConflictExit"),s(isPortConflictExit,"isPortConflictExit");function podChangeClock(t,a){const c=t.sourceSetChangedFor?.get(a);if(c!==void 0){const d=t.sourceSetChangesFor?.get(a);return{at:c,...d?{change:d}:{},scope:"project"}}return t.sourceSetScoped?.has(a)?{at:0,scope:"none"}:{at:t.sourceSetChangedAt??0,...t.sourceSetChange?{change:t.sourceSetChange}:{},scope:"workspace"}}v(podChangeClock,"podChangeClock"),s(podChangeClock,"podChangeClock");const changeEvidence=s(t=>{if(!t)return"no path evidence";const a=[...t.added.map(c=>`+${c}`),...t.removed.map(c=>`-${c}`)];return a.length?`${a.slice(0,3).join(", ")}${a.length>3?` (+${a.length-3} more)`:""}`:"no path evidence"},"changeEvidence");function createServiceLayer(t,a,c={}){const d=c.spawnProcess??node_child_process_1.spawn,f=c.probeAlive??pod_probe_1.probePodAlive,T=c.killTree??((e,r="SIGTERM")=>{try{process.kill(-e.pid,r)}catch{try{e.kill(r)}catch{}}}),E=c.sharedBuild??(0,shared_build_1.createSharedBuildRelay)(t),R=s((e,r="SIGTERM")=>{E.detach(e),T(e,r)},"killTree"),A=c.handoffDrainMs??(0,handoff_retire_1.handoffDrainBoundMs)(),I=c.treeAlive??(e=>{if(!e.pid)return!1;try{return process.kill(-e.pid,0),!0}catch(r){return r.code==="EPERM"}}),ne=c.retirePollMs??1e3,J=c.rollReadyMs??types_1.POD_ROLL_READY_MS,ae=c.rollPollMs??750,ce=c.maxConcurrentRolls??types_1.POD_MAX_CONCURRENT_ROLLS,B=ce>0?ce:Number.POSITIVE_INFINITY,w=c.maxConcurrentColdStarts??types_1.MAX_COLD_STARTS,y=w>0?w:Number.POSITIVE_INFINITY;let k=0;const L=[],O=s(()=>k<y?(k+=1,Promise.resolve()):new Promise(e=>{L.push(()=>{k+=1,e()})}),"acquireColdSlot"),D=s(()=>{k-=1,L.shift()?.()},"releaseColdSlot"),{emit:V}=a,Q=s(e=>e.wsKey===t.wsKey?void 0:path.join(e.root,".lensmcp","events.jsonl"),"serviceBus"),S=s((e,r,n,o,i)=>V(r,n,o,{wsKey:e.wsKey,...i},void 0,Q(e)),"emitSvc"),N=s(e=>(e.scannedAt=0,(0,discovery_1.pickSock)(e,t.scanTtlMs)?e.socks.length:0),"scanNow"),X=s((e,r)=>{let n;try{n=(0,node_child_process_1.execFileSync)("ps",["ax","-ww","-o","pid=,pgid=,command="],{encoding:"utf8",timeout:5e3})}catch{return 0}const o=e.proc?.pid;let i=0;for(const l of n.split(`
-`)){const p=l.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/);if(!p)continue;const u=Number(p[1]),$=Number(p[2]);if(isRunnerOf(p[3],r,e.root)&&!(u===process.pid||u===o)&&!(o!==void 0&&$===o)){try{process.kill(-$,"SIGKILL")}catch{try{process.kill(u,"SIGKILL")}catch{}}i+=1}}return i>0&&(console.log(`[gateway] ${r}: reaped ${i} stale runner tree(s) \u2014 a foreign generation of this service (holding nx's task lock, or its pool)`),S(e,"warning",`stale-runner reap: ${e.project} (${i} foreign tree(s))`,`cluster-service:${e.project}`,{kind:"stale-runner-reap",project:e.project,reaped:i})),i},"reapForeignRunners"),F=s((e,r)=>({...(0,pod_env_1.podBaseEnv)(process.env,t.root,e.root),...e.instanceEnv?{...e.instanceEnv,LENSMCP_SOCKET_ONLY:"1"}:{},...e.frozen?{LENSMCP_FREEZE:"1",LENSMCP_TYPECHECK:"off"}:{},NX_DAEMON:"false",NX_ISOLATE_PLUGINS:"false",...e.wsKey?{LENSMCP_WS_KEY:e.wsKey}:{},...e.sockNs?{LENSMCP_SOCK_NS:e.sockNs}:{},LENSMCP_EVENT_FILE:e.wsKey===t.wsKey?t.eventFile:path.join(e.root,".lensmcp","events.jsonl"),LENSMCP_GATEWAY_PID:String(process.pid),LENSMCP_POD_GENERATION:String(r),...eastWestEnv(t,e)}),"serviceEnv"),Z=s((e,r,n,o)=>{const i=F(e,r),l=e.wsKey||path.basename(e.root),p=(0,serve_launch_1.planServeLaunch)({stores:t.graphStores,root:e.root,project:e.project,target:e.decl.serveTarget??"serve-hmr",childCount:o,baseEnv:i});if(p.kind==="runner"){const h=E.offeredTo(e),_=d((0,process_name_1.namedExecPath)((0,process_name_1.processName)(l,e.project,serve_launch_1.SERVE_RUNNER_ROLE)),[...p.script.execArgv,p.script.path,...(0,serve_launch_1.serveRunnerArgs)(n,o,e.root)],{cwd:e.root,detached:!0,env:h?{...p.env,[protocol_1.SHARED_BUILD_ENV]:"1"}:p.env,stdio:["ignore","pipe","pipe","ipc"]});return h&&E.attach(e,r,_),_.send?.(p.message,j=>{j&&console.error(`[gateway] ${n}: could not hand the task to its runner (${j.message})`)}),{proc:_,launcher:"runner"}}console.log(`[gateway] ${n}: starting through nx run \u2014 ${p.reason}`);const u=path.join(e.root,"node_modules","nx","dist","bin","nx.js"),$=(0,process_name_1.namedExecPath)((0,process_name_1.processName)(l,e.project,"nx")),[M,m]=fs.existsSync(u)?[$,[u,"run",n,`--childCount=${o}`]]:["yarn",["nx","run",n,`--childCount=${o}`]];return{proc:d(M,m,{cwd:e.root,detached:!0,env:i,stdio:["ignore","pipe","pipe"]}),launcher:"nx"}},"startServiceTree"),ee=new WeakMap,H=s((e,r,n)=>{const o=`${e.project}:${e.decl.serveTarget??"serve-hmr"}`,i=Math.max(1,e.decl.minPods??1),{proc:l,launcher:p}=Z(e,r,o,i);ee.set(l,p),n==="active"&&(e.launcher=p),n==="active"&&(e.lastErrors=[],e.buildFailed=!1,e.faults=new pod_output_1.PodFaults);const u=s(m=>{const h=m.toString();for(const _ of h.split(`
-`)){const j=(0,pod_output_1.classifyPodLine)(_),{clean:C,broken:K,healthy:re}=j,g=C;if(K){e.lastErrors.push(C),e.lastErrors.length>12&&e.lastErrors.shift(),e.buildFailed=!0;const x=Date.now();(C!==e.lastErrorEmitSig||x-(e.lastErrorEmitAt??0)>=types_1.SERVICE_ERROR_THROTTLE_MS)&&(e.lastErrorEmitSig=C,e.lastErrorEmitAt=x,S(e,"error",`service error: ${e.project}`,`cluster-service:${e.project}`,{kind:"service-error",project:e.project,line:C,fault:j.fault}))}re&&(e.buildFailed=!1);const P=(e.faults??=new pod_output_1.PodFaults).observe(j);if(P){e.lastErrorEmitSig=void 0;const x=P.open.length===0;S(e,"info",`${x?"service recovered":`${P.closed.join(" + ")} recovered`}: ${e.project}`,`cluster-service:${e.project}`,{kind:x?"service-recovered":"service-fault-closed",project:e.project,line:C,closed:P.closed,open:P.open})}!e.lockBlockedAt&&/Waiting for .+ in another nx process/.test(g)&&(e.lockBlockedAt=Date.now(),console.log(`[gateway] ${o}: spawn blocked by a stale nx runner \u2014 breaking the lock + respawning`),S(e,"warning",`nx task lock: ${e.project} blocked by a stale runner \u2014 auto-breaking`,`cluster-service:${e.project}`,{kind:"nx-lock-break",project:e.project}),setTimeout(()=>{t.stopped()||!e.lockBlockedAt||(q(e,"parked on a foreign nx task lock \u2014 recycling",{announce:!1}),X(e,o),setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&U(e)},1500).unref?.())},250).unref?.())}},"sniff"),$=e.wsKey&&e.wsKey!==t.wsKey&&e.root?(0,workspace_log_1.workspaceLogWriter)(e.root):void 0,M=s(m=>{const h=(0,own_log_1.linePrefixer)(e.project);return _=>{if(t.stopped())return;const j=(0,inspector_secret_1.redactInspectorUrls)(_.toString());u(j);const C=h(j);m.write(C),$?.(C)}},"relay");return l.stdout?.on("data",M(process.stdout)),l.stderr?.on("data",M(process.stderr)),l.on("exit",(m,h)=>{if(e.proc!==l||(e.proc=void 0,e.state="down",t.stopped()))return;const _=(0,exit_reason_1.describeExit)(m,h);if(console.log(`[gateway] ${o} exited (${m??"no code"}${h?`, ${h}`:""}${e.inflight?`, ${e.inflight} request(s) in flight`:""})${_?` \u2014 ${_}`:""}`),S(e,m===0?"info":"warning",`service down: ${e.project} (exit ${m})`,`cluster-service:${e.project}`,{kind:"service-down",project:e.project,code:m,...h?{signal:h}:{}}),m===0||e.buildFailed)return;const j=Date.now();if(e.restarts=(e.restarts??[]).filter(K=>j-K<types_1.SERVICE_RESTART_WINDOW_MS),e.restarts.length>=types_1.SERVICE_MAX_RESTARTS){S(e,"error",`service crash-looping: ${e.project} \u2014 ${types_1.SERVICE_MAX_RESTARTS}+ restarts/min, not auto-respawning (the next request retries)`,`cluster-service:${e.project}`,{kind:"service-crash-loop",project:e.project});return}e.restarts.push(j);const C=Math.min(8e3,500*2**(e.restarts.length-1));setTimeout(()=>{t.stopped()||e.proc||e.state!=="down"||(console.log(`[gateway] self-heal: respawning ${e.project} (restart ${e.restarts.length}/${types_1.SERVICE_MAX_RESTARTS})`),S(e,"info",`self-heal: respawning ${e.project}`,`cluster-service:${e.project}`,{kind:"service-respawn",project:e.project,attempt:e.restarts.length}),U(e))},C)}),l},"launchServiceProcess"),U=s(e=>{if(t.stopped())return;const r=`${e.project}:${e.decl.serveTarget??"serve-hmr"}`,n=Math.max(1,e.decl.minPods??1);c.reapForeign!==!1&&X(e,r),e.lockBlockedAt=void 0,console.log(`[gateway] starting ${r} (pods=${n})\u2026`),S(e,"info",`service starting: ${e.project} (${n} pod${n>1?"s":""})`,`cluster-service:${e.project}`,{kind:"service-starting",project:e.project,pods:n}),e.state="starting",e.spawnAt=Date.now(),e.generation=(e.generation??0)+1,e.proc=H(e,e.generation,"active")},"spawnService"),ke=s(async e=>{const r=e.pool.socks[0];if(!r)return;const n=await(0,pod_probe_1.probePodAlive)(r,void 0,t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)]);n&&(e.activeWork=n.activeWork,n.watcher&&(e.watcher=n.watcher))},"refreshActiveWork"),je=s(e=>{try{const r=Number(fs.readFileSync(`${e}.owner`,"utf8").trim());return Number.isInteger(r)&&r>1?r:void 0}catch{return}},"readOwnerSidecar"),Re=s(async e=>{const r=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)];if(!r)return[];const n=(0,node_crypto_1.createHash)("sha256").update(r).digest("hex").slice(0,8),o=[];for(const i of e.pool.socks){const l=await(0,pod_probe_1.probePodAlive)(i,void 0,r);l&&(!l.keyFingerprint||l.keyFingerprint!==n)&&o.push({sock:i,pid:l.pid})}return o},"staleAdoptedSocks"),Ce=s((e,r)=>{const n=r.map(o=>o.sock);for(const{sock:o,pid:i}of r){const l=chooseStalePodKillTarget(i,je(o),process.pid);if(l!==void 0)try{process.kill(l,"SIGTERM")}catch{}try{fs.unlinkSync(o)}catch{}try{fs.unlinkSync(`${o}.owner`)}catch{}}e.pool.socks=e.pool.socks.filter(o=>!n.includes(o)),e.pool.idx=-1,e.pool.scannedAt=0},"evictStaleAdopted"),ie=s((e,r)=>{if(e.state==="up")return!1;e.state="up",e.wedgeRecycles=0;const n=e.pool.socks.length;return S(e,"info",`service up: ${e.project} (${n} pods, ${r})`,`cluster-service:${e.project}`,{kind:"service-up",project:e.project,pods:n,adopted:r==="adopted"}),!0},"markUp"),pe=s(async e=>{if(N(e.pool)>0){const o=await Re(e);if(o.length>0&&(console.error(`[gateway] ${e.project}: ${o.length} adopted pod(s) hold a foreign east-west key \u2014 evicting instead of adopting.`),S(e,"warning",`stale adoption: ${e.project} (foreign east-west key)`,`cluster-service:${e.project}`,{kind:"pod-recycle",project:e.project,reason:"stale-adoption",pods:o.length}),e.proc&&q(e,"stale adoption (foreign east-west key)"),Ce(e,o)),e.pool.socks.length>0)return ie(e,"adopted"),!0}if(t.stopped())return!1;let r=!1;const n=s(()=>e.state!=="starting"&&!e.proc,"notStarted");if(n())if(e.coldWait)await e.coldWait;else{k>=y&&(console.log(`[gateway] cold start queued: ${e.project}${e.wsKey&&e.wsKey!==t.wsKey?` (${e.wsKey})`:""} \u2014 ${k} starting, ${L.length} ahead (LENSMCP_MAX_COLD_STARTS=${y}).`),S(e,"info",`cold start queued: ${e.project}`,`cluster-service:${e.project}`,{kind:"cold-start-queued",project:e.project,inFlight:k,ahead:L.length}));const o=O();if(e.coldWait=o,await o,e.coldWait=void 0,r=!0,t.stopped())return D(),!1;n()&&U(e)}try{return await Ae(e)}finally{r&&D()}},"ensureUp"),Ae=s(async e=>{const r=Date.now()+t.startupTimeoutMs,n=Date.now()+12e3;for(;!t.stopped()&&Date.now()<r;){if(await sleep(Math.min(700,t.startupTimeoutMs/4)),e.buildFailed&&Date.now()>n)return!1;if(N(e.pool)>0){e.state="up",e.buildFailed=!1;const o=e.spawnAt?Date.now()-e.spawnAt:void 0;return console.log(`[gateway] ${e.project} up (${e.pool.socks.length} pods${o?`, cold start ${o}ms`:""})`),S(e,"info",`cold start: ${e.project} (${e.pool.socks.length} pods, ${o??"?"}ms)`,`cluster-service:${e.project}`,{kind:"cold-start",project:e.project,pods:e.pool.socks.length,durationMs:o}),!0}if(e.state==="down")return!1}return!1},"waitForPods"),Te=s(e=>{const r=e.decl.maxPods??e.decl.minPods??1,n=e.pool.socks.length,o=scaleTarget(e);!o||n===0||n>=r||e.inflight<=n||Date.now()-e.lastScaleAt<5e3||(e.lastScaleAt=Date.now(),postScale(o,n+1,c.scaleLookup),console.log(`[gateway] autoscale ${e.project}: ${n} \u2192 ${n+1} pods (inflight ${e.inflight})`),S(e,"info",`autoscale: ${e.project} ${n} \u2192 ${n+1} pods`,`cluster-service:${e.project}`,{kind:"autoscale",project:e.project,from:n,to:n+1,inflight:e.inflight}))},"scaleUp"),q=s((e,r,n)=>{if(!e.proc)return;console.log(`[gateway] ${e.project} ${r} \u2014 scaling to zero.`),n?.announce!==!1&&S(e,"info",`idle kill: ${e.project} (${r})`,`cluster-service:${e.project}`,{kind:"idle-kill",project:e.project,why:r});const o=e.proc;try{o.stdout?.removeAllListeners("data"),o.stderr?.removeAllListeners("data"),o.removeAllListeners("exit")}catch{}(0,discovery_1.stopRoutingTo)(e.pool),R(o,"SIGTERM"),e.proc=void 0,e.state="down",setTimeout(()=>{le(e),e.pool.socks=[],e.pool.scannedAt=0},1200).unref?.()},"killSpawned"),le=s(e=>{try{for(const r of fs.readdirSync(e.pool.dir)){if(!r.endsWith(".sock"))continue;const n=path.join(e.pool.dir,r);try{const o=Number(fs.readFileSync(`${n}.owner`,"utf8").trim());if(Number.isFinite(o)&&o>0)try{process.kill(o,0);continue}catch{}}catch{}try{fs.unlinkSync(n)}catch{}try{fs.unlinkSync(`${n}.owner`)}catch{}}}catch{}},"sweepDeadOwnerSocks");let Y=0;const ue=[],Me=s(()=>Y<B?(Y+=1,Promise.resolve()):new Promise(e=>{ue.push(()=>{Y+=1,e()})}),"acquireRollSlot"),xe=s(()=>{Y-=1,ue.shift()?.()},"releaseRollSlot"),_e=s(async(e,r,n)=>{N(e.pool);for(const o of e.pool.socks){const i=await f(o,void 0,n);if(i&&r(i.podGeneration))return!0}return!1},"anyPodAnswers"),he=new Map,Pe=s((e,r,n)=>{const o=A>0&&(0,handoff_retire_1.requestHandoffRetire)(e.pool.dir,n);if(R(r,"SIGTERM"),!o)return;const i=e.wsKey&&e.wsKey!==t.wsKey&&e.root?(0,workspace_log_1.workspaceLogWriter)(e.root):void 0,l=s(re=>g=>{if(t.stopped())return;const P=(0,inspector_secret_1.redactInspectorUrls)(g.toString()).replace(/^(?=.)/gm,`[${e.project} g${n} retiring] `);re.write(P),i?.(P)},"say"),p=l(process.stdout),u=l(process.stderr);r.stdout?.on("data",p),r.stderr?.on("data",u);let $,M=!1;const m=s(()=>{M=!0,clearTimeout($),r.stdout?.removeListener("data",p),r.stderr?.removeListener("data",u),r.removeListener("close",m)},"releaseLogs");r.once("close",m);let h=!1;const _=s(()=>{h||(h=!0,clearTimeout(j),clearInterval(K),t.stopped()?m():M||($=setTimeout(m,1e3),$.unref?.()),he.delete(r),(0,handoff_retire_1.clearHandoffRetire)(e.pool.dir,n),le(e),e.pool.scannedAt=0)},"end"),j=setTimeout(()=>{console.error(`[gateway] pod handoff: ${e.project} g${n} was still running ${A}ms after it was retired \u2014 killing it.`),T(r,"SIGKILL"),_()},A);j.unref?.();const C=s(()=>{if(!h){if(t.stopped()){_();return}I(r)||(console.log(`[gateway] pod handoff: ${e.project} g${n} finished its in-flight work and exited.`),_())}},"look"),K=setInterval(C,ne);K.unref?.(),r.once("exit",C),he.set(r,()=>{T(r,"SIGTERM"),_()})},"retireReplacedTree"),De=s(()=>{for(const e of[...he.values()])e()},"stopRetiringTrees"),ge=s((e,r)=>{if(e.rolling)return e.rolling;const n=e.proc;if(!n||t.stopped())return Promise.resolve(!1);if(e.launcher==="nx")return console.log(`[gateway] pod handoff skipped: ${e.project} runs through nx run, which starts one instance of a target at a time \u2014 no standby can overlap it (reason=${r}).`),Promise.resolve(!1);Y>=B&&console.log(`[gateway] pod handoff queued: ${e.project} reason=${r} \u2014 ${Y} roll(s) in flight, ${ue.length} ahead (LENSMCP_POD_MAX_CONCURRENT_ROLLS=${B}).`);const o=(async()=>{await Me();try{return await Le(e,n,r)}finally{xe()}})().finally(()=>{e.rolling=void 0});return e.rolling=o,o},"rollPod"),Le=s(async(e,r,n)=>{const o=e.proc;if(o!==r||t.stopped())return!1;const i=e.generation??0,l=i+1,p=Date.now();e.lastStaleRecycleAt=p,console.log(`[gateway] pod handoff start: ${e.project} g${i}\u2192g${l} reason=${n} \u2014 standby spawning, old generation keeps serving.`),S(e,"info",`pod handoff starting: ${e.project} (${n})`,`cluster-service:${e.project}`,{kind:"pod-handoff-start",project:e.project,reason:n,generation:l,fromGeneration:i});const u=H(e,l,"standby"),$=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)],M=p+J;let m;for(;!t.stopped()&&Date.now()<M&&!(u.exitCode!==null||u.signalCode!==null);){N(e.pool);for(const h of e.pool.socks)if((await f(h,void 0,$))?.podGeneration===l){m=h;break}if(m)break;await sleep(ae)}if(!m||t.stopped()){const h=t.stopped()?"gateway stopping":u.exitCode!==null||u.signalCode!==null?`standby exited (${u.exitCode??u.signalCode})`:`standby not ready within ${Math.round(J/1e3)}s`;try{u.stdout?.removeAllListeners("data"),u.stderr?.removeAllListeners("data")}catch{}(0,discovery_1.stopRoutingTo)(e.pool,C=>(0,pod_sock_1.podSockGeneration)(C)===l),R(u,"SIGTERM");const _=!t.stopped()&&e.proc===o&&o.exitCode===null&&o.signalCode===null&&await _e(e,C=>C!==l,$),j=_?`old generation g${i} kept serving`:`and the old generation g${i} is NOT serving (${e.proc!==o||o.exitCode!==null||o.signalCode!==null?"its tree is gone":"none of its pods answers"}) \u2014 the service is down until it recovers`;return console.error(`[gateway] pod handoff failed: ${e.project} g${l}: ${h} \u2014 ${j}.`),e.rollFailure=`${h} \u2014 ${j}`,S(e,_?"warning":"error",`pod handoff failed: ${e.project}`,`cluster-service:${e.project}`,{kind:"pod-handoff-failed",project:e.project,reason:n,generation:l,error:h,durationMs:Date.now()-p,oldServing:_}),setTimeout(()=>le(e),1200).unref?.(),!1}if(e.proc!==o)return(0,discovery_1.stopRoutingTo)(e.pool,h=>(0,pod_sock_1.podSockGeneration)(h)===l),R(u,"SIGTERM"),!1;e.proc=u,e.launcher=ee.get(u),e.generation=l,e.spawnAt=p,e.state="up",e.watcher=void 0,e.staleVerifiedAt=void 0,e.staleKeptAt=void 0,e.buildFailed=!1,e.lastErrors=[];try{o.stdout?.removeAllListeners("data"),o.stderr?.removeAllListeners("data"),o.removeAllListeners("exit")}catch{}return(0,discovery_1.stopRoutingTo)(e.pool,h=>(0,pod_sock_1.podSockGeneration)(h)!==l&&path.join(e.pool.dir,h)!==m),Pe(e,o,i),setTimeout(()=>{le(e),e.pool.scannedAt=0},1200).unref?.(),console.log(`[gateway] pod handoff ready: ${e.project} g${l} answered on ${path.basename(m)} in ${Date.now()-p}ms; retiring g${i}.`),S(e,"info",`pod handoff complete: ${e.project}`,`cluster-service:${e.project}`,{kind:"pod-handoff-complete",project:e.project,reason:n,generation:l,fromGeneration:i,durationMs:Date.now()-p}),!0},"runRoll"),be=s((e,r)=>{if(t.stopped())return{refused:"the gateway is stopping"};if(e.restart&&!e.restart.done)return{restart:e.restart,already:!0};if(!e.proc&&e.state!=="down")return{refused:`${e.project} is served by pods this gateway adopted from an earlier one \u2014 it has no tree of its own to replace. \`lensmcp gateway restart\` starts every service under this gateway.`};const n=e.proc?r.hard||e.launcher==="nx"||e.state!=="up"?"hard":"rolling":"start",o={how:n,reason:r.reason,startedAt:Date.now(),...e.proc&&e.generation!==void 0?{fromGeneration:e.generation}:{}};return e.restart=o,e.lastUsed=o.startedAt,console.log(`[gateway] restart asked: ${e.project}${e.wsKey!==t.wsKey?` (${e.wsKey})`:""} \u2014 ${n} (${r.reason}).`),S(e,"info",`restart asked: ${e.project} (${n})`,`cluster-service:${e.project}`,{kind:"service-restart",phase:"started",project:e.project,how:n,reason:r.reason,...o.fromGeneration!==void 0?{fromGeneration:o.fromGeneration}:{}}),s(async()=>{if(n==="start")return await pe(e)?{ok:!0}:{ok:!1,error:fe(e)};if(n==="rolling")return e.rolling&&await e.rolling,e.proc?(e.rollFailure=void 0,await ge(e,r.reason)?{ok:!0}:{ok:!1,error:e.rollFailure??"the new tree did not take over"}):await pe(e)?{ok:!0}:{ok:!1,error:fe(e)};if(q(e,`restart asked (${r.reason})`,{announce:!1}),await sleep(1500),t.stopped())return{ok:!1,error:"the gateway is stopping"};let i=!1;e.proc||(await O(),i=!0,!e.proc&&e.state!=="starting"&&U(e));try{const l=e.proc,p=e.generation,u=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)],$=Date.now()+t.startupTimeoutMs;for(;!t.stopped()&&Date.now()<$;){if(!l||e.proc!==l||l.exitCode!==null||l.signalCode!==null)return{ok:!1,error:fe(e)};if(await _e(e,M=>M===p,u))return ie(e,"observed"),e.buildFailed=!1,{ok:!0};await sleep(ae)}return{ok:!1,error:t.stopped()?"the gateway is stopping":`no pod of the new tree answered within ${Math.round(t.startupTimeoutMs/1e3)}s`}}finally{i&&D()}},"run")().catch(i=>({ok:!1,error:i.message})).then(i=>{o.done={ok:i.ok,at:Date.now(),...e.generation!==void 0?{generation:e.generation}:{},...i.error?{error:i.error}:{}},i.ok&&(e.lastUsed=o.done.at);const l=Math.round((o.done.at-o.startedAt)/100)/10;i.ok?console.log(`[gateway] restarted: ${e.project} \u2014 g${o.done.generation??"?"} serving after ${l}s (${n}).`):console.error(`[gateway] restart failed: ${e.project} \u2014 ${i.error} (${n}, after ${l}s).`),S(e,i.ok?"info":"error",i.ok?`restarted: ${e.project}`:`restart failed: ${e.project}`,`cluster-service:${e.project}`,{kind:"service-restart",phase:i.ok?"done":"failed",project:e.project,how:n,reason:r.reason,...o.fromGeneration!==void 0?{fromGeneration:o.fromGeneration}:{},...o.done.generation!==void 0?{generation:o.done.generation}:{},durationMs:o.done.at-o.startedAt,...i.error?{error:i.error}:{}})}),{restart:o,already:!1}},"restartService"),fe=s(e=>e.buildFailed?`its build failed${e.lastErrors?.length?`: ${e.lastErrors[e.lastErrors.length-1]}`:""}`:e.proc&&(e.proc.exitCode!==null||e.proc.signalCode!==null)?`its tree exited (${e.proc.exitCode??e.proc.signalCode})`:!e.proc&&e.state==="down"?"its tree exited before a pod answered":`no pod answered within ${Math.round(t.startupTimeoutMs/1e3)}s`,"startFailure"),Ee=s(e=>{const r=Date.now();e.restarts=(e.restarts??[]).filter(o=>r-o<types_1.SERVICE_RESTART_WINDOW_MS);const n=(e.wedgeRecycles??0)+1;if(e.restarts.length>=types_1.SERVICE_MAX_RESTARTS||n>types_1.SERVICE_MAX_RESTARTS){const o=e.restarts.length>=types_1.SERVICE_MAX_RESTARTS?`wedged ${types_1.SERVICE_MAX_RESTARTS}+\xD7/min`:`recycled ${types_1.SERVICE_MAX_RESTARTS}\xD7 in a row and no pod came up`;S(e,"error",`service crash-looping: ${e.project} \u2014 ${o}, not auto-recycling (the next request retries)`,`cluster-service:${e.project}`,{kind:"service-crash-loop",project:e.project,recycles:n-1}),q(e,"wedged (crash-loop cap) \u2014 leaving down for the next request",{announce:!1});return}e.wedgeRecycles=n,e.restarts.push(r),console.log(`[gateway] self-heal: recycling WEDGED ${e.project} (live parent, 0 live pods; attempt ${n}/${types_1.SERVICE_MAX_RESTARTS} with no pod up since the first, ${e.restarts.length} in the last minute; ${e.inflight??0} request(s) in flight \u2014 jobs held by the dead pod keep their locks until they expire)`),S(e,"warning",`self-heal: recycling wedged ${e.project} (0 live pods)`,`cluster-service:${e.project}`,{kind:"service-recycle",project:e.project,attempt:n,inWindow:e.restarts.length}),q(e,"wedged: 0 live pods \u2014 recycling",{announce:!1}),setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&U(e)},1500).unref?.()},"recycleWedged"),Ie={maxAgeMs:types_1.POD_MAX_AGE_MS,graceMs:types_1.POD_STALE_GRACE_MS,settleMs:types_1.POD_RECYCLE_SETTLE_MS,cooldownMs:types_1.POD_STALE_RECYCLE_COOLDOWN_MS};let we;const te=new Map,Se=new Map;let ve=!1,ye=0,de=0;const Oe=s(()=>{const e=t.projectRoots??{},r=new Set(t.services.map(o=>o.project));for(const o of t.routes)o.lens&&r.add(o.lens.project);const n=new Set;for(const o of r){const i=e[o];if(!i)continue;const l=path.relative(t.root,i).split(path.sep).filter(Boolean)[0];l&&!l.startsWith("..")&&n.add(l)}return[...n]},"managedBuckets"),Ne=s((e,r,n,o)=>{const i=t.projectSourceRoots?.[e];if(i?.length)return i;const l=t.projectRoots?.[e];if(!l)return;const p=sourceScopeDirs(t.root,l,n,r,o);return p.length>0?p:void 0},"scopeFor"),Fe=s(()=>{if(ve||Date.now()-ye<types_1.POD_STALE_SCAN_MS)return;ve=!0,ye=Date.now();const e=ye;(async()=>{const r=(0,scope_1.readSourceSetOptions)(t.root),n=r.exclude,o={ignoreTests:r.ignoreTests},i=sourceScanPass(t.root,n,o),l=await i.snapshot(),p=Date.now(),u=sourceChangeWindowMs(p,de,types_1.POD_STALE_SCAN_MS,types_1.POD_RECYCLE_SETTLE_MS),$=we?diffSourceSets(we,l,p,u):null;if($&&(t.sourceSetChangedAt=p,t.sourceSetChange=$,V("info","source file set changed \u2014 stale pods recycle when idle","cluster-gateway",{kind:"source-set-changed",added:$.added,removed:$.removed,truncated:$.truncated})),we=l,!t.projectRoots){de=e;return}let M;try{M=(await fs.promises.readdir(t.root,{withFileTypes:!0})).filter(g=>g.isDirectory()).map(g=>g.name)}catch{return}const m=Oe(),h=new Set(t.services.map(g=>g.project));for(const g of t.routes)g.lens&&h.add(g.lens.project);const _=new Map;for(const g of h){const P=Ne(g,M,m,n);if(!P)continue;const x=P.join("|"),W=_.get(x)??{dirs:P,projects:[]};W.projects.push(g),_.set(x,W)}const j=Date.now(),C=sourceChangeWindowMs(j,de,types_1.POD_STALE_SCAN_MS,types_1.POD_RECYCLE_SETTLE_MS);de=e;const K=s((g,P,x)=>{t.sourceSetChangedFor??=new Map,t.sourceSetChangesFor??=new Map;for(const W of g)t.sourceSetChangedFor.set(W,P),t.sourceSetChangesFor.set(W,x)},"stampScoped"),re=new Set([..._.values()].flatMap(g=>g.projects));for(const g of[...t.sourceSetScoped??[]])re.has(g)||t.sourceSetScoped.delete(g);for(const[g,{dirs:P,projects:x}]of _){const W=await i.snapshot(P),$e=te.get(g);if($e||x.some(b=>Se.has(b))){t.sourceSetScoped??=new Set;for(const b of x)t.sourceSetScoped.add(b)}te.set(g,W);const me=new Map;for(const b of x){const G=Se.get(b);if(G===void 0||G===g)continue;const z=me.get(G)??[];z.push(b),me.set(G,z)}for(const b of x)Se.set(b,g);for(const[b,G]of me){const z=te.get(b),se=z?diffSourceSets(z,W,j,C):$;se&&(K(G,z?j:p,se),V("info",`source file set changed \u2014 ${G.join(", ")}'s watch scope was re-keyed`,"cluster-gateway",{kind:"source-set-changed",scoped:!0,rescoped:!0,projects:G,added:se.added,removed:se.removed,truncated:se.truncated}))}if(!$e)continue;const oe=diffSourceSets($e,W,j,C);oe&&(K(x,j,oe),V("info",`source file set changed in ${x.join(", ")}'s scope`,"cluster-gateway",{kind:"source-set-changed",scoped:!0,projects:x,added:oe.added,removed:oe.removed,truncated:oe.truncated}))}for(const g of[...te.keys()])_.has(g)||te.delete(g)})().catch(()=>{}).finally(()=>{ve=!1})},"maybeScanSourceSet"),Ke=s((e,r)=>{const n=podChangeClock(t,e.project),o=n.change,i=`${n.scope==="project"?"in its scope":"in the workspace (its scope is not known)"}: ${changeEvidence(o)}`,l=o?.windowMs??types_1.POD_STALE_SCAN_MS+types_1.POD_RECYCLE_SETTLE_MS,p=podWatcherVerdict(e.watcher,r,l),u=s(M=>(e.staleKeptAt!==r&&(e.staleKeptAt=r,console.log(`[gateway] stale source set: pod ${e.project} ${M} \u2014 no recycle.`)),!1),"keep");if(p==="heard")return e.staleVerifiedAt=r,S(e,"info",`stale check: ${e.project} watcher confirmed \u2014 no recycle`,`cluster-service:${e.project}`,{kind:"pod-stale-verified",project:e.project,watcher:p}),u(`watcher CONFIRMED alive across the change (last build ${new Date(e.watcher?.lastBuildAt??0).toISOString()})`);if(p==="degraded")return console.log(`[gateway] stale source set: pod ${e.project} watcher DEGRADED (${e.watcher?.degraded}) \u2014 recycling.`),!0;const $=s(M=>(e.staleKeptAt!==r&&(e.staleKeptAt=r,console.log(`[gateway] stale source set: pod ${e.project} ${M}; the change ${i} \u2014 recycling.`)),!0),"recycling");return p==="silent"?types_1.POD_WATCHER_SILENT==="keep"?u("watcher alive but quiet across the change \u2014 keeping it (LENSMCP_POD_WATCHER_SILENT=keep; a live watcher cannot miss an add it will need)"):$("watcher alive but quiet across the change"):types_1.POD_WATCHER_UNVERIFIED==="keep"?u("unverifiable (no watcher testimony \u2014 older devserver bundle?) \u2014 keeping it (LENSMCP_POD_WATCHER_UNVERIFIED=keep)"):$("watcher unverifiable (no testimony yet \u2014 a pod that never rebuilt, or an older devserver bundle)")},"podStaleRecycleAllowed"),We=s((e,r)=>{if(e.lastStaleRecycleAt=Date.now(),types_1.POD_RECYCLE_MODE==="rolling"&&e.proc&&e.launcher!=="nx"){ge(e,r);return}console.log(`[gateway] zombie recycle: ${e.project} (${r}) \u2014 stale fs view; fresh pod incoming`),S(e,"warning",`zombie recycle: ${e.project} (${r})`,`cluster-service:${e.project}`,{kind:"pod-recycle",project:e.project,reason:r});const n=!!e.decl.eager;q(e,`${r} \u2014 recycling stale pod`,{announce:!1}),n&&setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&U(e)},1500).unref?.()},"recyclePod"),qe=s(()=>{const e=setInterval(()=>{Ge().catch(r=>console.error("[gateway] sweep error:",r))},t.sweepMs);return e.unref(),e},"startSweeper"),Ge=s(async()=>{{if(t.stopped())return;for(const e of t.graphStores?.all()??[])e.refreshIfStale();types_1.POD_RECYCLE_ENABLED&&Fe();for(const e of t.registry?t.registry.allServices():t.services){if(t.stopped()||!e.proc){e.wedgedSince=void 0;continue}const r=path.resolve(e.root)===path.resolve(t.root),n=(0,warm_start_1.instanceIdleSec)(e,!r&&!!t.registry?.get(e.wsKey)?.instance);if(n&&e.state==="up"&&(!e.restart||e.restart.done)&&e.inflight===0&&Date.now()-e.lastUsed>n*1e3){q(e,`idle ${n}s`),e.wedgedSince=void 0;continue}const o=e.state==="starting"&&e.spawnAt!==void 0&&Date.now()-e.spawnAt<t.startupTimeoutMs,i=N(e.pool)>0;if(i||o){if(e.wedgedSince=void 0,i&&ie(e,"observed"),i&&!o&&types_1.POD_RECYCLE_ENABLED&&r){const l=podChangeClock(t,e.project).at;await ke(e);const p=(e.staleVerifiedAt??0)>=l&&l>0,u=podRecycleReason(e,Date.now(),p?0:l,Ie);if(u==="stale-source-set"&&!Ke(e,l))continue;u&&We(e,u)}continue}e.wedgedSince??=Date.now(),Date.now()-e.wedgedSince>types_1.WEDGE_RECYCLE_GRACE_MS&&(e.wedgedSince=void 0,Ee(e))}}},"sweepOnce");return{scanNow:N,spawnService:U,ensureUp:pe,scaleUp:Te,killSpawned:q,markUp:ie,startSweeper:qe,rollPod:ge,restartService:be,recycleWedged:Ee,stopRetiringTrees:De,stopBuildHost:s(e=>e===void 0?E.stop():E.release(e),"stopBuildHost")}}v(createServiceLayer,"createServiceLayer"),s(createServiceLayer,"createServiceLayer");function chooseStalePodKillTarget(t,a,c){if(!(t===void 0||!Number.isInteger(t)||t<=1)&&!(a!==void 0&&a!==t)&&t!==c)return t}v(chooseStalePodKillTarget,"chooseStalePodKillTarget"),s(chooseStalePodKillTarget,"chooseStalePodKillTarget");
+"use strict";var it=Object.defineProperty;var m=(t,c)=>it(t,"name",{value:c,configurable:!0});var at=Object.defineProperty,a=m((t,c)=>at(t,"name",{value:c,configurable:!0}),"a");Object.defineProperty(exports,"__esModule",{value:!0}),exports.scaleTarget=scaleTarget,exports.isLoopbackHostname=isLoopbackHostname,exports.isOwnRunnerCommand=isOwnRunnerCommand,exports.isRunnerOf=isRunnerOf,exports.isTestSourcePath=isTestSourcePath,exports.sourceScanPass=sourceScanPass,exports.sourceSetSnapshot=sourceSetSnapshot,exports.sourceSetSignature=sourceSetSignature,exports.diffSourceSets=diffSourceSets,exports.sourceChangeWindowMs=sourceChangeWindowMs,exports.sourceScopeDirs=sourceScopeDirs,exports.podRecycleReason=podRecycleReason,exports.lensFrontendStale=lensFrontendStale,exports.lensWatcherVerdict=lensWatcherVerdict,exports.podWatcherVerdict=podWatcherVerdict,exports.lensChildWedged=lensChildWedged,exports.isPortConflictExit=isPortConflictExit,exports.podChangeClock=podChangeClock,exports.createServiceLayer=createServiceLayer,exports.chooseStalePodKillTarget=chooseStalePodKillTarget;const tslib_1=require("tslib"),node_child_process_1=require("node:child_process"),node_crypto_1=require("node:crypto"),fs=tslib_1.__importStar(require("node:fs")),http=tslib_1.__importStar(require("node:http")),net=tslib_1.__importStar(require("node:net")),path=tslib_1.__importStar(require("node:path")),node_util_1=require("node:util"),discovery_1=require("./discovery"),pod_sock_1=require("../../../pod-sock"),scope_1=require("./scope"),pod_env_1=require("./pod-env"),pod_probe_1=require("./pod-probe"),service_keys_1=require("./service-keys"),workspace_log_1=require("./workspace-log"),own_log_1=require("./own-log"),inspector_secret_1=require("../../../inspector-secret"),pod_output_1=require("./pod-output"),serve_launch_1=require("./serve-launch"),shared_build_1=require("./shared-build"),warm_start_1=require("./warm-start"),protocol_1=require("../../../shared-build/protocol"),types_1=require("./types"),exit_reason_1=require("../../../exit-reason"),handoff_retire_1=require("../../../handoff-retire"),process_name_1=require("../../../process-name"),orphan_reaper_1=require("./orphan-reaper");function eastWestEnv(t,c){const l={},p=t.serviceKeys[(0,service_keys_1.serviceKeyId)(c.wsKey,c.project)];if(p&&(l.LENSMCP_SERVICE_KEY=p),t.https){const f=require("../../../basic-ssl").caCertPath();fs.existsSync(f)&&(l.NODE_EXTRA_CA_CERTS=f)}return l}m(eastWestEnv,"eastWestEnv"),a(eastWestEnv,"eastWestEnv");const sleep=a(t=>new Promise(c=>setTimeout(c,t)),"sleep"),SRC_EXCLUDE=new Set(["node_modules","dist","tmp","coverage","build","out","target"]),SRC_EXT=new Set([".ts",".tsx",".js",".jsx",".mjs",".cjs",".css",".scss",".sass",".less",".json",".html",".htm",".vue",".svelte",".mdx"]),SCAN_FILE_CAP=1e5;function fnv1a(t){let c=2166136261;for(let l=0;l<t.length;l+=1)c^=t.charCodeAt(l),c=Math.imul(c,16777619);return c>>>0}m(fnv1a,"fnv1a"),a(fnv1a,"fnv1a");const execFileP=(0,node_util_1.promisify)(node_child_process_1.execFile);async function gitIgnoredPaths(t){try{const{stdout:c}=await execFileP("git",["ls-files","--others","--directory","--ignored","--exclude-standard","-z"],{cwd:t,maxBuffer:33554432,timeout:1e4,windowsHide:!0});return new Set(c.split("\0").filter(Boolean))}catch{return new Set}}m(gitIgnoredPaths,"gitIgnoredPaths"),a(gitIgnoredPaths,"gitIgnoredPaths");function scaleTarget(t,c=fs.existsSync){if(t.decl.adminUrl)return{url:t.decl.adminUrl};const l=path.join(t.pool.dir,"parent.sock");return c(l)?{socketPath:l}:t.decl.port?{url:`http://localhost:${t.decl.port}`}:void 0}m(scaleTarget,"scaleTarget"),a(scaleTarget,"scaleTarget");function isLoopbackHostname(t){const c=t.toLowerCase().replace(/^\[(.*)\]$/,"$1").replace(/\.$/,"");return c==="localhost"||c.endsWith(".localhost")?!0:net.isIPv4(c)?c.split(".")[0]==="127":net.isIPv6(c)&&new URL(`http://[${c}]/`).hostname==="[::1]"}m(isLoopbackHostname,"isLoopbackHostname"),a(isLoopbackHostname,"isLoopbackHostname");function postScale(t,c,l){let p;if("socketPath"in t)p=http.request({socketPath:t.socketPath,path:"/webpack/scale",method:"POST",headers:{"content-type":"application/json"}},f=>f.resume());else{const f=t.url.startsWith("https"),T=f?require("node:https"):http,_={method:"POST"};f&&isLoopbackHostname(new URL(t.url).hostname)&&(_.rejectUnauthorized=!1),l&&(_.lookup=l),p=T.request(`${t.url}/webpack/scale`,_,A=>A.resume())}return p.on("error",()=>{}),p.end(JSON.stringify({pods:c})),p}m(postScale,"postScale"),a(postScale,"postScale");function isOwnRunnerCommand(t,c){return(0,serve_launch_1.isServeRunnerCommand)(t)?(0,serve_launch_1.isServeRunnerCommandOf)(t,c):t.includes("/node_modules/nx/")?t.includes(`${c.replace(/\/+$/,"")}/node_modules/nx/`):!0}m(isOwnRunnerCommand,"isOwnRunnerCommand"),a(isOwnRunnerCommand,"isOwnRunnerCommand");function isRunnerOf(t,c,l){const p=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return!new RegExp(`(^|\\s)run ${p}(:|\\s|$)`).test(t)||!(0,serve_launch_1.isServeRunnerCommand)(t)&&!/\bnx(\.js)?\b/.test(t)?!1:isOwnRunnerCommand(t,l)}m(isRunnerOf,"isRunnerOf"),a(isRunnerOf,"isRunnerOf");const TEST_FILE_RE=/\.(spec|test|e2e|e2e-spec|stories)\.[cm]?[jt]sx?$/,TEST_DIR_SEGMENTS=new Set(["__tests__","__mocks__","__snapshots__"]);function isTestSourcePath(t){const c=t.split("/"),l=c[c.length-1]??"";if(TEST_FILE_RE.test(l))return!0;for(let p=0;p<c.length-1;p+=1)if(TEST_DIR_SEGMENTS.has(c[p]))return!0;return!1}m(isTestSourcePath,"isTestSourcePath"),a(isTestSourcePath,"isTestSourcePath");function sourceScanPass(t,c,l){const p=l?.ignoreTests??!0;let f;const T=new Map,_=`tree:${path.resolve(t)}`,A=a(y=>path.relative(t,y).split(path.sep).join("/"),"rel"),C=a(async(y,$)=>{const E=await(f??=gitIgnoredPaths(t)),b=[],F=new Set;let D=!1;const H=a(async(te,S)=>{if(b.length>=SCAN_FILE_CAP){D=!0;return}let O;try{O=await fs.promises.readdir(te,{withFileTypes:!0})}catch{return}S&&F.add(A(te));for(const z of O){if(b.length>=SCAN_FILE_CAP){D=!0;return}const W=z.name,re=path.join(te,W);if(z.isDirectory()){if(!S||W.startsWith(".")||SRC_EXCLUDE.has(W)||c?.has(W)||W.endsWith("-devserver")||E.has(`${A(re)}/`))continue;await H(re,!0)}else if(z.isFile()){const oe=W.lastIndexOf(".");if(oe<=0||!SRC_EXT.has(W.slice(oe)))continue;const J=A(re);if(E.has(J)||p&&isTestSourcePath(J))continue;b.push(J)}}},"walk");return await H(y,$),{paths:b,entered:F,truncated:D}},"walkDir"),L=a((y,$)=>{let E=T.get(y);return E||(E=$(),T.set(y,E)),E},"memo"),ie=a(async()=>{const y=T.get(_);if(!y)return;const $=await y;return $.truncated?void 0:$},"wideWalk"),B=new Set,le=a(()=>L(`files:${path.resolve(t)}`,async()=>{const y=await ie();return y?{paths:y.paths.filter($=>!$.includes("/")),entered:B,truncated:!1}:C(t,!1)}),"rootFiles"),de=a(y=>L(`tree:${path.resolve(y)}`,async()=>{const $=await ie(),E=A(y);if($?.entered.has(E)){const b=`${E}/`;return{paths:E===""?$.paths:$.paths.filter(F=>F.startsWith(b)),entered:B,truncated:!1}}return C(y,!0)}),"subtree"),X=a(y=>{const $=new Set;let E=0,b=y.some(D=>D.truncated);const F=a(D=>$.has(D)?!0:$.size>=SCAN_FILE_CAP?(b=!0,!1):($.add(D),E^=fnv1a(D),!0),"add");for(const D of y)if(!D.paths.every(F))break;return{signature:`${$.size}:${(E>>>0).toString(36)}`,paths:$,truncated:b}},"compose");return{async snapshot(y){if(!y||y.length===0)return X([await L(_,()=>C(t,!0))]);const $=[await le()];for(const E of y)$.push(await de(E));return X($)}}}m(sourceScanPass,"sourceScanPass"),a(sourceScanPass,"sourceScanPass");async function sourceSetSnapshot(t,c,l,p){return sourceScanPass(t,l,p).snapshot(c)}m(sourceSetSnapshot,"sourceSetSnapshot"),a(sourceSetSnapshot,"sourceSetSnapshot");async function sourceSetSignature(t,c,l,p){return(await sourceSetSnapshot(t,c,l,p)).signature}m(sourceSetSignature,"sourceSetSignature"),a(sourceSetSignature,"sourceSetSignature");const SOURCE_DIFF_PATH_CAP=128;function diffSourceSets(t,c,l,p){if(t.paths.size===c.paths.size){let A=!0;for(const C of t.paths)if(!c.paths.has(C)){A=!1;break}if(A)return null}const f=[],T=[];let _=t.truncated||c.truncated;for(const A of c.paths)t.paths.has(A)||(f.length+T.length<SOURCE_DIFF_PATH_CAP?f.push(A):_=!0);for(const A of t.paths)c.paths.has(A)||(f.length+T.length<SOURCE_DIFF_PATH_CAP?T.push(A):_=!0);return{changedAt:l,added:f,removed:T,truncated:_,...p!==void 0?{windowMs:p}:{}}}m(diffSourceSets,"diffSourceSets"),a(diffSourceSets,"diffSourceSets");function sourceChangeWindowMs(t,c,l,p){const f=c&&c>0?t-c:l;return Math.max(f,l)+p}m(sourceChangeWindowMs,"sourceChangeWindowMs"),a(sourceChangeWindowMs,"sourceChangeWindowMs");function sourceScopeDirs(t,c,l,p,f){const T=a(C=>{const L=path.relative(t,C).split(path.sep).filter(Boolean);return L.length===0||L[0].startsWith("..")?"":L[0]},"bucketOf")(c);if(!T)return[];const _=new Set(l.filter(Boolean)),A=new Set([T]);for(const C of p)C.startsWith(".")||SRC_EXCLUDE.has(C)||f?.has(C)||C.endsWith("-devserver")||_.has(C)||A.add(C);return[...A].sort().map(C=>path.join(t,C))}m(sourceScopeDirs,"sourceScopeDirs"),a(sourceScopeDirs,"sourceScopeDirs");function podRecycleReason(t,c,l,p){return!t.proc||t.state!=="up"||t.spawnAt==null||t.inflight>0||(t.activeWork??0)>0||c-t.lastUsed<p.settleMs||t.lastStaleRecycleAt!=null&&c-t.lastStaleRecycleAt<p.cooldownMs?null:l>0&&l-t.spawnAt>p.graceMs?"stale-source-set":p.maxAgeMs>0&&c-t.spawnAt>p.maxAgeMs?"max-age":null}m(podRecycleReason,"podRecycleReason"),a(podRecycleReason,"podRecycleReason");function lensFrontendStale(t,c,l,p,f){return!(p-c<f.cooldownMs||l<=0||l-t<=f.graceMs||p-l<f.settleMs)}m(lensFrontendStale,"lensFrontendStale"),a(lensFrontendStale,"lensFrontendStale");function lensWatcherVerdict(t,c,l,p=[]){if(!t)return"unknown";if("schemaVersion"in t&&t.schemaVersion>=2&&p.length>0){const f=c-l,T=new Set(t.recentEvents.filter(_=>_.at>=f).map(_=>path.resolve(_.path)));return p.every(_=>T.has(path.resolve(_)))?"heard":"missed"}return t.lastFsSetEventAt>=c-l?"heard":"missed"}m(lensWatcherVerdict,"lensWatcherVerdict"),a(lensWatcherVerdict,"lensWatcherVerdict");function podWatcherVerdict(t,c,l){if(!t)return"unknown";if(t.degraded)return"degraded";const p=c-l;return t.lastWatchRunAt>=p||t.lastBuildAt>=p?"heard":"silent"}m(podWatcherVerdict,"podWatcherVerdict"),a(podWatcherVerdict,"podWatcherVerdict");function lensChildWedged(t,c,l,p){const f=t.hasListened?p.graceMs:Math.max(p.graceMs,p.neverListenedGraceMs);return l-t.spawnedAt<f||l-c<p.cooldownMs||t.probeFails<Math.max(2,p.failureThreshold)?null:"wedged-no-listener"}m(lensChildWedged,"lensChildWedged"),a(lensChildWedged,"lensChildWedged");function isPortConflictExit(t,c){return t.signal||t.code===null||t.code===0||t.ranMs>c.fastFailMs?!1:t.portHeld}m(isPortConflictExit,"isPortConflictExit"),a(isPortConflictExit,"isPortConflictExit");function podChangeClock(t,c){const l=t.sourceSetChangedFor?.get(c);if(l!==void 0){const p=t.sourceSetChangesFor?.get(c);return{at:l,...p?{change:p}:{},scope:"project"}}return t.sourceSetScoped?.has(c)?{at:0,scope:"none"}:{at:t.sourceSetChangedAt??0,...t.sourceSetChange?{change:t.sourceSetChange}:{},scope:"workspace"}}m(podChangeClock,"podChangeClock"),a(podChangeClock,"podChangeClock");const changeEvidence=a(t=>{if(!t)return"no path evidence";const c=[...t.added.map(l=>`+${l}`),...t.removed.map(l=>`-${l}`)];return c.length?`${c.slice(0,3).join(", ")}${c.length>3?` (+${c.length-3} more)`:""}`:"no path evidence"},"changeEvidence");function createServiceLayer(t,c,l={}){const p=l.spawnProcess??node_child_process_1.spawn,f=l.probeAlive??pod_probe_1.probePodAlive,T=l.killTree??((e,r="SIGTERM")=>{try{process.kill(-e.pid,r)}catch{try{e.kill(r)}catch{}}}),_=l.sharedBuild??(0,shared_build_1.createSharedBuildRelay)(t),A=a((e,r="SIGTERM")=>{_.detach(e),T(e,r)},"killTree"),C=l.handoffDrainMs??(0,handoff_retire_1.handoffDrainBoundMs)(),L=l.treeAlive??(e=>{if(!e.pid||!(e instanceof node_child_process_1.ChildProcess))return!1;try{return process.kill(-e.pid,0),!0}catch(r){return r.code==="EPERM"}}),ie=l.retirePollMs??1e3,B=l.rollReadyMs??types_1.POD_ROLL_READY_MS,le=l.rollPollMs??750,de=l.maxConcurrentRolls??types_1.POD_MAX_CONCURRENT_ROLLS,X=de>0?de:Number.POSITIVE_INFINITY,y=l.maxConcurrentColdStarts??types_1.MAX_COLD_STARTS,$=y>0?y:Number.POSITIVE_INFINITY;let E=0;const b=[],F=a(()=>E<$?(E+=1,Promise.resolve()):new Promise(e=>{b.push(()=>{E+=1,e()})}),"acquireColdSlot"),D=a(()=>{E-=1,b.shift()?.()},"releaseColdSlot"),{emit:H}=c,te=a(e=>e.wsKey===t.wsKey?void 0:path.join(e.root,".lensmcp","events.jsonl"),"serviceBus"),S=a((e,r,o,s,i)=>H(r,o,s,{wsKey:e.wsKey,...i},void 0,te(e)),"emitSvc"),O=a(e=>(e.scannedAt=0,(0,discovery_1.pickSock)(e,t.scanTtlMs)?e.socks.length:0),"scanNow"),z=a((e,r)=>{let o;try{o=(0,node_child_process_1.execFileSync)("ps",["ax","-ww","-o","pid=,pgid=,command="],{encoding:"utf8",timeout:5e3})}catch{return 0}const s=e.proc?.pid;let i=0;for(const n of o.split(`
+`)){const d=n.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/);if(!d)continue;const u=Number(d[1]),w=Number(d[2]);if(isRunnerOf(d[3],r,e.root)&&!(u===process.pid||u===s)&&!(s!==void 0&&w===s)){try{process.kill(-w,"SIGKILL")}catch{try{process.kill(u,"SIGKILL")}catch{}}i+=1}}return i>0&&(console.log(`[gateway] ${r}: reaped ${i} stale runner tree(s) \u2014 a foreign generation of this service (holding nx's task lock, or its pool)`),S(e,"warning",`stale-runner reap: ${e.project} (${i} foreign tree(s))`,`cluster-service:${e.project}`,{kind:"stale-runner-reap",project:e.project,reaped:i})),i},"reapForeignRunners"),W=a((e,r)=>({...(0,pod_env_1.podBaseEnv)(process.env,t.root,e.root),...e.instanceEnv?{...e.instanceEnv,LENSMCP_SOCKET_ONLY:"1"}:{},...e.frozen?{LENSMCP_FREEZE:"1",LENSMCP_TYPECHECK:"off"}:{},NX_DAEMON:"false",NX_ISOLATE_PLUGINS:"false",...e.wsKey?{LENSMCP_WS_KEY:e.wsKey}:{},...e.sockNs?{LENSMCP_SOCK_NS:e.sockNs}:{},LENSMCP_EVENT_FILE:e.wsKey===t.wsKey?t.eventFile:path.join(e.root,".lensmcp","events.jsonl"),LENSMCP_GATEWAY_PID:String(process.pid),LENSMCP_POD_GENERATION:String(r),...eastWestEnv(t,e)}),"serviceEnv"),re=a((e,r,o,s)=>{const i=W(e,r),n=e.wsKey||path.basename(e.root),d=(0,serve_launch_1.planServeLaunch)({stores:t.graphStores,root:e.root,project:e.project,target:e.decl.serveTarget??"serve-hmr",childCount:s,baseEnv:i});if(d.kind==="runner"){const v=_.offeredTo(e),g=p((0,process_name_1.namedExecPath)((0,process_name_1.processName)(n,e.project,serve_launch_1.SERVE_RUNNER_ROLE)),[...d.script.execArgv,d.script.path,...(0,serve_launch_1.serveRunnerArgs)(o,s,e.root)],{cwd:e.root,detached:!0,env:v?{...d.env,[protocol_1.SHARED_BUILD_ENV]:"1"}:d.env,stdio:["ignore","pipe","pipe","ipc"]});return v&&_.attach(e,r,g),g.send?.(d.message,R=>{R&&console.error(`[gateway] ${o}: could not hand the task to its runner (${R.message})`)}),{proc:g,launcher:"runner"}}console.log(`[gateway] ${o}: starting through nx run \u2014 ${d.reason}`);const u=path.join(e.root,"node_modules","nx","dist","bin","nx.js"),w=(0,process_name_1.namedExecPath)((0,process_name_1.processName)(n,e.project,"nx")),[k,j]=fs.existsSync(u)?[w,[u,"run",o,`--childCount=${s}`]]:["yarn",["nx","run",o,`--childCount=${s}`]];return{proc:p(k,j,{cwd:e.root,detached:!0,env:i,stdio:["ignore","pipe","pipe"]}),launcher:"nx"}},"startServiceTree"),oe=new WeakMap,J=a((e,r,o)=>{const s=`${e.project}:${e.decl.serveTarget??"serve-hmr"}`,i=Math.max(1,e.decl.minPods??1),{proc:n,launcher:d}=re(e,r,s,i);oe.set(n,d),o==="active"&&(e.launcher=d),o==="active"&&(e.lastErrors=[],e.buildFailed=!1,e.faults=new pod_output_1.PodFaults);const u=a(j=>{const v=j.toString();for(const g of v.split(`
+`)){const R=(0,pod_output_1.classifyPodLine)(g),{clean:M,broken:G,healthy:ne}=R,h=M;if(G){e.lastErrors.push(M),e.lastErrors.length>12&&e.lastErrors.shift(),e.buildFailed=!0;const x=Date.now();(M!==e.lastErrorEmitSig||x-(e.lastErrorEmitAt??0)>=types_1.SERVICE_ERROR_THROTTLE_MS)&&(e.lastErrorEmitSig=M,e.lastErrorEmitAt=x,S(e,"error",`service error: ${e.project}`,`cluster-service:${e.project}`,{kind:"service-error",project:e.project,line:M,fault:R.fault}))}ne&&(e.buildFailed=!1);const P=(e.faults??=new pod_output_1.PodFaults).observe(R);if(P){e.lastErrorEmitSig=void 0;const x=P.open.length===0;S(e,"info",`${x?"service recovered":`${P.closed.join(" + ")} recovered`}: ${e.project}`,`cluster-service:${e.project}`,{kind:x?"service-recovered":"service-fault-closed",project:e.project,line:M,closed:P.closed,open:P.open})}!e.lockBlockedAt&&/Waiting for .+ in another nx process/.test(h)&&(e.lockBlockedAt=Date.now(),console.log(`[gateway] ${s}: spawn blocked by a stale nx runner \u2014 breaking the lock + respawning`),S(e,"warning",`nx task lock: ${e.project} blocked by a stale runner \u2014 auto-breaking`,`cluster-service:${e.project}`,{kind:"nx-lock-break",project:e.project}),setTimeout(()=>{t.stopped()||!e.lockBlockedAt||(V(e,"parked on a foreign nx task lock \u2014 recycling",{announce:!1}),z(e,s),setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&Y(e)},1500).unref?.())},250).unref?.())}},"sniff"),w=e.wsKey&&e.wsKey!==t.wsKey&&e.root?(0,workspace_log_1.workspaceLogWriter)(e.root):void 0,k=a(j=>{const v=(0,own_log_1.linePrefixer)(e.project);return g=>{if(t.stopped())return;const R=(0,inspector_secret_1.redactInspectorUrls)(g.toString());u(R);const M=v(R);j.write(M),w?.(M)}},"relay");return n.stdout?.on("data",k(process.stdout)),n.stderr?.on("data",k(process.stderr)),n.on("exit",(j,v)=>{if(e.proc!==n||(e.proc=void 0,e.state="down",t.stopped()))return;const g=(0,exit_reason_1.describeExit)(j,v);if(console.log(`[gateway] ${s} exited (${j??"no code"}${v?`, ${v}`:""}${e.inflight?`, ${e.inflight} request(s) in flight`:""})${g?` \u2014 ${g}`:""}`),S(e,j===0?"info":"warning",`service down: ${e.project} (exit ${j})`,`cluster-service:${e.project}`,{kind:"service-down",project:e.project,code:j,...v?{signal:v}:{}}),j===0||e.buildFailed)return;const R=Date.now();if(e.restarts=(e.restarts??[]).filter(G=>R-G<types_1.SERVICE_RESTART_WINDOW_MS),e.restarts.length>=types_1.SERVICE_MAX_RESTARTS){S(e,"error",`service crash-looping: ${e.project} \u2014 ${types_1.SERVICE_MAX_RESTARTS}+ restarts/min, not auto-respawning (the next request retries)`,`cluster-service:${e.project}`,{kind:"service-crash-loop",project:e.project});return}e.restarts.push(R);const M=Math.min(8e3,500*2**(e.restarts.length-1));setTimeout(()=>{t.stopped()||e.proc||e.state!=="down"||(console.log(`[gateway] self-heal: respawning ${e.project} (restart ${e.restarts.length}/${types_1.SERVICE_MAX_RESTARTS})`),S(e,"info",`self-heal: respawning ${e.project}`,`cluster-service:${e.project}`,{kind:"service-respawn",project:e.project,attempt:e.restarts.length}),Y(e))},M)}),n},"launchServiceProcess"),Y=a(e=>{if(t.stopped()||e.manualStopped)return;const r=`${e.project}:${e.decl.serveTarget??"serve-hmr"}`,o=Math.max(1,e.decl.minPods??1);l.reapForeign!==!1&&z(e,r),e.lockBlockedAt=void 0,console.log(`[gateway] starting ${r} (pods=${o})\u2026`),S(e,"info",`service starting: ${e.project} (${o} pod${o>1?"s":""})`,`cluster-service:${e.project}`,{kind:"service-starting",project:e.project,pods:o}),e.state="starting",e.spawnAt=Date.now(),e.generation=(e.generation??0)+1,e.proc=J(e,e.generation,"active")},"spawnService"),be=a(async e=>{const r=e.pool.socks[0];if(!r)return;const o=await(0,pod_probe_1.probePodAlive)(r,void 0,t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)]);o&&(e.activeWork=o.activeWork,o.watcher&&(e.watcher=o.watcher))},"refreshActiveWork"),Le=a(e=>{try{const r=Number(fs.readFileSync(`${e}.owner`,"utf8").trim());return Number.isInteger(r)&&r>1?r:void 0}catch{return}},"readOwnerSidecar"),Ie=a(async e=>{const r=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)];if(!r)return[];const o=(0,node_crypto_1.createHash)("sha256").update(r).digest("hex").slice(0,8),s=[];for(const i of e.pool.socks){const n=await(0,pod_probe_1.probePodAlive)(i,void 0,r);n&&(!n.keyFingerprint||n.keyFingerprint!==o)&&s.push({sock:i,pid:n.pid})}return s},"staleAdoptedSocks"),Oe=a((e,r)=>{const o=r.map(s=>s.sock);for(const{sock:s,pid:i}of r){const n=chooseStalePodKillTarget(i,Le(s),process.pid);if(n!==void 0)try{process.kill(n,"SIGTERM")}catch{}try{fs.unlinkSync(s)}catch{}try{fs.unlinkSync(`${s}.owner`)}catch{}}e.pool.socks=e.pool.socks.filter(s=>!o.includes(s)),e.pool.idx=-1,e.pool.scannedAt=0},"evictStaleAdopted"),pe=a((e,r)=>{if(e.state==="up")return!1;e.state="up",e.wedgeRecycles=0;const o=e.pool.socks.length;return S(e,"info",`service up: ${e.project} (${o} pods, ${r})`,`cluster-service:${e.project}`,{kind:"service-up",project:e.project,pods:o,adopted:r==="adopted"}),!0},"markUp"),we=a(async e=>{if(e.manualStopped)return!1;if(O(e.pool)>0){const s=await Ie(e);if(e.manualStopped)return!1;if(s.length>0&&(console.error(`[gateway] ${e.project}: ${s.length} adopted pod(s) hold a foreign east-west key \u2014 evicting instead of adopting.`),S(e,"warning",`stale adoption: ${e.project} (foreign east-west key)`,`cluster-service:${e.project}`,{kind:"pod-recycle",project:e.project,reason:"stale-adoption",pods:s.length}),e.proc&&V(e,"stale adoption (foreign east-west key)"),Oe(e,s)),e.pool.socks.length>0)return pe(e,"adopted"),!0}if(t.stopped())return!1;let r=!1;const o=a(()=>e.state!=="starting"&&!e.proc,"notStarted");if(o())if(e.coldWait)await e.coldWait;else{E>=$&&(console.log(`[gateway] cold start queued: ${e.project}${e.wsKey&&e.wsKey!==t.wsKey?` (${e.wsKey})`:""} \u2014 ${E} starting, ${b.length} ahead (LENSMCP_MAX_COLD_STARTS=${$}).`),S(e,"info",`cold start queued: ${e.project}`,`cluster-service:${e.project}`,{kind:"cold-start-queued",project:e.project,inFlight:E,ahead:b.length}));const s=F();if(e.coldWait=s,await s,e.coldWait=void 0,r=!0,t.stopped()||e.manualStopped)return D(),!1;o()&&Y(e)}try{return await Ne(e)}finally{r&&D()}},"ensureUp"),Ne=a(async e=>{const r=Date.now()+t.startupTimeoutMs,o=Date.now()+12e3;for(;!t.stopped()&&!e.manualStopped&&Date.now()<r;){if(await sleep(Math.min(700,t.startupTimeoutMs/4)),e.manualStopped||e.buildFailed&&Date.now()>o)return!1;if(O(e.pool)>0){e.state="up",e.buildFailed=!1;const s=e.spawnAt?Date.now()-e.spawnAt:void 0;return console.log(`[gateway] ${e.project} up (${e.pool.socks.length} pods${s?`, cold start ${s}ms`:""})`),S(e,"info",`cold start: ${e.project} (${e.pool.socks.length} pods, ${s??"?"}ms)`,`cluster-service:${e.project}`,{kind:"cold-start",project:e.project,pods:e.pool.socks.length,durationMs:s}),!0}if(e.state==="down")return!1}return!1},"waitForPods"),Ke=a(e=>{const r=e.decl.maxPods??e.decl.minPods??1,o=e.pool.socks.length,s=scaleTarget(e);!s||o===0||o>=r||e.inflight<=o||Date.now()-e.lastScaleAt<5e3||(e.lastScaleAt=Date.now(),postScale(s,o+1,l.scaleLookup),console.log(`[gateway] autoscale ${e.project}: ${o} \u2192 ${o+1} pods (inflight ${e.inflight})`),S(e,"info",`autoscale: ${e.project} ${o} \u2192 ${o+1} pods`,`cluster-service:${e.project}`,{kind:"autoscale",project:e.project,from:o,to:o+1,inflight:e.inflight}))},"scaleUp"),ue=l.killGraceMs??types_1.TREE_KILL_GRACE_MS,N=new Map,Ce=a(e=>{const r=N.get(e);if(!r||(N.delete(e),clearTimeout(r.timer),!L(e)))return;const{svc:o,why:s}=r;console.log(`[gateway] ${o.project}: its tree (pid ${e.pid}) was still running ${Math.round(ue/1e3)}s after it was told to stop (${s}) \u2014 killing it (SIGKILL).`),S(o,"warning",`tree force-killed: ${o.project} (it ignored SIGTERM)`,`cluster-service:${o.project}`,{kind:"tree-force-killed",project:o.project,why:s,pid:e.pid,graceMs:ue}),T(e,"SIGKILL")},"settleDyingTree"),Se=a((e,r,o)=>{if(A(r,"SIGTERM"),ue<=0||N.has(r))return;const s=setTimeout(()=>Ce(r),ue);s.unref?.(),N.set(r,{svc:e,why:o,timer:s})},"stopTree"),Fe=a(async(e=3e3)=>{const r=Date.now()+e;for(;N.size>0;){for(const[o,s]of[...N])L(o)||(clearTimeout(s.timer),N.delete(o));if(N.size===0||Date.now()>=r)break;await sleep(100)}for(const o of[...N.keys()])Ce(o)},"finishDyingTrees"),V=a((e,r,o)=>{if(!e.proc)return;console.log(`[gateway] ${e.project} ${r} \u2014 scaling to zero.`),o?.announce!==!1&&S(e,"info",`idle kill: ${e.project} (${r})`,`cluster-service:${e.project}`,{kind:"idle-kill",project:e.project,why:r});const s=e.proc;try{s.stdout?.removeAllListeners("data"),s.stderr?.removeAllListeners("data"),s.removeAllListeners("exit")}catch{}(0,discovery_1.stopRoutingTo)(e.pool),Se(e,s,r),e.proc=void 0,e.state="down",setTimeout(()=>{ge(e),e.pool.socks=[],e.pool.scannedAt=0},1200).unref?.()},"killSpawned"),ge=a(e=>{try{for(const r of fs.readdirSync(e.pool.dir)){if(!r.endsWith(".sock"))continue;const o=path.join(e.pool.dir,r);try{const s=Number(fs.readFileSync(`${o}.owner`,"utf8").trim());if(Number.isFinite(s)&&s>0)try{process.kill(s,0);continue}catch{}}catch{}try{fs.unlinkSync(o)}catch{}try{fs.unlinkSync(`${o}.owner`)}catch{}}}catch{}},"sweepDeadOwnerSocks");let Q=0;const ve=[],We=a(()=>Q<X?(Q+=1,Promise.resolve()):new Promise(e=>{ve.push(()=>{Q+=1,e()})}),"acquireRollSlot"),Ge=a(()=>{Q-=1,ve.shift()?.()},"releaseRollSlot"),ye=a(async(e,r,o)=>{O(e.pool);for(const s of e.pool.socks){const i=await f(s,void 0,o);if(i&&r(i.podGeneration))return!0}return!1},"anyPodAnswers"),he=new Map,qe=a((e,r,o)=>{const s=C>0&&(0,handoff_retire_1.requestHandoffRetire)(e.pool.dir,o);if(A(r,"SIGTERM"),!s)return;const i=e.wsKey&&e.wsKey!==t.wsKey&&e.root?(0,workspace_log_1.workspaceLogWriter)(e.root):void 0,n=a(ne=>h=>{if(t.stopped())return;const P=(0,inspector_secret_1.redactInspectorUrls)(h.toString()).replace(/^(?=.)/gm,`[${e.project} g${o} retiring] `);ne.write(P),i?.(P)},"say"),d=n(process.stdout),u=n(process.stderr);r.stdout?.on("data",d),r.stderr?.on("data",u);let w,k=!1;const j=a(()=>{k=!0,clearTimeout(w),r.stdout?.removeListener("data",d),r.stderr?.removeListener("data",u),r.removeListener("close",j)},"releaseLogs");r.once("close",j);let v=!1;const g=a(()=>{v||(v=!0,clearTimeout(R),clearInterval(G),t.stopped()?j():k||(w=setTimeout(j,1e3),w.unref?.()),he.delete(r),(0,handoff_retire_1.clearHandoffRetire)(e.pool.dir,o),ge(e),e.pool.scannedAt=0)},"end"),R=setTimeout(()=>{console.error(`[gateway] pod handoff: ${e.project} g${o} was still running ${C}ms after it was retired \u2014 killing it.`),T(r,"SIGKILL"),g()},C);R.unref?.();const M=a(()=>{if(!v){if(t.stopped()){g();return}L(r)||(console.log(`[gateway] pod handoff: ${e.project} g${o} finished its in-flight work and exited.`),g())}},"look"),G=setInterval(M,ie);G.unref?.(),r.once("exit",M),he.set(r,()=>{g(),Se(e,r,"the gateway is stopping")})},"retireReplacedTree"),Ve=l.standbyGraceMs??3e4,I=new Map,Ue=a((e,r,o,s,i)=>{const n=i+B+Ve,d=setTimeout(()=>Me(r),Math.max(0,n-Date.now()));d.unref?.(),I.set(r,{svc:e,generation:o,reason:s,beganAt:i,deadlineAt:n,timer:d})},"holdStandby"),Be=a(e=>{const r=I.get(e);r&&(clearTimeout(r.timer),I.delete(e))},"releaseStandby"),Z=a((e,r)=>{const o=I.get(e);if(!o)return!1;clearTimeout(o.timer),I.delete(e);const{svc:s,generation:i}=o;try{e.stdout?.removeAllListeners("data"),e.stderr?.removeAllListeners("data")}catch{}return(0,discovery_1.stopRoutingTo)(s.pool,n=>(0,pod_sock_1.podSockGeneration)(n)===i),Se(s,e,`standby g${i}: ${r}`),setTimeout(()=>ge(s),1200).unref?.(),!0},"endStandby"),Te=a((e,r)=>{const o=r.oldServing?`old generation g${r.fromGen} kept serving`:`and the old generation g${r.fromGen} is NOT serving (${r.oldGone?"its tree is gone":"none of its pods answers"}) \u2014 the service is down until it recovers`;console.error(`[gateway] pod handoff failed: ${e.project} g${r.generation}: ${r.why} \u2014 ${o}.`),e.rollFailure=`${r.why} \u2014 ${o}`,S(e,r.oldServing?"warning":"error",`pod handoff failed: ${e.project}`,`cluster-service:${e.project}`,{kind:"pod-handoff-failed",project:e.project,reason:r.reason,generation:r.generation,error:r.why,durationMs:Date.now()-r.beganAt,oldServing:r.oldServing})},"reportStandbyFailed"),Me=a(e=>{const r=I.get(e);if(!r)return;const{svc:o,generation:s,reason:i,beganAt:n}=r,d=`the standby outlived its roll (${Math.round((Date.now()-n)/1e3)}s; a roll waits ${Math.round(B/1e3)}s for it)`;Z(e,d);const u=o.proc,w=t.serviceKeys[(0,service_keys_1.serviceKeyId)(o.wsKey,o.project)];(async()=>{const k=!u||u.exitCode!==null||u.signalCode!==null,j=k?!1:await Promise.race([ye(o,g=>g!==s,w).catch(()=>!1),sleep(2e3).then(()=>{})]),v=!t.stopped()&&!k&&(j??O(o.pool)>0);Te(o,{reason:i,generation:s,fromGen:o.generation??s-1,why:d,beganAt:n,oldServing:v,oldGone:k})})()},"outlivedRoll"),He=a(()=>{for(const[e,r]of[...I])Date.now()>=r.deadlineAt?Me(e):!r.svc.rolling&&r.svc.proc!==e&&(console.error(`[gateway] pod handoff: ${r.svc.project} g${r.generation} is a standby whose roll is over \u2014 stopping it.`),Z(e,"its roll is over"))},"sweepStandbys"),Ye=a(()=>{for(const e of[...he.values()])e();for(const e of[...I.keys()])Z(e,"the gateway is stopping")},"stopRetiringTrees"),me=a((e,r)=>{if(e.rolling)return e.rolling;const o=e.proc;if(!o||t.stopped())return Promise.resolve(!1);if(e.launcher==="nx")return console.log(`[gateway] pod handoff skipped: ${e.project} runs through nx run, which starts one instance of a target at a time \u2014 no standby can overlap it (reason=${r}).`),Promise.resolve(!1);Q>=X&&console.log(`[gateway] pod handoff queued: ${e.project} reason=${r} \u2014 ${Q} roll(s) in flight, ${ve.length} ahead (LENSMCP_POD_MAX_CONCURRENT_ROLLS=${X}).`);const s=(async()=>{await We();try{return await Xe(e,o,r)}finally{Ge()}})().finally(()=>{e.rolling=void 0});return e.rolling=s,s},"rollPod"),Xe=a(async(e,r,o)=>{const s=e.proc;if(s!==r||t.stopped())return!1;const i=e.generation??0,n=i+1,d=Date.now();e.lastStaleRecycleAt=d,console.log(`[gateway] pod handoff start: ${e.project} g${i}\u2192g${n} reason=${o} \u2014 standby spawning, old generation keeps serving.`),S(e,"info",`pod handoff starting: ${e.project} (${o})`,`cluster-service:${e.project}`,{kind:"pod-handoff-start",project:e.project,reason:o,generation:n,fromGeneration:i});const u=J(e,n,"standby"),w=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)];Ue(e,u,n,o,d);let k=!1;try{const j=d+B;let v;for(;!t.stopped()&&Date.now()<j&&I.has(u)&&!(u.exitCode!==null||u.signalCode!==null);){O(e.pool);for(const g of e.pool.socks)if((await f(g,void 0,w))?.podGeneration===n){v=g;break}if(v)break;await sleep(le)}if(!I.has(u))return!1;if(!v||t.stopped()){const g=t.stopped()?"gateway stopping":u.exitCode!==null||u.signalCode!==null?`standby exited (${u.exitCode??u.signalCode})`:`standby not ready within ${Math.round(B/1e3)}s`;Z(u,g);const R=!t.stopped()&&e.proc===s&&s.exitCode===null&&s.signalCode===null&&await ye(e,M=>M!==n,w);return Te(e,{reason:o,generation:n,fromGen:i,why:g,beganAt:d,oldServing:R,oldGone:e.proc!==s||s.exitCode!==null||s.signalCode!==null}),!1}if(e.proc!==s)return Z(u,"the service was replaced underneath the roll"),!1;Be(u),k=!0,e.proc=u,e.launcher=oe.get(u),e.generation=n,e.spawnAt=d,e.state="up",e.watcher=void 0,e.staleVerifiedAt=void 0,e.staleKeptAt=void 0,e.buildFailed=!1,e.lastErrors=[];try{s.stdout?.removeAllListeners("data"),s.stderr?.removeAllListeners("data"),s.removeAllListeners("exit")}catch{}return(0,discovery_1.stopRoutingTo)(e.pool,g=>(0,pod_sock_1.podSockGeneration)(g)!==n&&path.join(e.pool.dir,g)!==v),qe(e,s,i),setTimeout(()=>{ge(e),e.pool.scannedAt=0},1200).unref?.(),console.log(`[gateway] pod handoff ready: ${e.project} g${n} answered on ${path.basename(v)} in ${Date.now()-d}ms; retiring g${i}.`),S(e,"info",`pod handoff complete: ${e.project}`,`cluster-service:${e.project}`,{kind:"pod-handoff-complete",project:e.project,reason:o,generation:n,fromGeneration:i,durationMs:Date.now()-d}),!0}finally{k||Z(u,"its roll ended without it")}},"runRoll"),ze=a((e,r)=>{if(t.stopped())return{refused:"the gateway is stopping"};if(e.manualStopped=!1,e.actionFailure=void 0,e.restart&&!e.restart.done)return{restart:e.restart,already:!0};if(!e.proc&&e.state!=="down")return{refused:`${e.project} is served by pods this gateway adopted from an earlier one \u2014 it has no tree of its own to replace. \`lensmcp gateway restart\` starts every service under this gateway.`};const o=e.proc?r.hard||e.launcher==="nx"||e.state!=="up"?"hard":"rolling":"start",s={how:o,reason:r.reason,startedAt:Date.now(),...e.proc&&e.generation!==void 0?{fromGeneration:e.generation}:{}};return e.restart=s,e.lastUsed=s.startedAt,console.log(`[gateway] restart asked: ${e.project}${e.wsKey!==t.wsKey?` (${e.wsKey})`:""} \u2014 ${o} (${r.reason}).`),S(e,"info",`restart asked: ${e.project} (${o})`,`cluster-service:${e.project}`,{kind:"service-restart",phase:"started",project:e.project,how:o,reason:r.reason,...s.fromGeneration!==void 0?{fromGeneration:s.fromGeneration}:{}}),a(async()=>{if(o==="start")return await we(e)?{ok:!0}:{ok:!1,error:$e(e)};if(o==="rolling")return e.rolling&&await e.rolling,e.proc?(e.rollFailure=void 0,await me(e,r.reason)?{ok:!0}:{ok:!1,error:e.rollFailure??"the new tree did not take over"}):await we(e)?{ok:!0}:{ok:!1,error:$e(e)};if(V(e,`restart asked (${r.reason})`,{announce:!1}),await sleep(1500),t.stopped())return{ok:!1,error:"the gateway is stopping"};let i=!1;e.proc||(await F(),i=!0,!e.proc&&e.state!=="starting"&&Y(e));try{const n=e.proc,d=e.generation,u=t.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)],w=Date.now()+t.startupTimeoutMs;for(;!t.stopped()&&Date.now()<w;){if(!n||e.proc!==n||n.exitCode!==null||n.signalCode!==null)return{ok:!1,error:$e(e)};if(await ye(e,k=>k===d,u))return pe(e,"observed"),e.buildFailed=!1,{ok:!0};await sleep(le)}return{ok:!1,error:t.stopped()?"the gateway is stopping":`no pod of the new tree answered within ${Math.round(t.startupTimeoutMs/1e3)}s`}}finally{i&&D()}},"run")().catch(i=>({ok:!1,error:i.message})).then(i=>{s.done={ok:i.ok,at:Date.now(),...e.generation!==void 0?{generation:e.generation}:{},...i.error?{error:i.error}:{}},i.ok&&(e.lastUsed=s.done.at);const n=Math.round((s.done.at-s.startedAt)/100)/10;i.ok?console.log(`[gateway] restarted: ${e.project} \u2014 g${s.done.generation??"?"} serving after ${n}s (${o}).`):console.error(`[gateway] restart failed: ${e.project} \u2014 ${i.error} (${o}, after ${n}s).`),S(e,i.ok?"info":"error",i.ok?`restarted: ${e.project}`:`restart failed: ${e.project}`,`cluster-service:${e.project}`,{kind:"service-restart",phase:i.ok?"done":"failed",project:e.project,how:o,reason:r.reason,...s.fromGeneration!==void 0?{fromGeneration:s.fromGeneration}:{},...s.done.generation!==void 0?{generation:s.done.generation}:{},durationMs:s.done.at-s.startedAt,...i.error?{error:i.error}:{}})}),{restart:s,already:!1}},"restartService"),$e=a(e=>e.buildFailed?`its build failed${e.lastErrors?.length?`: ${e.lastErrors[e.lastErrors.length-1]}`:""}`:e.proc&&(e.proc.exitCode!==null||e.proc.signalCode!==null)?`its tree exited (${e.proc.exitCode??e.proc.signalCode})`:!e.proc&&e.state==="down"?"its tree exited before a pod answered":`no pod answered within ${Math.round(t.startupTimeoutMs/1e3)}s`,"startFailure"),xe=a(e=>{const r=Date.now();e.restarts=(e.restarts??[]).filter(s=>r-s<types_1.SERVICE_RESTART_WINDOW_MS);const o=(e.wedgeRecycles??0)+1;if(e.restarts.length>=types_1.SERVICE_MAX_RESTARTS||o>types_1.SERVICE_MAX_RESTARTS){const s=e.restarts.length>=types_1.SERVICE_MAX_RESTARTS?`wedged ${types_1.SERVICE_MAX_RESTARTS}+\xD7/min`:`recycled ${types_1.SERVICE_MAX_RESTARTS}\xD7 in a row and no pod came up`;S(e,"error",`service crash-looping: ${e.project} \u2014 ${s}, not auto-recycling (the next request retries)`,`cluster-service:${e.project}`,{kind:"service-crash-loop",project:e.project,recycles:o-1}),V(e,"wedged (crash-loop cap) \u2014 leaving down for the next request",{announce:!1});return}e.wedgeRecycles=o,e.restarts.push(r),console.log(`[gateway] self-heal: recycling WEDGED ${e.project} (live parent, 0 live pods; attempt ${o}/${types_1.SERVICE_MAX_RESTARTS} with no pod up since the first, ${e.restarts.length} in the last minute; ${e.inflight??0} request(s) in flight \u2014 jobs held by the dead pod keep their locks until they expire)`),S(e,"warning",`self-heal: recycling wedged ${e.project} (0 live pods)`,`cluster-service:${e.project}`,{kind:"service-recycle",project:e.project,attempt:o,inWindow:e.restarts.length}),V(e,"wedged: 0 live pods \u2014 recycling",{announce:!1}),setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&Y(e)},1500).unref?.()},"recycleWedged"),Je={maxAgeMs:types_1.POD_MAX_AGE_MS,graceMs:types_1.POD_STALE_GRACE_MS,settleMs:types_1.POD_RECYCLE_SETTLE_MS,cooldownMs:types_1.POD_STALE_RECYCLE_COOLDOWN_MS};let _e;const se=new Map,Ee=new Map;let ke=!1,je=0,fe=0;const Qe=a(()=>{const e=t.projectRoots??{},r=new Set(t.services.map(s=>s.project));for(const s of t.routes)s.lens&&r.add(s.lens.project);const o=new Set;for(const s of r){const i=e[s];if(!i)continue;const n=path.relative(t.root,i).split(path.sep).filter(Boolean)[0];n&&!n.startsWith("..")&&o.add(n)}return[...o]},"managedBuckets"),Ze=a((e,r,o,s)=>{const i=t.projectSourceRoots?.[e];if(i?.length)return i;const n=t.projectRoots?.[e];if(!n)return;const d=sourceScopeDirs(t.root,n,o,r,s);return d.length>0?d:void 0},"scopeFor"),et=a(()=>{if(ke||Date.now()-je<types_1.POD_STALE_SCAN_MS)return;ke=!0,je=Date.now();const e=je;(async()=>{const r=(0,scope_1.readSourceSetOptions)(t.root),o=r.exclude,s={ignoreTests:r.ignoreTests},i=sourceScanPass(t.root,o,s),n=await i.snapshot(),d=Date.now(),u=sourceChangeWindowMs(d,fe,types_1.POD_STALE_SCAN_MS,types_1.POD_RECYCLE_SETTLE_MS),w=_e?diffSourceSets(_e,n,d,u):null;if(w&&(t.sourceSetChangedAt=d,t.sourceSetChange=w,H("info","source file set changed \u2014 stale pods recycle when idle","cluster-gateway",{kind:"source-set-changed",added:w.added,removed:w.removed,truncated:w.truncated})),_e=n,!t.projectRoots){fe=e;return}let k;try{k=(await fs.promises.readdir(t.root,{withFileTypes:!0})).filter(h=>h.isDirectory()).map(h=>h.name)}catch{return}const j=Qe(),v=new Set(t.services.map(h=>h.project));for(const h of t.routes)h.lens&&v.add(h.lens.project);const g=new Map;for(const h of v){const P=Ze(h,k,j,o);if(!P)continue;const x=P.join("|"),q=g.get(x)??{dirs:P,projects:[]};q.projects.push(h),g.set(x,q)}const R=Date.now(),M=sourceChangeWindowMs(R,fe,types_1.POD_STALE_SCAN_MS,types_1.POD_RECYCLE_SETTLE_MS);fe=e;const G=a((h,P,x)=>{t.sourceSetChangedFor??=new Map,t.sourceSetChangesFor??=new Map;for(const q of h)t.sourceSetChangedFor.set(q,P),t.sourceSetChangesFor.set(q,x)},"stampScoped"),ne=new Set([...g.values()].flatMap(h=>h.projects));for(const h of[...t.sourceSetScoped??[]])ne.has(h)||t.sourceSetScoped.delete(h);for(const[h,{dirs:P,projects:x}]of g){const q=await i.snapshot(P),Re=se.get(h);if(Re||x.some(K=>Ee.has(K))){t.sourceSetScoped??=new Set;for(const K of x)t.sourceSetScoped.add(K)}se.set(h,q);const Ae=new Map;for(const K of x){const U=Ee.get(K);if(U===void 0||U===h)continue;const ee=Ae.get(U)??[];ee.push(K),Ae.set(U,ee)}for(const K of x)Ee.set(K,h);for(const[K,U]of Ae){const ee=se.get(K),ce=ee?diffSourceSets(ee,q,R,M):w;ce&&(G(U,ee?R:d,ce),H("info",`source file set changed \u2014 ${U.join(", ")}'s watch scope was re-keyed`,"cluster-gateway",{kind:"source-set-changed",scoped:!0,rescoped:!0,projects:U,added:ce.added,removed:ce.removed,truncated:ce.truncated}))}if(!Re)continue;const ae=diffSourceSets(Re,q,R,M);ae&&(G(x,R,ae),H("info",`source file set changed in ${x.join(", ")}'s scope`,"cluster-gateway",{kind:"source-set-changed",scoped:!0,projects:x,added:ae.added,removed:ae.removed,truncated:ae.truncated}))}for(const h of[...se.keys()])g.has(h)||se.delete(h)})().catch(()=>{}).finally(()=>{ke=!1})},"maybeScanSourceSet"),tt=a((e,r)=>{const o=podChangeClock(t,e.project),s=o.change,i=`${o.scope==="project"?"in its scope":"in the workspace (its scope is not known)"}: ${changeEvidence(s)}`,n=s?.windowMs??types_1.POD_STALE_SCAN_MS+types_1.POD_RECYCLE_SETTLE_MS,d=podWatcherVerdict(e.watcher,r,n),u=a(k=>(e.staleKeptAt!==r&&(e.staleKeptAt=r,console.log(`[gateway] stale source set: pod ${e.project} ${k} \u2014 no recycle.`)),!1),"keep");if(d==="heard")return e.staleVerifiedAt=r,S(e,"info",`stale check: ${e.project} watcher confirmed \u2014 no recycle`,`cluster-service:${e.project}`,{kind:"pod-stale-verified",project:e.project,watcher:d}),u(`watcher CONFIRMED alive across the change (last build ${new Date(e.watcher?.lastBuildAt??0).toISOString()})`);if(d==="degraded")return console.log(`[gateway] stale source set: pod ${e.project} watcher DEGRADED (${e.watcher?.degraded}) \u2014 recycling.`),!0;const w=a(k=>(e.staleKeptAt!==r&&(e.staleKeptAt=r,console.log(`[gateway] stale source set: pod ${e.project} ${k}; the change ${i} \u2014 recycling.`)),!0),"recycling");return d==="silent"?types_1.POD_WATCHER_SILENT==="keep"?u("watcher alive but quiet across the change \u2014 keeping it (LENSMCP_POD_WATCHER_SILENT=keep; a live watcher cannot miss an add it will need)"):w("watcher alive but quiet across the change"):types_1.POD_WATCHER_UNVERIFIED==="keep"?u("unverifiable (no watcher testimony \u2014 older devserver bundle?) \u2014 keeping it (LENSMCP_POD_WATCHER_UNVERIFIED=keep)"):w("watcher unverifiable (no testimony yet \u2014 a pod that never rebuilt, or an older devserver bundle)")},"podStaleRecycleAllowed"),rt=a((e,r)=>{if(e.lastStaleRecycleAt=Date.now(),types_1.POD_RECYCLE_MODE==="rolling"&&e.proc&&e.launcher!=="nx"){me(e,r);return}console.log(`[gateway] zombie recycle: ${e.project} (${r}) \u2014 stale fs view; fresh pod incoming`),S(e,"warning",`zombie recycle: ${e.project} (${r})`,`cluster-service:${e.project}`,{kind:"pod-recycle",project:e.project,reason:r});const o=!!e.decl.eager;V(e,`${r} \u2014 recycling stale pod`,{announce:!1}),o&&setTimeout(()=>{!t.stopped()&&!e.proc&&e.state==="down"&&Y(e)},1500).unref?.()},"recyclePod"),ot=a(()=>{const e=setInterval(()=>{ct().catch(r=>console.error("[gateway] sweep error:",r))},t.sweepMs);return e.unref(),e},"startSweeper"),st=l.listProcesses??(()=>{try{return(0,orphan_reaper_1.parsePsRows)((0,node_child_process_1.execFileSync)("ps",orphan_reaper_1.PS_ROW_ARGS,{encoding:"utf8",timeout:5e3}))}catch{return[]}}),Pe=l.signalProcess??((e,r)=>{process.kill(e,r)});let De=0;const nt=a(()=>{if(typeof process.getuid!="function")return 0;const e=t.registry?t.registry.allServices():t.services;if(e.length===0)return 0;const r=e.map(n=>({label:n.wsKey||path.basename(n.root),project:n.project,root:n.root,svc:n})),o=new Set;for(const n of e)n.proc?.pid&&o.add(n.proc.pid);for(const n of[...he.keys(),...N.keys(),...I.keys()])n.pid&&o.add(n.pid);const s=[...new Set((0,process_name_1.defaultLinkDirs)().flatMap(n=>{try{return[n,fs.realpathSync(n)]}catch{return[n]}}))];let i=0;for(const n of(0,orphan_reaper_1.findOrphanExecutors)(st(),r,{linkDirs:s,uid:process.getuid(),selfPid:process.pid,ownGroups:o})){const{row:d}=n;try{Pe(n.group?-d.pgid:d.pid,"SIGKILL")}catch(w){if(w.code!=="ESRCH"||!n.group)continue;try{Pe(d.pid,"SIGKILL")}catch{continue}}i+=1;const u=n.owner.svc;console.log(`[gateway] reaped an orphaned executor of ${u.project} (${n.role}, pid ${d.pid}${n.group?`, group ${d.pgid}`:""}, parent gone) \u2014 a stopped tree it outlived: ${d.command.slice(0,160)}`),S(u,"warning",`orphan reap: ${u.project} (pid ${d.pid})`,`cluster-service:${u.project}`,{kind:"orphan-executor-reap",project:u.project,pid:d.pid,pgid:d.pgid,role:n.role,group:n.group})}return i},"reapOrphanExecutors"),ct=a(async()=>{{if(t.stopped())return;He(),l.reapForeign!==!1&&types_1.ORPHAN_REAP_EVERY_MS>0&&Date.now()-De>=types_1.ORPHAN_REAP_EVERY_MS&&(De=Date.now(),nt());for(const e of t.graphStores?.all()??[])e.refreshIfStale();types_1.POD_RECYCLE_ENABLED&&et();for(const e of t.registry?t.registry.allServices():t.services){if(t.stopped()||!e.proc){e.wedgedSince=void 0;continue}const r=path.resolve(e.root)===path.resolve(t.root),o=(0,warm_start_1.instanceIdleSec)(e,!r&&!!t.registry?.get(e.wsKey)?.instance);if(o&&e.state==="up"&&(!e.restart||e.restart.done)&&e.inflight===0&&Date.now()-e.lastUsed>o*1e3){V(e,`idle ${o}s`),e.wedgedSince=void 0;continue}const s=e.state==="starting"&&e.spawnAt!==void 0&&Date.now()-e.spawnAt<t.startupTimeoutMs,i=O(e.pool)>0;if(i||s){if(e.wedgedSince=void 0,i&&pe(e,"observed"),i&&!s&&types_1.POD_RECYCLE_ENABLED&&r){const n=podChangeClock(t,e.project).at;await be(e);const d=(e.staleVerifiedAt??0)>=n&&n>0,u=podRecycleReason(e,Date.now(),d?0:n,Je);if(u==="stale-source-set"&&!tt(e,n))continue;u&&rt(e,u)}continue}e.wedgedSince??=Date.now(),Date.now()-e.wedgedSince>types_1.WEDGE_RECYCLE_GRACE_MS&&(e.wedgedSince=void 0,xe(e))}}},"sweepOnce");return{scanNow:O,spawnService:Y,ensureUp:we,scaleUp:Ke,killSpawned:V,markUp:pe,startSweeper:ot,rollPod:me,restartService:ze,recycleWedged:xe,stopRetiringTrees:Ye,finishDyingTrees:Fe,stopBuildHost:a(e=>e===void 0?_.stop():_.release(e),"stopBuildHost")}}m(createServiceLayer,"createServiceLayer"),a(createServiceLayer,"createServiceLayer");function chooseStalePodKillTarget(t,c,l){if(!(t===void 0||!Number.isInteger(t)||t<=1)&&!(c!==void 0&&c!==t)&&t!==l)return t}m(chooseStalePodKillTarget,"chooseStalePodKillTarget"),a(chooseStalePodKillTarget,"chooseStalePodKillTarget");

package/executors/gateway/runtime/observability.js

@@ -1,2 +1,3 @@
-"use strict";var A=Object.defineProperty;var w=(s,c)=>A(s,"name",{value:c,configurable:!0});var _=Object.defineProperty,d=w((s,c)=>_(s,"name",{value:c,configurable:!0}),"d");Object.defineProperty(exports,"__esModule",{value:!0}),exports.EDGE_DURATION_SAMPLES=exports.LENS_DASHBOARD_PROJECT=void 0,exports.createObservability=createObservability;const tslib_1=require("tslib"),fs=tslib_1.__importStar(require("node:fs")),path=tslib_1.__importStar(require("node:path")),headless_flow_1=require("@lensmcp/core/headless-flow"),redact_1=require("@lensmcp/core/redact"),busHygiene=(()=>{try{return require("@lensmcp/node-instrumentation")}catch{return}})();exports.LENS_DASHBOARD_PROJECT="(lens-dashboard)",exports.EDGE_DURATION_SAMPLES=64;function ownerOf(s){return s?.svc?{root:s.svc.root,wsKey:s.svc.wsKey}:s?.lens?{...s.lens.workspaceRoot?{root:s.lens.workspaceRoot}:{},...s.lens.workspaceKey?{wsKey:s.lens.workspaceKey}:{}}:{}}w(ownerOf,"ownerOf"),d(ownerOf,"ownerOf");function busFileFor(s,c){const{root:l}=ownerOf(s);if(!l)return;const f=path.join(l,".lensmcp","events.jsonl");return f===c?void 0:f}w(busFileFor,"busFileFor"),d(busFileFor,"busFileFor");const hdr=d(s=>typeof s=="string"&&s!==""?s:Array.isArray(s)?s[0]:void 0,"hdr");function createObservability(s,c){const l=d(()=>Date.now().toString(36)+Math.random().toString(36).slice(2,12),"eid"),f=busHygiene?.createEventFloodGuard({burst:30,perSec:5}),y=d((e,t,r,u,n,a)=>{const o=a??s;try{const i=f?f.admit(r):{ok:!0,suppressed:0};if(!i.ok)return;fs.mkdirSync(path.dirname(o),{recursive:!0});const p=u&&(0,redact_1.redactDeep)(u),h=JSON.stringify({id:l(),sessionId:"pending",timestamp:Date.now(),source:"gateway",category:"cluster",severity:e,context:(0,redact_1.redactDeep)({sessionId:"pending",...n}),fingerprint:r,title:(0,redact_1.safeTitle)(t),raw:i.suppressed>0?{...p,floodSuppressed:i.suppressed}:p})+`
-`;busHygiene?.maybeRotateEventFile(o,h.length),fs.appendFileSync(o,h)}catch{}},"emit"),I=d(e=>{const t=hdr(e.headers["x-lensmcp-flow-id"]);let r=hdr(e.headers["x-request-id"]);if(!(!t&&!r&&!hdr(e.headers.traceparent)))return r||(r=l(),e.headers["x-request-id"]=r),{startedAt:Date.now(),method:(e.method??"GET").toUpperCase(),host:(e.headers.host||"").split(":")[0],url:e.url??"/",steps:[],...t?{flowId:t}:{},...r?{requestId:r}:{},...hdr(e.headers["x-lensmcp-parent-request-id"])?{parentRequestId:hdr(e.headers["x-lensmcp-parent-request-id"])}:{},...hdr(e.headers["x-lensmcp-origin-node-id"])?{originNodeId:hdr(e.headers["x-lensmcp-origin-node-id"])}:{}}},"beginTrace"),E=d((e,t,r,u)=>{if(r?.flowId||!t.pool||t.internal||t.project===exports.LENS_DASHBOARD_PROJECT||!(0,headless_flow_1.headlessFlowsEnabled)())return r;const n=e.url??"/";if((0,headless_flow_1.isProbePath)(n))return r;const a=hdr(e.headers["x-request-id"]);if(!a)return r;const o=(0,headless_flow_1.headlessFlowId)(a);e.headers["x-lensmcp-flow-id"]=o;const i=(e.method??"GET").toUpperCase(),p=(e.headers.host||"").split(":")[0],h=`request:${i} ${n.split("?")[0]}`,m=r??{startedAt:u,method:i,host:p,url:n,steps:[{step:"received",atMs:0,host:p,method:i,url:n}],requestId:a};return m.flowId=o,m.originNodeId=h,y("info",`${i} ${n.split("?")[0]}`,`gateway-origin:${t.project}`,{kind:"flow-origin",originNodeId:h,project:t.project,host:p,method:i,url:n,...ownerOf(t).wsKey?{wsKey:ownerOf(t).wsKey}:{}},{flowId:o,requestId:a,originType:"request",originNodeId:h},busFileFor(t,s)),m},"startHeadlessFlow"),S=d((e,t,r)=>{e&&e.steps.push({step:t,atMs:Date.now()-e.startedAt,...r})},"traceStep"),q=d((e,t,r,u,n)=>{if(!e||e.done||(e.done=!0,t===exports.LENS_DASHBOARD_PROJECT&&r<500))return;const a=Date.now()-e.startedAt;y(r>=500?"error":"info",`gateway ${e.method} ${e.host}${e.url} \u2192 ${r} (${a}ms)`,`gateway-request:${t}`,{kind:"gateway-request",project:t,host:e.host,method:e.method,url:e.url,status:r,durationMs:a,startedAt:e.startedAt,steps:e.steps,...ownerOf(n).wsKey?{wsKey:ownerOf(n).wsKey}:{},...u},{...e.flowId?{flowId:e.flowId}:{},...e.requestId?{requestId:e.requestId}:{},...e.parentRequestId?{parentRequestId:e.parentRequestId}:{},...e.originNodeId?{originNodeId:e.originNodeId}:{}},busFileFor(n,s))},"finishTrace"),g=new Map;return{emit:y,eid:l,beginTrace:I,startHeadlessFlow:E,traceStep:S,finishTrace:q,recordEdge:d((e,t,r,u)=>{if(e.project===exports.LENS_DASHBOARD_PROJECT&&r<500&&r!==0)return;const n=e.host??"(default)",a=(u?u+"\u21D2":"")+n+"\u2192"+e.project,o=g.get(a)??{host:n,project:e.project,caller:u,count:0,totalMs:0,errors:0,durations:[],...busFileFor(e,s)?{busFile:busFileFor(e,s)}:{},...ownerOf(e).wsKey?{wsKey:ownerOf(e).wsKey}:{}};if(o.count+=1,o.totalMs+=t,o.durations.length<exports.EDGE_DURATION_SAMPLES)o.durations.push(Math.round(t));else{const i=Math.floor(Math.random()*o.count);i<exports.EDGE_DURATION_SAMPLES&&(o.durations[i]=Math.round(t))}(r>=500||r===0)&&(o.errors+=1),g.set(a,o)},"recordEdge"),startEdgeFlusher:d(()=>{const e=setInterval(()=>{for(const t of g.values()){const r=Math.round(t.totalMs/t.count);y(t.errors?"warning":"info",`traffic ${t.host} \u2192 ${t.project}: ${t.count} req, ~${r}ms${t.errors?`, ${t.errors} errors`:""}`,`gateway-traffic:${t.host}->${t.project}`,{kind:"gateway-traffic",host:t.host,project:t.project,caller:t.caller,count:t.count,avgMs:r,errors:t.errors,durationsMs:t.durations,...t.wsKey?{wsKey:t.wsKey}:{}},void 0,t.busFile)}g.clear()},c.trafficFlushMs);return e.unref(),e},"startEdgeFlusher")}}w(createObservability,"createObservability"),d(createObservability,"createObservability");
+"use strict";var v=Object.defineProperty;var f=(s,u)=>v(s,"name",{value:u,configurable:!0});var $=Object.defineProperty,i=f((s,u)=>$(s,"name",{value:u,configurable:!0}),"i");Object.defineProperty(exports,"__esModule",{value:!0}),exports.EDGE_DURATION_SAMPLES=exports.LENS_DASHBOARD_PROJECT=void 0,exports.createObservability=createObservability;const tslib_1=require("tslib"),fs=tslib_1.__importStar(require("node:fs")),path=tslib_1.__importStar(require("node:path")),headless_flow_1=require("@lensmcp/core/headless-flow"),redact_1=require("@lensmcp/core/redact"),busHygiene=(()=>{try{return require("@lensmcp/node-instrumentation")}catch{return}})();exports.LENS_DASHBOARD_PROJECT="(lens-dashboard)",exports.EDGE_DURATION_SAMPLES=64;function ownerOf(s){return s?.svc?{root:s.svc.root,wsKey:s.svc.wsKey}:s?.lens?{...s.lens.workspaceRoot?{root:s.lens.workspaceRoot}:{},...s.lens.workspaceKey?{wsKey:s.lens.workspaceKey}:{}}:{}}f(ownerOf,"ownerOf"),i(ownerOf,"ownerOf");function busFileFor(s,u){const{root:l}=ownerOf(s);if(!l)return;const w=path.join(l,".lensmcp","events.jsonl");return w===u?void 0:w}f(busFileFor,"busFileFor"),i(busFileFor,"busFileFor");const hdr=i(s=>typeof s=="string"&&s!==""?s:Array.isArray(s)?s[0]:void 0,"hdr");function createObservability(s,u){const l=i(()=>Date.now().toString(36)+Math.random().toString(36).slice(2,12),"eid"),w=busHygiene?.createEventFloodGuard({burst:30,perSec:5}),g=new Map,S=i((e,t)=>{try{const r=g.get(e);r&&(fs.appendFileSync(e,JSON.stringify({id:l(),sessionId:"pending",timestamp:Date.now(),source:"gateway",category:"cluster",severity:"warning",context:{sessionId:"pending"},fingerprint:"cluster-gateway:bus-dropped",title:`lens bus: ${r.count} event${r.count===1?"":"s"} could not be written since ${new Date(r.since).toISOString()} (${r.code}) \u2014 they are lost`,raw:{kind:"bus-events-dropped",project:"gateway",dropped:r.count,since:r.since,until:Date.now(),code:r.code}})+`
+`),g.delete(e),console.warn(`[gateway] the lens bus ${e} takes events again \u2014 ${r.count} were lost (${r.code})`)),fs.appendFileSync(e,t)}catch(r){const a=g.get(e);if(a){a.count+=1;return}const o=r?.code??"EIO";g.set(e,{count:1,since:Date.now(),code:o}),console.warn(`[gateway] the lens bus ${e} cannot be written (${o}) \u2014 counting the events lost until it can`)}},"appendCounted"),y=i((e,t,r,a,o,d)=>{const n=d??s;try{const c=w?w.admit(r):{ok:!0,suppressed:0};if(!c.ok)return;fs.mkdirSync(path.dirname(n),{recursive:!0});const p=a&&(0,redact_1.redactDeep)(a),h=JSON.stringify({id:l(),sessionId:"pending",timestamp:Date.now(),source:"gateway",category:"cluster",severity:e,context:(0,redact_1.redactDeep)({sessionId:"pending",...o}),fingerprint:r,title:(0,redact_1.safeTitle)(t),raw:c.suppressed>0?{...p,floodSuppressed:c.suppressed}:p})+`
+`;busHygiene?.maybeRotateEventFile(n,h.length),S(n,h)}catch{}},"emit"),E=i(e=>{const t=hdr(e.headers["x-lensmcp-flow-id"]);let r=hdr(e.headers["x-request-id"]);if(!(!t&&!r&&!hdr(e.headers.traceparent)))return r||(r=l(),e.headers["x-request-id"]=r),{startedAt:Date.now(),method:(e.method??"GET").toUpperCase(),host:(e.headers.host||"").split(":")[0],url:e.url??"/",steps:[],...t?{flowId:t}:{},...r?{requestId:r}:{},...hdr(e.headers["x-lensmcp-parent-request-id"])?{parentRequestId:hdr(e.headers["x-lensmcp-parent-request-id"])}:{},...hdr(e.headers["x-lensmcp-origin-node-id"])?{originNodeId:hdr(e.headers["x-lensmcp-origin-node-id"])}:{}}},"beginTrace"),D=i((e,t,r,a)=>{if(r?.flowId||!t.pool||t.internal||t.project===exports.LENS_DASHBOARD_PROJECT||!(0,headless_flow_1.headlessFlowsEnabled)())return r;const o=e.url??"/";if((0,headless_flow_1.isProbePath)(o))return r;const d=hdr(e.headers["x-request-id"]);if(!d)return r;const n=(0,headless_flow_1.headlessFlowId)(d);e.headers["x-lensmcp-flow-id"]=n;const c=(e.method??"GET").toUpperCase(),p=(e.headers.host||"").split(":")[0],h=`request:${c} ${o.split("?")[0]}`,I=r??{startedAt:a,method:c,host:p,url:o,steps:[{step:"received",atMs:0,host:p,method:c,url:o}],requestId:d};return I.flowId=n,I.originNodeId=h,y("info",`${c} ${o.split("?")[0]}`,`gateway-origin:${t.project}`,{kind:"flow-origin",originNodeId:h,project:t.project,host:p,method:c,url:o,...ownerOf(t).wsKey?{wsKey:ownerOf(t).wsKey}:{}},{flowId:n,requestId:d,originType:"request",originNodeId:h},busFileFor(t,s)),I},"startHeadlessFlow"),b=i((e,t,r)=>{e&&e.steps.push({step:t,atMs:Date.now()-e.startedAt,...r})},"traceStep"),_=i((e,t,r,a,o)=>{if(!e||e.done||(e.done=!0,t===exports.LENS_DASHBOARD_PROJECT&&r<500))return;const d=Date.now()-e.startedAt;y(r>=500?"error":"info",`gateway ${e.method} ${e.host}${e.url} \u2192 ${r} (${d}ms)`,`gateway-request:${t}`,{kind:"gateway-request",project:t,host:e.host,method:e.method,url:e.url,status:r,durationMs:d,startedAt:e.startedAt,steps:e.steps,...ownerOf(o).wsKey?{wsKey:ownerOf(o).wsKey}:{},...a},{...e.flowId?{flowId:e.flowId}:{},...e.requestId?{requestId:e.requestId}:{},...e.parentRequestId?{parentRequestId:e.parentRequestId}:{},...e.originNodeId?{originNodeId:e.originNodeId}:{}},busFileFor(o,s))},"finishTrace"),m=new Map;return{emit:y,eid:l,beginTrace:E,startHeadlessFlow:D,traceStep:b,finishTrace:_,recordEdge:i((e,t,r,a)=>{if(e.project===exports.LENS_DASHBOARD_PROJECT&&r<500&&r!==0)return;const o=e.host??"(default)",d=(a?a+"\u21D2":"")+o+"\u2192"+e.project,n=m.get(d)??{host:o,project:e.project,caller:a,count:0,totalMs:0,errors:0,durations:[],...busFileFor(e,s)?{busFile:busFileFor(e,s)}:{},...ownerOf(e).wsKey?{wsKey:ownerOf(e).wsKey}:{}};if(n.count+=1,n.totalMs+=t,n.durations.length<exports.EDGE_DURATION_SAMPLES)n.durations.push(Math.round(t));else{const c=Math.floor(Math.random()*n.count);c<exports.EDGE_DURATION_SAMPLES&&(n.durations[c]=Math.round(t))}(r>=500||r===0)&&(n.errors+=1),m.set(d,n)},"recordEdge"),startEdgeFlusher:i(()=>{const e=setInterval(()=>{for(const t of m.values()){const r=Math.round(t.totalMs/t.count);y(t.errors?"warning":"info",`traffic ${t.host} \u2192 ${t.project}: ${t.count} req, ~${r}ms${t.errors?`, ${t.errors} errors`:""}`,`gateway-traffic:${t.host}->${t.project}`,{kind:"gateway-traffic",host:t.host,project:t.project,caller:t.caller,count:t.count,avgMs:r,errors:t.errors,durationsMs:t.durations,...t.wsKey?{wsKey:t.wsKey}:{}},void 0,t.busFile)}m.clear()},u.trafficFlushMs);return e.unref(),e},"startEdgeFlusher")}}f(createObservability,"createObservability"),i(createObservability,"createObservability");

package/executors/gateway/runtime/orphan-reaper.d.ts

@@ -0,0 +1,44 @@
+/** One process, as `ps` lists it. */
+export interface PsRow {
+    pid: number;
+    ppid: number;
+    pgid: number;
+    uid: number;
+    command: string;
+}
+/** The `ps` arguments {@link parsePsRows} reads. */
+export declare const PS_ROW_ARGS: string[];
+export declare function parsePsRows(out: string): PsRow[];
+/** A service whose orphaned executors may be reaped: the label its processes are named with, and its workspace root. */
+export interface OrphanOwner {
+    label: string;
+    project: string;
+    root: string;
+}
+/** The roles of the process that holds a service's executor: nx's (client and run-executor), and the serve runner. */
+export declare const ORPHAN_ROLES: readonly ["nx", "serve"];
+type OrphanRole = (typeof ORPHAN_ROLES)[number];
+export interface OrphanMatchOptions {
+    /** Where process-name links live (process-name.ts `defaultLinkDirs`). */
+    linkDirs: readonly string[];
+    /** This user. */
+    uid: number;
+    /** This gateway. */
+    selfPid: number;
+    /** The process groups of every tree this gateway holds — live, standby, retiring, stopping. */
+    ownGroups: ReadonlySet<number>;
+}
+export interface Orphan {
+    row: PsRow;
+    owner: OrphanOwner;
+    role: OrphanRole;
+    /** Kill its process group: it leads it, or the group's leader is gone. Otherwise only the process. */
+    group: boolean;
+}
+/**
+ * The orphaned executors among `rows` — of the services in `owners` only. A process this gateway holds, a process with
+ * a parent, another user's, and one whose name or arguments are not exactly a gateway's for one of `owners` are never
+ * returned.
+ */
+export declare function findOrphanExecutors(rows: readonly PsRow[], owners: readonly OrphanOwner[], opts: OrphanMatchOptions): Orphan[];
+export {};

package/executors/gateway/runtime/orphan-reaper.js

@@ -0,0 +1,2 @@
+"use strict";var x=Object.defineProperty;var u=(r,o)=>x(r,"name",{value:o,configurable:!0});var m=Object.defineProperty,d=u((r,o)=>m(r,"name",{value:o,configurable:!0}),"d");Object.defineProperty(exports,"__esModule",{value:!0}),exports.ORPHAN_ROLES=exports.PS_ROW_ARGS=void 0,exports.parsePsRows=parsePsRows,exports.findOrphanExecutors=findOrphanExecutors;const tslib_1=require("tslib"),fs=tslib_1.__importStar(require("node:fs")),path=tslib_1.__importStar(require("node:path")),process_name_1=require("../../../process-name"),serve_launch_1=require("./serve-launch");exports.PS_ROW_ARGS=["ax","-ww","-o","pid=,ppid=,pgid=,uid=,command="];function parsePsRows(r){const o=[];for(const t of r.split(`
+`)){const s=/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/.exec(t);s&&o.push({pid:Number(s[1]),ppid:Number(s[2]),pgid:Number(s[3]),uid:Number(s[4]),command:s[5]})}return o}u(parsePsRows,"parsePsRows"),d(parsePsRows,"parsePsRows"),exports.ORPHAN_ROLES=["nx",serve_launch_1.SERVE_RUNNER_ROLE];function namedExecutable(r,o){for(const t of o){const s=`${t.replace(/\/+$/,"")}/`;if(!r.startsWith(s))continue;const i=/^\d+-\d+\/([A-Za-z0-9._-]+)(?=\s|$)/.exec(r.slice(s.length));if(i)return{name:i[1],args:r.slice(s.length+i[0].length).trim().split(/\s+/).filter(Boolean)}}}u(namedExecutable,"namedExecutable"),d(namedExecutable,"namedExecutable");function rootForms(r){const o=new Set([path.resolve(r)]);try{o.add(fs.realpathSync(r))}catch{}return[...o]}u(rootForms,"rootForms"),d(rootForms,"rootForms");function isNxOf(r,o,t){const s=`${o}/node_modules/`,i=d(n=>n.startsWith(s)&&!n.slice(s.length).split("/").includes(".."),"under");if(r.some(n=>i(n)&&n.endsWith("/nx/dist/bin/run-executor.js")))return!0;const a=r.findIndex(n=>i(n)&&n.endsWith("/nx/dist/bin/nx.js"));return a>=0&&r[a+1]==="run"&&(r[a+2]??"").startsWith(`${t}:`)}u(isNxOf,"isNxOf"),d(isNxOf,"isNxOf");function findOrphanExecutors(r,o,t){const s=new Map;for(const e of o)for(const c of exports.ORPHAN_ROLES)for(const p of new Set([(0,process_name_1.processName)(e.label,e.project,c),(0,process_name_1.legacyProcessName)(e.label,e.project,c)])){if(!p)continue;const f=s.get(p)??[];f.push({owner:e,role:c}),s.set(p,f)}const i=new Set(r.map(e=>e.pid)),a=r.find(e=>e.pid===t.selfPid)?.pgid,n=[];for(const e of r){if(e.ppid!==1||e.uid!==t.uid||e.pid<=1||e.pgid<=1||e.pid===t.selfPid||t.ownGroups.has(e.pgid)||t.ownGroups.has(e.pid)||e.pgid===a)continue;const c=namedExecutable(e.command,t.linkDirs);if(!c)continue;const p=(s.get(c.name)??[]).find(({owner:f,role:h})=>rootForms(f.root).some(l=>h===serve_launch_1.SERVE_RUNNER_ROLE?(0,serve_launch_1.isServeRunnerCommandOf)(e.command,l):isNxOf(c.args,l,f.project)));p&&n.push({row:e,owner:p.owner,role:p.role,group:e.pid===e.pgid||!i.has(e.pgid)})}return n}u(findOrphanExecutors,"findOrphanExecutors"),d(findOrphanExecutors,"findOrphanExecutors");

package/executors/gateway/runtime/own-log.d.ts

@@ -19,6 +19,9 @@
  * still grew ~70 MB an hour on 1.25.9 (issues/closed/foodguard-77, reopened 2026-10-20). Truncating by path is safe for
  * the same reason as through the descriptor: every writer holds the launcher's append descriptor. A gateway run in a
  * terminal, or under a test, with neither, has nothing to rotate.
+ *
+ * Since issues/closed/foodguard-92 the gateway also opens that file itself and writes its stdout and stderr there
+ * ({@link takeOverGatewayLog}) — an append descriptor like the others, so the copy-and-truncate holds for it too.
  */
 export declare const OWN_LOG_MAX_BYTES: number;
 export declare const OWN_LOG_GENERATIONS = 3;
@@ -48,9 +51,77 @@ export declare function rotateOwnLog(file: string, opts?: OwnLogOptions): boolea
 export declare function startOwnLogRotation(root: string, opts?: OwnLogOptions & {
     everyMs?: number;
 }): () => void;
+/** What a log writer could not write since its last write that worked. */
+export interface DroppedOutput {
+    lines: number;
+    bytes: number;
+    firstAt: number;
+    lastAt: number;
+    /** The error of the last write that failed — `ENOSPC` on a full disk. */
+    code: string;
+}
+export interface LogWriter {
+    /** Write `data`. Never throws: a write that fails is counted. Returns whether it was written. */
+    write(data: string | Uint8Array): boolean;
+    /** What was lost since the last write that worked; nothing once a note has said so. */
+    dropped(): DroppedOutput | undefined;
+}
+/** The one line a log gets once it can be written again: how much of it is missing, when, and why. */
+export declare function droppedNote(d: DroppedOutput, file?: string): string;
+/**
+ * A log writer that outlives a failed write (issues/closed/foodguard-92): `append` may throw — `ENOSPC` on a full disk —
+ * and the writer counts what it could not write instead. The first write that works again is preceded by ONE line
+ * ({@link droppedNote}) saying how many lines were lost, between when and when, and why; then the writer goes on as before.
+ */
+export declare function createLogWriter(append: (data: Buffer) => void, opts?: {
+    now?: () => number;
+    file?: string;
+}): LogWriter;
+/** The gateway writing its own log ({@link takeOverGatewayLog}). */
+export interface GatewayLogTakeover {
+    readonly file: string;
+    /** The file at the path is not the one held open (removed, or renamed away): open the path again. Returns whether it did. */
+    reopenIfMoved(): boolean;
+    dropped(): DroppedOutput | undefined;
+    /** Give the streams their own `write` back and close the file. */
+    restore(): void;
+}
+/** A stream whose `write` the takeover replaces — `process.stdout` / `process.stderr`. */
+interface Writable {
+    write: (...args: never[]) => boolean;
+}
+export interface TakeOverOptions {
+    /** The log to write. Default: {@link GATEWAY_LOG_ENV}, the file the launcher opened. */
+    file?: string;
+    /** Default: this process's stdout and stderr. */
+    streams?: Writable[];
+    now?: () => number;
+    /** Write `data` to `fd`, returning how much was written — the spec makes it fail. */
+    writeFd?: (fd: number, data: Buffer) => number;
+}
+/** The takeover in force in this process, if any — a child whose output should reach the log is relayed through it. */
+export declare function gatewayLogTakeover(): GatewayLogTakeover | undefined;
+/**
+ * The gateway writes `gateway.log` itself (issues/closed/foodguard-92).
+ *
+ * Under `lensmcp gateway start` the gateway's stdout and stderr are a socket to the `nx run gateway:serve` client, and
+ * that client writes the file. One write of it that failed on a full disk (ENOSPC) was an unhandled error in nx, and nx
+ * died; from then on every line the gateway wrote went to a socket nobody read — its EPIPE swallowed as benign, nothing
+ * reopening anything — and the log stopped for good. FoodGuard, 2026-10-24: no line after 21:02:26Z (Node's fatal-exit
+ * footer, nx's), space free again from 21:07, nothing more until a restart; `events.jsonl`, which the gateway appends
+ * itself, went on.
+ *
+ * So when the launcher names the log it opened for append ({@link GATEWAY_LOG_ENV}), the gateway opens the same file
+ * for append and every write to its stdout and stderr goes there: nx is no longer between it and the file. A write
+ * that fails is counted ({@link createLogWriter}); the next one that works says how much was lost. Nothing is written to
+ * the old streams any more, so nothing is written twice. Returns `undefined` — the streams left as they are — when no
+ * log is named or it cannot be opened.
+ */
+export declare function takeOverGatewayLog(opts?: TakeOverOptions): GatewayLogTakeover | undefined;
 /**
  * A service's output, every line tagged with it. The relay used to tag a chunk — so a multi-line record (TypeORM's
  * «query is slow» with its SQL and parameters, a stack) had one `[engine]` on its first line and nothing on the rest,
  * and a chunk that began mid-line put the tag in the middle of one. Per stream: whether the next byte starts a line.
  */
 export declare function linePrefixer(project: string): (text: string) => string;
+export {};

package/executors/gateway/runtime/own-log.js

@@ -1,4 +1,6 @@
-"use strict";var O=Object.defineProperty;var a=(e,t)=>O(e,"name",{value:t,configurable:!0});var y=Object.defineProperty,o=a((e,t)=>y(e,"name",{value:t,configurable:!0}),"o");Object.defineProperty(exports,"__esModule",{value:!0}),exports.GATEWAY_LOG_ENV=exports.OWN_LOG_GENERATIONS=exports.OWN_LOG_MAX_BYTES=void 0,exports.shiftGenerations=shiftGenerations,exports.rotateOwnLog=rotateOwnLog,exports.startOwnLogRotation=startOwnLogRotation,exports.linePrefixer=linePrefixer;const tslib_1=require("tslib"),fs=tslib_1.__importStar(require("node:fs")),path=tslib_1.__importStar(require("node:path"));exports.OWN_LOG_MAX_BYTES=50*1024*1024,exports.OWN_LOG_GENERATIONS=3;const CHECK_EVERY_MS=3e4;exports.GATEWAY_LOG_ENV="LENSMCP_GATEWAY_LOG";const readEnv=o(e=>process.env[e],"readEnv");function shiftGenerations(e,t){try{fs.unlinkSync(`${e}.${t}`)}catch{}for(let r=t-1;r>=1;r--)try{fs.renameSync(`${e}.${r}`,`${e}.${r+1}`)}catch{}}a(shiftGenerations,"shiftGenerations"),o(shiftGenerations,"shiftGenerations");const sameFile=o((e,t)=>!!e&&e.isFile()&&e.dev===t.dev&&e.ino===t.ino,"sameFile"),statOr=o(e=>{try{return e()}catch{return}},"statOr");function rotateOwnLog(e,t={}){const r=t.fd??1,s=t.maxBytes??exports.OWN_LOG_MAX_BYTES,c=t.keep??exports.OWN_LOG_GENERATIONS,n=statOr(()=>fs.statSync(e));if(!n||n.size<=s)return!1;const i=sameFile(statOr(()=>fs.fstatSync(r)),n),u=t.launchedLog??readEnv(exports.GATEWAY_LOG_ENV);if(!i&&!(u&&sameFile(statOr(()=>fs.statSync(u)),n)))return!1;try{shiftGenerations(e,c),fs.copyFileSync(e,`${e}.1`);try{fs.chmodSync(`${e}.1`,384)}catch{}i?fs.ftruncateSync(r,0):fs.truncateSync(e,0)}catch{return!1}const p=i?_=>{try{fs.writeSync(r,`${_}
-`)}catch{}}:_=>{try{fs.appendFileSync(e,`${_}
-`)}catch{}};return(t.note??p)(`[gateway] ${path.basename(e)} passed ${Math.round(s/1048576)} MB \u2014 rotated; the lines before are in ${path.basename(e)}.1 (\u2026${c} kept)`),!0}a(rotateOwnLog,"rotateOwnLog"),o(rotateOwnLog,"rotateOwnLog");function startOwnLogRotation(e,t={}){const r=path.join(e,".lensmcp","gateway.log"),s=setInterval(()=>rotateOwnLog(r,t),t.everyMs??CHECK_EVERY_MS);return s.unref?.(),()=>clearInterval(s)}a(startOwnLogRotation,"startOwnLogRotation"),o(startOwnLogRotation,"startOwnLogRotation");function linePrefixer(e){let t=!0;const r=`[${e}] `;return s=>{let c="";for(const n of s.split(/(?<=\n)/))n&&(c+=(t?r:"")+n,t=n.endsWith(`
-`));return c}}a(linePrefixer,"linePrefixer"),o(linePrefixer,"linePrefixer");
+"use strict";var g=Object.defineProperty;var l=(t,e)=>g(t,"name",{value:e,configurable:!0});var S=Object.defineProperty,i=l((t,e)=>S(t,"name",{value:e,configurable:!0}),"i");Object.defineProperty(exports,"__esModule",{value:!0}),exports.GATEWAY_LOG_ENV=exports.OWN_LOG_GENERATIONS=exports.OWN_LOG_MAX_BYTES=void 0,exports.shiftGenerations=shiftGenerations,exports.rotateOwnLog=rotateOwnLog,exports.startOwnLogRotation=startOwnLogRotation,exports.droppedNote=droppedNote,exports.createLogWriter=createLogWriter,exports.gatewayLogTakeover=gatewayLogTakeover,exports.takeOverGatewayLog=takeOverGatewayLog,exports.linePrefixer=linePrefixer;const tslib_1=require("tslib"),fs=tslib_1.__importStar(require("node:fs")),path=tslib_1.__importStar(require("node:path"));exports.OWN_LOG_MAX_BYTES=50*1024*1024,exports.OWN_LOG_GENERATIONS=3;const CHECK_EVERY_MS=3e4;exports.GATEWAY_LOG_ENV="LENSMCP_GATEWAY_LOG";const readEnv=i(t=>process.env[t],"readEnv");function shiftGenerations(t,e){try{fs.unlinkSync(`${t}.${e}`)}catch{}for(let s=e-1;s>=1;s--)try{fs.renameSync(`${t}.${s}`,`${t}.${s+1}`)}catch{}}l(shiftGenerations,"shiftGenerations"),i(shiftGenerations,"shiftGenerations");const sameFile=i((t,e)=>!!t&&t.isFile()&&t.dev===e.dev&&t.ino===e.ino,"sameFile"),statOr=i(t=>{try{return t()}catch{return}},"statOr");function rotateOwnLog(t,e={}){const s=e.fd??1,r=e.maxBytes??exports.OWN_LOG_MAX_BYTES,p=e.keep??exports.OWN_LOG_GENERATIONS,d=statOr(()=>fs.statSync(t));if(!d||d.size<=r)return!1;const u=sameFile(statOr(()=>fs.fstatSync(s)),d),a=e.launchedLog??readEnv(exports.GATEWAY_LOG_ENV);if(!u&&!(a&&sameFile(statOr(()=>fs.statSync(a)),d)))return!1;try{shiftGenerations(t,p),fs.copyFileSync(t,`${t}.1`);try{fs.chmodSync(`${t}.1`,384)}catch{}u?fs.ftruncateSync(s,0):fs.truncateSync(t,0)}catch{return!1}const f=u?n=>{try{fs.writeSync(s,`${n}
+`)}catch{}}:n=>{try{fs.appendFileSync(t,`${n}
+`)}catch{}};return(e.note??f)(`[gateway] ${path.basename(t)} passed ${Math.round(r/1048576)} MB \u2014 rotated; the lines before are in ${path.basename(t)}.1 (\u2026${p} kept)`),!0}l(rotateOwnLog,"rotateOwnLog"),i(rotateOwnLog,"rotateOwnLog");function startOwnLogRotation(t,e={}){const s=path.join(t,".lensmcp","gateway.log"),r=setInterval(()=>{activeTakeover?.reopenIfMoved(),rotateOwnLog(s,e)},e.everyMs??CHECK_EVERY_MS);return r.unref?.(),()=>clearInterval(r)}l(startOwnLogRotation,"startOwnLogRotation"),i(startOwnLogRotation,"startOwnLogRotation");const sizeText=i(t=>t>=1048576?`${(t/1048576).toFixed(1)} MB`:t>=1024?`${Math.round(t/1024)} KB`:`${t} B`,"sizeText");function droppedNote(t,e="gateway.log"){const s=t.code==="ENOSPC"?"ENOSPC \u2014 the disk was full":t.code;return`[gateway] ${new Date(t.lastAt).toISOString()} ${e}: ${t.lines} line${t.lines===1?"":"s"} (${sizeText(t.bytes)}) could not be written between ${new Date(t.firstAt).toISOString()} and ${new Date(t.lastAt).toISOString()} (${s}) \u2014 they are lost
+`}l(droppedNote,"droppedNote"),i(droppedNote,"droppedNote");function createLogWriter(t,e={}){const s=e.now??Date.now;let r,p=!0;const d=i(a=>{try{t(a);return}catch(f){return f??new Error("the write failed")}},"attempt"),u=i((a,f)=>{const n=s();let o=0;for(const y of a)y===10&&(o+=1);o=Math.max(1,o);const c=f?.code??"EIO";r=r?{lines:r.lines+o,bytes:r.bytes+a.length,firstAt:r.firstAt,lastAt:n,code:c}:{lines:o,bytes:a.length,firstAt:n,lastAt:n,code:c},p=!1},"count");return{write(a){const f=typeof a=="string"?Buffer.from(a):Buffer.from(a.buffer,a.byteOffset,a.byteLength);if(f.length===0)return!0;if(r){const o=Buffer.from(`${p?"":`
+`}${droppedNote(r,e.file)}`),c=d(o);if(c)return u(f,c),!1;r=void 0,p=!0}const n=d(f);return n?(u(f,n),!1):(p=f[f.length-1]===10,!0)},dropped:i(()=>r?{...r}:void 0,"dropped")}}l(createLogWriter,"createLogWriter"),i(createLogWriter,"createLogWriter");let activeTakeover;function gatewayLogTakeover(){return activeTakeover}l(gatewayLogTakeover,"gatewayLogTakeover"),i(gatewayLogTakeover,"gatewayLogTakeover");function takeOverGatewayLog(t={}){const e=t.file??readEnv(exports.GATEWAY_LOG_ENV);if(!e||!path.isAbsolute(e))return;const s=i(()=>{try{const n=fs.openSync(e,"a",384);try{fs.fchmodSync(n,384)}catch{}return n}catch{return}},"open");let r=s();if(r===void 0)return;const p=t.writeFd??((n,o)=>fs.writeSync(n,o)),d=createLogWriter(i(n=>{if(r===void 0&&(r=s()),r===void 0)throw Object.assign(new Error(`${e} cannot be opened`),{code:"ENOENT"});let o=n;for(;o.length>0;){const c=p(r,o);if(!(c>0))throw Object.assign(new Error("nothing was written"),{code:"EIO"});o=o.subarray(c)}},"append"),{...t.now?{now:t.now}:{},file:path.basename(e)}),u=t.streams??[process.stdout,process.stderr],a=u.map(n=>n.write);for(const n of u)n.write=((o,c,y)=>{const O=typeof c=="function"?c:typeof y=="function"?y:void 0,w=typeof o=="string"?Buffer.from(o,typeof c=="string"&&Buffer.isEncoding(c)?c:"utf8"):o instanceof Uint8Array?o:Buffer.from(String(o));return d.write(w),O&&process.nextTick(O),!0});const f={file:e,reopenIfMoved(){const n=statOr(()=>fs.statSync(e)),o=r===void 0?void 0:statOr(()=>fs.fstatSync(r));if(n&&sameFile(o,n))return!1;const c=s();if(c===void 0)return!1;const y=r;if(r=c,y!==void 0)try{fs.closeSync(y)}catch{}return!0},dropped:i(()=>d.dropped(),"dropped"),restore(){if(u.forEach((n,o)=>{n.write=a[o]}),r!==void 0)try{fs.closeSync(r)}catch{}r=void 0,activeTakeover===f&&(activeTakeover=void 0)}};return t.streams||(activeTakeover=f),f}l(takeOverGatewayLog,"takeOverGatewayLog"),i(takeOverGatewayLog,"takeOverGatewayLog");function linePrefixer(t){let e=!0;const s=`[${t}] `;return r=>{let p="";for(const d of r.split(/(?<=\n)/))d&&(p+=(e?s:"")+d,e=d.endsWith(`
+`));return p}}l(linePrefixer,"linePrefixer"),i(linePrefixer,"linePrefixer");

package/executors/gateway/runtime/server.js

@@ -1,3 +1,3 @@
-"use strict";var fe=Object.defineProperty;var T=(r,t)=>fe(r,"name",{value:t,configurable:!0});var ye=Object.defineProperty,o=T((r,t)=>ye(r,"name",{value:t,configurable:!0}),"o");Object.defineProperty(exports,"__esModule",{value:!0}),exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT=exports.GATEWAY_MAX_H2_STREAMS_DEFAULT=exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT=exports.GATEWAY_MAX_CONNECTIONS_DEFAULT=exports.BENIGN_SOCKET_ERRNOS=void 0,exports.buildEdgeVerifierPool=buildEdgeVerifierPool,exports.edgeKeyWait=edgeKeyWait,exports.withBuiltinHooks=withBuiltinHooks,exports.classifyClientError=classifyClientError,exports.createClientErrorLogGate=createClientErrorLogGate,exports.retirePreviousWorkspace=retirePreviousWorkspace,exports.resolveMaxConnections=resolveMaxConnections,exports.h2BudgetFromEnv=h2BudgetFromEnv,exports.createH2Governor=createH2Governor,exports.startGateway=startGateway;const tslib_1=require("tslib"),http=tslib_1.__importStar(require("node:http")),path=tslib_1.__importStar(require("node:path")),discovery_1=require("./discovery"),scope_1=require("./scope"),route_registry_1=require("./route-registry"),act_1=require("./act"),control_1=require("./control"),workspace_middleware_1=require("./workspace-middleware"),workspace_registry_1=require("./workspace-registry"),service_keys_1=require("./service-keys"),source_poller_1=require("./source-poller"),client_ip_1=require("../client-ip"),workspace_log_1=require("./workspace-log"),own_log_1=require("./own-log"),observability_1=require("./observability"),lifecycle_1=require("./lifecycle"),process_names_1=require("./process-names"),project_graph_store_1=require("./project-graph-store"),serve_launch_1=require("./serve-launch"),process_name_1=require("../../../process-name"),protocol_1=require("../../../shared-build/protocol"),lens_children_1=require("./lens-children"),proxy_1=require("./proxy"),auth_1=require("./auth"),hooks_1=require("./hooks"),handler_1=require("./handler"),upgrade_1=require("./upgrade"),jwks_verify_1=require("../jwks-verify"),manifest_1=require("../manifest"),edge_1=require("./edge"),warm_start_1=require("./warm-start"),types_1=require("./types");function insecureLocalFetch(r,t){return new Promise((i,c)=>{const a=(r.startsWith("https:")?require("node:https"):require("node:http")).request(r,{method:"GET",headers:t?.headers,rejectUnauthorized:!1},_=>{const y=[];_.on("data",h=>y.push(h)),_.on("error",c),_.on("end",()=>{const h=_.statusCode??0,A=Buffer.concat(y).toString("utf8");i({ok:h>=200&&h<300,status:h,json:o(async()=>JSON.parse(A),"json")})})});a.on("error",c),t?.signal?.addEventListener("abort",()=>a.destroy(new Error(`JWKS ${r} \u2192 aborted`)),{once:!0}),a.end()})}T(insecureLocalFetch,"insecureLocalFetch"),o(insecureLocalFetch,"insecureLocalFetch");function buildEdgeVerifierPool(r,t={}){const i=process.env.LENSMCP_GW_JWKS_URL,c=process.env.LENSMCP_GW_JWT_ISS,a=new Map;let _,y=!1;const h=o((s,l)=>{console.log(`[gateway] edge JWT: verifying via JWKS ${s}${l}`);const d=(0,jwks_verify_1.createJwksVerifier)({jwksUrl:s,...c?{issuer:c}:{},fetchImpl:t.fetchImpl??insecureLocalFetch});let E=!1;return d.warm({...t.warm,onError:o((v,x,b)=>{E=!0,x===1&&b!==void 0&&console.warn(`[gateway] edge JWKS ${s} not loaded yet (${v.message}) \u2014 retrying in the background`),b===void 0&&console.warn(`[gateway] edge JWKS ${s} still not loaded after ${x} attempts (${v.message}) \u2014 the first token that needs it fetches it`)},"onError")}).then(v=>{v&&E&&console.log(`[gateway] edge JWKS ${s} loaded`)}),d},"make"),A=o(s=>{let l=a.get(s);return l||(l=h(`https://auth.${s}/jwks.json`,` (auto-derived for apex ${s})`),a.set(s,l)),l},"forApex");return{for(s){if(i)return _??=h(i,"");const l=s.split(":")[0].toLowerCase();let d;for(const E of idpApexes(r()))(l===E||l.endsWith(`.${E}`))&&(!d||E.length>d.length)&&(d=E);return d?A(d):void 0},warm(){if(!y){if(i){_??=h(i,"");return}for(const s of idpApexes(r()))A(s)}},stop(){y=!0,_?.stop();for(const s of a.values())s.stop()}}}T(buildEdgeVerifierPool,"buildEdgeVerifierPool"),o(buildEdgeVerifierPool,"buildEdgeVerifierPool");function idpApexes(r){const t=new Set;for(const i of r){const c=i.host;typeof c!="string"||!/^auth\./.test(c)||c.startsWith("internal.")||t.add(c.slice(5))}return t}T(idpApexes,"idpApexes"),o(idpApexes,"idpApexes");function edgeKeyWait(r){return async t=>{const i=t.route;if(!i?.auth||i.internal)return;const{req:c}=t,a=Array.isArray(c.headers.authorization)?c.headers.authorization[0]:c.headers.authorization,_=String(a??"");if(!_.startsWith("Bearer "))return;const y=c.url??"/";if((0,manifest_1.authRuleFor)(i.auth,(0,edge_1.canonicalPath)(y)??y.split("?")[0],(c.method??"GET").toUpperCase()).mode!=="jwt")return;const h=String(c.headers.host??"").split(":")[0].toLowerCase();await r.for(h)?.awaitKey(_.slice(7))}}T(edgeKeyWait,"edgeKeyWait"),o(edgeKeyWait,"edgeKeyWait");function withBuiltinHooks(r,t){return{...r,beforeAuth:[...r.beforeAuth??[],t.keyWait],afterAuth:[t.middleware,...r.afterAuth??[]]}}T(withBuiltinHooks,"withBuiltinHooks"),o(withBuiltinHooks,"withBuiltinHooks"),exports.BENIGN_SOCKET_ERRNOS=new Set(["ECONNRESET","EPIPE","ETIMEDOUT","ECONNABORTED","ECANCELED"]);function classifyClientError(r,t){return r&&exports.BENIGN_SOCKET_ERRNOS.has(r)?{status:null,report:!1,cause:"client-abort"}:r==="HPE_INVALID_EOF_STATE"?{status:null,report:!1,cause:"truncated-at-eof"}:r==="ERR_HTTP_REQUEST_TIMEOUT"?{status:408,report:!0,cause:"request-timeout"}:r==="HPE_HEADER_OVERFLOW"?{status:431,report:!0,cause:"header-overflow"}:r?.startsWith("HPE_")?{status:400,report:!0,cause:"parse-error"}:{status:400,report:!0,cause:r??t??"unknown"}}T(classifyClientError,"classifyClientError"),o(classifyClientError,"classifyClientError");const CLIENT_ERROR_REASON={400:"Bad Request",408:"Request Timeout",431:"Request Header Fields Too Large"};function createClientErrorLogGate(r=1e4){const t=new Map;return i=>{const c=Date.now(),a=t.get(i)??0;return c-a<r?!1:(t.set(i,c),!0)}}T(createClientErrorLogGate,"createClientErrorLogGate"),o(createClientErrorLogGate,"createClientErrorLogGate");function guardInboundSocket(r){r.on("error",t=>{exports.BENIGN_SOCKET_ERRNOS.has(t.code??"")||console.warn(`[gateway] inbound socket error (${t.code??t.message}) \u2014 dropping this connection only`),r.destroy()})}T(guardInboundSocket,"guardInboundSocket"),o(guardInboundSocket,"guardInboundSocket");function retirePreviousWorkspace(r,t,i){const c=r.get(t);if(!c)return!1;r.delete(t),i==="re-registered"&&console.log(`[gateway] workspace '${t}' re-registered \u2014 retiring the children of its previous registration first.`);try{c()}catch(a){console.warn(`[gateway] could not fully retire the previous registration of '${t}' (${i}): ${a.message}`)}return!0}T(retirePreviousWorkspace,"retirePreviousWorkspace"),o(retirePreviousWorkspace,"retirePreviousWorkspace"),exports.GATEWAY_MAX_CONNECTIONS_DEFAULT=512,exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT=64,exports.GATEWAY_MAX_H2_STREAMS_DEFAULT=4096,exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT=3e5;function parseBudget(r,t){if(r===void 0||r.trim()==="")return t;const i=Number(r);return!Number.isFinite(i)||i<0?t:Math.floor(i)}T(parseBudget,"parseBudget"),o(parseBudget,"parseBudget");function resolveMaxConnections(r){return parseBudget(r,exports.GATEWAY_MAX_CONNECTIONS_DEFAULT)}T(resolveMaxConnections,"resolveMaxConnections"),o(resolveMaxConnections,"resolveMaxConnections");function h2BudgetFromEnv(r=process.env){return{maxSessions:parseBudget(r.LENSMCP_MAX_H2_SESSIONS,exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT),maxStreams:parseBudget(r.LENSMCP_MAX_H2_STREAMS,exports.GATEWAY_MAX_H2_STREAMS_DEFAULT),idleSessionMs:parseBudget(r.LENSMCP_H2_IDLE_MS,exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT)}}T(h2BudgetFromEnv,"h2BudgetFromEnv"),o(h2BudgetFromEnv,"h2BudgetFromEnv");function createH2Governor(r,t){const{NGHTTP2_ENHANCE_YOUR_CALM:i,NGHTTP2_REFUSED_STREAM:c}=require("node:http2").constants,a=new Map;let _=0;const y=o(s=>{s.idle&&(clearTimeout(s.idle),s.idle=void 0)},"disarm"),h=o((s,l)=>{y(l),!(r.idleSessionMs<=0||l.open>0)&&(l.idle=setTimeout(()=>{s.destroyed||s.closed||(t("session-reclaimed",{idleMs:r.idleSessionMs,sessions:a.size}),a.delete(s),s.close())},r.idleSessionMs),l.idle.unref?.())},"arm"),A=o(()=>{for(const[s,l]of a)if(!(l.open>0||s.destroyed))return y(l),a.delete(s),t("session-reclaimed",{reason:"cap",sessions:a.size+1,maxSessions:r.maxSessions}),s.close(),!0;return!1},"reclaimOldestIdle");return{admitSession(s){if(r.maxSessions>0&&a.size>=r.maxSessions&&!A()){t("session-refused",{sessions:a.size,maxSessions:r.maxSessions,streams:_});try{s.goaway(i)}catch{}try{s.close()}catch{}return!1}const l={open:0};return a.set(s,l),h(s,l),s.once("close",()=>{y(l),a.delete(s)}),!0},admitStream(s,l){const d=a.get(s);if(!d)return!0;if(r.maxStreams>0&&_>=r.maxStreams){t("stream-refused",{streams:_,maxStreams:r.maxStreams,sessions:a.size});try{l.close(c)}catch{}return!1}return _+=1,d.open+=1,y(d),l.once("close",()=>{_=Math.max(0,_-1),d.open=Math.max(0,d.open-1),d.open===0&&a.has(s)&&h(s,d)}),!0},dispose(){for(const s of a.values())y(s);a.clear()}}}T(createH2Governor,"createH2Governor"),o(createH2Governor,"createH2Governor");async function startGateway(r,t){const i=r.https!==!1,c=r.ports?.length?r.ports:[443],a=r.scanTtlMs??1500,_=r.startupTimeoutMs??12e4,y=r.sweepMs??1e4,h=r.trafficFlushMs??5e3;let A=!1;const s=process.env.LENSMCP_EVENT_FILE??path.join(t.root,".lensmcp","events.jsonl"),l=(0,own_log_1.startOwnLogRotation)(t.root),d=(0,scope_1.readLensScope)(t.root),E=d.key,v=d.sockNs!==E?d.sockNs:void 0,x=t.projectsConfigurations?.projects??{},b=(0,discovery_1.projectSourceRoots)(t.root,x,t.projectDependencies),{routes:j,services:P,instance:Q}=(0,discovery_1.discoverRoutes)(t.root,x,void 0,E,b,v,d.instance);if(j.length===0)throw new Error("[gateway] no project declares a `cluster` field \u2014 nothing to route.");const K=new route_registry_1.RouteRegistry;K.register({wsKey:E,...v?{sockNs:v}:{},root:t.root,routes:j,services:P,...Q?{instance:Q}:{}});const R=[];let Z=o(()=>{},"refreshCert"),U=o(()=>{},"warmEdgeVerifiers");const ee=o(()=>{(0,route_registry_1.rebuildRoutesInPlace)(R,K),Z(),U()},"rebuildRoutes");ee();const{serviceKeys:le,keyToProject:ue,keyToWorkspace:de}=(0,service_keys_1.loadServiceKeys)(t.root,P,E),pe=(0,client_ip_1.createClientIpResolver)({...(0,client_ip_1.clientIpOptionsFromEnv)(),...r.trustProxyHops!==void 0?{trustProxyHops:r.trustProxyHops}:{},...r.clientIpHeader?{clientIpHeader:r.clientIpHeader}:{},...r.clientIpTrustedProxies?{clientIpTrustedProxies:r.clientIpTrustedProxies}:{}}),g={root:t.root,wsKey:E,...v?{sockNs:v}:{},eventFile:s,routes:R,registry:K,rebuildRoutes:ee,services:P,serviceKeys:le,keyToProject:ue,keyToWorkspace:de,clientIp:pe,bodyStallMs:r.bodyStallMs??types_1.BODY_STALL_MS,scanTtlMs:a,startupTimeoutMs:_,sweepMs:y,trafficFlushMs:h,https:i,projectRoots:Object.fromEntries(Object.entries(t.projectsConfigurations?.projects??{}).map(([e,n])=>[e,path.resolve(t.root,n.root)])),projectSourceRoots:b,...t.projectGraphStore?{graphStores:(0,project_graph_store_1.createProjectGraphStores)(t.projectGraphStore)}:{},...r.sharedCompiler!==void 0?{sharedCompiler:r.sharedCompiler}:{},stopped:o(()=>A,"stopped")};(0,process_name_1.prelinkProcessNames)((0,process_names_1.gatewayProcessNames)(P,{runner:(0,serve_launch_1.serveRunnerEnabled)()&&!!t.projectGraphStore,sharedBuild:(0,protocol_1.sharedCompilerEnabled)(process.env,r.sharedCompiler),label:E||path.basename(t.root),lens:{key:E,dashboard:r.dashboard!==!1,mcp:r.mcp!==!1&&j.some(e=>!!e.lens)}}));const m=(0,observability_1.createObservability)(s,{trafficFlushMs:h}),_e=createClientErrorLogGate(),L=h2BudgetFromEnv(),ge=createClientErrorLogGate(3e4),D=createH2Governor(L,(e,n)=>{ge(e)&&console.warn(`[gateway] h2 aggregate budget: ${e} (${JSON.stringify(n)}). These are the FRONT DOOR's ceilings, not an upstream's \u2014 raise/disable with LENSMCP_MAX_H2_SESSIONS / LENSMCP_MAX_H2_STREAMS / LENSMCP_H2_IDLE_MS / LENSMCP_MAX_CONNECTIONS (0 = unbounded).`),m.emit(e==="session-reclaimed"?"info":"warning",`h2 aggregate budget: ${e}`,`gateway-h2-budget:${e}`,{kind:"h2-aggregate-budget",event:e,project:"gateway",...n})}),M=(0,lifecycle_1.createServiceLayer)(g,m),O=(0,lens_children_1.createLensChildren)(g,m,r),F=new Map;let re=o(()=>{},"ownBooted");const Se=new Promise(e=>re=e),te=(0,warm_start_1.createWorktreeWarmer)({rt:g,obs:m,ready:Se,ensureUp:o(e=>M.ensureUp(e),"ensureUp")});let B;if(r.middleware){const e=require(path.resolve(t.root,r.middleware));B=e.default??e}const Y=(0,workspace_middleware_1.createWorkspaceMiddleware)({ownerKey:E,...B?{owner:B}:{},policy:(0,workspace_middleware_1.guestMiddlewarePolicy)()}),I=process.env.LENSMCP_GW_ACT==="0"?void 0:(0,act_1.createActStore)({dir:path.join(path.dirname((0,control_1.controlSocketPath)()),"identities")});let $;try{$=(0,control_1.startControlServer)({rt:g,killService:o((e,n)=>M.killSpawned(e,n),"killService"),restartService:o((e,n)=>M.restartService(e,n),"restartService"),...I?{act:I}:{},onRegister:o(async e=>{retirePreviousWorkspace(F,e.wsKey,"re-registered"),Y.set(e.wsKey,e.root,e.middleware);const n=await O.hostWorkspaceDashboard(e),w=await O.hostWorkspaceApps(e);F.set(e.wsKey,()=>{try{n?.()}finally{w()}}),te.onRegister(e)},"onRegister"),onUnregister:o(e=>{te.onUnregister(e.wsKey),retirePreviousWorkspace(F,e.wsKey,"unregistered"),Y.drop(e.wsKey),M.stopBuildHost(e.root)},"onUnregister")})}catch(e){console.warn("[gateway] control plane failed to start (multi-workspace register disabled):",e.message)}const H=(0,proxy_1.createProxy)(g,m,M),W=buildEdgeVerifierPool(()=>g.routes),se=(0,auth_1.createEdgeAuth)(g,m,W,I),he=o(e=>{const n=e.route?.wsKey,w=Y.for(n);if(w)try{const u=n?g.registry.get(n):void 0;w(e.req,{...n?{wsKey:n}:{},...u?{root:u.root}:{},env:u?.instance?.env??{}})}catch(u){m.traceStep(e.trace,"auth",{mode:"middleware",ok:!1,reason:u.message}),e.route&&m.finishTrace(e.trace,e.route.project,401),e.sendError(401,"unauthorized",`Gateway middleware rejected the request: ${u.message}`)}},"builtinMiddleware"),oe=(0,hooks_1.createHooks)(withBuiltinHooks(r.hooks??{},{keyWait:edgeKeyWait(W),middleware:he})),ne=(0,handler_1.createHandler)({rt:g,obs:m,auth:se,proxy:H,hooks:oe}),ae=(0,upgrade_1.createUpgrade)({rt:g,auth:se,proxy:H,hooks:oe,obs:m}),Ee=m.startEdgeFlusher(),me=M.startSweeper(),ie=(0,source_poller_1.createSourcePoller)({services:o(()=>g.registry.allServices(),"services"),serviceKey:o(e=>g.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)],"serviceKey"),emit:o((e,n,w,u,S)=>m.emit(n,w,u,{wsKey:e.wsKey,...S},void 0,e.wsKey===g.wsKey?void 0:path.join(e.root,".lensmcp","events.jsonl")),"emit"),scanTtlMs:a,stopped:o(()=>A,"stopped")});(0,source_poller_1.sourcesEnabled)()&&ie.start(),await O.bootSingletonsAndApps();const ce=P.filter(e=>e.decl.eager).map(e=>M.ensureUp(e));Promise.allSettled(ce).then(()=>setTimeout(re,ce.length?2e4:0).unref());let k,X,q;if(i){const e=require("../../../basic-ssl"),n=(0,service_keys_1.gatewayCacheDir)(t.root);q=o(()=>e.getCertificateSync(n,"gateway",g.routes.map(w=>w.host).filter(w=>!!w)),"mintCert"),k=q(),X=e.caCertPath()}const J=[],G=[],z=r.maxConnections??resolveMaxConnections(process.env.LENSMCP_MAX_CONNECTIONS);for(const e of c){const n=k?require("node:http2").createSecureServer({key:k,cert:k,allowHTTP1:!0,maxSessionMemory:512,settings:{maxConcurrentStreams:512},peerMaxConcurrentStreams:512,streamResetBurst:r.h2ResetBurst??types_1.GATEWAY_H2_RESET_BURST,streamResetRate:r.h2ResetRate??types_1.GATEWAY_H2_RESET_RATE},(u,S)=>ne(u,S)):http.createServer(ne);n.on("upgrade",ae),n.on("error",u=>console.error(`[gateway] port ${e}: ${u.code}`)),n.on("connection",guardInboundSocket),n.on("secureConnection",guardInboundSocket),n.on("tlsClientError",(u,S)=>S.destroy()),n.on("clientError",(u,S)=>{const p=u,f=classifyClientError(p.code,p.message);if(f.report&&_e(f.cause)){const N=S.remoteAddress??"unknown",C=f.status===null?"dropped":String(f.status);console.warn(`[gateway] inbound H1 clientError from ${N}: ${f.cause} (${p.code??p.message}${p.bytesParsed!==void 0?`, bytesParsed=${p.bytesParsed}`:""}) \u2192 ${C}. This status is the GATEWAY's, not the upstream service's.`),m.emit("warning",`inbound clientError: ${f.cause} \u2192 ${C}`,`gateway-client-error:${f.cause}`,{kind:"gateway-client-error",cause:f.cause,code:p.code??null,status:f.status,peer:N,bytesParsed:p.bytesParsed??null})}f.status!==null&&S.writable&&S.end(`HTTP/1.1 ${f.status} ${CLIENT_ERROR_REASON[f.status]}\r
+"use strict";var fe=Object.defineProperty;var T=(r,t)=>fe(r,"name",{value:t,configurable:!0});var ye=Object.defineProperty,o=T((r,t)=>ye(r,"name",{value:t,configurable:!0}),"o");Object.defineProperty(exports,"__esModule",{value:!0}),exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT=exports.GATEWAY_MAX_H2_STREAMS_DEFAULT=exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT=exports.GATEWAY_MAX_CONNECTIONS_DEFAULT=exports.BENIGN_SOCKET_ERRNOS=void 0,exports.buildEdgeVerifierPool=buildEdgeVerifierPool,exports.edgeKeyWait=edgeKeyWait,exports.withBuiltinHooks=withBuiltinHooks,exports.classifyClientError=classifyClientError,exports.createClientErrorLogGate=createClientErrorLogGate,exports.retirePreviousWorkspace=retirePreviousWorkspace,exports.resolveMaxConnections=resolveMaxConnections,exports.h2BudgetFromEnv=h2BudgetFromEnv,exports.createH2Governor=createH2Governor,exports.startGateway=startGateway;const tslib_1=require("tslib"),http=tslib_1.__importStar(require("node:http")),path=tslib_1.__importStar(require("node:path")),discovery_1=require("./discovery"),scope_1=require("./scope"),route_registry_1=require("./route-registry"),act_1=require("./act"),control_1=require("./control"),workspace_middleware_1=require("./workspace-middleware"),workspace_registry_1=require("./workspace-registry"),service_keys_1=require("./service-keys"),source_poller_1=require("./source-poller"),client_ip_1=require("../client-ip"),workspace_log_1=require("./workspace-log"),own_log_1=require("./own-log"),observability_1=require("./observability"),lifecycle_1=require("./lifecycle"),process_names_1=require("./process-names"),project_graph_store_1=require("./project-graph-store"),serve_launch_1=require("./serve-launch"),process_name_1=require("../../../process-name"),protocol_1=require("../../../shared-build/protocol"),lens_children_1=require("./lens-children"),proxy_1=require("./proxy"),auth_1=require("./auth"),hooks_1=require("./hooks"),handler_1=require("./handler"),upgrade_1=require("./upgrade"),jwks_verify_1=require("../jwks-verify"),manifest_1=require("../manifest"),edge_1=require("./edge"),warm_start_1=require("./warm-start"),types_1=require("./types");function insecureLocalFetch(r,t){return new Promise((i,c)=>{const a=(r.startsWith("https:")?require("node:https"):require("node:http")).request(r,{method:"GET",headers:t?.headers,rejectUnauthorized:!1},_=>{const y=[];_.on("data",h=>y.push(h)),_.on("error",c),_.on("end",()=>{const h=_.statusCode??0,v=Buffer.concat(y).toString("utf8");i({ok:h>=200&&h<300,status:h,json:o(async()=>JSON.parse(v),"json")})})});a.on("error",c),t?.signal?.addEventListener("abort",()=>a.destroy(new Error(`JWKS ${r} \u2192 aborted`)),{once:!0}),a.end()})}T(insecureLocalFetch,"insecureLocalFetch"),o(insecureLocalFetch,"insecureLocalFetch");function buildEdgeVerifierPool(r,t={}){const i=process.env.LENSMCP_GW_JWKS_URL,c=process.env.LENSMCP_GW_JWT_ISS,a=new Map;let _,y=!1;const h=o((s,l)=>{console.log(`[gateway] edge JWT: verifying via JWKS ${s}${l}`);const d=(0,jwks_verify_1.createJwksVerifier)({jwksUrl:s,...c?{issuer:c}:{},fetchImpl:t.fetchImpl??insecureLocalFetch});let E=!1;return d.warm({...t.warm,onError:o((M,x,b)=>{E=!0,x===1&&b!==void 0&&console.warn(`[gateway] edge JWKS ${s} not loaded yet (${M.message}) \u2014 retrying in the background`),b===void 0&&console.warn(`[gateway] edge JWKS ${s} still not loaded after ${x} attempts (${M.message}) \u2014 the first token that needs it fetches it`)},"onError")}).then(M=>{M&&E&&console.log(`[gateway] edge JWKS ${s} loaded`)}),d},"make"),v=o(s=>{let l=a.get(s);return l||(l=h(`https://auth.${s}/jwks.json`,` (auto-derived for apex ${s})`),a.set(s,l)),l},"forApex");return{for(s){if(i)return _??=h(i,"");const l=s.split(":")[0].toLowerCase();let d;for(const E of idpApexes(r()))(l===E||l.endsWith(`.${E}`))&&(!d||E.length>d.length)&&(d=E);return d?v(d):void 0},warm(){if(!y){if(i){_??=h(i,"");return}for(const s of idpApexes(r()))v(s)}},stop(){y=!0,_?.stop();for(const s of a.values())s.stop()}}}T(buildEdgeVerifierPool,"buildEdgeVerifierPool"),o(buildEdgeVerifierPool,"buildEdgeVerifierPool");function idpApexes(r){const t=new Set;for(const i of r){const c=i.host;typeof c!="string"||!/^auth\./.test(c)||c.startsWith("internal.")||t.add(c.slice(5))}return t}T(idpApexes,"idpApexes"),o(idpApexes,"idpApexes");function edgeKeyWait(r){return async t=>{const i=t.route;if(!i?.auth||i.internal)return;const{req:c}=t,a=Array.isArray(c.headers.authorization)?c.headers.authorization[0]:c.headers.authorization,_=String(a??"");if(!_.startsWith("Bearer "))return;const y=c.url??"/";if((0,manifest_1.authRuleFor)(i.auth,(0,edge_1.canonicalPath)(y)??y.split("?")[0],(c.method??"GET").toUpperCase()).mode!=="jwt")return;const h=String(c.headers.host??"").split(":")[0].toLowerCase();await r.for(h)?.awaitKey(_.slice(7))}}T(edgeKeyWait,"edgeKeyWait"),o(edgeKeyWait,"edgeKeyWait");function withBuiltinHooks(r,t){return{...r,beforeAuth:[...r.beforeAuth??[],t.keyWait],afterAuth:[t.middleware,...r.afterAuth??[]]}}T(withBuiltinHooks,"withBuiltinHooks"),o(withBuiltinHooks,"withBuiltinHooks"),exports.BENIGN_SOCKET_ERRNOS=new Set(["ECONNRESET","EPIPE","ETIMEDOUT","ECONNABORTED","ECANCELED"]);function classifyClientError(r,t){return r&&exports.BENIGN_SOCKET_ERRNOS.has(r)?{status:null,report:!1,cause:"client-abort"}:r==="HPE_INVALID_EOF_STATE"?{status:null,report:!1,cause:"truncated-at-eof"}:r==="ERR_HTTP_REQUEST_TIMEOUT"?{status:408,report:!0,cause:"request-timeout"}:r==="HPE_HEADER_OVERFLOW"?{status:431,report:!0,cause:"header-overflow"}:r?.startsWith("HPE_")?{status:400,report:!0,cause:"parse-error"}:{status:400,report:!0,cause:r??t??"unknown"}}T(classifyClientError,"classifyClientError"),o(classifyClientError,"classifyClientError");const CLIENT_ERROR_REASON={400:"Bad Request",408:"Request Timeout",431:"Request Header Fields Too Large"};function createClientErrorLogGate(r=1e4){const t=new Map;return i=>{const c=Date.now(),a=t.get(i)??0;return c-a<r?!1:(t.set(i,c),!0)}}T(createClientErrorLogGate,"createClientErrorLogGate"),o(createClientErrorLogGate,"createClientErrorLogGate");function guardInboundSocket(r){r.on("error",t=>{exports.BENIGN_SOCKET_ERRNOS.has(t.code??"")||console.warn(`[gateway] inbound socket error (${t.code??t.message}) \u2014 dropping this connection only`),r.destroy()})}T(guardInboundSocket,"guardInboundSocket"),o(guardInboundSocket,"guardInboundSocket");function retirePreviousWorkspace(r,t,i){const c=r.get(t);if(!c)return!1;r.delete(t),i==="re-registered"&&console.log(`[gateway] workspace '${t}' re-registered \u2014 retiring the children of its previous registration first.`);try{c()}catch(a){console.warn(`[gateway] could not fully retire the previous registration of '${t}' (${i}): ${a.message}`)}return!0}T(retirePreviousWorkspace,"retirePreviousWorkspace"),o(retirePreviousWorkspace,"retirePreviousWorkspace"),exports.GATEWAY_MAX_CONNECTIONS_DEFAULT=512,exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT=64,exports.GATEWAY_MAX_H2_STREAMS_DEFAULT=4096,exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT=3e5;function parseBudget(r,t){if(r===void 0||r.trim()==="")return t;const i=Number(r);return!Number.isFinite(i)||i<0?t:Math.floor(i)}T(parseBudget,"parseBudget"),o(parseBudget,"parseBudget");function resolveMaxConnections(r){return parseBudget(r,exports.GATEWAY_MAX_CONNECTIONS_DEFAULT)}T(resolveMaxConnections,"resolveMaxConnections"),o(resolveMaxConnections,"resolveMaxConnections");function h2BudgetFromEnv(r=process.env){return{maxSessions:parseBudget(r.LENSMCP_MAX_H2_SESSIONS,exports.GATEWAY_MAX_H2_SESSIONS_DEFAULT),maxStreams:parseBudget(r.LENSMCP_MAX_H2_STREAMS,exports.GATEWAY_MAX_H2_STREAMS_DEFAULT),idleSessionMs:parseBudget(r.LENSMCP_H2_IDLE_MS,exports.GATEWAY_H2_IDLE_SESSION_MS_DEFAULT)}}T(h2BudgetFromEnv,"h2BudgetFromEnv"),o(h2BudgetFromEnv,"h2BudgetFromEnv");function createH2Governor(r,t){const{NGHTTP2_ENHANCE_YOUR_CALM:i,NGHTTP2_REFUSED_STREAM:c}=require("node:http2").constants,a=new Map;let _=0;const y=o(s=>{s.idle&&(clearTimeout(s.idle),s.idle=void 0)},"disarm"),h=o((s,l)=>{y(l),!(r.idleSessionMs<=0||l.open>0)&&(l.idle=setTimeout(()=>{s.destroyed||s.closed||(t("session-reclaimed",{idleMs:r.idleSessionMs,sessions:a.size}),a.delete(s),s.close())},r.idleSessionMs),l.idle.unref?.())},"arm"),v=o(()=>{for(const[s,l]of a)if(!(l.open>0||s.destroyed))return y(l),a.delete(s),t("session-reclaimed",{reason:"cap",sessions:a.size+1,maxSessions:r.maxSessions}),s.close(),!0;return!1},"reclaimOldestIdle");return{admitSession(s){if(r.maxSessions>0&&a.size>=r.maxSessions&&!v()){t("session-refused",{sessions:a.size,maxSessions:r.maxSessions,streams:_});try{s.goaway(i)}catch{}try{s.close()}catch{}return!1}const l={open:0};return a.set(s,l),h(s,l),s.once("close",()=>{y(l),a.delete(s)}),!0},admitStream(s,l){const d=a.get(s);if(!d)return!0;if(r.maxStreams>0&&_>=r.maxStreams){t("stream-refused",{streams:_,maxStreams:r.maxStreams,sessions:a.size});try{l.close(c)}catch{}return!1}return _+=1,d.open+=1,y(d),l.once("close",()=>{_=Math.max(0,_-1),d.open=Math.max(0,d.open-1),d.open===0&&a.has(s)&&h(s,d)}),!0},dispose(){for(const s of a.values())y(s);a.clear()}}}T(createH2Governor,"createH2Governor"),o(createH2Governor,"createH2Governor");async function startGateway(r,t){const i=r.https!==!1,c=r.ports?.length?r.ports:[443],a=r.scanTtlMs??1500,_=r.startupTimeoutMs??12e4,y=r.sweepMs??1e4,h=r.trafficFlushMs??5e3;let v=!1;const s=process.env.LENSMCP_EVENT_FILE??path.join(t.root,".lensmcp","events.jsonl"),l=(0,own_log_1.startOwnLogRotation)(t.root),d=(0,scope_1.readLensScope)(t.root),E=d.key,M=d.sockNs!==E?d.sockNs:void 0,x=t.projectsConfigurations?.projects??{},b=(0,discovery_1.projectSourceRoots)(t.root,x,t.projectDependencies),{routes:j,services:P,instance:Q}=(0,discovery_1.discoverRoutes)(t.root,x,void 0,E,b,M,d.instance);if(j.length===0)throw new Error("[gateway] no project declares a `cluster` field \u2014 nothing to route.");const K=new route_registry_1.RouteRegistry;K.register({wsKey:E,...M?{sockNs:M}:{},root:t.root,routes:j,services:P,...Q?{instance:Q}:{}});const R=[];let Z=o(()=>{},"refreshCert"),U=o(()=>{},"warmEdgeVerifiers");const ee=o(()=>{(0,route_registry_1.rebuildRoutesInPlace)(R,K),Z(),U()},"rebuildRoutes");ee();const{serviceKeys:le,keyToProject:ue,keyToWorkspace:de}=(0,service_keys_1.loadServiceKeys)(t.root,P,E),pe=(0,client_ip_1.createClientIpResolver)({...(0,client_ip_1.clientIpOptionsFromEnv)(),...r.trustProxyHops!==void 0?{trustProxyHops:r.trustProxyHops}:{},...r.clientIpHeader?{clientIpHeader:r.clientIpHeader}:{},...r.clientIpTrustedProxies?{clientIpTrustedProxies:r.clientIpTrustedProxies}:{}}),g={root:t.root,wsKey:E,...M?{sockNs:M}:{},eventFile:s,routes:R,registry:K,rebuildRoutes:ee,services:P,serviceKeys:le,keyToProject:ue,keyToWorkspace:de,clientIp:pe,bodyStallMs:r.bodyStallMs??types_1.BODY_STALL_MS,scanTtlMs:a,startupTimeoutMs:_,sweepMs:y,trafficFlushMs:h,https:i,projectRoots:Object.fromEntries(Object.entries(t.projectsConfigurations?.projects??{}).map(([e,n])=>[e,path.resolve(t.root,n.root)])),projectSourceRoots:b,...t.projectGraphStore?{graphStores:(0,project_graph_store_1.createProjectGraphStores)(t.projectGraphStore)}:{},...r.sharedCompiler!==void 0?{sharedCompiler:r.sharedCompiler}:{},stopped:o(()=>v,"stopped")};(0,process_name_1.prelinkProcessNames)((0,process_names_1.gatewayProcessNames)(P,{runner:(0,serve_launch_1.serveRunnerEnabled)()&&!!t.projectGraphStore,sharedBuild:(0,protocol_1.sharedCompilerEnabled)(process.env,r.sharedCompiler),label:E||path.basename(t.root),lens:{key:E,dashboard:r.dashboard!==!1,mcp:r.mcp!==!1&&j.some(e=>!!e.lens)}}));const m=(0,observability_1.createObservability)(s,{trafficFlushMs:h}),_e=createClientErrorLogGate(),L=h2BudgetFromEnv(),ge=createClientErrorLogGate(3e4),D=createH2Governor(L,(e,n)=>{ge(e)&&console.warn(`[gateway] h2 aggregate budget: ${e} (${JSON.stringify(n)}). These are the FRONT DOOR's ceilings, not an upstream's \u2014 raise/disable with LENSMCP_MAX_H2_SESSIONS / LENSMCP_MAX_H2_STREAMS / LENSMCP_H2_IDLE_MS / LENSMCP_MAX_CONNECTIONS (0 = unbounded).`),m.emit(e==="session-reclaimed"?"info":"warning",`h2 aggregate budget: ${e}`,`gateway-h2-budget:${e}`,{kind:"h2-aggregate-budget",event:e,project:"gateway",...n})}),A=(0,lifecycle_1.createServiceLayer)(g,m),O=(0,lens_children_1.createLensChildren)(g,m,r),F=new Map;let re=o(()=>{},"ownBooted");const Se=new Promise(e=>re=e),te=(0,warm_start_1.createWorktreeWarmer)({rt:g,obs:m,ready:Se,ensureUp:o(e=>A.ensureUp(e),"ensureUp")});let B;if(r.middleware){const e=require(path.resolve(t.root,r.middleware));B=e.default??e}const Y=(0,workspace_middleware_1.createWorkspaceMiddleware)({ownerKey:E,...B?{owner:B}:{},policy:(0,workspace_middleware_1.guestMiddlewarePolicy)()}),I=process.env.LENSMCP_GW_ACT==="0"?void 0:(0,act_1.createActStore)({dir:path.join(path.dirname((0,control_1.controlSocketPath)()),"identities")});let $;try{$=(0,control_1.startControlServer)({rt:g,killService:o((e,n)=>A.killSpawned(e,n),"killService"),restartService:o((e,n)=>A.restartService(e,n),"restartService"),ensureService:o(e=>A.ensureUp(e),"ensureService"),...I?{act:I}:{},onRegister:o(async e=>{retirePreviousWorkspace(F,e.wsKey,"re-registered"),Y.set(e.wsKey,e.root,e.middleware);const n=await O.hostWorkspaceDashboard(e),w=await O.hostWorkspaceApps(e);F.set(e.wsKey,()=>{try{n?.()}finally{w()}}),te.onRegister(e)},"onRegister"),onUnregister:o(e=>{te.onUnregister(e.wsKey),retirePreviousWorkspace(F,e.wsKey,"unregistered"),Y.drop(e.wsKey),A.stopBuildHost(e.root)},"onUnregister")})}catch(e){console.warn("[gateway] control plane failed to start (multi-workspace register disabled):",e.message)}const H=(0,proxy_1.createProxy)(g,m,A),W=buildEdgeVerifierPool(()=>g.routes),se=(0,auth_1.createEdgeAuth)(g,m,W,I),he=o(e=>{const n=e.route?.wsKey,w=Y.for(n);if(w)try{const u=n?g.registry.get(n):void 0;w(e.req,{...n?{wsKey:n}:{},...u?{root:u.root}:{},env:u?.instance?.env??{}})}catch(u){m.traceStep(e.trace,"auth",{mode:"middleware",ok:!1,reason:u.message}),e.route&&m.finishTrace(e.trace,e.route.project,401),e.sendError(401,"unauthorized",`Gateway middleware rejected the request: ${u.message}`)}},"builtinMiddleware"),oe=(0,hooks_1.createHooks)(withBuiltinHooks(r.hooks??{},{keyWait:edgeKeyWait(W),middleware:he})),ne=(0,handler_1.createHandler)({rt:g,obs:m,auth:se,proxy:H,hooks:oe}),ae=(0,upgrade_1.createUpgrade)({rt:g,auth:se,proxy:H,hooks:oe,obs:m}),Ee=m.startEdgeFlusher(),me=A.startSweeper(),ie=(0,source_poller_1.createSourcePoller)({services:o(()=>g.registry.allServices(),"services"),serviceKey:o(e=>g.serviceKeys[(0,service_keys_1.serviceKeyId)(e.wsKey,e.project)],"serviceKey"),emit:o((e,n,w,u,S)=>m.emit(n,w,u,{wsKey:e.wsKey,...S},void 0,e.wsKey===g.wsKey?void 0:path.join(e.root,".lensmcp","events.jsonl")),"emit"),scanTtlMs:a,stopped:o(()=>v,"stopped")});(0,source_poller_1.sourcesEnabled)()&&ie.start(),await O.bootSingletonsAndApps();const ce=P.filter(e=>e.decl.eager).map(e=>A.ensureUp(e));Promise.allSettled(ce).then(()=>setTimeout(re,ce.length?2e4:0).unref());let k,X,q;if(i){const e=require("../../../basic-ssl"),n=(0,service_keys_1.gatewayCacheDir)(t.root);q=o(()=>e.getCertificateSync(n,"gateway",g.routes.map(w=>w.host).filter(w=>!!w)),"mintCert"),k=q(),X=e.caCertPath()}const J=[],G=[],z=r.maxConnections??resolveMaxConnections(process.env.LENSMCP_MAX_CONNECTIONS);for(const e of c){const n=k?require("node:http2").createSecureServer({key:k,cert:k,allowHTTP1:!0,maxSessionMemory:512,settings:{maxConcurrentStreams:512},peerMaxConcurrentStreams:512,streamResetBurst:r.h2ResetBurst??types_1.GATEWAY_H2_RESET_BURST,streamResetRate:r.h2ResetRate??types_1.GATEWAY_H2_RESET_RATE},(u,S)=>ne(u,S)):http.createServer(ne);n.on("upgrade",ae),n.on("error",u=>console.error(`[gateway] port ${e}: ${u.code}`)),n.on("connection",guardInboundSocket),n.on("secureConnection",guardInboundSocket),n.on("tlsClientError",(u,S)=>S.destroy()),n.on("clientError",(u,S)=>{const p=u,f=classifyClientError(p.code,p.message);if(f.report&&_e(f.cause)){const N=S.remoteAddress??"unknown",C=f.status===null?"dropped":String(f.status);console.warn(`[gateway] inbound H1 clientError from ${N}: ${f.cause} (${p.code??p.message}${p.bytesParsed!==void 0?`, bytesParsed=${p.bytesParsed}`:""}) \u2192 ${C}. This status is the GATEWAY's, not the upstream service's.`),m.emit("warning",`inbound clientError: ${f.cause} \u2192 ${C}`,`gateway-client-error:${f.cause}`,{kind:"gateway-client-error",cause:f.cause,code:p.code??null,status:f.status,peer:N,bytesParsed:p.bytesParsed??null})}f.status!==null&&S.writable&&S.end(`HTTP/1.1 ${f.status} ${CLIENT_ERROR_REASON[f.status]}\r
 \r
-`),S.destroy()}),n.keepAliveTimeout=types_1.GATEWAY_KEEPALIVE_TIMEOUT_MS,n.headersTimeout=r.headersTimeoutMs??types_1.GATEWAY_HEADERS_TIMEOUT_MS,z>0&&(n.maxConnections=z);const w=r.connectionsCheckingIntervalMs??types_1.GATEWAY_CONN_CHECK_MS;if(w>0&&(n.connectionsCheckingInterval=w),k!=null){const u=n,{NGHTTP2_CANCEL:S}=require("node:http2").constants;u.on("session",p=>{if(!D.admitSession(p)){p.on("error",()=>{});return}let f=0,N=0;p.on("stream",C=>{D.admitStream(p,C)&&C.once("close",()=>{p.destroyed?N+=1:C.rstCode===S&&(f+=1)})}),p.once("close",()=>{N!==0&&(console.warn(`[gateway] h2 session closed with ${N} stream(s) in flight after ${f} client reset(s) \u2014 every in-flight fetch on that connection failed at once (ERR_HTTP2_PROTOCOL_ERROR in the browser). Reset budget: LENSMCP_H2_RESET_BURST/LENSMCP_H2_RESET_RATE (issues/closed/foodguard-14).`),m.emit("warning",`h2 session lost with ${N} stream(s) in flight`,"gateway-h2-session-lost",{kind:"h2-session-lost",project:"gateway",inflight:N,clientResets:f}))}),p.on("error",C=>{const we=C.code??C.message;console.warn(`[gateway] h2 session error (${we}) \u2014 dropping this session only`),p.destroyed||p.destroy()})}),u.on("sessionError",p=>{console.warn(`[gateway] h2 sessionError: ${p.code??p.message}`)})}await new Promise(u=>{n.listen(e,()=>{const S=n.address()?.port??e;G.push(S),console.log(`[gateway] Listening on ${k?"https":"http"}://localhost:${S}`),u()})}),J.push(n)}q&&(Z=o(()=>{const e=q();for(const n of J)n.setSecureContext?.({key:e,cert:e})},"refreshCert")),console.log("[gateway] Routes:");for(const e of R){const n=e.pool?`pods@${e.pool.dir}`:e.target;console.log(`[gateway]   ${e.host??"(default)"} \u2192 ${n}${e.prependPrefix?`  (+${e.prependPrefix})`:""}${e.internal?"  [internal: no middleware]":""}  [${e.project}]`)}X&&console.log(`[gateway] CA: ${X} \u2014 one-time setup via the trust executor.`),U=o(()=>W.warm(),"warmEdgeVerifiers"),U();const V=o(e=>e>0?String(e):"unbounded","unbounded");return console.log(`[gateway] front-door budget: \u2264${V(z)} connections, \u2264${V(L.maxSessions)} h2 sessions (each \u2264512 MB session memory, \u2264512 streams), \u2264${V(L.maxStreams)} open streams in total, idle h2 session reclaimed after ${L.idleSessionMs>0?`${Math.round(L.idleSessionMs/1e3)}s`:"never"}. Override: LENSMCP_MAX_CONNECTIONS / LENSMCP_MAX_H2_SESSIONS / LENSMCP_MAX_H2_STREAMS / LENSMCP_H2_IDLE_MS (0 = unbounded).`),m.emit("info",`gateway up: ${R.length} routes on ${G.join("/")}`,"cluster-gateway",{kind:"gateway-up",project:"gateway",ports:G,routes:R.map(e=>({host:e.host??"(default)",project:e.project,mode:e.pool?"pods":"tcp",internal:!!e.internal,prependPrefix:e.prependPrefix}))}),$&&process.env.LENSMCP_GUEST_REPLAY!=="0"&&$.replay().catch(e=>console.warn("[gateway] guest replay failed:",e.message)),{ports:G,routes:R,services:P,stop:o(async()=>{if(!A){A=!0,clearInterval(Ee),clearInterval(me),ie.stop(),m.emit("info","gateway down","cluster-gateway",{kind:"gateway-down",project:"gateway"}),W.stop(),$?.close(),I?.flush(),(0,workspace_registry_1.unregisterWorkspace)(E);for(const e of P)M.killSpawned(e,"gateway shutdown");M.stopRetiringTrees(),await M.stopBuildHost(),await O.reapAll(),ae.closeTunnels(),await Promise.all(J.map(e=>new Promise(n=>{let w=!1;const u=o(()=>{w||(w=!0,n())},"done"),S=e;e.close(()=>u());try{S.closeIdleConnections?.()}catch{}setTimeout(()=>{try{S.closeAllConnections?.()}catch{}u()},types_1.GATEWAY_STOP_GRACE_MS).unref?.()}))),H.closeProxy(),H.destroyAgents(),D.dispose(),(0,workspace_log_1.closeWorkspaceLogs)(),l()}},"stop"),...g.lensSupervisor?{lensSupervisor:g.lensSupervisor}:{},...g.graphStores?{graphStores:g.graphStores}:{}}}T(startGateway,"startGateway"),o(startGateway,"startGateway");
+`),S.destroy()}),n.keepAliveTimeout=types_1.GATEWAY_KEEPALIVE_TIMEOUT_MS,n.headersTimeout=r.headersTimeoutMs??types_1.GATEWAY_HEADERS_TIMEOUT_MS,z>0&&(n.maxConnections=z);const w=r.connectionsCheckingIntervalMs??types_1.GATEWAY_CONN_CHECK_MS;if(w>0&&(n.connectionsCheckingInterval=w),k!=null){const u=n,{NGHTTP2_CANCEL:S}=require("node:http2").constants;u.on("session",p=>{if(!D.admitSession(p)){p.on("error",()=>{});return}let f=0,N=0;p.on("stream",C=>{D.admitStream(p,C)&&C.once("close",()=>{p.destroyed?N+=1:C.rstCode===S&&(f+=1)})}),p.once("close",()=>{N!==0&&(console.warn(`[gateway] h2 session closed with ${N} stream(s) in flight after ${f} client reset(s) \u2014 every in-flight fetch on that connection failed at once (ERR_HTTP2_PROTOCOL_ERROR in the browser). Reset budget: LENSMCP_H2_RESET_BURST/LENSMCP_H2_RESET_RATE (issues/closed/foodguard-14).`),m.emit("warning",`h2 session lost with ${N} stream(s) in flight`,"gateway-h2-session-lost",{kind:"h2-session-lost",project:"gateway",inflight:N,clientResets:f}))}),p.on("error",C=>{const we=C.code??C.message;console.warn(`[gateway] h2 session error (${we}) \u2014 dropping this session only`),p.destroyed||p.destroy()})}),u.on("sessionError",p=>{console.warn(`[gateway] h2 sessionError: ${p.code??p.message}`)})}await new Promise(u=>{n.listen(e,()=>{const S=n.address()?.port??e;G.push(S),console.log(`[gateway] Listening on ${k?"https":"http"}://localhost:${S}`),u()})}),J.push(n)}q&&(Z=o(()=>{const e=q();for(const n of J)n.setSecureContext?.({key:e,cert:e})},"refreshCert")),console.log("[gateway] Routes:");for(const e of R){const n=e.pool?`pods@${e.pool.dir}`:e.target;console.log(`[gateway]   ${e.host??"(default)"} \u2192 ${n}${e.prependPrefix?`  (+${e.prependPrefix})`:""}${e.internal?"  [internal: no middleware]":""}  [${e.project}]`)}X&&console.log(`[gateway] CA: ${X} \u2014 one-time setup via the trust executor.`),U=o(()=>W.warm(),"warmEdgeVerifiers"),U();const V=o(e=>e>0?String(e):"unbounded","unbounded");return console.log(`[gateway] front-door budget: \u2264${V(z)} connections, \u2264${V(L.maxSessions)} h2 sessions (each \u2264512 MB session memory, \u2264512 streams), \u2264${V(L.maxStreams)} open streams in total, idle h2 session reclaimed after ${L.idleSessionMs>0?`${Math.round(L.idleSessionMs/1e3)}s`:"never"}. Override: LENSMCP_MAX_CONNECTIONS / LENSMCP_MAX_H2_SESSIONS / LENSMCP_MAX_H2_STREAMS / LENSMCP_H2_IDLE_MS (0 = unbounded).`),m.emit("info",`gateway up: ${R.length} routes on ${G.join("/")}`,"cluster-gateway",{kind:"gateway-up",project:"gateway",ports:G,routes:R.map(e=>({host:e.host??"(default)",project:e.project,mode:e.pool?"pods":"tcp",internal:!!e.internal,prependPrefix:e.prependPrefix}))}),$&&process.env.LENSMCP_GUEST_REPLAY!=="0"&&$.replay().catch(e=>console.warn("[gateway] guest replay failed:",e.message)),{ports:G,routes:R,services:P,stop:o(async()=>{if(!v){v=!0,clearInterval(Ee),clearInterval(me),ie.stop(),m.emit("info","gateway down","cluster-gateway",{kind:"gateway-down",project:"gateway"}),W.stop(),$?.close(),I?.flush(),(0,workspace_registry_1.unregisterWorkspace)(E);for(const e of P)A.killSpawned(e,"gateway shutdown");A.stopRetiringTrees(),await A.stopBuildHost(),await A.finishDyingTrees(),await O.reapAll(),ae.closeTunnels(),await Promise.all(J.map(e=>new Promise(n=>{let w=!1;const u=o(()=>{w||(w=!0,n())},"done"),S=e;e.close(()=>u());try{S.closeIdleConnections?.()}catch{}setTimeout(()=>{try{S.closeAllConnections?.()}catch{}u()},types_1.GATEWAY_STOP_GRACE_MS).unref?.()}))),H.closeProxy(),H.destroyAgents(),D.dispose(),(0,workspace_log_1.closeWorkspaceLogs)(),l()}},"stop"),...g.lensSupervisor?{lensSupervisor:g.lensSupervisor}:{},...g.graphStores?{graphStores:g.graphStores}:{}}}T(startGateway,"startGateway"),o(startGateway,"startGateway");

package/executors/gateway/runtime/types.d.ts

@@ -315,6 +315,12 @@ export interface ServiceCtl {
     pool: SockPool;
     proc?: ChildProcess;
     state: 'down' | 'starting' | 'up';
+    /** An explicit stop suppresses cold starts and self-healing until start/restart is requested. */
+    manualStopped?: boolean;
+    stoppingPid?: number;
+    stoppingUntil?: number;
+    actionFailure?: string;
+    actionStarting?: boolean;
     lastUsed: number;
     inflight: number;
     /** In-process background work the pod reports (queue jobs). Refreshed on the sweep from
@@ -557,6 +563,16 @@ export declare const POD_ROLL_READY_MS: number;
  * `0` removes the cap. Default 2.
  */
 export declare const POD_MAX_CONCURRENT_ROLLS: number;
+/**
+ * How long a service tree told to stop (SIGTERM to its process group) has before what is left of the group is
+ * SIGKILLed (issues/closed/foodguard-93). A tree's own shutdown takes under a second; an executor that ignores the
+ * signal — nx's run-executor under a serve executor that only stopped its devserver — never ends, and lived for two days
+ * at 0.6 GB. Override `LENSMCP_TREE_KILL_GRACE_MS`; `0` sends SIGTERM only. Default 10 s.
+ */
+export declare const TREE_KILL_GRACE_MS: number;
+/** How often the sweep looks for orphaned service executors (orphan-reaper.ts); the first look is the first sweep.
+ *  Override `LENSMCP_ORPHAN_REAP_MS`; `0` turns the reaper off. Default 5 min. */
+export declare const ORPHAN_REAP_EVERY_MS: number;
 /**
  * How many services may COLD START at once, daemon-wide — every workspace's and every worktree instance's
  * together. A cold start is a whole `serve-hmr` tree and a cold webpack compile; a page that touches a dozen lazy