@shogo-ai/worker 1.14.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs
CHANGED
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
import{createRequire as
|
|
1
|
+
import{createRequire as F$}from"node:module";var L$=Object.defineProperty;var U$=($)=>$;function N$($,X){this[$]=U$.bind(null,X)}var A$=($,X)=>{for(var Y in X)L$($,Y,{get:X[Y],enumerable:!0,configurable:!0,set:N$.bind(X,Y)})};var B$=($,X)=>()=>($&&(X=$($=0)),X);var y0=F$(import.meta.url);var U2={};A$(U2,{runGit:()=>y,isGitRepo:()=>X0,gitIsAvailable:()=>E0,gitFetchUnshallow:()=>O0,gitFetchAndReset:()=>$0,commitAndPush:()=>$6,cloneProject:()=>v0,buildGitUrl:()=>t});import{spawn as r$,execFile as e$}from"node:child_process";import{existsSync as R0}from"node:fs";import{join as b0}from"node:path";import{promisify as s$}from"node:util";async function E0($=!1){if(s!==null&&!$)return s;try{await t$("git",["--version"],{timeout:3000}),s=!0}catch{s=!1}return s}function t($,X){return`${$.replace(/\/+$/,"")}/api/projects/${X}/git`}function y($,X={}){let{cwd:Y,env:Z,timeoutMs:z=300000,logger:J}=X;return new Promise((K,V)=>{let H=r$("git",$,{cwd:Y,env:{...process.env,...Z},stdio:["ignore","pipe","pipe"]}),q="",Q="",G=[],W=[];H.stdout.setEncoding("utf-8"),H.stderr.setEncoding("utf-8"),H.stdout.on("data",(k)=>{if(G.push(k),J)J.log(k.trimEnd())}),H.stderr.on("data",(k)=>{if(W.push(k),J)J.warn(k.trimEnd())});let C=setTimeout(()=>{try{H.kill("SIGKILL")}catch{}V(Error(`git ${$[0]} timed out after ${z}ms`))},z);H.on("error",(k)=>{clearTimeout(C),V(k)}),H.on("close",(k)=>{clearTimeout(C),q=G.join(""),Q=W.join("");let L=k??-1;if(L===0)K({stdout:q,stderr:Q,exitCode:L});else{let O=Error(`git ${$[0]} exited with code ${L}: ${Q.slice(0,500)}`);O.exitCode=L,O.stdout=q,O.stderr=Q,V(O)}})})}async function v0($){let{apiUrl:X,apiKey:Y,projectId:Z,localDir:z,shallow:J=!0,logger:K,timeoutMs:V}=$;if(R0(b0(z,".git")))throw Error(`cloneProject: ${z}/.git already exists; use gitFetchAndReset instead`);let H=t(X,Z),q=["-c",`http.extraHeader=Authorization: Bearer ${Y}`,"clone"];if(J)q.push("--depth=1");return q.push(H,z),await y(q,{logger:K,timeoutMs:V}),{commitSha:(await y(["rev-parse","HEAD"],{cwd:z,logger:K,timeoutMs:1e4})).stdout.trim()}}async function $0($){let{apiUrl:X,apiKey:Y,projectId:Z,localDir:z,branch:J="HEAD",logger:K,timeoutMs:V}=$,H=t(X,Z),q=["-c",`http.extraHeader=Authorization: Bearer ${Y}`];return await y([...q,"fetch",H,J],{cwd:z,logger:K,timeoutMs:V}),await y(["reset","--hard","FETCH_HEAD"],{cwd:z,logger:K,timeoutMs:V}),{commitSha:(await y(["rev-parse","HEAD"],{cwd:z,logger:K,timeoutMs:1e4})).stdout.trim()}}async function O0($){let{apiUrl:X,apiKey:Y,projectId:Z,localDir:z,logger:J,timeoutMs:K}=$,V=t(X,Z),H=["-c",`http.extraHeader=Authorization: Bearer ${Y}`];if(R0(b0(z,".git","shallow")))await y([...H,"fetch","--unshallow",V],{cwd:z,logger:J,timeoutMs:K})}async function $6($){let{apiUrl:X,apiKey:Y,projectId:Z,localDir:z,message:J,branch:K="HEAD",authorEmail:V,authorName:H,logger:q,timeoutMs:Q}=$,G=t(X,Z),W={};if(V)W.GIT_AUTHOR_EMAIL=V,W.GIT_COMMITTER_EMAIL=V;if(H)W.GIT_AUTHOR_NAME=H,W.GIT_COMMITTER_NAME=H;await y(["add","-A"],{cwd:z,env:W,logger:q,timeoutMs:Q});let C=!1;try{await y(["diff","--cached","--quiet"],{cwd:z,env:W,logger:q,timeoutMs:Q})}catch{C=!0}if(!C)return{committed:!1};await y(["commit","-m",J,"--no-verify"],{cwd:z,env:W,logger:q,timeoutMs:Q});let L=(await y(["rev-parse","HEAD"],{cwd:z,env:W,logger:q,timeoutMs:1e4})).stdout.trim(),O=["-c",`http.extraHeader=Authorization: Bearer ${Y}`];return await y([...O,"push",G,K],{cwd:z,env:W,logger:q,timeoutMs:Q}),{committed:!0,commitSha:L}}function X0($){return R0(b0($,".git"))}var t$,s=null;var w0=B$(()=>{t$=s$(e$)});import{Command as g1}from"commander";import w$ from"picocolors";import w from"picocolors";import{existsSync as v6}from"node:fs";import{readFileSync as y$,writeFileSync as _$,existsSync as D$,chmodSync as x$}from"node:fs";import{hostname as M$}from"node:os";import{homedir as T$}from"node:os";import{join as D}from"node:path";import{mkdirSync as V0}from"node:fs";var h=D(T$(),".shogo"),j=D(h,"config.json"),r1=D(h,"credentials.json"),o=D(h,"device-id"),a=D(h,"worker.pid"),_0=D(h,"logs"),q0=D(_0,"worker.log"),H0=D(_0,"worker.err.log"),l=D(h,"runtime"),S=D(l,process.platform==="win32"?"agent-runtime.exe":"agent-runtime"),r=D(l,"version.json"),Q0=D(h,"projects");function g($,X=Q0){return D(X,$)}function m(){V0(h,{recursive:!0,mode:448}),V0(_0,{recursive:!0,mode:448})}function s0(){m(),V0(l,{recursive:!0,mode:448})}function t0($=Q0){m(),V0($,{recursive:!0,mode:448})}var D0={cloudUrl:"https://studio.shogo.ai",port:8002,projectsDir:Q0};function I(){if(!D$(j))return{};try{return JSON.parse(y$(j,"utf-8"))}catch($){throw Error(`Corrupt config at ${j}: ${$.message}`)}}function e($){m(),_$(j,JSON.stringify($,null,2),{mode:384}),x$(j,384)}function $2($,X){return{...$,...Object.fromEntries(Object.entries(X).filter(([,Y])=>Y!==void 0))}}function R($={}){let X=I(),Y={apiKey:process.env.SHOGO_API_KEY,cloudUrl:process.env.SHOGO_CLOUD_URL,name:process.env.SHOGO_INSTANCE_NAME,workerDir:process.env.SHOGO_WORKER_DIR,port:process.env.PORT?parseInt(process.env.PORT,10):void 0,projectsDir:process.env.SHOGO_PROJECTS_DIR},Z=$2($2(X,Y),$);if(!Z.apiKey)throw Error("No API key. Run `shogo login` or pass --api-key / set SHOGO_API_KEY.");return{apiKey:Z.apiKey,cloudUrl:Z.cloudUrl??D0.cloudUrl,name:Z.name??M$(),workerDir:Z.workerDir??process.cwd(),port:Z.port??D0.port,projectsDir:Z.projectsDir??D0.projectsDir}}import{spawn as P$}from"node:child_process";import{existsSync as S$,openSync as X2,readFileSync as R$,unlinkSync as b$,writeFileSync as E$}from"node:fs";function G0(){if(!S$(a))return null;let $=R$(a,"utf-8").trim(),X=parseInt($,10);return Number.isFinite(X)?X:null}function k0($){try{return process.kill($,0),!0}catch{return!1}}function x0(){try{b$(a)}catch{}}function Y2($){m();let X=G0();if(X&&k0(X))throw Error(`Worker already running (pid=${X}). Run \`shogo worker stop\` first.`);if(X)x0();let Y=$.inheritStdio?["ignore","inherit","inherit"]:["ignore",X2(q0,"a"),X2(H0,"a")],Z=P$($.runner,[$.entry],{cwd:$.cwd,env:$.env,detached:!!$.detach,stdio:Y});if(!Z.pid)throw Error("Failed to spawn worker process.");if(E$(a,String(Z.pid),{mode:384}),$.detach)Z.unref();return{pid:Z.pid,child:Z}}function Z2($="SIGTERM"){let X=G0();if(!X)return{killedPid:null};if(!k0(X))return x0(),{killedPid:null};try{process.kill(X,$)}catch{}return x0(),{killedPid:X}}import x from"picocolors";import{existsSync as h$}from"node:fs";import{request as v$}from"node:http";function J2($,X=process.env){let Y=(J)=>J&&J.trim().length>0?J.trim():void 0,Z=Y($);if(Z)return{url:z2(Z),source:"flag"};let z=[["HTTPS_PROXY",X.HTTPS_PROXY],["https_proxy",X.https_proxy],["HTTP_PROXY",X.HTTP_PROXY],["http_proxy",X.http_proxy]];for(let[J,K]of z){let V=Y(K);if(V)return{url:z2(V),source:J}}return null}function z2($){if(/^https?:\/\//i.test($))return $;return`http://${$}`}function M0($,X){if(!X)return $;return{...$,HTTPS_PROXY:$.HTTPS_PROXY??X.url,https_proxy:$.https_proxy??X.url,HTTP_PROXY:$.HTTP_PROXY??X.url,http_proxy:$.http_proxy??X.url}}async function K2($,X="api.shogo.ai",Y=5000){return new Promise((Z)=>{let z=!1,J=(K)=>{if(z)return;z=!0,Z(K)};try{let K=new URL($.url),V=K.hostname,H=K.port?parseInt(K.port,10):K.protocol==="https:"?443:80,q=v$({method:"CONNECT",host:V,port:H,path:`${X}:443`,timeout:Y});q.on("connect",(Q)=>{q.destroy();let G=Q.statusCode??0;if(G===200)J({ok:!0,detail:`CONNECT ${X}:443 → 200 (via ${V}:${H})`});else if(G===407)J({ok:!1,detail:`407 Proxy Authentication Required — check credentials in ${$.source}`});else J({ok:!1,detail:`CONNECT → HTTP ${G}`})}),q.on("timeout",()=>{q.destroy(),J({ok:!1,detail:`timeout after ${Y}ms`})}),q.on("error",(Q)=>{let G=Q.code?` (${Q.code})`:"";J({ok:!1,detail:`${Q.message}${G}`})}),q.end()}catch(K){J({ok:!1,detail:K?.message??"unknown error"})}})}function V2($){let X;try{X=new URL($)}catch{return[]}let Y=X.host,Z=X.protocol.replace(":","")||"https",z=X.hostname.split("."),J=z.length>=2?z.slice(-2).join("."):X.hostname;return[{url:`${Z}://${Y}`,host:Y,purpose:"control",criticality:"fatal"},{url:`${Z}://api-direct.${J}`,host:`api-direct.${J}`,purpose:"tunnel-direct",criticality:"graceful"},{url:`${Z}://artifacts.${J}`,host:`artifacts.${J}`,purpose:"artifacts",criticality:"graceful"}]}var j$=5000;async function m$($,X){try{let Y=new AbortController,Z=setTimeout(()=>Y.abort(),X),z=await fetch(`${$.replace(/\/$/,"")}/health`,{signal:Y.signal}).catch(()=>null);if(clearTimeout(Z),!z)return{ok:!1,detail:"no response (firewall? DNS?)"};return{ok:!0,detail:`HTTP ${z.status}`}}catch(Y){return{ok:!1,detail:Y?.message??"unknown error"}}}var q2=($)=>{let X=[{name:"Runtime (node >= 20)",criticality:"fatal",async run(){let[Z]=process.versions.node.split(".").map(Number);return(Z??0)>=20?{ok:!0,detail:`node v${process.versions.node}`}:{ok:!1,detail:`node v${process.versions.node} — need >=20`}}},{name:"Worker directory exists",criticality:"fatal",async run(){return h$($.workerDir)?{ok:!0,detail:$.workerDir}:{ok:!1,detail:`${$.workerDir} does not exist`}}}];if($.proxy)X.push({name:`Proxy reachable (${I$($.proxy.url)})`,criticality:"fatal",async run(){return K2($.proxy)}});let Y=V2($.cloudUrl);for(let Z of Y)X.push({name:`Reach ${Z.host}${Z.purpose!=="control"?x.dim(` (${Z.purpose})`):""}`,criticality:Z.criticality,run:()=>m$(Z.url,j$)});return X.push({name:"API key valid",criticality:"fatal",async run(){try{let Z=await fetch(`${$.cloudUrl}/api/instances/heartbeat`,{method:"POST",headers:{"content-type":"application/json","x-api-key":$.apiKey},body:JSON.stringify({hostname:"preflight",name:"preflight",os:"preflight",arch:"preflight",metadata:{preflight:!0}})});if(Z.status===401||Z.status===403)return{ok:!1,detail:`HTTP ${Z.status} — key rejected`};return Z.ok?{ok:!0,detail:`HTTP ${Z.status}`}:{ok:!1,detail:`HTTP ${Z.status}`}}catch(Z){return{ok:!1,detail:Z?.message??"unknown error"}}}}),X};function I$($){try{return new URL($).host}catch{return $}}async function H2($){console.log(x.bold(`
|
|
2
2
|
Shogo Worker — Preflight
|
|
3
|
-
`));let X=!1,Y=0;for(let Z of $){process.stdout.write(` ${
|
|
3
|
+
`));let X=!1,Y=0;for(let Z of $){process.stdout.write(` ${x.dim("...")} ${Z.name}`);let z=await Z.run();if(process.stdout.write("\r"),z.ok)console.log(` ${x.green("✓")} ${Z.name}${z.detail?x.dim(` — ${z.detail}`):""}`);else if(Z.criticality==="graceful")console.log(` ${x.yellow("◦")} ${Z.name}${z.detail?x.dim(` — ${z.detail}`):""}${x.yellow(" (optional — worker will still start)")}`),Y++;else console.log(` ${x.red("✗")} ${Z.name}${z.detail?x.dim(` — ${z.detail}`):""}`),X=!0}if(!X&&Y===0)console.log(x.green(`
|
|
4
4
|
All checks passed.
|
|
5
|
-
`));else if(!X)console.log(
|
|
5
|
+
`));else if(!X)console.log(x.yellow(`
|
|
6
6
|
Starting with ${Y} optional host(s) blocked. See docs/my-machines-networking.md.
|
|
7
|
-
`));else console.log(
|
|
7
|
+
`));else console.log(x.red(`
|
|
8
8
|
Preflight failed — fix blocking issues before starting.
|
|
9
|
-
`));return!X}import{accessSync as
|
|
10
|
-
`)}import{spawn as
|
|
11
|
-
`)){let
|
|
12
|
-
`)}class n0{opts;log;runtimes=new Map;usedPorts=new Set;spawnCommand;resolved=null;stopped=!1;resolvedLimits=void 0;watchers=new Map;pulledProjects=new Set;syncModes=new Map;constructor($={}){this.opts=$,this.log=$.logger??console,this.spawnCommand=$.spawnCommand??y6}resolveBinary(){if(!this.resolved)this.resolved=this.opts.resolveBin?this.opts.resolveBin():u({flag:this.opts.runtimeBin,env:this.opts.env});return this.resolved}async resolveLocalUrl($,X){let{pathname:Y,search:Z}=D2($);if(!(Y.startsWith("/agent/")||Y==="/agent"))return null;if(!X){let J=this.getActiveProjects();if(J.length!==1)return null;X=J[0]}let z=await this.spawnConfigFor(X);if(!z)return this.log.warn(`[WorkerRuntimeManager] No spawn config for ${X} — set defaultSpawnConfig or enrichSpawnConfig`),null;let K=await this.ensureRunning(X,z);if(!K.agentPort)return null;return this.touch(X),`http://127.0.0.1:${K.agentPort}${Y}${Z}`}deriveRuntimeToken($){return y2($)}describeRejection($,X){let{pathname:Y}=D2($);if(!(Y.startsWith("/agent/")||Y==="/agent"))return{code:"CLI_WORKER_HAS_NO_DATA_API",message:`cli-worker only serves /agent/* paths; tried: ${Y}`};return{code:"CLI_WORKER_NO_PROJECT_FOR_PATH",message:`cli-worker received an /agent path without a single active project; projectId=${X??"none"}, path=${Y}`}}async spawnConfigFor($){let X=this.opts.defaultSpawnConfig;if(!X)return null;if(this.opts.enrichSpawnConfig)try{return await this.opts.enrichSpawnConfig($,X)}catch(Y){this.log.warn(`[WorkerRuntimeManager] enrichSpawnConfig failed for ${$}: ${Y?.message??Y}`)}return X}async ensureRunning($,X){if(this.stopped)throw Error("WorkerRuntimeManager is stopped");let Y=this.runtimes.get($);if(Y?.status==="failed")throw Error(`[WorkerRuntimeManager] cannot ensureRunning(${$}): ${Y.lastError??"runtime is in failed state"}. Call resetFailure(${$}) or stop(${$}) before retrying.`);X=await this.maybeAutoPull($,X);let Z=this.runtimes.get($);if(Z?.status==="running")return this.touch($),this.snapshot(Z);if(Z?.startPromise){let K=await Z.startPromise;return this.snapshot(K)}let z=Z??this.makeSlot($,X);if(!Z)this.runtimes.set($,z);z.spawnConfig=X,z.startPromise=this.doStart(z);try{let K=await z.startPromise;return this.enforceMaxRuntimes($),this.snapshot(K)}finally{z.startPromise=null}}enforceMaxRuntimes($){let X=this.opts.maxRuntimes;if(X==null||!Number.isFinite(X)||X<=0)return;let Y=Date.now(),Z=Array.from(this.runtimes.values()).filter((J)=>J.status==="running");if(Z.length<=X)return;let z=Z.filter((J)=>J.projectId!==$&&Y-J.lastUsedAt>=C6).sort((J,V)=>J.lastUsedAt-V.lastUsedAt),K=Z.length-X;for(let J of z){if(K<=0)break;let V=Y-J.lastUsedAt;this.log.log(`[WorkerRuntimeManager] maxRuntimes=${X} exceeded (${Z.length} running) — `+`LRU-evicting ${J.projectId} (idle ${Math.round(V/1000)}s)`),this.stop(J.projectId).catch((Q)=>{this.log.warn(`[WorkerRuntimeManager] maxRuntimes eviction of ${J.projectId} failed: ${Q?.message??Q}`)}),K--}if(K>0)this.log.log(`[WorkerRuntimeManager] maxRuntimes=${X} still exceeded by ${K} after eviction pass — `+"remaining over-cap slots are mid-stream; will retry on next spawn / idle reap")}async ensurePulled($,X){return this.maybeAutoPull($,X)}async maybeAutoPull($,X){let Y=this.opts.autoPull;if(X.projectDir&&h0(X.projectDir))return X;if(!Y)throw Error(f0($,"no-auto-pull-config",null));if(!Y.projectsDir)throw Error(f0($,"no-projects-dir",null));if(!Y.enabled){let J=Z0(Y.projectsDir,$);if(h0(J)&&!M2(J))return{...X,projectDir:J};throw Error(f0($,"auto-pull-disabled",J))}if(this.pulledProjects.has($))return{...X,projectDir:Z0(Y.projectsDir,$)};let Z=Z0(Y.projectsDir,$),z=Y.logger??this.log,K={cloneProject:Y.gitOps?.cloneProject??v0,gitIsAvailable:Y.gitOps?.gitIsAvailable??b0,isGitRepo:Y.gitOps?.isGitRepo??X0};this.pulledProjects.add($);try{U2(Z,{recursive:!0});let J=M2(Z),V=K.isGitRepo(Z),H=(Y.useGit!==!1?await K.gitIsAvailable():!1)&&(J||V)?"git":"files";if(this.syncModes.set($,H),H==="git"){if(J){z.log(`[WorkerRuntimeManager] auto-pull: git clone project ${$} into ${Z}`);try{let W=await K.cloneProject({apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$,localDir:Z,shallow:!0,logger:z});z.log(`[WorkerRuntimeManager] auto-pull: ${$} cloned at ${W.commitSha.slice(0,8)}`)}catch(W){z.warn(`[WorkerRuntimeManager] auto-pull: git clone failed for ${$} (${W?.message??W}); falling back to CloudFileTransport.downloadAll`),this.syncModes.set($,"files"),await this.fileTransportClone($,Z,X,z)}}else if(V)z.log(`[WorkerRuntimeManager] auto-pull: ${$} already has .git/; skipping clone`);if(this.syncModes.get($)==="git")await this.topUpShogoState($,Z,X,z)}else if(J)await this.fileTransportClone($,Z,X,z);else z.log(`[WorkerRuntimeManager] auto-pull: ${$} workspace already populated; skipping clone`);if(Y.watch!==!1&&!this.watchers.has($))try{let W=new j0({apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$,localDir:Z}),O=this.syncModes.get($)??"files",C=new Y0({rootDir:Z,transport:W,logger:z,mode:O,git:O==="git"?{apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$}:void 0});C.start(),this.watchers.set($,C)}catch(W){z.warn(`[WorkerRuntimeManager] auto-pull: watcher start failed for ${$}: ${W?.message??W}`)}}catch(J){z.warn(`[WorkerRuntimeManager] auto-pull: failed for ${$} — runtime will fall back to template defaults. `+`(${J?.message??J})`)}return{...X,projectDir:Z}}async fileTransportClone($,X,Y,Z){Z.log(`[WorkerRuntimeManager] auto-pull: file-transport clone of ${$} into ${X}`);let K=await new j0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:X}).downloadAll();Z.log(`[WorkerRuntimeManager] auto-pull: ${$} downloaded ${K.downloaded} files (${K.errors.length} errors)`)}async topUpShogoState($,X,Y,Z){try{let z=new j0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:X}),J=(await z.listManifest()).filter((Q)=>Q.path===".shogo"||Q.path.startsWith(".shogo/"));if(J.length===0)return;let V=await z.downloadFiles(J);Z.log(`[WorkerRuntimeManager] auto-pull: ${$} .shogo/ top-up downloaded ${V.downloaded} files (${V.errors.length} errors)`)}catch(z){Z.warn(`[WorkerRuntimeManager] auto-pull: .shogo top-up failed for ${$}: ${z?.message??z}`)}}shouldDeferToCloudSync($){return!!this.opts.autoPull?.enabled&&this.pulledProjects.has($)}status($){let X=this.runtimes.get($);return X?this.snapshot(X):null}getActiveProjects(){return Array.from(this.runtimes.keys()).filter(($)=>{let X=this.runtimes.get($);return X&&(X.status==="running"||X.status==="starting"||X.status==="restarting")})}touch($){let X=this.runtimes.get($);if(!X)return;X.lastUsedAt=Date.now(),this.armIdleTimer(X)}async stop($,X="SIGTERM"){let Y=this.runtimes.get($);if(!Y)return;if(Y.status="stopping",Y.restartTimer)clearTimeout(Y.restartTimer),Y.restartTimer=null;if(Y.idleTimer)clearTimeout(Y.idleTimer),Y.idleTimer=null;if(Y.graceTimer)clearTimeout(Y.graceTimer),Y.graceTimer=null;if(Y.limitWatchdog)Y.limitWatchdog.stop(),Y.limitWatchdog=null;if(Y.proc){this.killProcessGroup(Y,X);try{Y.proc.kill(X)}catch{}await this.waitForExit(Y.proc,5000),this.killProcessGroup(Y,"SIGKILL")}Y.pid=null,this.releasePort(Y.agentPort),this.runtimes.delete($)}resetFailure($){let X=this.runtimes.get($);if(!X||X.status!=="failed")return!1;if(X.restartTimer)clearTimeout(X.restartTimer),X.restartTimer=null;if(X.idleTimer)clearTimeout(X.idleTimer),X.idleTimer=null;if(X.graceTimer)clearTimeout(X.graceTimer),X.graceTimer=null;if(X.limitWatchdog)X.limitWatchdog.stop(),X.limitWatchdog=null;return this.runtimes.delete($),this.log.log(`[WorkerRuntimeManager] resetFailure: ${$} cleared, next ensureRunning will respawn`),!0}async stopAll($="SIGTERM"){this.stopped=!0;let X=Array.from(this.watchers.keys());await Promise.all(X.map(async(Z)=>{let z=this.watchers.get(Z);if(this.watchers.delete(Z),z)try{await z.stop()}catch(K){this.log.warn(`[WorkerRuntimeManager] watcher stop ${Z}: ${K?.message??K}`)}}));let Y=Array.from(this.runtimes.keys());await Promise.all(Y.map((Z)=>this.stop(Z,$).catch((z)=>{this.log.error(`[WorkerRuntimeManager] Failed to stop ${Z}: ${z?.message??z}`)})))}makeSlot($,X){return{projectId:$,agentPort:0,apiServerPort:0,status:"starting",proc:null,pid:null,startedAt:0,lastStdoutAt:0,lastUsedAt:Date.now(),restarts:0,consecutiveFailures:0,lastFailureAt:0,graceTimer:null,restartTimer:null,idleTimer:null,limitWatchdog:null,spawnConfig:X,startPromise:null}}async doStart($){let X=this.resolveBinary();if(!X)throw $.status="error",$.lastError="agent-runtime binary not found (run `shogo runtime install`)",Error($.lastError);if(!$.agentPort)$.agentPort=await this.allocatePort(),$.apiServerPort=$.agentPort+B2;let Y=this.buildEnv($,X.path),Z=this.resolveCwd($),z=this.spawnCommand(X.path),K=this.resolveLimits(),J=z.command,V=z.args,Q=!1;if(K){let G=w2({command:z.command,args:z.args,limits:K,scopeName:`shogo-rt-${$.projectId.slice(0,8)}-${Date.now().toString(36)}`,logger:this.log});J=G.command,V=G.args,Q=G.wrapped}this.log.log(`[WorkerRuntimeManager] Spawning agent-runtime for ${$.projectId} via ${J} ${V.join(" ")} (port=${$.agentPort}, source=${X.source}${K?`, memCeil=${K.memoryMB}M`:""})`);let q=process.platform!=="win32",H=V6(J,V,{cwd:Z,env:Y,detached:q,stdio:["ignore","pipe","pipe"]});if(q)try{H.unref()}catch{}$.proc=H,$.pid=H.pid??null,$.status="starting",$.startedAt=Date.now();let W=$.spawnConfig.extraEnv?.SHOGO_OPEN_ID;if(L0("agent-spawned",$.projectId,W,{pid:$.pid,command:J,args:V,source:X.source}),K)try{this.applyPostSpawnLimits($,K,Q)}catch(G){this.log.warn(`[WorkerRuntimeManager] failed to apply resource limits for ${$.projectId}: ${G?.message??G}`)}$.lastStdoutAt=$.startedAt,H.on("error",(G)=>{$.lastError=G?.message??String(G),this.log.error(`[WorkerRuntimeManager] spawn error for ${$.projectId}: ${$.lastError}`)}),H.on("exit",(G,L)=>{this.handleExit($,G,L)});let O=`[runtime:${$.projectId.slice(0,8)}]`,C=!0;H.stdout?.on("data",(G)=>{if($.lastStdoutAt=Date.now(),C)C=!1,L0("agent-first-output",$.projectId,W,{elapsedMs:Date.now()-($.startedAt??Date.now())});for(let L of G.toString().trimEnd().split(`
|
|
13
|
-
`))if(L)this.log.log(`${
|
|
14
|
-
`))if(L)this.log.error(`${O} ${L}`)});try{return await this.waitForHealth($,B6),L0("agent-health-ready",$.projectId,W,{elapsedMs:Date.now()-($.startedAt??Date.now()),agentPort:$.agentPort}),$.status="running",$.lastUsedAt=Date.now(),this.armIdleTimer($),this.armGraceTimer($),$}catch(G){$.status="error",$.lastError=G?.message??String(G),this.killProcessGroup($,"SIGTERM");try{H.kill("SIGTERM")}catch{}throw this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,G}}resolveLimits(){if(this.resolvedLimits===void 0)this.resolvedLimits=O2(this.opts.env??process.env);return this.resolvedLimits}applyPostSpawnLimits($,X,Y){let Z=$.pid;if(!Z)return;if(process.platform==="linux"){if(Y)return;if(C2({pid:Z,limits:X,logger:this.log}))return}else if(process.platform==="win32"){if(L2({pid:Z,limits:X,logger:this.log}))return}$.limitWatchdog=new R0({pid:Z,ceilingMB:X.memoryMB,logger:this.log,onBreach:(z)=>this.handleLimitBreach($,z,X.memoryMB)}),$.limitWatchdog.start()}handleLimitBreach($,X,Y){if(this.log.warn(`[WorkerRuntimeManager] runtime ${$.projectId} exceeded memory ceiling (${X}MB > ${Y}MB) — restarting`),$.lastError=`memory ceiling exceeded (${X}MB > ${Y}MB)`,$.limitWatchdog)$.limitWatchdog.stop(),$.limitWatchdog=null;this.killProcessGroup($,"SIGTERM");try{$.proc?.kill("SIGTERM")}catch{}let Z=setTimeout(()=>this.killProcessGroup($,"SIGKILL"),5000);try{Z.unref?.()}catch{}}killProcessGroup($,X){if(!$.pid)return;if(process.platform==="win32"){try{q6("taskkill",["/F","/T","/PID",String($.pid)],{stdio:"ignore",windowsHide:!0})}catch{}return}try{process.kill(-$.pid,X)}catch{}}armGraceTimer($){if($.graceTimer)clearTimeout($.graceTimer),$.graceTimer=null;$.graceTimer=setTimeout(()=>{if($.graceTimer=null,$.consecutiveFailures>0)$.consecutiveFailures=0},N6);try{$.graceTimer.unref?.()}catch{}}buildEnv($,X){let Y=$.spawnConfig,Z={...this.opts.env??process.env,PROJECT_ID:$.projectId,PORT:String($.agentPort),API_SERVER_PORT:String($.apiServerPort),SKILL_SERVER_PORT:String($.apiServerPort),WORKSPACE_API_PORT_BASE:String($.agentPort+k6),NODE_ENV:"production",STARTUP_TIME:String(Date.now()),SHOGO_CLOUD_URL:Y.cloudUrl,SHOGO_API_URL:Y.cloudUrl,SHOGO_API_KEY:Y.apiKey,RUNTIME_AUTH_SECRET:y2($.projectId),WEBHOOK_TOKEN:D6($.projectId)};if(Y.projectDir)Z.PROJECT_DIR=Y.projectDir,Z.WORKSPACE_DIR=Y.projectDir;if(this.shouldDeferToCloudSync($.projectId))Z.SHOGO_CLOUD_SYNC="1";if(Y.aiProxyUrl)Z.AI_PROXY_URL=Y.aiProxyUrl;if(Y.aiProxyToken)Z.AI_PROXY_TOKEN=Y.aiProxyToken;if(Y.techStackId)Z.TECH_STACK_ID=Y.techStackId;if(Y.name)Z.AGENT_NAME=Y.name;if(Y.workspaceId)Z.WORKSPACE_ID=Y.workspaceId;if(!Z.TREE_SITTER_WASM_DIR)Z.TREE_SITTER_WASM_DIR=Z0(O6(X),"tree-sitter-wasm");if(Y.extraEnv)Object.assign(Z,Y.extraEnv);let z=this.resolveLimits();if(z)k2(Z,z.memoryMB);return Z}resolveCwd($){let X=$.spawnConfig;if(X.projectDir&&h0(X.projectDir))return X.projectDir;let Y=this.opts.runtimeWorkDir??Z0(G6(),"shogo-runtime",$.projectId);return U2(Y,{recursive:!0}),Y}handleExit($,X,Y){let Z=Y===null&&X===0;if(this.log.log(`[WorkerRuntimeManager] runtime ${$.projectId} exited (code=${X}, signal=${Y})`),$.proc=null,$.graceTimer)clearTimeout($.graceTimer),$.graceTimer=null;if($.limitWatchdog)$.limitWatchdog.stop(),$.limitWatchdog=null;if($.status==="stopping"||this.stopped){$.status="stopped",$.pid=null,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0;return}if(Z){$.status="stopped",$.pid=null,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,this.runtimes.delete($.projectId);return}this.killProcessGroup($,"SIGKILL"),$.pid=null;let z=Date.now(),K=z-$.lastFailureAt<=T2;if($.consecutiveFailures=K?$.consecutiveFailures+1:1,$.lastFailureAt=z,$.restarts+=1,$.lastError=`exited code=${X} signal=${Y}`,$.consecutiveFailures>=F2){if($.status="failed",$.lastError=`Circuit breaker tripped: ${$.consecutiveFailures} consecutive non-clean exits within ${Math.round(T2/1000)}s (last: code=${X} signal=${Y}). Most recent on macOS is jetsam OOM (signal=SIGKILL with code=null); the previous incarnation's vite/tsserver/preview-manager children were reaped to prevent further RSS growth. Stop, fix the workspace, and call resetFailure(projectId) (or stop(projectId)) to allow another spawn attempt.`,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,$.restartTimer)clearTimeout($.restartTimer),$.restartTimer=null;if($.idleTimer)clearTimeout($.idleTimer),$.idleTimer=null;this.log.error(`[WorkerRuntimeManager] ${$.lastError}`);return}let J=this.restartBackoffMs($.restarts);if($.status="restarting",this.log.warn(`[WorkerRuntimeManager] restarting ${$.projectId} in ${Math.round(J/1000)}s (restart #${$.restarts}, consecutive failures ${$.consecutiveFailures}/${F2})`),$.restartTimer)clearTimeout($.restartTimer);$.restartTimer=setTimeout(()=>{if($.restartTimer=null,$.status==="failed"||this.stopped)return;$.startPromise=this.doStart($).then((V)=>{return $.startPromise=null,V}).catch((V)=>{return $.startPromise=null,this.log.error(`[WorkerRuntimeManager] restart of ${$.projectId} failed: ${V?.message??V}`),$})},J);try{$.restartTimer.unref?.()}catch{}}restartBackoffMs($){let X=Math.min(L6*Math.pow(2,Math.max(0,$-1)),A6),Y=X*0.2*Math.random();return X+Y}armIdleTimer($){if($.idleTimer)clearTimeout($.idleTimer),$.idleTimer=null;let X=this.opts.idleMs??w6;if(!Number.isFinite(X)||X<=0)return;$.idleTimer=setTimeout(()=>{let Y=Date.now()-$.lastUsedAt;if(Y<X){this.armIdleTimer($);return}this.log.log(`[WorkerRuntimeManager] idle-evicting ${$.projectId} after ${Math.round(Y/1000)}s`),this.stop($.projectId).catch((Z)=>{this.log.warn(`[WorkerRuntimeManager] idle stop failed: ${Z?.message??Z}`)})},X)}async allocatePort(){let $=I0-m0,X=Math.min($,50);for(let Y=0;Y<X;Y++){let Z=m0+Math.floor(Math.random()*$);if(Z+C0-1>I0)continue;let z=!0;for(let V=0;V<C0;V++)if(this.usedPorts.has(Z+V)){z=!1;break}if(!z)continue;let K=await this.isPortListening(Z),J=await this.isPortListening(Z+B2);if(K||J)continue;for(let V=0;V<C0;V++)this.usedPorts.add(Z+V);return Z}throw Error(`Cannot allocate port in range ${m0}-${I0} after ${X} attempts`)}releasePort($){if(!$)return;for(let X=0;X<C0;X++)this.usedPorts.delete($+X)}async isPortListening($){let X=new AbortController,Y=setTimeout(()=>X.abort(),250);try{return await fetch(`http://127.0.0.1:${$}/`,{method:"HEAD",signal:X.signal}),clearTimeout(Y),!0}catch{return clearTimeout(Y),!1}}tcpProbe($){return new Promise((X)=>{let Y=!1,Z=(K)=>{if(Y)return;Y=!0;try{z.destroy()}catch{}X(K)},z=W6({host:"127.0.0.1",port:$});z.setTimeout(T6),z.once("connect",()=>Z(!0)),z.once("error",()=>Z(!1)),z.once("timeout",()=>Z(!1))})}async waitForHealth($,X){let{agentPort:Y,proc:Z}=$;if(!Z)throw Error(`waitForHealth: slot ${$.projectId} has no spawned process`);let z=Date.now(),K=z+X,J=null,V=!1,Q=0,q=0,H=0,W=0,O=z;while(Date.now()<K){if(W++,Z.exitCode!==null||Z.signalCode!=null||Z.killed)throw Error(`agent-runtime exited (code=${Z.exitCode}, signal=${Z.signalCode}) before becoming healthy on port ${Y}`);q++;let C=new AbortController,G=setTimeout(()=>C.abort(),1500);try{let k=await fetch(`http://127.0.0.1:${Y}/health`,{method:"GET",signal:C.signal});if(clearTimeout(G),k.ok){this.log.log(`[WorkerRuntimeManager] /health ready for ${$.projectId} on port ${Y} (HTTP ${k.status} after ${Date.now()-z}ms, ${W} iter, ${q} http)`);return}J=`HTTP /health returned ${k.status}`}catch(k){clearTimeout(G);let T=k?.name??"Error",c=k?.code??k?.cause?.code;J=`HTTP /health failed: ${T}${c?`(${c})`:""}: ${k?.message??k}`}if(H++,V=await this.tcpProbe(Y),V){Q=Date.now();let k=Date.now()-$.lastStdoutAt;if(k>=_2)throw Error(`agent-runtime wedged on port ${Y}: TCP listening but stdout silent for ${k}ms (> ${_2}ms window); ${q} /health attempts, last error: ${J??"n/a"}`)}let L=Date.now();if(L-O>=F6){let k=L-z,T=L-$.lastStdoutAt,c=Q>0?L-Q:null;this.log.log(`[WorkerRuntimeManager] still waiting for /health on ${$.projectId} port ${Y} (${(k/1000).toFixed(1)}s elapsed, tcpListening=${V}${c!=null?`(${c}ms ago)`:""}, lastStdout=${T}ms ago, ${q} http, ${H} tcp, lastError=${J??"n/a"})`),O=L}await new Promise((k)=>setTimeout(k,U6))}if(Z.exitCode!==null||Z.signalCode!=null||Z.killed)throw Error(`agent-runtime exited (code=${Z.exitCode}, signal=${Z.signalCode}) before becoming healthy on port ${Y}`);throw Error(`Timeout waiting for agent-runtime /health on port ${Y} after ${W} iter (httpAttempts=${q}, tcpAttempts=${H}, tcpListening=${V}, lastError=${J??"n/a"})`)}async waitForExit($,X){if($.exitCode!==null||$.signalCode!=null||$.killed)return;await new Promise((Y)=>{let Z=setTimeout(()=>{try{$.kill("SIGKILL")}catch{}Y()},X);$.once("exit",()=>{clearTimeout(Z),Y()})})}snapshot($){return{projectId:$.projectId,status:$.status,agentPort:$.agentPort||void 0,apiServerPort:$.apiServerPort||void 0,pid:$.proc?.pid,startedAt:$.startedAt||void 0,lastUsedAt:$.lastUsedAt,restarts:$.restarts,lastError:$.lastError}}}import{hostname as d0,platform as g0,arch as i0}from"node:os";function d(){if(typeof __SHOGO_WORKER_VERSION__==="string"&&__SHOGO_WORKER_VERSION__.length>0)return`shogo-cli/${__SHOGO_WORKER_VERSION__}`;return"shogo-cli/unknown"}class b2 extends Error{code="TUNNEL_WS_HEADERS_UNSUPPORTED";constructor(){super("Tunnel WebSocket auth requires a runtime with WebSocket header support (Bun or Node >= 21). Current runtime does not support WebSocket constructor headers.");this.name="TunnelWebSocketHeaderSupportError"}}var z0=60,A0=300,p0=3,M6=p0,x6=1800000,S6=25000,R2=1000,P2=60000,E2=3;class c0{opts;log;pollTimer=null;ws=null;wsIdleTimer=null;heartbeatTimer=null;stopped=!1;currentPollInterval=z0;wsReconnectAttempt=0;lastHeartbeatError=null;consecutiveAuthFailures=0;consecutiveAuthSuccesses=0;serverPublishedWsUrl=null;activeAbortControllers=new Map;constructor($){this.opts=$,this.log=$.logger??console}start(){if(!this.opts.apiKey){this.log.log("[WorkerTunnel] No API key set, skipping tunnel");return}this.stopped=!1,this.wsReconnectAttempt=0,this.currentPollInterval=z0,this.log.log("[WorkerTunnel] Starting heartbeat polling to Shogo Cloud..."),this.heartbeatLoop()}stop(){if(this.stopped=!0,this.pollTimer)clearTimeout(this.pollTimer),this.pollTimer=null;this.cleanupWs(),this.log.log("[WorkerTunnel] Tunnel stopped")}isConnected(){if(this.ws!==null&&this.ws.readyState===WebSocket.OPEN)return!0;return!this.stopped&&!!this.opts.apiKey&&this.lastHeartbeatError===null&&this.pollTimer!==null}getCloudUrl(){return this.opts.cloudUrl.replace(/\/$/,"")}getWsBaseUrl(){let $=(this.opts.wsUrlOverride||process.env.SHOGO_TUNNEL_WS_URL||"").trim();if($)return $.replace(/\/$/,"");if(this.serverPublishedWsUrl)return this.serverPublishedWsUrl.replace(/\/$/,"");return this.getCloudUrl().replace(/^http/,"ws")}buildWsUrl(){return`${this.getWsBaseUrl()}/api/instances/ws`}supportsWebSocketConstructorHeaders($=globalThis){if(typeof $.Bun<"u"||typeof $.process?.versions?.bun==="string")return!0;let X=$.process?.versions?.node;if(X){if(parseInt(X.split(".")[0]??"0",10)>=21)return!0}return!1}createTunnelWebSocket($,X,Y=globalThis){if(!this.supportsWebSocketConstructorHeaders(Y))throw new b2;return new WebSocket($,X)}getReconnectDelayMs(){let $=Math.min(R2*Math.pow(2,this.wsReconnectAttempt),P2),X=$*0.2*Math.random();return $+X}async collectMetadata(){let $={hostname:d0(),os:g0(),arch:i0(),uptime:process.uptime(),protocolVersion:E2,tunnelStatus:this.ws?.readyState===WebSocket.OPEN?"connected":"polling",kind:this.opts.kind??"cli-worker"};try{let X=this.opts.resolver.getActiveProjects();$.activeProjects=X.length,$.projects=X.map((Y)=>{let Z=this.opts.resolver.status(Y);return{projectId:Y,status:Z?.status??"unknown",agentPort:Z?.agentPort}})}catch{$.activeProjects=0}return $}async sendHeartbeat(){let $=this.getCloudUrl(),X=await this.collectMetadata(),Y=d0(),Z=this.opts.name??Y,z=await fetch(`${$}/api/instances/heartbeat`,{method:"POST",headers:{"Content-Type":"application/json","User-Agent":d(),"x-api-key":this.opts.apiKey,"x-shogo-kind":this.opts.kind??"cli-worker"},body:JSON.stringify({hostname:Y,name:Z,os:g0(),arch:i0(),kind:this.opts.kind??"cli-worker",metadata:X})});if(!z.ok)throw Error(`Heartbeat failed: HTTP ${z.status}`);let K=await z.json();if(typeof K.wsUrl==="string"&&K.wsUrl.length>0){if(K.wsUrl!==this.serverPublishedWsUrl)this.log.log(`[WorkerTunnel] Cloud advertised tunnel WS URL: ${K.wsUrl}`);this.serverPublishedWsUrl=K.wsUrl}return K}scheduleNextPoll($){if(this.stopped)return;if(this.pollTimer)clearTimeout(this.pollTimer);let X=($??this.currentPollInterval)*1000;this.pollTimer=setTimeout(()=>void this.heartbeatLoop(),X)}async heartbeatLoop(){if(this.stopped)return;if(this.ws&&this.ws.readyState===WebSocket.OPEN){this.scheduleNextPoll(this.currentPollInterval);return}try{let $=await this.sendHeartbeat(),X=$.nextPollIn||z0;if(this.consecutiveAuthFailures>=p0){if(this.consecutiveAuthSuccesses++,this.consecutiveAuthSuccesses<M6){this.currentPollInterval=A0,this.scheduleNextPoll();return}}if(this.currentPollInterval=X,this.lastHeartbeatError)this.log.log("[WorkerTunnel] Heartbeat recovered"),this.lastHeartbeatError=null;if(this.consecutiveAuthFailures=0,this.consecutiveAuthSuccesses=0,$.wsRequested&&!this.ws){this.log.log("[WorkerTunnel] Cloud requested WebSocket — connecting..."),this.connectWs();return}}catch($){let X=$?.message??String($);if(/HTTP 40[13]\b/.test(X))this.consecutiveAuthFailures++,this.consecutiveAuthSuccesses=0;else this.consecutiveAuthFailures=0,this.consecutiveAuthSuccesses=0;if(X!==this.lastHeartbeatError)this.log.error(`[WorkerTunnel] Heartbeat error: ${X}`),this.lastHeartbeatError=X;if(this.consecutiveAuthFailures>=p0){let Z=`tunnel saw ${this.consecutiveAuthFailures} consecutive auth failures from Shogo Cloud`;try{this.opts.onAuthRevoked?.(Z)}catch(z){this.log.warn(`[WorkerTunnel] onAuthRevoked threw: ${z?.message??z}`)}if(this.currentPollInterval!==A0)this.log.warn(`[WorkerTunnel] ${this.consecutiveAuthFailures} consecutive auth failures — `+`backing off to ${A0}s. Re-authenticate with \`shogo login\`.`);this.currentPollInterval=A0}else this.currentPollInterval=z0}this.scheduleNextPoll()}async handleRequest($){if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;this.resetWsIdleTimer();let X=new AbortController;this.activeAbortControllers.set($.requestId,X);try{let Y=await this.resolveLocalUrl($.path,$.projectId);if(!Y){let J=this.opts.resolver.describeRejection?this.opts.resolver.describeRejection($.path,$.projectId):{code:"NO_LOCAL_RUNTIME",message:`no local runtime available for path: ${$.path}`};this.sendFrame({type:"response",requestId:$.requestId,status:502,headers:{"content-type":"application/json"},body:JSON.stringify({code:J.code,message:J.message,path:$.path})});return}let Z={...$.headers??{}};if($.projectId&&($.path.startsWith("/agent/")||$.path==="/agent")){let J=this.opts.resolver.deriveRuntimeToken($.projectId);if(J)Z["x-runtime-token"]=J}let z={method:$.method,headers:Z,signal:X.signal};if($.body&&$.method!=="GET"&&$.method!=="HEAD")z.body=$.body;let K=await fetch(Y,z);if($.stream){let J=K.body?.getReader();if(!J){this.sendFrame({type:"stream-error",requestId:$.requestId,error:"No response body for stream"});return}let V=new TextDecoder;try{while(!0){let{done:Q,value:q}=await J.read();if(Q)break;if(this.ws?.readyState!==WebSocket.OPEN)break;this.sendFrame({type:"stream-chunk",requestId:$.requestId,data:V.decode(q,{stream:!0})})}if(this.ws?.readyState===WebSocket.OPEN)this.sendFrame({type:"stream-end",requestId:$.requestId})}catch(Q){if(Q?.name!=="AbortError"&&this.ws?.readyState===WebSocket.OPEN)this.sendFrame({type:"stream-error",requestId:$.requestId,error:Q?.message??String(Q)})}}else{let J=await K.text(),V={};K.headers.forEach((Q,q)=>{V[q]=Q}),this.sendFrame({type:"response",requestId:$.requestId,status:K.status,headers:V,body:J})}}catch(Y){if(Y?.name==="AbortError")return;if(this.ws?.readyState!==WebSocket.OPEN)return;let Z=$.stream?{type:"stream-error",requestId:$.requestId,error:Y?.message??String(Y)}:{type:"response",requestId:$.requestId,status:502,body:JSON.stringify({error:Y?.message??String(Y)})};this.sendFrame(Z)}finally{this.activeAbortControllers.delete($.requestId)}}async resolveLocalUrl($,X){return this.opts.resolver.resolveLocalUrl($,X)}sendFrame($){if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;try{this.ws.send(JSON.stringify($))}catch(X){this.log.warn(`[WorkerTunnel] Frame send failed: ${X?.message??X}`)}}resetWsIdleTimer(){if(this.wsIdleTimer)clearTimeout(this.wsIdleTimer);this.wsIdleTimer=setTimeout(()=>{if(this.ws&&this.ws.readyState===WebSocket.OPEN){this.log.log("[WorkerTunnel] WebSocket idle timeout — closing, returning to polling");try{this.ws.close(1000,"Idle timeout")}catch{}}},x6)}startWsHeartbeat(){if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.heartbeatTimer=setInterval(async()=>{if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;try{let $=await this.collectMetadata();this.sendFrame({type:"heartbeat",metadata:$})}catch{}},S6)}connectWs(){if(this.stopped||this.ws)return;let $=this.buildWsUrl(),X=d0(),Y=g0(),Z=i0(),z=this.opts.name??X;this.log.log(`[WorkerTunnel] Opening WebSocket to ${$} (hostname=${X})`);let K={headers:{Authorization:`Bearer ${this.opts.apiKey}`,"User-Agent":d(),"x-shogo-hostname":X,"x-shogo-name":z,"x-shogo-os":Y,"x-shogo-arch":Z,"x-shogo-kind":this.opts.kind??"cli-worker"}},J;try{J=this.createTunnelWebSocket($,K)}catch(V){this.log.error(`[WorkerTunnel] WebSocket creation failed: ${V?.message??V}`),this.ws=null,this.scheduleNextPoll(5);return}this.ws=J,J.onopen=()=>{this.log.log("[WorkerTunnel] WebSocket connected — session active"),this.wsReconnectAttempt=0,this.startWsHeartbeat(),this.resetWsIdleTimer()},J.onmessage=(V)=>{let Q;try{let q=typeof V.data==="string"?V.data:V.data.toString();Q=JSON.parse(q)}catch{return}if(Q.type==="ping"){this.sendFrame({type:"pong"}),this.resetWsIdleTimer();return}if(Q.type==="cancel"){let q=this.activeAbortControllers.get(Q.requestId);if(q)q.abort();return}if(Q.type==="request"){this.handleRequest(Q).catch((q)=>{this.log.error(`[WorkerTunnel] Error handling request: ${q?.message??q}`)});return}},J.onclose=(V)=>{if(this.log.log(`[WorkerTunnel] WebSocket closed: code=${V.code} reason=${V.reason||"none"}`),this.cleanupWs(),this.stopped)return;if(V.code===1000||V.code===4000)this.scheduleNextPoll(this.currentPollInterval);else{this.wsReconnectAttempt++;let Q=this.getReconnectDelayMs();this.log.log(`[WorkerTunnel] Reconnecting in ${Math.round(Q/1000)}s (attempt ${this.wsReconnectAttempt})`),this.scheduleNextPoll(Math.ceil(Q/1000))}},J.onerror=(V)=>{this.log.error("[WorkerTunnel] WebSocket error:",V?.message??"unknown")}}cleanupWs(){if(this.heartbeatTimer)clearInterval(this.heartbeatTimer),this.heartbeatTimer=null;if(this.wsIdleTimer)clearTimeout(this.wsIdleTimer),this.wsIdleTimer=null;for(let[,$]of this.activeAbortControllers)try{$.abort()}catch{}this.activeAbortControllers.clear(),this.ws=null}_testing(){let $=this;return{sendHeartbeat:()=>$.sendHeartbeat(),heartbeatLoop:()=>$.heartbeatLoop(),connectWs:()=>$.connectWs(),cleanupWs:()=>$.cleanupWs(),handleRequest:(X)=>$.handleRequest(X),installFakeWs:(X)=>{$.ws=X},getCloudUrl:()=>$.getCloudUrl(),getWsBaseUrl:()=>$.getWsBaseUrl(),buildWsUrl:()=>$.buildWsUrl(),getReconnectDelayMs:()=>$.getReconnectDelayMs(),supportsWebSocketConstructorHeaders:(X)=>$.supportsWebSocketConstructorHeaders(X),createTunnelWebSocket:(X,Y,Z)=>$.createTunnelWebSocket(X,Y,Z),DEFAULT_POLL_INTERVAL_S:z0,BACKOFF_BASE_MS:R2,BACKOFF_MAX_MS:P2,TUNNEL_PROTOCOL_VERSION:E2,get currentPollInterval(){return $.currentPollInterval},set currentPollInterval(X){$.currentPollInterval=X},get wsReconnectAttempt(){return $.wsReconnectAttempt},set wsReconnectAttempt(X){$.wsReconnectAttempt=X},get ws(){return $.ws},get stopped(){return $.stopped},get serverPublishedWsUrl(){return $.serverPublishedWsUrl},set serverPublishedWsUrl(X){$.serverPublishedWsUrl=X}}}}async function v2($){let X=P({name:$.name,workerDir:$.workerDir,apiKey:$.apiKey,cloudUrl:$.cloudUrl,port:$.port?parseInt($.port,10):void 0,projectsDir:$.projectsDir}),Y=K2($.proxy);if(!$.foreground)return P6({cfg:X,proxy:Y,flags:$});let Z=u({flag:$.runtimeBin});if(!Z)console.error(w.red(n({flag:$.runtimeBin}))),process.exit(1);if(x0(process.env,Y),$.debug){if(!await Q2(q2({cloudUrl:X.cloudUrl,apiKey:X.apiKey,workerDir:X.workerDir,proxy:Y})))process.exit(1)}let z=!$.noAutoPull,K=!$.noGit;if(console.log(w.bold(`
|
|
15
|
-
Shogo Worker — Starting`)),console.log(w.dim(" name ")+X.name),console.log(w.dim(" worker-dir ")+X.workerDir),console.log(w.dim(" cloud ")+X.cloudUrl),console.log(w.dim(" runtime ")+`${Z.path} ${w.dim(`(via ${Z.source})`)}`),console.log(w.dim(" auto-pull ")+(z?w.green("on")+w.dim(` → ${X.projectsDir}`):w.yellow("off"))),console.log(w.dim(" sync mode ")+(
|
|
16
|
-
Received ${
|
|
17
|
-
Shogo Worker — Started`)),console.log(w.dim(" pid: ")+V),console.log(w.dim(" name: ")+$.name),console.log(w.dim(" logs: ")+"~/.shogo/logs/worker.log"),console.log(w.dim(" stop: ")+"shogo worker stop")}function E6($){let X=["worker","start","--foreground"];if($.name)X.push("--name",$.name);if($.workerDir)X.push("--worker-dir",$.workerDir);if($.apiKey)X.push("--api-key",$.apiKey);if($.cloudUrl)X.push("--cloud-url",$.cloudUrl);if($.port)X.push("--port",$.port);if($.proxy)X.push("--proxy",$.proxy);if($.project)X.push("--project",$.project);if($.runtimeBin)X.push("--runtime-bin",$.runtimeBin);if($.debug)X.push("--debug");if($.noAutoPull)X.push("--no-auto-pull");if($.projectsDir)X.push("--projects-dir",$.projectsDir);if($.noGit)X.push("--no-git");return X}function b6(){let $=process.execPath,X=/\bbun(?:-[^/\\]*)?$/.test($)||typeof globalThis.Bun<"u",Y=process.argv[1];if(Y&&R6(Y)){let z=/\.ts$/.test(Y);return{entry:Y,runner:X?"bun":z?"tsx":"node"}}return{entry:new URL("../../bin/shogo.mjs",import.meta.url).pathname,runner:X?"bun":"node"}}import h2 from"picocolors";async function j2(){let{killedPid:$}=Z2("SIGTERM");if($===null){console.log(h2.dim("No worker running."));return}console.log(h2.green(`✓ Worker stopped (pid=${$}).`))}import v from"picocolors";async function m2(){let $=W0();if(!$){console.log(v.yellow("● stopped")+v.dim(" (no pid file)"));return}if(!G0($)){console.log(v.red("● dead")+v.dim(` (stale pid ${$})`));return}let X=I();if(console.log(v.green("● running")+v.dim(` (pid ${$})`)),X.name)console.log(v.dim(" name: ")+X.name);if(X.cloudUrl)console.log(v.dim(" cloud: ")+X.cloudUrl);if(X.workerDir)console.log(v.dim(" dir: ")+X.workerDir)}import{spawn as v6}from"node:child_process";import{existsSync as h6,statSync as j6}from"node:fs";import I2 from"picocolors";async function u2($){let X=$.err?Q0:q0;if(!h6(X)){console.log(I2.dim("No logs yet."));return}let Y=$.follow?["-F","-n","200",X]:["-n","200",X];if(v6("tail",Y,{stdio:"inherit"}).on("exit",(z)=>process.exit(z??0)),!$.follow){if(j6(X).size===0)console.log(I2.dim("(log file is empty)"))}}import y from"picocolors";import{spawn as m6}from"node:child_process";import{hostname as I6,platform as u6,arch as f6}from"node:os";import f from"picocolors";class x extends Error{kind;constructor($,X){super($);this.kind=X;this.name="CloudLoginError"}}async function f2($){let X=$.cloudUrl.replace(/\/$/,""),Y=$.client??"cli",Z=$.deviceName??I6(),z=$.devicePlatform??`${u6()}-${f6()}`,K=$.appVersion??g6(),J=$.log??((k)=>console.log(k)),V=$.fetchImpl??fetch,Q=await V(`${X}/api/cli/login/start`,{method:"POST",headers:{"content-type":"application/json","user-agent":d()},body:JSON.stringify({deviceId:$.deviceId,deviceName:Z,devicePlatform:z,deviceAppVersion:K,workspaceId:$.workspaceId,client:Y}),signal:AbortSignal.timeout(1e4)}).catch((k)=>{throw new x(`Cannot reach Shogo Cloud at ${X}: ${k?.message??k}`,"transport")});if(!Q.ok){let k=await Q.text().catch(()=>"");throw new x(`Cloud rejected /api/cli/login/start (HTTP ${Q.status}): ${k||"no body"}`,"transport")}let q=await Q.json().catch(()=>({}));if(!q?.ok||!q.state||!q.authUrl)throw new x(`Cloud returned a malformed start response: ${q?.error??JSON.stringify(q)}`,"transport");let H=n6($.pollIntervalMs??q.pollIntervalMs),W=$.timeoutMs??q.expiresInMs,O=Date.now()+W;if(J(""),J(f.bold("Sign in to Shogo Cloud")),J(f.dim(" cloud: ")+X),J(f.dim(" device: ")+`${Z} (${z})`),J(f.dim(" user code: ")+f.cyan(q.userCode)),J(""),J(" Open this URL in your browser to approve:"),J(" "+f.cyan(q.authUrl)),J(""),$.openBrowser!==!1)if(typeof $.openBrowser==="function")try{await $.openBrowser(q.authUrl)}catch{}else i6(q.authUrl).catch(()=>{});J(f.dim("Waiting for approval..."));let C=$.installSignalHandlers??!0,G=()=>{L.abort()},L=new AbortController;if($.abortSignal)if($.abortSignal.aborted)L.abort();else $.abortSignal.addEventListener("abort",()=>L.abort(),{once:!0});if(C)process.once("SIGINT",G),process.once("SIGTERM",G);try{while(!0){if(L.signal.aborted)throw new x("Aborted by caller.","cancelled");if(Date.now()>=O)throw new x(`Timed out after ${Math.round(W/1000)}s waiting for approval.`,"timeout");let k=await V(`${X}/api/cli/login/poll?state=${encodeURIComponent(q.state)}`,{method:"GET",headers:{accept:"application/json","user-agent":d()},signal:AbortSignal.timeout(1e4)}).catch((T)=>{return J(f.dim(` (poll error: ${T?.message??T} — retrying)`)),null});if(k&&k.ok){let T=await k.json().catch(()=>({}));if(T?.status==="approved"&&T.key)return{key:T.key,email:T.email??null,workspace:T.workspace??null,deviceId:T.deviceId??$.deviceId};if(T?.status==="denied")throw new x("Sign-in was denied in the browser.","denied");if(T?.status==="expired")throw new x("Sign-in request expired before it was approved.","expired")}await d6(H,L.signal)}}finally{if(C)process.removeListener("SIGINT",G),process.removeListener("SIGTERM",G)}}function n6($){if(!Number.isFinite($)||$<=0)return 2000;return Math.min(Math.max($,1000),1e4)}function d6($,X){return new Promise((Y,Z)=>{if(X?.aborted)return Z(new x("Aborted by caller.","cancelled"));let z=setTimeout(()=>{X?.removeEventListener("abort",K),Y()},$),K=()=>{clearTimeout(z),Z(new x("Aborted by caller.","cancelled"))};X?.addEventListener("abort",K,{once:!0})})}function g6(){return d()}function i6($){return new Promise((X)=>{let Y=process.platform==="darwin"?"open":process.platform==="win32"?"cmd":"xdg-open",Z=process.platform==="win32"?["/c","start",'""',$]:[$];try{let z=m6(Y,Z,{stdio:"ignore",detached:!0});z.on("error",()=>X()),z.unref(),X()}catch{X()}})}import{readFileSync as p6,writeFileSync as c6,existsSync as o6,chmodSync as a6}from"node:fs";import{randomUUID as l6}from"node:crypto";function n2(){if(o6(o)){let X=p6(o,"utf-8").trim();if(X)return X}m();let $=l6();return c6(o,$,{mode:384}),a6(o,384),$}var r6="https://studio.shogo.ai";async function d2($){let X=I(),Y=($.cloudUrl||X.cloudUrl||r6).replace(/\/$/,""),Z=$.apiKey||process.env.SHOGO_API_KEY;if(Z){await e6({key:Z,cloudUrl:Y,cfg:X,name:$.name});return}let z=n2(),K;try{K=await f2({cloudUrl:Y,deviceId:z,deviceName:$.name,workspaceId:$.workspace,openBrowser:!$.noBrowser})}catch(J){if(J instanceof x){if(console.error(y.red(`✗ ${J.message}`)),J.kind==="transport")console.error(y.dim(" If your network blocks browsers, run with --api-key <key> instead."));process.exit(1)}throw J}if(X.apiKey=K.key,X.cloudUrl=Y,$.name)X.name=$.name;if(e(X),console.log(""),console.log(y.green("✓ Signed in to Shogo Cloud")),K.workspace)console.log(y.dim(" workspace: ")+K.workspace);if(K.email)console.log(y.dim(" email: ")+K.email);console.log(y.dim(" saved to: ")+"~/.shogo/config.json"),console.log(""),console.log(y.dim(" next: ")+"shogo worker start")}async function e6({key:$,cloudUrl:X,cfg:Y,name:Z}){if(!/^shogo_sk_/.test($))console.error(y.red('✗ API key should start with "shogo_sk_". Copy it verbatim from the API Keys page.')),process.exit(1);let z;try{let K=await fetch(`${X}/api/api-keys/validate`,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({key:$}),signal:AbortSignal.timeout(1e4)});if(z=await K.json().catch(()=>({})),!K.ok||!z?.valid)console.error(y.red(`✗ ${z?.error||`Cloud rejected the key (HTTP ${K.status}).`}`)),process.exit(1)}catch(K){console.error(y.red(`✗ Cannot reach Shogo Cloud at ${X}: ${K?.message??K}`)),process.exit(1)}if(Y.apiKey=$,Y.cloudUrl=X,Z)Y.name=Z;if(e(Y),console.log(y.green("✓ API key saved to ~/.shogo/config.json")),z.workspace?.name)console.log(y.dim(" workspace: ")+z.workspace.name);if(z.user?.email)console.log(y.dim(" email: ")+z.user.email);console.log(y.dim(" next: ")+"shogo worker start")}import K0 from"picocolors";async function g2(){let $=I();if(Object.keys($).length===0){console.log(K0.dim("(no config — run `shogo login` or `shogo config set`)"));return}let X={...$,apiKey:$.apiKey?`***${$.apiKey.slice(-4)}`:void 0};console.log(K0.dim(`file: ${j}`)),console.log(JSON.stringify(X,null,2))}async function i2($,X){let Y=["apiKey","cloudUrl","name","workerDir","port"];if(!Y.includes($))console.error(K0.red(`Unknown key: ${$}`)),console.error(K0.dim(`Allowed: ${Y.join(", ")}`)),process.exit(1);let Z=I();Z[$]=$==="port"?parseInt(X,10):X,e(Z),console.log(K0.green(`✓ ${$} set`))}import U from"picocolors";import{spawn as s6}from"node:child_process";import{createHash as t6}from"node:crypto";import{createWriteStream as $1,existsSync as J0,mkdirSync as o0,readFileSync as o2,renameSync as p2,rmSync as a0,writeFileSync as X1}from"node:fs";import{tmpdir as Y1}from"node:os";import{dirname as Z1,join as N0}from"node:path";import{Readable as z1}from"node:stream";import{pipeline as K1}from"node:stream/promises";var J1="https://github.com/shogo-labs/shogo-ai/releases/download";function a2(){let{platform:$,arch:X}=process,Y;if($==="darwin")Y="darwin";else if($==="linux")Y="linux";else if($==="win32")Y="windows";else throw Error(`Unsupported platform: ${$}`);let Z;if(X==="arm64")Z="arm64";else if(X==="x64")Z="x64";else throw Error(`Unsupported arch: ${X} (need arm64 or x64)`);return`${Y}-${Z}`}function U0(){if(!J0(r))return null;try{let $=o2(r,"utf-8");return JSON.parse($)}catch{return null}}async function V1($,X){let Y=X.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download/);if(!Y)throw Error(`Cannot auto-resolve latest version from non-GitHub baseUrl '${X}'. Pass --version explicitly.`);let Z=Y[1],z=Y[2];if($==="stable"){let H=`https://api.github.com/repos/${Z}/${z}/releases/latest`,W=await fetch(H,{headers:{Accept:"application/vnd.github+json"}});if(!W.ok)throw Error(`GitHub API ${W.status} for ${H}`);let O=await W.json();if(!O.tag_name)throw Error("GitHub API did not return tag_name");return c2(O.tag_name)}let K=`https://api.github.com/repos/${Z}/${z}/releases?per_page=30`,J=await fetch(K,{headers:{Accept:"application/vnd.github+json"}});if(!J.ok)throw Error(`GitHub API ${J.status} for ${K}`);let V=await J.json(),Q=$==="beta"?"-beta":"-nightly",q=V.find((H)=>H.prerelease&&/^v\d+\.\d+\.\d+-/.test(H.tag_name)&&H.tag_name.includes(Q));if(!q)throw Error(`No ${$} runtime release found`);return c2(q.tag_name)}function c2($){if(!/^v\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/.test($))throw Error(`Unexpected app tag '${$}' (expected vX.Y.Z[-prerelease])`);return $.slice(1)}function q1($,X,Y){let Z=`shogo-agent-runtime-${X}.tar.gz`,z=`v${$}`,K=`${Y.replace(/\/$/,"")}/${z}/${Z}`;return{tarball:K,sha256:`${K}.sha256`,assetName:Z}}async function Q1($,X){let Y=await fetch($,{redirect:"follow"});if(!Y.ok)throw Error(`Download failed: HTTP ${Y.status} for ${$}`);if(!Y.body)throw Error(`Download failed: empty body for ${$}`);await K1(z1.fromWeb(Y.body),$1(X))}async function H1($){let X=await fetch($,{redirect:"follow"});if(!X.ok)throw Error(`SHA256 sidecar fetch failed: HTTP ${X.status} for ${$}`);let Z=(await X.text()).trim().split(/\s+/)[0];if(!/^[0-9a-f]{64}$/i.test(Z))throw Error(`SHA256 sidecar at ${$} did not contain a 64-char hex digest`);return Z.toLowerCase()}function W1($){let X=o2($);return t6("sha256").update(X).digest("hex")}async function G1($,X){await new Promise((Y,Z)=>{let z=s6("tar",["-xzf",$,"-C",X],{stdio:["ignore","pipe","pipe"]}),K="";z.stderr.on("data",(J)=>{K+=J.toString()}),z.on("error",Z),z.on("exit",(J)=>{if(J===0)Y();else Z(Error(`tar exited ${J}: ${K.trim()}`))})})}function O1($,X){s0();let Y=Z1(X);if(!J0(Y))o0(Y,{recursive:!0,mode:448});let Z=`${X}.next`;if(J0(Z))a0(Z,{force:!0});p2($,Z);try{if(process.platform!=="win32"){let{chmodSync:z}=_0("node:fs");z(Z,493)}}catch{}if(J0(X))a0(X,{force:!0});p2(Z,X)}async function l2($={}){let X=$.logger??console,Y=$.channel??"stable",Z=$.baseUrl??process.env.SHOGO_RUNTIME_RELEASES_URL??J1,z=$.target??a2(),K=$.version;if(!K)X.log(`[runtime install] Resolving latest ${Y} version...`),K=await V1(Y,Z),X.log(`[runtime install] Latest ${Y} = ${K}`);let J=U0();if(J&&J.version===K&&J.target===z&&!$.force)return X.log(`[runtime install] ${K} (${z}) already installed at ${R} — pass --force to reinstall`),{version:K,target:z,binPath:R,source:J.source,sha256:J.sha256,channel:Y};let V=q1(K,z,Z);X.log(`[runtime install] Downloading ${V.tarball}`);let Q=N0(Y1(),`shogo-runtime-install-${process.pid}-${Date.now()}`);o0(Q,{recursive:!0});try{let q=N0(Q,V.assetName);await Q1(V.tarball,q),X.log("[runtime install] Verifying SHA-256...");let H=await H1(V.sha256),W=W1(q);if(W!==H)throw Error(`SHA-256 mismatch for ${V.assetName}
|
|
18
|
-
expected: ${
|
|
19
|
-
actual: ${
|
|
20
|
-
`,{mode:384}),X.log(`[runtime install] Installed agent-runtime ${
|
|
21
|
-
shogo project pull ${
|
|
22
|
-
Watching for local changes...`));let V=new Y0({rootDir:Z,transport:
|
|
23
|
-
Received ${
|
|
24
|
-
${Z}`),console.log(
|
|
25
|
-
shogo project push ${B.cyan($)}`)),console.log(B.dim(" cloud ")+Y.cloudUrl),console.log(B.dim(" from ")+Z),z?.length)console.log(B.dim(" include ")+z.join(", "));if(X.deleteRemote)console.log(B.yellow(" --delete-remote: remote files not present locally will be DELETED"));console.log("");let
|
|
26
|
-
${Z}`),console.log(B.dim(" uploaded: ")+X.uploaded),X.deleted)console.log(B.dim(" deleted: ")+X.deleted);if(!Y){console.log(B.dim(" errors: ")+X.errors.length);for(let z of X.errors.slice(0,5))console.log(B.dim(" ")+B.red(`${z.path}: ${z.message}`));if(X.errors.length>5)console.log(B.dim(` ... and ${X.errors.length-5} more`))}}function
|
|
27
|
-
shogo project checkout ${
|
|
9
|
+
`));return!X}import{accessSync as u$,constants as f$,existsSync as n$}from"node:fs";import{delimiter as d$,join as g$}from"node:path";function u($={}){let X=$.env??process.env,Y=i$($);for(let J of Y)if(G2(J.path))return J;let Z=$.systemBinName??Q2(),z=p$(Z,X.PATH);if(z)return{path:z,source:"path"};return null}function i$($){let X=$.env??process.env,Y=[];if($.flag&&$.flag.trim())Y.push({path:$.flag.trim(),source:"flag"});let Z=X.SHOGO_AGENT_RUNTIME_BIN?.trim();if(Z)Y.push({path:Z,source:"env"});return Y.push({path:S,source:"home"}),Y}function Q2(){return process.platform==="win32"?"shogo-agent-runtime.exe":"shogo-agent-runtime"}function G2($){if(!n$($))return!1;try{if(process.platform==="win32")return!0;return u$($,f$.X_OK),!0}catch{return!1}}function p$($,X){if(!X)return null;for(let Y of X.split(d$)){if(!Y)continue;let Z=g$(Y,$);if(G2(Z))return Z}return null}function n($={}){let X=$.env??process.env,Y=[];if(Y.push("Error: agent-runtime binary not found."),Y.push(""),Y.push("Looked in (priority order):"),$.flag)Y.push(` --runtime-bin ${$.flag}`);if(X.SHOGO_AGENT_RUNTIME_BIN)Y.push(` $SHOGO_AGENT_RUNTIME_BIN = ${X.SHOGO_AGENT_RUNTIME_BIN}`);return Y.push(` ${S} (default install location)`),Y.push(` ${Q2()} on $PATH`),Y.push(""),Y.push("Fix:"),Y.push(" - Run `shogo runtime install` to download the latest binary, or"),Y.push(" - Pass `--runtime-bin <path>` to point at an existing build."),Y.join(`
|
|
10
|
+
`)}import{spawn as q6,spawnSync as H6}from"node:child_process";import{createHmac as P2,randomBytes as Q6}from"node:crypto";import{existsSync as h0,mkdirSync as A2,readdirSync as G6}from"node:fs";import{createConnection as k6}from"node:net";import{tmpdir as W6}from"node:os";import{dirname as O6,join as Z0}from"node:path";import{spawnSync as P0}from"node:child_process";import{existsSync as c$,mkdirSync as o$,writeFileSync as W0}from"node:fs";import{join as i}from"node:path";var k2=1e5;function W2($){let X=parseInt($.RUNTIME_MEMORY_MB||"",10);if(!Number.isFinite(X)||X<=0)return null;let Y=parseInt($.RUNTIME_CPU_PERCENT||"0",10),Z=Number.isFinite(Y)&&Y>0?Y:0,z=parseInt($.RUNTIME_TASKS_MAX||"",10),J=Number.isFinite(z)&&z>0?z:4096;return{memoryMB:X,cpuPercent:Z,tasksMax:J}}function O2($,X){let Z=`--max-old-space-size=${Math.max(256,Math.floor(X*0.75))}`,z=$.NODE_OPTIONS?.trim();if(z){if(/--max-old-space-size=/.test(z))return;$.NODE_OPTIONS=`${z} ${Z}`}else $.NODE_OPTIONS=Z}function a$(){if(process.platform!=="linux")return!1;try{return P0("systemd-run",["--version"],{stdio:"ignore",timeout:3000}).status===0}catch{return!1}}function w2($){let{command:X,args:Y,limits:Z,scopeName:z,logger:J}=$;if(process.platform!=="linux"||!a$())return{command:X,args:Y,wrapped:!1};let K=["-p",`MemoryMax=${Z.memoryMB}M`,"-p",`MemoryHigh=${Math.max(128,Math.floor(Z.memoryMB*0.9))}M`,"-p",`TasksMax=${Z.tasksMax}`];if(Z.cpuPercent>0)K.push("-p",`CPUQuota=${Z.cpuPercent}%`);let V=["--user","--scope","--quiet","--collect",`--unit=${z}`,...K,"--",X,...Y];return J?.log(`[resource-limits] wrapping spawn in systemd scope ${z} (MemoryMax=${Z.memoryMB}M)`),{command:"systemd-run",args:V,wrapped:!0}}function C2($){let{pid:X,limits:Y,logger:Z}=$;if(process.platform!=="linux")return!1;let z="/sys/fs/cgroup";if(!c$(i(z,"cgroup.controllers")))return!1;let J=i(z,"shogo",`rt-${X}`);try{if(o$(J,{recursive:!0}),W0(i(J,"memory.max"),String(Y.memoryMB*1024*1024)),Y.cpuPercent>0){let K=Math.floor(Y.cpuPercent/100*k2);W0(i(J,"cpu.max"),`${K} ${k2}`)}try{W0(i(J,"pids.max"),String(Y.tasksMax))}catch{}return W0(i(J,"cgroup.procs"),String(X)),Z?.log(`[resource-limits] attached pid ${X} to cgroup ${J} (memory.max=${Y.memoryMB}M)`),!0}catch(K){return Z?.warn(`[resource-limits] direct cgroup attach failed for pid ${X}: ${K?.message??K}`),!1}}function L2($){let{pid:X,limits:Y,logger:Z}=$;if(process.platform!=="win32")return!1;let z=process.env.SHOGO_WINJOB_HELPER||"@shogo/winjob";try{let J=y0(z);if(typeof J.limitProcess!=="function")return!1;let K=J.limitProcess(X,{memoryBytes:Y.memoryMB*1024*1024,killOnClose:!0});if(K)Z?.log(`[resource-limits] attached pid ${X} to Job Object (${Y.memoryMB}M)`);return K}catch{return!1}}function l$($,X=process.platform){try{if(X==="win32"){let J=`$ErrorActionPreference='SilentlyContinue';$root=${$};$all=Get-CimInstance Win32_Process;$seen=@{};$stack=New-Object System.Collections.Stack;$stack.Push($root);$sum=0;while($stack.Count -gt 0){$id=$stack.Pop();if($seen[$id]){continue};$seen[$id]=$true;foreach($p in $all){if($p.ProcessId -eq $id){$sum+=$p.WorkingSetSize};if($p.ParentProcessId -eq $id){$stack.Push($p.ProcessId)}}};[math]::Floor($sum/1024)`,K=P0("powershell.exe",["-NoProfile","-NonInteractive","-Command",J],{encoding:"utf-8",timeout:1e4,windowsHide:!0});if(K.status!==0||!K.stdout)return null;let V=parseInt(K.stdout.trim(),10);return Number.isFinite(V)?V:null}let Y=P0("ps",["-o","rss=","-g",String($)],{encoding:"utf-8",timeout:5000});if(Y.status!==0||!Y.stdout)return null;let Z=0,z=!1;for(let J of Y.stdout.split(`
|
|
11
|
+
`)){let K=parseInt(J.trim(),10);if(Number.isFinite(K))Z+=K,z=!0}return z?Z:null}catch{return null}}class S0{timer=null;consecutive=0;fired=!1;pid;ceilingMB;platform;intervalMs;strikes;logger;onBreach;constructor($){this.pid=$.pid,this.ceilingMB=$.ceilingMB,this.platform=$.platform??process.platform,this.intervalMs=$.intervalMs??parseInt(process.env.RUNTIME_RSS_WATCHDOG_MS||"15000",10),this.strikes=$.strikes??parseInt(process.env.RUNTIME_RSS_WATCHDOG_STRIKES||"2",10),this.logger=$.logger,this.onBreach=$.onBreach}start(){if(this.timer)return;this.timer=setInterval(()=>this.sample(),Math.max(2000,this.intervalMs));try{this.timer.unref?.()}catch{}}stop(){if(this.timer)clearInterval(this.timer),this.timer=null}sample(){if(this.fired)return;let $=l$(this.pid,this.platform);if($==null){this.consecutive=0;return}let X=Math.floor($/1024);if(X<=this.ceilingMB){this.consecutive=0;return}if(this.consecutive+=1,this.logger?.warn(`[resource-limits] pid ${this.pid} RSS ${X}MB over ceiling ${this.ceilingMB}MB (strike ${this.consecutive}/${this.strikes})`),this.consecutive>=this.strikes)this.fired=!0,this.stop(),this.onBreach(X)}}import{CloudFileTransport as j0}from"@shogo-ai/sdk/cloud-file-transport";import{watch as N2,statSync as X6}from"node:fs";import{relative as xX,sep as Y6,posix as Z6}from"node:path";var z6=1500,J6=new Set(["node_modules",".git","dist","build",".vite",".cache"]),K6=[".shogo/"];function V6($){if(!$)return!0;return $.split("/").some((X)=>J6.has(X))}class Y0{rootDir;transport;debounceMs;logger;onFlush;mode;git;commitAndPush;watcher=null;pending=new Set;timer=null;flushing=!1;stopped=!1;constructor($){if(this.rootDir=$.rootDir,this.transport=$.transport,this.debounceMs=$.debounceMs??z6,this.logger=$.logger??console,this.onFlush=$.onFlush,this.mode=$.mode??"files",this.mode==="git"&&!$.git)throw Error('CloudSyncWatcher: mode: "git" requires the `git` option block');this.git=$.git,this.commitAndPush=$.commitAndPush??(async(X)=>{return(await Promise.resolve().then(() => (w0(),U2))).commitAndPush(X)})}start(){if(this.watcher)return;if(this.stopped)throw Error("CloudSyncWatcher: already stopped, build a fresh instance");try{this.watcher=N2(this.rootDir,{recursive:!0},($,X)=>{if(!X)return;this.handleEvent(String(X))})}catch($){this.logger.warn(`[CloudSyncWatcher] recursive watch failed (${$?.message??$}); falling back to root-only watch.`),this.watcher=N2(this.rootDir,(X,Y)=>{if(!Y)return;this.handleEvent(String(Y))})}}async stop(){if(this.stopped)return;if(this.stopped=!0,this.timer)clearTimeout(this.timer),this.timer=null;if(this.watcher){try{this.watcher.close()}catch{}this.watcher=null}if(this.pending.size>0)await this.flush()}handleEvent($){if(this.stopped)return;let X=$.split(Y6).join(Z6.sep);if(V6(X))return;try{let Y=`${this.rootDir}/${X}`;if(X6(Y).isDirectory())return}catch{return}this.pending.add(X),this.scheduleFlush()}scheduleFlush(){if(this.timer)return;this.timer=setTimeout(()=>{this.timer=null,this.flush()},this.debounceMs)}async flush(){if(this.flushing){this.scheduleFlush();return}if(this.pending.size===0)return;this.flushing=!0;let $=Array.from(this.pending);this.pending.clear();try{if(this.mode==="git")await this.flushGit($);else await this.flushFiles($)}catch(X){this.logger.error(`[CloudSyncWatcher] flush failed: ${X?.message??X}`);for(let Y of $)this.pending.add(Y);this.scheduleFlush()}finally{this.flushing=!1}}async flushFiles($){if($.length===0)return;let X=await this.transport.uploadFiles($);if(this.onFlush?.({uploaded:$,errors:X.errors.length}),X.errors.length>0)for(let Y of X.errors)this.logger.warn(`[CloudSyncWatcher] upload ${Y.path}: ${Y.message}`)}async flushGit($){if(!this.git)throw Error("CloudSyncWatcher: missing git options");let X=[],Y=[];for(let J of $)if(K6.some((K)=>J===K.slice(0,-1)||J.startsWith(K)))X.push(J);else Y.push(J);if(X.length>0){let J=await this.transport.uploadFiles(X);if(J.errors.length>0)for(let K of J.errors)this.logger.warn(`[CloudSyncWatcher] upload ${K.path}: ${K.message}`)}let Z=!1,z;if(Y.length>0){let J=`auto: ${new Date().toISOString()}`;try{let K=await this.commitAndPush({apiUrl:this.git.apiUrl,apiKey:this.git.apiKey,projectId:this.git.projectId,localDir:this.rootDir,message:J,branch:this.git.branch,authorEmail:this.git.authorEmail,authorName:this.git.authorName,logger:this.logger});Z=K.committed,z=K.commitSha}catch(K){throw K}}this.onFlush?.({uploaded:$,errors:0,committed:Z,commitSha:z})}}w0();var m0=37100,I0=37900,B2=1,w6=2,C0=16,C6=900000,L6=30000,U6=1000,N6=60000,F2=8,T2=300000,y2=3600000,A6=8,B6=120000,F6=900000,T6=60000,y6=500,_6=30000,_2=25000,D6=5000,x6=500,M6=process.env.SHOGO_PERF_LOG==="1";function L0($,X,Y,Z){if(!M6)return;console.log(`[shogo-perf] ${JSON.stringify({perf:"open",source:"worker",phase:$,projectId:X,openAttemptId:Y,atMs:Date.now(),...Z})}`)}var P6=($)=>({command:$,args:[]}),u0=null;function S2(){if(!u0)u0=Q6(32).toString("hex");return u0}function D2($){return P2("sha256",S2()).update(`runtime:${$}`).digest("hex")}function S6($){return P2("sha256",S2()).update(`webhook:${$}`).digest("hex")}function x2($){let X=$.indexOf("?");if(X===-1)return{pathname:$,search:""};return{pathname:$.slice(0,X),search:$.slice(X)}}function M2($){try{return G6($).length===0}catch{return!0}}function f0($,X,Y){let Z=[];switch(Z.push(`Cannot spawn agent-runtime for project ${$}: no workspace directory available.`),Z.push(""),X){case"no-auto-pull-config":Z.push(" Reason: WorkerRuntimeManager was constructed without an `autoPull` config and no caller-provided `projectDir` was found on disk."),Z.push(" This usually means a programmatic embedder forgot to wire up enrichSpawnConfig "+"or autoPull. CLI users should not see this — please file a bug.");break;case"no-projects-dir":Z.push(" Reason: auto-pull is configured but `projectsDir` is empty. The worker needs a persistent root directory under which it can store cloned project workspaces.");break;case"auto-pull-disabled":Z.push(` Reason: auto-pull was disabled (--no-auto-pull) and the expected pre-pulled workspace at ${Y} is missing or empty.`);break}return Z.push(""),Z.push(" How to fix (pick one):"),Z.push(" 1. Re-enable auto-pull (default). Drop the --no-auto-pull flag and restart"),Z.push(" the worker. The first inbound request for this project will clone its"),Z.push(" workspace from Shogo Cloud into <projectsDir>/<projectId>/."),Z.push(""),Z.push(" 2. Pre-pull manually with `shogo project pull <projectId>` before starting"),Z.push(" the worker. Use this when you want full control over when the clone runs"),Z.push(" (slow links, scheduled maintenance windows, etc.)."),Z.push(""),Z.push(" 3. Point the worker at an existing workspace by setting either:"),Z.push(" --projects-dir <path> (per-invocation flag)"),Z.push(" SHOGO_PROJECTS_DIR=<path> (env var, persists across restarts)"),Z.push(" shogo config set projectsDir <path>"),Z.push(" Whichever path you pick must contain a subdirectory named after the "),Z.push(` project id (e.g. <path>/${$}/) populated with the project's source.`),Z.push(""),Z.push(" Docs: https://shogo.ai/docs/self-hosted-worker#workspace-seeding"),Z.join(`
|
|
12
|
+
`)}class n0{opts;log;runtimes=new Map;usedPorts=new Set;spawnCommand;resolved=null;stopped=!1;resolvedLimits=void 0;watchers=new Map;pulledProjects=new Set;syncModes=new Map;constructor($={}){this.opts=$,this.log=$.logger??console,this.spawnCommand=$.spawnCommand??P6}resolveBinary(){if(!this.resolved)this.resolved=this.opts.resolveBin?this.opts.resolveBin():u({flag:this.opts.runtimeBin,env:this.opts.env});return this.resolved}async resolveLocalUrl($,X){let{pathname:Y,search:Z}=x2($);if(!(Y.startsWith("/agent/")||Y==="/agent"))return null;if(!X){let K=this.getActiveProjects();if(K.length!==1)return null;X=K[0]}let z=await this.spawnConfigFor(X);if(!z)return this.log.warn(`[WorkerRuntimeManager] No spawn config for ${X} — set defaultSpawnConfig or enrichSpawnConfig`),null;let J=await this.ensureRunning(X,z);if(!J.agentPort)return null;return this.touch(X),`http://127.0.0.1:${J.agentPort}${Y}${Z}`}deriveRuntimeToken($){return D2($)}describeRejection($,X){let{pathname:Y}=x2($);if(!(Y.startsWith("/agent/")||Y==="/agent"))return{code:"CLI_WORKER_HAS_NO_DATA_API",message:`cli-worker only serves /agent/* paths; tried: ${Y}`};return{code:"CLI_WORKER_NO_PROJECT_FOR_PATH",message:`cli-worker received an /agent path without a single active project; projectId=${X??"none"}, path=${Y}`}}async spawnConfigFor($){let X=this.opts.defaultSpawnConfig;if(!X)return null;if(this.opts.enrichSpawnConfig)try{return await this.opts.enrichSpawnConfig($,X)}catch(Y){this.log.warn(`[WorkerRuntimeManager] enrichSpawnConfig failed for ${$}: ${Y?.message??Y}`)}return X}async ensureRunning($,X){if(this.stopped)throw Error("WorkerRuntimeManager is stopped");let Y=this.runtimes.get($);if(Y?.status==="failed"){let J=this.breakerCooldownMs(Y.breakerTrips),K=Date.now()-Y.lastFailureAt;if(K>=J)this.log.log(`[WorkerRuntimeManager] auto-heal: ${$} cooldown elapsed (${Math.round(K/1000)}s since trip #${Y.breakerTrips}) — `+"clearing circuit breaker and attempting a fresh spawn"),this.healBreaker(Y);else{let V=J-K;throw Error(`[WorkerRuntimeManager] cannot ensureRunning(${$}): ${Y.lastError??"runtime is in failed state"} Will auto-retry in ~${Math.ceil(V/1000)}s; call resetFailure(${$}) to retry immediately, or stop(${$}) to give up.`)}}X=await this.maybeAutoPull($,X);let Z=this.runtimes.get($);if(Z?.status==="running")return this.touch($),this.snapshot(Z);if(Z?.startPromise){let J=await Z.startPromise;return this.snapshot(J)}let z=Z??this.makeSlot($,X);if(!Z)this.runtimes.set($,z);z.spawnConfig=X,z.startPromise=this.doStart(z);try{let J=await z.startPromise;return this.enforceMaxRuntimes($),this.snapshot(J)}finally{z.startPromise=null}}enforceMaxRuntimes($){let X=this.opts.maxRuntimes;if(X==null||!Number.isFinite(X)||X<=0)return;let Y=Date.now(),Z=Array.from(this.runtimes.values()).filter((K)=>K.status==="running");if(Z.length<=X)return;let z=Z.filter((K)=>K.projectId!==$&&Y-K.lastUsedAt>=L6).sort((K,V)=>K.lastUsedAt-V.lastUsedAt),J=Z.length-X;for(let K of z){if(J<=0)break;let V=Y-K.lastUsedAt;this.log.log(`[WorkerRuntimeManager] maxRuntimes=${X} exceeded (${Z.length} running) — `+`LRU-evicting ${K.projectId} (idle ${Math.round(V/1000)}s)`),this.stop(K.projectId).catch((H)=>{this.log.warn(`[WorkerRuntimeManager] maxRuntimes eviction of ${K.projectId} failed: ${H?.message??H}`)}),J--}if(J>0)this.log.log(`[WorkerRuntimeManager] maxRuntimes=${X} still exceeded by ${J} after eviction pass — `+"remaining over-cap slots are mid-stream; will retry on next spawn / idle reap")}async ensurePulled($,X){return this.maybeAutoPull($,X)}async maybeAutoPull($,X){let Y=this.opts.autoPull;if(X.projectDir&&h0(X.projectDir))return X;if(!Y)throw Error(f0($,"no-auto-pull-config",null));if(!Y.projectsDir)throw Error(f0($,"no-projects-dir",null));if(!Y.enabled){let K=Z0(Y.projectsDir,$);if(h0(K)&&!M2(K))return{...X,projectDir:K};throw Error(f0($,"auto-pull-disabled",K))}if(this.pulledProjects.has($))return{...X,projectDir:Z0(Y.projectsDir,$)};let Z=Z0(Y.projectsDir,$),z=Y.logger??this.log,J={cloneProject:Y.gitOps?.cloneProject??v0,gitIsAvailable:Y.gitOps?.gitIsAvailable??E0,isGitRepo:Y.gitOps?.isGitRepo??X0};this.pulledProjects.add($);try{A2(Z,{recursive:!0});let K=M2(Z),V=J.isGitRepo(Z),Q=(Y.useGit!==!1?await J.gitIsAvailable():!1)&&(K||V)?"git":"files";if(this.syncModes.set($,Q),Q==="git"){if(K){z.log(`[WorkerRuntimeManager] auto-pull: git clone project ${$} into ${Z}`);try{let G=await J.cloneProject({apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$,localDir:Z,shallow:!0,logger:z});z.log(`[WorkerRuntimeManager] auto-pull: ${$} cloned at ${G.commitSha.slice(0,8)}`)}catch(G){z.warn(`[WorkerRuntimeManager] auto-pull: git clone failed for ${$} (${G?.message??G}); falling back to CloudFileTransport.downloadAll`),this.syncModes.set($,"files"),await this.fileTransportClone($,Z,X,z)}}else if(V)z.log(`[WorkerRuntimeManager] auto-pull: ${$} already has .git/; skipping clone`);if(this.syncModes.get($)==="git")await this.topUpShogoState($,Z,X,z)}else if(K)await this.fileTransportClone($,Z,X,z);else z.log(`[WorkerRuntimeManager] auto-pull: ${$} workspace already populated; skipping clone`);if(Y.watch!==!1&&!this.watchers.has($))try{let G=new j0({apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$,localDir:Z}),W=this.syncModes.get($)??"files",C=new Y0({rootDir:Z,transport:G,logger:z,mode:W,git:W==="git"?{apiUrl:X.cloudUrl,apiKey:X.apiKey,projectId:$}:void 0});C.start(),this.watchers.set($,C)}catch(G){z.warn(`[WorkerRuntimeManager] auto-pull: watcher start failed for ${$}: ${G?.message??G}`)}}catch(K){z.warn(`[WorkerRuntimeManager] auto-pull: failed for ${$} — runtime will fall back to template defaults. `+`(${K?.message??K})`)}return{...X,projectDir:Z}}async fileTransportClone($,X,Y,Z){Z.log(`[WorkerRuntimeManager] auto-pull: file-transport clone of ${$} into ${X}`);let J=await new j0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:X}).downloadAll();Z.log(`[WorkerRuntimeManager] auto-pull: ${$} downloaded ${J.downloaded} files (${J.errors.length} errors)`)}async topUpShogoState($,X,Y,Z){try{let z=new j0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:X}),K=(await z.listManifest()).filter((H)=>H.path===".shogo"||H.path.startsWith(".shogo/"));if(K.length===0)return;let V=await z.downloadFiles(K);Z.log(`[WorkerRuntimeManager] auto-pull: ${$} .shogo/ top-up downloaded ${V.downloaded} files (${V.errors.length} errors)`)}catch(z){Z.warn(`[WorkerRuntimeManager] auto-pull: .shogo top-up failed for ${$}: ${z?.message??z}`)}}shouldDeferToCloudSync($){return!!this.opts.autoPull?.enabled&&this.pulledProjects.has($)}status($){let X=this.runtimes.get($);return X?this.snapshot(X):null}getActiveProjects(){return Array.from(this.runtimes.keys()).filter(($)=>{let X=this.runtimes.get($);return X&&(X.status==="running"||X.status==="starting"||X.status==="restarting")})}touch($){let X=this.runtimes.get($);if(!X)return;X.lastUsedAt=Date.now(),this.armIdleTimer(X)}async stop($,X="SIGTERM"){let Y=this.runtimes.get($);if(!Y)return;if(Y.status="stopping",Y.restartTimer)clearTimeout(Y.restartTimer),Y.restartTimer=null;if(Y.idleTimer)clearTimeout(Y.idleTimer),Y.idleTimer=null;if(Y.graceTimer)clearTimeout(Y.graceTimer),Y.graceTimer=null;if(Y.limitWatchdog)Y.limitWatchdog.stop(),Y.limitWatchdog=null;if(Y.proc){this.killProcessGroup(Y,X);try{Y.proc.kill(X)}catch{}await this.waitForExit(Y.proc,5000),this.killProcessGroup(Y,"SIGKILL")}Y.pid=null,this.releasePort(Y.agentPort),this.runtimes.delete($)}resetFailure($){let X=this.runtimes.get($);if(!X||X.status!=="failed")return!1;if(X.restartTimer)clearTimeout(X.restartTimer),X.restartTimer=null;if(X.idleTimer)clearTimeout(X.idleTimer),X.idleTimer=null;if(X.graceTimer)clearTimeout(X.graceTimer),X.graceTimer=null;if(X.limitWatchdog)X.limitWatchdog.stop(),X.limitWatchdog=null;return this.runtimes.delete($),this.log.log(`[WorkerRuntimeManager] resetFailure: ${$} cleared, next ensureRunning will respawn`),!0}breakerCooldownMs($){let X=Math.max(0,$-1),Y=B6*Math.pow(2,X);return Math.min(Y,F6)}healBreaker($){$.status="stopped",$.consecutiveFailures=0}async stopAll($="SIGTERM"){this.stopped=!0;let X=Array.from(this.watchers.keys());await Promise.all(X.map(async(Z)=>{let z=this.watchers.get(Z);if(this.watchers.delete(Z),z)try{await z.stop()}catch(J){this.log.warn(`[WorkerRuntimeManager] watcher stop ${Z}: ${J?.message??J}`)}}));let Y=Array.from(this.runtimes.keys());await Promise.all(Y.map((Z)=>this.stop(Z,$).catch((z)=>{this.log.error(`[WorkerRuntimeManager] Failed to stop ${Z}: ${z?.message??z}`)})))}makeSlot($,X){return{projectId:$,agentPort:0,apiServerPort:0,status:"starting",proc:null,pid:null,startedAt:0,lastStdoutAt:0,lastUsedAt:Date.now(),restarts:0,consecutiveFailures:0,lastFailureAt:0,breakerTrips:0,restartTimestamps:[],graceTimer:null,restartTimer:null,idleTimer:null,limitWatchdog:null,spawnConfig:X,startPromise:null}}async doStart($){let X=this.resolveBinary();if(!X)throw $.status="error",$.lastError="agent-runtime binary not found (run `shogo runtime install`)",Error($.lastError);if(!$.agentPort)$.agentPort=await this.allocatePort(),$.apiServerPort=$.agentPort+B2;let Y=this.buildEnv($,X.path),Z=this.resolveCwd($),z=this.spawnCommand(X.path),J=this.resolveLimits(),K=z.command,V=z.args,H=!1;if(J){let k=w2({command:z.command,args:z.args,limits:J,scopeName:`shogo-rt-${$.projectId.slice(0,8)}-${Date.now().toString(36)}`,logger:this.log});K=k.command,V=k.args,H=k.wrapped}this.log.log(`[WorkerRuntimeManager] Spawning agent-runtime for ${$.projectId} via ${K} ${V.join(" ")} (port=${$.agentPort}, source=${X.source}${J?`, memCeil=${J.memoryMB}M`:""})`);let q=process.platform!=="win32",Q=q6(K,V,{cwd:Z,env:Y,detached:q,stdio:["ignore","pipe","pipe"]});if(q)try{Q.unref()}catch{}$.proc=Q,$.pid=Q.pid??null,$.status="starting",$.startedAt=Date.now();let G=$.spawnConfig.extraEnv?.SHOGO_OPEN_ID;if(L0("agent-spawned",$.projectId,G,{pid:$.pid,command:K,args:V,source:X.source}),J)try{this.applyPostSpawnLimits($,J,H)}catch(k){this.log.warn(`[WorkerRuntimeManager] failed to apply resource limits for ${$.projectId}: ${k?.message??k}`)}$.lastStdoutAt=$.startedAt,Q.on("error",(k)=>{$.lastError=k?.message??String(k),this.log.error(`[WorkerRuntimeManager] spawn error for ${$.projectId}: ${$.lastError}`)}),Q.on("exit",(k,L)=>{this.handleExit($,k,L)});let W=`[runtime:${$.projectId.slice(0,8)}]`,C=!0;Q.stdout?.on("data",(k)=>{if($.lastStdoutAt=Date.now(),C)C=!1,L0("agent-first-output",$.projectId,G,{elapsedMs:Date.now()-($.startedAt??Date.now())});for(let L of k.toString().trimEnd().split(`
|
|
13
|
+
`))if(L)this.log.log(`${W} ${L}`)}),Q.stderr?.on("data",(k)=>{if($.lastStdoutAt=Date.now(),C)C=!1,L0("agent-first-output",$.projectId,G,{elapsedMs:Date.now()-($.startedAt??Date.now()),stream:"stderr"});for(let L of k.toString().trimEnd().split(`
|
|
14
|
+
`))if(L)this.log.error(`${W} ${L}`)});try{return await this.waitForHealth($,_6),L0("agent-health-ready",$.projectId,G,{elapsedMs:Date.now()-($.startedAt??Date.now()),agentPort:$.agentPort}),$.status="running",$.lastUsedAt=Date.now(),this.armIdleTimer($),this.armGraceTimer($),$}catch(k){$.status="error",$.lastError=k?.message??String(k),this.killProcessGroup($,"SIGTERM");try{Q.kill("SIGTERM")}catch{}throw this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,k}}resolveLimits(){if(this.resolvedLimits===void 0)this.resolvedLimits=W2(this.opts.env??process.env);return this.resolvedLimits}applyPostSpawnLimits($,X,Y){let Z=$.pid;if(!Z)return;if(process.platform==="linux"){if(Y)return;if(C2({pid:Z,limits:X,logger:this.log}))return}else if(process.platform==="win32"){if(L2({pid:Z,limits:X,logger:this.log}))return}$.limitWatchdog=new S0({pid:Z,ceilingMB:X.memoryMB,logger:this.log,onBreach:(z)=>this.handleLimitBreach($,z,X.memoryMB)}),$.limitWatchdog.start()}handleLimitBreach($,X,Y){if(this.log.warn(`[WorkerRuntimeManager] runtime ${$.projectId} exceeded memory ceiling (${X}MB > ${Y}MB) — restarting`),$.lastError=`memory ceiling exceeded (${X}MB > ${Y}MB)`,$.limitWatchdog)$.limitWatchdog.stop(),$.limitWatchdog=null;this.killProcessGroup($,"SIGTERM");try{$.proc?.kill("SIGTERM")}catch{}let Z=setTimeout(()=>this.killProcessGroup($,"SIGKILL"),5000);try{Z.unref?.()}catch{}}killProcessGroup($,X){if(!$.pid)return;if(process.platform==="win32"){try{H6("taskkill",["/F","/T","/PID",String($.pid)],{stdio:"ignore",windowsHide:!0})}catch{}return}try{process.kill(-$.pid,X)}catch{}}armGraceTimer($){if($.graceTimer)clearTimeout($.graceTimer),$.graceTimer=null;$.graceTimer=setTimeout(()=>{if($.graceTimer=null,$.consecutiveFailures>0)$.consecutiveFailures=0;$.breakerTrips=0,$.restartTimestamps=[]},T6);try{$.graceTimer.unref?.()}catch{}}buildEnv($,X){let Y=$.spawnConfig,Z={...this.opts.env??process.env,PROJECT_ID:$.projectId,PORT:String($.agentPort),API_SERVER_PORT:String($.apiServerPort),SKILL_SERVER_PORT:String($.apiServerPort),WORKSPACE_API_PORT_BASE:String($.agentPort+w6),NODE_ENV:"production",STARTUP_TIME:String(Date.now()),SHOGO_CLOUD_URL:Y.cloudUrl,SHOGO_API_URL:Y.cloudUrl,SHOGO_API_KEY:Y.apiKey,RUNTIME_AUTH_SECRET:D2($.projectId),WEBHOOK_TOKEN:S6($.projectId)};if(Y.projectDir)Z.PROJECT_DIR=Y.projectDir,Z.WORKSPACE_DIR=Y.projectDir;if(this.shouldDeferToCloudSync($.projectId))Z.SHOGO_CLOUD_SYNC="1";if(Y.aiProxyUrl)Z.AI_PROXY_URL=Y.aiProxyUrl;if(Y.aiProxyToken)Z.AI_PROXY_TOKEN=Y.aiProxyToken;if(Y.techStackId)Z.TECH_STACK_ID=Y.techStackId;if(Y.name)Z.AGENT_NAME=Y.name;if(Y.workspaceId)Z.WORKSPACE_ID=Y.workspaceId;if(!Z.TREE_SITTER_WASM_DIR)Z.TREE_SITTER_WASM_DIR=Z0(O6(X),"tree-sitter-wasm");if(Y.extraEnv)Object.assign(Z,Y.extraEnv);let z=this.resolveLimits();if(z)O2(Z,z.memoryMB);return Z}resolveCwd($){let X=$.spawnConfig;if(X.projectDir&&h0(X.projectDir))return X.projectDir;let Y=this.opts.runtimeWorkDir??Z0(W6(),"shogo-runtime",$.projectId);return A2(Y,{recursive:!0}),Y}handleExit($,X,Y){let Z=Y===null&&X===0;if(this.log.log(`[WorkerRuntimeManager] runtime ${$.projectId} exited (code=${X}, signal=${Y})`),$.proc=null,$.graceTimer)clearTimeout($.graceTimer),$.graceTimer=null;if($.limitWatchdog)$.limitWatchdog.stop(),$.limitWatchdog=null;if($.status==="stopping"||this.stopped){$.status="stopped",$.pid=null,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0;return}if(Z){$.status="stopped",$.pid=null,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,this.runtimes.delete($.projectId);return}this.killProcessGroup($,"SIGKILL"),$.pid=null;let z=Date.now(),J=z-$.lastFailureAt<=T2;$.consecutiveFailures=J?$.consecutiveFailures+1:1,$.lastFailureAt=z,$.restarts+=1,$.lastError=`exited code=${X} signal=${Y}`,$.restartTimestamps.push(z),$.restartTimestamps=$.restartTimestamps.filter((q)=>z-q<=y2);let K=$.consecutiveFailures>=F2,V=$.restartTimestamps.length>=A6;if(K||V){$.status="failed",$.breakerTrips+=1;let q=this.breakerCooldownMs($.breakerTrips),Q=K?`${$.consecutiveFailures} consecutive non-clean exits within ${Math.round(T2/1000)}s`:`${$.restartTimestamps.length} non-clean exits within the last ${Math.round(y2/60000)}m (slow-bleed loop — gaps between crashes kept clearing the tight-loop window)`;if($.lastError=`Circuit breaker tripped (trip #${$.breakerTrips}): ${Q} (last: code=${X} signal=${Y}). Most recent on macOS is jetsam OOM (signal=SIGKILL with code=null); the previous incarnation's vite/tsserver/preview-manager children were reaped to prevent further RSS growth. Will auto-retry in ~${Math.round(q/1000)}s; call resetFailure(${$.projectId}) to retry immediately, or stop(${$.projectId}) to give up.`,this.releasePort($.agentPort),$.agentPort=0,$.apiServerPort=0,$.restartTimer)clearTimeout($.restartTimer),$.restartTimer=null;if($.idleTimer)clearTimeout($.idleTimer),$.idleTimer=null;this.log.error(`[WorkerRuntimeManager] ${$.lastError}`);return}let H=this.restartBackoffMs($.restarts);if($.status="restarting",this.log.warn(`[WorkerRuntimeManager] restarting ${$.projectId} in ${Math.round(H/1000)}s (restart #${$.restarts}, consecutive failures ${$.consecutiveFailures}/${F2})`),$.restartTimer)clearTimeout($.restartTimer);$.restartTimer=setTimeout(()=>{if($.restartTimer=null,$.status==="failed"||this.stopped)return;$.startPromise=this.doStart($).then((q)=>{return $.startPromise=null,q}).catch((q)=>{return $.startPromise=null,this.log.error(`[WorkerRuntimeManager] restart of ${$.projectId} failed: ${q?.message??q}`),$})},H);try{$.restartTimer.unref?.()}catch{}}restartBackoffMs($){let X=Math.min(U6*Math.pow(2,Math.max(0,$-1)),N6),Y=X*0.2*Math.random();return X+Y}armIdleTimer($){if($.idleTimer)clearTimeout($.idleTimer),$.idleTimer=null;let X=this.opts.idleMs??C6;if(!Number.isFinite(X)||X<=0)return;$.idleTimer=setTimeout(()=>{let Y=Date.now()-$.lastUsedAt;if(Y<X){this.armIdleTimer($);return}this.log.log(`[WorkerRuntimeManager] idle-evicting ${$.projectId} after ${Math.round(Y/1000)}s`),this.stop($.projectId).catch((Z)=>{this.log.warn(`[WorkerRuntimeManager] idle stop failed: ${Z?.message??Z}`)})},X)}async allocatePort(){let $=I0-m0,X=Math.min($,50);for(let Y=0;Y<X;Y++){let Z=m0+Math.floor(Math.random()*$);if(Z+C0-1>I0)continue;let z=!0;for(let V=0;V<C0;V++)if(this.usedPorts.has(Z+V)){z=!1;break}if(!z)continue;let J=await this.isPortListening(Z),K=await this.isPortListening(Z+B2);if(J||K)continue;for(let V=0;V<C0;V++)this.usedPorts.add(Z+V);return Z}throw Error(`Cannot allocate port in range ${m0}-${I0} after ${X} attempts`)}releasePort($){if(!$)return;for(let X=0;X<C0;X++)this.usedPorts.delete($+X)}async isPortListening($){let X=new AbortController,Y=setTimeout(()=>X.abort(),250);try{return await fetch(`http://127.0.0.1:${$}/`,{method:"HEAD",signal:X.signal}),clearTimeout(Y),!0}catch{return clearTimeout(Y),!1}}tcpProbe($){return new Promise((X)=>{let Y=!1,Z=(J)=>{if(Y)return;Y=!0;try{z.destroy()}catch{}X(J)},z=k6({host:"127.0.0.1",port:$});z.setTimeout(x6),z.once("connect",()=>Z(!0)),z.once("error",()=>Z(!1)),z.once("timeout",()=>Z(!1))})}async waitForHealth($,X){let{agentPort:Y,proc:Z}=$;if(!Z)throw Error(`waitForHealth: slot ${$.projectId} has no spawned process`);let z=Date.now(),J=z+X,K=null,V=!1,H=0,q=0,Q=0,G=0,W=z;while(Date.now()<J){if(G++,Z.exitCode!==null||Z.signalCode!=null||Z.killed)throw Error(`agent-runtime exited (code=${Z.exitCode}, signal=${Z.signalCode}) before becoming healthy on port ${Y}`);q++;let C=new AbortController,k=setTimeout(()=>C.abort(),1500);try{let O=await fetch(`http://127.0.0.1:${Y}/health`,{method:"GET",signal:C.signal});if(clearTimeout(k),O.ok){this.log.log(`[WorkerRuntimeManager] /health ready for ${$.projectId} on port ${Y} (HTTP ${O.status} after ${Date.now()-z}ms, ${G} iter, ${q} http)`);return}K=`HTTP /health returned ${O.status}`}catch(O){clearTimeout(k);let T=O?.name??"Error",c=O?.code??O?.cause?.code;K=`HTTP /health failed: ${T}${c?`(${c})`:""}: ${O?.message??O}`}if(Q++,V=await this.tcpProbe(Y),V){H=Date.now();let O=Date.now()-$.lastStdoutAt;if(O>=_2)throw Error(`agent-runtime wedged on port ${Y}: TCP listening but stdout silent for ${O}ms (> ${_2}ms window); ${q} /health attempts, last error: ${K??"n/a"}`)}let L=Date.now();if(L-W>=D6){let O=L-z,T=L-$.lastStdoutAt,c=H>0?L-H:null;this.log.log(`[WorkerRuntimeManager] still waiting for /health on ${$.projectId} port ${Y} (${(O/1000).toFixed(1)}s elapsed, tcpListening=${V}${c!=null?`(${c}ms ago)`:""}, lastStdout=${T}ms ago, ${q} http, ${Q} tcp, lastError=${K??"n/a"})`),W=L}await new Promise((O)=>setTimeout(O,y6))}if(Z.exitCode!==null||Z.signalCode!=null||Z.killed)throw Error(`agent-runtime exited (code=${Z.exitCode}, signal=${Z.signalCode}) before becoming healthy on port ${Y}`);throw Error(`Timeout waiting for agent-runtime /health on port ${Y} after ${G} iter (httpAttempts=${q}, tcpAttempts=${Q}, tcpListening=${V}, lastError=${K??"n/a"})`)}async waitForExit($,X){if($.exitCode!==null||$.signalCode!=null||$.killed)return;await new Promise((Y)=>{let Z=setTimeout(()=>{try{$.kill("SIGKILL")}catch{}Y()},X);$.once("exit",()=>{clearTimeout(Z),Y()})})}snapshot($){return{projectId:$.projectId,status:$.status,agentPort:$.agentPort||void 0,apiServerPort:$.apiServerPort||void 0,pid:$.proc?.pid,startedAt:$.startedAt||void 0,lastUsedAt:$.lastUsedAt,restarts:$.restarts,lastError:$.lastError}}}import{hostname as d0,platform as g0,arch as i0}from"node:os";function d(){if(typeof __SHOGO_WORKER_VERSION__==="string"&&__SHOGO_WORKER_VERSION__.length>0)return`shogo-cli/${__SHOGO_WORKER_VERSION__}`;return"shogo-cli/unknown"}class v2 extends Error{code="TUNNEL_WS_HEADERS_UNSUPPORTED";constructor(){super("Tunnel WebSocket auth requires a runtime with WebSocket header support (Bun or Node >= 21). Current runtime does not support WebSocket constructor headers.");this.name="TunnelWebSocketHeaderSupportError"}}var z0=60,U0=300,p0=3,R6=p0,b6=1800000,E6=25000,R2=1000,b2=60000,E2=3;class c0{opts;log;pollTimer=null;ws=null;wsIdleTimer=null;heartbeatTimer=null;stopped=!1;currentPollInterval=z0;wsReconnectAttempt=0;lastHeartbeatError=null;consecutiveAuthFailures=0;consecutiveAuthSuccesses=0;serverPublishedWsUrl=null;activeAbortControllers=new Map;constructor($){this.opts=$,this.log=$.logger??console}start(){if(!this.opts.apiKey){this.log.log("[WorkerTunnel] No API key set, skipping tunnel");return}this.stopped=!1,this.wsReconnectAttempt=0,this.currentPollInterval=z0,this.log.log("[WorkerTunnel] Starting heartbeat polling to Shogo Cloud..."),this.heartbeatLoop()}stop(){if(this.stopped=!0,this.pollTimer)clearTimeout(this.pollTimer),this.pollTimer=null;this.cleanupWs(),this.log.log("[WorkerTunnel] Tunnel stopped")}isConnected(){if(this.ws!==null&&this.ws.readyState===WebSocket.OPEN)return!0;return!this.stopped&&!!this.opts.apiKey&&this.lastHeartbeatError===null&&this.pollTimer!==null}getCloudUrl(){return this.opts.cloudUrl.replace(/\/$/,"")}getWsBaseUrl(){let $=(this.opts.wsUrlOverride||process.env.SHOGO_TUNNEL_WS_URL||"").trim();if($)return $.replace(/\/$/,"");if(this.serverPublishedWsUrl)return this.serverPublishedWsUrl.replace(/\/$/,"");return this.getCloudUrl().replace(/^http/,"ws")}buildWsUrl(){return`${this.getWsBaseUrl()}/api/instances/ws`}supportsWebSocketConstructorHeaders($=globalThis){if(typeof $.Bun<"u"||typeof $.process?.versions?.bun==="string")return!0;let X=$.process?.versions?.node;if(X){if(parseInt(X.split(".")[0]??"0",10)>=21)return!0}return!1}createTunnelWebSocket($,X,Y=globalThis){if(!this.supportsWebSocketConstructorHeaders(Y))throw new v2;return new WebSocket($,X)}getReconnectDelayMs(){let $=Math.min(R2*Math.pow(2,this.wsReconnectAttempt),b2),X=$*0.2*Math.random();return $+X}async collectMetadata(){let $={hostname:d0(),os:g0(),arch:i0(),uptime:process.uptime(),protocolVersion:E2,tunnelStatus:this.ws?.readyState===WebSocket.OPEN?"connected":"polling",kind:this.opts.kind??"cli-worker"};try{let X=this.opts.resolver.getActiveProjects();$.activeProjects=X.length,$.projects=X.map((Y)=>{let Z=this.opts.resolver.status(Y);return{projectId:Y,status:Z?.status??"unknown",agentPort:Z?.agentPort}})}catch{$.activeProjects=0}return $}async sendHeartbeat(){let $=this.getCloudUrl(),X=await this.collectMetadata(),Y=d0(),Z=this.opts.name??Y,z=await fetch(`${$}/api/instances/heartbeat`,{method:"POST",headers:{"Content-Type":"application/json","User-Agent":d(),"x-api-key":this.opts.apiKey,"x-shogo-kind":this.opts.kind??"cli-worker"},body:JSON.stringify({hostname:Y,name:Z,os:g0(),arch:i0(),kind:this.opts.kind??"cli-worker",metadata:X})});if(!z.ok)throw Error(`Heartbeat failed: HTTP ${z.status}`);let J=await z.json();if(typeof J.wsUrl==="string"&&J.wsUrl.length>0){if(J.wsUrl!==this.serverPublishedWsUrl)this.log.log(`[WorkerTunnel] Cloud advertised tunnel WS URL: ${J.wsUrl}`);this.serverPublishedWsUrl=J.wsUrl}return J}scheduleNextPoll($){if(this.stopped)return;if(this.pollTimer)clearTimeout(this.pollTimer);let X=($??this.currentPollInterval)*1000;this.pollTimer=setTimeout(()=>void this.heartbeatLoop(),X)}async heartbeatLoop(){if(this.stopped)return;if(this.ws&&this.ws.readyState===WebSocket.OPEN){this.scheduleNextPoll(this.currentPollInterval);return}try{let $=await this.sendHeartbeat(),X=$.nextPollIn||z0;if(this.consecutiveAuthFailures>=p0){if(this.consecutiveAuthSuccesses++,this.consecutiveAuthSuccesses<R6){this.currentPollInterval=U0,this.scheduleNextPoll();return}}if(this.currentPollInterval=X,this.lastHeartbeatError)this.log.log("[WorkerTunnel] Heartbeat recovered"),this.lastHeartbeatError=null;if(this.consecutiveAuthFailures=0,this.consecutiveAuthSuccesses=0,$.wsRequested&&!this.ws){this.log.log("[WorkerTunnel] Cloud requested WebSocket — connecting..."),this.connectWs();return}}catch($){let X=$?.message??String($);if(/HTTP 40[13]\b/.test(X))this.consecutiveAuthFailures++,this.consecutiveAuthSuccesses=0;else this.consecutiveAuthFailures=0,this.consecutiveAuthSuccesses=0;if(X!==this.lastHeartbeatError)this.log.error(`[WorkerTunnel] Heartbeat error: ${X}`),this.lastHeartbeatError=X;if(this.consecutiveAuthFailures>=p0){let Z=`tunnel saw ${this.consecutiveAuthFailures} consecutive auth failures from Shogo Cloud`;try{this.opts.onAuthRevoked?.(Z)}catch(z){this.log.warn(`[WorkerTunnel] onAuthRevoked threw: ${z?.message??z}`)}if(this.currentPollInterval!==U0)this.log.warn(`[WorkerTunnel] ${this.consecutiveAuthFailures} consecutive auth failures — `+`backing off to ${U0}s. Re-authenticate with \`shogo login\`.`);this.currentPollInterval=U0}else this.currentPollInterval=z0}this.scheduleNextPoll()}async handleRequest($){if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;this.resetWsIdleTimer();let X=new AbortController;this.activeAbortControllers.set($.requestId,X);try{let Y=await this.resolveLocalUrl($.path,$.projectId);if(!Y){let K=this.opts.resolver.describeRejection?this.opts.resolver.describeRejection($.path,$.projectId):{code:"NO_LOCAL_RUNTIME",message:`no local runtime available for path: ${$.path}`};this.sendFrame({type:"response",requestId:$.requestId,status:502,headers:{"content-type":"application/json"},body:JSON.stringify({code:K.code,message:K.message,path:$.path})});return}let Z={...$.headers??{}};if($.projectId&&($.path.startsWith("/agent/")||$.path==="/agent")){let K=this.opts.resolver.deriveRuntimeToken($.projectId);if(K)Z["x-runtime-token"]=K}let z={method:$.method,headers:Z,signal:X.signal};if($.body&&$.method!=="GET"&&$.method!=="HEAD")z.body=$.body;let J=await fetch(Y,z);if($.stream){let K=J.body?.getReader();if(!K){this.sendFrame({type:"stream-error",requestId:$.requestId,error:"No response body for stream"});return}let V=new TextDecoder;try{while(!0){let{done:H,value:q}=await K.read();if(H)break;if(this.ws?.readyState!==WebSocket.OPEN)break;this.sendFrame({type:"stream-chunk",requestId:$.requestId,data:V.decode(q,{stream:!0})})}if(this.ws?.readyState===WebSocket.OPEN)this.sendFrame({type:"stream-end",requestId:$.requestId})}catch(H){if(H?.name!=="AbortError"&&this.ws?.readyState===WebSocket.OPEN)this.sendFrame({type:"stream-error",requestId:$.requestId,error:H?.message??String(H)})}}else{let K=await J.text(),V={};J.headers.forEach((H,q)=>{V[q]=H}),this.sendFrame({type:"response",requestId:$.requestId,status:J.status,headers:V,body:K})}}catch(Y){if(Y?.name==="AbortError")return;if(this.ws?.readyState!==WebSocket.OPEN)return;let Z=$.stream?{type:"stream-error",requestId:$.requestId,error:Y?.message??String(Y)}:{type:"response",requestId:$.requestId,status:502,body:JSON.stringify({error:Y?.message??String(Y)})};this.sendFrame(Z)}finally{this.activeAbortControllers.delete($.requestId)}}async resolveLocalUrl($,X){return this.opts.resolver.resolveLocalUrl($,X)}sendFrame($){if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;try{this.ws.send(JSON.stringify($))}catch(X){this.log.warn(`[WorkerTunnel] Frame send failed: ${X?.message??X}`)}}resetWsIdleTimer(){if(this.wsIdleTimer)clearTimeout(this.wsIdleTimer);this.wsIdleTimer=setTimeout(()=>{if(this.ws&&this.ws.readyState===WebSocket.OPEN){this.log.log("[WorkerTunnel] WebSocket idle timeout — closing, returning to polling");try{this.ws.close(1000,"Idle timeout")}catch{}}},b6)}startWsHeartbeat(){if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.heartbeatTimer=setInterval(async()=>{if(!this.ws||this.ws.readyState!==WebSocket.OPEN)return;try{let $=await this.collectMetadata();this.sendFrame({type:"heartbeat",metadata:$})}catch{}},E6)}connectWs(){if(this.stopped||this.ws)return;let $=this.buildWsUrl(),X=d0(),Y=g0(),Z=i0(),z=this.opts.name??X;this.log.log(`[WorkerTunnel] Opening WebSocket to ${$} (hostname=${X})`);let J={headers:{Authorization:`Bearer ${this.opts.apiKey}`,"User-Agent":d(),"x-shogo-hostname":X,"x-shogo-name":z,"x-shogo-os":Y,"x-shogo-arch":Z,"x-shogo-kind":this.opts.kind??"cli-worker"}},K;try{K=this.createTunnelWebSocket($,J)}catch(V){this.log.error(`[WorkerTunnel] WebSocket creation failed: ${V?.message??V}`),this.ws=null,this.scheduleNextPoll(5);return}this.ws=K,K.onopen=()=>{this.log.log("[WorkerTunnel] WebSocket connected — session active"),this.wsReconnectAttempt=0,this.startWsHeartbeat(),this.resetWsIdleTimer()},K.onmessage=(V)=>{let H;try{let q=typeof V.data==="string"?V.data:V.data.toString();H=JSON.parse(q)}catch{return}if(H.type==="ping"){this.sendFrame({type:"pong"}),this.resetWsIdleTimer();return}if(H.type==="cancel"){let q=this.activeAbortControllers.get(H.requestId);if(q)q.abort();return}if(H.type==="request"){this.handleRequest(H).catch((q)=>{this.log.error(`[WorkerTunnel] Error handling request: ${q?.message??q}`)});return}},K.onclose=(V)=>{if(this.log.log(`[WorkerTunnel] WebSocket closed: code=${V.code} reason=${V.reason||"none"}`),this.cleanupWs(),this.stopped)return;if(V.code===1000||V.code===4000)this.scheduleNextPoll(this.currentPollInterval);else{this.wsReconnectAttempt++;let H=this.getReconnectDelayMs();this.log.log(`[WorkerTunnel] Reconnecting in ${Math.round(H/1000)}s (attempt ${this.wsReconnectAttempt})`),this.scheduleNextPoll(Math.ceil(H/1000))}},K.onerror=(V)=>{this.log.error("[WorkerTunnel] WebSocket error:",V?.message??"unknown")}}cleanupWs(){if(this.heartbeatTimer)clearInterval(this.heartbeatTimer),this.heartbeatTimer=null;if(this.wsIdleTimer)clearTimeout(this.wsIdleTimer),this.wsIdleTimer=null;for(let[,$]of this.activeAbortControllers)try{$.abort()}catch{}this.activeAbortControllers.clear(),this.ws=null}_testing(){let $=this;return{sendHeartbeat:()=>$.sendHeartbeat(),heartbeatLoop:()=>$.heartbeatLoop(),connectWs:()=>$.connectWs(),cleanupWs:()=>$.cleanupWs(),handleRequest:(X)=>$.handleRequest(X),installFakeWs:(X)=>{$.ws=X},getCloudUrl:()=>$.getCloudUrl(),getWsBaseUrl:()=>$.getWsBaseUrl(),buildWsUrl:()=>$.buildWsUrl(),getReconnectDelayMs:()=>$.getReconnectDelayMs(),supportsWebSocketConstructorHeaders:(X)=>$.supportsWebSocketConstructorHeaders(X),createTunnelWebSocket:(X,Y,Z)=>$.createTunnelWebSocket(X,Y,Z),DEFAULT_POLL_INTERVAL_S:z0,BACKOFF_BASE_MS:R2,BACKOFF_MAX_MS:b2,TUNNEL_PROTOCOL_VERSION:E2,get currentPollInterval(){return $.currentPollInterval},set currentPollInterval(X){$.currentPollInterval=X},get wsReconnectAttempt(){return $.wsReconnectAttempt},set wsReconnectAttempt(X){$.wsReconnectAttempt=X},get ws(){return $.ws},get stopped(){return $.stopped},get serverPublishedWsUrl(){return $.serverPublishedWsUrl},set serverPublishedWsUrl(X){$.serverPublishedWsUrl=X}}}}async function h2($){let X=R({name:$.name,workerDir:$.workerDir,apiKey:$.apiKey,cloudUrl:$.cloudUrl,port:$.port?parseInt($.port,10):void 0,projectsDir:$.projectsDir}),Y=J2($.proxy);if(!$.foreground)return h6({cfg:X,proxy:Y,flags:$});let Z=u({flag:$.runtimeBin});if(!Z)console.error(w.red(n({flag:$.runtimeBin}))),process.exit(1);if(M0(process.env,Y),$.debug){if(!await H2(q2({cloudUrl:X.cloudUrl,apiKey:X.apiKey,workerDir:X.workerDir,proxy:Y})))process.exit(1)}let z=!$.noAutoPull,J=!$.noGit;if(console.log(w.bold(`
|
|
15
|
+
Shogo Worker — Starting`)),console.log(w.dim(" name ")+X.name),console.log(w.dim(" worker-dir ")+X.workerDir),console.log(w.dim(" cloud ")+X.cloudUrl),console.log(w.dim(" runtime ")+`${Z.path} ${w.dim(`(via ${Z.source})`)}`),console.log(w.dim(" auto-pull ")+(z?w.green("on")+w.dim(` → ${X.projectsDir}`):w.yellow("off"))),console.log(w.dim(" sync mode ")+(J?w.green("git")+w.dim(" (falls back to file transport if git is missing)"):w.yellow("files-only"))),$.project)console.log(w.dim(" project ")+$.project);if(Y)console.log(w.dim(" proxy ")+`${Y.url} ${w.dim(`(from ${Y.source})`)}`);console.log("");let K=`${X.cloudUrl.replace(/\/+$/,"")}/api/ai/v1`,V={cloudUrl:X.cloudUrl,apiKey:X.apiKey,aiProxyUrl:K,aiProxyToken:X.apiKey},H=new n0({runtimeBin:$.runtimeBin,defaultSpawnConfig:V,autoPull:{enabled:z,projectsDir:X.projectsDir,watch:!0,useGit:J}});if(!H.resolveBinary())console.error(w.red(n({flag:$.runtimeBin}))),process.exit(1);let q=new c0({apiKey:X.apiKey,cloudUrl:X.cloudUrl,name:X.name,kind:"cli-worker",resolver:H,onAuthRevoked:(W)=>{console.error(w.red(`✗ Cloud auth revoked: ${W}`)),console.error(w.dim(" Run `shogo login` to re-authenticate; this worker will keep polling at the auth-failure backoff until then."))}}),Q=!1,G=async(W)=>{if(Q)return;Q=!0,console.log(w.dim(`
|
|
16
|
+
Received ${W} — shutting down...`));try{q.stop()}catch{}try{await H.stopAll()}catch(C){console.warn(w.yellow(`stopAll: ${C?.message??C}`))}process.exit(0)};process.once("SIGINT",()=>void G("SIGINT")),process.once("SIGTERM",()=>void G("SIGTERM")),process.once("SIGHUP",()=>void G("SIGHUP")),q.start(),console.log(w.green("✓ Worker running. Ctrl-C to stop.")),await new Promise(()=>{})}function h6({cfg:$,proxy:X,flags:Y}){let{entry:Z,runner:z}=m6(),J=M0({...process.env,SHOGO_API_KEY:$.apiKey,SHOGO_CLOUD_URL:$.cloudUrl,SHOGO_INSTANCE_NAME:$.name,SHOGO_WORKER_DIR:$.workerDir,SHOGO_LOCAL_MODE:"true",PORT:String($.port)},X),K=j6(Y),{pid:V}=Y2({entry:Z,runner:z,env:{...J,SHOGO_DETACHED_ARGS:K.join(" ")},cwd:$.workerDir,detach:!0,inheritStdio:!1});console.log(w.bold(`
|
|
17
|
+
Shogo Worker — Started`)),console.log(w.dim(" pid: ")+V),console.log(w.dim(" name: ")+$.name),console.log(w.dim(" logs: ")+"~/.shogo/logs/worker.log"),console.log(w.dim(" stop: ")+"shogo worker stop")}function j6($){let X=["worker","start","--foreground"];if($.name)X.push("--name",$.name);if($.workerDir)X.push("--worker-dir",$.workerDir);if($.apiKey)X.push("--api-key",$.apiKey);if($.cloudUrl)X.push("--cloud-url",$.cloudUrl);if($.port)X.push("--port",$.port);if($.proxy)X.push("--proxy",$.proxy);if($.project)X.push("--project",$.project);if($.runtimeBin)X.push("--runtime-bin",$.runtimeBin);if($.debug)X.push("--debug");if($.noAutoPull)X.push("--no-auto-pull");if($.projectsDir)X.push("--projects-dir",$.projectsDir);if($.noGit)X.push("--no-git");return X}function m6(){let $=process.execPath,X=/\bbun(?:-[^/\\]*)?$/.test($)||typeof globalThis.Bun<"u",Y=process.argv[1];if(Y&&v6(Y)){let z=/\.ts$/.test(Y);return{entry:Y,runner:X?"bun":z?"tsx":"node"}}return{entry:new URL("../../bin/shogo.mjs",import.meta.url).pathname,runner:X?"bun":"node"}}import j2 from"picocolors";async function m2(){let{killedPid:$}=Z2("SIGTERM");if($===null){console.log(j2.dim("No worker running."));return}console.log(j2.green(`✓ Worker stopped (pid=${$}).`))}import v from"picocolors";async function I2(){let $=G0();if(!$){console.log(v.yellow("● stopped")+v.dim(" (no pid file)"));return}if(!k0($)){console.log(v.red("● dead")+v.dim(` (stale pid ${$})`));return}let X=I();if(console.log(v.green("● running")+v.dim(` (pid ${$})`)),X.name)console.log(v.dim(" name: ")+X.name);if(X.cloudUrl)console.log(v.dim(" cloud: ")+X.cloudUrl);if(X.workerDir)console.log(v.dim(" dir: ")+X.workerDir)}import{spawn as I6}from"node:child_process";import{existsSync as u6,statSync as f6}from"node:fs";import u2 from"picocolors";async function f2($){let X=$.err?H0:q0;if(!u6(X)){console.log(u2.dim("No logs yet."));return}let Y=$.follow?["-F","-n","200",X]:["-n","200",X];if(I6("tail",Y,{stdio:"inherit"}).on("exit",(z)=>process.exit(z??0)),!$.follow){if(f6(X).size===0)console.log(u2.dim("(log file is empty)"))}}import _ from"picocolors";import{spawn as n6}from"node:child_process";import{hostname as d6,platform as g6,arch as i6}from"node:os";import f from"picocolors";class M extends Error{kind;constructor($,X){super($);this.kind=X;this.name="CloudLoginError"}}async function n2($){let X=$.cloudUrl.replace(/\/$/,""),Y=$.client??"cli",Z=$.deviceName??d6(),z=$.devicePlatform??`${g6()}-${i6()}`,J=$.appVersion??o6(),K=$.log??((O)=>console.log(O)),V=$.fetchImpl??fetch,H=await V(`${X}/api/cli/login/start`,{method:"POST",headers:{"content-type":"application/json","user-agent":d()},body:JSON.stringify({deviceId:$.deviceId,deviceName:Z,devicePlatform:z,deviceAppVersion:J,workspaceId:$.workspaceId,client:Y}),signal:AbortSignal.timeout(1e4)}).catch((O)=>{throw new M(`Cannot reach Shogo Cloud at ${X}: ${O?.message??O}`,"transport")});if(!H.ok){let O=await H.text().catch(()=>"");throw new M(`Cloud rejected /api/cli/login/start (HTTP ${H.status}): ${O||"no body"}`,"transport")}let q=await H.json().catch(()=>({}));if(!q?.ok||!q.state||!q.authUrl)throw new M(`Cloud returned a malformed start response: ${q?.error??JSON.stringify(q)}`,"transport");let Q=p6($.pollIntervalMs??q.pollIntervalMs),G=$.timeoutMs??q.expiresInMs,W=Date.now()+G;if(K(""),K(f.bold("Sign in to Shogo Cloud")),K(f.dim(" cloud: ")+X),K(f.dim(" device: ")+`${Z} (${z})`),K(f.dim(" user code: ")+f.cyan(q.userCode)),K(""),K(" Open this URL in your browser to approve:"),K(" "+f.cyan(q.authUrl)),K(""),$.openBrowser!==!1)if(typeof $.openBrowser==="function")try{await $.openBrowser(q.authUrl)}catch{}else a6(q.authUrl).catch(()=>{});K(f.dim("Waiting for approval..."));let C=$.installSignalHandlers??!0,k=()=>{L.abort()},L=new AbortController;if($.abortSignal)if($.abortSignal.aborted)L.abort();else $.abortSignal.addEventListener("abort",()=>L.abort(),{once:!0});if(C)process.once("SIGINT",k),process.once("SIGTERM",k);try{while(!0){if(L.signal.aborted)throw new M("Aborted by caller.","cancelled");if(Date.now()>=W)throw new M(`Timed out after ${Math.round(G/1000)}s waiting for approval.`,"timeout");let O=await V(`${X}/api/cli/login/poll?state=${encodeURIComponent(q.state)}`,{method:"GET",headers:{accept:"application/json","user-agent":d()},signal:AbortSignal.timeout(1e4)}).catch((T)=>{return K(f.dim(` (poll error: ${T?.message??T} — retrying)`)),null});if(O&&O.ok){let T=await O.json().catch(()=>({}));if(T?.status==="approved"&&T.key)return{key:T.key,email:T.email??null,workspace:T.workspace??null,deviceId:T.deviceId??$.deviceId};if(T?.status==="denied")throw new M("Sign-in was denied in the browser.","denied");if(T?.status==="expired")throw new M("Sign-in request expired before it was approved.","expired")}await c6(Q,L.signal)}}finally{if(C)process.removeListener("SIGINT",k),process.removeListener("SIGTERM",k)}}function p6($){if(!Number.isFinite($)||$<=0)return 2000;return Math.min(Math.max($,1000),1e4)}function c6($,X){return new Promise((Y,Z)=>{if(X?.aborted)return Z(new M("Aborted by caller.","cancelled"));let z=setTimeout(()=>{X?.removeEventListener("abort",J),Y()},$),J=()=>{clearTimeout(z),Z(new M("Aborted by caller.","cancelled"))};X?.addEventListener("abort",J,{once:!0})})}function o6(){return d()}function a6($){return new Promise((X)=>{let Y=process.platform==="darwin"?"open":process.platform==="win32"?"cmd":"xdg-open",Z=process.platform==="win32"?["/c","start",'""',$]:[$];try{let z=n6(Y,Z,{stdio:"ignore",detached:!0});z.on("error",()=>X()),z.unref(),X()}catch{X()}})}import{readFileSync as l6,writeFileSync as r6,existsSync as e6,chmodSync as s6}from"node:fs";import{randomUUID as t6}from"node:crypto";function d2(){if(e6(o)){let X=l6(o,"utf-8").trim();if(X)return X}m();let $=t6();return r6(o,$,{mode:384}),s6(o,384),$}var $1="https://studio.shogo.ai";async function g2($){let X=I(),Y=($.cloudUrl||X.cloudUrl||$1).replace(/\/$/,""),Z=$.apiKey||process.env.SHOGO_API_KEY;if(Z){await X1({key:Z,cloudUrl:Y,cfg:X,name:$.name});return}let z=d2(),J;try{J=await n2({cloudUrl:Y,deviceId:z,deviceName:$.name,workspaceId:$.workspace,openBrowser:!$.noBrowser})}catch(K){if(K instanceof M){if(console.error(_.red(`✗ ${K.message}`)),K.kind==="transport")console.error(_.dim(" If your network blocks browsers, run with --api-key <key> instead."));process.exit(1)}throw K}if(X.apiKey=J.key,X.cloudUrl=Y,$.name)X.name=$.name;if(e(X),console.log(""),console.log(_.green("✓ Signed in to Shogo Cloud")),J.workspace)console.log(_.dim(" workspace: ")+J.workspace);if(J.email)console.log(_.dim(" email: ")+J.email);console.log(_.dim(" saved to: ")+"~/.shogo/config.json"),console.log(""),console.log(_.dim(" next: ")+"shogo worker start")}async function X1({key:$,cloudUrl:X,cfg:Y,name:Z}){if(!/^shogo_sk_/.test($))console.error(_.red('✗ API key should start with "shogo_sk_". Copy it verbatim from the API Keys page.')),process.exit(1);let z;try{let J=await fetch(`${X}/api/api-keys/validate`,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({key:$}),signal:AbortSignal.timeout(1e4)});if(z=await J.json().catch(()=>({})),!J.ok||!z?.valid)console.error(_.red(`✗ ${z?.error||`Cloud rejected the key (HTTP ${J.status}).`}`)),process.exit(1)}catch(J){console.error(_.red(`✗ Cannot reach Shogo Cloud at ${X}: ${J?.message??J}`)),process.exit(1)}if(Y.apiKey=$,Y.cloudUrl=X,Z)Y.name=Z;if(e(Y),console.log(_.green("✓ API key saved to ~/.shogo/config.json")),z.workspace?.name)console.log(_.dim(" workspace: ")+z.workspace.name);if(z.user?.email)console.log(_.dim(" email: ")+z.user.email);console.log(_.dim(" next: ")+"shogo worker start")}import J0 from"picocolors";async function i2(){let $=I();if(Object.keys($).length===0){console.log(J0.dim("(no config — run `shogo login` or `shogo config set`)"));return}let X={...$,apiKey:$.apiKey?`***${$.apiKey.slice(-4)}`:void 0};console.log(J0.dim(`file: ${j}`)),console.log(JSON.stringify(X,null,2))}async function p2($,X){let Y=["apiKey","cloudUrl","name","workerDir","port"];if(!Y.includes($))console.error(J0.red(`Unknown key: ${$}`)),console.error(J0.dim(`Allowed: ${Y.join(", ")}`)),process.exit(1);let Z=I();Z[$]=$==="port"?parseInt(X,10):X,e(Z),console.log(J0.green(`✓ ${$} set`))}import A from"picocolors";import{spawn as Y1}from"node:child_process";import{createHash as Z1}from"node:crypto";import{createWriteStream as z1,existsSync as K0,mkdirSync as o0,readFileSync as a2,renameSync as c2,rmSync as a0,writeFileSync as J1}from"node:fs";import{tmpdir as K1}from"node:os";import{dirname as V1,join as N0}from"node:path";import{Readable as q1}from"node:stream";import{pipeline as H1}from"node:stream/promises";var Q1="https://github.com/shogo-labs/shogo-ai/releases/download";function l2(){let{platform:$,arch:X}=process,Y;if($==="darwin")Y="darwin";else if($==="linux")Y="linux";else if($==="win32")Y="windows";else throw Error(`Unsupported platform: ${$}`);let Z;if(X==="arm64")Z="arm64";else if(X==="x64")Z="x64";else throw Error(`Unsupported arch: ${X} (need arm64 or x64)`);return`${Y}-${Z}`}function A0(){if(!K0(r))return null;try{let $=a2(r,"utf-8");return JSON.parse($)}catch{return null}}async function G1($,X){let Y=X.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download/);if(!Y)throw Error(`Cannot auto-resolve latest version from non-GitHub baseUrl '${X}'. Pass --version explicitly.`);let Z=Y[1],z=Y[2];if($==="stable"){let Q=`https://api.github.com/repos/${Z}/${z}/releases/latest`,G=await fetch(Q,{headers:{Accept:"application/vnd.github+json"}});if(!G.ok)throw Error(`GitHub API ${G.status} for ${Q}`);let W=await G.json();if(!W.tag_name)throw Error("GitHub API did not return tag_name");return o2(W.tag_name)}let J=`https://api.github.com/repos/${Z}/${z}/releases?per_page=30`,K=await fetch(J,{headers:{Accept:"application/vnd.github+json"}});if(!K.ok)throw Error(`GitHub API ${K.status} for ${J}`);let V=await K.json(),H=$==="beta"?"-beta":"-nightly",q=V.find((Q)=>Q.prerelease&&/^v\d+\.\d+\.\d+-/.test(Q.tag_name)&&Q.tag_name.includes(H));if(!q)throw Error(`No ${$} runtime release found`);return o2(q.tag_name)}function o2($){if(!/^v\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/.test($))throw Error(`Unexpected app tag '${$}' (expected vX.Y.Z[-prerelease])`);return $.slice(1)}function k1($,X,Y){let Z=`shogo-agent-runtime-${X}.tar.gz`,z=`v${$}`,J=`${Y.replace(/\/$/,"")}/${z}/${Z}`;return{tarball:J,sha256:`${J}.sha256`,assetName:Z}}async function W1($,X){let Y=await fetch($,{redirect:"follow"});if(!Y.ok)throw Error(`Download failed: HTTP ${Y.status} for ${$}`);if(!Y.body)throw Error(`Download failed: empty body for ${$}`);await H1(q1.fromWeb(Y.body),z1(X))}async function O1($){let X=await fetch($,{redirect:"follow"});if(!X.ok)throw Error(`SHA256 sidecar fetch failed: HTTP ${X.status} for ${$}`);let Z=(await X.text()).trim().split(/\s+/)[0];if(!/^[0-9a-f]{64}$/i.test(Z))throw Error(`SHA256 sidecar at ${$} did not contain a 64-char hex digest`);return Z.toLowerCase()}function w1($){let X=a2($);return Z1("sha256").update(X).digest("hex")}async function C1($,X){await new Promise((Y,Z)=>{let z=Y1("tar",["-xzf",$,"-C",X],{stdio:["ignore","pipe","pipe"]}),J="";z.stderr.on("data",(K)=>{J+=K.toString()}),z.on("error",Z),z.on("exit",(K)=>{if(K===0)Y();else Z(Error(`tar exited ${K}: ${J.trim()}`))})})}function L1($,X){s0();let Y=V1(X);if(!K0(Y))o0(Y,{recursive:!0,mode:448});let Z=`${X}.next`;if(K0(Z))a0(Z,{force:!0});c2($,Z);try{if(process.platform!=="win32"){let{chmodSync:z}=y0("node:fs");z(Z,493)}}catch{}if(K0(X))a0(X,{force:!0});c2(Z,X)}async function r2($={}){let X=$.logger??console,Y=$.channel??"stable",Z=$.baseUrl??process.env.SHOGO_RUNTIME_RELEASES_URL??Q1,z=$.target??l2(),J=$.version;if(!J)X.log(`[runtime install] Resolving latest ${Y} version...`),J=await G1(Y,Z),X.log(`[runtime install] Latest ${Y} = ${J}`);let K=A0();if(K&&K.version===J&&K.target===z&&!$.force)return X.log(`[runtime install] ${J} (${z}) already installed at ${S} — pass --force to reinstall`),{version:J,target:z,binPath:S,source:K.source,sha256:K.sha256,channel:Y};let V=k1(J,z,Z);X.log(`[runtime install] Downloading ${V.tarball}`);let H=N0(K1(),`shogo-runtime-install-${process.pid}-${Date.now()}`);o0(H,{recursive:!0});try{let q=N0(H,V.assetName);await W1(V.tarball,q),X.log("[runtime install] Verifying SHA-256...");let Q=await O1(V.sha256),G=w1(q);if(G!==Q)throw Error(`SHA-256 mismatch for ${V.assetName}
|
|
18
|
+
expected: ${Q}
|
|
19
|
+
actual: ${G}`);let W=N0(H,"extract");o0(W,{recursive:!0}),await C1(q,W);let C=N0(W,"agent-runtime");if(!K0(C))throw Error(`Tarball ${V.assetName} did not contain ./agent-runtime`);L1(C,S);let k={version:J,target:z,installedAt:new Date().toISOString(),channel:Y,source:V.tarball,sha256:G};return J1(r,JSON.stringify(k,null,2)+`
|
|
20
|
+
`,{mode:384}),X.log(`[runtime install] Installed agent-runtime ${J} (${z}) to ${S}`),{version:J,target:z,binPath:S,source:V.tarball,sha256:G,channel:Y}}finally{try{a0(H,{recursive:!0,force:!0})}catch{}}}function e2(){return{runtimeDir:l,runtimeBin:S,versionFile:r}}async function l0($={}){let X=await r2({channel:$.channel,version:$.version,baseUrl:$.baseUrl,force:$.force});console.log(),console.log(A.green("✓"),`agent-runtime ${A.bold(X.version)} installed (${X.target})`),console.log(` ${A.dim("path: ")} ${X.binPath}`),console.log(` ${A.dim("source: ")} ${X.source}`),console.log(` ${A.dim("sha256: ")} ${X.sha256}`),console.log(` ${A.dim("channel: ")} ${X.channel}`)}function s2(){let $=A0();if(!$){console.log(A.yellow("No agent-runtime installed.")),console.log(`Run ${A.cyan("shogo runtime install")} to download the latest.`),process.exitCode=1;return}console.log(`${A.bold("agent-runtime")} ${$.version}`),console.log(` ${A.dim("target: ")} ${$.target}`),console.log(` ${A.dim("channel: ")} ${$.channel}`),console.log(` ${A.dim("installed at:")} ${$.installedAt}`),console.log(` ${A.dim("source: ")} ${$.source}`)}function t2(){let $=u(),X=e2();if(!$){console.log(A.yellow("agent-runtime binary not found on this machine.")),console.log(),console.log(A.dim("Default install path:"),X.runtimeBin),console.log(),console.log(n()),process.exitCode=1;return}if(console.log($.path),process.env.SHOGO_DEBUG||process.env.VERBOSE)console.log(A.dim(` (resolved via: ${$.source})`))}async function $$($={}){let X=A0(),Y=$.channel??X?.channel??"stable";if(X)console.log(`${A.dim("current:")} ${X.version} (${X.target}, ${X.channel})`);else console.log(A.dim("No existing install — installing fresh..."));await l0({channel:Y,baseUrl:$.baseUrl,force:!0})}import{existsSync as U1,mkdirSync as N1}from"node:fs";import{resolve as A1}from"node:path";import U from"picocolors";import{CloudFileTransport as B1}from"@shogo-ai/sdk/cloud-file-transport";async function X$($,X){if(!$)throw Error("projectId is required");let Y=R({apiKey:X.apiKey,cloudUrl:X.cloudUrl});t0(Y.projectsDir);let Z=A1(X.into??g($,Y.projectsDir));if(!U1(Z))N1(Z,{recursive:!0});let z=X.include?.split(",").map((V)=>V.trim()).filter(Boolean);if(console.log(U.bold(`
|
|
21
|
+
shogo project pull ${U.cyan($)}`)),console.log(U.dim(" cloud ")+Y.cloudUrl),console.log(U.dim(" into ")+Z),z?.length)console.log(U.dim(" include ")+z.join(", "));console.log("");let J=new B1({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z,include:z,onProgress:F1()}),K=await J.downloadAll();if(T1("Pull",K),X.watch){console.log(U.bold(`
|
|
22
|
+
Watching for local changes...`));let V=new Y0({rootDir:Z,transport:J,onFlush:({uploaded:Q,errors:G})=>{let W=Q.length===1?Q[0]:`${Q.length} files`,C=G>0?U.red(` (${G} errors)`):"";console.log(U.dim(` ↑ ${W}${C}`))}});V.start();let H=!1,q=async(Q)=>{if(H)return;H=!0,console.log(U.dim(`
|
|
23
|
+
Received ${Q} — flushing pending uploads...`));try{await V.stop()}catch(G){console.warn(U.yellow(`watcher.stop: ${G?.message??G}`))}process.exit(0)};process.once("SIGINT",()=>void q("SIGINT")),process.once("SIGTERM",()=>void q("SIGTERM")),process.once("SIGHUP",()=>void q("SIGHUP")),await new Promise(()=>{})}}function F1(){return($)=>{let X=$.kind==="download"?"↓":$.kind==="upload"?"↑":$.kind==="delete"?"✗":"·",Y=$.total>0?` ${$.index+1}/${$.total}`:"",Z=$.bytes!=null?U.dim(` (${y1($.bytes)})`):"";console.log(U.dim(` ${X}${Y} ${$.path}${Z}`))}}function T1($,X){let Y=X.errors.length===0,Z=Y?U.green(`✓ ${$} complete`):U.red(`✗ ${$} completed with errors`);if(console.log(`
|
|
24
|
+
${Z}`),console.log(U.dim(" downloaded: ")+X.downloaded),X.uploaded)console.log(U.dim(" uploaded: ")+X.uploaded);if(X.deleted)console.log(U.dim(" deleted: ")+X.deleted);if(X.skipped)console.log(U.dim(" skipped: ")+X.skipped);if(!Y){console.log(U.dim(" errors: ")+X.errors.length);for(let z of X.errors.slice(0,5))console.log(U.dim(" ")+U.red(`${z.path}: ${z.message}`));if(X.errors.length>5)console.log(U.dim(` ... and ${X.errors.length-5} more`))}}function y1($){if($<1024)return`${$}B`;if($<1048576)return`${($/1024).toFixed(1)}KB`;return`${($/1024/1024).toFixed(2)}MB`}import{existsSync as _1}from"node:fs";import{resolve as D1}from"node:path";import B from"picocolors";import{CloudFileTransport as x1}from"@shogo-ai/sdk/cloud-file-transport";async function Y$($,X){if(!$)throw Error("projectId is required");let Y=R({apiKey:X.apiKey,cloudUrl:X.cloudUrl}),Z=D1(X.from??g($,Y.projectsDir));if(!_1(Z))throw Error(`Source directory does not exist: ${Z}`);let z=X.include?.split(",").map((V)=>V.trim()).filter(Boolean);if(console.log(B.bold(`
|
|
25
|
+
shogo project push ${B.cyan($)}`)),console.log(B.dim(" cloud ")+Y.cloudUrl),console.log(B.dim(" from ")+Z),z?.length)console.log(B.dim(" include ")+z.join(", "));if(X.deleteRemote)console.log(B.yellow(" --delete-remote: remote files not present locally will be DELETED"));console.log("");let K=await new x1({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z,include:z,onProgress:(V)=>{let H=V.kind==="upload"?"↑":V.kind==="delete"?"✗":"·",q=V.total>0?` ${V.index+1}/${V.total}`:"",Q=V.bytes!=null?B.dim(` (${P1(V.bytes)})`):"";console.log(B.dim(` ${H}${q} ${V.path}${Q}`))}}).uploadAll({deleteRemote:X.deleteRemote});M1("Push",K)}function M1($,X){let Y=X.errors.length===0,Z=Y?B.green(`✓ ${$} complete`):B.red(`✗ ${$} completed with errors`);if(console.log(`
|
|
26
|
+
${Z}`),console.log(B.dim(" uploaded: ")+X.uploaded),X.deleted)console.log(B.dim(" deleted: ")+X.deleted);if(!Y){console.log(B.dim(" errors: ")+X.errors.length);for(let z of X.errors.slice(0,5))console.log(B.dim(" ")+B.red(`${z.path}: ${z.message}`));if(X.errors.length>5)console.log(B.dim(` ... and ${X.errors.length-5} more`))}}function P1($){if($<1024)return`${$}B`;if($<1048576)return`${($/1024).toFixed(1)}KB`;return`${($/1024/1024).toFixed(2)}MB`}import{existsSync as S1}from"node:fs";import{resolve as R1}from"node:path";import b from"picocolors";w0();async function Z$($,X){if(!$)throw Error("projectId is required");let Y=R({apiKey:X.apiKey,cloudUrl:X.cloudUrl}),Z=R1(X.into??g($,Y.projectsDir));if(!S1(Z))throw Error(`Local project dir does not exist: ${Z} — run \`shogo project pull\` first`);if(!X0(Z))throw Error(`${Z} is not a git repo. \`shogo project checkout\` requires the git sync path; run \`shogo project pull\` after installing git, or use \`shogo project pull --include\` for file-only restore.`);if(console.log(b.bold(`
|
|
27
|
+
shogo project checkout ${b.cyan($)}`)),console.log(b.dim(" cloud ")+Y.cloudUrl),console.log(b.dim(" local ")+Z),X.at)console.log(b.dim(" at ")+X.at);if(console.log(""),X.unshallow)console.log(b.dim("Unshallowing repo (this may take a moment)...")),await O0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z});if(!X.at){let J=await $0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z});console.log(b.green(`✓ Reset to ${J.commitSha.slice(0,8)} (remote HEAD)`));return}let z;try{z=(await y(["rev-parse","--verify",`${X.at}^{commit}`],{cwd:Z})).stdout.trim()}catch{z=await b1(Y.cloudUrl,Y.apiKey,$,X.at),console.log(b.dim(` resolved checkpoint "${X.at}" → ${z.slice(0,8)}`))}try{await $0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z,branch:z})}catch{if(!X.unshallow)console.log(b.yellow(" fetch failed; unshallowing and retrying...")),await O0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z}),await $0({apiUrl:Y.cloudUrl,apiKey:Y.apiKey,projectId:$,localDir:Z,branch:z});else throw Error(`Cannot reach commit ${z} in local clone`)}await y(["reset","--hard",z],{cwd:Z}),console.log(b.green(`✓ Checked out ${z.slice(0,8)}`))}async function b1($,X,Y,Z){let z=`${$.replace(/\/+$/,"")}/api/projects/${encodeURIComponent(Y)}/checkpoints?limit=100`,J=await fetch(z,{headers:{Authorization:`Bearer ${X}`}});if(!J.ok)throw Error(`Failed to list checkpoints: HTTP ${J.status}`);let K=await J.json(),V=Z.toLowerCase(),H=K.checkpoints.find((q)=>{if(q.name&&q.name.toLowerCase()===V)return!0;if(q.commitSha.startsWith(Z))return!0;if(q.commitMessage&&q.commitMessage.toLowerCase().includes(V))return!0;return!1});if(!H)throw Error(`No checkpoint matches "${Z}" (searched ${K.checkpoints.length} most recent)`);return H.commitSha}import{createInterface as m1}from"node:readline";import N from"picocolors";import{execFileSync as J$}from"node:child_process";import{existsSync as p,copyFileSync as z$}from"node:fs";import P from"node:path";import{homedir as K$,platform as V$}from"node:os";function q$($,X,Y){let z=J$($,["-e",Y],{env:{...process.env,DBP:X},encoding:"utf-8",stdio:["ignore","pipe","pipe"],timeout:5000}).trim();if(!z)throw Error(`bun script returned empty output for ${X}`);return JSON.parse(z)}function B0($,X){if(!p(X))return[];return q$($,X,`
|
|
28
28
|
import { Database } from 'bun:sqlite';
|
|
29
29
|
try {
|
|
30
30
|
const db = new Database(process.env.DBP, { readonly: true });
|
|
@@ -38,7 +38,7 @@ shogo project checkout ${E.cyan($)}`)),console.log(E.dim(" cloud ")+Y.cloudUrl
|
|
|
38
38
|
console.error(String(e?.stack || e));
|
|
39
39
|
process.exit(2);
|
|
40
40
|
}
|
|
41
|
-
`)}function
|
|
41
|
+
`)}function E1($){if(!p($))throw Error(`Database does not exist at ${$} — refusing to back up nothing`);let X=P.dirname($),Y=P.basename($),Z=new Date().toISOString().replace(/[:.]/g,"-"),z=P.join(X,`${Y}.bak-${Z}`);z$($,z);for(let J of["-wal","-shm"]){let K=`${$}${J}`;if(p(K))z$(K,`${z}${J}`)}return z}function v1($,X,Y){if(Y.length===0)return 0;if(!p(X))throw Error(`Cannot repair: database does not exist at ${X}`);let z=`
|
|
42
42
|
import { Database } from 'bun:sqlite';
|
|
43
43
|
const names = ${JSON.stringify(Y)};
|
|
44
44
|
if (!Array.isArray(names) || names.some(n => typeof n !== 'string')) {
|
|
@@ -57,8 +57,8 @@ shogo project checkout ${E.cyan($)}`)),console.log(E.dim(" cloud ")+Y.cloudUrl
|
|
|
57
57
|
console.error(String(e?.stack || e));
|
|
58
58
|
process.exit(3);
|
|
59
59
|
}
|
|
60
|
-
`;return
|
|
61
|
-
`)[0]?.trim();if(console.log(` ${N.yellow("•")} ${N.bold(X.name)} ${N.dim(`(attempted ${Y})`)}`),Z)console.log(` ${N.dim(Z)}`)}}async function
|
|
62
|
-
`+" (e.g. /Applications/Shogo.app/Contents/Resources/bun/bun on macOS).");console.log(N.dim(`Database: ${X}`)),console.log(N.dim(`Bun: ${Y}`)),console.log();let Z=B0(Y,X);if(Z.length===0){console.log(N.green("✓ No failed migrations detected — your local database looks healthy."));return}if(console.log(N.red(`✗ Found ${Z.length} failed migration(s):`)),
|
|
63
|
-
→ Relaunch the Shogo app to finish applying migrations.`))}else{if(console.log(N.red(`✗ ${z.message}`)),z.backupPath)console.log(N.dim(` Backup: ${z.backupPath}`));process.exitCode=1}}import{spawn as
|
|
64
|
-
(use --help for usage)`));
|
|
60
|
+
`;return q$($,X,z).deleted}function H$($){let X=$.log??(()=>{}),{bunPath:Y,dbPath:Z}=$;if(!p(Z))return{status:"no-database",detected:[],cleared:[],remaining:[],message:`No database found at ${Z}. Nothing to repair — launch Shogo once to create it.`};let z=B0(Y,Z);if(z.length===0)return{status:"healthy",detected:[],cleared:[],remaining:[],message:"No failed migrations detected — the local database looks healthy."};X(`Found ${z.length} failed migration(s): ${z.map((G)=>G.name).join(", ")}`);let J;if(!$.skipBackup)J=E1(Z),X(`Backed up database to ${J}`);let K=z.map((G)=>G.name),V=v1(Y,Z,K);X(`Cleared ${V} failed migration row(s).`);let H=B0(Y,Z),q=H.length===0?"repaired":"failed",Q=q==="repaired"?"Cleared the failed migration record. Relaunch Shogo to re-apply migrations cleanly.":`Repair incomplete — ${H.length} migration(s) still failed: ${H.map((G)=>G.name).join(", ")}.`;return{status:q,detected:z,backupPath:J,cleared:K.slice(0,V),remaining:H,message:Q}}function h1(){let $=K$(),X=V$(),Y;if(X==="darwin")Y=P.join($,"Library","Application Support");else if(X==="win32")Y=process.env.APPDATA??P.join($,"AppData","Roaming");else Y=process.env.XDG_CONFIG_HOME??P.join($,".config");return P.join(Y,"Shogo","data")}function Q$(){return P.join(h1(),"shogo.db")}function j1(){let $=V$(),X=$==="win32"?"bun.exe":"bun",Y=K$(),Z=[];if($==="darwin")Z.push(P.join("/Applications","Shogo.app","Contents","Resources","bun",X),P.join(Y,"Applications","Shogo.app","Contents","Resources","bun",X));else if($==="linux")Z.push(P.join("/opt","Shogo","resources","bun",X),P.join("/usr","lib","shogo","resources","bun",X));return Z}function r0($){try{return J$($,["--version"],{stdio:"ignore",timeout:5000}),!0}catch{return!1}}function G$($){if($)return r0($)?$:null;if(process.versions.bun&&process.execPath)return process.execPath;for(let X of j1())if(p(X)&&r0(X))return X;if(r0("bun"))return"bun";return null}function I1($){for(let X of $){let Y=Number.isFinite(X.startedAt)?new Date(X.startedAt).toISOString():"unknown time",Z=(X.errorExcerpt??"").split(`
|
|
61
|
+
`)[0]?.trim();if(console.log(` ${N.yellow("•")} ${N.bold(X.name)} ${N.dim(`(attempted ${Y})`)}`),Z)console.log(` ${N.dim(Z)}`)}}async function u1($){if(!process.stdin.isTTY)return console.log(N.dim("(stdin is not a TTY — re-run with --yes to repair non-interactively)")),!1;let X=m1({input:process.stdin,output:process.stdout});try{let Y=await new Promise((Z)=>X.question($,Z));return/^y(es)?$/i.test(Y.trim())}finally{X.close()}}async function k$($={}){let X=$.db??Q$(),Y=G$($.bun);if(!Y)throw Error("Could not find a usable `bun` binary (needed to inspect the SQLite database).\n"+" • Install Bun (https://bun.sh) so `bun` is on your PATH, or\n"+` • pass --bun <path> pointing at the bun shipped inside the Shogo app
|
|
62
|
+
`+" (e.g. /Applications/Shogo.app/Contents/Resources/bun/bun on macOS).");console.log(N.dim(`Database: ${X}`)),console.log(N.dim(`Bun: ${Y}`)),console.log();let Z=B0(Y,X);if(Z.length===0){console.log(N.green("✓ No failed migrations detected — your local database looks healthy."));return}if(console.log(N.red(`✗ Found ${Z.length} failed migration(s):`)),I1(Z),console.log(),$.check){console.log(N.dim("Run `shogo doctor` (without --check) to back up the database and clear these records.")),process.exitCode=1;return}if(!$.yes){let J=$.backup===!1?N.red("This will NOT create a backup (--no-backup)."):"Your database will be backed up to a .bak-<timestamp> file first.";if(console.log(J),!await u1(N.bold("Clear these failed migration records and repair? [y/N] "))){console.log(N.dim("Aborted — no changes made.")),process.exitCode=1;return}console.log()}let z=H$({bunPath:Y,dbPath:X,skipBackup:$.backup===!1,log:(J)=>console.log(N.dim(` ${J}`))});if(console.log(),z.status==="repaired"){if(console.log(N.green(`✓ ${z.message}`)),z.backupPath)console.log(N.dim(` Backup: ${z.backupPath}`));console.log(N.bold(`
|
|
63
|
+
→ Relaunch the Shogo app to finish applying migrations.`))}else{if(console.log(N.red(`✗ ${z.message}`)),z.backupPath)console.log(N.dim(` Backup: ${z.backupPath}`));process.exitCode=1}}import{spawn as f1}from"node:child_process";import{resolve as n1}from"node:path";import W$ from"picocolors";function d1($){let{flags:X,config:Y,runtime:Z}=$,z=X.cwd?n1(X.cwd):process.cwd(),J=["interactive"];if(X.print!==void 0)J.push("-p",X.print);if(X.noTui)J.push("--no-tui");let K=Y.cloudUrl.replace(/\/$/,""),V={...$.baseEnv??process.env,SHOGO_INTERACTIVE:"1",SHOGO_INTERACTIVE_CWD:z,PROJECT_DIR:z,WORKSPACE_DIR:z,SHOGO_API_URL:K,SHOGO_CLOUD_URL:K,SHOGO_API_KEY:Y.apiKey,AI_PROXY_URL:`${K}/api/ai/v1`,AI_PROXY_TOKEN:Y.apiKey,NODE_ENV:"production"};if(X.model)V.SHOGO_MODEL=X.model;if(X.print!==void 0)V.SHOGO_PRINT_PROMPT=X.print;return{bin:Z.path,args:J,env:V,cwd:z}}async function O$($={}){let X=R({apiKey:$.apiKey,cloudUrl:$.cloudUrl}),Y=u({flag:$.runtimeBin});if(!Y)console.error(W$.red(n({flag:$.runtimeBin}))),process.exit(1);let Z=d1({flags:$,config:{apiKey:X.apiKey,cloudUrl:X.cloudUrl},runtime:Y}),z=await new Promise((J)=>{let K=f1(Z.bin,Z.args,{cwd:Z.cwd,env:Z.env,stdio:"inherit"});K.on("error",(V)=>{console.error(W$.red(`✗ Failed to launch agent-runtime: ${V.message}`)),J(1)}),K.on("exit",(V)=>J(V??0))});process.exit(z)}var i1="0.1.0",E=new g1;E.name("shogo").description("Shogo Cloud Agent Worker — run Shogo agents on your own machine.").version(i1);E.command("login").description("Pair this machine with Shogo Cloud (browser device flow, or --api-key for CI)").option("--api-key <key>","CI escape hatch — skip the browser flow and use a key directly").option("--cloud-url <url>","Shogo Cloud URL (default: https://studio.shogo.ai)").option("--name <name>","Device label shown in the dashboard (default: hostname)").option("--workspace <id>","Pre-select a workspace on the bridge picker").option("--no-browser","Do not auto-open the browser; print the URL instead").action(($)=>F(()=>g2($)));var F0=E.command("worker").description("Manage the local worker process");F0.command("start").description("Start the worker and pair with Shogo Cloud").option("--name <name>","Instance name shown in the dashboard (default: hostname)").option("--worker-dir <path>","Working directory for the worker (default: $PWD)").option("--api-key <key>","API key (overrides config/env)").option("--cloud-url <url>","Shogo Cloud URL").option("--port <port>","Local HTTP port for the embedded API").option("--proxy <url>","HTTPS proxy (overrides HTTPS_PROXY env)").option("--project <id>","Pin to a single project (default: multi-project on demand)").option("--runtime-bin <path>","Override the agent-runtime binary path").option("--debug","Run preflight checks before starting").option("--foreground","Run in foreground (don't detach)").option("--no-auto-pull","Disable auto-clone of project workspaces on first request").option("--projects-dir <path>","Root directory for cloned project workspaces (default: ~/.shogo/projects)").option("--no-git","Force the file-transport sync path even when git is available").action(($)=>F(()=>h2($)));F0.command("stop").description("Stop the running worker").action(()=>F(m2));F0.command("status").description("Show worker status").action(()=>F(I2));F0.command("logs").description("Tail worker logs from ~/.shogo/logs/worker.log").option("-f, --follow","Follow the log (tail -F)").option("--err","Show stderr log instead").action(($)=>F(()=>f2($)));var T0=E.command("runtime").description("Install / inspect the local agent-runtime binary");T0.command("install").description("Download + verify the agent-runtime tarball into ~/.shogo/runtime/").option("--channel <channel>","stable | beta | nightly (default: stable)").option("--version <version>","install a specific version (e.g. 0.1.0)").option("--base-url <url>","override release base URL (default: GitHub Releases)").option("--force","reinstall even if the same version is already on disk").action(($)=>F(()=>l0($)));T0.command("version").description("Print the installed agent-runtime version").action(()=>F(()=>s2()));T0.command("where").description("Print the resolved agent-runtime binary path").action(()=>F(()=>t2()));T0.command("update").description("Update agent-runtime to the latest in its channel").option("--channel <channel>","override channel (defaults to whatever is installed)").option("--base-url <url>","override release base URL").action(($)=>F(()=>$$($)));var C$=E.command("config").description("Inspect or modify ~/.shogo/config.json");C$.command("show").description("Print current config (API key masked)").action(()=>F(i2));C$.command("set <key> <value>").description("Set apiKey / cloudUrl / name / workerDir / port / projectsDir").action(($,X)=>F(()=>p2($,X)));var e0=E.command("project").description("Clone/sync project workspaces between Shogo Cloud and this machine");e0.command("pull <projectId>").description("Clone a project from Shogo Cloud into ~/.shogo/projects/<projectId>/").option("--into <dir>","Destination directory (default: ~/.shogo/projects/<projectId>)").option("--watch","After pull, watch the local dir and push edits back to cloud").option("--include <patterns>",'Comma-separated glob patterns (e.g. "src/**,*.md")').option("--api-key <key>","Override API key for this run").option("--cloud-url <url>","Override Shogo Cloud URL for this run").action(($,X)=>F(()=>X$($,X)));e0.command("push <projectId>").description("Upload local workspace edits back to Shogo Cloud").option("--from <dir>","Source directory (default: ~/.shogo/projects/<projectId>)").option("--delete-remote","Mirror local deletions to cloud (DESTRUCTIVE)").option("--include <patterns>","Comma-separated glob patterns").option("--api-key <key>","Override API key for this run").option("--cloud-url <url>","Override Shogo Cloud URL for this run").action(($,X)=>F(()=>Y$($,X)));e0.command("checkout <projectId>").description("Roll the local workspace to a specific git checkpoint (SHA or named checkpoint)").option("--at <ref>","Target SHA or checkpoint name (default: remote HEAD)").option("--unshallow","Fetch full history before checking out (needed for old SHAs)").option("--into <dir>","Local dir override (default: ~/.shogo/projects/<projectId>)").option("--api-key <key>","Override API key for this run").option("--cloud-url <url>","Override Shogo Cloud URL for this run").action(($,X)=>F(()=>Z$($,X)));E.command("doctor").description("Diagnose & repair a wedged local Shogo database (clears failed migrations so the app can reboot)").option("--check","Detect only — never modify the database").option("--yes","Repair without the confirmation prompt").option("--db <path>","Path to shogo.db (default: the desktop app's local database)").option("--bun <path>","Path to a bun binary (default: bundled/PATH bun)").option("--no-backup","Skip the pre-repair database backup (discouraged)").action(($)=>F(()=>k$($)));E.command("chat",{isDefault:!0}).description("Start the interactive coding agent in the current directory").argument("[prompt...]","Optional one-shot prompt (runs non-interactively, like -p)").option("-p, --print <prompt>","Headless one-shot: run a single turn and print the result").option("--model <model>","Model id to use for new turns").option("--cwd <dir>","Working directory to operate in (default: $PWD)").option("--runtime-bin <path>","Override the agent-runtime binary path").option("--no-tui","Disable the rich TUI; use a plain readline renderer").option("--api-key <key>","Override API key for this run").option("--cloud-url <url>","Override Shogo Cloud URL for this run").action(($,X)=>F(()=>O$({print:X.print??($.length?$.join(" "):void 0),model:X.model,cwd:X.cwd,runtimeBin:X.runtimeBin,noTui:X.tui===!1,apiKey:X.apiKey,cloudUrl:X.cloudUrl})));E.showHelpAfterError(w$.dim(`
|
|
64
|
+
(use --help for usage)`));E.parseAsync(process.argv);async function F($){try{await $()}catch(X){if(console.error(w$.red(`✗ ${X?.message??X}`)),process.env.SHOGO_DEBUG)console.error(X?.stack);process.exit(1)}}
|
package/package.json
CHANGED
|
@@ -18,19 +18,25 @@
|
|
|
18
18
|
* ~1/minute respawn loop until the user quit the app. Symptom:
|
|
19
19
|
* `restart #54` in the same project's log.
|
|
20
20
|
*
|
|
21
|
-
* This file pins
|
|
21
|
+
* This file pins four contracts:
|
|
22
22
|
*
|
|
23
23
|
* 1. Every non-clean exit reaps the runtime's process group via the
|
|
24
24
|
* recorded PID (the orphan-sweep that wasn't happening before).
|
|
25
25
|
* 2. After {@link MAX_CONSECUTIVE_RESTARTS} consecutive non-clean
|
|
26
26
|
* exits within {@link RESTART_FAILURE_WINDOW_MS}, the slot
|
|
27
27
|
* transitions to `'failed'` and the manager stops scheduling
|
|
28
|
-
* restarts.
|
|
29
|
-
* respawn until the operator calls `resetFailure(...)`.
|
|
28
|
+
* restarts.
|
|
30
29
|
* 3. A clean exit (code=0, signal=null) does NOT increment the
|
|
31
30
|
* failure budget — bounded chat sessions that exit normally
|
|
32
31
|
* after their work is done should leave the slot in the same
|
|
33
32
|
* "healthy" state they found it in.
|
|
33
|
+
* 4. A `'failed'` slot is NOT terminal: it self-heals. A tripped
|
|
34
|
+
* breaker refuses `ensureRunning(...)` immediately, but once an
|
|
35
|
+
* escalating cooldown (2m/4m/8m/15m-cap, see `breakerCooldownMs`)
|
|
36
|
+
* elapses, the NEXT `ensureRunning(...)` call auto-clears it and
|
|
37
|
+
* attempts a fresh spawn with a full failure budget — no operator
|
|
38
|
+
* or caller has to remember to invoke `resetFailure(...)`, though
|
|
39
|
+
* it remains available for an immediate manual retry.
|
|
34
40
|
*/
|
|
35
41
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
36
42
|
import { WorkerRuntimeManager } from '../runtime-manager.ts';
|
|
@@ -79,6 +85,8 @@ function insertRunningSlot(mgr: WorkerRuntimeManager, projectId: string, pid = 9
|
|
|
79
85
|
restarts: 0,
|
|
80
86
|
consecutiveFailures: 0,
|
|
81
87
|
lastFailureAt: 0,
|
|
88
|
+
breakerTrips: 0,
|
|
89
|
+
restartTimestamps: [] as number[],
|
|
82
90
|
graceTimer: null,
|
|
83
91
|
restartTimer: null,
|
|
84
92
|
idleTimer: null,
|
|
@@ -308,3 +316,91 @@ describe('WorkerRuntimeManager.ensureRunning refuses failed slots', () => {
|
|
|
308
316
|
).rejects.toThrow(/Circuit breaker tripped|resetFailure/);
|
|
309
317
|
});
|
|
310
318
|
});
|
|
319
|
+
|
|
320
|
+
const spawnConfig = {
|
|
321
|
+
cloudUrl: 'https://example.invalid',
|
|
322
|
+
apiKey: 'unused',
|
|
323
|
+
} as never;
|
|
324
|
+
|
|
325
|
+
describe('WorkerRuntimeManager auto-heal (self-healing circuit breaker)', () => {
|
|
326
|
+
it('still refuses a failed slot before its cooldown has elapsed', async () => {
|
|
327
|
+
const mgr = new WorkerRuntimeManager({ logger: SILENT });
|
|
328
|
+
const slot = insertRunningSlot(mgr, 'proj-too-soon', 55551);
|
|
329
|
+
for (let i = 0; i < 8; i++) handleExit(mgr, slot, null, 'SIGKILL');
|
|
330
|
+
expect(slot.status).toBe('failed');
|
|
331
|
+
expect(slot.breakerTrips).toBe(1);
|
|
332
|
+
|
|
333
|
+
await expect(mgr.ensureRunning('proj-too-soon', spawnConfig)).rejects.toThrow(
|
|
334
|
+
/Will auto-retry in ~\d+s/,
|
|
335
|
+
);
|
|
336
|
+
// Refusing doesn't itself count as another trip.
|
|
337
|
+
expect(slot.breakerTrips).toBe(1);
|
|
338
|
+
expect(slot.status).toBe('failed');
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
it('auto-heals once the cooldown elapses, without anyone calling resetFailure', async () => {
|
|
342
|
+
const mgr = new WorkerRuntimeManager({ logger: SILENT });
|
|
343
|
+
const slot = insertRunningSlot(mgr, 'proj-cooldown', 55552);
|
|
344
|
+
for (let i = 0; i < 8; i++) handleExit(mgr, slot, null, 'SIGKILL');
|
|
345
|
+
expect(slot.status).toBe('failed');
|
|
346
|
+
|
|
347
|
+
// Backdate past the 2-minute base cooldown for trip #1.
|
|
348
|
+
slot.lastFailureAt = Date.now() - (2 * 60 * 1000 + 1_000);
|
|
349
|
+
|
|
350
|
+
// No `autoPull`/`projectDir` is configured on this bare manager, so
|
|
351
|
+
// the call still fails — but on a DIFFERENT, unrelated error (a
|
|
352
|
+
// workspace-misconfiguration, not the circuit breaker). That proves
|
|
353
|
+
// `ensureRunning` fell through the breaker guard and actually
|
|
354
|
+
// attempted a fresh start instead of refusing outright.
|
|
355
|
+
let caught: Error | null = null;
|
|
356
|
+
try {
|
|
357
|
+
await mgr.ensureRunning('proj-cooldown', spawnConfig);
|
|
358
|
+
} catch (err) {
|
|
359
|
+
caught = err as Error;
|
|
360
|
+
}
|
|
361
|
+
expect(caught).not.toBeNull();
|
|
362
|
+
expect(caught!.message).not.toMatch(/cannot ensureRunning|Circuit breaker tripped/);
|
|
363
|
+
// The slot moved off `'failed'` — the breaker is no longer open.
|
|
364
|
+
expect(slot.status).not.toBe('failed');
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it('escalates the cooldown on repeated trips (2m -> 4m)', async () => {
|
|
368
|
+
const mgr = new WorkerRuntimeManager({ logger: SILENT });
|
|
369
|
+
const slot = insertRunningSlot(mgr, 'proj-escalate', 55553);
|
|
370
|
+
for (let i = 0; i < 8; i++) handleExit(mgr, slot, null, 'SIGKILL');
|
|
371
|
+
expect(slot.breakerTrips).toBe(1);
|
|
372
|
+
|
|
373
|
+
// Elapse the first (2m) cooldown and let it auto-heal + immediately
|
|
374
|
+
// re-trip (still no projectDir configured, but that's irrelevant here
|
|
375
|
+
// — we only care about the breaker bookkeeping around the 2nd trip).
|
|
376
|
+
slot.lastFailureAt = Date.now() - (2 * 60 * 1000 + 1_000);
|
|
377
|
+
try { await mgr.ensureRunning('proj-escalate', spawnConfig); } catch { /* expected */ }
|
|
378
|
+
|
|
379
|
+
// Re-insert as a running slot (ensureRunning's failed attempt didn't
|
|
380
|
+
// leave it running) and drive it back into `'failed'` a second time.
|
|
381
|
+
const slot2 = insertRunningSlot(mgr, 'proj-escalate', 55554);
|
|
382
|
+
slot2.breakerTrips = slot.breakerTrips; // carry the trip count forward
|
|
383
|
+
for (let i = 0; i < 8; i++) handleExit(mgr, slot2, null, 'SIGKILL');
|
|
384
|
+
expect(slot2.breakerTrips).toBe(2);
|
|
385
|
+
|
|
386
|
+
// Trip #2's cooldown should be double trip #1's (4m, not 2m): backdating
|
|
387
|
+
// by just over 2m must NOT be enough to auto-heal yet.
|
|
388
|
+
slot2.lastFailureAt = Date.now() - (2 * 60 * 1000 + 1_000);
|
|
389
|
+
await expect(mgr.ensureRunning('proj-escalate', spawnConfig)).rejects.toThrow(
|
|
390
|
+
/Will auto-retry/,
|
|
391
|
+
);
|
|
392
|
+
expect(slot2.status).toBe('failed');
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
it('healBreaker gives the project a fresh consecutiveFailures budget', async () => {
|
|
396
|
+
const mgr = new WorkerRuntimeManager({ logger: SILENT });
|
|
397
|
+
const slot = insertRunningSlot(mgr, 'proj-fresh-budget', 55555);
|
|
398
|
+
for (let i = 0; i < 8; i++) handleExit(mgr, slot, null, 'SIGKILL');
|
|
399
|
+
expect(slot.consecutiveFailures).toBe(8);
|
|
400
|
+
|
|
401
|
+
slot.lastFailureAt = Date.now() - (2 * 60 * 1000 + 1_000);
|
|
402
|
+
try { await mgr.ensureRunning('proj-fresh-budget', spawnConfig); } catch { /* expected */ }
|
|
403
|
+
|
|
404
|
+
expect(slot.consecutiveFailures).toBe(0);
|
|
405
|
+
});
|
|
406
|
+
});
|
|
@@ -116,6 +116,57 @@ const RESTART_BACKOFF_MAX_MS = 60_000;
|
|
|
116
116
|
*/
|
|
117
117
|
const MAX_CONSECUTIVE_RESTARTS = 8;
|
|
118
118
|
const RESTART_FAILURE_WINDOW_MS = 5 * 60 * 1000;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Second, independent trip condition: too many restarts in a long rolling
|
|
122
|
+
* window, regardless of the gaps between them.
|
|
123
|
+
*
|
|
124
|
+
* `MAX_CONSECUTIVE_RESTARTS`/`RESTART_FAILURE_WINDOW_MS` only catches a
|
|
125
|
+
* *tight* crash loop — repeated exits closer together than
|
|
126
|
+
* {@link STARTUP_GRACE_MS} (60s), which is what resets `consecutiveFailures`
|
|
127
|
+
* back to 0. A real production episode (main.log, 2026-09-19 19:44–20:52)
|
|
128
|
+
* showed macOS jetsam SIGKILLing several unrelated project runtimes (AND an
|
|
129
|
+
* idle warm-pool slot) in lockstep every 1–17 minutes for 68 minutes
|
|
130
|
+
* straight — individually every gap cleared the 60s grace window, so
|
|
131
|
+
* `consecutiveFailures` kept resetting to 0/1 and the tight breaker NEVER
|
|
132
|
+
* tripped, even after 15 restarts. Every one of those restarts still killed
|
|
133
|
+
* whatever chat turn was in flight.
|
|
134
|
+
*
|
|
135
|
+
* `slot.restartTimestamps` tracks restarts independently of the grace
|
|
136
|
+
* timer — pruned to this window, not reset by a "survived 60s" incarnation
|
|
137
|
+
* — so a slow, sustained bleed like that episode also trips the breaker
|
|
138
|
+
* (and gets the same auto-heal cooldown as a tight loop) instead of
|
|
139
|
+
* silently degrading chat for over an hour with no safety valve.
|
|
140
|
+
*/
|
|
141
|
+
const SLOW_BLEED_WINDOW_MS = 60 * 60 * 1000;
|
|
142
|
+
const SLOW_BLEED_MAX_RESTARTS = 8;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Self-heal cooldown for a `'failed'` (circuit-broken) slot.
|
|
146
|
+
*
|
|
147
|
+
* The breaker tripping is not evidence the project is permanently broken —
|
|
148
|
+
* the dominant real-world cause (see {@link MAX_CONSECUTIVE_RESTARTS}'s
|
|
149
|
+
* comment) is a transient jetsam OOM sweep that can clear itself up within
|
|
150
|
+
* minutes once memory pressure elsewhere on the box eases. Without this,
|
|
151
|
+
* `'failed'` was a terminal state: nothing ever called `resetFailure()`
|
|
152
|
+
* (no caller wires it to a route or the UI), so a project's chat was dead
|
|
153
|
+
* until the whole app/worker restarted — indistinguishable from a real bug
|
|
154
|
+
* to the end user.
|
|
155
|
+
*
|
|
156
|
+
* `ensureRunning()` treats a `'failed'` slot whose cooldown has elapsed as
|
|
157
|
+
* an implicit `resetFailure()` + fresh spawn attempt, with the SAME full
|
|
158
|
+
* `MAX_CONSECUTIVE_RESTARTS` budget (the cooldown itself is the rate
|
|
159
|
+
* limit — a project that's actually broken re-trips almost immediately and
|
|
160
|
+
* pays an escalating cooldown instead of hammering the OS every retry).
|
|
161
|
+
*
|
|
162
|
+
* Escalates 2m → 4m → 8m → 15m(cap) per repeated trip via
|
|
163
|
+
* {@link breakerCooldownMs}, and resets back to the 2m base the next time
|
|
164
|
+
* the project survives a full {@link STARTUP_GRACE_MS} healthy run (see
|
|
165
|
+
* `armGraceTimer`) — a project that recovers for good isn't penalized by
|
|
166
|
+
* cooldowns it earned months ago.
|
|
167
|
+
*/
|
|
168
|
+
const CIRCUIT_BREAKER_COOLDOWN_BASE_MS = 2 * 60 * 1000;
|
|
169
|
+
const CIRCUIT_BREAKER_COOLDOWN_MAX_MS = 15 * 60 * 1000;
|
|
119
170
|
/**
|
|
120
171
|
* If a runtime stays up at least this long after the /health-gated
|
|
121
172
|
* `'running'` transition, we treat it as "recovered" and reset the
|
|
@@ -211,10 +262,14 @@ export type RuntimeStatus =
|
|
|
211
262
|
| 'stopped'
|
|
212
263
|
| 'error'
|
|
213
264
|
/**
|
|
214
|
-
*
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
*
|
|
265
|
+
* The circuit breaker tripped. The slot stays in the `runtimes` map
|
|
266
|
+
* (so `status(projectId)` keeps reporting it) and refuses new spawns
|
|
267
|
+
* — but this is NOT terminal: the next `ensureRunning()` call after
|
|
268
|
+
* {@link CIRCUIT_BREAKER_COOLDOWN_BASE_MS} (escalating per repeat
|
|
269
|
+
* trip, see `breakerCooldownMs`) auto-heals it and tries again with a
|
|
270
|
+
* fresh budget. Callers that want to skip the wait can still call
|
|
271
|
+
* {@link WorkerRuntimeManager.resetFailure} directly, or `stop()` to
|
|
272
|
+
* give up on the project entirely.
|
|
218
273
|
*/
|
|
219
274
|
| 'failed';
|
|
220
275
|
|
|
@@ -410,8 +465,26 @@ interface InternalRuntime {
|
|
|
410
465
|
/** Consecutive non-clean exits since the last healthy run. */
|
|
411
466
|
consecutiveFailures: number;
|
|
412
467
|
/** Timestamp (Date.now) of the most recent non-clean exit. Used to
|
|
413
|
-
* detect "loop within the failure window" for the circuit breaker
|
|
468
|
+
* detect "loop within the failure window" for the circuit breaker,
|
|
469
|
+
* and (once tripped) as the cooldown clock for auto-heal. */
|
|
414
470
|
lastFailureAt: number;
|
|
471
|
+
/**
|
|
472
|
+
* Number of times the circuit breaker has tripped into `'failed'` for
|
|
473
|
+
* this project since it last survived a full {@link STARTUP_GRACE_MS}
|
|
474
|
+
* healthy run. Drives the escalating auto-heal cooldown in
|
|
475
|
+
* {@link WorkerRuntimeManager.breakerCooldownMs} — repeat offenders back
|
|
476
|
+
* off further instead of retrying every 2 minutes forever.
|
|
477
|
+
*/
|
|
478
|
+
breakerTrips: number;
|
|
479
|
+
/**
|
|
480
|
+
* Wall-clock timestamps (Date.now) of the last several non-clean-exit
|
|
481
|
+
* restarts, pruned to {@link SLOW_BLEED_WINDOW_MS}. Independent of
|
|
482
|
+
* `consecutiveFailures` (which the 60s startup grace resets on every
|
|
483
|
+
* incarnation that survives a minute) — this is what catches a crash
|
|
484
|
+
* that recurs every few minutes for an hour: individually every gap
|
|
485
|
+
* clears the tight breaker, but the rolling count here keeps growing.
|
|
486
|
+
*/
|
|
487
|
+
restartTimestamps: number[];
|
|
415
488
|
/** Timer that resets `consecutiveFailures` to 0 once a fresh run has
|
|
416
489
|
* survived for {@link STARTUP_GRACE_MS}. */
|
|
417
490
|
graceTimer: ReturnType<typeof setTimeout> | null;
|
|
@@ -678,18 +751,36 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
678
751
|
async ensureRunning(projectId: string, config: ProjectSpawnConfig): Promise<RuntimeStatusInfo> {
|
|
679
752
|
if (this.stopped) throw new Error('WorkerRuntimeManager is stopped');
|
|
680
753
|
|
|
681
|
-
//
|
|
682
|
-
//
|
|
683
|
-
//
|
|
684
|
-
//
|
|
754
|
+
// A circuit-broken slot gets one chance to heal itself before we
|
|
755
|
+
// refuse it: if its cooldown has elapsed, treat this call as an
|
|
756
|
+
// implicit resetFailure() and fall through to a fresh spawn attempt
|
|
757
|
+
// below. Otherwise refuse BEFORE auto-pull so we don't churn the
|
|
758
|
+
// network/disk on a project we already know we won't spawn — the
|
|
759
|
+
// parked-state message (with a concrete retry ETA) lets the caller
|
|
760
|
+
// (tunnel proxy, desktop UI) render an actionable error instead of a
|
|
685
761
|
// generic auto-pull / spawn failure.
|
|
686
762
|
const failedExisting = this.runtimes.get(projectId);
|
|
687
763
|
if (failedExisting?.status === 'failed') {
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
764
|
+
const cooldownMs = this.breakerCooldownMs(failedExisting.breakerTrips);
|
|
765
|
+
const sinceTripMs = Date.now() - failedExisting.lastFailureAt;
|
|
766
|
+
if (sinceTripMs >= cooldownMs) {
|
|
767
|
+
this.log.log(
|
|
768
|
+
`[WorkerRuntimeManager] auto-heal: ${projectId} cooldown elapsed ` +
|
|
769
|
+
`(${Math.round(sinceTripMs / 1000)}s since trip #${failedExisting.breakerTrips}) — ` +
|
|
770
|
+
`clearing circuit breaker and attempting a fresh spawn`,
|
|
771
|
+
);
|
|
772
|
+
this.healBreaker(failedExisting);
|
|
773
|
+
// Falls through — `failedExisting` is the same object referenced
|
|
774
|
+
// below as `existing`, now with status reset off `'failed'`.
|
|
775
|
+
} else {
|
|
776
|
+
const retryInMs = cooldownMs - sinceTripMs;
|
|
777
|
+
throw new Error(
|
|
778
|
+
`[WorkerRuntimeManager] cannot ensureRunning(${projectId}): ` +
|
|
779
|
+
`${failedExisting.lastError ?? 'runtime is in failed state'} ` +
|
|
780
|
+
`Will auto-retry in ~${Math.ceil(retryInMs / 1000)}s; ` +
|
|
781
|
+
`call resetFailure(${projectId}) to retry immediately, or stop(${projectId}) to give up.`,
|
|
782
|
+
);
|
|
783
|
+
}
|
|
693
784
|
}
|
|
694
785
|
|
|
695
786
|
// Apply auto-pull before any runtime spawn so the runtime's PROJECT_DIR
|
|
@@ -1076,15 +1167,21 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1076
1167
|
}
|
|
1077
1168
|
|
|
1078
1169
|
/**
|
|
1079
|
-
* Re-arm a runtime that the circuit breaker parked in `'failed'
|
|
1080
|
-
*
|
|
1081
|
-
*
|
|
1170
|
+
* Re-arm a runtime that the circuit breaker parked in `'failed'`,
|
|
1171
|
+
* bypassing the auto-heal cooldown (see `breakerCooldownMs`) for an
|
|
1172
|
+
* IMMEDIATE retry. Drops the slot from the map so the next
|
|
1173
|
+
* `ensureRunning(projectId, …)` call performs a fresh `doStart()` with a
|
|
1174
|
+
* zeroed failure budget (though `breakerTrips` — and therefore the NEXT
|
|
1175
|
+
* cooldown if it re-trips — carries over; only a full healthy run via
|
|
1176
|
+
* `armGraceTimer` resets that).
|
|
1082
1177
|
*
|
|
1083
|
-
*
|
|
1084
|
-
*
|
|
1085
|
-
*
|
|
1086
|
-
*
|
|
1087
|
-
*
|
|
1178
|
+
* `ensureRunning()` itself now auto-heals a `'failed'` slot once its
|
|
1179
|
+
* cooldown elapses, so this is no longer the only way out of `'failed'`
|
|
1180
|
+
* — it's for callers that don't want to wait: the desktop's "reopen
|
|
1181
|
+
* project" flow, or an operator who fixed whatever was crashing the
|
|
1182
|
+
* runtime (e.g. freed memory, deleted a corrupted workspace file) and
|
|
1183
|
+
* wants to retry right now without tearing down the whole worker. No-op
|
|
1184
|
+
* if the project isn't in `'failed'`.
|
|
1088
1185
|
*/
|
|
1089
1186
|
resetFailure(projectId: string): boolean {
|
|
1090
1187
|
const r = this.runtimes.get(projectId);
|
|
@@ -1098,6 +1195,35 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1098
1195
|
return true;
|
|
1099
1196
|
}
|
|
1100
1197
|
|
|
1198
|
+
/**
|
|
1199
|
+
* Escalating cooldown for the Nth breaker trip: 2m, 4m, 8m, 15m(cap), …
|
|
1200
|
+
* `tripCount` is `breakerTrips` *before* incrementing for this trip (so
|
|
1201
|
+
* the very first trip — `tripCount === 0` going into `handleExit`, which
|
|
1202
|
+
* increments it to `1` — cools down for the base 2m, not 4m).
|
|
1203
|
+
*/
|
|
1204
|
+
private breakerCooldownMs(tripCount: number): number {
|
|
1205
|
+
const exponent = Math.max(0, tripCount - 1);
|
|
1206
|
+
const scaled = CIRCUIT_BREAKER_COOLDOWN_BASE_MS * Math.pow(2, exponent);
|
|
1207
|
+
return Math.min(scaled, CIRCUIT_BREAKER_COOLDOWN_MAX_MS);
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* Clear a `'failed'` slot's breaker state in place (used by both the
|
|
1212
|
+
* auto-heal path in `ensureRunning` and could be called directly by a
|
|
1213
|
+
* future explicit "retry now" caller). Unlike {@link resetFailure} this
|
|
1214
|
+
* does NOT remove the slot from `this.runtimes` — the caller is about to
|
|
1215
|
+
* reuse the same object as the `existing` slot for a fresh `doStart()`,
|
|
1216
|
+
* so timers/state on it are about to be overwritten by that path anyway.
|
|
1217
|
+
* `breakerTrips` and `restartTimestamps` are deliberately left alone:
|
|
1218
|
+
* they're what makes cooldowns escalate for a project that keeps
|
|
1219
|
+
* re-tripping, and `armGraceTimer` is what resets them back to zero once
|
|
1220
|
+
* the project actually recovers.
|
|
1221
|
+
*/
|
|
1222
|
+
private healBreaker(slot: InternalRuntime): void {
|
|
1223
|
+
slot.status = 'stopped';
|
|
1224
|
+
slot.consecutiveFailures = 0;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1101
1227
|
async stopAll(signal: NodeJS.Signals = 'SIGTERM'): Promise<void> {
|
|
1102
1228
|
this.stopped = true;
|
|
1103
1229
|
// Stop watchers FIRST so their final flush has a chance to PUT before
|
|
@@ -1137,6 +1263,8 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1137
1263
|
restarts: 0,
|
|
1138
1264
|
consecutiveFailures: 0,
|
|
1139
1265
|
lastFailureAt: 0,
|
|
1266
|
+
breakerTrips: 0,
|
|
1267
|
+
restartTimestamps: [],
|
|
1140
1268
|
graceTimer: null,
|
|
1141
1269
|
restartTimer: null,
|
|
1142
1270
|
idleTimer: null,
|
|
@@ -1429,6 +1557,12 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1429
1557
|
* survived for {@link STARTUP_GRACE_MS}. Re-armed on every successful
|
|
1430
1558
|
* /health transition; cleared on any non-clean exit so a crash
|
|
1431
1559
|
* inside the grace window counts toward the circuit breaker.
|
|
1560
|
+
*
|
|
1561
|
+
* Also resets `breakerTrips` and `restartTimestamps` on that same
|
|
1562
|
+
* survived-a-full-grace-window signal: a project that's actually
|
|
1563
|
+
* recovered for good shouldn't carry an escalated auto-heal cooldown
|
|
1564
|
+
* (see `breakerCooldownMs`) or slow-bleed history from an incident
|
|
1565
|
+
* months ago into its next unrelated hiccup.
|
|
1432
1566
|
*/
|
|
1433
1567
|
private armGraceTimer(slot: InternalRuntime): void {
|
|
1434
1568
|
if (slot.graceTimer) {
|
|
@@ -1440,6 +1574,8 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1440
1574
|
if (slot.consecutiveFailures > 0) {
|
|
1441
1575
|
slot.consecutiveFailures = 0;
|
|
1442
1576
|
}
|
|
1577
|
+
slot.breakerTrips = 0;
|
|
1578
|
+
slot.restartTimestamps = [];
|
|
1443
1579
|
}, STARTUP_GRACE_MS);
|
|
1444
1580
|
try { slot.graceTimer.unref?.(); } catch { /* unref is best-effort */ }
|
|
1445
1581
|
}
|
|
@@ -1586,15 +1722,30 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1586
1722
|
slot.restarts += 1;
|
|
1587
1723
|
slot.lastError = `exited code=${code} signal=${signal}`;
|
|
1588
1724
|
|
|
1589
|
-
|
|
1725
|
+
// Slow-bleed tracking: independent of `consecutiveFailures`, which the
|
|
1726
|
+
// 60s startup grace timer resets on every incarnation that survives a
|
|
1727
|
+
// minute. Pruned to a rolling hour so a project that crashed a lot last
|
|
1728
|
+
// month doesn't stay flagged forever.
|
|
1729
|
+
slot.restartTimestamps.push(now);
|
|
1730
|
+
slot.restartTimestamps = slot.restartTimestamps.filter((t) => now - t <= SLOW_BLEED_WINDOW_MS);
|
|
1731
|
+
|
|
1732
|
+
const tightLoopTripped = slot.consecutiveFailures >= MAX_CONSECUTIVE_RESTARTS;
|
|
1733
|
+
const slowBleedTripped = slot.restartTimestamps.length >= SLOW_BLEED_MAX_RESTARTS;
|
|
1734
|
+
|
|
1735
|
+
if (tightLoopTripped || slowBleedTripped) {
|
|
1590
1736
|
slot.status = 'failed';
|
|
1737
|
+
slot.breakerTrips += 1;
|
|
1738
|
+
const cooldownMs = this.breakerCooldownMs(slot.breakerTrips);
|
|
1739
|
+
const cause = tightLoopTripped
|
|
1740
|
+
? `${slot.consecutiveFailures} consecutive non-clean exits within ${Math.round(RESTART_FAILURE_WINDOW_MS / 1000)}s`
|
|
1741
|
+
: `${slot.restartTimestamps.length} non-clean exits within the last ${Math.round(SLOW_BLEED_WINDOW_MS / 60_000)}m (slow-bleed loop — gaps between crashes kept clearing the tight-loop window)`;
|
|
1591
1742
|
slot.lastError =
|
|
1592
|
-
`Circuit breaker tripped
|
|
1593
|
-
`
|
|
1594
|
-
`
|
|
1595
|
-
`
|
|
1596
|
-
`
|
|
1597
|
-
`
|
|
1743
|
+
`Circuit breaker tripped (trip #${slot.breakerTrips}): ${cause} ` +
|
|
1744
|
+
`(last: code=${code} signal=${signal}). Most recent on macOS is jetsam OOM ` +
|
|
1745
|
+
`(signal=SIGKILL with code=null); the previous incarnation's vite/tsserver/` +
|
|
1746
|
+
`preview-manager children were reaped to prevent further RSS growth. ` +
|
|
1747
|
+
`Will auto-retry in ~${Math.round(cooldownMs / 1000)}s; call resetFailure(${slot.projectId}) ` +
|
|
1748
|
+
`to retry immediately, or stop(${slot.projectId}) to give up.`;
|
|
1598
1749
|
this.releasePort(slot.agentPort);
|
|
1599
1750
|
slot.agentPort = 0;
|
|
1600
1751
|
slot.apiServerPort = 0;
|