@botbuddy/cli 1.63.14 → 1.63.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +1 -1
- package/src/daemon/callback-release.mjs +50 -0
- package/src/daemon/installer.mjs +30 -0
- package/src/fixtures/bot2775-callback-contract.json +32 -0
- package/src/fixtures/bot2779-handler-status-2026-10-27.json +47 -0
- package/src/stack-callback-gateway.mjs +44 -0
- package/src/stack-native-callback.mjs +126 -0
- package/src/stack-tunnel.mjs +4 -2
- package/src/stack.mjs +64 -13
|
Binary file
|
|
Binary file
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.63.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.63.16",
|
|
3
|
+
"source_commit": "1b763461c5262553e1214f40d1c1da94f6761b9a",
|
|
4
|
+
"source_sha256": "9925e4a7e228d64aee885a9334ca45684b8ff6f423441ddf4825a30ad878723e",
|
|
5
5
|
"target": "darwin-arm64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "8516955d8e25a9430f715bdeb3afca7ee9df991f0f4a97ed504ed81b204ae4c6"
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.63.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.63.16",
|
|
3
|
+
"source_commit": "1b763461c5262553e1214f40d1c1da94f6761b9a",
|
|
4
|
+
"source_sha256": "9925e4a7e228d64aee885a9334ca45684b8ff6f423441ddf4825a30ad878723e",
|
|
5
5
|
"target": "linux-x64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "a6d5ffb7686b855ed77819c58343122bcb906c1d6fcb3431d34cf3862525e7e1"
|
|
7
7
|
}
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": 1,
|
|
3
|
-
"release_version": "1.63.
|
|
4
|
-
"source_commit": "
|
|
5
|
-
"source_sha256": "
|
|
3
|
+
"release_version": "1.63.16",
|
|
4
|
+
"source_commit": "1b763461c5262553e1214f40d1c1da94f6761b9a",
|
|
5
|
+
"source_sha256": "9925e4a7e228d64aee885a9334ca45684b8ff6f423441ddf4825a30ad878723e",
|
|
6
6
|
"targets": {
|
|
7
7
|
"darwin-arm64": {
|
|
8
8
|
"goos": "darwin",
|
|
9
9
|
"goarch": "arm64",
|
|
10
10
|
"binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
|
|
11
|
-
"sha256": "
|
|
11
|
+
"sha256": "8516955d8e25a9430f715bdeb3afca7ee9df991f0f4a97ed504ed81b204ae4c6",
|
|
12
12
|
"mode": "0755"
|
|
13
13
|
},
|
|
14
14
|
"linux-x64": {
|
|
15
15
|
"goos": "linux",
|
|
16
16
|
"goarch": "amd64",
|
|
17
17
|
"binary": "daemon/bin/linux-x64/botbuddy-daemon",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "a6d5ffb7686b855ed77819c58343122bcb906c1d6fcb3431d34cf3862525e7e1",
|
|
19
19
|
"mode": "0755"
|
|
20
20
|
}
|
|
21
21
|
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// BOT-2775: authenticate an older installed runtime against its exact published
|
|
2
|
+
// npm package. Only archive data are read; no downloaded code is executed.
|
|
3
|
+
import {createHash} from 'node:crypto';
|
|
4
|
+
import {execFile as execFileCallback} from 'node:child_process';
|
|
5
|
+
import {promisify} from 'node:util';
|
|
6
|
+
import {mkdtemp,lstat,readFile,mkdir,writeFile,rm} from 'node:fs/promises';
|
|
7
|
+
import {join} from 'node:path';
|
|
8
|
+
import {tmpdir} from 'node:os';
|
|
9
|
+
import {validateDaemonRelease} from './release.mjs';
|
|
10
|
+
const execFile=promisify(execFileCallback),MAX_ARCHIVE=128*1024*1024,MAX_BINARY=64*1024*1024;
|
|
11
|
+
const fail=reason=>{throw Error('callback_release_'+reason)};
|
|
12
|
+
const matches=(r,v,c,s)=>r.releaseVersion===v&&r.sourceCommit===c&&r.sha256===s;
|
|
13
|
+
async function metadataJSON(response){
|
|
14
|
+
if(!response.ok||!response.body)fail('metadata');let size=0;const chunks=[];
|
|
15
|
+
for await(const chunk of response.body){size+=chunk.length;if(size>65536)fail('metadata_size');chunks.push(chunk)}
|
|
16
|
+
try{return JSON.parse(Buffer.concat(chunks).toString('utf8'))}catch{fail('metadata')}
|
|
17
|
+
}
|
|
18
|
+
export async function verifiedCallbackRelease({version,sourceCommit,sha256,releaseRoot,run=execFile,fetchImpl=fetch,validateRelease=validateDaemonRelease}={}){
|
|
19
|
+
if(!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)||!/^[0-9a-f]{40}$/.test(sourceCommit)||!/^[0-9a-f]{64}$/.test(sha256))fail('identity');
|
|
20
|
+
try{const current=await validateRelease({releaseRoot},{platform:'linux',arch:'x64'});if(matches(current,version,sourceCommit,sha256))return current}catch{}
|
|
21
|
+
const controller=new AbortController(),deadline=setTimeout(()=>controller.abort(),60_000);deadline.unref();
|
|
22
|
+
let root;
|
|
23
|
+
try{
|
|
24
|
+
const metadata=await metadataJSON(await fetchImpl('https://registry.npmjs.org/@botbuddy%2fcli/'+encodeURIComponent(version),{signal:controller.signal,redirect:'error'}));
|
|
25
|
+
const integrity=metadata?.dist?.integrity;
|
|
26
|
+
if(metadata?.name!=='@botbuddy/cli'||metadata.version!==version||metadata.dist?.tarball!==`https://registry.npmjs.org/@botbuddy/cli/-/cli-${version}.tgz`||typeof integrity!=='string'||!/^sha512-[A-Za-z0-9+/]{86}==$/.test(integrity))fail('metadata');
|
|
27
|
+
root=await mkdtemp(join(tmpdir(),'botbuddy-callback-release-'));
|
|
28
|
+
const packed=await run('npm',['pack','@botbuddy/cli@'+version,'--ignore-scripts','--json','--pack-destination',root,'--registry=https://registry.npmjs.org','--fetch-retries=0','--fetch-timeout=30000'],{signal:controller.signal,timeout:45_000,maxBuffer:65536,encoding:'utf8'});
|
|
29
|
+
let result;try{result=JSON.parse(packed.stdout)}catch{fail('pack')}
|
|
30
|
+
if(!Array.isArray(result)||result.length!==1||result[0].name!=='@botbuddy/cli'||result[0].version!==version||result[0].integrity!==integrity||!/^[-A-Za-z0-9.]+\.tgz$/.test(result[0].filename))fail('pack');
|
|
31
|
+
const archive=join(root,result[0].filename),stat=await lstat(archive);
|
|
32
|
+
if(!stat.isFile()||stat.isSymbolicLink()||stat.size>MAX_ARCHIVE)fail('archive');
|
|
33
|
+
if('sha512-'+createHash('sha512').update(await readFile(archive)).digest('base64')!==integrity)fail('integrity');
|
|
34
|
+
const tar=async(args,maxBuffer)=>run('tar',args,{signal:controller.signal,timeout:10_000,maxBuffer,encoding:'buffer'});
|
|
35
|
+
const names=(await tar(['-tzf',archive],4*1024*1024)).stdout.toString('utf8').trimEnd().split('\n');
|
|
36
|
+
const details=(await tar(['-tvzf',archive],8*1024*1024)).stdout.toString('utf8').trimEnd().split('\n');
|
|
37
|
+
if(names.length!==details.length||new Set(names).size!==names.length||names.some((name,i)=>!/^package\//.test(name)||/[\\\x00-\x20\x7f]/.test(name)||name.split('/').some(part=>part==='..'||part==='.')||!['-','d'].includes(details[i][0])))fail('archive_entries');
|
|
38
|
+
const entries=['package/package.json','package/daemon-release/manifest.json','package/daemon/bin/linux-x64/botbuddy-daemon'];
|
|
39
|
+
if(entries.some(name=>!names.includes(name)||details[names.indexOf(name)][0]!=='-'))fail('archive_entries');
|
|
40
|
+
const buffers=[];for(const name of entries)buffers.push((await tar(['-xOzf',archive,'--',name],name.endsWith('botbuddy-daemon')?MAX_BINARY:65536)).stdout);
|
|
41
|
+
let pkg;try{pkg=JSON.parse(buffers[0])}catch{fail('package')}
|
|
42
|
+
if(pkg.name!=='@botbuddy/cli'||pkg.version!==version)fail('package');
|
|
43
|
+
const verifiedRoot=join(root,'verified');await mkdir(join(verifiedRoot,'daemon-release'),{recursive:true,mode:0o700});await mkdir(join(verifiedRoot,'daemon/bin/linux-x64'),{recursive:true,mode:0o700});
|
|
44
|
+
await writeFile(join(verifiedRoot,'daemon-release/manifest.json'),buffers[1],{mode:0o600});await writeFile(join(verifiedRoot,'daemon/bin/linux-x64/botbuddy-daemon'),buffers[2],{mode:0o755});
|
|
45
|
+
const release=await validateDaemonRelease({releaseRoot:verifiedRoot},{platform:'linux',arch:'x64'});
|
|
46
|
+
if(!matches(release,version,sourceCommit,sha256))fail('identity');
|
|
47
|
+
// No path or payload escapes the temporary verification scope.
|
|
48
|
+
return Object.freeze({releaseVersion:release.releaseVersion,sourceCommit:release.sourceCommit,sha256:release.sha256,target:release.target});
|
|
49
|
+
}finally{clearTimeout(deadline);controller.abort();if(root)await rm(root,{recursive:true,force:true})}
|
|
50
|
+
}
|
package/src/daemon/installer.mjs
CHANGED
|
@@ -436,3 +436,33 @@ export async function serviceStatus({ platform, host, exec = execFile } = {}) {
|
|
|
436
436
|
throw error;
|
|
437
437
|
}
|
|
438
438
|
}
|
|
439
|
+
|
|
440
|
+
// BOT-2775 uses the same immutable release layout, root ancestry and external
|
|
441
|
+
// checksum checks as installation before invoking a producer-published mode.
|
|
442
|
+
// This read-only check runs as the already-authorized SSH/service user, no sudo.
|
|
443
|
+
export const remoteVerifyInstalledCallbackScript = String.raw`set -eu
|
|
444
|
+
binary=$1
|
|
445
|
+
digest=$2
|
|
446
|
+
service_uid=$3
|
|
447
|
+
test "$(uname -s)" = Linux
|
|
448
|
+
test "$(id -u)" = "$service_uid"
|
|
449
|
+
parent=$(dirname "$binary")
|
|
450
|
+
while :; do
|
|
451
|
+
test ! -L "$parent"
|
|
452
|
+
test -d "$parent"
|
|
453
|
+
test "$(stat -c '%u' "$parent")" = 0
|
|
454
|
+
mode=$(stat -c '%a' "$parent")
|
|
455
|
+
case "$mode" in *[2367][0-7]|*[0-7][2367]) exit 82 ;; esac
|
|
456
|
+
test "$parent" != / || break
|
|
457
|
+
parent=$(dirname "$parent")
|
|
458
|
+
done
|
|
459
|
+
test ! -L "$binary"
|
|
460
|
+
test -f "$binary"
|
|
461
|
+
test -x "$binary"
|
|
462
|
+
test "$(stat -c '%u' "$binary")" = 0
|
|
463
|
+
mode=$(stat -c '%a' "$binary")
|
|
464
|
+
case "$mode" in *[2367][0-7]|*[0-7][2367]) exit 82 ;; esac
|
|
465
|
+
actual=$(sha256sum "$binary" | awk '{print $1}')
|
|
466
|
+
test "$actual" = "$digest"
|
|
467
|
+
printf '%s\n' '{"verified":true}'
|
|
468
|
+
`;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fixture_scope": "Synthetic contract fixture only; not an actual installed producer receipt.",
|
|
3
|
+
"callback_transport": "native_origin_v1",
|
|
4
|
+
"callback_current_authorization": true,
|
|
5
|
+
"callback_context": {
|
|
6
|
+
"owner": "operator@example.test",
|
|
7
|
+
"tenant_id": "botbuddy",
|
|
8
|
+
"requester_machine_id": "30000000-0000-4000-8000-000000000001",
|
|
9
|
+
"descriptor": {
|
|
10
|
+
"protocol": 1,
|
|
11
|
+
"binary_path": "/opt/botbuddy/daemon/releases/1.99.0/linux-x64/botbuddy-daemon",
|
|
12
|
+
"artifact_sha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
13
|
+
"config_path": "/home/botbuddy/.botbuddy-daemon/runtime.json",
|
|
14
|
+
"service_uid": 1001,
|
|
15
|
+
"runtime_version": "1.99.0",
|
|
16
|
+
"runtime_git_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
17
|
+
"runtime_target": "linux-amd64",
|
|
18
|
+
"host_key": "bbh-cccccccccccccccccccccccc",
|
|
19
|
+
"machine_id": "exact-linux-machine-id-spelling",
|
|
20
|
+
"engine_id": "engine-fixture",
|
|
21
|
+
"lease_id": "10000000-0000-4000-8000-000000000001",
|
|
22
|
+
"job_id": "20000000-0000-4000-8000-000000000001",
|
|
23
|
+
"attempt": 1,
|
|
24
|
+
"tenant_id": "botbuddy",
|
|
25
|
+
"project_id": "bbbot2775fixture",
|
|
26
|
+
"callback_port": 4600,
|
|
27
|
+
"producer_pid": 4242,
|
|
28
|
+
"producer_start_ticks": "100000",
|
|
29
|
+
"producer_boot_id": "70000000-0000-4000-8000-000000000001"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"provenance": {
|
|
3
|
+
"captured_at": "2026-10-27",
|
|
4
|
+
"source": "BOT-2779 actual isolated get_stack_lease handler/SQL fixture run 1afcb247",
|
|
5
|
+
"producer_descriptor": "synthetic, not a native producer receipt",
|
|
6
|
+
"projection": "sanitized by backend test owner; only context/project metadata"
|
|
7
|
+
},
|
|
8
|
+
"response": {
|
|
9
|
+
"lease_id": "74704da2-d98a-45c2-9dcc-f3d69e901d24",
|
|
10
|
+
"state": "active",
|
|
11
|
+
"host_key": "bbh-09093a8a9efa4f45691a61e7",
|
|
12
|
+
"execution_mode": "remote",
|
|
13
|
+
"callback_port": 4600,
|
|
14
|
+
"callback_transport": "native_origin_v1",
|
|
15
|
+
"callback_current_authorization": true,
|
|
16
|
+
"callback_context": {
|
|
17
|
+
"owner": "jono@bot-buddy.ai",
|
|
18
|
+
"tenant_id": "botbuddy",
|
|
19
|
+
"requester_machine_id": "e0c36c64-5055-4456-9df8-5c1d1b60fa43",
|
|
20
|
+
"descriptor": {
|
|
21
|
+
"protocol": 1,
|
|
22
|
+
"binary_path": "/opt/botbuddy/daemon/releases/1.99.0/linux-x64/botbuddy-daemon",
|
|
23
|
+
"artifact_sha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
24
|
+
"config_path": "/home/botbuddy/.botbuddy-daemon/runtime.json",
|
|
25
|
+
"service_uid": 1001,
|
|
26
|
+
"runtime_version": "1.99.0",
|
|
27
|
+
"runtime_git_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
28
|
+
"runtime_target": "linux-amd64",
|
|
29
|
+
"host_key": "bbh-09093a8a9efa4f45691a61e7",
|
|
30
|
+
"machine_id": "57d36d8c-a574-4971-98d1-3bea0def5a6f",
|
|
31
|
+
"engine_id": "engine-fixture",
|
|
32
|
+
"lease_id": "74704da2-d98a-45c2-9dcc-f3d69e901d24",
|
|
33
|
+
"job_id": "098b322e-d162-49d9-8f5a-44d31441d88f",
|
|
34
|
+
"attempt": 1,
|
|
35
|
+
"tenant_id": "botbuddy",
|
|
36
|
+
"project_id": "bb214530925b62",
|
|
37
|
+
"callback_port": 4600,
|
|
38
|
+
"producer_pid": 4242,
|
|
39
|
+
"producer_start_ticks": "100000",
|
|
40
|
+
"producer_boot_id": "70000000-0000-4000-8000-000000000001"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"connection": {
|
|
44
|
+
"project_id": "bb214530925b62"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// BOT-2775: private requester gateway for one same-owner SSH callback generation.
|
|
2
|
+
// The caller supplies only a binding from the authenticated CURRENT lease read.
|
|
3
|
+
import {createServer, request} from 'node:http';
|
|
4
|
+
import {createHash,createHmac, timingSafeEqual} from 'node:crypto';
|
|
5
|
+
import {Transform} from 'node:stream';
|
|
6
|
+
export const CALLBACK_CAP_HEADER='x-botbuddy-callback-capability';
|
|
7
|
+
const digest=value=>createHash('sha256').update(value).digest();
|
|
8
|
+
export const CALLBACK_MAX_STREAM_BYTES=32*1024*1024;
|
|
9
|
+
const sameBinding=(a,b)=>b&&typeof b==='object'&&Object.keys(a).length===Object.keys(b).length&&Object.keys(a).every(key=>a[key]===b[key]);
|
|
10
|
+
export async function openCallbackGateway({callbackPort,binding,capability,lifetimeMs=30_000,now=Date.now,activate=true,readinessNonce=null}){
|
|
11
|
+
if(!Number.isInteger(callbackPort)||callbackPort<1024||callbackPort>65535||!binding||typeof binding!=='object'||!Object.isFrozen(binding)||typeof capability!=='string'||!/^[0-9a-f]{64}$/.test(capability)||lifetimeMs<1||lifetimeMs>30_000)throw Error('callback gateway admission refused');
|
|
12
|
+
if(readinessNonce!==null&&!/^[0-9a-f]{32}$/.test(readinessNonce))throw Error('callback readiness identity refused');
|
|
13
|
+
const bootstrapDeadline=now()+30_000,readinessPath=readinessNonce?'/.botbuddy/callback-ready/'+readinessNonce:null;
|
|
14
|
+
const readinessProof=readinessNonce?createHmac('sha256',Buffer.from(capability,'hex')).update('native_origin_v1/readiness/'+readinessNonce).digest('hex'):null;
|
|
15
|
+
const expected=digest(capability),active=new Set(),closeListeners=new Set();let deadline=0,timer,closed=false;
|
|
16
|
+
const server=createServer((incoming,outgoing)=>{
|
|
17
|
+
const presented=incoming.headers[CALLBACK_CAP_HEADER];
|
|
18
|
+
const readiness=readinessPath&&incoming.method==='HEAD'&&incoming.url===readinessPath;
|
|
19
|
+
if(closed||(now()>=deadline&&!(readiness&&deadline===0&&now()<bootstrapDeadline))||typeof presented!=='string'||!/^[0-9a-f]{64}$/.test(presented)||!timingSafeEqual(digest(presented),expected)||!incoming.url?.startsWith('/')||incoming.url.startsWith('//')||incoming.method==='TRACE'||incoming.headers.upgrade){outgoing.writeHead(403);outgoing.end('CALLBACK_REFUSED');return;}
|
|
20
|
+
if(readiness){outgoing.writeHead(204,{'x-botbuddy-callback-ready':readinessNonce,'x-botbuddy-callback-proof':readinessProof});outgoing.end();return;}
|
|
21
|
+
if(Number(incoming.headers['content-length'])>CALLBACK_MAX_STREAM_BYTES){outgoing.writeHead(413,{connection:'close'});outgoing.end('CALLBACK_STREAM_LIMIT');return;}
|
|
22
|
+
const headers={...incoming.headers,host:'127.0.0.1:'+callbackPort};delete headers[CALLBACK_CAP_HEADER];delete headers['x-botbuddy-callback-ready'];delete headers['x-botbuddy-callback-proof'];delete headers['proxy-authorization'];delete headers['proxy-connection'];
|
|
23
|
+
const abort=new AbortController();active.add(abort);let limited=false;
|
|
24
|
+
const finish=()=>{active.delete(abort);abort.abort();};outgoing.once('close',finish);incoming.once('aborted',finish);
|
|
25
|
+
const limitStream=()=>{let bytes=0;return new Transform({transform(chunk,_encoding,done){bytes+=chunk.length;if(bytes>CALLBACK_MAX_STREAM_BYTES)done(Error('CALLBACK_STREAM_LIMIT'));else done(null,chunk);}});};
|
|
26
|
+
const upstream=request({host:'127.0.0.1',port:callbackPort,path:incoming.url,method:incoming.method,headers,signal:abort.signal},response=>{
|
|
27
|
+
const responseHeaders={...response.headers};delete responseHeaders[CALLBACK_CAP_HEADER];delete responseHeaders['x-botbuddy-callback-ready'];delete responseHeaders['x-botbuddy-callback-proof'];
|
|
28
|
+
if(incoming.method!=='HEAD'&&Number(response.headers['content-length'])>CALLBACK_MAX_STREAM_BYTES){limited=true;response.destroy();outgoing.writeHead(502,{connection:'close'});outgoing.end('CALLBACK_STREAM_LIMIT');return;}
|
|
29
|
+
const counter=limitStream();counter.on('error',()=>{limited=true;response.destroy();outgoing.destroy();finish();});
|
|
30
|
+
response.on('error',()=>outgoing.destroy());outgoing.writeHead(response.statusCode,responseHeaders);response.pipe(counter).pipe(outgoing);
|
|
31
|
+
});
|
|
32
|
+
upstream.on('error',()=>{if(limited)return;if(!outgoing.headersSent){outgoing.writeHead(503);outgoing.end('CALLBACK_UPSTREAM_UNAVAILABLE');}else outgoing.destroy();});
|
|
33
|
+
// Count both streaming directions without buffering. Application auth and
|
|
34
|
+
// HMAC/body bytes are preserved; a stream over32MiB is aborted.
|
|
35
|
+
const counter=limitStream();counter.on('error',()=>{limited=true;upstream.destroy();incoming.unpipe(counter);if(!outgoing.headersSent){outgoing.writeHead(413,{connection:'close'});outgoing.end('CALLBACK_STREAM_LIMIT',()=>incoming.destroy());}else outgoing.destroy();finish();});incoming.pipe(counter).pipe(upstream);
|
|
36
|
+
});
|
|
37
|
+
server.maxConnections=64;server.maxHeadersCount=100;server.headersTimeout=10_000;server.requestTimeout=0;
|
|
38
|
+
const denySocket=(_req,socket)=>socket.end('HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n');server.on('connect',denySocket);server.on('upgrade',denySocket);
|
|
39
|
+
let closePromise;const close=()=>{if(closePromise)return closePromise;closed=true;clearTimeout(timer);for(const controller of active)controller.abort();active.clear();server.closeAllConnections();closePromise=new Promise(resolve=>server.close(resolve));for(const listener of closeListeners)listener();return closePromise;};
|
|
40
|
+
const refresh=(current,validForMs=lifetimeMs)=>{let matches=false;try{matches=sameBinding(binding,current)}catch{}if(closed||!matches||!Number.isFinite(validForMs)||validForMs<=0||validForMs>lifetimeMs){close();throw Error('callback binding changed');}deadline=now()+validForMs;clearTimeout(timer);timer=setTimeout(close,validForMs);timer.unref();};
|
|
41
|
+
await new Promise((resolve,reject)=>{server.once('error',reject);server.listen(0,'127.0.0.1',resolve);});
|
|
42
|
+
if(activate)refresh(binding);else{timer=setTimeout(close,30_000);timer.unref();}
|
|
43
|
+
return {port:server.address().port,refresh,close,onClose(listener){closeListeners.add(listener);if(closed)listener()},isLive:()=>!closed&&now()<deadline};
|
|
44
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// BOT-2775 same-owner, required native HTTP callback transport. No child capability.
|
|
2
|
+
import {execFile as execFileCallback,spawn} from 'node:child_process';
|
|
3
|
+
import {promisify} from 'node:util';
|
|
4
|
+
import {randomBytes,randomUUID} from 'node:crypto';
|
|
5
|
+
import {createInterface} from 'node:readline';
|
|
6
|
+
import {posix} from 'node:path';
|
|
7
|
+
import {isIP as netIsIP} from 'node:net';
|
|
8
|
+
import {fileURLToPath} from 'node:url';
|
|
9
|
+
import {validateDaemonRelease} from './daemon/release.mjs';
|
|
10
|
+
import {verifiedCallbackRelease} from './daemon/callback-release.mjs';
|
|
11
|
+
import {assertSshTarget,SSH_FORWARD_OPTIONS} from './stack-tunnel.mjs';
|
|
12
|
+
import {installedBinaryPath,remoteVerifyInstalledCallbackScript} from './daemon/installer.mjs';
|
|
13
|
+
import {openCallbackGateway} from './stack-callback-gateway.mjs';
|
|
14
|
+
export const NATIVE_CALLBACK_TRANSPORT='native_origin_v1';
|
|
15
|
+
const execFile=promisify(execFileCallback),uuid=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
16
|
+
const safePath=value=>typeof value==='string'&&value.startsWith('/')&&!/[\x00-\x1f\x7f]/.test(value)&&posix.normalize(value)===value;
|
|
17
|
+
const text=value=>typeof value==='string'&&value.length>0&&value.length<=512&&!/[\x00-\x20\x7f]/.test(value);
|
|
18
|
+
const version=/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
|
|
19
|
+
const fields=['protocol','binary_path','artifact_sha256','config_path','service_uid','runtime_version','runtime_git_sha','runtime_target','host_key','machine_id','engine_id','lease_id','job_id','attempt','tenant_id','project_id','callback_port','producer_pid','producer_start_ticks','producer_boot_id'];
|
|
20
|
+
const fail=(code,message)=>{throw Object.assign(Error(message),{code});};
|
|
21
|
+
const quote=value=>"'"+String(value).replaceAll("'","'\\''")+"'";
|
|
22
|
+
const canonicalMachine=value=>typeof value==='string'&&/^(?:[0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i.test(value)?value.replaceAll('-','').toLowerCase():null;
|
|
23
|
+
|
|
24
|
+
export function validatedCallbackDescriptor(value){
|
|
25
|
+
if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).length!==fields.length||fields.some(field=>!(field in value)))fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','native callback descriptor missing or unknown');
|
|
26
|
+
const d=Object.fromEntries(fields.map(field=>[field,value[field]]));
|
|
27
|
+
if(d.protocol!==1||typeof d.runtime_version!=='string'||!version.test(d.runtime_version)||d.runtime_target!=='linux-amd64'||d.binary_path!==installedBinaryPath({releaseVersion:d.runtime_version,target:'linux-x64'})||!safePath(d.config_path)||!/^sha256:[0-9a-f]{64}$/.test(d.artifact_sha256)||!Number.isSafeInteger(d.service_uid)||d.service_uid<1||!Number.isSafeInteger(d.producer_pid)||d.producer_pid<=1||typeof d.producer_start_ticks!=='string'||!/^[1-9][0-9]*$/.test(d.producer_start_ticks)||!uuid.test(d.producer_boot_id)||!text(d.host_key)||!text(d.machine_id)||!text(d.engine_id)||!uuid.test(d.lease_id)||!uuid.test(d.job_id)||!Number.isSafeInteger(d.attempt)||d.attempt<1||!text(d.tenant_id)||typeof d.project_id!=='string'||!/^[a-z0-9][a-z0-9_-]{1,62}$/.test(d.project_id)||!Number.isInteger(d.callback_port)||d.callback_port<1024||d.callback_port>65535||!/^[0-9a-f]{40}$/.test(d.runtime_git_sha))fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','native callback runtime identity refused');
|
|
28
|
+
return Object.freeze(d);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function currentCallbackBinding(current,{leaseId,hostKey,callbackPort,requesterMachineId,routeGeneration}){
|
|
32
|
+
if(current?.callback_transport!==NATIVE_CALLBACK_TRANSPORT)fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','backend did not acknowledge required callback transport');
|
|
33
|
+
if(current.callback_current_authorization!==true||current.state!=='active'||current.execution_mode!=='remote')fail('CALLBACK_AUTHORIZATION_LOST','current callback authorization unavailable');
|
|
34
|
+
const context=current.callback_context,d=validatedCallbackDescriptor(context?.descriptor);
|
|
35
|
+
if(current.lease_id!==leaseId||current.host_key!==hostKey||d.lease_id!==leaseId||d.host_key!==hostKey||d.callback_port!==callbackPort||current.callback_port!==callbackPort||d.project_id!==current.connection?.project_id||!context||context.tenant_id!==d.tenant_id||!text(context.owner)||!uuid.test(context.requester_machine_id)||canonicalMachine(requesterMachineId)!==canonicalMachine(context.requester_machine_id)||!uuid.test(routeGeneration))fail('CALLBACK_AUTHORIZATION_LOST','current callback context changed');
|
|
36
|
+
return Object.freeze({owner:context.owner,tenant_id:context.tenant_id,requester_machine_id:context.requester_machine_id,route_generation:routeGeneration,origin:'requester_http',descriptor:d});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function verifyInstalledCallbackRuntime({descriptor,sshTarget,sshBin='ssh',run=execFile,env=process.env,releaseRoot=fileURLToPath(new URL('../',import.meta.url)),validateRelease=validateDaemonRelease,resolveRelease=verifiedCallbackRelease}){
|
|
40
|
+
const d=validatedCallbackDescriptor(descriptor),target=assertSshTarget(sshTarget);
|
|
41
|
+
let release;try{release=await resolveRelease({version:d.runtime_version,sourceCommit:d.runtime_git_sha,sha256:d.artifact_sha256.slice(7),releaseRoot,validateRelease});}catch{fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','verified callback release unavailable');}
|
|
42
|
+
if(release.releaseVersion!==d.runtime_version||release.sourceCommit!==d.runtime_git_sha||'sha256:'+release.sha256!==d.artifact_sha256)fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','callback runtime differs from verified release');
|
|
43
|
+
const command='sh -c '+quote(remoteVerifyInstalledCallbackScript)+' sh '+[d.binary_path,d.artifact_sha256.slice(7),d.service_uid].map(quote).join(' ');
|
|
44
|
+
let result;try{result=await run(sshBin,['-T','-o','BatchMode=yes','-o','ControlMaster=no','-o','ControlPath=none','--',target,command],{env,timeout:15_000,maxBuffer:4096});}catch{fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','installed callback executable verification failed');}
|
|
45
|
+
if(result.stdout.trim()!=='{"verified":true}')fail('HOST_CALLBACK_TRANSPORT_UNSUPPORTED','installed callback executable was not verified');
|
|
46
|
+
return d;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const privateIPv4=value=>netIsIP(value)===4&&(value.startsWith('10.')||value.startsWith('192.168.')||/^172\.(?:1[6-9]|2[0-9]|3[01])\./.test(value));
|
|
50
|
+
function callbackReadiness(value,callbackPort){
|
|
51
|
+
let url;try{url=new URL(value.callback_url)}catch{fail('CALLBACK_NOT_READY','native callback readiness refused');}
|
|
52
|
+
if(value.ready!==true||url.protocol!=='http:'||!privateIPv4(url.hostname)||Number(url.port)!==callbackPort||url.pathname!=='/'||url.search||url.hash||url.username||url.password||!privateIPv4(value.peer_ip)||!['container_id','network_id','endpoint_id'].every(key=>/^[0-9a-f]{64}$/.test(value[key]??'')))fail('CALLBACK_NOT_READY','native callback endpoint refused');
|
|
53
|
+
return url.origin;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Renewal cadence and acknowledgement budgets. The first renewal also runs the relay's
|
|
57
|
+
// consuming-namespace probe (validation + docker exec), so it gets the larger budget.
|
|
58
|
+
const GENERATION_MS=30_000,RENEW_INTERVAL_MS=10_000,FIRST_ACK_MS=15_000,RENEW_ACK_MS=6_000,STATUS_DEADLINE_MS=5_000,TRANSIENT_RETRY_MS=1_000;
|
|
59
|
+
// A transient status read is retried after TRANSIENT_RETRY_MS only while that retry (read +
|
|
60
|
+
// acknowledgement) can still land inside the current 30 s generation; otherwise the route closes.
|
|
61
|
+
const TRANSIENT_GRACE_MS=GENERATION_MS-TRANSIENT_RETRY_MS-STATUS_DEADLINE_MS-RENEW_ACK_MS;
|
|
62
|
+
export async function openNativeCallbackTransport({current,leaseId,hostKey,callbackPort,requesterMachineId,sshTarget,readCurrent,sshBin='ssh',env=process.env,run=execFile,spawnImpl=spawn,verifyRuntime=verifyInstalledCallbackRuntime,openGateway=openCallbackGateway,renewIntervalMs=RENEW_INTERVAL_MS,firstAckMs=FIRST_ACK_MS,renewAckMs=RENEW_ACK_MS,statusDeadlineMs=STATUS_DEADLINE_MS,transientRetryMs=TRANSIENT_RETRY_MS,transientGraceMs=TRANSIENT_GRACE_MS}){
|
|
63
|
+
if(typeof readCurrent!=='function')fail('CALLBACK_AUTHORIZATION_LOST','current callback status reader required');
|
|
64
|
+
const routeGeneration=randomUUID(),options={leaseId,hostKey,callbackPort,requesterMachineId,routeGeneration};
|
|
65
|
+
const binding=currentCallbackBinding(current,options),snapshot=JSON.stringify(binding),target=assertSshTarget(sshTarget),d=binding.descriptor;
|
|
66
|
+
await verifyRuntime({descriptor:d,sshTarget:target,sshBin,env,run});
|
|
67
|
+
const capability=randomBytes(32).toString('hex'),readinessNonce=randomBytes(16).toString('hex'),directory='/tmp/bb-callback-'+randomUUID(),socket=directory+'/callback.sock';
|
|
68
|
+
const sshArgs=['-T','-o','BatchMode=yes','-o','ControlMaster=no','-o','ControlPath=none','--',target];
|
|
69
|
+
let gateway,child,terminal,refreshTimer,statusAbort,pendingReady,pendingAck,closed=false,refreshing=false,cleanupPromise,exitResult;const exits=new Set();
|
|
70
|
+
const reportExit=exit=>{exitResult=exit;for(const listener of exits)listener(exit)};
|
|
71
|
+
const remote=async command=>{try{return await run(sshBin,[...sshArgs,command],{env,timeout:15_000,maxBuffer:4096})}catch{fail('CALLBACK_REMOTE_OPERATION_FAILED','owned callback operation failed');}};
|
|
72
|
+
const retire=async()=>{
|
|
73
|
+
// Every check tolerates the relay's own EXIT trap removing the socket/directory concurrently
|
|
74
|
+
// (a SIGKILLed local ssh): absence is success, anything present must be exactly ours.
|
|
75
|
+
const command=`set -eu; directory=${quote(directory)}; socket=${quote(socket)}; gone(){ [ ! -e "$directory" ] && [ ! -L "$directory" ]; }; if gone; then exit 0; fi; test ! -L "$directory"; if ! mode=$(stat -c '%u:%a' "$directory" 2>/dev/null); then gone; exit 0; fi; if [ ! -d "$directory" ]; then gone; exit 0; fi; test "$mode" = ${quote(d.service_uid+':700')}; if [ -e "$socket" ] || [ -L "$socket" ]; then test ! -L "$socket"; if smode=$(stat -c '%u:%a' "$socket" 2>/dev/null); then if [ -e "$socket" ]; then test -S "$socket"; fi; test "$smode" = ${quote(d.service_uid+':600')}; fi; rm -f -- "$socket"; fi; rmdir -- "$directory" 2>/dev/null || gone`;
|
|
76
|
+
await remote(command);
|
|
77
|
+
};
|
|
78
|
+
const close=()=>cleanupPromise??=(async()=>{
|
|
79
|
+
closed=true;clearInterval(refreshTimer);statusAbort?.abort();pendingReady?.reject(Error('callback closed'));pendingAck?.reject(Error('callback closed'));await gateway?.close();
|
|
80
|
+
if(child){child.stdin.end();const term=setTimeout(()=>child.kill('SIGTERM'),3000);const kill=setTimeout(()=>child.kill('SIGKILL'),6000);term.unref();kill.unref();try{await terminal}finally{clearTimeout(term);clearTimeout(kill)}}
|
|
81
|
+
try{await retire()}catch{fail('CALLBACK_CLEANUP_UNCERTAIN','owned callback cleanup could not be verified');}
|
|
82
|
+
})();
|
|
83
|
+
try{
|
|
84
|
+
await remote(`set -eu; test "$(id -u)" = ${quote(d.service_uid)}; umask 077; mkdir -- ${quote(directory)}; chmod 0700 -- ${quote(directory)}`);
|
|
85
|
+
gateway=await openGateway({callbackPort,binding:Object.freeze({snapshot}),capability,activate:false,readinessNonce});
|
|
86
|
+
const command=`set -eu; relayPid=; cleanup(){ trap - EXIT HUP INT TERM; if [ -n "$relayPid" ]; then kill -TERM "$relayPid" 2>/dev/null || :; wait "$relayPid" 2>/dev/null || :; fi; rm -f -- ${quote(socket)}; rmdir -- ${quote(directory)}; }; trap cleanup EXIT HUP INT TERM; ${quote(d.binary_path)} --callback-relay --config ${quote(d.config_path)} <&0 & relayPid=$!; status=0; wait "$relayPid" || status=$?; relayPid=; exit "$status"`;
|
|
87
|
+
child=spawnImpl(sshBin,[...SSH_FORWARD_OPTIONS.filter(arg=>arg!=='-N'),'-o','StreamLocalBindMask=0177','-R',socket+':127.0.0.1:'+gateway.port,'--',target,command],{env,stdio:['pipe','pipe','pipe']});
|
|
88
|
+
child.stdin.on('error',()=>{});child.stderr.on('data',()=>{});
|
|
89
|
+
let spawnError=false;child.once('error',()=>{spawnError=true});terminal=new Promise(resolve=>child.once('close',(code,signal)=>resolve({code,signal,reason:spawnError?'callback_process_error':'callback_process_exit'})));
|
|
90
|
+
terminal.then(exit=>{if(!closed){reportExit(exit);void close().catch(()=>reportExit({code:null,reason:'callback_cleanup_uncertain'}))}});
|
|
91
|
+
const lines=createInterface({input:child.stdout});
|
|
92
|
+
lines.on('line',line=>{try{const event=JSON.parse(line);if(event.refused===true&&event.code==='CALLBACK_CONSUMING_NAMESPACE_UNREACHABLE'&&pendingAck){const waiter=pendingAck;pendingAck=undefined;const refusal=Error('consuming namespace callback probe failed');refusal.code='CALLBACK_CONSUMING_NAMESPACE_UNREACHABLE';waiter.reject(refusal);}else if(pendingReady){const waiter=pendingReady;pendingReady=undefined;waiter.resolve(event);}else if(pendingAck&&event.renewed===true&&event.route_generation===routeGeneration&&Number.isFinite(Date.parse(event.expires_at))){const waiter=pendingAck;pendingAck=undefined;waiter.resolve(event);}else throw Error('unexpected callback frame');}catch{reportExit({code:null,reason:'callback_protocol_error'});void close().catch(()=>{})}});
|
|
93
|
+
terminal.then(()=>{lines.close();pendingReady?.reject(Error('callback process ended'));pendingAck?.reject(Error('callback process ended'));});
|
|
94
|
+
const ready=await new Promise((resolve,reject)=>{
|
|
95
|
+
const timeout=setTimeout(()=>reject(Object.assign(Error('callback readiness deadline'),{code:'CALLBACK_NOT_READY'})),20_000);timeout.unref();
|
|
96
|
+
pendingReady={resolve:value=>{clearTimeout(timeout);resolve(value)},reject:error=>{clearTimeout(timeout);reject(error)}};
|
|
97
|
+
child.stdin.write(JSON.stringify({binding,capability,socket_path:socket,readiness_nonce:readinessNonce})+'\n');
|
|
98
|
+
});
|
|
99
|
+
const callbackUrl=callbackReadiness(ready,callbackPort);
|
|
100
|
+
gateway.onClose(()=>{if(!closed){reportExit({code:null,reason:'callback_gateway_closed'});void close().catch(()=>{})}});
|
|
101
|
+
let renewedAt=null;
|
|
102
|
+
const refresh=async(first=false)=>{
|
|
103
|
+
if(closed||refreshing)return;refreshing=true;
|
|
104
|
+
try{
|
|
105
|
+
statusAbort=new AbortController();let statusTimeout,status;
|
|
106
|
+
try{status=await new Promise((resolve,reject)=>{statusTimeout=setTimeout(()=>{statusAbort.abort();reject(Object.assign(Error('callback status deadline'),{transient:true}));},statusDeadlineMs);statusTimeout.unref();Promise.resolve().then(()=>readCurrent(statusAbort.signal)).then(resolve,reject).finally(()=>clearTimeout(statusTimeout));});}
|
|
107
|
+
catch(error){
|
|
108
|
+
// A failed status READ is not an authorization answer: keep the live generation (relay and
|
|
109
|
+
// gateway still expire it at 30 s) and retry on the next tick while inside the grace window.
|
|
110
|
+
if(!first&&error?.transient===true&&renewedAt!==null&&performance.now()-renewedAt<transientGraceMs){const retry=setTimeout(()=>{void refresh()},transientRetryMs);retry.unref();return;}
|
|
111
|
+
throw error;
|
|
112
|
+
}
|
|
113
|
+
if(closed)return;const renewed=currentCallbackBinding(status,options);if(JSON.stringify(renewed)!==snapshot)fail('CALLBACK_AUTHORIZATION_LOST','callback context changed');
|
|
114
|
+
const sentAt=performance.now();
|
|
115
|
+
await new Promise((resolve,reject)=>{const timeout=setTimeout(()=>reject(Error('callback renewal acknowledgement deadline')),first?firstAckMs:renewAckMs);timeout.unref();pendingAck={resolve:value=>{clearTimeout(timeout);resolve(value)},reject:error=>{clearTimeout(timeout);reject(error)}};child.stdin.write(JSON.stringify({binding:renewed,renew:true})+'\n');});
|
|
116
|
+
if(closed)return;renewedAt=sentAt;gateway.refresh(Object.freeze({snapshot}),30_000-(performance.now()-sentAt));
|
|
117
|
+
}
|
|
118
|
+
// The first renewal carries the consuming-namespace probe: its uncoded failure is readiness.
|
|
119
|
+
catch(error){if(!closed){reportExit({code:null,reason:'callback_authorization_lost',...(error.code?{code:error.code}:first?{code:'CALLBACK_NOT_READY'}:{})});void close().catch(()=>{});}}
|
|
120
|
+
finally{refreshing=false}
|
|
121
|
+
};
|
|
122
|
+
await refresh(true);if(closed||!gateway.isLive())fail(exitResult?.code??'CALLBACK_NOT_READY','callback route not ready after readiness');
|
|
123
|
+
refreshTimer=setInterval(refresh,renewIntervalMs);refreshTimer.unref();
|
|
124
|
+
return{callbackUrl,close,isLive:()=>!closed&&gateway.isLive(),onExit(listener){exits.add(listener);if(exitResult)listener(exitResult)}};
|
|
125
|
+
}catch(error){try{await close()}catch{fail('CALLBACK_CLEANUP_UNCERTAIN','owned callback cleanup could not be verified');}throw error;}
|
|
126
|
+
}
|
package/src/stack-tunnel.mjs
CHANGED
|
@@ -469,7 +469,7 @@ export async function probeGatewayPorts({
|
|
|
469
469
|
* @returns {{ forwards, connection, argv, attempts, onExit, close, pid }}
|
|
470
470
|
*/
|
|
471
471
|
export async function establishSshTunnel({
|
|
472
|
-
connection, sshTarget, hostKey = null, callbackPort = null, sshBin = null, env = process.env,
|
|
472
|
+
connection, sshTarget, hostKey = null, callbackPort = null, reserveLocalPorts = [], sshBin = null, env = process.env,
|
|
473
473
|
attempts = 3, readyTimeoutMs = DEFAULT_TUNNEL_READY_MS, isFree = isLocalPortFree, pickFree = pickFreeLocalPort, spawnImpl = spawn,
|
|
474
474
|
runRemote = spawnSync, probe = probeGatewayPorts, probeReadyTimeoutMs = DEFAULT_PROBE_READY_MS,
|
|
475
475
|
}) {
|
|
@@ -500,7 +500,9 @@ export async function establishSshTunnel({
|
|
|
500
500
|
}
|
|
501
501
|
// Codex #1134 P2: a remapped stack forward must never take the callback port locally; ssh
|
|
502
502
|
// would bind it before the caller's callback service and the -R traffic would loop into it.
|
|
503
|
-
|
|
503
|
+
// BOT-2775 (Codex #1402 P2): a native-callback run opens no -R, but its child still serves the
|
|
504
|
+
// callback on that requester port, so the planner keeps it reserved via reserveLocalPorts.
|
|
505
|
+
let avoid = new Set([...(callbackPort != null ? [Number(callbackPort)] : []), ...reserveLocalPorts.map(Number)]);
|
|
504
506
|
let lastError = null;
|
|
505
507
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
506
508
|
const forwards = await planForwards(remotePorts, { isFree, pickFree, avoid });
|
package/src/stack.mjs
CHANGED
|
@@ -52,6 +52,7 @@ import { checkReadoptedStackMounts } from "./stack-mount-health.mjs";
|
|
|
52
52
|
import { machineUuid } from "./machine-id.mjs";
|
|
53
53
|
import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
|
|
54
54
|
import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
|
|
55
|
+
import { NATIVE_CALLBACK_TRANSPORT, openNativeCallbackTransport } from "./stack-native-callback.mjs";
|
|
55
56
|
import { FORWARD_ATTESTATION_ENV, startForwardAttestation, readForwardAttestation } from "./stack-forward-attestation.mjs";
|
|
56
57
|
import { stackServicesForForwards } from "./stack-service-discovery.mjs";
|
|
57
58
|
import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
|
|
@@ -80,6 +81,8 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
|
|
|
80
81
|
* probe (`probeGatewayPorts` in stack-tunnel.mjs) before the real `-R` exists, and refuses
|
|
81
82
|
* with exit 5 CALLBACK_NOT_LOOPBACK when the host would expose it. The code below is kept
|
|
82
83
|
* only so older automation that matched on it still resolves; no path emits it any more.
|
|
84
|
+
* BOT-2775: an explicit beta remote callback skips the -R path and uses the verified native
|
|
85
|
+
* private route (stack-native-callback.mjs); stable/default remote callbacks keep the probe.
|
|
83
86
|
*/
|
|
84
87
|
export const CALLBACK_PORT_UNSUPPORTED_REMOTE = "CALLBACK_PORT_UNSUPPORTED_REMOTE";
|
|
85
88
|
export const DEFAULT_RECEIPT_MAX_BYTES = 10240;
|
|
@@ -259,11 +262,16 @@ ${bold("run OPTIONS")}
|
|
|
259
262
|
it as connection.requester_origin. The lease is requested through the
|
|
260
263
|
identified admission (exact replay after a lost response, no fallback).
|
|
261
264
|
--callback-port N The caller runs a listener the stack calls back through
|
|
262
|
-
host.docker.internal:N (e.g. 4600): sent as callback_port. On a
|
|
263
|
-
lease run first probes the host with a throwaway reverse forward
|
|
264
|
-
-R 127.0.0.1:N:127.0.0.1:N only when the host keeps it on loopback
|
|
265
|
-
GatewayPorts no/clientspecified); otherwise exit 5
|
|
266
|
-
(BOT-2184).
|
|
265
|
+
host.docker.internal:N (e.g. 4600): sent as callback_port. On a stable or
|
|
266
|
+
auto REMOTE lease run first probes the host with a throwaway reverse forward
|
|
267
|
+
and opens -R 127.0.0.1:N:127.0.0.1:N only when the host keeps it on loopback
|
|
268
|
+
(sshd GatewayPorts no/clientspecified); otherwise exit 5
|
|
269
|
+
CALLBACK_NOT_LOOPBACK (BOT-2184). With --remote --execution-ring beta it
|
|
270
|
+
instead requires a Linux Go runtime with native callbacks (protocol 5): a
|
|
271
|
+
private SSH Unix-socket route to a fixed requester gateway, renewed against
|
|
272
|
+
current owned authorization (BOT-2775). Unknown/older hosts or a backend
|
|
273
|
+
that does not acknowledge it fail HOST_CALLBACK_TRANSPORT_UNSUPPORTED; the
|
|
274
|
+
child never receives the route capability.
|
|
267
275
|
--bundle-max-mb N Remote-lease bundle size cap in MB (default 50, compressed). The bundle
|
|
268
276
|
excludes .bb-stack.json, node_modules, supabase/.temp|.branches and
|
|
269
277
|
untracked-secret dotenv files (.env, .env.local, .env.*.local).
|
|
@@ -324,9 +332,15 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
|
|
|
324
332
|
4 STACK_EXECUTION_MODE_MISMATCH the slot already holds a lease in another mode/host — stack done it first
|
|
325
333
|
(the four codes above create NO lease)
|
|
326
334
|
5 BUNDLE_UPLOAD_FAILED the signed bundle upload failed (the lease is released first)
|
|
327
|
-
5 CALLBACK_NOT_LOOPBACK --callback-port: the GatewayPorts probe (before any real -R)
|
|
328
|
-
post-forward check found the host's -R listener not loopback-only
|
|
335
|
+
5 CALLBACK_NOT_LOOPBACK --callback-port (stable/auto): the GatewayPorts probe (before any real -R)
|
|
336
|
+
or the post-forward check found the host's -R listener not loopback-only
|
|
329
337
|
(sshd GatewayPorts yes), or could not check it (no lsof, no listener)
|
|
338
|
+
5 HOST_CALLBACK_TRANSPORT_UNSUPPORTED --callback-port (beta remote): native callback transport is
|
|
339
|
+
unavailable; needs the installed Linux Go runtime (protocol 5) and backend
|
|
340
|
+
5 CALLBACK_NOT_READY / CALLBACK_CONSUMING_NAMESPACE_UNREACHABLE the native route did not
|
|
341
|
+
answer from the stack's own network before the child (lease reaped)
|
|
342
|
+
6 CALLBACK_TRANSPORT_LOST / CALLBACK_AUTHORIZATION_LOST the native route or its current
|
|
343
|
+
authorization closed; the child is stopped and the lease reaped
|
|
330
344
|
5 CALLBACK_PORT_BUSY --callback-port is already listening on the serving host (another run or
|
|
331
345
|
service), the host refused this run's -R because another run took the
|
|
332
346
|
port first, or another listener shares the port; this run holds no -R —
|
|
@@ -3668,7 +3682,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3668
3682
|
// BOT-2780 (beta requester origin): BOT-2499's identified admission, never a fallback.
|
|
3669
3683
|
requestIdentified: (args) => rpc.call("request_stack_lease_identified", args),
|
|
3670
3684
|
getAdmission: (args) => rpc.call("get_stack_lease_admission", args),
|
|
3671
|
-
get: (leaseId) => rpc.call("get_stack_lease", { lease_id: leaseId }),
|
|
3685
|
+
get: (leaseId, { signal } = {}) => rpc.call("get_stack_lease", { lease_id: leaseId }, { signal }),
|
|
3672
3686
|
// BOT-1803: a heartbeat carries the measured client use when there is one.
|
|
3673
3687
|
touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
|
|
3674
3688
|
// BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
|
|
@@ -3736,6 +3750,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3736
3750
|
const buildBundle = adapters.buildBundle ?? buildStackBundle;
|
|
3737
3751
|
const uploadBundle = adapters.uploadBundle ?? uploadStackBundle;
|
|
3738
3752
|
const openTunnel = adapters.openTunnel ?? ((args) => establishSshTunnel(args));
|
|
3753
|
+
const openCallback = adapters.openNativeCallbackTransport ?? openNativeCallbackTransport;
|
|
3739
3754
|
const clock = adapters.clock ?? { setInterval, clearInterval, setTimeout, clearTimeout };
|
|
3740
3755
|
const signals = adapters.signals ?? process;
|
|
3741
3756
|
const provisionTimeout = opts.provisionTimeout ?? opts.timeout;
|
|
@@ -3758,6 +3773,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3758
3773
|
let activeWaitAbort = null;
|
|
3759
3774
|
// BOT-2142: the remote lease's forwards, and whether they died under a running child.
|
|
3760
3775
|
let tunnel = null;
|
|
3776
|
+
let nativeCallback = null;
|
|
3777
|
+
let callbackLost = null;
|
|
3778
|
+
let callbackCleanupError = null;
|
|
3761
3779
|
let forwardAttestation = null;
|
|
3762
3780
|
let tunnelLost = null;
|
|
3763
3781
|
// BOT-2780: the requester origin closed under a live run (SSH loss or forward failure).
|
|
@@ -3777,6 +3795,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3777
3795
|
const closeTunnel = async () => {
|
|
3778
3796
|
await closeOrigin();
|
|
3779
3797
|
try {
|
|
3798
|
+
if (nativeCallback) { const route = nativeCallback; nativeCallback = null; try { await route.close(); } catch { callbackCleanupError = "CALLBACK_CLEANUP_UNCERTAIN"; } }
|
|
3780
3799
|
if (forwardAttestation) {
|
|
3781
3800
|
const owner = forwardAttestation;
|
|
3782
3801
|
forwardAttestation = null;
|
|
@@ -4019,6 +4038,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4019
4038
|
"an idle stack on this slot will be re-provisioned rather than reused.");
|
|
4020
4039
|
}
|
|
4021
4040
|
idleLease.stackDigest = stackDigest;
|
|
4041
|
+
// Runtime-first rollout: only explicit beta callers activate the new transport.
|
|
4042
|
+
const requireNativeCallback = opts.callbackPort != null && opts.executionRing === "beta" && executionRequest !== "local";
|
|
4022
4043
|
const leaseArgs = {
|
|
4023
4044
|
slot, host_key: opts.host || undefined, repo: opts.repo, ticket_id: opts.ticket,
|
|
4024
4045
|
ticket_url: opts.ticketUrl || undefined, pr_id: opts.prId || undefined, pr_url: opts.prUrl || undefined,
|
|
@@ -4033,7 +4054,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4033
4054
|
require_stack_digest: true,
|
|
4034
4055
|
// BOT-2269: what the bundle carries beyond the defaults (paths, sha256, key NAMES).
|
|
4035
4056
|
...(bundle?.included?.length ? { bundle_included: bundle.included } : {}),
|
|
4036
|
-
...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
|
|
4057
|
+
...(opts.callbackPort != null ? { callback_port: opts.callbackPort, ...(requireNativeCallback ? { callback_transport: NATIVE_CALLBACK_TRANSPORT } : {}) } : {}),
|
|
4037
4058
|
};
|
|
4038
4059
|
// BOT-2780: the identified admission key and a digest of the immutable origin context.
|
|
4039
4060
|
// The intent (including the bundle digest that names the reserved origin) is fingerprinted
|
|
@@ -4355,6 +4376,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4355
4376
|
placement.mode = request.data.execution_mode ?? null;
|
|
4356
4377
|
placement.hostKey = request.data.host_key ?? null;
|
|
4357
4378
|
placement.requesterHost = request.data.requester_host ?? null;
|
|
4379
|
+
if (remoteLease && requireNativeCallback && request.data.callback_transport !== NATIVE_CALLBACK_TRANSPORT) {
|
|
4380
|
+
const cleanup = await requestCleanup();
|
|
4381
|
+
return { exitCode: cleanup.ok ? EXIT.BACKEND : EXIT.CLEANUP_FAILED, outcome: "error", code: "HOST_CALLBACK_TRANSPORT_UNSUPPORTED", leaseId, error: "backend did not acknowledge required remote callback transport", cleanup };
|
|
4382
|
+
}
|
|
4358
4383
|
// BOT-2207: why the server placed (or queued) it there, and every host it weighed.
|
|
4359
4384
|
placement.placementReason = request.data.placement_reason ?? null;
|
|
4360
4385
|
placement.considered = Array.isArray(request.data.considered) ? request.data.considered : null;
|
|
@@ -4624,11 +4649,30 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4624
4649
|
// BOT-2142: forward every stack port over ONE ssh and hand the child the local ports.
|
|
4625
4650
|
sshTarget = current.data.connection?.ssh_target ?? current.data.ssh_target ?? sshTarget;
|
|
4626
4651
|
try {
|
|
4627
|
-
tunnel = await openTunnel({ connection: current.data.connection, sshTarget, hostKey: placement.hostKey, callbackPort:
|
|
4652
|
+
tunnel = await openTunnel({ connection: current.data.connection, sshTarget, hostKey: placement.hostKey, callbackPort: requireNativeCallback ? null : opts.callbackPort,
|
|
4653
|
+
...(requireNativeCallback ? { reserveLocalPorts: [opts.callbackPort] } : {}) });
|
|
4628
4654
|
} catch (error) {
|
|
4629
4655
|
const cleanup = await requestCleanup();
|
|
4630
4656
|
return { exitCode: EXIT.BACKEND, outcome: "error", code: error?.code ?? "REMOTE_UNREACHABLE", leaseId, error: String(error?.message ?? error), cleanup };
|
|
4631
4657
|
}
|
|
4658
|
+
if (requireNativeCallback) {
|
|
4659
|
+
try {
|
|
4660
|
+
nativeCallback = await openCallback({ current: current.data, leaseId, hostKey: placement.hostKey, callbackPort: opts.callbackPort, requesterMachineId: machineUuidFn(), sshTarget,
|
|
4661
|
+
readCurrent: async (signal) => { let observed; try { observed = await api.get(leaseId, { signal }); } catch (error) { throw Object.assign(new Error(`current callback status unreadable (${error?.message ?? error})`), { transient: true }); } if (!observed?.ok) {
|
|
4662
|
+
// Only an unanswered read is transient (transport, 5xx, 408/429, unreadable body). A 401/403
|
|
4663
|
+
// that survived the auth refresh or a typed JSON-RPC refusal is a definitive answer.
|
|
4664
|
+
const transient = !observed?.auth && (observed?.status == null || observed.status >= 500 || observed.status === 408 || observed.status === 429 || observed.error === "invalid_json");
|
|
4665
|
+
throw Object.assign(new Error(transient ? "current callback status unreadable" : "current callback authorization refused"), transient ? { transient: true } : { code: "CALLBACK_AUTHORIZATION_LOST" });
|
|
4666
|
+
}
|
|
4667
|
+
if (observed.data?.success !== true) throw Object.assign(new Error("current callback authorization unavailable"), { code: "CALLBACK_AUTHORIZATION_LOST" });
|
|
4668
|
+
return observed.data; },
|
|
4669
|
+
});
|
|
4670
|
+
nativeCallback.onExit((exit) => { if (childResult) return; callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
|
|
4671
|
+
} catch (error) {
|
|
4672
|
+
await closeTunnel(); const cleanup = await requestCleanup();
|
|
4673
|
+
return { exitCode: cleanup.ok && !callbackCleanupError && error?.code !== "CALLBACK_CLEANUP_UNCERTAIN" ? EXIT.BACKEND : EXIT.CLEANUP_FAILED, outcome: "error", code: error?.code ?? "CALLBACK_NOT_READY", leaseId, error: String(error?.message ?? "callback transport unavailable"), cleanup };
|
|
4674
|
+
}
|
|
4675
|
+
}
|
|
4632
4676
|
placement.transport = "ssh-forward";
|
|
4633
4677
|
placement.forwards = tunnel.forwards.map(({ name, remote, local }) => ({ name, remote, local }));
|
|
4634
4678
|
for (const f of tunnel.forwards.filter((x) => x.remapped)) stderrLine(`stack: forward ${f.name} remapped: local ${f.local} → ${placement.hostKey ?? "remote"} ${f.remote}`);
|
|
@@ -4707,6 +4751,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4707
4751
|
return { exitCode: cleanup.ok ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "error", code: originLost?.code ?? "REQUESTER_ORIGIN_TUNNEL_LOST", leaseId, cleanup,
|
|
4708
4752
|
error: "the requester origin closed before the child started; the lease was released" };
|
|
4709
4753
|
}
|
|
4754
|
+
if (callbackLost || nativeCallback && !nativeCallback.isLive()) {
|
|
4755
|
+
await closeTunnel(); const cleanup = await requestCleanup();
|
|
4756
|
+
return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "error", code: "CALLBACK_TRANSPORT_LOST", leaseId, error: "callback transport closed before child start", cleanup };
|
|
4757
|
+
}
|
|
4710
4758
|
if (tunnelLost) {
|
|
4711
4759
|
// The forwards died between being verified and the child starting.
|
|
4712
4760
|
await closeTunnel();
|
|
@@ -4758,7 +4806,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4758
4806
|
// BOT-1803: a run that ended on its own (the child exited, success or not) leaves its
|
|
4759
4807
|
// lease IDLE for the next run on this slot — no reap wait. `--reap`, a signal, the
|
|
4760
4808
|
// hard TTL, a fenced/preempted heartbeat and a lost tunnel keep the eager signed reap.
|
|
4761
|
-
const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && forcedStopCode == null;
|
|
4809
|
+
const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && !callbackCleanupError && forcedStopCode == null;
|
|
4762
4810
|
const cleanup = await requestCleanup({ idle: idleOnEnd });
|
|
4763
4811
|
if (cleanup?.idle) {
|
|
4764
4812
|
// The stack is still running: keep its directory and host port block reserved well
|
|
@@ -4771,6 +4819,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4771
4819
|
return { exitCode: cleanup.ok ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: originLost.code, leaseId, childExitCode: childResult.code, cleanup, fenced: true,
|
|
4772
4820
|
error: "the requester origin closed while the child was running, so the child was stopped and the lease released" };
|
|
4773
4821
|
}
|
|
4822
|
+
if (callbackLost && !receivedSignal) {
|
|
4823
|
+
return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: callbackLost.reason === "callback_authorization_lost" ? "CALLBACK_AUTHORIZATION_LOST" : "CALLBACK_TRANSPORT_LOST", leaseId, childExitCode: childResult.code, cleanup, fenced: true, error: "callback transport authorization or runtime context was lost; child stopped" };
|
|
4824
|
+
}
|
|
4774
4825
|
if (tunnelLost && !receivedSignal) {
|
|
4775
4826
|
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
|
|
4776
4827
|
}
|
|
@@ -4782,8 +4833,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4782
4833
|
...(parkIncident ? { incident: parkIncident } : {}),
|
|
4783
4834
|
};
|
|
4784
4835
|
}
|
|
4785
|
-
const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
|
|
4786
|
-
return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
4836
|
+
const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : callbackCleanupError ? EXIT.CLEANUP_FAILED : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : callbackCleanupError || !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
|
|
4837
|
+
return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, ...(callbackCleanupError ? { code: callbackCleanupError } : {}), fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
4787
4838
|
} catch (error) {
|
|
4788
4839
|
const cleanup = await requestCleanup();
|
|
4789
4840
|
// BOT-1750 (Codex R10 P2): a thrown auth/self-heal failure (e.g. a refresh that
|