@panorama-ai/gateway 2.28.50 → 2.29.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/dist/child-process-startup.d.ts +4 -0
- package/dist/child-process-startup.d.ts.map +1 -0
- package/dist/child-process-startup.js +33 -0
- package/dist/child-process-startup.js.map +1 -0
- package/dist/cli-providers/codex.d.ts.map +1 -1
- package/dist/cli-providers/codex.js +1 -0
- package/dist/cli-providers/codex.js.map +1 -1
- package/dist/cli-providers/gemini.d.ts.map +1 -1
- package/dist/cli-providers/gemini.js +12 -0
- package/dist/cli-providers/gemini.js.map +1 -1
- package/dist/database.types.d.ts +628 -0
- package/dist/database.types.d.ts.map +1 -1
- package/dist/database.types.js.map +1 -1
- package/dist/gateway-operational-status.d.ts +15 -0
- package/dist/gateway-operational-status.d.ts.map +1 -0
- package/dist/gateway-operational-status.js +31 -0
- package/dist/gateway-operational-status.js.map +1 -0
- package/dist/index.js +1897 -310
- package/dist/index.js.map +1 -1
- package/dist/job-queue.d.ts +23 -0
- package/dist/job-queue.d.ts.map +1 -0
- package/dist/job-queue.js +70 -0
- package/dist/job-queue.js.map +1 -0
- package/dist/remote-shell-resource.d.ts +19 -0
- package/dist/remote-shell-resource.d.ts.map +1 -0
- package/dist/remote-shell-resource.js +51 -0
- package/dist/remote-shell-resource.js.map +1 -0
- package/dist/stream-json-collector.d.ts +29 -0
- package/dist/stream-json-collector.d.ts.map +1 -0
- package/dist/stream-json-collector.js +79 -0
- package/dist/stream-json-collector.js.map +1 -0
- package/dist/subagent-adapters/claude-code.d.ts.map +1 -1
- package/dist/subagent-adapters/claude-code.js +54 -12
- package/dist/subagent-adapters/claude-code.js.map +1 -1
- package/dist/subagent-adapters/claude-support.d.ts +2 -0
- package/dist/subagent-adapters/claude-support.d.ts.map +1 -1
- package/dist/subagent-adapters/claude-support.js +4 -0
- package/dist/subagent-adapters/claude-support.js.map +1 -1
- package/dist/subagent-adapters/codex.d.ts.map +1 -1
- package/dist/subagent-adapters/codex.js +44 -14
- package/dist/subagent-adapters/codex.js.map +1 -1
- package/dist/subagent-adapters/gemini.d.ts.map +1 -1
- package/dist/subagent-adapters/gemini.js +31 -3
- package/dist/subagent-adapters/gemini.js.map +1 -1
- package/dist/subagent-adapters/types.d.ts +1 -0
- package/dist/subagent-adapters/types.d.ts.map +1 -1
- package/dist/subagent-cancel-target.d.ts +15 -0
- package/dist/subagent-cancel-target.d.ts.map +1 -0
- package/dist/subagent-cancel-target.js +22 -0
- package/dist/subagent-cancel-target.js.map +1 -0
- package/dist/subagent-full-control.d.ts +7 -0
- package/dist/subagent-full-control.d.ts.map +1 -0
- package/dist/subagent-full-control.js +47 -0
- package/dist/subagent-full-control.js.map +1 -0
- package/dist/subagent-process-control.d.ts +9 -0
- package/dist/subagent-process-control.d.ts.map +1 -0
- package/dist/subagent-process-control.js +32 -0
- package/dist/subagent-process-control.js.map +1 -0
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import path from 'node:path';
|
|
|
10
10
|
import process from 'node:process';
|
|
11
11
|
import { promisify } from 'node:util';
|
|
12
12
|
import { fileURLToPath } from 'node:url';
|
|
13
|
+
import { buildRequestedCancelMetadata, getSubagentCancelAdapter, resolveCancelRequest, GATEWAY_CAPABILITIES_SCHEMA_VERSION, } from '@panorama/shared';
|
|
13
14
|
import { buildMissingSupabaseConfigError, resolveSupabaseConfig, } from './supabase-config.js';
|
|
14
15
|
import { DEFAULT_CLAUDE_SUPPORT, coerceClaudeSupport, parseClaudeSupport, toFlagRecord, } from './subagent-adapters/claude-support.js';
|
|
15
16
|
import { getSubagentAdapter } from './subagent-adapters/registry.js';
|
|
@@ -19,24 +20,41 @@ import { DEFAULT_MODEL_SENTINEL } from './cli-providers/types.js';
|
|
|
19
20
|
import { resolveGatewayConfigDir, resolveGatewayConfigPath, resolveGatewayLogPath, resolveGatewayPidPath, resolveGatewayTmpDir, resolveGatewayWorkdirRoot, resolveModelWorkdirRoot, resolveSubagentWorkdirRoot as resolveSubagentWorkdirRootBase, resolveValidationWorkdirRoot, } from './runtime-paths.js';
|
|
20
21
|
import { sanitizeGatewayDebugPayload } from './debug-redaction.js';
|
|
21
22
|
import { buildGatewayChildProcessEnv } from './child-process-env.js';
|
|
23
|
+
import { evaluateGatewayProviderFullControlSupport } from './subagent-full-control.js';
|
|
24
|
+
import { GatewayJobQueue } from './job-queue.js';
|
|
25
|
+
import { computeGatewayOperationalStatus, } from './gateway-operational-status.js';
|
|
22
26
|
import { ensureOwnerOnlyDirectory, ensureOwnerOnlyFile, shouldEnforceOwnerOnlyPermissions, writeOwnerOnlyFile, } from './local-security.js';
|
|
27
|
+
import { requestChildTermination as terminateChildProcess } from './subagent-process-control.js';
|
|
28
|
+
import { validateRemoteShellResourceRow } from './remote-shell-resource.js';
|
|
29
|
+
import { StreamJsonCollector } from './stream-json-collector.js';
|
|
30
|
+
import { waitForChildStartup } from './child-process-startup.js';
|
|
23
31
|
const execFileAsync = promisify(execFile);
|
|
24
32
|
const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
25
33
|
const DEFAULT_CLAUDE_TIMEOUT_MS = 10_000;
|
|
26
34
|
const DEFAULT_SUBAGENT_TIMEOUT_MS = 30 * 60_000;
|
|
27
35
|
const DEFAULT_MODEL_RUN_TIMEOUT_MS = 110_000;
|
|
36
|
+
const MAX_ACTIVE_SHELL_SESSIONS = 64;
|
|
37
|
+
const MAX_SHELL_SESSION_EVENTS = 800;
|
|
38
|
+
const MAX_SHELL_SESSION_BUFFER_BYTES = 1_000_000;
|
|
39
|
+
const SHELL_SESSION_RETENTION_MS = 15 * 60_000;
|
|
40
|
+
const DEFAULT_SHELL_COLUMNS = 120;
|
|
41
|
+
const DEFAULT_SHELL_ROWS = 40;
|
|
28
42
|
const DEFAULT_GATEWAY_CONCURRENCY = 10;
|
|
29
43
|
const MAX_SUBAGENT_OUTPUT_BYTES = 5_000_000;
|
|
44
|
+
const MAX_SUBAGENT_STREAM_EVENTS = 2000;
|
|
45
|
+
const MAX_SUBAGENT_STREAM_BUFFER_CHARS = 200_000;
|
|
30
46
|
const MAX_SUBAGENT_EVENT_PAYLOAD_BYTES = 200_000;
|
|
31
47
|
const MAX_GATEWAY_LOG_BYTES = 200_000;
|
|
32
48
|
const MAX_GATEWAY_EVENT_MESSAGE_CHARS = 2000;
|
|
33
|
-
const SUBAGENT_EVENT_BATCH_SIZE = 200;
|
|
34
49
|
const SUBAGENT_CANCEL_KILL_TIMEOUT_MS = 5_000;
|
|
35
50
|
const PROCESS_KILL_GRACE_MS = 2_000;
|
|
51
|
+
const DEFAULT_GATEWAY_SHUTDOWN_GRACE_MS = 15_000;
|
|
52
|
+
const DEFAULT_JOB_RECONCILE_INTERVAL_MS = 5_000;
|
|
53
|
+
const DEFAULT_JOB_CHANNEL_RECONNECT_MS = 5_000;
|
|
36
54
|
const PROVIDER_VALIDATION_TIMEOUT_MS = 30_000;
|
|
37
|
-
const GATEWAY_CAPABILITIES_SCHEMA_VERSION = 'v1';
|
|
38
55
|
const DEFAULT_RESTART_CHECK_INTERVAL_MS = 30_000;
|
|
39
56
|
const MAX_CLAUDE_WRAPPER_BYTES = 4096;
|
|
57
|
+
const GATEWAY_HEARTBEAT_STALE_MS = 90_000;
|
|
40
58
|
const PROVIDER_VALIDATION_PROMPT = 'Say hi.';
|
|
41
59
|
const PROVIDER_VALIDATION_SCHEMA = {
|
|
42
60
|
type: 'object',
|
|
@@ -61,7 +79,23 @@ const GATEWAY_CONCURRENCY = (() => {
|
|
|
61
79
|
return DEFAULT_GATEWAY_CONCURRENCY;
|
|
62
80
|
return parsed;
|
|
63
81
|
})();
|
|
82
|
+
const SUBAGENT_RUN_EVENT_TYPES = [
|
|
83
|
+
'message',
|
|
84
|
+
'system',
|
|
85
|
+
'tool_call',
|
|
86
|
+
'tool_result',
|
|
87
|
+
'run_started',
|
|
88
|
+
'run_progress',
|
|
89
|
+
'cancellation_requested',
|
|
90
|
+
'cancellation_acknowledged',
|
|
91
|
+
'run_completed',
|
|
92
|
+
'run_failed',
|
|
93
|
+
'run_cancelled',
|
|
94
|
+
];
|
|
95
|
+
const SUBAGENT_RUN_EVENT_TYPE_SET = new Set(SUBAGENT_RUN_EVENT_TYPES);
|
|
64
96
|
const activeSubagentRuns = new Map();
|
|
97
|
+
const activeModelRuns = new Map();
|
|
98
|
+
const activeShellSessions = new Map();
|
|
65
99
|
const pendingCancelByRunId = new Map();
|
|
66
100
|
const pendingCancelBySubagentId = new Map();
|
|
67
101
|
const pendingGatewayEvents = [];
|
|
@@ -69,6 +103,9 @@ let CURRENT_CAPABILITIES = null;
|
|
|
69
103
|
let CURRENT_DEVICE_NAME = null;
|
|
70
104
|
let CURRENT_CONFIG = null;
|
|
71
105
|
let CURRENT_PROVIDER_HEALTH = null;
|
|
106
|
+
let CURRENT_JOB_INGRESS_STATE = null;
|
|
107
|
+
let CURRENT_ACCEPTING_JOBS = true;
|
|
108
|
+
let CURRENT_RUNTIME_OPTIONS = null;
|
|
72
109
|
let ACTIVE_OPTIONS = null;
|
|
73
110
|
function parseArgs(argv) {
|
|
74
111
|
const normalizedArgv = argv[0] === '--' ? argv.slice(1) : argv;
|
|
@@ -108,7 +145,7 @@ function parseArgs(argv) {
|
|
|
108
145
|
return { command, positional, options };
|
|
109
146
|
}
|
|
110
147
|
function printHelp() {
|
|
111
|
-
console.log(`\nPanorama Gateway\n\nUsage:\n panorama-gateway pair <PAIRING_CODE> [--device-name \"My Mac\"]\n panorama-gateway start [--device-name \"My Mac\"] [--foreground|--daemon]\n panorama-gateway stop\n panorama-gateway status\n panorama-gateway logs [--lines 200] [--no-follow]\n panorama-gateway doctor\n\nNotes:\n Pair automatically starts the gateway when possible.\n Start runs in the background by default for the built CLI. Use --foreground to keep it attached.\n\nOutput options:\n --verbose, -v Show technical details (paths, IDs, PIDs)\n\nEnvironment options:\n --env <local|dev|test|stage|prod> Load .env.<env> from repo root (defaults to .env)\n --env-file <path> Load a specific env file\n PANORAMA_ENV Same as --env\n PANORAMA_ENV_FILE Same as --env-file\n\nCLI overrides:\n --config-dir <path> Override config directory (default: ~/.panorama/gateway)\n --config-path <path> Override gateway config file\n --log-path <path> Override gateway log file\n --pid-path <path> Override gateway pid file\n --claude-cli <path> Override Claude CLI command\n --codex-cli <path> Override Codex CLI command\n --gemini-cli <path> Override Gemini CLI command\n\nEnvironment overrides:\n PANORAMA_GATEWAY_CONFIG_DIR\n PANORAMA_GATEWAY_CONFIG_PATH\n PANORAMA_GATEWAY_LOG_PATH\n PANORAMA_GATEWAY_PID_PATH\n PANORAMA_CLAUDE_CLI or CLAUDE_CLI\n PANORAMA_CODEX_CLI or CODEX_CLI\n PANORAMA_GEMINI_CLI or GEMINI_CLI\n PANORAMA_GATEWAY_AUTO_RESTART (default: on for built CLI, off for dev)\n PANORAMA_GATEWAY_RESTART_CHECK_MS (default: 30000)\n PANORAMA_GATEWAY_RESTART_MAX_WAIT_MS (optional)\n`);
|
|
148
|
+
console.log(`\nPanorama Gateway\n\nUsage:\n panorama-gateway pair <PAIRING_CODE> [--device-name \"My Mac\"]\n panorama-gateway start [--device-name \"My Mac\"] [--foreground|--daemon]\n panorama-gateway stop\n panorama-gateway status\n panorama-gateway logs [--lines 200] [--no-follow]\n panorama-gateway doctor\n panorama-gateway full-control <enable|disable|status>\n\nNotes:\n Pair automatically starts the gateway when possible.\n Start runs in the background by default for the built CLI. Use --foreground to keep it attached.\n\nOutput options:\n --verbose, -v Show technical details (paths, IDs, PIDs)\n\nEnvironment options:\n --env <local|dev|test|stage|prod> Load .env.<env> from repo root (defaults to .env)\n --env-file <path> Load a specific env file\n PANORAMA_ENV Same as --env\n PANORAMA_ENV_FILE Same as --env-file\n\nCLI overrides:\n --config-dir <path> Override config directory (default: ~/.panorama/gateway)\n --config-path <path> Override gateway config file\n --log-path <path> Override gateway log file\n --pid-path <path> Override gateway pid file\n --claude-cli <path> Override Claude CLI command\n --codex-cli <path> Override Codex CLI command\n --gemini-cli <path> Override Gemini CLI command\n\nEnvironment overrides:\n PANORAMA_GATEWAY_CONFIG_DIR\n PANORAMA_GATEWAY_CONFIG_PATH\n PANORAMA_GATEWAY_LOG_PATH\n PANORAMA_GATEWAY_PID_PATH\n PANORAMA_CLAUDE_CLI or CLAUDE_CLI\n PANORAMA_CODEX_CLI or CODEX_CLI\n PANORAMA_GEMINI_CLI or GEMINI_CLI\n PANORAMA_GATEWAY_AUTO_RESTART (default: on for built CLI, off for dev)\n PANORAMA_GATEWAY_RESTART_CHECK_MS (default: 30000)\n PANORAMA_GATEWAY_RESTART_MAX_WAIT_MS (optional)\n`);
|
|
112
149
|
}
|
|
113
150
|
function getStringOption(options, key) {
|
|
114
151
|
const value = options[key];
|
|
@@ -116,6 +153,20 @@ function getStringOption(options, key) {
|
|
|
116
153
|
return value;
|
|
117
154
|
return undefined;
|
|
118
155
|
}
|
|
156
|
+
function parseBooleanInput(value) {
|
|
157
|
+
if (typeof value === 'boolean')
|
|
158
|
+
return value;
|
|
159
|
+
if (typeof value === 'string') {
|
|
160
|
+
const normalized = value.trim().toLowerCase();
|
|
161
|
+
if (normalized === 'true' || normalized === '1' || normalized === 'yes' || normalized === 'on') {
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
if (normalized === 'false' || normalized === '0' || normalized === 'no' || normalized === 'off') {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
119
170
|
function getActiveOptions(options) {
|
|
120
171
|
return options ?? ACTIVE_OPTIONS ?? {};
|
|
121
172
|
}
|
|
@@ -1110,12 +1161,44 @@ async function loadConfig(options) {
|
|
|
1110
1161
|
const { configPath } = resolveGatewayPaths(options);
|
|
1111
1162
|
await ensureOwnerOnlyFile(configPath);
|
|
1112
1163
|
const raw = await fs.readFile(configPath, 'utf-8');
|
|
1113
|
-
|
|
1164
|
+
const parsed = JSON.parse(raw);
|
|
1165
|
+
return {
|
|
1166
|
+
...(parsed ?? {}),
|
|
1167
|
+
supabaseUrl: parsed?.supabaseUrl ?? '',
|
|
1168
|
+
supabaseAnonKey: parsed?.supabaseAnonKey ?? '',
|
|
1169
|
+
accessToken: parsed?.accessToken ?? '',
|
|
1170
|
+
refreshToken: parsed?.refreshToken ?? '',
|
|
1171
|
+
gatewayId: parsed?.gatewayId ?? '',
|
|
1172
|
+
teamId: parsed?.teamId ?? '',
|
|
1173
|
+
full_control_enabled: parsed?.full_control_enabled === true,
|
|
1174
|
+
};
|
|
1114
1175
|
}
|
|
1115
1176
|
async function saveConfig(config, options) {
|
|
1116
1177
|
const { configPath } = resolveGatewayPaths(options);
|
|
1117
1178
|
await writeOwnerOnlyFile(configPath, JSON.stringify(config, null, 2));
|
|
1118
1179
|
}
|
|
1180
|
+
async function refreshRuntimeInstallFullControl(config) {
|
|
1181
|
+
const options = CURRENT_RUNTIME_OPTIONS ?? undefined;
|
|
1182
|
+
try {
|
|
1183
|
+
const latestConfig = await loadConfig(options);
|
|
1184
|
+
const nextValue = latestConfig.full_control_enabled === true;
|
|
1185
|
+
if (config.full_control_enabled !== nextValue) {
|
|
1186
|
+
config.full_control_enabled = nextValue;
|
|
1187
|
+
if (CURRENT_CONFIG) {
|
|
1188
|
+
CURRENT_CONFIG.full_control_enabled = nextValue;
|
|
1189
|
+
}
|
|
1190
|
+
logInfo('Reloaded gateway install full-control setting', {
|
|
1191
|
+
enabled: nextValue,
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
catch (error) {
|
|
1196
|
+
logError('Failed to refresh gateway install full-control setting', {
|
|
1197
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
return config.full_control_enabled === true;
|
|
1201
|
+
}
|
|
1119
1202
|
function shouldEmitDebugOutput(stream) {
|
|
1120
1203
|
if (!stream.isTTY)
|
|
1121
1204
|
return true;
|
|
@@ -1237,6 +1320,15 @@ async function pairGateway(code, options, configResolution) {
|
|
|
1237
1320
|
if (!body?.access_token || !body?.refresh_token || !body?.gateway_id || !body?.team_id) {
|
|
1238
1321
|
throw new Error('Pairing response missing required fields');
|
|
1239
1322
|
}
|
|
1323
|
+
const requestedFullControl = parseBooleanInput(options['full-control']);
|
|
1324
|
+
let existingConfig = null;
|
|
1325
|
+
try {
|
|
1326
|
+
existingConfig = await loadConfig(options);
|
|
1327
|
+
}
|
|
1328
|
+
catch {
|
|
1329
|
+
existingConfig = null;
|
|
1330
|
+
}
|
|
1331
|
+
const installFullControlEnabled = requestedFullControl ?? existingConfig?.full_control_enabled ?? false;
|
|
1240
1332
|
const config = {
|
|
1241
1333
|
supabaseUrl,
|
|
1242
1334
|
supabaseAnonKey,
|
|
@@ -1244,6 +1336,7 @@ async function pairGateway(code, options, configResolution) {
|
|
|
1244
1336
|
refreshToken: body.refresh_token,
|
|
1245
1337
|
gatewayId: body.gateway_id,
|
|
1246
1338
|
teamId: body.team_id,
|
|
1339
|
+
full_control_enabled: installFullControlEnabled,
|
|
1247
1340
|
deviceName,
|
|
1248
1341
|
};
|
|
1249
1342
|
await saveConfig(config, options);
|
|
@@ -1255,6 +1348,7 @@ async function pairGateway(code, options, configResolution) {
|
|
|
1255
1348
|
await discoverGatewayCliCommands(options);
|
|
1256
1349
|
const { providerHealth: resolvedHealth, anyProviderReady: ready } = await buildCapabilities({
|
|
1257
1350
|
validationMode: 'full',
|
|
1351
|
+
installFullControlEnabled: config.full_control_enabled,
|
|
1258
1352
|
onProgress: (event) => {
|
|
1259
1353
|
if (event.stage === 'start') {
|
|
1260
1354
|
cliInfo(`Validating ${formatProviderLabel(event.providerId)}...`, options);
|
|
@@ -1282,6 +1376,11 @@ async function pairGateway(code, options, configResolution) {
|
|
|
1282
1376
|
{ label: 'Gateway ID', value: body.gateway_id, verboseOnly: true },
|
|
1283
1377
|
{ label: 'Team ID', value: body.team_id, verboseOnly: true },
|
|
1284
1378
|
{ label: 'Device', value: deviceName, verboseOnly: true },
|
|
1379
|
+
{
|
|
1380
|
+
label: 'Install full control',
|
|
1381
|
+
value: installFullControlEnabled ? 'Enabled' : 'Disabled',
|
|
1382
|
+
verboseOnly: true,
|
|
1383
|
+
},
|
|
1285
1384
|
...(providerHealthSummary
|
|
1286
1385
|
? [{ label: 'Provider health', value: providerHealthSummary, verboseOnly: true }]
|
|
1287
1386
|
: []),
|
|
@@ -1567,10 +1666,12 @@ async function buildCapabilities(params) {
|
|
|
1567
1666
|
return false;
|
|
1568
1667
|
return entry.health?.status !== 'unavailable' && entry.health?.status !== 'unhealthy';
|
|
1569
1668
|
});
|
|
1669
|
+
const gatewayVersion = await getGatewayVersion();
|
|
1570
1670
|
const runtime = {
|
|
1571
1671
|
platform: process.platform,
|
|
1572
1672
|
arch: process.arch,
|
|
1573
1673
|
node_version: process.version,
|
|
1674
|
+
...(gatewayVersion ? { gateway_version: gatewayVersion } : {}),
|
|
1574
1675
|
};
|
|
1575
1676
|
const host = {
|
|
1576
1677
|
kind: 'gateway',
|
|
@@ -1579,6 +1680,9 @@ async function buildCapabilities(params) {
|
|
|
1579
1680
|
if (params?.deviceName) {
|
|
1580
1681
|
host.label = params.deviceName;
|
|
1581
1682
|
}
|
|
1683
|
+
host.metadata = {
|
|
1684
|
+
install_full_control_enabled: params?.installFullControlEnabled === true,
|
|
1685
|
+
};
|
|
1582
1686
|
return {
|
|
1583
1687
|
capabilities: {
|
|
1584
1688
|
schema_version: GATEWAY_CAPABILITIES_SCHEMA_VERSION,
|
|
@@ -1623,21 +1727,6 @@ function buildProviderFailureMessage(providerId, detail) {
|
|
|
1623
1727
|
const detailSuffix = detail ? ` Details: ${detail}` : '';
|
|
1624
1728
|
return `Gateway provider ${label} is unhealthy. Try another provider or run "panorama-gateway doctor".${detailSuffix}`;
|
|
1625
1729
|
}
|
|
1626
|
-
function isProviderReady(entry) {
|
|
1627
|
-
if (!entry)
|
|
1628
|
-
return false;
|
|
1629
|
-
if (entry.health?.status === 'healthy')
|
|
1630
|
-
return true;
|
|
1631
|
-
if (!entry.available)
|
|
1632
|
-
return false;
|
|
1633
|
-
if (!entry.health)
|
|
1634
|
-
return true;
|
|
1635
|
-
return entry.health.status !== 'unavailable' && entry.health.status !== 'unhealthy';
|
|
1636
|
-
}
|
|
1637
|
-
function computeGatewayStatusFromCapabilities(providers) {
|
|
1638
|
-
const anyReady = Object.values(providers).some((entry) => isProviderReady(entry));
|
|
1639
|
-
return anyReady ? 'ready' : 'error';
|
|
1640
|
-
}
|
|
1641
1730
|
function resolveRuntimeProviderHealth(providerId) {
|
|
1642
1731
|
if (CURRENT_PROVIDER_HEALTH && CURRENT_PROVIDER_HEALTH[providerId]) {
|
|
1643
1732
|
return CURRENT_PROVIDER_HEALTH[providerId];
|
|
@@ -1698,7 +1787,11 @@ async function updateRuntimeProviderHealth(params) {
|
|
|
1698
1787
|
...(CURRENT_PROVIDER_HEALTH ?? {}),
|
|
1699
1788
|
[params.providerId]: nextHealth,
|
|
1700
1789
|
};
|
|
1701
|
-
const nextStatusOverall =
|
|
1790
|
+
const nextStatusOverall = computeGatewayOperationalStatus({
|
|
1791
|
+
capabilities: CURRENT_CAPABILITIES,
|
|
1792
|
+
jobIngressState: CURRENT_JOB_INGRESS_STATE,
|
|
1793
|
+
acceptingJobs: CURRENT_ACCEPTING_JOBS,
|
|
1794
|
+
});
|
|
1702
1795
|
if (CURRENT_CAPABILITIES && CURRENT_DEVICE_NAME) {
|
|
1703
1796
|
await sendHeartbeat(params.supabase, nextStatusOverall, CURRENT_CAPABILITIES, CURRENT_DEVICE_NAME);
|
|
1704
1797
|
}
|
|
@@ -1728,8 +1821,13 @@ function buildGatewayChildEnv(baseEnv) {
|
|
|
1728
1821
|
const tmpDir = resolveGatewayTmpDir();
|
|
1729
1822
|
return buildGatewayChildProcessEnv({ baseEnv, tmpDir });
|
|
1730
1823
|
}
|
|
1731
|
-
function resolveChildProcessEnv(
|
|
1732
|
-
|
|
1824
|
+
function resolveChildProcessEnv(envOverrides) {
|
|
1825
|
+
const tmpDir = resolveGatewayTmpDir();
|
|
1826
|
+
return buildGatewayChildProcessEnv({
|
|
1827
|
+
baseEnv: process.env,
|
|
1828
|
+
tmpDir,
|
|
1829
|
+
overrides: envOverrides,
|
|
1830
|
+
});
|
|
1733
1831
|
}
|
|
1734
1832
|
async function runSubagentCommand(command, args, options) {
|
|
1735
1833
|
const start = Date.now();
|
|
@@ -1781,14 +1879,11 @@ async function runSubagentCommand(command, args, options) {
|
|
|
1781
1879
|
const timer = timeoutMs && Number.isFinite(timeoutMs)
|
|
1782
1880
|
? setTimeout(() => {
|
|
1783
1881
|
timedOut = true;
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
}
|
|
1790
|
-
}, PROCESS_KILL_GRACE_MS);
|
|
1791
|
-
}
|
|
1882
|
+
requestChildTerminationWithLogging(child, {
|
|
1883
|
+
reason: 'process_timeout',
|
|
1884
|
+
graceMs: PROCESS_KILL_GRACE_MS,
|
|
1885
|
+
context: 'runSubagentCommand',
|
|
1886
|
+
});
|
|
1792
1887
|
}, timeoutMs)
|
|
1793
1888
|
: null;
|
|
1794
1889
|
const finalize = (exitCode, error) => {
|
|
@@ -1822,65 +1917,23 @@ async function runSubagentCommandStreaming(command, args, options) {
|
|
|
1822
1917
|
const start = Date.now();
|
|
1823
1918
|
const timeoutMs = options.timeoutMs ?? DEFAULT_SUBAGENT_TIMEOUT_MS;
|
|
1824
1919
|
return await new Promise((resolve) => {
|
|
1825
|
-
const stdoutChunks = [];
|
|
1826
1920
|
const stderrChunks = [];
|
|
1827
|
-
const events = [];
|
|
1828
|
-
let stdoutBytes = 0;
|
|
1829
1921
|
let stderrBytes = 0;
|
|
1830
|
-
let stdoutTruncated = false;
|
|
1831
1922
|
let stderrTruncated = false;
|
|
1832
1923
|
let timedOut = false;
|
|
1833
|
-
|
|
1834
|
-
|
|
1924
|
+
const collector = new StreamJsonCollector({
|
|
1925
|
+
maxOutputBytes: MAX_SUBAGENT_OUTPUT_BYTES,
|
|
1926
|
+
maxEvents: MAX_SUBAGENT_STREAM_EVENTS,
|
|
1927
|
+
maxBufferChars: MAX_SUBAGENT_STREAM_BUFFER_CHARS,
|
|
1928
|
+
});
|
|
1835
1929
|
const child = spawn(command, args, {
|
|
1836
1930
|
cwd: options.cwd,
|
|
1837
1931
|
env: resolveChildProcessEnv(options.env),
|
|
1838
1932
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1839
1933
|
});
|
|
1840
1934
|
options.onStart?.(child);
|
|
1841
|
-
const captureStdout = (chunk) => {
|
|
1842
|
-
if (stdoutBytes >= MAX_SUBAGENT_OUTPUT_BYTES) {
|
|
1843
|
-
stdoutTruncated = true;
|
|
1844
|
-
return;
|
|
1845
|
-
}
|
|
1846
|
-
const remaining = MAX_SUBAGENT_OUTPUT_BYTES - stdoutBytes;
|
|
1847
|
-
if (chunk.length > remaining) {
|
|
1848
|
-
stdoutChunks.push(chunk.slice(0, remaining));
|
|
1849
|
-
stdoutBytes = MAX_SUBAGENT_OUTPUT_BYTES;
|
|
1850
|
-
stdoutTruncated = true;
|
|
1851
|
-
return;
|
|
1852
|
-
}
|
|
1853
|
-
stdoutChunks.push(chunk);
|
|
1854
|
-
stdoutBytes += chunk.length;
|
|
1855
|
-
};
|
|
1856
|
-
const parseLine = (line) => {
|
|
1857
|
-
const trimmed = line.trim();
|
|
1858
|
-
if (!trimmed)
|
|
1859
|
-
return;
|
|
1860
|
-
let event = null;
|
|
1861
|
-
try {
|
|
1862
|
-
event = JSON.parse(trimmed);
|
|
1863
|
-
}
|
|
1864
|
-
catch {
|
|
1865
|
-
event = null;
|
|
1866
|
-
}
|
|
1867
|
-
events.push({
|
|
1868
|
-
sequence,
|
|
1869
|
-
raw: trimmed,
|
|
1870
|
-
event,
|
|
1871
|
-
});
|
|
1872
|
-
sequence += 1;
|
|
1873
|
-
};
|
|
1874
1935
|
child.stdout?.on('data', (chunk) => {
|
|
1875
|
-
|
|
1876
|
-
buffer += chunk.toString('utf-8');
|
|
1877
|
-
let newlineIndex = buffer.indexOf('\n');
|
|
1878
|
-
while (newlineIndex >= 0) {
|
|
1879
|
-
const line = buffer.slice(0, newlineIndex);
|
|
1880
|
-
buffer = buffer.slice(newlineIndex + 1);
|
|
1881
|
-
parseLine(line);
|
|
1882
|
-
newlineIndex = buffer.indexOf('\n');
|
|
1883
|
-
}
|
|
1936
|
+
collector.append(chunk);
|
|
1884
1937
|
});
|
|
1885
1938
|
child.stderr?.on('data', (chunk) => {
|
|
1886
1939
|
if (stderrBytes >= MAX_SUBAGENT_OUTPUT_BYTES) {
|
|
@@ -1900,36 +1953,30 @@ async function runSubagentCommandStreaming(command, args, options) {
|
|
|
1900
1953
|
const timer = timeoutMs && Number.isFinite(timeoutMs)
|
|
1901
1954
|
? setTimeout(() => {
|
|
1902
1955
|
timedOut = true;
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
}
|
|
1909
|
-
}, PROCESS_KILL_GRACE_MS);
|
|
1910
|
-
}
|
|
1956
|
+
requestChildTerminationWithLogging(child, {
|
|
1957
|
+
reason: 'process_timeout',
|
|
1958
|
+
graceMs: PROCESS_KILL_GRACE_MS,
|
|
1959
|
+
context: 'runSubagentCommandStreaming',
|
|
1960
|
+
});
|
|
1911
1961
|
}, timeoutMs)
|
|
1912
1962
|
: null;
|
|
1913
1963
|
const finalize = (exitCode, error) => {
|
|
1914
1964
|
if (timer)
|
|
1915
1965
|
clearTimeout(timer);
|
|
1916
|
-
|
|
1917
|
-
parseLine(buffer);
|
|
1918
|
-
}
|
|
1966
|
+
const streamResult = collector.finalize();
|
|
1919
1967
|
const durationMs = Date.now() - start;
|
|
1920
|
-
const stdout = Buffer.concat(stdoutChunks).toString('utf-8');
|
|
1921
1968
|
const stderr = Buffer.concat(stderrChunks).toString('utf-8');
|
|
1922
1969
|
const ok = !timedOut && exitCode === 0 && !error;
|
|
1923
1970
|
resolve({
|
|
1924
1971
|
ok,
|
|
1925
|
-
stdout,
|
|
1972
|
+
stdout: streamResult.stdout,
|
|
1926
1973
|
stderr,
|
|
1927
1974
|
exitCode,
|
|
1928
1975
|
durationMs,
|
|
1929
1976
|
timedOut,
|
|
1930
|
-
stdoutTruncated,
|
|
1977
|
+
stdoutTruncated: streamResult.stdoutTruncated || streamResult.eventsTruncated,
|
|
1931
1978
|
stderrTruncated,
|
|
1932
|
-
events,
|
|
1979
|
+
events: streamResult.events,
|
|
1933
1980
|
error,
|
|
1934
1981
|
});
|
|
1935
1982
|
};
|
|
@@ -2166,6 +2213,11 @@ function describeGatewayJob(job) {
|
|
|
2166
2213
|
if (job.job_type === 'subagent_cancel') {
|
|
2167
2214
|
return { job_kind: 'subagent_cancel' };
|
|
2168
2215
|
}
|
|
2216
|
+
if (job.job_type === 'shell_session') {
|
|
2217
|
+
const payload = job.payload ?? {};
|
|
2218
|
+
const action = typeof payload.action === 'string' ? payload.action : null;
|
|
2219
|
+
return { job_kind: 'shell_session', shell_action: action };
|
|
2220
|
+
}
|
|
2169
2221
|
return { job_kind: job.job_type };
|
|
2170
2222
|
}
|
|
2171
2223
|
function inferProviderFromSubagentType(subagentType) {
|
|
@@ -2224,15 +2276,86 @@ function classifySubagentRetryable(params) {
|
|
|
2224
2276
|
return null;
|
|
2225
2277
|
}
|
|
2226
2278
|
async function getNextRunSequence(supabase, subagentId) {
|
|
2227
|
-
const { data } = await supabase
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2279
|
+
const { data, error } = await supabase.rpc('allocate_subagent_run_sequence', {
|
|
2280
|
+
p_subagent_id: subagentId,
|
|
2281
|
+
});
|
|
2282
|
+
if (error) {
|
|
2283
|
+
logError('Failed to allocate subagent run sequence; falling back to read-based sequence', {
|
|
2284
|
+
subagentId,
|
|
2285
|
+
error: error.message,
|
|
2286
|
+
});
|
|
2287
|
+
const { data: fallback } = await supabase
|
|
2288
|
+
.from('subagent_runs')
|
|
2289
|
+
.select('run_sequence')
|
|
2290
|
+
.eq('subagent_id', subagentId)
|
|
2291
|
+
.order('run_sequence', { ascending: false })
|
|
2292
|
+
.limit(1)
|
|
2293
|
+
.maybeSingle();
|
|
2294
|
+
const lastSequence = fallback && typeof fallback.run_sequence === 'number' ? fallback.run_sequence : 0;
|
|
2295
|
+
return lastSequence + 1;
|
|
2296
|
+
}
|
|
2297
|
+
return typeof data === 'number' && Number.isFinite(data) ? data : 1;
|
|
2298
|
+
}
|
|
2299
|
+
async function getNextSubagentRunEventSequence(supabase, runId) {
|
|
2300
|
+
const { data, error } = await supabase
|
|
2301
|
+
.from('subagent_run_events')
|
|
2302
|
+
.select('sequence')
|
|
2303
|
+
.eq('run_id', runId)
|
|
2304
|
+
.order('sequence', { ascending: false })
|
|
2232
2305
|
.limit(1)
|
|
2233
2306
|
.maybeSingle();
|
|
2234
|
-
|
|
2235
|
-
|
|
2307
|
+
if (error) {
|
|
2308
|
+
logError('Failed to fetch subagent run event sequence; defaulting to 0', {
|
|
2309
|
+
runId,
|
|
2310
|
+
error: error.message,
|
|
2311
|
+
});
|
|
2312
|
+
return 0;
|
|
2313
|
+
}
|
|
2314
|
+
if (!data || typeof data.sequence !== 'number' || !Number.isFinite(data.sequence)) {
|
|
2315
|
+
return 0;
|
|
2316
|
+
}
|
|
2317
|
+
return data.sequence + 1;
|
|
2318
|
+
}
|
|
2319
|
+
function normalizeSubagentRunEventType(value, fallback = 'system') {
|
|
2320
|
+
if (typeof value !== 'string')
|
|
2321
|
+
return fallback;
|
|
2322
|
+
const normalized = value.trim().toLowerCase();
|
|
2323
|
+
return SUBAGENT_RUN_EVENT_TYPE_SET.has(normalized) ? normalized : fallback;
|
|
2324
|
+
}
|
|
2325
|
+
async function insertSubagentRunEvent(params) {
|
|
2326
|
+
const normalizedType = normalizeSubagentRunEventType(params.eventType, 'system');
|
|
2327
|
+
const insertWithSequence = async (sequence) => params.supabase.from('subagent_run_events').insert({
|
|
2328
|
+
run_id: params.runId,
|
|
2329
|
+
subagent_id: params.subagentId,
|
|
2330
|
+
team_id: params.teamId,
|
|
2331
|
+
gateway_id: params.gatewayId,
|
|
2332
|
+
sequence,
|
|
2333
|
+
event_type: normalizedType,
|
|
2334
|
+
payload: asJson(trimEventPayload(params.payload)),
|
|
2335
|
+
});
|
|
2336
|
+
const { error } = await insertWithSequence(params.sequence);
|
|
2337
|
+
if (!error)
|
|
2338
|
+
return params.sequence;
|
|
2339
|
+
if (error.code === '23505') {
|
|
2340
|
+
const retrySequence = await getNextSubagentRunEventSequence(params.supabase, params.runId);
|
|
2341
|
+
const { error: retryError } = await insertWithSequence(retrySequence);
|
|
2342
|
+
if (!retryError)
|
|
2343
|
+
return retrySequence;
|
|
2344
|
+
logError('Failed to insert subagent run event after retry', {
|
|
2345
|
+
runId: params.runId,
|
|
2346
|
+
subagentId: params.subagentId,
|
|
2347
|
+
event_type: normalizedType,
|
|
2348
|
+
error: retryError.message,
|
|
2349
|
+
});
|
|
2350
|
+
return null;
|
|
2351
|
+
}
|
|
2352
|
+
logError('Failed to insert subagent run event', {
|
|
2353
|
+
runId: params.runId,
|
|
2354
|
+
subagentId: params.subagentId,
|
|
2355
|
+
event_type: normalizedType,
|
|
2356
|
+
error: error.message,
|
|
2357
|
+
});
|
|
2358
|
+
return null;
|
|
2236
2359
|
}
|
|
2237
2360
|
function mergeMetadata(base, update) {
|
|
2238
2361
|
return {
|
|
@@ -2243,19 +2366,61 @@ function mergeMetadata(base, update) {
|
|
|
2243
2366
|
function asJson(value) {
|
|
2244
2367
|
return value;
|
|
2245
2368
|
}
|
|
2369
|
+
async function loadGatewayControlState(supabase, config) {
|
|
2370
|
+
const { data, error } = await supabase
|
|
2371
|
+
.from('team_gateways')
|
|
2372
|
+
.select('full_control_enabled, status, last_seen_at')
|
|
2373
|
+
.eq('id', config.gatewayId)
|
|
2374
|
+
.eq('team_id', config.teamId)
|
|
2375
|
+
.maybeSingle();
|
|
2376
|
+
if (error) {
|
|
2377
|
+
logError('Failed to load gateway control state', {
|
|
2378
|
+
gatewayId: config.gatewayId,
|
|
2379
|
+
teamId: config.teamId,
|
|
2380
|
+
error: error.message,
|
|
2381
|
+
});
|
|
2382
|
+
return null;
|
|
2383
|
+
}
|
|
2384
|
+
if (!data)
|
|
2385
|
+
return null;
|
|
2386
|
+
return {
|
|
2387
|
+
full_control_enabled: data.full_control_enabled === true,
|
|
2388
|
+
status: typeof data.status === 'string' ? data.status : 'unknown',
|
|
2389
|
+
last_seen_at: typeof data.last_seen_at === 'string' ? data.last_seen_at : null,
|
|
2390
|
+
};
|
|
2391
|
+
}
|
|
2392
|
+
function isHeartbeatFresh(lastSeenAt) {
|
|
2393
|
+
if (!lastSeenAt)
|
|
2394
|
+
return false;
|
|
2395
|
+
const parsed = Date.parse(lastSeenAt);
|
|
2396
|
+
if (!Number.isFinite(parsed))
|
|
2397
|
+
return false;
|
|
2398
|
+
return Date.now() - parsed <= GATEWAY_HEARTBEAT_STALE_MS;
|
|
2399
|
+
}
|
|
2400
|
+
function requestChildTerminationWithLogging(child, options) {
|
|
2401
|
+
terminateChildProcess(child, {
|
|
2402
|
+
reason: options.reason,
|
|
2403
|
+
graceMs: options.graceMs,
|
|
2404
|
+
onError: (phase, error) => {
|
|
2405
|
+
logError('Failed to send child process termination signal', {
|
|
2406
|
+
context: options.context,
|
|
2407
|
+
reason: options.reason,
|
|
2408
|
+
phase,
|
|
2409
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2410
|
+
});
|
|
2411
|
+
},
|
|
2412
|
+
});
|
|
2413
|
+
}
|
|
2246
2414
|
function requestSubagentCancel(subagentId, runId, reason) {
|
|
2247
2415
|
const activeRun = activeSubagentRuns.get(subagentId);
|
|
2248
2416
|
if (activeRun && (!runId || activeRun.runId === runId)) {
|
|
2249
2417
|
activeRun.cancelled = true;
|
|
2250
2418
|
activeRun.cancelReason = reason;
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
}
|
|
2257
|
-
}, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
|
|
2258
|
-
}
|
|
2419
|
+
requestChildTerminationWithLogging(activeRun.child, {
|
|
2420
|
+
reason,
|
|
2421
|
+
graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
|
|
2422
|
+
context: 'gateway_cancel_request',
|
|
2423
|
+
});
|
|
2259
2424
|
return { active: true };
|
|
2260
2425
|
}
|
|
2261
2426
|
if (runId) {
|
|
@@ -2273,6 +2438,9 @@ function resolvePendingCancel(subagentId, runId, metadata) {
|
|
|
2273
2438
|
? metadata.cancel_reason
|
|
2274
2439
|
: 'Cancelled';
|
|
2275
2440
|
}
|
|
2441
|
+
return resolvePendingCancelFromQueue(subagentId, runId);
|
|
2442
|
+
}
|
|
2443
|
+
function resolvePendingCancelFromQueue(subagentId, runId) {
|
|
2276
2444
|
const pendingByRun = pendingCancelByRunId.get(runId);
|
|
2277
2445
|
if (pendingByRun)
|
|
2278
2446
|
return pendingByRun.reason;
|
|
@@ -2281,6 +2449,13 @@ function resolvePendingCancel(subagentId, runId, metadata) {
|
|
|
2281
2449
|
return pendingBySubagent.reason;
|
|
2282
2450
|
return null;
|
|
2283
2451
|
}
|
|
2452
|
+
function consumePendingCancelFromQueue(subagentId, runId) {
|
|
2453
|
+
const reason = resolvePendingCancelFromQueue(subagentId, runId);
|
|
2454
|
+
if (!reason)
|
|
2455
|
+
return null;
|
|
2456
|
+
clearPendingCancel(subagentId, runId);
|
|
2457
|
+
return reason;
|
|
2458
|
+
}
|
|
2284
2459
|
function clearPendingCancel(subagentId, runId) {
|
|
2285
2460
|
pendingCancelByRunId.delete(runId);
|
|
2286
2461
|
pendingCancelBySubagentId.delete(subagentId);
|
|
@@ -2294,7 +2469,17 @@ function buildErrorOutput(message) {
|
|
|
2294
2469
|
};
|
|
2295
2470
|
}
|
|
2296
2471
|
function isJobTargeted(job, gatewayId) {
|
|
2297
|
-
return
|
|
2472
|
+
return job.gateway_id === gatewayId;
|
|
2473
|
+
}
|
|
2474
|
+
function resolveJobSubagentId(job) {
|
|
2475
|
+
if (job.job_type !== 'subagent_run')
|
|
2476
|
+
return null;
|
|
2477
|
+
const payload = job.payload ?? {};
|
|
2478
|
+
return typeof payload.subagent_id === 'string'
|
|
2479
|
+
? payload.subagent_id
|
|
2480
|
+
: typeof payload.subagentId === 'string'
|
|
2481
|
+
? payload.subagentId
|
|
2482
|
+
: null;
|
|
2298
2483
|
}
|
|
2299
2484
|
async function sendHeartbeat(supabase, status, capabilities, deviceName) {
|
|
2300
2485
|
const { error } = await supabase.functions.invoke('gateway-heartbeat', {
|
|
@@ -2322,7 +2507,7 @@ async function claimJob(supabase, config, job) {
|
|
|
2322
2507
|
.eq('id', job.id)
|
|
2323
2508
|
.eq('team_id', config.teamId)
|
|
2324
2509
|
.eq('status', 'queued')
|
|
2325
|
-
.
|
|
2510
|
+
.eq('gateway_id', config.gatewayId)
|
|
2326
2511
|
.select('*')
|
|
2327
2512
|
.maybeSingle();
|
|
2328
2513
|
if (error) {
|
|
@@ -2410,7 +2595,7 @@ async function handleDiagnosticJob() {
|
|
|
2410
2595
|
},
|
|
2411
2596
|
};
|
|
2412
2597
|
}
|
|
2413
|
-
async function handleSubagentCancelJob(supabase, job) {
|
|
2598
|
+
async function handleSubagentCancelJob(supabase, config, job) {
|
|
2414
2599
|
const payload = job.payload ?? {};
|
|
2415
2600
|
const subagentId = typeof payload.subagent_id === 'string'
|
|
2416
2601
|
? payload.subagent_id
|
|
@@ -2432,12 +2617,35 @@ async function handleSubagentCancelJob(supabase, job) {
|
|
|
2432
2617
|
const reason = typeof payload.reason === 'string' && payload.reason.trim().length > 0
|
|
2433
2618
|
? payload.reason.trim()
|
|
2434
2619
|
: 'Cancelled';
|
|
2435
|
-
const { data: subagent } = await supabase
|
|
2620
|
+
const { data: subagent, error: subagentError } = await supabase
|
|
2436
2621
|
.from('subagents')
|
|
2437
|
-
.select('status, metadata
|
|
2622
|
+
.select('id, team_id, gateway_id, subagent_type, status, metadata')
|
|
2438
2623
|
.eq('id', subagentId)
|
|
2439
2624
|
.maybeSingle();
|
|
2440
|
-
if (
|
|
2625
|
+
if (subagentError) {
|
|
2626
|
+
return {
|
|
2627
|
+
ok: false,
|
|
2628
|
+
result: { message: 'Failed to load subagent for cancellation', subagent_id: subagentId },
|
|
2629
|
+
error: subagentError.message,
|
|
2630
|
+
};
|
|
2631
|
+
}
|
|
2632
|
+
if (!subagent) {
|
|
2633
|
+
return {
|
|
2634
|
+
ok: true,
|
|
2635
|
+
result: {
|
|
2636
|
+
subagent_id: subagentId,
|
|
2637
|
+
run_id: runId,
|
|
2638
|
+
cancelled_active_run: false,
|
|
2639
|
+
reason,
|
|
2640
|
+
cancel_result: 'no_active_run',
|
|
2641
|
+
cancel_state: 'none',
|
|
2642
|
+
cancel_supported: false,
|
|
2643
|
+
status: 'not_found',
|
|
2644
|
+
message: 'Subagent not found; no active run to cancel.',
|
|
2645
|
+
},
|
|
2646
|
+
};
|
|
2647
|
+
}
|
|
2648
|
+
if (subagent.gateway_id !== config.gatewayId) {
|
|
2441
2649
|
return {
|
|
2442
2650
|
ok: true,
|
|
2443
2651
|
result: {
|
|
@@ -2445,72 +2653,216 @@ async function handleSubagentCancelJob(supabase, job) {
|
|
|
2445
2653
|
run_id: runId,
|
|
2446
2654
|
cancelled_active_run: false,
|
|
2447
2655
|
reason,
|
|
2448
|
-
|
|
2656
|
+
cancel_result: 'no_active_run',
|
|
2657
|
+
cancel_state: 'none',
|
|
2658
|
+
cancel_supported: false,
|
|
2659
|
+
status: 'running',
|
|
2660
|
+
message: 'Subagent is assigned to another gateway; skipping stale cancel request.',
|
|
2449
2661
|
},
|
|
2450
2662
|
};
|
|
2451
2663
|
}
|
|
2452
2664
|
const metadata = (subagent.metadata ?? {});
|
|
2453
|
-
const
|
|
2454
|
-
const
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2665
|
+
const status = typeof subagent.status === 'string' ? subagent.status : 'unknown';
|
|
2666
|
+
const hasGateway = !!subagent.gateway_id;
|
|
2667
|
+
const cancelAdapter = getSubagentCancelAdapter(subagent.subagent_type);
|
|
2668
|
+
const cancelSupported = cancelAdapter.supported &&
|
|
2669
|
+
(cancelAdapter.support !== 'gateway' || hasGateway);
|
|
2670
|
+
const cancelSupportMode = cancelAdapter.support;
|
|
2671
|
+
const cancellationScope = cancelAdapter.scope;
|
|
2672
|
+
let activeRun = null;
|
|
2673
|
+
if (status === 'running') {
|
|
2461
2674
|
const { data: runningRun, error: runningRunError } = await supabase
|
|
2462
2675
|
.from('subagent_runs')
|
|
2463
|
-
.select('id,
|
|
2464
|
-
.eq('
|
|
2676
|
+
.select('id, metadata')
|
|
2677
|
+
.eq('subagent_id', subagentId)
|
|
2465
2678
|
.eq('status', 'running')
|
|
2679
|
+
.order('run_sequence', { ascending: false })
|
|
2680
|
+
.limit(1)
|
|
2466
2681
|
.maybeSingle();
|
|
2467
2682
|
if (runningRunError) {
|
|
2468
|
-
|
|
2683
|
+
return {
|
|
2684
|
+
ok: false,
|
|
2685
|
+
result: {
|
|
2686
|
+
message: 'Failed to resolve active subagent run for cancellation',
|
|
2687
|
+
subagent_id: subagentId,
|
|
2688
|
+
},
|
|
2469
2689
|
error: runningRunError.message,
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2690
|
+
};
|
|
2691
|
+
}
|
|
2692
|
+
activeRun = runningRun;
|
|
2693
|
+
}
|
|
2694
|
+
const resolution = resolveCancelRequest({
|
|
2695
|
+
subagent_status: status,
|
|
2696
|
+
requested_run_id: runId,
|
|
2697
|
+
active_run_id: activeRun?.id ?? null,
|
|
2698
|
+
cancel_supported: cancelSupported,
|
|
2699
|
+
});
|
|
2700
|
+
if (!resolution.accepted) {
|
|
2701
|
+
return {
|
|
2702
|
+
ok: true,
|
|
2703
|
+
result: {
|
|
2704
|
+
subagent_id: subagentId,
|
|
2705
|
+
run_id: resolution.resolved_run_id,
|
|
2706
|
+
active_run_id: activeRun?.id ?? null,
|
|
2707
|
+
cancelled_active_run: false,
|
|
2708
|
+
reason,
|
|
2709
|
+
cancel_result: resolution.cancel_result,
|
|
2710
|
+
cancel_state: resolution.cancel_state,
|
|
2711
|
+
cancel_supported: cancelSupported,
|
|
2712
|
+
status: resolution.status,
|
|
2713
|
+
message: resolution.message,
|
|
2714
|
+
},
|
|
2715
|
+
};
|
|
2716
|
+
}
|
|
2717
|
+
const resolvedRunId = resolution.resolved_run_id;
|
|
2718
|
+
if (!resolvedRunId) {
|
|
2719
|
+
return {
|
|
2720
|
+
ok: false,
|
|
2721
|
+
result: {
|
|
2722
|
+
message: 'Cancel accepted without an active run_id',
|
|
2723
|
+
subagent_id: subagentId,
|
|
2724
|
+
},
|
|
2725
|
+
error: 'Cancel accepted without an active run_id',
|
|
2726
|
+
};
|
|
2727
|
+
}
|
|
2728
|
+
const nowIso = new Date().toISOString();
|
|
2729
|
+
const mergedMetadata = buildRequestedCancelMetadata({
|
|
2730
|
+
base: metadata,
|
|
2731
|
+
run_id: resolvedRunId,
|
|
2732
|
+
reason,
|
|
2733
|
+
requested_at: nowIso,
|
|
2734
|
+
cancel_support: cancelSupportMode,
|
|
2735
|
+
cancellation_scope: cancellationScope,
|
|
2736
|
+
});
|
|
2737
|
+
const { data: updatedSubagent, error: subagentUpdateError } = await supabase
|
|
2738
|
+
.from('subagents')
|
|
2739
|
+
.update({
|
|
2740
|
+
metadata: asJson(mergedMetadata),
|
|
2741
|
+
})
|
|
2742
|
+
.eq('status', 'running')
|
|
2743
|
+
.eq('id', subagentId)
|
|
2744
|
+
.select('id')
|
|
2745
|
+
.maybeSingle();
|
|
2746
|
+
if (subagentUpdateError) {
|
|
2747
|
+
return {
|
|
2748
|
+
ok: false,
|
|
2749
|
+
result: {
|
|
2750
|
+
message: 'Failed to persist subagent cancellation metadata',
|
|
2751
|
+
subagent_id: subagentId,
|
|
2752
|
+
run_id: resolvedRunId,
|
|
2753
|
+
},
|
|
2754
|
+
error: subagentUpdateError.message,
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
if (!updatedSubagent) {
|
|
2758
|
+
const { data: latestSubagent, error: latestSubagentError } = await supabase
|
|
2759
|
+
.from('subagents')
|
|
2760
|
+
.select('status')
|
|
2761
|
+
.eq('id', subagentId)
|
|
2762
|
+
.maybeSingle();
|
|
2763
|
+
if (latestSubagentError) {
|
|
2764
|
+
return {
|
|
2765
|
+
ok: false,
|
|
2766
|
+
result: {
|
|
2767
|
+
message: 'Failed to refresh subagent status after cancel race',
|
|
2768
|
+
subagent_id: subagentId,
|
|
2769
|
+
run_id: resolvedRunId,
|
|
2770
|
+
},
|
|
2771
|
+
error: latestSubagentError.message,
|
|
2772
|
+
};
|
|
2473
2773
|
}
|
|
2474
|
-
|
|
2475
|
-
|
|
2774
|
+
const latestStatus = typeof latestSubagent?.status === 'string' ? latestSubagent.status : status;
|
|
2775
|
+
let latestActiveRunId = null;
|
|
2776
|
+
if (latestStatus === 'running') {
|
|
2777
|
+
const { data: latestRunningRun, error: latestRunningRunError } = await supabase
|
|
2476
2778
|
.from('subagent_runs')
|
|
2477
|
-
.
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
.update({
|
|
2494
|
-
status: 'failed',
|
|
2495
|
-
output: asJson(cancelledOutput),
|
|
2496
|
-
error: reason,
|
|
2497
|
-
completed_at: nowIso,
|
|
2498
|
-
metadata: asJson(mergeMetadata(metadata, {
|
|
2499
|
-
cancel_run_id: resolvedRunId,
|
|
2500
|
-
cancel_requested_at: nowIso,
|
|
2501
|
-
cancel_reason: reason,
|
|
2502
|
-
})),
|
|
2503
|
-
})
|
|
2504
|
-
.eq('id', subagentId);
|
|
2505
|
-
if (subagentUpdateError) {
|
|
2506
|
-
logError('Failed to mark subagent as cancelled', {
|
|
2507
|
-
error: subagentUpdateError.message,
|
|
2508
|
-
subagentId,
|
|
2509
|
-
runId: resolvedRunId,
|
|
2510
|
-
});
|
|
2779
|
+
.select('id')
|
|
2780
|
+
.eq('subagent_id', subagentId)
|
|
2781
|
+
.eq('status', 'running')
|
|
2782
|
+
.order('run_sequence', { ascending: false })
|
|
2783
|
+
.limit(1)
|
|
2784
|
+
.maybeSingle();
|
|
2785
|
+
if (latestRunningRunError) {
|
|
2786
|
+
return {
|
|
2787
|
+
ok: false,
|
|
2788
|
+
result: {
|
|
2789
|
+
message: 'Failed to refresh active subagent run after cancel race',
|
|
2790
|
+
subagent_id: subagentId,
|
|
2791
|
+
run_id: resolvedRunId,
|
|
2792
|
+
},
|
|
2793
|
+
error: latestRunningRunError.message,
|
|
2794
|
+
};
|
|
2511
2795
|
}
|
|
2796
|
+
latestActiveRunId = latestRunningRun?.id ?? null;
|
|
2797
|
+
}
|
|
2798
|
+
const latestResolution = resolveCancelRequest({
|
|
2799
|
+
subagent_status: latestStatus,
|
|
2800
|
+
requested_run_id: runId,
|
|
2801
|
+
active_run_id: latestActiveRunId,
|
|
2802
|
+
cancel_supported: cancelSupported,
|
|
2803
|
+
});
|
|
2804
|
+
return {
|
|
2805
|
+
ok: true,
|
|
2806
|
+
result: {
|
|
2807
|
+
subagent_id: subagentId,
|
|
2808
|
+
run_id: latestResolution.resolved_run_id,
|
|
2809
|
+
cancelled_active_run: false,
|
|
2810
|
+
reason,
|
|
2811
|
+
cancel_result: latestResolution.cancel_result,
|
|
2812
|
+
cancel_state: latestResolution.cancel_state,
|
|
2813
|
+
cancel_supported: cancelSupported,
|
|
2814
|
+
status: latestResolution.status,
|
|
2815
|
+
message: latestResolution.message,
|
|
2816
|
+
},
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
if (activeRun) {
|
|
2820
|
+
const runMetadata = (activeRun.metadata ?? {});
|
|
2821
|
+
const { error: runUpdateError } = await supabase
|
|
2822
|
+
.from('subagent_runs')
|
|
2823
|
+
.update({
|
|
2824
|
+
metadata: asJson(buildRequestedCancelMetadata({
|
|
2825
|
+
base: runMetadata,
|
|
2826
|
+
run_id: resolvedRunId,
|
|
2827
|
+
reason,
|
|
2828
|
+
requested_at: nowIso,
|
|
2829
|
+
cancel_support: cancelSupportMode,
|
|
2830
|
+
cancellation_scope: cancellationScope,
|
|
2831
|
+
})),
|
|
2832
|
+
})
|
|
2833
|
+
.eq('id', resolvedRunId)
|
|
2834
|
+
.eq('status', 'running');
|
|
2835
|
+
if (runUpdateError) {
|
|
2836
|
+
return {
|
|
2837
|
+
ok: false,
|
|
2838
|
+
result: {
|
|
2839
|
+
message: 'Failed to persist run cancellation metadata',
|
|
2840
|
+
subagent_id: subagentId,
|
|
2841
|
+
run_id: resolvedRunId,
|
|
2842
|
+
},
|
|
2843
|
+
error: runUpdateError.message,
|
|
2844
|
+
};
|
|
2512
2845
|
}
|
|
2513
2846
|
}
|
|
2847
|
+
const { active } = requestSubagentCancel(subagentId, resolvedRunId, reason);
|
|
2848
|
+
if (activeRun) {
|
|
2849
|
+
const nextEventSequence = await getNextSubagentRunEventSequence(supabase, resolvedRunId);
|
|
2850
|
+
await insertSubagentRunEvent({
|
|
2851
|
+
supabase,
|
|
2852
|
+
runId: resolvedRunId,
|
|
2853
|
+
subagentId,
|
|
2854
|
+
teamId: subagent.team_id,
|
|
2855
|
+
gatewayId: subagent.gateway_id,
|
|
2856
|
+
sequence: nextEventSequence,
|
|
2857
|
+
eventType: 'cancellation_requested',
|
|
2858
|
+
payload: {
|
|
2859
|
+
reason,
|
|
2860
|
+
source: 'gateway_cancel_job',
|
|
2861
|
+
active_process: active,
|
|
2862
|
+
cancel_state: 'requested',
|
|
2863
|
+
},
|
|
2864
|
+
});
|
|
2865
|
+
}
|
|
2514
2866
|
return {
|
|
2515
2867
|
ok: true,
|
|
2516
2868
|
result: {
|
|
@@ -2518,10 +2870,17 @@ async function handleSubagentCancelJob(supabase, job) {
|
|
|
2518
2870
|
run_id: resolvedRunId,
|
|
2519
2871
|
cancelled_active_run: active,
|
|
2520
2872
|
reason,
|
|
2873
|
+
cancel_result: 'accepted',
|
|
2874
|
+
cancel_state: 'requested',
|
|
2875
|
+
cancel_supported: cancelSupported,
|
|
2876
|
+
status: resolution.status,
|
|
2877
|
+
message: cancelSupportMode === 'gateway'
|
|
2878
|
+
? 'Cancel requested and forwarded to gateway.'
|
|
2879
|
+
: 'Cancel requested. Worker will stop the run on its next cancellation check.',
|
|
2521
2880
|
},
|
|
2522
2881
|
};
|
|
2523
2882
|
}
|
|
2524
|
-
async function handleSubagentRunJob(supabase, job) {
|
|
2883
|
+
async function handleSubagentRunJob(supabase, config, job) {
|
|
2525
2884
|
const payload = job.payload ?? {};
|
|
2526
2885
|
const subagentId = typeof payload.subagent_id === 'string'
|
|
2527
2886
|
? payload.subagent_id
|
|
@@ -2544,9 +2903,26 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2544
2903
|
return {
|
|
2545
2904
|
ok: false,
|
|
2546
2905
|
result: { message: 'Subagent not found', subagent_id: subagentId },
|
|
2906
|
+
jobStatus: 'cancelled',
|
|
2907
|
+
skipSubagentFailure: true,
|
|
2547
2908
|
error: subagentError?.message ?? 'Subagent not found',
|
|
2548
2909
|
};
|
|
2549
2910
|
}
|
|
2911
|
+
if (subagent.gateway_id !== config.gatewayId) {
|
|
2912
|
+
const message = 'Subagent is assigned to a different gateway';
|
|
2913
|
+
return {
|
|
2914
|
+
ok: false,
|
|
2915
|
+
result: {
|
|
2916
|
+
message,
|
|
2917
|
+
subagent_id: subagentId,
|
|
2918
|
+
assigned_gateway_id: subagent.gateway_id,
|
|
2919
|
+
gateway_id: config.gatewayId,
|
|
2920
|
+
},
|
|
2921
|
+
jobStatus: 'cancelled',
|
|
2922
|
+
skipSubagentFailure: true,
|
|
2923
|
+
error: message,
|
|
2924
|
+
};
|
|
2925
|
+
}
|
|
2550
2926
|
const subagentType = subagent.subagent_type;
|
|
2551
2927
|
const adapter = getSubagentAdapter(subagentType);
|
|
2552
2928
|
if (!adapter) {
|
|
@@ -2572,6 +2948,59 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2572
2948
|
}
|
|
2573
2949
|
const configPayload = (subagent.config ?? {});
|
|
2574
2950
|
const metadata = (subagent.metadata ?? {});
|
|
2951
|
+
const subagentFullControlRequested = parseBooleanInput(configPayload.full_control) ??
|
|
2952
|
+
parseBooleanInput(configPayload.fullControl) ??
|
|
2953
|
+
false;
|
|
2954
|
+
const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
|
|
2955
|
+
const gatewayControlState = await loadGatewayControlState(supabase, config);
|
|
2956
|
+
if (!gatewayControlState) {
|
|
2957
|
+
return {
|
|
2958
|
+
ok: false,
|
|
2959
|
+
result: { message: 'Gateway control state unavailable', subagent_id: subagentId },
|
|
2960
|
+
error: 'Gateway control state unavailable',
|
|
2961
|
+
};
|
|
2962
|
+
}
|
|
2963
|
+
const gatewayHeartbeatFresh = gatewayControlState.status === 'ready' &&
|
|
2964
|
+
isHeartbeatFresh(gatewayControlState.last_seen_at);
|
|
2965
|
+
if (!gatewayHeartbeatFresh) {
|
|
2966
|
+
return {
|
|
2967
|
+
ok: false,
|
|
2968
|
+
result: {
|
|
2969
|
+
message: 'Gateway is offline or heartbeat is stale',
|
|
2970
|
+
subagent_id: subagentId,
|
|
2971
|
+
},
|
|
2972
|
+
error: 'Gateway is offline or heartbeat is stale',
|
|
2973
|
+
};
|
|
2974
|
+
}
|
|
2975
|
+
const adapterId = adapter.id;
|
|
2976
|
+
const providerSupport = PROVIDER_CAPABILITIES?.[adapterId]?.supported_flags ??
|
|
2977
|
+
(adapterId === 'claude_code'
|
|
2978
|
+
? toFlagRecord(CLAUDE_SUPPORT ?? DEFAULT_CLAUDE_SUPPORT)
|
|
2979
|
+
: {});
|
|
2980
|
+
const providerFullControlSupport = evaluateGatewayProviderFullControlSupport(adapterId, providerSupport);
|
|
2981
|
+
const layeredFullControlEnabled = installFullControlEnabled && gatewayControlState.full_control_enabled;
|
|
2982
|
+
const effectiveFullControl = subagentFullControlRequested &&
|
|
2983
|
+
layeredFullControlEnabled &&
|
|
2984
|
+
providerFullControlSupport.supported;
|
|
2985
|
+
let fullControlDeniedReason = null;
|
|
2986
|
+
if (subagentFullControlRequested && !effectiveFullControl) {
|
|
2987
|
+
fullControlDeniedReason = !installFullControlEnabled
|
|
2988
|
+
? 'Gateway install full control is disabled. Run "panorama-gateway full-control enable" on the host.'
|
|
2989
|
+
: !gatewayControlState.full_control_enabled
|
|
2990
|
+
? 'Gateway connection full control is disabled by team settings.'
|
|
2991
|
+
: providerFullControlSupport.reason ?? 'Gateway provider full control is unavailable.';
|
|
2992
|
+
}
|
|
2993
|
+
logInfo('Resolved subagent full-control state', {
|
|
2994
|
+
jobId: job.id,
|
|
2995
|
+
subagentId,
|
|
2996
|
+
providerId: adapterId,
|
|
2997
|
+
fullControlRequested: subagentFullControlRequested,
|
|
2998
|
+
installFullControlEnabled,
|
|
2999
|
+
gatewayFullControlEnabled: gatewayControlState.full_control_enabled,
|
|
3000
|
+
providerFullControlSupported: providerFullControlSupport.supported,
|
|
3001
|
+
fullControlDeniedReason,
|
|
3002
|
+
fullControlEffective: effectiveFullControl,
|
|
3003
|
+
});
|
|
2575
3004
|
const inputPayload = (subagent.input ?? {});
|
|
2576
3005
|
const inputRunId = typeof inputPayload.run_id === 'string' ? inputPayload.run_id.trim() : null;
|
|
2577
3006
|
const rawRunSequence = typeof inputPayload.run_sequence === 'number'
|
|
@@ -2608,10 +3037,33 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2608
3037
|
const runMetadata = {
|
|
2609
3038
|
schema_version: SUBAGENT_SCHEMA_VERSION,
|
|
2610
3039
|
runner: 'gateway',
|
|
2611
|
-
adapter_id:
|
|
3040
|
+
adapter_id: adapterId,
|
|
2612
3041
|
subagent_type: subagentType,
|
|
2613
3042
|
run_id: resolvedRunId,
|
|
2614
3043
|
run_sequence: runSequence,
|
|
3044
|
+
cancel_state: 'none',
|
|
3045
|
+
full_control_requested: subagentFullControlRequested,
|
|
3046
|
+
full_control_effective: effectiveFullControl,
|
|
3047
|
+
full_control_denied_reason: fullControlDeniedReason,
|
|
3048
|
+
};
|
|
3049
|
+
let nextEventSequence = null;
|
|
3050
|
+
const emitRunEvent = async (eventType, eventPayload) => {
|
|
3051
|
+
if (nextEventSequence == null) {
|
|
3052
|
+
nextEventSequence = await getNextSubagentRunEventSequence(supabase, resolvedRunId);
|
|
3053
|
+
}
|
|
3054
|
+
const insertedSequence = await insertSubagentRunEvent({
|
|
3055
|
+
supabase,
|
|
3056
|
+
runId: resolvedRunId,
|
|
3057
|
+
subagentId,
|
|
3058
|
+
teamId: subagent.team_id,
|
|
3059
|
+
gatewayId: subagent.gateway_id,
|
|
3060
|
+
sequence: nextEventSequence,
|
|
3061
|
+
eventType,
|
|
3062
|
+
payload: eventPayload,
|
|
3063
|
+
});
|
|
3064
|
+
if (typeof insertedSequence === 'number') {
|
|
3065
|
+
nextEventSequence = insertedSequence + 1;
|
|
3066
|
+
}
|
|
2615
3067
|
};
|
|
2616
3068
|
const pendingCancelReason = resolvePendingCancel(subagentId, resolvedRunId, metadata);
|
|
2617
3069
|
if (pendingCancelReason) {
|
|
@@ -2633,6 +3085,10 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2633
3085
|
metadata: asJson({
|
|
2634
3086
|
...runMetadata,
|
|
2635
3087
|
cancel_reason: pendingCancelReason,
|
|
3088
|
+
cancel_support: 'gateway',
|
|
3089
|
+
cancel_state: 'acknowledged',
|
|
3090
|
+
cancellation_scope: 'gateway',
|
|
3091
|
+
cancelled: true,
|
|
2636
3092
|
}),
|
|
2637
3093
|
});
|
|
2638
3094
|
if (cancelInsertError) {
|
|
@@ -2642,6 +3098,21 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2642
3098
|
error: cancelInsertError.message,
|
|
2643
3099
|
};
|
|
2644
3100
|
}
|
|
3101
|
+
await emitRunEvent('cancellation_requested', {
|
|
3102
|
+
reason: pendingCancelReason,
|
|
3103
|
+
source: 'pending_cancel',
|
|
3104
|
+
stage: 'before_start',
|
|
3105
|
+
});
|
|
3106
|
+
await emitRunEvent('cancellation_acknowledged', {
|
|
3107
|
+
reason: pendingCancelReason,
|
|
3108
|
+
source: 'gateway',
|
|
3109
|
+
cancellation_scope: 'gateway',
|
|
3110
|
+
});
|
|
3111
|
+
await emitRunEvent('run_cancelled', {
|
|
3112
|
+
reason: pendingCancelReason,
|
|
3113
|
+
source: 'gateway',
|
|
3114
|
+
cancellation_scope: 'gateway',
|
|
3115
|
+
});
|
|
2645
3116
|
return {
|
|
2646
3117
|
ok: true,
|
|
2647
3118
|
result: {
|
|
@@ -2664,6 +3135,41 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2664
3135
|
metadata: asJson(runMetadata),
|
|
2665
3136
|
});
|
|
2666
3137
|
if (runInsertError) {
|
|
3138
|
+
if (runInsertError.code === '23505') {
|
|
3139
|
+
const { data: existingRunningRun, error: existingRunningRunError } = await supabase
|
|
3140
|
+
.from('subagent_runs')
|
|
3141
|
+
.select('id, status')
|
|
3142
|
+
.eq('subagent_id', subagentId)
|
|
3143
|
+
.eq('status', 'running')
|
|
3144
|
+
.order('run_sequence', { ascending: false })
|
|
3145
|
+
.limit(1)
|
|
3146
|
+
.maybeSingle();
|
|
3147
|
+
if (existingRunningRunError) {
|
|
3148
|
+
return {
|
|
3149
|
+
ok: false,
|
|
3150
|
+
result: {
|
|
3151
|
+
message: 'Failed to resolve conflicting subagent run after insert conflict',
|
|
3152
|
+
subagent_id: subagentId,
|
|
3153
|
+
run_id: resolvedRunId,
|
|
3154
|
+
},
|
|
3155
|
+
error: existingRunningRunError.message,
|
|
3156
|
+
};
|
|
3157
|
+
}
|
|
3158
|
+
if (existingRunningRun?.id && existingRunningRun.id !== resolvedRunId) {
|
|
3159
|
+
return {
|
|
3160
|
+
ok: false,
|
|
3161
|
+
result: {
|
|
3162
|
+
message: 'Skipped duplicate subagent job because another run is already active',
|
|
3163
|
+
subagent_id: subagentId,
|
|
3164
|
+
run_id: resolvedRunId,
|
|
3165
|
+
active_run_id: existingRunningRun.id,
|
|
3166
|
+
},
|
|
3167
|
+
jobStatus: 'cancelled',
|
|
3168
|
+
skipSubagentFailure: true,
|
|
3169
|
+
error: 'Another subagent run is already active',
|
|
3170
|
+
};
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
2667
3173
|
return {
|
|
2668
3174
|
ok: false,
|
|
2669
3175
|
result: { message: 'Failed to create subagent run', subagent_id: subagentId },
|
|
@@ -2675,7 +3181,12 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2675
3181
|
cancelled: false,
|
|
2676
3182
|
};
|
|
2677
3183
|
activeSubagentRuns.set(subagentId, activeRun);
|
|
2678
|
-
|
|
3184
|
+
const latePendingCancelReason = consumePendingCancelFromQueue(subagentId, resolvedRunId);
|
|
3185
|
+
if (latePendingCancelReason) {
|
|
3186
|
+
activeRun.cancelled = true;
|
|
3187
|
+
activeRun.cancelReason = latePendingCancelReason;
|
|
3188
|
+
}
|
|
3189
|
+
const { error: startSubagentUpdateError } = await supabase
|
|
2679
3190
|
.from('subagents')
|
|
2680
3191
|
.update({
|
|
2681
3192
|
status: 'running',
|
|
@@ -2686,41 +3197,94 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2686
3197
|
})),
|
|
2687
3198
|
})
|
|
2688
3199
|
.eq('id', subagentId);
|
|
2689
|
-
|
|
3200
|
+
if (startSubagentUpdateError) {
|
|
3201
|
+
activeSubagentRuns.delete(subagentId);
|
|
3202
|
+
return {
|
|
3203
|
+
ok: false,
|
|
3204
|
+
result: {
|
|
3205
|
+
message: 'Failed to mark subagent as running',
|
|
3206
|
+
subagent_id: subagentId,
|
|
3207
|
+
run_id: resolvedRunId,
|
|
3208
|
+
},
|
|
3209
|
+
error: startSubagentUpdateError.message,
|
|
3210
|
+
};
|
|
3211
|
+
}
|
|
3212
|
+
await emitRunEvent('run_started', {
|
|
3213
|
+
runner: 'gateway',
|
|
3214
|
+
adapter_id: adapterId,
|
|
3215
|
+
subagent_type: subagentType,
|
|
3216
|
+
run_id: resolvedRunId,
|
|
3217
|
+
run_sequence: runSequence,
|
|
3218
|
+
});
|
|
3219
|
+
const failRun = async (message, options) => {
|
|
2690
3220
|
const failureOutput = buildErrorOutput(message);
|
|
2691
3221
|
const completedAt = new Date().toISOString();
|
|
2692
|
-
|
|
3222
|
+
const metadataUpdate = mergeMetadata(runMetadata, options?.metadata ?? {});
|
|
3223
|
+
const { error: runFailureUpdateError } = await supabase
|
|
2693
3224
|
.from('subagent_runs')
|
|
2694
3225
|
.update({
|
|
2695
3226
|
status: 'failed',
|
|
2696
3227
|
output: asJson(failureOutput),
|
|
2697
3228
|
error: message,
|
|
2698
3229
|
completed_at: completedAt,
|
|
3230
|
+
metadata: asJson(metadataUpdate),
|
|
2699
3231
|
})
|
|
2700
3232
|
.eq('id', resolvedRunId);
|
|
2701
|
-
|
|
3233
|
+
if (runFailureUpdateError) {
|
|
3234
|
+
logError('Failed to persist failed subagent run state', {
|
|
3235
|
+
subagentId,
|
|
3236
|
+
runId: resolvedRunId,
|
|
3237
|
+
error: runFailureUpdateError.message,
|
|
3238
|
+
});
|
|
3239
|
+
}
|
|
3240
|
+
const { error: subagentFailureUpdateError } = await supabase
|
|
2702
3241
|
.from('subagents')
|
|
2703
3242
|
.update({
|
|
2704
3243
|
status: 'failed',
|
|
2705
3244
|
output: asJson(failureOutput),
|
|
2706
3245
|
error: message,
|
|
2707
3246
|
completed_at: completedAt,
|
|
3247
|
+
metadata: asJson(mergeMetadata(metadata, options?.metadata ?? {})),
|
|
2708
3248
|
})
|
|
2709
3249
|
.eq('id', subagentId);
|
|
3250
|
+
if (subagentFailureUpdateError) {
|
|
3251
|
+
logError('Failed to persist failed subagent state', {
|
|
3252
|
+
subagentId,
|
|
3253
|
+
runId: resolvedRunId,
|
|
3254
|
+
error: subagentFailureUpdateError.message,
|
|
3255
|
+
});
|
|
3256
|
+
}
|
|
3257
|
+
await emitRunEvent('run_failed', {
|
|
3258
|
+
reason: message,
|
|
3259
|
+
source: 'gateway',
|
|
3260
|
+
stage: 'pre_execution',
|
|
3261
|
+
});
|
|
2710
3262
|
activeSubagentRuns.delete(subagentId);
|
|
3263
|
+
const persistenceErrors = [
|
|
3264
|
+
runFailureUpdateError ? `run update: ${runFailureUpdateError.message}` : null,
|
|
3265
|
+
subagentFailureUpdateError ? `subagent update: ${subagentFailureUpdateError.message}` : null,
|
|
3266
|
+
].filter((value) => !!value);
|
|
3267
|
+
const error = persistenceErrors.length > 0
|
|
3268
|
+
? `${message} (persistence errors: ${persistenceErrors.join('; ')})`
|
|
3269
|
+
: message;
|
|
2711
3270
|
return {
|
|
2712
3271
|
ok: false,
|
|
2713
3272
|
result: { message, subagent_id: subagentId },
|
|
2714
|
-
error
|
|
3273
|
+
error,
|
|
2715
3274
|
};
|
|
2716
3275
|
};
|
|
3276
|
+
if (subagentFullControlRequested && !effectiveFullControl) {
|
|
3277
|
+
const deniedReason = fullControlDeniedReason ?? 'Gateway full control is unavailable.';
|
|
3278
|
+
return await failRun(deniedReason, {
|
|
3279
|
+
metadata: {
|
|
3280
|
+
full_control_requested: true,
|
|
3281
|
+
full_control_effective: false,
|
|
3282
|
+
full_control_denied_reason: deniedReason,
|
|
3283
|
+
},
|
|
3284
|
+
});
|
|
3285
|
+
}
|
|
2717
3286
|
let runPlan;
|
|
2718
3287
|
try {
|
|
2719
|
-
const adapterId = adapter.id;
|
|
2720
|
-
const providerSupport = PROVIDER_CAPABILITIES?.[adapterId]?.supported_flags ??
|
|
2721
|
-
(adapterId === 'claude_code'
|
|
2722
|
-
? toFlagRecord(CLAUDE_SUPPORT ?? DEFAULT_CLAUDE_SUPPORT)
|
|
2723
|
-
: {});
|
|
2724
3288
|
const command = adapterId === 'codex'
|
|
2725
3289
|
? resolveCodexCommand()
|
|
2726
3290
|
: adapterId === 'gemini'
|
|
@@ -2731,6 +3295,7 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2731
3295
|
config: configPayload,
|
|
2732
3296
|
command,
|
|
2733
3297
|
support: providerSupport,
|
|
3298
|
+
fullControl: effectiveFullControl,
|
|
2734
3299
|
resumeSessionId,
|
|
2735
3300
|
});
|
|
2736
3301
|
}
|
|
@@ -2764,6 +3329,11 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2764
3329
|
activeResult = null;
|
|
2765
3330
|
normalized = null;
|
|
2766
3331
|
let executionError = null;
|
|
3332
|
+
await emitRunEvent('run_progress', {
|
|
3333
|
+
phase: 'attempt_started',
|
|
3334
|
+
attempt,
|
|
3335
|
+
max_attempts: retryConfig.maxAttempts,
|
|
3336
|
+
});
|
|
2767
3337
|
try {
|
|
2768
3338
|
if (runPlan.outputFormat === 'stream-json') {
|
|
2769
3339
|
streamResult = await runSubagentCommandStreaming(runPlan.command, runPlan.args, {
|
|
@@ -2773,12 +3343,11 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2773
3343
|
onStart: (child) => {
|
|
2774
3344
|
activeRun.child = child;
|
|
2775
3345
|
if (activeRun.cancelled) {
|
|
2776
|
-
child
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
}, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
|
|
3346
|
+
requestChildTerminationWithLogging(child, {
|
|
3347
|
+
reason: activeRun.cancelReason ?? 'Cancelled',
|
|
3348
|
+
graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
|
|
3349
|
+
context: 'subagent_run_streaming_on_start',
|
|
3350
|
+
});
|
|
2782
3351
|
}
|
|
2783
3352
|
},
|
|
2784
3353
|
});
|
|
@@ -2791,12 +3360,11 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2791
3360
|
onStart: (child) => {
|
|
2792
3361
|
activeRun.child = child;
|
|
2793
3362
|
if (activeRun.cancelled) {
|
|
2794
|
-
child
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
}, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
|
|
3363
|
+
requestChildTerminationWithLogging(child, {
|
|
3364
|
+
reason: activeRun.cancelReason ?? 'Cancelled',
|
|
3365
|
+
graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
|
|
3366
|
+
context: 'subagent_run_on_start',
|
|
3367
|
+
});
|
|
2800
3368
|
}
|
|
2801
3369
|
},
|
|
2802
3370
|
});
|
|
@@ -2866,6 +3434,13 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2866
3434
|
reason: retryDecision.reason,
|
|
2867
3435
|
delay_ms: delayMs,
|
|
2868
3436
|
});
|
|
3437
|
+
await emitRunEvent('run_progress', {
|
|
3438
|
+
phase: 'retry_scheduled',
|
|
3439
|
+
attempt,
|
|
3440
|
+
max_attempts: retryConfig.maxAttempts,
|
|
3441
|
+
reason: retryDecision.reason,
|
|
3442
|
+
delay_ms: delayMs,
|
|
3443
|
+
});
|
|
2869
3444
|
await sleep(delayMs);
|
|
2870
3445
|
continue;
|
|
2871
3446
|
}
|
|
@@ -2898,6 +3473,7 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2898
3473
|
const completedAt = new Date().toISOString();
|
|
2899
3474
|
const parseStrict = runPlan.outputFormat === 'json';
|
|
2900
3475
|
const shouldFail = !activeResult.ok || (parseStrict && !!parseError);
|
|
3476
|
+
const effectiveShouldFail = !wasCancelled && shouldFail;
|
|
2901
3477
|
const stderrMessage = activeResult.stderr && activeResult.stderr.trim().length > 0
|
|
2902
3478
|
? activeResult.stderr.trim()
|
|
2903
3479
|
: null;
|
|
@@ -2909,7 +3485,7 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2909
3485
|
: parseError ?? null;
|
|
2910
3486
|
const providerId = adapter.id;
|
|
2911
3487
|
if (!wasCancelled) {
|
|
2912
|
-
if (
|
|
3488
|
+
if (effectiveShouldFail) {
|
|
2913
3489
|
const stderrInfo = truncateLog(activeResult.stderr ?? '');
|
|
2914
3490
|
const stdoutInfo = truncateLog(activeResult.stdout ?? '');
|
|
2915
3491
|
logError('Subagent run failed', {
|
|
@@ -2954,6 +3530,9 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2954
3530
|
}
|
|
2955
3531
|
}
|
|
2956
3532
|
const failureOutput = buildErrorOutput(wasCancelled ? cancelReason : errorMessage ?? 'Subagent run failed');
|
|
3533
|
+
const finalStatus = wasCancelled ? 'cancelled' : effectiveShouldFail ? 'failed' : 'completed';
|
|
3534
|
+
const finalOutput = wasCancelled || effectiveShouldFail ? failureOutput : output;
|
|
3535
|
+
const finalError = wasCancelled ? cancelReason : effectiveShouldFail ? errorMessage : null;
|
|
2957
3536
|
const metadataUpdate = {
|
|
2958
3537
|
schema_version: SUBAGENT_SCHEMA_VERSION,
|
|
2959
3538
|
runner: 'gateway',
|
|
@@ -2977,6 +3556,12 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2977
3556
|
cancel_requested_at: null,
|
|
2978
3557
|
cancel_reason: wasCancelled ? cancelReason : null,
|
|
2979
3558
|
cancelled: wasCancelled,
|
|
3559
|
+
cancel_state: wasCancelled ? 'acknowledged' : 'none',
|
|
3560
|
+
cancel_support: wasCancelled ? 'gateway' : null,
|
|
3561
|
+
cancellation_scope: wasCancelled ? 'gateway' : null,
|
|
3562
|
+
full_control_requested: subagentFullControlRequested,
|
|
3563
|
+
full_control_effective: effectiveFullControl,
|
|
3564
|
+
full_control_denied_reason: null,
|
|
2980
3565
|
};
|
|
2981
3566
|
if (activeResult.stderr) {
|
|
2982
3567
|
metadataUpdate.stderr = activeResult.stderr;
|
|
@@ -2985,69 +3570,95 @@ async function handleSubagentRunJob(supabase, job) {
|
|
|
2985
3570
|
const runMetadataUpdate = mergeMetadata(mergeMetadata(runMetadata, normalized.metadata), metadataUpdate);
|
|
2986
3571
|
if (streamResult?.events?.length && adapter.normalizeStreamEvents) {
|
|
2987
3572
|
const normalizedEvents = adapter.normalizeStreamEvents(streamResult.events);
|
|
2988
|
-
const
|
|
2989
|
-
|
|
2990
|
-
subagent_id: subagentId,
|
|
2991
|
-
team_id: subagent.team_id,
|
|
2992
|
-
gateway_id: subagent.gateway_id,
|
|
2993
|
-
sequence: index,
|
|
2994
|
-
event_type: entry.event_type,
|
|
2995
|
-
payload: asJson(trimEventPayload(entry.payload)),
|
|
2996
|
-
}));
|
|
2997
|
-
for (let i = 0; i < eventRows.length; i += SUBAGENT_EVENT_BATCH_SIZE) {
|
|
2998
|
-
const chunk = eventRows.slice(i, i + SUBAGENT_EVENT_BATCH_SIZE);
|
|
2999
|
-
const { error: eventsError } = await supabase
|
|
3000
|
-
.from('subagent_run_events')
|
|
3001
|
-
.insert(chunk);
|
|
3002
|
-
if (eventsError) {
|
|
3003
|
-
logError('Failed to insert subagent run events', {
|
|
3004
|
-
error: eventsError.message,
|
|
3005
|
-
subagentId,
|
|
3006
|
-
runId: resolvedRunId,
|
|
3007
|
-
});
|
|
3008
|
-
}
|
|
3573
|
+
for (const entry of normalizedEvents) {
|
|
3574
|
+
await emitRunEvent(entry.event_type, entry.payload);
|
|
3009
3575
|
}
|
|
3010
3576
|
}
|
|
3011
|
-
|
|
3577
|
+
if (wasCancelled) {
|
|
3578
|
+
await emitRunEvent('cancellation_acknowledged', {
|
|
3579
|
+
reason: cancelReason,
|
|
3580
|
+
source: 'gateway',
|
|
3581
|
+
cancellation_scope: 'gateway',
|
|
3582
|
+
});
|
|
3583
|
+
await emitRunEvent('run_cancelled', {
|
|
3584
|
+
reason: cancelReason,
|
|
3585
|
+
source: 'gateway',
|
|
3586
|
+
cancellation_scope: 'gateway',
|
|
3587
|
+
duration_ms: activeResult.durationMs,
|
|
3588
|
+
});
|
|
3589
|
+
}
|
|
3590
|
+
else if (effectiveShouldFail) {
|
|
3591
|
+
await emitRunEvent('run_failed', {
|
|
3592
|
+
reason: errorMessage ?? 'Subagent run failed',
|
|
3593
|
+
duration_ms: activeResult.durationMs,
|
|
3594
|
+
parse_error: parseError,
|
|
3595
|
+
});
|
|
3596
|
+
}
|
|
3597
|
+
else {
|
|
3598
|
+
await emitRunEvent('run_completed', {
|
|
3599
|
+
duration_ms: activeResult.durationMs,
|
|
3600
|
+
parse_error: parseError,
|
|
3601
|
+
retry_count: retryCount,
|
|
3602
|
+
});
|
|
3603
|
+
}
|
|
3604
|
+
const { error: runFinalizeError } = await supabase
|
|
3012
3605
|
.from('subagent_runs')
|
|
3013
3606
|
.update({
|
|
3014
|
-
status:
|
|
3015
|
-
output: asJson(
|
|
3016
|
-
error:
|
|
3607
|
+
status: finalStatus,
|
|
3608
|
+
output: asJson(finalOutput),
|
|
3609
|
+
error: finalError,
|
|
3017
3610
|
completed_at: completedAt,
|
|
3018
3611
|
metadata: asJson(runMetadataUpdate),
|
|
3019
3612
|
})
|
|
3020
3613
|
.eq('id', resolvedRunId);
|
|
3614
|
+
if (runFinalizeError) {
|
|
3615
|
+
activeSubagentRuns.delete(subagentId);
|
|
3616
|
+
return {
|
|
3617
|
+
ok: false,
|
|
3618
|
+
result: {
|
|
3619
|
+
message: 'Failed to persist terminal subagent run state',
|
|
3620
|
+
subagent_id: subagentId,
|
|
3621
|
+
run_id: resolvedRunId,
|
|
3622
|
+
},
|
|
3623
|
+
error: runFinalizeError.message,
|
|
3624
|
+
};
|
|
3625
|
+
}
|
|
3021
3626
|
const { error: subagentUpdateError } = await supabase
|
|
3022
3627
|
.from('subagents')
|
|
3023
3628
|
.update({
|
|
3024
|
-
status:
|
|
3025
|
-
output: asJson(
|
|
3026
|
-
error:
|
|
3629
|
+
status: finalStatus,
|
|
3630
|
+
output: asJson(finalOutput),
|
|
3631
|
+
error: finalError,
|
|
3027
3632
|
completed_at: completedAt,
|
|
3028
3633
|
metadata: asJson(mergedMetadata),
|
|
3029
3634
|
})
|
|
3030
3635
|
.eq('id', subagentId);
|
|
3031
3636
|
if (subagentUpdateError) {
|
|
3032
|
-
|
|
3637
|
+
activeSubagentRuns.delete(subagentId);
|
|
3638
|
+
return {
|
|
3639
|
+
ok: false,
|
|
3640
|
+
result: {
|
|
3641
|
+
message: 'Failed to update subagent after run',
|
|
3642
|
+
subagent_id: subagentId,
|
|
3643
|
+
run_id: resolvedRunId,
|
|
3644
|
+
},
|
|
3033
3645
|
error: subagentUpdateError.message,
|
|
3034
|
-
|
|
3035
|
-
runId: resolvedRunId,
|
|
3036
|
-
});
|
|
3646
|
+
};
|
|
3037
3647
|
}
|
|
3038
3648
|
activeSubagentRuns.delete(subagentId);
|
|
3039
3649
|
return {
|
|
3040
|
-
ok: !
|
|
3650
|
+
ok: !effectiveShouldFail,
|
|
3041
3651
|
result: {
|
|
3042
3652
|
subagent_id: subagentId,
|
|
3043
|
-
status:
|
|
3044
|
-
output:
|
|
3045
|
-
error:
|
|
3653
|
+
status: finalStatus,
|
|
3654
|
+
output: effectiveShouldFail ? null : finalOutput,
|
|
3655
|
+
error: finalError,
|
|
3046
3656
|
},
|
|
3047
|
-
error:
|
|
3657
|
+
error: effectiveShouldFail ? errorMessage ?? 'Subagent run failed' : undefined,
|
|
3048
3658
|
};
|
|
3049
3659
|
}
|
|
3050
3660
|
finally {
|
|
3661
|
+
activeSubagentRuns.delete(subagentId);
|
|
3051
3662
|
if (workDir) {
|
|
3052
3663
|
await cleanupWorkDir(workDir, workRoot);
|
|
3053
3664
|
}
|
|
@@ -3111,6 +3722,10 @@ async function handleModelRunJob(_supabase, job) {
|
|
|
3111
3722
|
});
|
|
3112
3723
|
const workRoot = resolveModelWorkdirRoot();
|
|
3113
3724
|
let workDir = '';
|
|
3725
|
+
const activeModelRun = {
|
|
3726
|
+
shutdownRequested: false,
|
|
3727
|
+
};
|
|
3728
|
+
activeModelRuns.set(job.id, activeModelRun);
|
|
3114
3729
|
try {
|
|
3115
3730
|
workDir = await createWorkDir(workRoot, [{ value: job.id, label: 'job id' }]);
|
|
3116
3731
|
assertSafeOutputPath(workDir, runPlan.outputPath);
|
|
@@ -3137,6 +3752,16 @@ async function handleModelRunJob(_supabase, job) {
|
|
|
3137
3752
|
timeoutMs: runPlan.timeoutMs ?? DEFAULT_MODEL_RUN_TIMEOUT_MS,
|
|
3138
3753
|
cwd: workDir,
|
|
3139
3754
|
env: resolvedEnv,
|
|
3755
|
+
onStart: (child) => {
|
|
3756
|
+
activeModelRun.child = child;
|
|
3757
|
+
if (activeModelRun.shutdownRequested) {
|
|
3758
|
+
requestChildTerminationWithLogging(child, {
|
|
3759
|
+
reason: activeModelRun.shutdownReason ?? 'Gateway shutting down',
|
|
3760
|
+
graceMs: PROCESS_KILL_GRACE_MS,
|
|
3761
|
+
context: 'model_run_shutdown',
|
|
3762
|
+
});
|
|
3763
|
+
}
|
|
3764
|
+
},
|
|
3140
3765
|
});
|
|
3141
3766
|
const stdoutInfo = truncateLog(runResult.stdout);
|
|
3142
3767
|
const stderrInfo = truncateLog(runResult.stderr);
|
|
@@ -3214,11 +3839,620 @@ async function handleModelRunJob(_supabase, job) {
|
|
|
3214
3839
|
};
|
|
3215
3840
|
}
|
|
3216
3841
|
finally {
|
|
3842
|
+
activeModelRuns.delete(job.id);
|
|
3217
3843
|
if (workDir) {
|
|
3218
3844
|
await cleanupWorkDir(workDir, workRoot);
|
|
3219
3845
|
}
|
|
3220
3846
|
}
|
|
3221
3847
|
}
|
|
3848
|
+
function normalizeCommandArgs(value) {
|
|
3849
|
+
if (value === undefined || value === null)
|
|
3850
|
+
return [];
|
|
3851
|
+
if (!Array.isArray(value))
|
|
3852
|
+
return null;
|
|
3853
|
+
const normalized = [];
|
|
3854
|
+
for (const item of value) {
|
|
3855
|
+
if (typeof item !== 'string')
|
|
3856
|
+
return null;
|
|
3857
|
+
normalized.push(item);
|
|
3858
|
+
}
|
|
3859
|
+
return normalized;
|
|
3860
|
+
}
|
|
3861
|
+
function normalizeCommandEnv(value) {
|
|
3862
|
+
if (value === undefined || value === null)
|
|
3863
|
+
return {};
|
|
3864
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
3865
|
+
return null;
|
|
3866
|
+
const normalized = {};
|
|
3867
|
+
for (const [key, raw] of Object.entries(value)) {
|
|
3868
|
+
if (typeof raw === 'string') {
|
|
3869
|
+
normalized[key] = raw;
|
|
3870
|
+
continue;
|
|
3871
|
+
}
|
|
3872
|
+
if (typeof raw === 'number' || typeof raw === 'boolean') {
|
|
3873
|
+
normalized[key] = String(raw);
|
|
3874
|
+
continue;
|
|
3875
|
+
}
|
|
3876
|
+
return null;
|
|
3877
|
+
}
|
|
3878
|
+
return normalized;
|
|
3879
|
+
}
|
|
3880
|
+
function coercePositiveInteger(value, fallback, options) {
|
|
3881
|
+
const min = options?.min ?? 1;
|
|
3882
|
+
const max = options?.max ?? Number.MAX_SAFE_INTEGER;
|
|
3883
|
+
const parsed = typeof value === 'number'
|
|
3884
|
+
? value
|
|
3885
|
+
: typeof value === 'string'
|
|
3886
|
+
? Number.parseInt(value, 10)
|
|
3887
|
+
: NaN;
|
|
3888
|
+
if (!Number.isFinite(parsed))
|
|
3889
|
+
return fallback;
|
|
3890
|
+
return Math.max(min, Math.min(max, Math.trunc(parsed)));
|
|
3891
|
+
}
|
|
3892
|
+
function coerceNonNegativeInteger(value, fallback, options) {
|
|
3893
|
+
const max = options?.max ?? Number.MAX_SAFE_INTEGER;
|
|
3894
|
+
const parsed = typeof value === 'number'
|
|
3895
|
+
? value
|
|
3896
|
+
: typeof value === 'string'
|
|
3897
|
+
? Number.parseInt(value, 10)
|
|
3898
|
+
: NaN;
|
|
3899
|
+
if (!Number.isFinite(parsed))
|
|
3900
|
+
return fallback;
|
|
3901
|
+
return Math.max(0, Math.min(max, Math.trunc(parsed)));
|
|
3902
|
+
}
|
|
3903
|
+
function getRunningShellSessionCount() {
|
|
3904
|
+
let count = 0;
|
|
3905
|
+
for (const session of activeShellSessions.values()) {
|
|
3906
|
+
if (session.status === 'running')
|
|
3907
|
+
count += 1;
|
|
3908
|
+
}
|
|
3909
|
+
return count;
|
|
3910
|
+
}
|
|
3911
|
+
function estimateShellEventBytes(event) {
|
|
3912
|
+
return Buffer.byteLength(event.data, 'utf-8') + 64;
|
|
3913
|
+
}
|
|
3914
|
+
function appendShellSessionEvent(session, stream, data, timestamp = new Date().toISOString()) {
|
|
3915
|
+
if (!data)
|
|
3916
|
+
return;
|
|
3917
|
+
const event = {
|
|
3918
|
+
sequence: session.nextSequence,
|
|
3919
|
+
stream,
|
|
3920
|
+
data,
|
|
3921
|
+
timestamp,
|
|
3922
|
+
};
|
|
3923
|
+
session.nextSequence += 1;
|
|
3924
|
+
session.events.push(event);
|
|
3925
|
+
session.bufferedBytes += estimateShellEventBytes(event);
|
|
3926
|
+
session.updatedAt = timestamp;
|
|
3927
|
+
while (session.events.length > MAX_SHELL_SESSION_EVENTS ||
|
|
3928
|
+
session.bufferedBytes > MAX_SHELL_SESSION_BUFFER_BYTES) {
|
|
3929
|
+
const removed = session.events.shift();
|
|
3930
|
+
if (!removed)
|
|
3931
|
+
break;
|
|
3932
|
+
session.bufferedBytes = Math.max(0, session.bufferedBytes - estimateShellEventBytes(removed));
|
|
3933
|
+
}
|
|
3934
|
+
}
|
|
3935
|
+
function readShellSessionEvents(session, options) {
|
|
3936
|
+
const sinceSequence = options?.sinceSequence ?? 0;
|
|
3937
|
+
const maxEvents = options?.maxEvents ?? 200;
|
|
3938
|
+
const pending = session.events.filter((event) => event.sequence > sinceSequence);
|
|
3939
|
+
const events = pending.slice(0, maxEvents);
|
|
3940
|
+
return {
|
|
3941
|
+
events,
|
|
3942
|
+
hasMore: pending.length > events.length,
|
|
3943
|
+
latestSequence: session.nextSequence > 0 ? session.nextSequence - 1 : 0,
|
|
3944
|
+
};
|
|
3945
|
+
}
|
|
3946
|
+
function purgeExpiredShellSessions(nowMs = Date.now()) {
|
|
3947
|
+
for (const [sessionId, session] of activeShellSessions.entries()) {
|
|
3948
|
+
if (session.status === 'running')
|
|
3949
|
+
continue;
|
|
3950
|
+
const updatedAtMs = Date.parse(session.updatedAt);
|
|
3951
|
+
if (!Number.isFinite(updatedAtMs)) {
|
|
3952
|
+
activeShellSessions.delete(sessionId);
|
|
3953
|
+
continue;
|
|
3954
|
+
}
|
|
3955
|
+
if (nowMs - updatedAtMs >= SHELL_SESSION_RETENTION_MS) {
|
|
3956
|
+
activeShellSessions.delete(sessionId);
|
|
3957
|
+
}
|
|
3958
|
+
}
|
|
3959
|
+
}
|
|
3960
|
+
async function validateMachineControlState(supabase, config) {
|
|
3961
|
+
const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
|
|
3962
|
+
const gatewayControlState = await loadGatewayControlState(supabase, config);
|
|
3963
|
+
if (!gatewayControlState) {
|
|
3964
|
+
return {
|
|
3965
|
+
ok: false,
|
|
3966
|
+
message: 'Gateway control state unavailable',
|
|
3967
|
+
};
|
|
3968
|
+
}
|
|
3969
|
+
if (!installFullControlEnabled) {
|
|
3970
|
+
return {
|
|
3971
|
+
ok: false,
|
|
3972
|
+
message: 'Gateway install full control is disabled. Run "panorama-gateway full-control enable" on the host.',
|
|
3973
|
+
};
|
|
3974
|
+
}
|
|
3975
|
+
if (!gatewayControlState.full_control_enabled) {
|
|
3976
|
+
return {
|
|
3977
|
+
ok: false,
|
|
3978
|
+
message: 'Gateway connection full control is disabled by team settings.',
|
|
3979
|
+
};
|
|
3980
|
+
}
|
|
3981
|
+
if (gatewayControlState.status !== 'ready' ||
|
|
3982
|
+
!isHeartbeatFresh(gatewayControlState.last_seen_at)) {
|
|
3983
|
+
return {
|
|
3984
|
+
ok: false,
|
|
3985
|
+
message: 'Gateway is offline or heartbeat is stale',
|
|
3986
|
+
};
|
|
3987
|
+
}
|
|
3988
|
+
return { ok: true };
|
|
3989
|
+
}
|
|
3990
|
+
function normalizeShellSessionAction(value) {
|
|
3991
|
+
if (typeof value !== 'string')
|
|
3992
|
+
return null;
|
|
3993
|
+
const normalized = value.trim().toLowerCase();
|
|
3994
|
+
if (normalized === 'open' ||
|
|
3995
|
+
normalized === 'write' ||
|
|
3996
|
+
normalized === 'read' ||
|
|
3997
|
+
normalized === 'close' ||
|
|
3998
|
+
normalized === 'status') {
|
|
3999
|
+
return normalized;
|
|
4000
|
+
}
|
|
4001
|
+
return null;
|
|
4002
|
+
}
|
|
4003
|
+
function resolveDefaultShellCommand() {
|
|
4004
|
+
if (typeof process.env.SHELL === 'string' && process.env.SHELL.trim().length > 0) {
|
|
4005
|
+
return process.env.SHELL.trim();
|
|
4006
|
+
}
|
|
4007
|
+
return '/bin/bash';
|
|
4008
|
+
}
|
|
4009
|
+
function buildShellSessionSummary(session) {
|
|
4010
|
+
return {
|
|
4011
|
+
session_id: session.id,
|
|
4012
|
+
resource_id: session.resourceId,
|
|
4013
|
+
status: session.status,
|
|
4014
|
+
shell: session.shell,
|
|
4015
|
+
shell_args: session.shellArgs,
|
|
4016
|
+
cwd: session.cwd,
|
|
4017
|
+
created_at: session.createdAt,
|
|
4018
|
+
updated_at: session.updatedAt,
|
|
4019
|
+
exit_code: session.exitCode,
|
|
4020
|
+
error: session.error,
|
|
4021
|
+
latest_sequence: session.nextSequence > 0 ? session.nextSequence - 1 : 0,
|
|
4022
|
+
};
|
|
4023
|
+
}
|
|
4024
|
+
function cleanupShellSession(sessionId, reason) {
|
|
4025
|
+
const session = activeShellSessions.get(sessionId);
|
|
4026
|
+
if (!session)
|
|
4027
|
+
return;
|
|
4028
|
+
if (session.child) {
|
|
4029
|
+
requestChildTerminationWithLogging(session.child, {
|
|
4030
|
+
reason,
|
|
4031
|
+
graceMs: PROCESS_KILL_GRACE_MS,
|
|
4032
|
+
context: 'shell_session_cleanup',
|
|
4033
|
+
});
|
|
4034
|
+
session.child = undefined;
|
|
4035
|
+
}
|
|
4036
|
+
session.status = session.status === 'running' ? 'closed' : session.status;
|
|
4037
|
+
session.updatedAt = new Date().toISOString();
|
|
4038
|
+
}
|
|
4039
|
+
async function loadGatewayJobStatus(supabase, jobId) {
|
|
4040
|
+
const { data, error } = await supabase
|
|
4041
|
+
.from('gateway_jobs')
|
|
4042
|
+
.select('status')
|
|
4043
|
+
.eq('id', jobId)
|
|
4044
|
+
.maybeSingle();
|
|
4045
|
+
if (error) {
|
|
4046
|
+
logError('Failed to fetch gateway job status', {
|
|
4047
|
+
jobId,
|
|
4048
|
+
error: error.message,
|
|
4049
|
+
});
|
|
4050
|
+
return null;
|
|
4051
|
+
}
|
|
4052
|
+
if (!data || typeof data.status !== 'string') {
|
|
4053
|
+
return null;
|
|
4054
|
+
}
|
|
4055
|
+
const status = data.status;
|
|
4056
|
+
if (status === 'queued' ||
|
|
4057
|
+
status === 'running' ||
|
|
4058
|
+
status === 'completed' ||
|
|
4059
|
+
status === 'failed' ||
|
|
4060
|
+
status === 'cancelled') {
|
|
4061
|
+
return status;
|
|
4062
|
+
}
|
|
4063
|
+
return null;
|
|
4064
|
+
}
|
|
4065
|
+
async function isGatewayJobCancelled(supabase, jobId) {
|
|
4066
|
+
const status = await loadGatewayJobStatus(supabase, jobId);
|
|
4067
|
+
return status === 'cancelled';
|
|
4068
|
+
}
|
|
4069
|
+
async function loadValidatedRemoteShellResource(supabase, config, resourceId, requiredAccess) {
|
|
4070
|
+
const { data, error } = await supabase
|
|
4071
|
+
.from('team_resources')
|
|
4072
|
+
.select('id, team_id, resource_type, max_access_level, metadata')
|
|
4073
|
+
.eq('id', resourceId)
|
|
4074
|
+
.eq('team_id', config.teamId)
|
|
4075
|
+
.maybeSingle();
|
|
4076
|
+
if (error) {
|
|
4077
|
+
logError('Failed to load remote shell resource for gateway validation', {
|
|
4078
|
+
resourceId,
|
|
4079
|
+
gatewayId: config.gatewayId,
|
|
4080
|
+
teamId: config.teamId,
|
|
4081
|
+
error: error.message,
|
|
4082
|
+
});
|
|
4083
|
+
return { ok: false, message: 'Failed to load remote shell resource' };
|
|
4084
|
+
}
|
|
4085
|
+
return validateRemoteShellResourceRow(data ?? null, {
|
|
4086
|
+
teamId: config.teamId,
|
|
4087
|
+
gatewayId: config.gatewayId,
|
|
4088
|
+
requiredAccess,
|
|
4089
|
+
});
|
|
4090
|
+
}
|
|
4091
|
+
function revokeShellSession(session, message) {
|
|
4092
|
+
appendShellSessionEvent(session, 'system', `Session closed: ${message}`);
|
|
4093
|
+
cleanupShellSession(session.id, 'shell_session_resource_revoked');
|
|
4094
|
+
session.status = 'closed';
|
|
4095
|
+
session.updatedAt = new Date().toISOString();
|
|
4096
|
+
}
|
|
4097
|
+
async function handleShellSessionJob(supabase, config, job) {
|
|
4098
|
+
purgeExpiredShellSessions();
|
|
4099
|
+
if (await isGatewayJobCancelled(supabase, job.id)) {
|
|
4100
|
+
const message = 'Gateway shell session job was cancelled before execution';
|
|
4101
|
+
return {
|
|
4102
|
+
ok: false,
|
|
4103
|
+
result: { message, cancelled: true },
|
|
4104
|
+
error: message,
|
|
4105
|
+
};
|
|
4106
|
+
}
|
|
4107
|
+
const controlValidation = await validateMachineControlState(supabase, config);
|
|
4108
|
+
if (!controlValidation.ok) {
|
|
4109
|
+
const message = controlValidation.message ?? 'Gateway machine control unavailable';
|
|
4110
|
+
return {
|
|
4111
|
+
ok: false,
|
|
4112
|
+
result: { message },
|
|
4113
|
+
error: message,
|
|
4114
|
+
};
|
|
4115
|
+
}
|
|
4116
|
+
const payload = job.payload ?? {};
|
|
4117
|
+
const action = normalizeShellSessionAction(payload.action);
|
|
4118
|
+
if (!action) {
|
|
4119
|
+
return {
|
|
4120
|
+
ok: false,
|
|
4121
|
+
result: { message: 'Invalid shell session action' },
|
|
4122
|
+
error: 'Invalid shell session action',
|
|
4123
|
+
};
|
|
4124
|
+
}
|
|
4125
|
+
const resourceId = typeof payload.resource_id === 'string' && payload.resource_id.trim().length > 0
|
|
4126
|
+
? payload.resource_id.trim()
|
|
4127
|
+
: null;
|
|
4128
|
+
if (!resourceId) {
|
|
4129
|
+
return {
|
|
4130
|
+
ok: false,
|
|
4131
|
+
result: { message: 'Missing resource_id in payload' },
|
|
4132
|
+
error: 'Missing resource_id in payload',
|
|
4133
|
+
};
|
|
4134
|
+
}
|
|
4135
|
+
if (action === 'open') {
|
|
4136
|
+
const resourceValidation = await loadValidatedRemoteShellResource(supabase, config, resourceId, 'write');
|
|
4137
|
+
if (!resourceValidation.ok) {
|
|
4138
|
+
return {
|
|
4139
|
+
ok: false,
|
|
4140
|
+
result: { message: resourceValidation.message, resource_id: resourceId },
|
|
4141
|
+
error: resourceValidation.message,
|
|
4142
|
+
};
|
|
4143
|
+
}
|
|
4144
|
+
if (getRunningShellSessionCount() >= MAX_ACTIVE_SHELL_SESSIONS) {
|
|
4145
|
+
const message = `Too many active shell sessions (max ${MAX_ACTIVE_SHELL_SESSIONS})`;
|
|
4146
|
+
return {
|
|
4147
|
+
ok: false,
|
|
4148
|
+
result: { message, max_active_sessions: MAX_ACTIVE_SHELL_SESSIONS },
|
|
4149
|
+
error: message,
|
|
4150
|
+
};
|
|
4151
|
+
}
|
|
4152
|
+
const shell = typeof payload.shell === 'string' && payload.shell.trim().length > 0
|
|
4153
|
+
? payload.shell.trim()
|
|
4154
|
+
: resolveDefaultShellCommand();
|
|
4155
|
+
const shellArgs = normalizeCommandArgs(payload.shell_args);
|
|
4156
|
+
if (!shellArgs) {
|
|
4157
|
+
return {
|
|
4158
|
+
ok: false,
|
|
4159
|
+
result: { message: 'Invalid shell_args in payload (expected string array)' },
|
|
4160
|
+
error: 'Invalid shell_args in payload',
|
|
4161
|
+
};
|
|
4162
|
+
}
|
|
4163
|
+
const env = normalizeCommandEnv(payload.env);
|
|
4164
|
+
if (!env) {
|
|
4165
|
+
return {
|
|
4166
|
+
ok: false,
|
|
4167
|
+
result: { message: 'Invalid env in payload (expected string/number/boolean map)' },
|
|
4168
|
+
error: 'Invalid env in payload',
|
|
4169
|
+
};
|
|
4170
|
+
}
|
|
4171
|
+
const cwd = typeof payload.cwd === 'string' && payload.cwd.trim().length > 0
|
|
4172
|
+
? payload.cwd.trim()
|
|
4173
|
+
: null;
|
|
4174
|
+
const requestedSessionId = typeof payload.session_id === 'string' && payload.session_id.trim().length > 0
|
|
4175
|
+
? payload.session_id.trim()
|
|
4176
|
+
: null;
|
|
4177
|
+
let sessionId = requestedSessionId ?? randomUUID();
|
|
4178
|
+
if (requestedSessionId && activeShellSessions.has(requestedSessionId)) {
|
|
4179
|
+
return {
|
|
4180
|
+
ok: false,
|
|
4181
|
+
result: {
|
|
4182
|
+
message: 'Shell session ID already exists',
|
|
4183
|
+
session_id: requestedSessionId,
|
|
4184
|
+
},
|
|
4185
|
+
error: 'Shell session ID already exists',
|
|
4186
|
+
};
|
|
4187
|
+
}
|
|
4188
|
+
if (!requestedSessionId) {
|
|
4189
|
+
let retries = 0;
|
|
4190
|
+
while (activeShellSessions.has(sessionId) && retries < 5) {
|
|
4191
|
+
sessionId = randomUUID();
|
|
4192
|
+
retries += 1;
|
|
4193
|
+
}
|
|
4194
|
+
if (activeShellSessions.has(sessionId)) {
|
|
4195
|
+
return {
|
|
4196
|
+
ok: false,
|
|
4197
|
+
result: {
|
|
4198
|
+
message: 'Failed to allocate unique shell session ID',
|
|
4199
|
+
},
|
|
4200
|
+
error: 'Failed to allocate unique shell session ID',
|
|
4201
|
+
};
|
|
4202
|
+
}
|
|
4203
|
+
}
|
|
4204
|
+
const nowIso = new Date().toISOString();
|
|
4205
|
+
const session = {
|
|
4206
|
+
id: sessionId,
|
|
4207
|
+
teamId: config.teamId,
|
|
4208
|
+
gatewayId: config.gatewayId,
|
|
4209
|
+
resourceId,
|
|
4210
|
+
shell,
|
|
4211
|
+
shellArgs,
|
|
4212
|
+
cwd,
|
|
4213
|
+
status: 'running',
|
|
4214
|
+
createdAt: nowIso,
|
|
4215
|
+
updatedAt: nowIso,
|
|
4216
|
+
exitCode: null,
|
|
4217
|
+
error: null,
|
|
4218
|
+
child: undefined,
|
|
4219
|
+
events: [],
|
|
4220
|
+
nextSequence: 1,
|
|
4221
|
+
bufferedBytes: 0,
|
|
4222
|
+
};
|
|
4223
|
+
const sessionEnv = {
|
|
4224
|
+
...env,
|
|
4225
|
+
};
|
|
4226
|
+
if (!Object.hasOwn(sessionEnv, 'TERM')) {
|
|
4227
|
+
sessionEnv.TERM = 'xterm-256color';
|
|
4228
|
+
}
|
|
4229
|
+
if (!Object.hasOwn(sessionEnv, 'COLUMNS')) {
|
|
4230
|
+
sessionEnv.COLUMNS = String(DEFAULT_SHELL_COLUMNS);
|
|
4231
|
+
}
|
|
4232
|
+
if (!Object.hasOwn(sessionEnv, 'LINES')) {
|
|
4233
|
+
sessionEnv.LINES = String(DEFAULT_SHELL_ROWS);
|
|
4234
|
+
}
|
|
4235
|
+
try {
|
|
4236
|
+
const child = spawn(shell, shellArgs, {
|
|
4237
|
+
cwd: cwd ?? undefined,
|
|
4238
|
+
env: resolveChildProcessEnv(sessionEnv),
|
|
4239
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
4240
|
+
});
|
|
4241
|
+
child.on('error', (error) => {
|
|
4242
|
+
const now = new Date().toISOString();
|
|
4243
|
+
session.status = 'failed';
|
|
4244
|
+
session.error = error.message;
|
|
4245
|
+
session.updatedAt = now;
|
|
4246
|
+
session.child = undefined;
|
|
4247
|
+
appendShellSessionEvent(session, 'system', `Session error: ${error.message}`, now);
|
|
4248
|
+
});
|
|
4249
|
+
child.on('close', (code) => {
|
|
4250
|
+
const now = new Date().toISOString();
|
|
4251
|
+
session.exitCode = code ?? null;
|
|
4252
|
+
if (session.status === 'running') {
|
|
4253
|
+
session.status = 'exited';
|
|
4254
|
+
}
|
|
4255
|
+
session.updatedAt = now;
|
|
4256
|
+
session.child = undefined;
|
|
4257
|
+
appendShellSessionEvent(session, 'system', `Session exited with code ${code ?? 'unknown'}`, now);
|
|
4258
|
+
});
|
|
4259
|
+
child.stdout?.on('data', (chunk) => {
|
|
4260
|
+
appendShellSessionEvent(session, 'stdout', chunk.toString('utf-8'));
|
|
4261
|
+
});
|
|
4262
|
+
child.stderr?.on('data', (chunk) => {
|
|
4263
|
+
appendShellSessionEvent(session, 'stderr', chunk.toString('utf-8'));
|
|
4264
|
+
});
|
|
4265
|
+
await waitForChildStartup(child);
|
|
4266
|
+
if (session.status !== 'running') {
|
|
4267
|
+
const message = session.error ??
|
|
4268
|
+
(session.exitCode !== null
|
|
4269
|
+
? `Shell session exited during startup with code ${session.exitCode}`
|
|
4270
|
+
: 'Shell session exited during startup');
|
|
4271
|
+
return {
|
|
4272
|
+
ok: false,
|
|
4273
|
+
result: {
|
|
4274
|
+
message,
|
|
4275
|
+
...buildShellSessionSummary(session),
|
|
4276
|
+
},
|
|
4277
|
+
error: message,
|
|
4278
|
+
};
|
|
4279
|
+
}
|
|
4280
|
+
session.child = child;
|
|
4281
|
+
activeShellSessions.set(sessionId, session);
|
|
4282
|
+
appendShellSessionEvent(session, 'system', `Session opened (${shell}${shellArgs.length > 0 ? ` ${shellArgs.join(' ')}` : ''})`);
|
|
4283
|
+
return {
|
|
4284
|
+
ok: true,
|
|
4285
|
+
result: {
|
|
4286
|
+
message: 'Shell session opened',
|
|
4287
|
+
...buildShellSessionSummary(session),
|
|
4288
|
+
default_columns: DEFAULT_SHELL_COLUMNS,
|
|
4289
|
+
default_rows: DEFAULT_SHELL_ROWS,
|
|
4290
|
+
},
|
|
4291
|
+
};
|
|
4292
|
+
}
|
|
4293
|
+
catch (error) {
|
|
4294
|
+
activeShellSessions.delete(sessionId);
|
|
4295
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4296
|
+
return {
|
|
4297
|
+
ok: false,
|
|
4298
|
+
result: {
|
|
4299
|
+
message: `Failed to open shell session: ${message}`,
|
|
4300
|
+
},
|
|
4301
|
+
error: `Failed to open shell session: ${message}`,
|
|
4302
|
+
};
|
|
4303
|
+
}
|
|
4304
|
+
}
|
|
4305
|
+
const sessionId = typeof payload.session_id === 'string' && payload.session_id.trim().length > 0
|
|
4306
|
+
? payload.session_id.trim()
|
|
4307
|
+
: null;
|
|
4308
|
+
if (!sessionId) {
|
|
4309
|
+
return {
|
|
4310
|
+
ok: false,
|
|
4311
|
+
result: { message: 'Missing session_id in payload' },
|
|
4312
|
+
error: 'Missing session_id in payload',
|
|
4313
|
+
};
|
|
4314
|
+
}
|
|
4315
|
+
const session = activeShellSessions.get(sessionId);
|
|
4316
|
+
if (!session) {
|
|
4317
|
+
return {
|
|
4318
|
+
ok: false,
|
|
4319
|
+
result: { message: 'Shell session not found or expired', session_id: sessionId },
|
|
4320
|
+
error: 'Shell session not found or expired',
|
|
4321
|
+
};
|
|
4322
|
+
}
|
|
4323
|
+
if (session.teamId !== config.teamId || session.gatewayId !== config.gatewayId) {
|
|
4324
|
+
return {
|
|
4325
|
+
ok: false,
|
|
4326
|
+
result: { message: 'Shell session belongs to another gateway context', session_id: sessionId },
|
|
4327
|
+
error: 'Shell session belongs to another gateway context',
|
|
4328
|
+
};
|
|
4329
|
+
}
|
|
4330
|
+
if (resourceId !== session.resourceId) {
|
|
4331
|
+
return {
|
|
4332
|
+
ok: false,
|
|
4333
|
+
result: { message: 'Shell session resource mismatch', session_id: sessionId },
|
|
4334
|
+
error: 'Shell session resource mismatch',
|
|
4335
|
+
};
|
|
4336
|
+
}
|
|
4337
|
+
const requiredAccess = action === 'read' || action === 'status' ? 'read' : 'write';
|
|
4338
|
+
const resourceValidation = await loadValidatedRemoteShellResource(supabase, config, resourceId, requiredAccess);
|
|
4339
|
+
if (!resourceValidation.ok) {
|
|
4340
|
+
revokeShellSession(session, resourceValidation.message);
|
|
4341
|
+
return {
|
|
4342
|
+
ok: false,
|
|
4343
|
+
result: {
|
|
4344
|
+
message: resourceValidation.message,
|
|
4345
|
+
...buildShellSessionSummary(session),
|
|
4346
|
+
},
|
|
4347
|
+
error: resourceValidation.message,
|
|
4348
|
+
};
|
|
4349
|
+
}
|
|
4350
|
+
if (action === 'status') {
|
|
4351
|
+
return {
|
|
4352
|
+
ok: true,
|
|
4353
|
+
result: {
|
|
4354
|
+
message: 'Shell session status',
|
|
4355
|
+
...buildShellSessionSummary(session),
|
|
4356
|
+
},
|
|
4357
|
+
};
|
|
4358
|
+
}
|
|
4359
|
+
if (action === 'read') {
|
|
4360
|
+
const sinceSequence = coerceNonNegativeInteger(payload.since_sequence, 0);
|
|
4361
|
+
const maxEvents = coercePositiveInteger(payload.max_events, 200, {
|
|
4362
|
+
min: 1,
|
|
4363
|
+
max: 1000,
|
|
4364
|
+
});
|
|
4365
|
+
const readResult = readShellSessionEvents(session, { sinceSequence, maxEvents });
|
|
4366
|
+
return {
|
|
4367
|
+
ok: true,
|
|
4368
|
+
result: {
|
|
4369
|
+
message: 'Shell session output',
|
|
4370
|
+
...buildShellSessionSummary(session),
|
|
4371
|
+
since_sequence: sinceSequence,
|
|
4372
|
+
max_events: maxEvents,
|
|
4373
|
+
has_more: readResult.hasMore,
|
|
4374
|
+
events: readResult.events,
|
|
4375
|
+
latest_sequence: readResult.latestSequence,
|
|
4376
|
+
},
|
|
4377
|
+
};
|
|
4378
|
+
}
|
|
4379
|
+
if (action === 'write') {
|
|
4380
|
+
const input = typeof payload.input === 'string' ? payload.input : null;
|
|
4381
|
+
if (input === null) {
|
|
4382
|
+
return {
|
|
4383
|
+
ok: false,
|
|
4384
|
+
result: { message: 'Missing input in payload', session_id: sessionId },
|
|
4385
|
+
error: 'Missing input in payload',
|
|
4386
|
+
};
|
|
4387
|
+
}
|
|
4388
|
+
if (session.status !== 'running' || !session.child || !session.child.stdin) {
|
|
4389
|
+
const message = `Shell session is not writable (status: ${session.status})`;
|
|
4390
|
+
return {
|
|
4391
|
+
ok: false,
|
|
4392
|
+
result: {
|
|
4393
|
+
message,
|
|
4394
|
+
...buildShellSessionSummary(session),
|
|
4395
|
+
},
|
|
4396
|
+
error: message,
|
|
4397
|
+
};
|
|
4398
|
+
}
|
|
4399
|
+
const appendNewline = payload.append_newline === true;
|
|
4400
|
+
const toWrite = appendNewline ? `${input}\n` : input;
|
|
4401
|
+
const bytesWritten = Buffer.byteLength(toWrite, 'utf-8');
|
|
4402
|
+
try {
|
|
4403
|
+
await new Promise((resolve, reject) => {
|
|
4404
|
+
session.child?.stdin?.write(toWrite, (error) => {
|
|
4405
|
+
if (error) {
|
|
4406
|
+
reject(error);
|
|
4407
|
+
return;
|
|
4408
|
+
}
|
|
4409
|
+
resolve();
|
|
4410
|
+
});
|
|
4411
|
+
});
|
|
4412
|
+
session.updatedAt = new Date().toISOString();
|
|
4413
|
+
return {
|
|
4414
|
+
ok: true,
|
|
4415
|
+
result: {
|
|
4416
|
+
message: 'Shell session input sent',
|
|
4417
|
+
...buildShellSessionSummary(session),
|
|
4418
|
+
bytes_written: bytesWritten,
|
|
4419
|
+
append_newline: appendNewline,
|
|
4420
|
+
},
|
|
4421
|
+
};
|
|
4422
|
+
}
|
|
4423
|
+
catch (error) {
|
|
4424
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4425
|
+
return {
|
|
4426
|
+
ok: false,
|
|
4427
|
+
result: {
|
|
4428
|
+
message: `Failed to write to shell session: ${message}`,
|
|
4429
|
+
...buildShellSessionSummary(session),
|
|
4430
|
+
},
|
|
4431
|
+
error: `Failed to write to shell session: ${message}`,
|
|
4432
|
+
};
|
|
4433
|
+
}
|
|
4434
|
+
}
|
|
4435
|
+
if (action === 'close') {
|
|
4436
|
+
if (session.status === 'running') {
|
|
4437
|
+
cleanupShellSession(session.id, 'shell_session_close');
|
|
4438
|
+
appendShellSessionEvent(session, 'system', 'Session closed by request');
|
|
4439
|
+
}
|
|
4440
|
+
session.status = 'closed';
|
|
4441
|
+
session.updatedAt = new Date().toISOString();
|
|
4442
|
+
return {
|
|
4443
|
+
ok: true,
|
|
4444
|
+
result: {
|
|
4445
|
+
message: 'Shell session closed',
|
|
4446
|
+
...buildShellSessionSummary(session),
|
|
4447
|
+
},
|
|
4448
|
+
};
|
|
4449
|
+
}
|
|
4450
|
+
return {
|
|
4451
|
+
ok: false,
|
|
4452
|
+
result: { message: `Unsupported shell session action: ${action}` },
|
|
4453
|
+
error: `Unsupported shell session action: ${action}`,
|
|
4454
|
+
};
|
|
4455
|
+
}
|
|
3222
4456
|
async function failSubagentRun(supabase, job, errorMessage) {
|
|
3223
4457
|
const payload = job.payload ?? {};
|
|
3224
4458
|
const subagentId = typeof payload.subagent_id === 'string'
|
|
@@ -3256,15 +4490,7 @@ async function failSubagentRun(supabase, job, errorMessage) {
|
|
|
3256
4490
|
.limit(1)
|
|
3257
4491
|
.maybeSingle();
|
|
3258
4492
|
if (runningRun?.id) {
|
|
3259
|
-
const {
|
|
3260
|
-
.from('subagent_run_events')
|
|
3261
|
-
.select('sequence')
|
|
3262
|
-
.eq('run_id', runningRun.id)
|
|
3263
|
-
.order('sequence', { ascending: false })
|
|
3264
|
-
.limit(1)
|
|
3265
|
-
.maybeSingle();
|
|
3266
|
-
const nextSequence = lastEvent && typeof lastEvent.sequence === 'number' ? lastEvent.sequence + 1 : 0;
|
|
3267
|
-
await supabase
|
|
4493
|
+
const { error: runFailUpdateError } = await supabase
|
|
3268
4494
|
.from('subagent_runs')
|
|
3269
4495
|
.update({
|
|
3270
4496
|
status: 'failed',
|
|
@@ -3274,15 +4500,24 @@ async function failSubagentRun(supabase, job, errorMessage) {
|
|
|
3274
4500
|
metadata: asJson(mergedMetadata),
|
|
3275
4501
|
})
|
|
3276
4502
|
.eq('id', runningRun.id);
|
|
4503
|
+
if (runFailUpdateError) {
|
|
4504
|
+
logError('Failed to mark running subagent run as failed after gateway error', {
|
|
4505
|
+
subagentId,
|
|
4506
|
+
runId: runningRun.id,
|
|
4507
|
+
error: runFailUpdateError.message,
|
|
4508
|
+
});
|
|
4509
|
+
}
|
|
3277
4510
|
if (subagent.gateway_id) {
|
|
3278
|
-
await supabase.
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
4511
|
+
const nextSequence = await getNextSubagentRunEventSequence(supabase, runningRun.id);
|
|
4512
|
+
await insertSubagentRunEvent({
|
|
4513
|
+
supabase,
|
|
4514
|
+
runId: runningRun.id,
|
|
4515
|
+
subagentId: subagent.id,
|
|
4516
|
+
teamId: subagent.team_id,
|
|
4517
|
+
gatewayId: subagent.gateway_id,
|
|
3283
4518
|
sequence: nextSequence,
|
|
3284
|
-
|
|
3285
|
-
payload:
|
|
4519
|
+
eventType: 'run_failed',
|
|
4520
|
+
payload: { reason: errorMessage, source: 'gateway_job_failure' },
|
|
3286
4521
|
});
|
|
3287
4522
|
}
|
|
3288
4523
|
}
|
|
@@ -3328,6 +4563,7 @@ async function processJob(supabase, config, job) {
|
|
|
3328
4563
|
provider: jobDetails.provider ?? undefined,
|
|
3329
4564
|
model: jobDetails.model ?? undefined,
|
|
3330
4565
|
subagentType: jobDetails.subagent_type ?? undefined,
|
|
4566
|
+
shellAction: jobDetails.shell_action ?? undefined,
|
|
3331
4567
|
});
|
|
3332
4568
|
if (claimed.job_type === 'diagnostic') {
|
|
3333
4569
|
const diagnostic = await handleDiagnosticJob();
|
|
@@ -3340,12 +4576,15 @@ async function processJob(supabase, config, job) {
|
|
|
3340
4576
|
return;
|
|
3341
4577
|
}
|
|
3342
4578
|
if (claimed.job_type === 'subagent_run') {
|
|
3343
|
-
const result = await handleSubagentRunJob(supabase, claimed);
|
|
4579
|
+
const result = await handleSubagentRunJob(supabase, config, claimed);
|
|
3344
4580
|
if (result.ok) {
|
|
3345
4581
|
await completeJob(supabase, claimed.id, 'completed', result.result);
|
|
3346
4582
|
}
|
|
3347
4583
|
else {
|
|
3348
|
-
await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
|
|
4584
|
+
await completeJob(supabase, claimed.id, result.jobStatus ?? 'failed', result.result, result.error);
|
|
4585
|
+
if (!result.skipSubagentFailure) {
|
|
4586
|
+
await failSubagentRun(supabase, claimed, result.error ?? 'Subagent run failed');
|
|
4587
|
+
}
|
|
3349
4588
|
}
|
|
3350
4589
|
return;
|
|
3351
4590
|
}
|
|
@@ -3360,7 +4599,17 @@ async function processJob(supabase, config, job) {
|
|
|
3360
4599
|
return;
|
|
3361
4600
|
}
|
|
3362
4601
|
if (claimed.job_type === 'subagent_cancel') {
|
|
3363
|
-
const result = await handleSubagentCancelJob(supabase, claimed);
|
|
4602
|
+
const result = await handleSubagentCancelJob(supabase, config, claimed);
|
|
4603
|
+
if (result.ok) {
|
|
4604
|
+
await completeJob(supabase, claimed.id, 'completed', result.result);
|
|
4605
|
+
}
|
|
4606
|
+
else {
|
|
4607
|
+
await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
|
|
4608
|
+
}
|
|
4609
|
+
return;
|
|
4610
|
+
}
|
|
4611
|
+
if (claimed.job_type === 'shell_session') {
|
|
4612
|
+
const result = await handleShellSessionJob(supabase, config, claimed);
|
|
3364
4613
|
if (result.ok) {
|
|
3365
4614
|
await completeJob(supabase, claimed.id, 'completed', result.result);
|
|
3366
4615
|
}
|
|
@@ -3431,6 +4680,7 @@ async function startGateway(options, configResolution) {
|
|
|
3431
4680
|
}
|
|
3432
4681
|
}
|
|
3433
4682
|
cliInfo('Starting gateway...', options);
|
|
4683
|
+
CURRENT_RUNTIME_OPTIONS = options;
|
|
3434
4684
|
let config;
|
|
3435
4685
|
try {
|
|
3436
4686
|
config = await loadConfig(options);
|
|
@@ -3506,10 +4756,11 @@ async function startGateway(options, configResolution) {
|
|
|
3506
4756
|
await ensureGatewayTmpDir();
|
|
3507
4757
|
await discoverGatewayCliCommands(options);
|
|
3508
4758
|
cliInfo('Validating providers...', options);
|
|
3509
|
-
const { capabilities: initialCapabilities, providerHealth
|
|
4759
|
+
const { capabilities: initialCapabilities, providerHealth } = await buildCapabilities({
|
|
3510
4760
|
validationMode: 'auto',
|
|
3511
4761
|
previousHealth: config.providerHealth,
|
|
3512
4762
|
deviceName,
|
|
4763
|
+
installFullControlEnabled: config.full_control_enabled,
|
|
3513
4764
|
onProgress: (event) => {
|
|
3514
4765
|
if (event.stage === 'start') {
|
|
3515
4766
|
cliInfo(` ${formatProviderLabel(event.providerId)}...`, options);
|
|
@@ -3526,8 +4777,16 @@ async function startGateway(options, configResolution) {
|
|
|
3526
4777
|
let currentProviderHealth = providerHealth;
|
|
3527
4778
|
CURRENT_CAPABILITIES = initialCapabilities;
|
|
3528
4779
|
CURRENT_PROVIDER_HEALTH = providerHealth;
|
|
4780
|
+
CURRENT_JOB_INGRESS_STATE = {
|
|
4781
|
+
pending: true,
|
|
4782
|
+
ready: false,
|
|
4783
|
+
};
|
|
3529
4784
|
const providerStatus = currentCapabilities.providers;
|
|
3530
|
-
const initialStatus =
|
|
4785
|
+
const initialStatus = computeGatewayOperationalStatus({
|
|
4786
|
+
capabilities: currentCapabilities,
|
|
4787
|
+
jobIngressState: CURRENT_JOB_INGRESS_STATE,
|
|
4788
|
+
acceptingJobs: CURRENT_ACCEPTING_JOBS,
|
|
4789
|
+
});
|
|
3531
4790
|
logInfo('Gateway capabilities loaded', {
|
|
3532
4791
|
providers: Object.fromEntries(Object.entries(providerStatus).map(([id, entry]) => [
|
|
3533
4792
|
id,
|
|
@@ -3555,19 +4814,32 @@ async function startGateway(options, configResolution) {
|
|
|
3555
4814
|
if (ranFullValidation) {
|
|
3556
4815
|
await emitProviderHealthEvents(supabase, config, providerHealth, 'startup');
|
|
3557
4816
|
}
|
|
3558
|
-
await sendHeartbeat(supabase, initialStatus, currentCapabilities, deviceName);
|
|
3559
4817
|
let heartbeatStatus = initialStatus;
|
|
4818
|
+
await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
|
|
3560
4819
|
await writePidFile(process.pid, options);
|
|
3561
4820
|
let heartbeatInFlight = false;
|
|
4821
|
+
const syncOperationalHeartbeatIfChanged = async () => {
|
|
4822
|
+
const nextStatus = computeGatewayOperationalStatus({
|
|
4823
|
+
capabilities: currentCapabilities,
|
|
4824
|
+
jobIngressState: CURRENT_JOB_INGRESS_STATE,
|
|
4825
|
+
acceptingJobs: CURRENT_ACCEPTING_JOBS,
|
|
4826
|
+
});
|
|
4827
|
+
if (nextStatus === heartbeatStatus)
|
|
4828
|
+
return;
|
|
4829
|
+
heartbeatStatus = nextStatus;
|
|
4830
|
+
await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
|
|
4831
|
+
};
|
|
3562
4832
|
const refreshHeartbeat = async () => {
|
|
3563
4833
|
if (heartbeatInFlight)
|
|
3564
4834
|
return;
|
|
3565
4835
|
heartbeatInFlight = true;
|
|
3566
4836
|
try {
|
|
3567
|
-
const
|
|
4837
|
+
const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
|
|
4838
|
+
const { capabilities: nextCapabilities, providerHealth: nextProviderHealth, } = await buildCapabilities({
|
|
3568
4839
|
validationMode: 'light',
|
|
3569
4840
|
previousHealth: CURRENT_PROVIDER_HEALTH ?? currentProviderHealth,
|
|
3570
4841
|
deviceName,
|
|
4842
|
+
installFullControlEnabled,
|
|
3571
4843
|
onProgress: (event) => {
|
|
3572
4844
|
if (event.stage === 'start')
|
|
3573
4845
|
return;
|
|
@@ -3584,7 +4856,11 @@ async function startGateway(options, configResolution) {
|
|
|
3584
4856
|
config.providerHealth = nextProviderHealth;
|
|
3585
4857
|
CURRENT_CAPABILITIES = nextCapabilities;
|
|
3586
4858
|
CURRENT_PROVIDER_HEALTH = nextProviderHealth;
|
|
3587
|
-
heartbeatStatus =
|
|
4859
|
+
heartbeatStatus = computeGatewayOperationalStatus({
|
|
4860
|
+
capabilities: nextCapabilities,
|
|
4861
|
+
jobIngressState: CURRENT_JOB_INGRESS_STATE,
|
|
4862
|
+
acceptingJobs: CURRENT_ACCEPTING_JOBS,
|
|
4863
|
+
});
|
|
3588
4864
|
await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
|
|
3589
4865
|
}
|
|
3590
4866
|
catch (error) {
|
|
@@ -3602,12 +4878,63 @@ async function startGateway(options, configResolution) {
|
|
|
3602
4878
|
void refreshHeartbeat();
|
|
3603
4879
|
}, DEFAULT_HEARTBEAT_INTERVAL_MS);
|
|
3604
4880
|
let acceptingJobs = true;
|
|
4881
|
+
CURRENT_ACCEPTING_JOBS = true;
|
|
3605
4882
|
let restartRequested = false;
|
|
3606
4883
|
let restartDetectedAt = null;
|
|
3607
4884
|
let restartTimer = null;
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
const
|
|
4885
|
+
let reconcileTimer = null;
|
|
4886
|
+
let channelReconnectTimer = null;
|
|
4887
|
+
const jobQueue = new GatewayJobQueue(resolveJobSubagentId);
|
|
4888
|
+
const activeClaimedJobs = new Map();
|
|
4889
|
+
let lastJobIngressSuccessAtMs = null;
|
|
4890
|
+
let lastJobIngressError = null;
|
|
4891
|
+
let reconcileInFlight = false;
|
|
4892
|
+
let shutdownInProgress = false;
|
|
4893
|
+
const updateJobIngressState = () => {
|
|
4894
|
+
const ready = lastJobIngressSuccessAtMs !== null &&
|
|
4895
|
+
Date.now() - lastJobIngressSuccessAtMs <= DEFAULT_JOB_RECONCILE_INTERVAL_MS * 3;
|
|
4896
|
+
const pending = lastJobIngressSuccessAtMs === null && lastJobIngressError === null;
|
|
4897
|
+
CURRENT_JOB_INGRESS_STATE = {
|
|
4898
|
+
pending,
|
|
4899
|
+
ready,
|
|
4900
|
+
};
|
|
4901
|
+
};
|
|
4902
|
+
const reconcileQueuedJobs = async (reason) => {
|
|
4903
|
+
if (reconcileInFlight || shutdownInProgress)
|
|
4904
|
+
return;
|
|
4905
|
+
reconcileInFlight = true;
|
|
4906
|
+
try {
|
|
4907
|
+
const { data, error } = await supabase
|
|
4908
|
+
.from('gateway_jobs')
|
|
4909
|
+
.select('*')
|
|
4910
|
+
.eq('team_id', config.teamId)
|
|
4911
|
+
.eq('status', 'queued')
|
|
4912
|
+
.eq('gateway_id', config.gatewayId)
|
|
4913
|
+
.order('created_at', { ascending: true });
|
|
4914
|
+
if (error) {
|
|
4915
|
+
throw error;
|
|
4916
|
+
}
|
|
4917
|
+
lastJobIngressSuccessAtMs = Date.now();
|
|
4918
|
+
lastJobIngressError = null;
|
|
4919
|
+
updateJobIngressState();
|
|
4920
|
+
for (const row of data ?? []) {
|
|
4921
|
+
enqueueJob(row);
|
|
4922
|
+
}
|
|
4923
|
+
await syncOperationalHeartbeatIfChanged();
|
|
4924
|
+
}
|
|
4925
|
+
catch (error) {
|
|
4926
|
+
lastJobIngressError = error instanceof Error ? error.message : String(error);
|
|
4927
|
+
updateJobIngressState();
|
|
4928
|
+
logError('Failed to reconcile queued gateway jobs', {
|
|
4929
|
+
error: lastJobIngressError,
|
|
4930
|
+
reason,
|
|
4931
|
+
});
|
|
4932
|
+
await syncOperationalHeartbeatIfChanged();
|
|
4933
|
+
}
|
|
4934
|
+
finally {
|
|
4935
|
+
reconcileInFlight = false;
|
|
4936
|
+
}
|
|
4937
|
+
};
|
|
3611
4938
|
const entryPath = process.argv[1] || '';
|
|
3612
4939
|
let modulePath = '';
|
|
3613
4940
|
try {
|
|
@@ -3635,7 +4962,10 @@ async function startGateway(options, configResolution) {
|
|
|
3635
4962
|
const maybeRestart = async () => {
|
|
3636
4963
|
if (!restartRequested)
|
|
3637
4964
|
return;
|
|
3638
|
-
if (
|
|
4965
|
+
if (jobQueue.activeCount() > 0 ||
|
|
4966
|
+
activeSubagentRuns.size > 0 ||
|
|
4967
|
+
activeModelRuns.size > 0 ||
|
|
4968
|
+
getRunningShellSessionCount() > 0) {
|
|
3639
4969
|
return;
|
|
3640
4970
|
}
|
|
3641
4971
|
await restartGateway('all jobs complete');
|
|
@@ -3646,13 +4976,16 @@ async function startGateway(options, configResolution) {
|
|
|
3646
4976
|
restartRequested = true;
|
|
3647
4977
|
restartDetectedAt = Date.now();
|
|
3648
4978
|
acceptingJobs = false;
|
|
3649
|
-
|
|
3650
|
-
|
|
4979
|
+
CURRENT_ACCEPTING_JOBS = false;
|
|
4980
|
+
jobQueue.clearPending();
|
|
3651
4981
|
logInfo('Gateway update detected; draining before restart', {
|
|
3652
4982
|
reason,
|
|
3653
|
-
active_jobs:
|
|
4983
|
+
active_jobs: jobQueue.activeCount(),
|
|
3654
4984
|
active_subagent_runs: activeSubagentRuns.size,
|
|
4985
|
+
active_model_runs: activeModelRuns.size,
|
|
4986
|
+
active_shell_sessions: getRunningShellSessionCount(),
|
|
3655
4987
|
});
|
|
4988
|
+
void syncOperationalHeartbeatIfChanged();
|
|
3656
4989
|
void maybeRestart();
|
|
3657
4990
|
};
|
|
3658
4991
|
const checkForUpgrade = () => {
|
|
@@ -3680,7 +5013,7 @@ async function startGateway(options, configResolution) {
|
|
|
3680
5013
|
const claimed = await claimJob(supabase, config, job);
|
|
3681
5014
|
if (!claimed)
|
|
3682
5015
|
return;
|
|
3683
|
-
const result = await handleSubagentCancelJob(supabase, claimed);
|
|
5016
|
+
const result = await handleSubagentCancelJob(supabase, config, claimed);
|
|
3684
5017
|
if (result.ok) {
|
|
3685
5018
|
await completeJob(supabase, claimed.id, 'completed', result.result);
|
|
3686
5019
|
}
|
|
@@ -3688,7 +5021,8 @@ async function startGateway(options, configResolution) {
|
|
|
3688
5021
|
await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
|
|
3689
5022
|
}
|
|
3690
5023
|
};
|
|
3691
|
-
const processJobWithHandling = async (nextJob) => {
|
|
5024
|
+
const processJobWithHandling = async (nextJob, reservedSubagentId) => {
|
|
5025
|
+
activeClaimedJobs.set(nextJob.id, nextJob);
|
|
3692
5026
|
try {
|
|
3693
5027
|
await processJob(supabase, config, nextJob);
|
|
3694
5028
|
}
|
|
@@ -3704,7 +5038,8 @@ async function startGateway(options, configResolution) {
|
|
|
3704
5038
|
}
|
|
3705
5039
|
}
|
|
3706
5040
|
finally {
|
|
3707
|
-
|
|
5041
|
+
activeClaimedJobs.delete(nextJob.id);
|
|
5042
|
+
jobQueue.markDone(nextJob.id, reservedSubagentId);
|
|
3708
5043
|
void processQueue();
|
|
3709
5044
|
void maybeRestart();
|
|
3710
5045
|
}
|
|
@@ -3712,15 +5047,9 @@ async function startGateway(options, configResolution) {
|
|
|
3712
5047
|
const processQueue = async () => {
|
|
3713
5048
|
if (!acceptingJobs)
|
|
3714
5049
|
return;
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
break;
|
|
3719
|
-
queuedJobIds.delete(nextJob.id);
|
|
3720
|
-
if (activeJobIds.has(nextJob.id))
|
|
3721
|
-
continue;
|
|
3722
|
-
activeJobIds.add(nextJob.id);
|
|
3723
|
-
void processJobWithHandling(nextJob);
|
|
5050
|
+
const readyJobs = jobQueue.takeAvailable(GATEWAY_CONCURRENCY);
|
|
5051
|
+
for (const nextJob of readyJobs) {
|
|
5052
|
+
void processJobWithHandling(nextJob.job, nextJob.reservedSubagentId);
|
|
3724
5053
|
}
|
|
3725
5054
|
};
|
|
3726
5055
|
const enqueueJob = (job) => {
|
|
@@ -3733,73 +5062,275 @@ async function startGateway(options, configResolution) {
|
|
|
3733
5062
|
if (!acceptingJobs) {
|
|
3734
5063
|
return;
|
|
3735
5064
|
}
|
|
3736
|
-
if (
|
|
3737
|
-
|
|
5065
|
+
if (jobQueue.enqueue(job)) {
|
|
5066
|
+
void processQueue();
|
|
3738
5067
|
}
|
|
3739
|
-
pendingJobs.push(job);
|
|
3740
|
-
queuedJobIds.add(job.id);
|
|
3741
|
-
void processQueue();
|
|
3742
5068
|
};
|
|
3743
5069
|
let channel = null;
|
|
3744
|
-
const
|
|
3745
|
-
|
|
3746
|
-
.select('*')
|
|
3747
|
-
.eq('team_id', config.teamId)
|
|
3748
|
-
.eq('status', 'queued')
|
|
3749
|
-
.or(`gateway_id.is.null,gateway_id.eq.${config.gatewayId}`)
|
|
3750
|
-
.order('created_at', { ascending: true });
|
|
3751
|
-
if (backlogError) {
|
|
3752
|
-
logError('Failed to fetch queued gateway jobs', { error: backlogError.message });
|
|
3753
|
-
}
|
|
3754
|
-
else if (backlog && backlog.length > 0) {
|
|
3755
|
-
backlog.forEach((job) => enqueueJob(job));
|
|
3756
|
-
}
|
|
3757
|
-
channel = supabase
|
|
3758
|
-
.channel(`gateway_jobs_${config.gatewayId}`)
|
|
3759
|
-
.on('postgres_changes', {
|
|
3760
|
-
event: 'INSERT',
|
|
3761
|
-
schema: 'public',
|
|
3762
|
-
table: 'gateway_jobs',
|
|
3763
|
-
filter: `team_id=eq.${config.teamId}`,
|
|
3764
|
-
}, (payload) => {
|
|
3765
|
-
const job = payload.new;
|
|
3766
|
-
if (job.status !== 'queued')
|
|
5070
|
+
const scheduleChannelReconnect = (reason) => {
|
|
5071
|
+
if (shutdownInProgress || channelReconnectTimer)
|
|
3767
5072
|
return;
|
|
3768
|
-
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
|
|
3772
|
-
|
|
5073
|
+
channelReconnectTimer = setTimeout(() => {
|
|
5074
|
+
channelReconnectTimer = null;
|
|
5075
|
+
void connectJobChannel(`reconnect:${reason}`);
|
|
5076
|
+
}, DEFAULT_JOB_CHANNEL_RECONNECT_MS);
|
|
5077
|
+
};
|
|
5078
|
+
const connectJobChannel = async (reason) => {
|
|
5079
|
+
if (shutdownInProgress)
|
|
5080
|
+
return;
|
|
5081
|
+
if (channel) {
|
|
5082
|
+
try {
|
|
5083
|
+
await channel.unsubscribe();
|
|
5084
|
+
}
|
|
5085
|
+
catch {
|
|
5086
|
+
// ignore unsubscribe failures during reconnect
|
|
5087
|
+
}
|
|
3773
5088
|
}
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
5089
|
+
channel = supabase
|
|
5090
|
+
.channel(`gateway_jobs_${config.gatewayId}`)
|
|
5091
|
+
.on('postgres_changes', {
|
|
5092
|
+
event: 'INSERT',
|
|
5093
|
+
schema: 'public',
|
|
5094
|
+
table: 'gateway_jobs',
|
|
5095
|
+
filter: `team_id=eq.${config.teamId}`,
|
|
5096
|
+
}, (payload) => {
|
|
5097
|
+
const job = payload.new;
|
|
5098
|
+
if (job.status !== 'queued')
|
|
5099
|
+
return;
|
|
5100
|
+
enqueueJob(job);
|
|
5101
|
+
})
|
|
5102
|
+
.subscribe((status) => {
|
|
5103
|
+
if (status === 'SUBSCRIBED') {
|
|
5104
|
+
logInfo('Gateway subscribed to job queue', { reason });
|
|
5105
|
+
void reconcileQueuedJobs(`channel_subscribed:${reason}`);
|
|
5106
|
+
return;
|
|
5107
|
+
}
|
|
5108
|
+
if (status === 'CHANNEL_ERROR' ||
|
|
5109
|
+
status === 'TIMED_OUT' ||
|
|
5110
|
+
status === 'CLOSED') {
|
|
5111
|
+
logError('Gateway realtime channel unavailable', { status, reason });
|
|
5112
|
+
scheduleChannelReconnect(status.toLowerCase());
|
|
5113
|
+
}
|
|
5114
|
+
});
|
|
5115
|
+
};
|
|
5116
|
+
const requestActiveJobShutdown = (reason) => {
|
|
5117
|
+
for (const activeRun of activeSubagentRuns.values()) {
|
|
5118
|
+
activeRun.cancelled = true;
|
|
5119
|
+
activeRun.cancelReason = reason;
|
|
5120
|
+
requestChildTerminationWithLogging(activeRun.child, {
|
|
5121
|
+
reason,
|
|
5122
|
+
graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
|
|
5123
|
+
context: 'gateway_shutdown_subagent',
|
|
5124
|
+
});
|
|
3777
5125
|
}
|
|
3778
|
-
|
|
3779
|
-
|
|
5126
|
+
for (const activeModelRun of activeModelRuns.values()) {
|
|
5127
|
+
activeModelRun.shutdownRequested = true;
|
|
5128
|
+
activeModelRun.shutdownReason = reason;
|
|
5129
|
+
requestChildTerminationWithLogging(activeModelRun.child, {
|
|
5130
|
+
reason,
|
|
5131
|
+
graceMs: PROCESS_KILL_GRACE_MS,
|
|
5132
|
+
context: 'gateway_shutdown_model_run',
|
|
5133
|
+
});
|
|
5134
|
+
}
|
|
5135
|
+
};
|
|
5136
|
+
const waitForActiveJobsToDrain = async (graceMs) => {
|
|
5137
|
+
const deadline = Date.now() + graceMs;
|
|
5138
|
+
while (Date.now() < deadline) {
|
|
5139
|
+
if (activeClaimedJobs.size === 0 &&
|
|
5140
|
+
activeSubagentRuns.size === 0 &&
|
|
5141
|
+
activeModelRuns.size === 0) {
|
|
5142
|
+
return;
|
|
5143
|
+
}
|
|
5144
|
+
await sleep(100);
|
|
5145
|
+
}
|
|
5146
|
+
};
|
|
5147
|
+
const cancelSubagentRunOnShutdown = async (subagentId, runId, reason) => {
|
|
5148
|
+
const nowIso = new Date().toISOString();
|
|
5149
|
+
const cancelledOutput = buildErrorOutput(reason);
|
|
5150
|
+
const { data: subagentRow, error: subagentLoadError } = await supabase
|
|
5151
|
+
.from('subagents')
|
|
5152
|
+
.select('team_id, gateway_id, metadata')
|
|
5153
|
+
.eq('id', subagentId)
|
|
5154
|
+
.maybeSingle();
|
|
5155
|
+
if (subagentLoadError) {
|
|
5156
|
+
logError('Failed to load subagent during shutdown cancellation', {
|
|
5157
|
+
subagentId,
|
|
5158
|
+
runId,
|
|
5159
|
+
error: subagentLoadError.message,
|
|
5160
|
+
});
|
|
5161
|
+
return;
|
|
5162
|
+
}
|
|
5163
|
+
const { data: runRow, error: runLoadError } = await supabase
|
|
5164
|
+
.from('subagent_runs')
|
|
5165
|
+
.select('team_id, gateway_id, metadata')
|
|
5166
|
+
.eq('id', runId)
|
|
5167
|
+
.maybeSingle();
|
|
5168
|
+
if (runLoadError) {
|
|
5169
|
+
logError('Failed to load subagent run during shutdown cancellation', {
|
|
5170
|
+
subagentId,
|
|
5171
|
+
runId,
|
|
5172
|
+
error: runLoadError.message,
|
|
5173
|
+
});
|
|
5174
|
+
return;
|
|
5175
|
+
}
|
|
5176
|
+
const updatedRunMetadata = {
|
|
5177
|
+
...buildRequestedCancelMetadata({
|
|
5178
|
+
base: (runRow?.metadata ?? {}),
|
|
5179
|
+
run_id: runId,
|
|
5180
|
+
reason,
|
|
5181
|
+
requested_at: nowIso,
|
|
5182
|
+
cancel_support: 'gateway',
|
|
5183
|
+
cancellation_scope: 'gateway',
|
|
5184
|
+
}),
|
|
5185
|
+
cancel_state: 'acknowledged',
|
|
5186
|
+
cancelled: true,
|
|
5187
|
+
last_run_at: nowIso,
|
|
5188
|
+
};
|
|
5189
|
+
const { data: updatedRun, error: runUpdateError } = await supabase
|
|
5190
|
+
.from('subagent_runs')
|
|
5191
|
+
.update({
|
|
5192
|
+
status: 'cancelled',
|
|
5193
|
+
output: asJson(cancelledOutput),
|
|
5194
|
+
error: reason,
|
|
5195
|
+
completed_at: nowIso,
|
|
5196
|
+
metadata: asJson(updatedRunMetadata),
|
|
5197
|
+
})
|
|
5198
|
+
.eq('id', runId)
|
|
5199
|
+
.eq('status', 'running')
|
|
5200
|
+
.select('id')
|
|
5201
|
+
.maybeSingle();
|
|
5202
|
+
if (runUpdateError) {
|
|
5203
|
+
logError('Failed to cancel subagent run during shutdown', {
|
|
5204
|
+
subagentId,
|
|
5205
|
+
runId,
|
|
5206
|
+
error: runUpdateError.message,
|
|
5207
|
+
});
|
|
5208
|
+
}
|
|
5209
|
+
const updatedSubagentMetadata = mergeMetadata((subagentRow?.metadata ?? {}), {
|
|
5210
|
+
cancel_run_id: runId,
|
|
5211
|
+
cancel_requested_at: nowIso,
|
|
5212
|
+
cancel_reason: reason,
|
|
5213
|
+
cancel_state: 'acknowledged',
|
|
5214
|
+
cancel_support: 'gateway',
|
|
5215
|
+
cancellation_scope: 'gateway',
|
|
5216
|
+
cancelled: true,
|
|
5217
|
+
last_run_at: nowIso,
|
|
5218
|
+
});
|
|
5219
|
+
const { error: subagentUpdateError } = await supabase
|
|
5220
|
+
.from('subagents')
|
|
5221
|
+
.update({
|
|
5222
|
+
status: 'cancelled',
|
|
5223
|
+
output: asJson(cancelledOutput),
|
|
5224
|
+
error: reason,
|
|
5225
|
+
completed_at: nowIso,
|
|
5226
|
+
metadata: asJson(updatedSubagentMetadata),
|
|
5227
|
+
})
|
|
5228
|
+
.eq('id', subagentId)
|
|
5229
|
+
.eq('status', 'running');
|
|
5230
|
+
if (subagentUpdateError) {
|
|
5231
|
+
logError('Failed to cancel subagent during shutdown', {
|
|
5232
|
+
subagentId,
|
|
5233
|
+
runId,
|
|
5234
|
+
error: subagentUpdateError.message,
|
|
5235
|
+
});
|
|
5236
|
+
}
|
|
5237
|
+
if (updatedRun?.id && subagentRow?.team_id) {
|
|
5238
|
+
const nextSequence = await getNextSubagentRunEventSequence(supabase, runId);
|
|
5239
|
+
await insertSubagentRunEvent({
|
|
5240
|
+
supabase,
|
|
5241
|
+
runId,
|
|
5242
|
+
subagentId,
|
|
5243
|
+
teamId: subagentRow.team_id,
|
|
5244
|
+
gatewayId: subagentRow.gateway_id,
|
|
5245
|
+
sequence: nextSequence,
|
|
5246
|
+
eventType: 'cancellation_acknowledged',
|
|
5247
|
+
payload: {
|
|
5248
|
+
reason,
|
|
5249
|
+
source: 'gateway_shutdown',
|
|
5250
|
+
cancellation_scope: 'gateway',
|
|
5251
|
+
},
|
|
5252
|
+
});
|
|
5253
|
+
await insertSubagentRunEvent({
|
|
5254
|
+
supabase,
|
|
5255
|
+
runId,
|
|
5256
|
+
subagentId,
|
|
5257
|
+
teamId: subagentRow.team_id,
|
|
5258
|
+
gatewayId: subagentRow.gateway_id,
|
|
5259
|
+
sequence: nextSequence + 1,
|
|
5260
|
+
eventType: 'run_cancelled',
|
|
5261
|
+
payload: {
|
|
5262
|
+
reason,
|
|
5263
|
+
source: 'gateway_shutdown',
|
|
5264
|
+
cancellation_scope: 'gateway',
|
|
5265
|
+
},
|
|
5266
|
+
});
|
|
5267
|
+
}
|
|
5268
|
+
};
|
|
5269
|
+
const finalizeResidualGatewayJobs = async (reason) => {
|
|
5270
|
+
for (const [jobId, jobRow] of activeClaimedJobs.entries()) {
|
|
5271
|
+
if (jobRow.job_type === 'subagent_run') {
|
|
5272
|
+
const subagentId = resolveJobSubagentId(jobRow);
|
|
5273
|
+
const activeRun = subagentId ? activeSubagentRuns.get(subagentId) : null;
|
|
5274
|
+
if (subagentId && activeRun) {
|
|
5275
|
+
await cancelSubagentRunOnShutdown(subagentId, activeRun.runId, reason);
|
|
5276
|
+
}
|
|
5277
|
+
await completeJob(supabase, jobId, 'cancelled', { message: reason, cancelled: true }, reason);
|
|
5278
|
+
continue;
|
|
5279
|
+
}
|
|
5280
|
+
await completeJob(supabase, jobId, 'cancelled', { message: reason, cancelled: true }, reason);
|
|
5281
|
+
}
|
|
5282
|
+
};
|
|
5283
|
+
await connectJobChannel('startup');
|
|
5284
|
+
await reconcileQueuedJobs('startup');
|
|
5285
|
+
reconcileTimer = setInterval(() => {
|
|
5286
|
+
void reconcileQueuedJobs('interval');
|
|
5287
|
+
}, DEFAULT_JOB_RECONCILE_INTERVAL_MS);
|
|
3780
5288
|
const performShutdown = async (signal) => {
|
|
3781
5289
|
if (shutdownInProgress)
|
|
3782
5290
|
return;
|
|
3783
5291
|
shutdownInProgress = true;
|
|
5292
|
+
acceptingJobs = false;
|
|
5293
|
+
CURRENT_ACCEPTING_JOBS = false;
|
|
5294
|
+
jobQueue.clearPending();
|
|
3784
5295
|
logInfo('Gateway shutting down', { signal });
|
|
3785
5296
|
clearInterval(heartbeatTimer);
|
|
5297
|
+
if (reconcileTimer) {
|
|
5298
|
+
clearInterval(reconcileTimer);
|
|
5299
|
+
}
|
|
5300
|
+
if (channelReconnectTimer) {
|
|
5301
|
+
clearTimeout(channelReconnectTimer);
|
|
5302
|
+
}
|
|
3786
5303
|
if (restartTimer) {
|
|
3787
5304
|
clearInterval(restartTimer);
|
|
3788
5305
|
}
|
|
5306
|
+
await syncOperationalHeartbeatIfChanged();
|
|
5307
|
+
requestActiveJobShutdown(`Gateway shutting down (${signal})`);
|
|
5308
|
+
await waitForActiveJobsToDrain(DEFAULT_GATEWAY_SHUTDOWN_GRACE_MS);
|
|
5309
|
+
await finalizeResidualGatewayJobs(`Gateway shutting down (${signal})`);
|
|
5310
|
+
for (const sessionId of activeShellSessions.keys()) {
|
|
5311
|
+
cleanupShellSession(sessionId, `gateway_shutdown_${signal}`);
|
|
5312
|
+
}
|
|
3789
5313
|
heartbeatStatus = 'offline';
|
|
5314
|
+
CURRENT_JOB_INGRESS_STATE = {
|
|
5315
|
+
pending: false,
|
|
5316
|
+
ready: false,
|
|
5317
|
+
};
|
|
3790
5318
|
await sendHeartbeat(supabase, 'offline', currentCapabilities, deviceName);
|
|
3791
5319
|
if (channel) {
|
|
3792
5320
|
await channel.unsubscribe();
|
|
3793
5321
|
}
|
|
3794
5322
|
await removePidFile(options);
|
|
5323
|
+
CURRENT_RUNTIME_OPTIONS = null;
|
|
3795
5324
|
};
|
|
3796
5325
|
async function restartGateway(reason) {
|
|
3797
5326
|
if (shutdownInProgress)
|
|
3798
5327
|
return;
|
|
3799
5328
|
logInfo('Gateway restarting', {
|
|
3800
5329
|
reason,
|
|
3801
|
-
active_jobs:
|
|
5330
|
+
active_jobs: jobQueue.activeCount(),
|
|
3802
5331
|
active_subagent_runs: activeSubagentRuns.size,
|
|
5332
|
+
active_model_runs: activeModelRuns.size,
|
|
5333
|
+
active_shell_sessions: getRunningShellSessionCount(),
|
|
3803
5334
|
});
|
|
3804
5335
|
const child = spawn(process.execPath, process.argv.slice(1), {
|
|
3805
5336
|
stdio: 'inherit',
|
|
@@ -3827,8 +5358,10 @@ async function startGateway(options, configResolution) {
|
|
|
3827
5358
|
if (elapsedMs >= restartMaxWaitMs) {
|
|
3828
5359
|
logInfo('Gateway restart max wait reached; restarting anyway', {
|
|
3829
5360
|
wait_ms: elapsedMs,
|
|
3830
|
-
active_jobs:
|
|
5361
|
+
active_jobs: jobQueue.activeCount(),
|
|
3831
5362
|
active_subagent_runs: activeSubagentRuns.size,
|
|
5363
|
+
active_model_runs: activeModelRuns.size,
|
|
5364
|
+
active_shell_sessions: getRunningShellSessionCount(),
|
|
3832
5365
|
});
|
|
3833
5366
|
void restartGateway('restart max wait exceeded');
|
|
3834
5367
|
}
|
|
@@ -3848,12 +5381,14 @@ async function startGateway(options, configResolution) {
|
|
|
3848
5381
|
gatewayId: config.gatewayId,
|
|
3849
5382
|
teamId: config.teamId,
|
|
3850
5383
|
deviceName,
|
|
5384
|
+
installFullControlEnabled: config.full_control_enabled,
|
|
3851
5385
|
heartbeatIntervalMs: DEFAULT_HEARTBEAT_INTERVAL_MS,
|
|
3852
5386
|
concurrency: GATEWAY_CONCURRENCY,
|
|
3853
5387
|
});
|
|
3854
5388
|
cliSuccess('Gateway running', options, [
|
|
3855
5389
|
{ label: 'Mode', value: 'Foreground' },
|
|
3856
5390
|
{ label: 'Device', value: deviceName },
|
|
5391
|
+
{ label: 'Install full control', value: config.full_control_enabled ? 'Enabled' : 'Disabled' },
|
|
3857
5392
|
{ label: 'Gateway ID', value: config.gatewayId, verboseOnly: true },
|
|
3858
5393
|
{ label: 'Team ID', value: config.teamId, verboseOnly: true },
|
|
3859
5394
|
]);
|
|
@@ -3911,6 +5446,7 @@ async function statusGateway(options) {
|
|
|
3911
5446
|
gatewayId: config.gatewayId,
|
|
3912
5447
|
teamId: config.teamId,
|
|
3913
5448
|
deviceName: config.deviceName ?? undefined,
|
|
5449
|
+
fullControlEnabled: config.full_control_enabled === true,
|
|
3914
5450
|
};
|
|
3915
5451
|
}
|
|
3916
5452
|
catch {
|
|
@@ -3925,6 +5461,14 @@ async function statusGateway(options) {
|
|
|
3925
5461
|
{ label: 'Version', value: version ?? undefined },
|
|
3926
5462
|
{ label: 'Uptime', value: uptime ?? undefined },
|
|
3927
5463
|
{ label: 'Paired', value: configExists ? 'Yes' : 'No' },
|
|
5464
|
+
{
|
|
5465
|
+
label: 'Install full control',
|
|
5466
|
+
value: configSummary == null
|
|
5467
|
+
? undefined
|
|
5468
|
+
: configSummary.fullControlEnabled
|
|
5469
|
+
? 'Enabled'
|
|
5470
|
+
: 'Disabled',
|
|
5471
|
+
},
|
|
3928
5472
|
{ label: 'Device', value: configSummary?.deviceName ?? undefined, verboseOnly: true },
|
|
3929
5473
|
{ label: 'Gateway ID', value: configSummary?.gatewayId ?? undefined, verboseOnly: true },
|
|
3930
5474
|
{ label: 'Team ID', value: configSummary?.teamId ?? undefined, verboseOnly: true },
|
|
@@ -3935,6 +5479,40 @@ async function statusGateway(options) {
|
|
|
3935
5479
|
{ label: 'PID Path', value: formatPathForDisplay(pidPath), verboseOnly: true },
|
|
3936
5480
|
]);
|
|
3937
5481
|
}
|
|
5482
|
+
async function fullControlGateway(options, positional) {
|
|
5483
|
+
const action = (positional[0] ?? 'status').trim().toLowerCase();
|
|
5484
|
+
let config;
|
|
5485
|
+
try {
|
|
5486
|
+
config = await loadConfig(options);
|
|
5487
|
+
}
|
|
5488
|
+
catch (error) {
|
|
5489
|
+
const details = error instanceof Error ? error.message : String(error);
|
|
5490
|
+
throw new GatewayCliError('Gateway config not found. Run "pair" first.', details);
|
|
5491
|
+
}
|
|
5492
|
+
if (action === 'status') {
|
|
5493
|
+
cliInfo('Gateway install full control', options, [
|
|
5494
|
+
{ label: 'Enabled', value: config.full_control_enabled ? 'Yes' : 'No' },
|
|
5495
|
+
{ label: 'Gateway ID', value: config.gatewayId, verboseOnly: true },
|
|
5496
|
+
]);
|
|
5497
|
+
return;
|
|
5498
|
+
}
|
|
5499
|
+
const enabled = action === 'enable' || action === 'on' || action === 'true';
|
|
5500
|
+
const disabled = action === 'disable' || action === 'off' || action === 'false';
|
|
5501
|
+
if (!enabled && !disabled) {
|
|
5502
|
+
throw new GatewayCliError('Unknown full-control command.', 'Use one of: enable, disable, status');
|
|
5503
|
+
}
|
|
5504
|
+
const nextValue = enabled;
|
|
5505
|
+
if (config.full_control_enabled === nextValue) {
|
|
5506
|
+
cliInfo(`Gateway install full control already ${nextValue ? 'enabled' : 'disabled'}.`, options);
|
|
5507
|
+
return;
|
|
5508
|
+
}
|
|
5509
|
+
config.full_control_enabled = nextValue;
|
|
5510
|
+
await saveConfig(config, options);
|
|
5511
|
+
if (CURRENT_CONFIG && CURRENT_CONFIG.gatewayId === config.gatewayId) {
|
|
5512
|
+
CURRENT_CONFIG.full_control_enabled = nextValue;
|
|
5513
|
+
}
|
|
5514
|
+
cliSuccess(`Gateway install full control ${nextValue ? 'enabled' : 'disabled'}.`, options, [{ label: 'Gateway ID', value: config.gatewayId, verboseOnly: true }]);
|
|
5515
|
+
}
|
|
3938
5516
|
async function doctorGateway(options, configResolution) {
|
|
3939
5517
|
const { configPath, configDir } = resolveGatewayPaths(options);
|
|
3940
5518
|
const configExists = fsSync.existsSync(configPath);
|
|
@@ -3953,6 +5531,10 @@ async function doctorGateway(options, configResolution) {
|
|
|
3953
5531
|
cliInfo('Gateway doctor report', options, [
|
|
3954
5532
|
{ label: 'Config dir', value: formatPathForDisplay(configDir) },
|
|
3955
5533
|
{ label: 'Paired', value: configExists ? 'Yes' : 'No' },
|
|
5534
|
+
{
|
|
5535
|
+
label: 'Install full control',
|
|
5536
|
+
value: config == null ? undefined : config.full_control_enabled ? 'Enabled' : 'Disabled',
|
|
5537
|
+
},
|
|
3956
5538
|
{ label: 'Gateway ID', value: config?.gatewayId ?? undefined, verboseOnly: true },
|
|
3957
5539
|
{ label: 'Team ID', value: config?.teamId ?? undefined, verboseOnly: true },
|
|
3958
5540
|
]);
|
|
@@ -3961,6 +5543,7 @@ async function doctorGateway(options, configResolution) {
|
|
|
3961
5543
|
const { providerHealth, anyProviderReady } = await buildCapabilities({
|
|
3962
5544
|
validationMode: 'full',
|
|
3963
5545
|
previousHealth: config?.providerHealth,
|
|
5546
|
+
installFullControlEnabled: config?.full_control_enabled === true,
|
|
3964
5547
|
onProgress: (event) => {
|
|
3965
5548
|
if (event.stage === 'start') {
|
|
3966
5549
|
cliInfo(` ${formatProviderLabel(event.providerId)}...`, options);
|
|
@@ -4101,6 +5684,10 @@ async function run() {
|
|
|
4101
5684
|
await startGateway(parsed.options, configResolution);
|
|
4102
5685
|
return;
|
|
4103
5686
|
}
|
|
5687
|
+
if (parsed.command === 'full-control') {
|
|
5688
|
+
await fullControlGateway(parsed.options, parsed.positional);
|
|
5689
|
+
return;
|
|
5690
|
+
}
|
|
4104
5691
|
if (parsed.command === 'stop') {
|
|
4105
5692
|
await stopGateway(parsed.options);
|
|
4106
5693
|
return;
|