@panorama-ai/gateway 2.28.50 → 2.29.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +28 -9
  2. package/dist/child-process-startup.d.ts +4 -0
  3. package/dist/child-process-startup.d.ts.map +1 -0
  4. package/dist/child-process-startup.js +33 -0
  5. package/dist/child-process-startup.js.map +1 -0
  6. package/dist/cli-providers/codex.d.ts.map +1 -1
  7. package/dist/cli-providers/codex.js +1 -0
  8. package/dist/cli-providers/codex.js.map +1 -1
  9. package/dist/cli-providers/gemini.d.ts.map +1 -1
  10. package/dist/cli-providers/gemini.js +12 -0
  11. package/dist/cli-providers/gemini.js.map +1 -1
  12. package/dist/database.types.d.ts +628 -0
  13. package/dist/database.types.d.ts.map +1 -1
  14. package/dist/database.types.js.map +1 -1
  15. package/dist/gateway-operational-status.d.ts +15 -0
  16. package/dist/gateway-operational-status.d.ts.map +1 -0
  17. package/dist/gateway-operational-status.js +31 -0
  18. package/dist/gateway-operational-status.js.map +1 -0
  19. package/dist/index.js +1897 -310
  20. package/dist/index.js.map +1 -1
  21. package/dist/job-queue.d.ts +23 -0
  22. package/dist/job-queue.d.ts.map +1 -0
  23. package/dist/job-queue.js +70 -0
  24. package/dist/job-queue.js.map +1 -0
  25. package/dist/remote-shell-resource.d.ts +19 -0
  26. package/dist/remote-shell-resource.d.ts.map +1 -0
  27. package/dist/remote-shell-resource.js +51 -0
  28. package/dist/remote-shell-resource.js.map +1 -0
  29. package/dist/stream-json-collector.d.ts +29 -0
  30. package/dist/stream-json-collector.d.ts.map +1 -0
  31. package/dist/stream-json-collector.js +79 -0
  32. package/dist/stream-json-collector.js.map +1 -0
  33. package/dist/subagent-adapters/claude-code.d.ts.map +1 -1
  34. package/dist/subagent-adapters/claude-code.js +54 -12
  35. package/dist/subagent-adapters/claude-code.js.map +1 -1
  36. package/dist/subagent-adapters/claude-support.d.ts +2 -0
  37. package/dist/subagent-adapters/claude-support.d.ts.map +1 -1
  38. package/dist/subagent-adapters/claude-support.js +4 -0
  39. package/dist/subagent-adapters/claude-support.js.map +1 -1
  40. package/dist/subagent-adapters/codex.d.ts.map +1 -1
  41. package/dist/subagent-adapters/codex.js +44 -14
  42. package/dist/subagent-adapters/codex.js.map +1 -1
  43. package/dist/subagent-adapters/gemini.d.ts.map +1 -1
  44. package/dist/subagent-adapters/gemini.js +31 -3
  45. package/dist/subagent-adapters/gemini.js.map +1 -1
  46. package/dist/subagent-adapters/types.d.ts +1 -0
  47. package/dist/subagent-adapters/types.d.ts.map +1 -1
  48. package/dist/subagent-cancel-target.d.ts +15 -0
  49. package/dist/subagent-cancel-target.d.ts.map +1 -0
  50. package/dist/subagent-cancel-target.js +22 -0
  51. package/dist/subagent-cancel-target.js.map +1 -0
  52. package/dist/subagent-full-control.d.ts +7 -0
  53. package/dist/subagent-full-control.d.ts.map +1 -0
  54. package/dist/subagent-full-control.js +47 -0
  55. package/dist/subagent-full-control.js.map +1 -0
  56. package/dist/subagent-process-control.d.ts +9 -0
  57. package/dist/subagent-process-control.d.ts.map +1 -0
  58. package/dist/subagent-process-control.js +32 -0
  59. package/dist/subagent-process-control.js.map +1 -0
  60. package/package.json +5 -5
package/dist/index.js CHANGED
@@ -10,6 +10,7 @@ import path from 'node:path';
10
10
  import process from 'node:process';
11
11
  import { promisify } from 'node:util';
12
12
  import { fileURLToPath } from 'node:url';
13
+ import { buildRequestedCancelMetadata, getSubagentCancelAdapter, resolveCancelRequest, GATEWAY_CAPABILITIES_SCHEMA_VERSION, } from '@panorama/shared';
13
14
  import { buildMissingSupabaseConfigError, resolveSupabaseConfig, } from './supabase-config.js';
14
15
  import { DEFAULT_CLAUDE_SUPPORT, coerceClaudeSupport, parseClaudeSupport, toFlagRecord, } from './subagent-adapters/claude-support.js';
15
16
  import { getSubagentAdapter } from './subagent-adapters/registry.js';
@@ -19,24 +20,41 @@ import { DEFAULT_MODEL_SENTINEL } from './cli-providers/types.js';
19
20
  import { resolveGatewayConfigDir, resolveGatewayConfigPath, resolveGatewayLogPath, resolveGatewayPidPath, resolveGatewayTmpDir, resolveGatewayWorkdirRoot, resolveModelWorkdirRoot, resolveSubagentWorkdirRoot as resolveSubagentWorkdirRootBase, resolveValidationWorkdirRoot, } from './runtime-paths.js';
20
21
  import { sanitizeGatewayDebugPayload } from './debug-redaction.js';
21
22
  import { buildGatewayChildProcessEnv } from './child-process-env.js';
23
+ import { evaluateGatewayProviderFullControlSupport } from './subagent-full-control.js';
24
+ import { GatewayJobQueue } from './job-queue.js';
25
+ import { computeGatewayOperationalStatus, } from './gateway-operational-status.js';
22
26
  import { ensureOwnerOnlyDirectory, ensureOwnerOnlyFile, shouldEnforceOwnerOnlyPermissions, writeOwnerOnlyFile, } from './local-security.js';
27
+ import { requestChildTermination as terminateChildProcess } from './subagent-process-control.js';
28
+ import { validateRemoteShellResourceRow } from './remote-shell-resource.js';
29
+ import { StreamJsonCollector } from './stream-json-collector.js';
30
+ import { waitForChildStartup } from './child-process-startup.js';
23
31
  const execFileAsync = promisify(execFile);
24
32
  const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
25
33
  const DEFAULT_CLAUDE_TIMEOUT_MS = 10_000;
26
34
  const DEFAULT_SUBAGENT_TIMEOUT_MS = 30 * 60_000;
27
35
  const DEFAULT_MODEL_RUN_TIMEOUT_MS = 110_000;
36
+ const MAX_ACTIVE_SHELL_SESSIONS = 64;
37
+ const MAX_SHELL_SESSION_EVENTS = 800;
38
+ const MAX_SHELL_SESSION_BUFFER_BYTES = 1_000_000;
39
+ const SHELL_SESSION_RETENTION_MS = 15 * 60_000;
40
+ const DEFAULT_SHELL_COLUMNS = 120;
41
+ const DEFAULT_SHELL_ROWS = 40;
28
42
  const DEFAULT_GATEWAY_CONCURRENCY = 10;
29
43
  const MAX_SUBAGENT_OUTPUT_BYTES = 5_000_000;
44
+ const MAX_SUBAGENT_STREAM_EVENTS = 2000;
45
+ const MAX_SUBAGENT_STREAM_BUFFER_CHARS = 200_000;
30
46
  const MAX_SUBAGENT_EVENT_PAYLOAD_BYTES = 200_000;
31
47
  const MAX_GATEWAY_LOG_BYTES = 200_000;
32
48
  const MAX_GATEWAY_EVENT_MESSAGE_CHARS = 2000;
33
- const SUBAGENT_EVENT_BATCH_SIZE = 200;
34
49
  const SUBAGENT_CANCEL_KILL_TIMEOUT_MS = 5_000;
35
50
  const PROCESS_KILL_GRACE_MS = 2_000;
51
+ const DEFAULT_GATEWAY_SHUTDOWN_GRACE_MS = 15_000;
52
+ const DEFAULT_JOB_RECONCILE_INTERVAL_MS = 5_000;
53
+ const DEFAULT_JOB_CHANNEL_RECONNECT_MS = 5_000;
36
54
  const PROVIDER_VALIDATION_TIMEOUT_MS = 30_000;
37
- const GATEWAY_CAPABILITIES_SCHEMA_VERSION = 'v1';
38
55
  const DEFAULT_RESTART_CHECK_INTERVAL_MS = 30_000;
39
56
  const MAX_CLAUDE_WRAPPER_BYTES = 4096;
57
+ const GATEWAY_HEARTBEAT_STALE_MS = 90_000;
40
58
  const PROVIDER_VALIDATION_PROMPT = 'Say hi.';
41
59
  const PROVIDER_VALIDATION_SCHEMA = {
42
60
  type: 'object',
@@ -61,7 +79,23 @@ const GATEWAY_CONCURRENCY = (() => {
61
79
  return DEFAULT_GATEWAY_CONCURRENCY;
62
80
  return parsed;
63
81
  })();
82
+ const SUBAGENT_RUN_EVENT_TYPES = [
83
+ 'message',
84
+ 'system',
85
+ 'tool_call',
86
+ 'tool_result',
87
+ 'run_started',
88
+ 'run_progress',
89
+ 'cancellation_requested',
90
+ 'cancellation_acknowledged',
91
+ 'run_completed',
92
+ 'run_failed',
93
+ 'run_cancelled',
94
+ ];
95
+ const SUBAGENT_RUN_EVENT_TYPE_SET = new Set(SUBAGENT_RUN_EVENT_TYPES);
64
96
  const activeSubagentRuns = new Map();
97
+ const activeModelRuns = new Map();
98
+ const activeShellSessions = new Map();
65
99
  const pendingCancelByRunId = new Map();
66
100
  const pendingCancelBySubagentId = new Map();
67
101
  const pendingGatewayEvents = [];
@@ -69,6 +103,9 @@ let CURRENT_CAPABILITIES = null;
69
103
  let CURRENT_DEVICE_NAME = null;
70
104
  let CURRENT_CONFIG = null;
71
105
  let CURRENT_PROVIDER_HEALTH = null;
106
+ let CURRENT_JOB_INGRESS_STATE = null;
107
+ let CURRENT_ACCEPTING_JOBS = true;
108
+ let CURRENT_RUNTIME_OPTIONS = null;
72
109
  let ACTIVE_OPTIONS = null;
73
110
  function parseArgs(argv) {
74
111
  const normalizedArgv = argv[0] === '--' ? argv.slice(1) : argv;
@@ -108,7 +145,7 @@ function parseArgs(argv) {
108
145
  return { command, positional, options };
109
146
  }
110
147
  function printHelp() {
111
- console.log(`\nPanorama Gateway\n\nUsage:\n panorama-gateway pair <PAIRING_CODE> [--device-name \"My Mac\"]\n panorama-gateway start [--device-name \"My Mac\"] [--foreground|--daemon]\n panorama-gateway stop\n panorama-gateway status\n panorama-gateway logs [--lines 200] [--no-follow]\n panorama-gateway doctor\n\nNotes:\n Pair automatically starts the gateway when possible.\n Start runs in the background by default for the built CLI. Use --foreground to keep it attached.\n\nOutput options:\n --verbose, -v Show technical details (paths, IDs, PIDs)\n\nEnvironment options:\n --env <local|dev|test|stage|prod> Load .env.<env> from repo root (defaults to .env)\n --env-file <path> Load a specific env file\n PANORAMA_ENV Same as --env\n PANORAMA_ENV_FILE Same as --env-file\n\nCLI overrides:\n --config-dir <path> Override config directory (default: ~/.panorama/gateway)\n --config-path <path> Override gateway config file\n --log-path <path> Override gateway log file\n --pid-path <path> Override gateway pid file\n --claude-cli <path> Override Claude CLI command\n --codex-cli <path> Override Codex CLI command\n --gemini-cli <path> Override Gemini CLI command\n\nEnvironment overrides:\n PANORAMA_GATEWAY_CONFIG_DIR\n PANORAMA_GATEWAY_CONFIG_PATH\n PANORAMA_GATEWAY_LOG_PATH\n PANORAMA_GATEWAY_PID_PATH\n PANORAMA_CLAUDE_CLI or CLAUDE_CLI\n PANORAMA_CODEX_CLI or CODEX_CLI\n PANORAMA_GEMINI_CLI or GEMINI_CLI\n PANORAMA_GATEWAY_AUTO_RESTART (default: on for built CLI, off for dev)\n PANORAMA_GATEWAY_RESTART_CHECK_MS (default: 30000)\n PANORAMA_GATEWAY_RESTART_MAX_WAIT_MS (optional)\n`);
148
+ console.log(`\nPanorama Gateway\n\nUsage:\n panorama-gateway pair <PAIRING_CODE> [--device-name \"My Mac\"]\n panorama-gateway start [--device-name \"My Mac\"] [--foreground|--daemon]\n panorama-gateway stop\n panorama-gateway status\n panorama-gateway logs [--lines 200] [--no-follow]\n panorama-gateway doctor\n panorama-gateway full-control <enable|disable|status>\n\nNotes:\n Pair automatically starts the gateway when possible.\n Start runs in the background by default for the built CLI. Use --foreground to keep it attached.\n\nOutput options:\n --verbose, -v Show technical details (paths, IDs, PIDs)\n\nEnvironment options:\n --env <local|dev|test|stage|prod> Load .env.<env> from repo root (defaults to .env)\n --env-file <path> Load a specific env file\n PANORAMA_ENV Same as --env\n PANORAMA_ENV_FILE Same as --env-file\n\nCLI overrides:\n --config-dir <path> Override config directory (default: ~/.panorama/gateway)\n --config-path <path> Override gateway config file\n --log-path <path> Override gateway log file\n --pid-path <path> Override gateway pid file\n --claude-cli <path> Override Claude CLI command\n --codex-cli <path> Override Codex CLI command\n --gemini-cli <path> Override Gemini CLI command\n\nEnvironment overrides:\n PANORAMA_GATEWAY_CONFIG_DIR\n PANORAMA_GATEWAY_CONFIG_PATH\n PANORAMA_GATEWAY_LOG_PATH\n PANORAMA_GATEWAY_PID_PATH\n PANORAMA_CLAUDE_CLI or CLAUDE_CLI\n PANORAMA_CODEX_CLI or CODEX_CLI\n PANORAMA_GEMINI_CLI or GEMINI_CLI\n PANORAMA_GATEWAY_AUTO_RESTART (default: on for built CLI, off for dev)\n PANORAMA_GATEWAY_RESTART_CHECK_MS (default: 30000)\n PANORAMA_GATEWAY_RESTART_MAX_WAIT_MS (optional)\n`);
112
149
  }
113
150
  function getStringOption(options, key) {
114
151
  const value = options[key];
@@ -116,6 +153,20 @@ function getStringOption(options, key) {
116
153
  return value;
117
154
  return undefined;
118
155
  }
156
+ function parseBooleanInput(value) {
157
+ if (typeof value === 'boolean')
158
+ return value;
159
+ if (typeof value === 'string') {
160
+ const normalized = value.trim().toLowerCase();
161
+ if (normalized === 'true' || normalized === '1' || normalized === 'yes' || normalized === 'on') {
162
+ return true;
163
+ }
164
+ if (normalized === 'false' || normalized === '0' || normalized === 'no' || normalized === 'off') {
165
+ return false;
166
+ }
167
+ }
168
+ return null;
169
+ }
119
170
  function getActiveOptions(options) {
120
171
  return options ?? ACTIVE_OPTIONS ?? {};
121
172
  }
@@ -1110,12 +1161,44 @@ async function loadConfig(options) {
1110
1161
  const { configPath } = resolveGatewayPaths(options);
1111
1162
  await ensureOwnerOnlyFile(configPath);
1112
1163
  const raw = await fs.readFile(configPath, 'utf-8');
1113
- return JSON.parse(raw);
1164
+ const parsed = JSON.parse(raw);
1165
+ return {
1166
+ ...(parsed ?? {}),
1167
+ supabaseUrl: parsed?.supabaseUrl ?? '',
1168
+ supabaseAnonKey: parsed?.supabaseAnonKey ?? '',
1169
+ accessToken: parsed?.accessToken ?? '',
1170
+ refreshToken: parsed?.refreshToken ?? '',
1171
+ gatewayId: parsed?.gatewayId ?? '',
1172
+ teamId: parsed?.teamId ?? '',
1173
+ full_control_enabled: parsed?.full_control_enabled === true,
1174
+ };
1114
1175
  }
1115
1176
  async function saveConfig(config, options) {
1116
1177
  const { configPath } = resolveGatewayPaths(options);
1117
1178
  await writeOwnerOnlyFile(configPath, JSON.stringify(config, null, 2));
1118
1179
  }
1180
+ async function refreshRuntimeInstallFullControl(config) {
1181
+ const options = CURRENT_RUNTIME_OPTIONS ?? undefined;
1182
+ try {
1183
+ const latestConfig = await loadConfig(options);
1184
+ const nextValue = latestConfig.full_control_enabled === true;
1185
+ if (config.full_control_enabled !== nextValue) {
1186
+ config.full_control_enabled = nextValue;
1187
+ if (CURRENT_CONFIG) {
1188
+ CURRENT_CONFIG.full_control_enabled = nextValue;
1189
+ }
1190
+ logInfo('Reloaded gateway install full-control setting', {
1191
+ enabled: nextValue,
1192
+ });
1193
+ }
1194
+ }
1195
+ catch (error) {
1196
+ logError('Failed to refresh gateway install full-control setting', {
1197
+ error: error instanceof Error ? error.message : String(error),
1198
+ });
1199
+ }
1200
+ return config.full_control_enabled === true;
1201
+ }
1119
1202
  function shouldEmitDebugOutput(stream) {
1120
1203
  if (!stream.isTTY)
1121
1204
  return true;
@@ -1237,6 +1320,15 @@ async function pairGateway(code, options, configResolution) {
1237
1320
  if (!body?.access_token || !body?.refresh_token || !body?.gateway_id || !body?.team_id) {
1238
1321
  throw new Error('Pairing response missing required fields');
1239
1322
  }
1323
+ const requestedFullControl = parseBooleanInput(options['full-control']);
1324
+ let existingConfig = null;
1325
+ try {
1326
+ existingConfig = await loadConfig(options);
1327
+ }
1328
+ catch {
1329
+ existingConfig = null;
1330
+ }
1331
+ const installFullControlEnabled = requestedFullControl ?? existingConfig?.full_control_enabled ?? false;
1240
1332
  const config = {
1241
1333
  supabaseUrl,
1242
1334
  supabaseAnonKey,
@@ -1244,6 +1336,7 @@ async function pairGateway(code, options, configResolution) {
1244
1336
  refreshToken: body.refresh_token,
1245
1337
  gatewayId: body.gateway_id,
1246
1338
  teamId: body.team_id,
1339
+ full_control_enabled: installFullControlEnabled,
1247
1340
  deviceName,
1248
1341
  };
1249
1342
  await saveConfig(config, options);
@@ -1255,6 +1348,7 @@ async function pairGateway(code, options, configResolution) {
1255
1348
  await discoverGatewayCliCommands(options);
1256
1349
  const { providerHealth: resolvedHealth, anyProviderReady: ready } = await buildCapabilities({
1257
1350
  validationMode: 'full',
1351
+ installFullControlEnabled: config.full_control_enabled,
1258
1352
  onProgress: (event) => {
1259
1353
  if (event.stage === 'start') {
1260
1354
  cliInfo(`Validating ${formatProviderLabel(event.providerId)}...`, options);
@@ -1282,6 +1376,11 @@ async function pairGateway(code, options, configResolution) {
1282
1376
  { label: 'Gateway ID', value: body.gateway_id, verboseOnly: true },
1283
1377
  { label: 'Team ID', value: body.team_id, verboseOnly: true },
1284
1378
  { label: 'Device', value: deviceName, verboseOnly: true },
1379
+ {
1380
+ label: 'Install full control',
1381
+ value: installFullControlEnabled ? 'Enabled' : 'Disabled',
1382
+ verboseOnly: true,
1383
+ },
1285
1384
  ...(providerHealthSummary
1286
1385
  ? [{ label: 'Provider health', value: providerHealthSummary, verboseOnly: true }]
1287
1386
  : []),
@@ -1567,10 +1666,12 @@ async function buildCapabilities(params) {
1567
1666
  return false;
1568
1667
  return entry.health?.status !== 'unavailable' && entry.health?.status !== 'unhealthy';
1569
1668
  });
1669
+ const gatewayVersion = await getGatewayVersion();
1570
1670
  const runtime = {
1571
1671
  platform: process.platform,
1572
1672
  arch: process.arch,
1573
1673
  node_version: process.version,
1674
+ ...(gatewayVersion ? { gateway_version: gatewayVersion } : {}),
1574
1675
  };
1575
1676
  const host = {
1576
1677
  kind: 'gateway',
@@ -1579,6 +1680,9 @@ async function buildCapabilities(params) {
1579
1680
  if (params?.deviceName) {
1580
1681
  host.label = params.deviceName;
1581
1682
  }
1683
+ host.metadata = {
1684
+ install_full_control_enabled: params?.installFullControlEnabled === true,
1685
+ };
1582
1686
  return {
1583
1687
  capabilities: {
1584
1688
  schema_version: GATEWAY_CAPABILITIES_SCHEMA_VERSION,
@@ -1623,21 +1727,6 @@ function buildProviderFailureMessage(providerId, detail) {
1623
1727
  const detailSuffix = detail ? ` Details: ${detail}` : '';
1624
1728
  return `Gateway provider ${label} is unhealthy. Try another provider or run "panorama-gateway doctor".${detailSuffix}`;
1625
1729
  }
1626
- function isProviderReady(entry) {
1627
- if (!entry)
1628
- return false;
1629
- if (entry.health?.status === 'healthy')
1630
- return true;
1631
- if (!entry.available)
1632
- return false;
1633
- if (!entry.health)
1634
- return true;
1635
- return entry.health.status !== 'unavailable' && entry.health.status !== 'unhealthy';
1636
- }
1637
- function computeGatewayStatusFromCapabilities(providers) {
1638
- const anyReady = Object.values(providers).some((entry) => isProviderReady(entry));
1639
- return anyReady ? 'ready' : 'error';
1640
- }
1641
1730
  function resolveRuntimeProviderHealth(providerId) {
1642
1731
  if (CURRENT_PROVIDER_HEALTH && CURRENT_PROVIDER_HEALTH[providerId]) {
1643
1732
  return CURRENT_PROVIDER_HEALTH[providerId];
@@ -1698,7 +1787,11 @@ async function updateRuntimeProviderHealth(params) {
1698
1787
  ...(CURRENT_PROVIDER_HEALTH ?? {}),
1699
1788
  [params.providerId]: nextHealth,
1700
1789
  };
1701
- const nextStatusOverall = computeGatewayStatusFromCapabilities(providerCapabilities);
1790
+ const nextStatusOverall = computeGatewayOperationalStatus({
1791
+ capabilities: CURRENT_CAPABILITIES,
1792
+ jobIngressState: CURRENT_JOB_INGRESS_STATE,
1793
+ acceptingJobs: CURRENT_ACCEPTING_JOBS,
1794
+ });
1702
1795
  if (CURRENT_CAPABILITIES && CURRENT_DEVICE_NAME) {
1703
1796
  await sendHeartbeat(params.supabase, nextStatusOverall, CURRENT_CAPABILITIES, CURRENT_DEVICE_NAME);
1704
1797
  }
@@ -1728,8 +1821,13 @@ function buildGatewayChildEnv(baseEnv) {
1728
1821
  const tmpDir = resolveGatewayTmpDir();
1729
1822
  return buildGatewayChildProcessEnv({ baseEnv, tmpDir });
1730
1823
  }
1731
- function resolveChildProcessEnv(env) {
1732
- return buildGatewayChildEnv(env);
1824
+ function resolveChildProcessEnv(envOverrides) {
1825
+ const tmpDir = resolveGatewayTmpDir();
1826
+ return buildGatewayChildProcessEnv({
1827
+ baseEnv: process.env,
1828
+ tmpDir,
1829
+ overrides: envOverrides,
1830
+ });
1733
1831
  }
1734
1832
  async function runSubagentCommand(command, args, options) {
1735
1833
  const start = Date.now();
@@ -1781,14 +1879,11 @@ async function runSubagentCommand(command, args, options) {
1781
1879
  const timer = timeoutMs && Number.isFinite(timeoutMs)
1782
1880
  ? setTimeout(() => {
1783
1881
  timedOut = true;
1784
- if (!child.killed) {
1785
- child.kill('SIGTERM');
1786
- setTimeout(() => {
1787
- if (!child.killed) {
1788
- child.kill('SIGKILL');
1789
- }
1790
- }, PROCESS_KILL_GRACE_MS);
1791
- }
1882
+ requestChildTerminationWithLogging(child, {
1883
+ reason: 'process_timeout',
1884
+ graceMs: PROCESS_KILL_GRACE_MS,
1885
+ context: 'runSubagentCommand',
1886
+ });
1792
1887
  }, timeoutMs)
1793
1888
  : null;
1794
1889
  const finalize = (exitCode, error) => {
@@ -1822,65 +1917,23 @@ async function runSubagentCommandStreaming(command, args, options) {
1822
1917
  const start = Date.now();
1823
1918
  const timeoutMs = options.timeoutMs ?? DEFAULT_SUBAGENT_TIMEOUT_MS;
1824
1919
  return await new Promise((resolve) => {
1825
- const stdoutChunks = [];
1826
1920
  const stderrChunks = [];
1827
- const events = [];
1828
- let stdoutBytes = 0;
1829
1921
  let stderrBytes = 0;
1830
- let stdoutTruncated = false;
1831
1922
  let stderrTruncated = false;
1832
1923
  let timedOut = false;
1833
- let buffer = '';
1834
- let sequence = 0;
1924
+ const collector = new StreamJsonCollector({
1925
+ maxOutputBytes: MAX_SUBAGENT_OUTPUT_BYTES,
1926
+ maxEvents: MAX_SUBAGENT_STREAM_EVENTS,
1927
+ maxBufferChars: MAX_SUBAGENT_STREAM_BUFFER_CHARS,
1928
+ });
1835
1929
  const child = spawn(command, args, {
1836
1930
  cwd: options.cwd,
1837
1931
  env: resolveChildProcessEnv(options.env),
1838
1932
  stdio: ['ignore', 'pipe', 'pipe'],
1839
1933
  });
1840
1934
  options.onStart?.(child);
1841
- const captureStdout = (chunk) => {
1842
- if (stdoutBytes >= MAX_SUBAGENT_OUTPUT_BYTES) {
1843
- stdoutTruncated = true;
1844
- return;
1845
- }
1846
- const remaining = MAX_SUBAGENT_OUTPUT_BYTES - stdoutBytes;
1847
- if (chunk.length > remaining) {
1848
- stdoutChunks.push(chunk.slice(0, remaining));
1849
- stdoutBytes = MAX_SUBAGENT_OUTPUT_BYTES;
1850
- stdoutTruncated = true;
1851
- return;
1852
- }
1853
- stdoutChunks.push(chunk);
1854
- stdoutBytes += chunk.length;
1855
- };
1856
- const parseLine = (line) => {
1857
- const trimmed = line.trim();
1858
- if (!trimmed)
1859
- return;
1860
- let event = null;
1861
- try {
1862
- event = JSON.parse(trimmed);
1863
- }
1864
- catch {
1865
- event = null;
1866
- }
1867
- events.push({
1868
- sequence,
1869
- raw: trimmed,
1870
- event,
1871
- });
1872
- sequence += 1;
1873
- };
1874
1935
  child.stdout?.on('data', (chunk) => {
1875
- captureStdout(chunk);
1876
- buffer += chunk.toString('utf-8');
1877
- let newlineIndex = buffer.indexOf('\n');
1878
- while (newlineIndex >= 0) {
1879
- const line = buffer.slice(0, newlineIndex);
1880
- buffer = buffer.slice(newlineIndex + 1);
1881
- parseLine(line);
1882
- newlineIndex = buffer.indexOf('\n');
1883
- }
1936
+ collector.append(chunk);
1884
1937
  });
1885
1938
  child.stderr?.on('data', (chunk) => {
1886
1939
  if (stderrBytes >= MAX_SUBAGENT_OUTPUT_BYTES) {
@@ -1900,36 +1953,30 @@ async function runSubagentCommandStreaming(command, args, options) {
1900
1953
  const timer = timeoutMs && Number.isFinite(timeoutMs)
1901
1954
  ? setTimeout(() => {
1902
1955
  timedOut = true;
1903
- if (!child.killed) {
1904
- child.kill('SIGTERM');
1905
- setTimeout(() => {
1906
- if (!child.killed) {
1907
- child.kill('SIGKILL');
1908
- }
1909
- }, PROCESS_KILL_GRACE_MS);
1910
- }
1956
+ requestChildTerminationWithLogging(child, {
1957
+ reason: 'process_timeout',
1958
+ graceMs: PROCESS_KILL_GRACE_MS,
1959
+ context: 'runSubagentCommandStreaming',
1960
+ });
1911
1961
  }, timeoutMs)
1912
1962
  : null;
1913
1963
  const finalize = (exitCode, error) => {
1914
1964
  if (timer)
1915
1965
  clearTimeout(timer);
1916
- if (buffer.trim().length > 0) {
1917
- parseLine(buffer);
1918
- }
1966
+ const streamResult = collector.finalize();
1919
1967
  const durationMs = Date.now() - start;
1920
- const stdout = Buffer.concat(stdoutChunks).toString('utf-8');
1921
1968
  const stderr = Buffer.concat(stderrChunks).toString('utf-8');
1922
1969
  const ok = !timedOut && exitCode === 0 && !error;
1923
1970
  resolve({
1924
1971
  ok,
1925
- stdout,
1972
+ stdout: streamResult.stdout,
1926
1973
  stderr,
1927
1974
  exitCode,
1928
1975
  durationMs,
1929
1976
  timedOut,
1930
- stdoutTruncated,
1977
+ stdoutTruncated: streamResult.stdoutTruncated || streamResult.eventsTruncated,
1931
1978
  stderrTruncated,
1932
- events,
1979
+ events: streamResult.events,
1933
1980
  error,
1934
1981
  });
1935
1982
  };
@@ -2166,6 +2213,11 @@ function describeGatewayJob(job) {
2166
2213
  if (job.job_type === 'subagent_cancel') {
2167
2214
  return { job_kind: 'subagent_cancel' };
2168
2215
  }
2216
+ if (job.job_type === 'shell_session') {
2217
+ const payload = job.payload ?? {};
2218
+ const action = typeof payload.action === 'string' ? payload.action : null;
2219
+ return { job_kind: 'shell_session', shell_action: action };
2220
+ }
2169
2221
  return { job_kind: job.job_type };
2170
2222
  }
2171
2223
  function inferProviderFromSubagentType(subagentType) {
@@ -2224,15 +2276,86 @@ function classifySubagentRetryable(params) {
2224
2276
  return null;
2225
2277
  }
2226
2278
  async function getNextRunSequence(supabase, subagentId) {
2227
- const { data } = await supabase
2228
- .from('subagent_runs')
2229
- .select('run_sequence')
2230
- .eq('subagent_id', subagentId)
2231
- .order('run_sequence', { ascending: false })
2279
+ const { data, error } = await supabase.rpc('allocate_subagent_run_sequence', {
2280
+ p_subagent_id: subagentId,
2281
+ });
2282
+ if (error) {
2283
+ logError('Failed to allocate subagent run sequence; falling back to read-based sequence', {
2284
+ subagentId,
2285
+ error: error.message,
2286
+ });
2287
+ const { data: fallback } = await supabase
2288
+ .from('subagent_runs')
2289
+ .select('run_sequence')
2290
+ .eq('subagent_id', subagentId)
2291
+ .order('run_sequence', { ascending: false })
2292
+ .limit(1)
2293
+ .maybeSingle();
2294
+ const lastSequence = fallback && typeof fallback.run_sequence === 'number' ? fallback.run_sequence : 0;
2295
+ return lastSequence + 1;
2296
+ }
2297
+ return typeof data === 'number' && Number.isFinite(data) ? data : 1;
2298
+ }
2299
+ async function getNextSubagentRunEventSequence(supabase, runId) {
2300
+ const { data, error } = await supabase
2301
+ .from('subagent_run_events')
2302
+ .select('sequence')
2303
+ .eq('run_id', runId)
2304
+ .order('sequence', { ascending: false })
2232
2305
  .limit(1)
2233
2306
  .maybeSingle();
2234
- const lastSequence = data && typeof data.run_sequence === 'number' ? data.run_sequence : 0;
2235
- return lastSequence + 1;
2307
+ if (error) {
2308
+ logError('Failed to fetch subagent run event sequence; defaulting to 0', {
2309
+ runId,
2310
+ error: error.message,
2311
+ });
2312
+ return 0;
2313
+ }
2314
+ if (!data || typeof data.sequence !== 'number' || !Number.isFinite(data.sequence)) {
2315
+ return 0;
2316
+ }
2317
+ return data.sequence + 1;
2318
+ }
2319
+ function normalizeSubagentRunEventType(value, fallback = 'system') {
2320
+ if (typeof value !== 'string')
2321
+ return fallback;
2322
+ const normalized = value.trim().toLowerCase();
2323
+ return SUBAGENT_RUN_EVENT_TYPE_SET.has(normalized) ? normalized : fallback;
2324
+ }
2325
+ async function insertSubagentRunEvent(params) {
2326
+ const normalizedType = normalizeSubagentRunEventType(params.eventType, 'system');
2327
+ const insertWithSequence = async (sequence) => params.supabase.from('subagent_run_events').insert({
2328
+ run_id: params.runId,
2329
+ subagent_id: params.subagentId,
2330
+ team_id: params.teamId,
2331
+ gateway_id: params.gatewayId,
2332
+ sequence,
2333
+ event_type: normalizedType,
2334
+ payload: asJson(trimEventPayload(params.payload)),
2335
+ });
2336
+ const { error } = await insertWithSequence(params.sequence);
2337
+ if (!error)
2338
+ return params.sequence;
2339
+ if (error.code === '23505') {
2340
+ const retrySequence = await getNextSubagentRunEventSequence(params.supabase, params.runId);
2341
+ const { error: retryError } = await insertWithSequence(retrySequence);
2342
+ if (!retryError)
2343
+ return retrySequence;
2344
+ logError('Failed to insert subagent run event after retry', {
2345
+ runId: params.runId,
2346
+ subagentId: params.subagentId,
2347
+ event_type: normalizedType,
2348
+ error: retryError.message,
2349
+ });
2350
+ return null;
2351
+ }
2352
+ logError('Failed to insert subagent run event', {
2353
+ runId: params.runId,
2354
+ subagentId: params.subagentId,
2355
+ event_type: normalizedType,
2356
+ error: error.message,
2357
+ });
2358
+ return null;
2236
2359
  }
2237
2360
  function mergeMetadata(base, update) {
2238
2361
  return {
@@ -2243,19 +2366,61 @@ function mergeMetadata(base, update) {
2243
2366
  function asJson(value) {
2244
2367
  return value;
2245
2368
  }
2369
+ async function loadGatewayControlState(supabase, config) {
2370
+ const { data, error } = await supabase
2371
+ .from('team_gateways')
2372
+ .select('full_control_enabled, status, last_seen_at')
2373
+ .eq('id', config.gatewayId)
2374
+ .eq('team_id', config.teamId)
2375
+ .maybeSingle();
2376
+ if (error) {
2377
+ logError('Failed to load gateway control state', {
2378
+ gatewayId: config.gatewayId,
2379
+ teamId: config.teamId,
2380
+ error: error.message,
2381
+ });
2382
+ return null;
2383
+ }
2384
+ if (!data)
2385
+ return null;
2386
+ return {
2387
+ full_control_enabled: data.full_control_enabled === true,
2388
+ status: typeof data.status === 'string' ? data.status : 'unknown',
2389
+ last_seen_at: typeof data.last_seen_at === 'string' ? data.last_seen_at : null,
2390
+ };
2391
+ }
2392
+ function isHeartbeatFresh(lastSeenAt) {
2393
+ if (!lastSeenAt)
2394
+ return false;
2395
+ const parsed = Date.parse(lastSeenAt);
2396
+ if (!Number.isFinite(parsed))
2397
+ return false;
2398
+ return Date.now() - parsed <= GATEWAY_HEARTBEAT_STALE_MS;
2399
+ }
2400
+ function requestChildTerminationWithLogging(child, options) {
2401
+ terminateChildProcess(child, {
2402
+ reason: options.reason,
2403
+ graceMs: options.graceMs,
2404
+ onError: (phase, error) => {
2405
+ logError('Failed to send child process termination signal', {
2406
+ context: options.context,
2407
+ reason: options.reason,
2408
+ phase,
2409
+ error: error instanceof Error ? error.message : String(error),
2410
+ });
2411
+ },
2412
+ });
2413
+ }
2246
2414
  function requestSubagentCancel(subagentId, runId, reason) {
2247
2415
  const activeRun = activeSubagentRuns.get(subagentId);
2248
2416
  if (activeRun && (!runId || activeRun.runId === runId)) {
2249
2417
  activeRun.cancelled = true;
2250
2418
  activeRun.cancelReason = reason;
2251
- if (activeRun.child) {
2252
- activeRun.child.kill('SIGTERM');
2253
- setTimeout(() => {
2254
- if (!activeRun.child?.killed) {
2255
- activeRun.child?.kill('SIGKILL');
2256
- }
2257
- }, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
2258
- }
2419
+ requestChildTerminationWithLogging(activeRun.child, {
2420
+ reason,
2421
+ graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
2422
+ context: 'gateway_cancel_request',
2423
+ });
2259
2424
  return { active: true };
2260
2425
  }
2261
2426
  if (runId) {
@@ -2273,6 +2438,9 @@ function resolvePendingCancel(subagentId, runId, metadata) {
2273
2438
  ? metadata.cancel_reason
2274
2439
  : 'Cancelled';
2275
2440
  }
2441
+ return resolvePendingCancelFromQueue(subagentId, runId);
2442
+ }
2443
+ function resolvePendingCancelFromQueue(subagentId, runId) {
2276
2444
  const pendingByRun = pendingCancelByRunId.get(runId);
2277
2445
  if (pendingByRun)
2278
2446
  return pendingByRun.reason;
@@ -2281,6 +2449,13 @@ function resolvePendingCancel(subagentId, runId, metadata) {
2281
2449
  return pendingBySubagent.reason;
2282
2450
  return null;
2283
2451
  }
2452
+ function consumePendingCancelFromQueue(subagentId, runId) {
2453
+ const reason = resolvePendingCancelFromQueue(subagentId, runId);
2454
+ if (!reason)
2455
+ return null;
2456
+ clearPendingCancel(subagentId, runId);
2457
+ return reason;
2458
+ }
2284
2459
  function clearPendingCancel(subagentId, runId) {
2285
2460
  pendingCancelByRunId.delete(runId);
2286
2461
  pendingCancelBySubagentId.delete(subagentId);
@@ -2294,7 +2469,17 @@ function buildErrorOutput(message) {
2294
2469
  };
2295
2470
  }
2296
2471
  function isJobTargeted(job, gatewayId) {
2297
- return !job.gateway_id || job.gateway_id === gatewayId;
2472
+ return job.gateway_id === gatewayId;
2473
+ }
2474
+ function resolveJobSubagentId(job) {
2475
+ if (job.job_type !== 'subagent_run')
2476
+ return null;
2477
+ const payload = job.payload ?? {};
2478
+ return typeof payload.subagent_id === 'string'
2479
+ ? payload.subagent_id
2480
+ : typeof payload.subagentId === 'string'
2481
+ ? payload.subagentId
2482
+ : null;
2298
2483
  }
2299
2484
  async function sendHeartbeat(supabase, status, capabilities, deviceName) {
2300
2485
  const { error } = await supabase.functions.invoke('gateway-heartbeat', {
@@ -2322,7 +2507,7 @@ async function claimJob(supabase, config, job) {
2322
2507
  .eq('id', job.id)
2323
2508
  .eq('team_id', config.teamId)
2324
2509
  .eq('status', 'queued')
2325
- .or(`gateway_id.is.null,gateway_id.eq.${config.gatewayId}`)
2510
+ .eq('gateway_id', config.gatewayId)
2326
2511
  .select('*')
2327
2512
  .maybeSingle();
2328
2513
  if (error) {
@@ -2410,7 +2595,7 @@ async function handleDiagnosticJob() {
2410
2595
  },
2411
2596
  };
2412
2597
  }
2413
- async function handleSubagentCancelJob(supabase, job) {
2598
+ async function handleSubagentCancelJob(supabase, config, job) {
2414
2599
  const payload = job.payload ?? {};
2415
2600
  const subagentId = typeof payload.subagent_id === 'string'
2416
2601
  ? payload.subagent_id
@@ -2432,12 +2617,35 @@ async function handleSubagentCancelJob(supabase, job) {
2432
2617
  const reason = typeof payload.reason === 'string' && payload.reason.trim().length > 0
2433
2618
  ? payload.reason.trim()
2434
2619
  : 'Cancelled';
2435
- const { data: subagent } = await supabase
2620
+ const { data: subagent, error: subagentError } = await supabase
2436
2621
  .from('subagents')
2437
- .select('status, metadata, input')
2622
+ .select('id, team_id, gateway_id, subagent_type, status, metadata')
2438
2623
  .eq('id', subagentId)
2439
2624
  .maybeSingle();
2440
- if (!subagent || subagent.status !== 'running') {
2625
+ if (subagentError) {
2626
+ return {
2627
+ ok: false,
2628
+ result: { message: 'Failed to load subagent for cancellation', subagent_id: subagentId },
2629
+ error: subagentError.message,
2630
+ };
2631
+ }
2632
+ if (!subagent) {
2633
+ return {
2634
+ ok: true,
2635
+ result: {
2636
+ subagent_id: subagentId,
2637
+ run_id: runId,
2638
+ cancelled_active_run: false,
2639
+ reason,
2640
+ cancel_result: 'no_active_run',
2641
+ cancel_state: 'none',
2642
+ cancel_supported: false,
2643
+ status: 'not_found',
2644
+ message: 'Subagent not found; no active run to cancel.',
2645
+ },
2646
+ };
2647
+ }
2648
+ if (subagent.gateway_id !== config.gatewayId) {
2441
2649
  return {
2442
2650
  ok: true,
2443
2651
  result: {
@@ -2445,72 +2653,216 @@ async function handleSubagentCancelJob(supabase, job) {
2445
2653
  run_id: runId,
2446
2654
  cancelled_active_run: false,
2447
2655
  reason,
2448
- message: 'Subagent not running',
2656
+ cancel_result: 'no_active_run',
2657
+ cancel_state: 'none',
2658
+ cancel_supported: false,
2659
+ status: 'running',
2660
+ message: 'Subagent is assigned to another gateway; skipping stale cancel request.',
2449
2661
  },
2450
2662
  };
2451
2663
  }
2452
2664
  const metadata = (subagent.metadata ?? {});
2453
- const input = (subagent.input ?? {});
2454
- const resolvedRunId = runId ||
2455
- (typeof metadata.current_run_id === 'string' ? metadata.current_run_id : null) ||
2456
- (typeof input.run_id === 'string' ? input.run_id : null);
2457
- const { active } = requestSubagentCancel(subagentId, resolvedRunId, reason);
2458
- if (resolvedRunId) {
2459
- const nowIso = new Date().toISOString();
2460
- const cancelledOutput = buildErrorOutput(reason);
2665
+ const status = typeof subagent.status === 'string' ? subagent.status : 'unknown';
2666
+ const hasGateway = !!subagent.gateway_id;
2667
+ const cancelAdapter = getSubagentCancelAdapter(subagent.subagent_type);
2668
+ const cancelSupported = cancelAdapter.supported &&
2669
+ (cancelAdapter.support !== 'gateway' || hasGateway);
2670
+ const cancelSupportMode = cancelAdapter.support;
2671
+ const cancellationScope = cancelAdapter.scope;
2672
+ let activeRun = null;
2673
+ if (status === 'running') {
2461
2674
  const { data: runningRun, error: runningRunError } = await supabase
2462
2675
  .from('subagent_runs')
2463
- .select('id, status')
2464
- .eq('id', resolvedRunId)
2676
+ .select('id, metadata')
2677
+ .eq('subagent_id', subagentId)
2465
2678
  .eq('status', 'running')
2679
+ .order('run_sequence', { ascending: false })
2680
+ .limit(1)
2466
2681
  .maybeSingle();
2467
2682
  if (runningRunError) {
2468
- logError('Failed to fetch running subagent run for cancel', {
2683
+ return {
2684
+ ok: false,
2685
+ result: {
2686
+ message: 'Failed to resolve active subagent run for cancellation',
2687
+ subagent_id: subagentId,
2688
+ },
2469
2689
  error: runningRunError.message,
2470
- subagentId,
2471
- runId: resolvedRunId,
2472
- });
2690
+ };
2691
+ }
2692
+ activeRun = runningRun;
2693
+ }
2694
+ const resolution = resolveCancelRequest({
2695
+ subagent_status: status,
2696
+ requested_run_id: runId,
2697
+ active_run_id: activeRun?.id ?? null,
2698
+ cancel_supported: cancelSupported,
2699
+ });
2700
+ if (!resolution.accepted) {
2701
+ return {
2702
+ ok: true,
2703
+ result: {
2704
+ subagent_id: subagentId,
2705
+ run_id: resolution.resolved_run_id,
2706
+ active_run_id: activeRun?.id ?? null,
2707
+ cancelled_active_run: false,
2708
+ reason,
2709
+ cancel_result: resolution.cancel_result,
2710
+ cancel_state: resolution.cancel_state,
2711
+ cancel_supported: cancelSupported,
2712
+ status: resolution.status,
2713
+ message: resolution.message,
2714
+ },
2715
+ };
2716
+ }
2717
+ const resolvedRunId = resolution.resolved_run_id;
2718
+ if (!resolvedRunId) {
2719
+ return {
2720
+ ok: false,
2721
+ result: {
2722
+ message: 'Cancel accepted without an active run_id',
2723
+ subagent_id: subagentId,
2724
+ },
2725
+ error: 'Cancel accepted without an active run_id',
2726
+ };
2727
+ }
2728
+ const nowIso = new Date().toISOString();
2729
+ const mergedMetadata = buildRequestedCancelMetadata({
2730
+ base: metadata,
2731
+ run_id: resolvedRunId,
2732
+ reason,
2733
+ requested_at: nowIso,
2734
+ cancel_support: cancelSupportMode,
2735
+ cancellation_scope: cancellationScope,
2736
+ });
2737
+ const { data: updatedSubagent, error: subagentUpdateError } = await supabase
2738
+ .from('subagents')
2739
+ .update({
2740
+ metadata: asJson(mergedMetadata),
2741
+ })
2742
+ .eq('status', 'running')
2743
+ .eq('id', subagentId)
2744
+ .select('id')
2745
+ .maybeSingle();
2746
+ if (subagentUpdateError) {
2747
+ return {
2748
+ ok: false,
2749
+ result: {
2750
+ message: 'Failed to persist subagent cancellation metadata',
2751
+ subagent_id: subagentId,
2752
+ run_id: resolvedRunId,
2753
+ },
2754
+ error: subagentUpdateError.message,
2755
+ };
2756
+ }
2757
+ if (!updatedSubagent) {
2758
+ const { data: latestSubagent, error: latestSubagentError } = await supabase
2759
+ .from('subagents')
2760
+ .select('status')
2761
+ .eq('id', subagentId)
2762
+ .maybeSingle();
2763
+ if (latestSubagentError) {
2764
+ return {
2765
+ ok: false,
2766
+ result: {
2767
+ message: 'Failed to refresh subagent status after cancel race',
2768
+ subagent_id: subagentId,
2769
+ run_id: resolvedRunId,
2770
+ },
2771
+ error: latestSubagentError.message,
2772
+ };
2473
2773
  }
2474
- if (runningRun?.id) {
2475
- const { error: runUpdateError } = await supabase
2774
+ const latestStatus = typeof latestSubagent?.status === 'string' ? latestSubagent.status : status;
2775
+ let latestActiveRunId = null;
2776
+ if (latestStatus === 'running') {
2777
+ const { data: latestRunningRun, error: latestRunningRunError } = await supabase
2476
2778
  .from('subagent_runs')
2477
- .update({
2478
- status: 'cancelled',
2479
- output: asJson(cancelledOutput),
2480
- error: reason,
2481
- completed_at: nowIso,
2482
- })
2483
- .eq('id', resolvedRunId);
2484
- if (runUpdateError) {
2485
- logError('Failed to mark subagent run as cancelled', {
2486
- error: runUpdateError.message,
2487
- subagentId,
2488
- runId: resolvedRunId,
2489
- });
2490
- }
2491
- const { error: subagentUpdateError } = await supabase
2492
- .from('subagents')
2493
- .update({
2494
- status: 'failed',
2495
- output: asJson(cancelledOutput),
2496
- error: reason,
2497
- completed_at: nowIso,
2498
- metadata: asJson(mergeMetadata(metadata, {
2499
- cancel_run_id: resolvedRunId,
2500
- cancel_requested_at: nowIso,
2501
- cancel_reason: reason,
2502
- })),
2503
- })
2504
- .eq('id', subagentId);
2505
- if (subagentUpdateError) {
2506
- logError('Failed to mark subagent as cancelled', {
2507
- error: subagentUpdateError.message,
2508
- subagentId,
2509
- runId: resolvedRunId,
2510
- });
2779
+ .select('id')
2780
+ .eq('subagent_id', subagentId)
2781
+ .eq('status', 'running')
2782
+ .order('run_sequence', { ascending: false })
2783
+ .limit(1)
2784
+ .maybeSingle();
2785
+ if (latestRunningRunError) {
2786
+ return {
2787
+ ok: false,
2788
+ result: {
2789
+ message: 'Failed to refresh active subagent run after cancel race',
2790
+ subagent_id: subagentId,
2791
+ run_id: resolvedRunId,
2792
+ },
2793
+ error: latestRunningRunError.message,
2794
+ };
2511
2795
  }
2796
+ latestActiveRunId = latestRunningRun?.id ?? null;
2797
+ }
2798
+ const latestResolution = resolveCancelRequest({
2799
+ subagent_status: latestStatus,
2800
+ requested_run_id: runId,
2801
+ active_run_id: latestActiveRunId,
2802
+ cancel_supported: cancelSupported,
2803
+ });
2804
+ return {
2805
+ ok: true,
2806
+ result: {
2807
+ subagent_id: subagentId,
2808
+ run_id: latestResolution.resolved_run_id,
2809
+ cancelled_active_run: false,
2810
+ reason,
2811
+ cancel_result: latestResolution.cancel_result,
2812
+ cancel_state: latestResolution.cancel_state,
2813
+ cancel_supported: cancelSupported,
2814
+ status: latestResolution.status,
2815
+ message: latestResolution.message,
2816
+ },
2817
+ };
2818
+ }
2819
+ if (activeRun) {
2820
+ const runMetadata = (activeRun.metadata ?? {});
2821
+ const { error: runUpdateError } = await supabase
2822
+ .from('subagent_runs')
2823
+ .update({
2824
+ metadata: asJson(buildRequestedCancelMetadata({
2825
+ base: runMetadata,
2826
+ run_id: resolvedRunId,
2827
+ reason,
2828
+ requested_at: nowIso,
2829
+ cancel_support: cancelSupportMode,
2830
+ cancellation_scope: cancellationScope,
2831
+ })),
2832
+ })
2833
+ .eq('id', resolvedRunId)
2834
+ .eq('status', 'running');
2835
+ if (runUpdateError) {
2836
+ return {
2837
+ ok: false,
2838
+ result: {
2839
+ message: 'Failed to persist run cancellation metadata',
2840
+ subagent_id: subagentId,
2841
+ run_id: resolvedRunId,
2842
+ },
2843
+ error: runUpdateError.message,
2844
+ };
2512
2845
  }
2513
2846
  }
2847
+ const { active } = requestSubagentCancel(subagentId, resolvedRunId, reason);
2848
+ if (activeRun) {
2849
+ const nextEventSequence = await getNextSubagentRunEventSequence(supabase, resolvedRunId);
2850
+ await insertSubagentRunEvent({
2851
+ supabase,
2852
+ runId: resolvedRunId,
2853
+ subagentId,
2854
+ teamId: subagent.team_id,
2855
+ gatewayId: subagent.gateway_id,
2856
+ sequence: nextEventSequence,
2857
+ eventType: 'cancellation_requested',
2858
+ payload: {
2859
+ reason,
2860
+ source: 'gateway_cancel_job',
2861
+ active_process: active,
2862
+ cancel_state: 'requested',
2863
+ },
2864
+ });
2865
+ }
2514
2866
  return {
2515
2867
  ok: true,
2516
2868
  result: {
@@ -2518,10 +2870,17 @@ async function handleSubagentCancelJob(supabase, job) {
2518
2870
  run_id: resolvedRunId,
2519
2871
  cancelled_active_run: active,
2520
2872
  reason,
2873
+ cancel_result: 'accepted',
2874
+ cancel_state: 'requested',
2875
+ cancel_supported: cancelSupported,
2876
+ status: resolution.status,
2877
+ message: cancelSupportMode === 'gateway'
2878
+ ? 'Cancel requested and forwarded to gateway.'
2879
+ : 'Cancel requested. Worker will stop the run on its next cancellation check.',
2521
2880
  },
2522
2881
  };
2523
2882
  }
2524
- async function handleSubagentRunJob(supabase, job) {
2883
+ async function handleSubagentRunJob(supabase, config, job) {
2525
2884
  const payload = job.payload ?? {};
2526
2885
  const subagentId = typeof payload.subagent_id === 'string'
2527
2886
  ? payload.subagent_id
@@ -2544,9 +2903,26 @@ async function handleSubagentRunJob(supabase, job) {
2544
2903
  return {
2545
2904
  ok: false,
2546
2905
  result: { message: 'Subagent not found', subagent_id: subagentId },
2906
+ jobStatus: 'cancelled',
2907
+ skipSubagentFailure: true,
2547
2908
  error: subagentError?.message ?? 'Subagent not found',
2548
2909
  };
2549
2910
  }
2911
+ if (subagent.gateway_id !== config.gatewayId) {
2912
+ const message = 'Subagent is assigned to a different gateway';
2913
+ return {
2914
+ ok: false,
2915
+ result: {
2916
+ message,
2917
+ subagent_id: subagentId,
2918
+ assigned_gateway_id: subagent.gateway_id,
2919
+ gateway_id: config.gatewayId,
2920
+ },
2921
+ jobStatus: 'cancelled',
2922
+ skipSubagentFailure: true,
2923
+ error: message,
2924
+ };
2925
+ }
2550
2926
  const subagentType = subagent.subagent_type;
2551
2927
  const adapter = getSubagentAdapter(subagentType);
2552
2928
  if (!adapter) {
@@ -2572,6 +2948,59 @@ async function handleSubagentRunJob(supabase, job) {
2572
2948
  }
2573
2949
  const configPayload = (subagent.config ?? {});
2574
2950
  const metadata = (subagent.metadata ?? {});
2951
+ const subagentFullControlRequested = parseBooleanInput(configPayload.full_control) ??
2952
+ parseBooleanInput(configPayload.fullControl) ??
2953
+ false;
2954
+ const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
2955
+ const gatewayControlState = await loadGatewayControlState(supabase, config);
2956
+ if (!gatewayControlState) {
2957
+ return {
2958
+ ok: false,
2959
+ result: { message: 'Gateway control state unavailable', subagent_id: subagentId },
2960
+ error: 'Gateway control state unavailable',
2961
+ };
2962
+ }
2963
+ const gatewayHeartbeatFresh = gatewayControlState.status === 'ready' &&
2964
+ isHeartbeatFresh(gatewayControlState.last_seen_at);
2965
+ if (!gatewayHeartbeatFresh) {
2966
+ return {
2967
+ ok: false,
2968
+ result: {
2969
+ message: 'Gateway is offline or heartbeat is stale',
2970
+ subagent_id: subagentId,
2971
+ },
2972
+ error: 'Gateway is offline or heartbeat is stale',
2973
+ };
2974
+ }
2975
+ const adapterId = adapter.id;
2976
+ const providerSupport = PROVIDER_CAPABILITIES?.[adapterId]?.supported_flags ??
2977
+ (adapterId === 'claude_code'
2978
+ ? toFlagRecord(CLAUDE_SUPPORT ?? DEFAULT_CLAUDE_SUPPORT)
2979
+ : {});
2980
+ const providerFullControlSupport = evaluateGatewayProviderFullControlSupport(adapterId, providerSupport);
2981
+ const layeredFullControlEnabled = installFullControlEnabled && gatewayControlState.full_control_enabled;
2982
+ const effectiveFullControl = subagentFullControlRequested &&
2983
+ layeredFullControlEnabled &&
2984
+ providerFullControlSupport.supported;
2985
+ let fullControlDeniedReason = null;
2986
+ if (subagentFullControlRequested && !effectiveFullControl) {
2987
+ fullControlDeniedReason = !installFullControlEnabled
2988
+ ? 'Gateway install full control is disabled. Run "panorama-gateway full-control enable" on the host.'
2989
+ : !gatewayControlState.full_control_enabled
2990
+ ? 'Gateway connection full control is disabled by team settings.'
2991
+ : providerFullControlSupport.reason ?? 'Gateway provider full control is unavailable.';
2992
+ }
2993
+ logInfo('Resolved subagent full-control state', {
2994
+ jobId: job.id,
2995
+ subagentId,
2996
+ providerId: adapterId,
2997
+ fullControlRequested: subagentFullControlRequested,
2998
+ installFullControlEnabled,
2999
+ gatewayFullControlEnabled: gatewayControlState.full_control_enabled,
3000
+ providerFullControlSupported: providerFullControlSupport.supported,
3001
+ fullControlDeniedReason,
3002
+ fullControlEffective: effectiveFullControl,
3003
+ });
2575
3004
  const inputPayload = (subagent.input ?? {});
2576
3005
  const inputRunId = typeof inputPayload.run_id === 'string' ? inputPayload.run_id.trim() : null;
2577
3006
  const rawRunSequence = typeof inputPayload.run_sequence === 'number'
@@ -2608,10 +3037,33 @@ async function handleSubagentRunJob(supabase, job) {
2608
3037
  const runMetadata = {
2609
3038
  schema_version: SUBAGENT_SCHEMA_VERSION,
2610
3039
  runner: 'gateway',
2611
- adapter_id: adapter.id,
3040
+ adapter_id: adapterId,
2612
3041
  subagent_type: subagentType,
2613
3042
  run_id: resolvedRunId,
2614
3043
  run_sequence: runSequence,
3044
+ cancel_state: 'none',
3045
+ full_control_requested: subagentFullControlRequested,
3046
+ full_control_effective: effectiveFullControl,
3047
+ full_control_denied_reason: fullControlDeniedReason,
3048
+ };
3049
+ let nextEventSequence = null;
3050
+ const emitRunEvent = async (eventType, eventPayload) => {
3051
+ if (nextEventSequence == null) {
3052
+ nextEventSequence = await getNextSubagentRunEventSequence(supabase, resolvedRunId);
3053
+ }
3054
+ const insertedSequence = await insertSubagentRunEvent({
3055
+ supabase,
3056
+ runId: resolvedRunId,
3057
+ subagentId,
3058
+ teamId: subagent.team_id,
3059
+ gatewayId: subagent.gateway_id,
3060
+ sequence: nextEventSequence,
3061
+ eventType,
3062
+ payload: eventPayload,
3063
+ });
3064
+ if (typeof insertedSequence === 'number') {
3065
+ nextEventSequence = insertedSequence + 1;
3066
+ }
2615
3067
  };
2616
3068
  const pendingCancelReason = resolvePendingCancel(subagentId, resolvedRunId, metadata);
2617
3069
  if (pendingCancelReason) {
@@ -2633,6 +3085,10 @@ async function handleSubagentRunJob(supabase, job) {
2633
3085
  metadata: asJson({
2634
3086
  ...runMetadata,
2635
3087
  cancel_reason: pendingCancelReason,
3088
+ cancel_support: 'gateway',
3089
+ cancel_state: 'acknowledged',
3090
+ cancellation_scope: 'gateway',
3091
+ cancelled: true,
2636
3092
  }),
2637
3093
  });
2638
3094
  if (cancelInsertError) {
@@ -2642,6 +3098,21 @@ async function handleSubagentRunJob(supabase, job) {
2642
3098
  error: cancelInsertError.message,
2643
3099
  };
2644
3100
  }
3101
+ await emitRunEvent('cancellation_requested', {
3102
+ reason: pendingCancelReason,
3103
+ source: 'pending_cancel',
3104
+ stage: 'before_start',
3105
+ });
3106
+ await emitRunEvent('cancellation_acknowledged', {
3107
+ reason: pendingCancelReason,
3108
+ source: 'gateway',
3109
+ cancellation_scope: 'gateway',
3110
+ });
3111
+ await emitRunEvent('run_cancelled', {
3112
+ reason: pendingCancelReason,
3113
+ source: 'gateway',
3114
+ cancellation_scope: 'gateway',
3115
+ });
2645
3116
  return {
2646
3117
  ok: true,
2647
3118
  result: {
@@ -2664,6 +3135,41 @@ async function handleSubagentRunJob(supabase, job) {
2664
3135
  metadata: asJson(runMetadata),
2665
3136
  });
2666
3137
  if (runInsertError) {
3138
+ if (runInsertError.code === '23505') {
3139
+ const { data: existingRunningRun, error: existingRunningRunError } = await supabase
3140
+ .from('subagent_runs')
3141
+ .select('id, status')
3142
+ .eq('subagent_id', subagentId)
3143
+ .eq('status', 'running')
3144
+ .order('run_sequence', { ascending: false })
3145
+ .limit(1)
3146
+ .maybeSingle();
3147
+ if (existingRunningRunError) {
3148
+ return {
3149
+ ok: false,
3150
+ result: {
3151
+ message: 'Failed to resolve conflicting subagent run after insert conflict',
3152
+ subagent_id: subagentId,
3153
+ run_id: resolvedRunId,
3154
+ },
3155
+ error: existingRunningRunError.message,
3156
+ };
3157
+ }
3158
+ if (existingRunningRun?.id && existingRunningRun.id !== resolvedRunId) {
3159
+ return {
3160
+ ok: false,
3161
+ result: {
3162
+ message: 'Skipped duplicate subagent job because another run is already active',
3163
+ subagent_id: subagentId,
3164
+ run_id: resolvedRunId,
3165
+ active_run_id: existingRunningRun.id,
3166
+ },
3167
+ jobStatus: 'cancelled',
3168
+ skipSubagentFailure: true,
3169
+ error: 'Another subagent run is already active',
3170
+ };
3171
+ }
3172
+ }
2667
3173
  return {
2668
3174
  ok: false,
2669
3175
  result: { message: 'Failed to create subagent run', subagent_id: subagentId },
@@ -2675,7 +3181,12 @@ async function handleSubagentRunJob(supabase, job) {
2675
3181
  cancelled: false,
2676
3182
  };
2677
3183
  activeSubagentRuns.set(subagentId, activeRun);
2678
- await supabase
3184
+ const latePendingCancelReason = consumePendingCancelFromQueue(subagentId, resolvedRunId);
3185
+ if (latePendingCancelReason) {
3186
+ activeRun.cancelled = true;
3187
+ activeRun.cancelReason = latePendingCancelReason;
3188
+ }
3189
+ const { error: startSubagentUpdateError } = await supabase
2679
3190
  .from('subagents')
2680
3191
  .update({
2681
3192
  status: 'running',
@@ -2686,41 +3197,94 @@ async function handleSubagentRunJob(supabase, job) {
2686
3197
  })),
2687
3198
  })
2688
3199
  .eq('id', subagentId);
2689
- const failRun = async (message) => {
3200
+ if (startSubagentUpdateError) {
3201
+ activeSubagentRuns.delete(subagentId);
3202
+ return {
3203
+ ok: false,
3204
+ result: {
3205
+ message: 'Failed to mark subagent as running',
3206
+ subagent_id: subagentId,
3207
+ run_id: resolvedRunId,
3208
+ },
3209
+ error: startSubagentUpdateError.message,
3210
+ };
3211
+ }
3212
+ await emitRunEvent('run_started', {
3213
+ runner: 'gateway',
3214
+ adapter_id: adapterId,
3215
+ subagent_type: subagentType,
3216
+ run_id: resolvedRunId,
3217
+ run_sequence: runSequence,
3218
+ });
3219
+ const failRun = async (message, options) => {
2690
3220
  const failureOutput = buildErrorOutput(message);
2691
3221
  const completedAt = new Date().toISOString();
2692
- await supabase
3222
+ const metadataUpdate = mergeMetadata(runMetadata, options?.metadata ?? {});
3223
+ const { error: runFailureUpdateError } = await supabase
2693
3224
  .from('subagent_runs')
2694
3225
  .update({
2695
3226
  status: 'failed',
2696
3227
  output: asJson(failureOutput),
2697
3228
  error: message,
2698
3229
  completed_at: completedAt,
3230
+ metadata: asJson(metadataUpdate),
2699
3231
  })
2700
3232
  .eq('id', resolvedRunId);
2701
- await supabase
3233
+ if (runFailureUpdateError) {
3234
+ logError('Failed to persist failed subagent run state', {
3235
+ subagentId,
3236
+ runId: resolvedRunId,
3237
+ error: runFailureUpdateError.message,
3238
+ });
3239
+ }
3240
+ const { error: subagentFailureUpdateError } = await supabase
2702
3241
  .from('subagents')
2703
3242
  .update({
2704
3243
  status: 'failed',
2705
3244
  output: asJson(failureOutput),
2706
3245
  error: message,
2707
3246
  completed_at: completedAt,
3247
+ metadata: asJson(mergeMetadata(metadata, options?.metadata ?? {})),
2708
3248
  })
2709
3249
  .eq('id', subagentId);
3250
+ if (subagentFailureUpdateError) {
3251
+ logError('Failed to persist failed subagent state', {
3252
+ subagentId,
3253
+ runId: resolvedRunId,
3254
+ error: subagentFailureUpdateError.message,
3255
+ });
3256
+ }
3257
+ await emitRunEvent('run_failed', {
3258
+ reason: message,
3259
+ source: 'gateway',
3260
+ stage: 'pre_execution',
3261
+ });
2710
3262
  activeSubagentRuns.delete(subagentId);
3263
+ const persistenceErrors = [
3264
+ runFailureUpdateError ? `run update: ${runFailureUpdateError.message}` : null,
3265
+ subagentFailureUpdateError ? `subagent update: ${subagentFailureUpdateError.message}` : null,
3266
+ ].filter((value) => !!value);
3267
+ const error = persistenceErrors.length > 0
3268
+ ? `${message} (persistence errors: ${persistenceErrors.join('; ')})`
3269
+ : message;
2711
3270
  return {
2712
3271
  ok: false,
2713
3272
  result: { message, subagent_id: subagentId },
2714
- error: message,
3273
+ error,
2715
3274
  };
2716
3275
  };
3276
+ if (subagentFullControlRequested && !effectiveFullControl) {
3277
+ const deniedReason = fullControlDeniedReason ?? 'Gateway full control is unavailable.';
3278
+ return await failRun(deniedReason, {
3279
+ metadata: {
3280
+ full_control_requested: true,
3281
+ full_control_effective: false,
3282
+ full_control_denied_reason: deniedReason,
3283
+ },
3284
+ });
3285
+ }
2717
3286
  let runPlan;
2718
3287
  try {
2719
- const adapterId = adapter.id;
2720
- const providerSupport = PROVIDER_CAPABILITIES?.[adapterId]?.supported_flags ??
2721
- (adapterId === 'claude_code'
2722
- ? toFlagRecord(CLAUDE_SUPPORT ?? DEFAULT_CLAUDE_SUPPORT)
2723
- : {});
2724
3288
  const command = adapterId === 'codex'
2725
3289
  ? resolveCodexCommand()
2726
3290
  : adapterId === 'gemini'
@@ -2731,6 +3295,7 @@ async function handleSubagentRunJob(supabase, job) {
2731
3295
  config: configPayload,
2732
3296
  command,
2733
3297
  support: providerSupport,
3298
+ fullControl: effectiveFullControl,
2734
3299
  resumeSessionId,
2735
3300
  });
2736
3301
  }
@@ -2764,6 +3329,11 @@ async function handleSubagentRunJob(supabase, job) {
2764
3329
  activeResult = null;
2765
3330
  normalized = null;
2766
3331
  let executionError = null;
3332
+ await emitRunEvent('run_progress', {
3333
+ phase: 'attempt_started',
3334
+ attempt,
3335
+ max_attempts: retryConfig.maxAttempts,
3336
+ });
2767
3337
  try {
2768
3338
  if (runPlan.outputFormat === 'stream-json') {
2769
3339
  streamResult = await runSubagentCommandStreaming(runPlan.command, runPlan.args, {
@@ -2773,12 +3343,11 @@ async function handleSubagentRunJob(supabase, job) {
2773
3343
  onStart: (child) => {
2774
3344
  activeRun.child = child;
2775
3345
  if (activeRun.cancelled) {
2776
- child.kill('SIGTERM');
2777
- setTimeout(() => {
2778
- if (!child.killed) {
2779
- child.kill('SIGKILL');
2780
- }
2781
- }, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
3346
+ requestChildTerminationWithLogging(child, {
3347
+ reason: activeRun.cancelReason ?? 'Cancelled',
3348
+ graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
3349
+ context: 'subagent_run_streaming_on_start',
3350
+ });
2782
3351
  }
2783
3352
  },
2784
3353
  });
@@ -2791,12 +3360,11 @@ async function handleSubagentRunJob(supabase, job) {
2791
3360
  onStart: (child) => {
2792
3361
  activeRun.child = child;
2793
3362
  if (activeRun.cancelled) {
2794
- child.kill('SIGTERM');
2795
- setTimeout(() => {
2796
- if (!child.killed) {
2797
- child.kill('SIGKILL');
2798
- }
2799
- }, SUBAGENT_CANCEL_KILL_TIMEOUT_MS);
3363
+ requestChildTerminationWithLogging(child, {
3364
+ reason: activeRun.cancelReason ?? 'Cancelled',
3365
+ graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
3366
+ context: 'subagent_run_on_start',
3367
+ });
2800
3368
  }
2801
3369
  },
2802
3370
  });
@@ -2866,6 +3434,13 @@ async function handleSubagentRunJob(supabase, job) {
2866
3434
  reason: retryDecision.reason,
2867
3435
  delay_ms: delayMs,
2868
3436
  });
3437
+ await emitRunEvent('run_progress', {
3438
+ phase: 'retry_scheduled',
3439
+ attempt,
3440
+ max_attempts: retryConfig.maxAttempts,
3441
+ reason: retryDecision.reason,
3442
+ delay_ms: delayMs,
3443
+ });
2869
3444
  await sleep(delayMs);
2870
3445
  continue;
2871
3446
  }
@@ -2898,6 +3473,7 @@ async function handleSubagentRunJob(supabase, job) {
2898
3473
  const completedAt = new Date().toISOString();
2899
3474
  const parseStrict = runPlan.outputFormat === 'json';
2900
3475
  const shouldFail = !activeResult.ok || (parseStrict && !!parseError);
3476
+ const effectiveShouldFail = !wasCancelled && shouldFail;
2901
3477
  const stderrMessage = activeResult.stderr && activeResult.stderr.trim().length > 0
2902
3478
  ? activeResult.stderr.trim()
2903
3479
  : null;
@@ -2909,7 +3485,7 @@ async function handleSubagentRunJob(supabase, job) {
2909
3485
  : parseError ?? null;
2910
3486
  const providerId = adapter.id;
2911
3487
  if (!wasCancelled) {
2912
- if (shouldFail) {
3488
+ if (effectiveShouldFail) {
2913
3489
  const stderrInfo = truncateLog(activeResult.stderr ?? '');
2914
3490
  const stdoutInfo = truncateLog(activeResult.stdout ?? '');
2915
3491
  logError('Subagent run failed', {
@@ -2954,6 +3530,9 @@ async function handleSubagentRunJob(supabase, job) {
2954
3530
  }
2955
3531
  }
2956
3532
  const failureOutput = buildErrorOutput(wasCancelled ? cancelReason : errorMessage ?? 'Subagent run failed');
3533
+ const finalStatus = wasCancelled ? 'cancelled' : effectiveShouldFail ? 'failed' : 'completed';
3534
+ const finalOutput = wasCancelled || effectiveShouldFail ? failureOutput : output;
3535
+ const finalError = wasCancelled ? cancelReason : effectiveShouldFail ? errorMessage : null;
2957
3536
  const metadataUpdate = {
2958
3537
  schema_version: SUBAGENT_SCHEMA_VERSION,
2959
3538
  runner: 'gateway',
@@ -2977,6 +3556,12 @@ async function handleSubagentRunJob(supabase, job) {
2977
3556
  cancel_requested_at: null,
2978
3557
  cancel_reason: wasCancelled ? cancelReason : null,
2979
3558
  cancelled: wasCancelled,
3559
+ cancel_state: wasCancelled ? 'acknowledged' : 'none',
3560
+ cancel_support: wasCancelled ? 'gateway' : null,
3561
+ cancellation_scope: wasCancelled ? 'gateway' : null,
3562
+ full_control_requested: subagentFullControlRequested,
3563
+ full_control_effective: effectiveFullControl,
3564
+ full_control_denied_reason: null,
2980
3565
  };
2981
3566
  if (activeResult.stderr) {
2982
3567
  metadataUpdate.stderr = activeResult.stderr;
@@ -2985,69 +3570,95 @@ async function handleSubagentRunJob(supabase, job) {
2985
3570
  const runMetadataUpdate = mergeMetadata(mergeMetadata(runMetadata, normalized.metadata), metadataUpdate);
2986
3571
  if (streamResult?.events?.length && adapter.normalizeStreamEvents) {
2987
3572
  const normalizedEvents = adapter.normalizeStreamEvents(streamResult.events);
2988
- const eventRows = normalizedEvents.map((entry, index) => ({
2989
- run_id: resolvedRunId,
2990
- subagent_id: subagentId,
2991
- team_id: subagent.team_id,
2992
- gateway_id: subagent.gateway_id,
2993
- sequence: index,
2994
- event_type: entry.event_type,
2995
- payload: asJson(trimEventPayload(entry.payload)),
2996
- }));
2997
- for (let i = 0; i < eventRows.length; i += SUBAGENT_EVENT_BATCH_SIZE) {
2998
- const chunk = eventRows.slice(i, i + SUBAGENT_EVENT_BATCH_SIZE);
2999
- const { error: eventsError } = await supabase
3000
- .from('subagent_run_events')
3001
- .insert(chunk);
3002
- if (eventsError) {
3003
- logError('Failed to insert subagent run events', {
3004
- error: eventsError.message,
3005
- subagentId,
3006
- runId: resolvedRunId,
3007
- });
3008
- }
3573
+ for (const entry of normalizedEvents) {
3574
+ await emitRunEvent(entry.event_type, entry.payload);
3009
3575
  }
3010
3576
  }
3011
- await supabase
3577
+ if (wasCancelled) {
3578
+ await emitRunEvent('cancellation_acknowledged', {
3579
+ reason: cancelReason,
3580
+ source: 'gateway',
3581
+ cancellation_scope: 'gateway',
3582
+ });
3583
+ await emitRunEvent('run_cancelled', {
3584
+ reason: cancelReason,
3585
+ source: 'gateway',
3586
+ cancellation_scope: 'gateway',
3587
+ duration_ms: activeResult.durationMs,
3588
+ });
3589
+ }
3590
+ else if (effectiveShouldFail) {
3591
+ await emitRunEvent('run_failed', {
3592
+ reason: errorMessage ?? 'Subagent run failed',
3593
+ duration_ms: activeResult.durationMs,
3594
+ parse_error: parseError,
3595
+ });
3596
+ }
3597
+ else {
3598
+ await emitRunEvent('run_completed', {
3599
+ duration_ms: activeResult.durationMs,
3600
+ parse_error: parseError,
3601
+ retry_count: retryCount,
3602
+ });
3603
+ }
3604
+ const { error: runFinalizeError } = await supabase
3012
3605
  .from('subagent_runs')
3013
3606
  .update({
3014
- status: wasCancelled ? 'cancelled' : shouldFail ? 'failed' : 'completed',
3015
- output: asJson(wasCancelled ? failureOutput : shouldFail ? failureOutput : output),
3016
- error: wasCancelled ? cancelReason : shouldFail ? errorMessage : null,
3607
+ status: finalStatus,
3608
+ output: asJson(finalOutput),
3609
+ error: finalError,
3017
3610
  completed_at: completedAt,
3018
3611
  metadata: asJson(runMetadataUpdate),
3019
3612
  })
3020
3613
  .eq('id', resolvedRunId);
3614
+ if (runFinalizeError) {
3615
+ activeSubagentRuns.delete(subagentId);
3616
+ return {
3617
+ ok: false,
3618
+ result: {
3619
+ message: 'Failed to persist terminal subagent run state',
3620
+ subagent_id: subagentId,
3621
+ run_id: resolvedRunId,
3622
+ },
3623
+ error: runFinalizeError.message,
3624
+ };
3625
+ }
3021
3626
  const { error: subagentUpdateError } = await supabase
3022
3627
  .from('subagents')
3023
3628
  .update({
3024
- status: wasCancelled ? 'failed' : shouldFail ? 'failed' : 'completed',
3025
- output: asJson(wasCancelled ? failureOutput : shouldFail ? failureOutput : output),
3026
- error: wasCancelled ? cancelReason : shouldFail ? errorMessage : null,
3629
+ status: finalStatus,
3630
+ output: asJson(finalOutput),
3631
+ error: finalError,
3027
3632
  completed_at: completedAt,
3028
3633
  metadata: asJson(mergedMetadata),
3029
3634
  })
3030
3635
  .eq('id', subagentId);
3031
3636
  if (subagentUpdateError) {
3032
- logError('Failed to update subagent after run', {
3637
+ activeSubagentRuns.delete(subagentId);
3638
+ return {
3639
+ ok: false,
3640
+ result: {
3641
+ message: 'Failed to update subagent after run',
3642
+ subagent_id: subagentId,
3643
+ run_id: resolvedRunId,
3644
+ },
3033
3645
  error: subagentUpdateError.message,
3034
- subagentId,
3035
- runId: resolvedRunId,
3036
- });
3646
+ };
3037
3647
  }
3038
3648
  activeSubagentRuns.delete(subagentId);
3039
3649
  return {
3040
- ok: !shouldFail,
3650
+ ok: !effectiveShouldFail,
3041
3651
  result: {
3042
3652
  subagent_id: subagentId,
3043
- status: shouldFail ? 'failed' : 'completed',
3044
- output: shouldFail ? null : output,
3045
- error: shouldFail ? errorMessage : null,
3653
+ status: finalStatus,
3654
+ output: effectiveShouldFail ? null : finalOutput,
3655
+ error: finalError,
3046
3656
  },
3047
- error: shouldFail ? errorMessage ?? 'Subagent run failed' : undefined,
3657
+ error: effectiveShouldFail ? errorMessage ?? 'Subagent run failed' : undefined,
3048
3658
  };
3049
3659
  }
3050
3660
  finally {
3661
+ activeSubagentRuns.delete(subagentId);
3051
3662
  if (workDir) {
3052
3663
  await cleanupWorkDir(workDir, workRoot);
3053
3664
  }
@@ -3111,6 +3722,10 @@ async function handleModelRunJob(_supabase, job) {
3111
3722
  });
3112
3723
  const workRoot = resolveModelWorkdirRoot();
3113
3724
  let workDir = '';
3725
+ const activeModelRun = {
3726
+ shutdownRequested: false,
3727
+ };
3728
+ activeModelRuns.set(job.id, activeModelRun);
3114
3729
  try {
3115
3730
  workDir = await createWorkDir(workRoot, [{ value: job.id, label: 'job id' }]);
3116
3731
  assertSafeOutputPath(workDir, runPlan.outputPath);
@@ -3137,6 +3752,16 @@ async function handleModelRunJob(_supabase, job) {
3137
3752
  timeoutMs: runPlan.timeoutMs ?? DEFAULT_MODEL_RUN_TIMEOUT_MS,
3138
3753
  cwd: workDir,
3139
3754
  env: resolvedEnv,
3755
+ onStart: (child) => {
3756
+ activeModelRun.child = child;
3757
+ if (activeModelRun.shutdownRequested) {
3758
+ requestChildTerminationWithLogging(child, {
3759
+ reason: activeModelRun.shutdownReason ?? 'Gateway shutting down',
3760
+ graceMs: PROCESS_KILL_GRACE_MS,
3761
+ context: 'model_run_shutdown',
3762
+ });
3763
+ }
3764
+ },
3140
3765
  });
3141
3766
  const stdoutInfo = truncateLog(runResult.stdout);
3142
3767
  const stderrInfo = truncateLog(runResult.stderr);
@@ -3214,11 +3839,620 @@ async function handleModelRunJob(_supabase, job) {
3214
3839
  };
3215
3840
  }
3216
3841
  finally {
3842
+ activeModelRuns.delete(job.id);
3217
3843
  if (workDir) {
3218
3844
  await cleanupWorkDir(workDir, workRoot);
3219
3845
  }
3220
3846
  }
3221
3847
  }
3848
+ function normalizeCommandArgs(value) {
3849
+ if (value === undefined || value === null)
3850
+ return [];
3851
+ if (!Array.isArray(value))
3852
+ return null;
3853
+ const normalized = [];
3854
+ for (const item of value) {
3855
+ if (typeof item !== 'string')
3856
+ return null;
3857
+ normalized.push(item);
3858
+ }
3859
+ return normalized;
3860
+ }
3861
+ function normalizeCommandEnv(value) {
3862
+ if (value === undefined || value === null)
3863
+ return {};
3864
+ if (!value || typeof value !== 'object' || Array.isArray(value))
3865
+ return null;
3866
+ const normalized = {};
3867
+ for (const [key, raw] of Object.entries(value)) {
3868
+ if (typeof raw === 'string') {
3869
+ normalized[key] = raw;
3870
+ continue;
3871
+ }
3872
+ if (typeof raw === 'number' || typeof raw === 'boolean') {
3873
+ normalized[key] = String(raw);
3874
+ continue;
3875
+ }
3876
+ return null;
3877
+ }
3878
+ return normalized;
3879
+ }
3880
+ function coercePositiveInteger(value, fallback, options) {
3881
+ const min = options?.min ?? 1;
3882
+ const max = options?.max ?? Number.MAX_SAFE_INTEGER;
3883
+ const parsed = typeof value === 'number'
3884
+ ? value
3885
+ : typeof value === 'string'
3886
+ ? Number.parseInt(value, 10)
3887
+ : NaN;
3888
+ if (!Number.isFinite(parsed))
3889
+ return fallback;
3890
+ return Math.max(min, Math.min(max, Math.trunc(parsed)));
3891
+ }
3892
+ function coerceNonNegativeInteger(value, fallback, options) {
3893
+ const max = options?.max ?? Number.MAX_SAFE_INTEGER;
3894
+ const parsed = typeof value === 'number'
3895
+ ? value
3896
+ : typeof value === 'string'
3897
+ ? Number.parseInt(value, 10)
3898
+ : NaN;
3899
+ if (!Number.isFinite(parsed))
3900
+ return fallback;
3901
+ return Math.max(0, Math.min(max, Math.trunc(parsed)));
3902
+ }
3903
+ function getRunningShellSessionCount() {
3904
+ let count = 0;
3905
+ for (const session of activeShellSessions.values()) {
3906
+ if (session.status === 'running')
3907
+ count += 1;
3908
+ }
3909
+ return count;
3910
+ }
3911
+ function estimateShellEventBytes(event) {
3912
+ return Buffer.byteLength(event.data, 'utf-8') + 64;
3913
+ }
3914
+ function appendShellSessionEvent(session, stream, data, timestamp = new Date().toISOString()) {
3915
+ if (!data)
3916
+ return;
3917
+ const event = {
3918
+ sequence: session.nextSequence,
3919
+ stream,
3920
+ data,
3921
+ timestamp,
3922
+ };
3923
+ session.nextSequence += 1;
3924
+ session.events.push(event);
3925
+ session.bufferedBytes += estimateShellEventBytes(event);
3926
+ session.updatedAt = timestamp;
3927
+ while (session.events.length > MAX_SHELL_SESSION_EVENTS ||
3928
+ session.bufferedBytes > MAX_SHELL_SESSION_BUFFER_BYTES) {
3929
+ const removed = session.events.shift();
3930
+ if (!removed)
3931
+ break;
3932
+ session.bufferedBytes = Math.max(0, session.bufferedBytes - estimateShellEventBytes(removed));
3933
+ }
3934
+ }
3935
+ function readShellSessionEvents(session, options) {
3936
+ const sinceSequence = options?.sinceSequence ?? 0;
3937
+ const maxEvents = options?.maxEvents ?? 200;
3938
+ const pending = session.events.filter((event) => event.sequence > sinceSequence);
3939
+ const events = pending.slice(0, maxEvents);
3940
+ return {
3941
+ events,
3942
+ hasMore: pending.length > events.length,
3943
+ latestSequence: session.nextSequence > 0 ? session.nextSequence - 1 : 0,
3944
+ };
3945
+ }
3946
+ function purgeExpiredShellSessions(nowMs = Date.now()) {
3947
+ for (const [sessionId, session] of activeShellSessions.entries()) {
3948
+ if (session.status === 'running')
3949
+ continue;
3950
+ const updatedAtMs = Date.parse(session.updatedAt);
3951
+ if (!Number.isFinite(updatedAtMs)) {
3952
+ activeShellSessions.delete(sessionId);
3953
+ continue;
3954
+ }
3955
+ if (nowMs - updatedAtMs >= SHELL_SESSION_RETENTION_MS) {
3956
+ activeShellSessions.delete(sessionId);
3957
+ }
3958
+ }
3959
+ }
3960
+ async function validateMachineControlState(supabase, config) {
3961
+ const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
3962
+ const gatewayControlState = await loadGatewayControlState(supabase, config);
3963
+ if (!gatewayControlState) {
3964
+ return {
3965
+ ok: false,
3966
+ message: 'Gateway control state unavailable',
3967
+ };
3968
+ }
3969
+ if (!installFullControlEnabled) {
3970
+ return {
3971
+ ok: false,
3972
+ message: 'Gateway install full control is disabled. Run "panorama-gateway full-control enable" on the host.',
3973
+ };
3974
+ }
3975
+ if (!gatewayControlState.full_control_enabled) {
3976
+ return {
3977
+ ok: false,
3978
+ message: 'Gateway connection full control is disabled by team settings.',
3979
+ };
3980
+ }
3981
+ if (gatewayControlState.status !== 'ready' ||
3982
+ !isHeartbeatFresh(gatewayControlState.last_seen_at)) {
3983
+ return {
3984
+ ok: false,
3985
+ message: 'Gateway is offline or heartbeat is stale',
3986
+ };
3987
+ }
3988
+ return { ok: true };
3989
+ }
3990
+ function normalizeShellSessionAction(value) {
3991
+ if (typeof value !== 'string')
3992
+ return null;
3993
+ const normalized = value.trim().toLowerCase();
3994
+ if (normalized === 'open' ||
3995
+ normalized === 'write' ||
3996
+ normalized === 'read' ||
3997
+ normalized === 'close' ||
3998
+ normalized === 'status') {
3999
+ return normalized;
4000
+ }
4001
+ return null;
4002
+ }
4003
+ function resolveDefaultShellCommand() {
4004
+ if (typeof process.env.SHELL === 'string' && process.env.SHELL.trim().length > 0) {
4005
+ return process.env.SHELL.trim();
4006
+ }
4007
+ return '/bin/bash';
4008
+ }
4009
+ function buildShellSessionSummary(session) {
4010
+ return {
4011
+ session_id: session.id,
4012
+ resource_id: session.resourceId,
4013
+ status: session.status,
4014
+ shell: session.shell,
4015
+ shell_args: session.shellArgs,
4016
+ cwd: session.cwd,
4017
+ created_at: session.createdAt,
4018
+ updated_at: session.updatedAt,
4019
+ exit_code: session.exitCode,
4020
+ error: session.error,
4021
+ latest_sequence: session.nextSequence > 0 ? session.nextSequence - 1 : 0,
4022
+ };
4023
+ }
4024
+ function cleanupShellSession(sessionId, reason) {
4025
+ const session = activeShellSessions.get(sessionId);
4026
+ if (!session)
4027
+ return;
4028
+ if (session.child) {
4029
+ requestChildTerminationWithLogging(session.child, {
4030
+ reason,
4031
+ graceMs: PROCESS_KILL_GRACE_MS,
4032
+ context: 'shell_session_cleanup',
4033
+ });
4034
+ session.child = undefined;
4035
+ }
4036
+ session.status = session.status === 'running' ? 'closed' : session.status;
4037
+ session.updatedAt = new Date().toISOString();
4038
+ }
4039
+ async function loadGatewayJobStatus(supabase, jobId) {
4040
+ const { data, error } = await supabase
4041
+ .from('gateway_jobs')
4042
+ .select('status')
4043
+ .eq('id', jobId)
4044
+ .maybeSingle();
4045
+ if (error) {
4046
+ logError('Failed to fetch gateway job status', {
4047
+ jobId,
4048
+ error: error.message,
4049
+ });
4050
+ return null;
4051
+ }
4052
+ if (!data || typeof data.status !== 'string') {
4053
+ return null;
4054
+ }
4055
+ const status = data.status;
4056
+ if (status === 'queued' ||
4057
+ status === 'running' ||
4058
+ status === 'completed' ||
4059
+ status === 'failed' ||
4060
+ status === 'cancelled') {
4061
+ return status;
4062
+ }
4063
+ return null;
4064
+ }
4065
+ async function isGatewayJobCancelled(supabase, jobId) {
4066
+ const status = await loadGatewayJobStatus(supabase, jobId);
4067
+ return status === 'cancelled';
4068
+ }
4069
+ async function loadValidatedRemoteShellResource(supabase, config, resourceId, requiredAccess) {
4070
+ const { data, error } = await supabase
4071
+ .from('team_resources')
4072
+ .select('id, team_id, resource_type, max_access_level, metadata')
4073
+ .eq('id', resourceId)
4074
+ .eq('team_id', config.teamId)
4075
+ .maybeSingle();
4076
+ if (error) {
4077
+ logError('Failed to load remote shell resource for gateway validation', {
4078
+ resourceId,
4079
+ gatewayId: config.gatewayId,
4080
+ teamId: config.teamId,
4081
+ error: error.message,
4082
+ });
4083
+ return { ok: false, message: 'Failed to load remote shell resource' };
4084
+ }
4085
+ return validateRemoteShellResourceRow(data ?? null, {
4086
+ teamId: config.teamId,
4087
+ gatewayId: config.gatewayId,
4088
+ requiredAccess,
4089
+ });
4090
+ }
4091
+ function revokeShellSession(session, message) {
4092
+ appendShellSessionEvent(session, 'system', `Session closed: ${message}`);
4093
+ cleanupShellSession(session.id, 'shell_session_resource_revoked');
4094
+ session.status = 'closed';
4095
+ session.updatedAt = new Date().toISOString();
4096
+ }
4097
+ async function handleShellSessionJob(supabase, config, job) {
4098
+ purgeExpiredShellSessions();
4099
+ if (await isGatewayJobCancelled(supabase, job.id)) {
4100
+ const message = 'Gateway shell session job was cancelled before execution';
4101
+ return {
4102
+ ok: false,
4103
+ result: { message, cancelled: true },
4104
+ error: message,
4105
+ };
4106
+ }
4107
+ const controlValidation = await validateMachineControlState(supabase, config);
4108
+ if (!controlValidation.ok) {
4109
+ const message = controlValidation.message ?? 'Gateway machine control unavailable';
4110
+ return {
4111
+ ok: false,
4112
+ result: { message },
4113
+ error: message,
4114
+ };
4115
+ }
4116
+ const payload = job.payload ?? {};
4117
+ const action = normalizeShellSessionAction(payload.action);
4118
+ if (!action) {
4119
+ return {
4120
+ ok: false,
4121
+ result: { message: 'Invalid shell session action' },
4122
+ error: 'Invalid shell session action',
4123
+ };
4124
+ }
4125
+ const resourceId = typeof payload.resource_id === 'string' && payload.resource_id.trim().length > 0
4126
+ ? payload.resource_id.trim()
4127
+ : null;
4128
+ if (!resourceId) {
4129
+ return {
4130
+ ok: false,
4131
+ result: { message: 'Missing resource_id in payload' },
4132
+ error: 'Missing resource_id in payload',
4133
+ };
4134
+ }
4135
+ if (action === 'open') {
4136
+ const resourceValidation = await loadValidatedRemoteShellResource(supabase, config, resourceId, 'write');
4137
+ if (!resourceValidation.ok) {
4138
+ return {
4139
+ ok: false,
4140
+ result: { message: resourceValidation.message, resource_id: resourceId },
4141
+ error: resourceValidation.message,
4142
+ };
4143
+ }
4144
+ if (getRunningShellSessionCount() >= MAX_ACTIVE_SHELL_SESSIONS) {
4145
+ const message = `Too many active shell sessions (max ${MAX_ACTIVE_SHELL_SESSIONS})`;
4146
+ return {
4147
+ ok: false,
4148
+ result: { message, max_active_sessions: MAX_ACTIVE_SHELL_SESSIONS },
4149
+ error: message,
4150
+ };
4151
+ }
4152
+ const shell = typeof payload.shell === 'string' && payload.shell.trim().length > 0
4153
+ ? payload.shell.trim()
4154
+ : resolveDefaultShellCommand();
4155
+ const shellArgs = normalizeCommandArgs(payload.shell_args);
4156
+ if (!shellArgs) {
4157
+ return {
4158
+ ok: false,
4159
+ result: { message: 'Invalid shell_args in payload (expected string array)' },
4160
+ error: 'Invalid shell_args in payload',
4161
+ };
4162
+ }
4163
+ const env = normalizeCommandEnv(payload.env);
4164
+ if (!env) {
4165
+ return {
4166
+ ok: false,
4167
+ result: { message: 'Invalid env in payload (expected string/number/boolean map)' },
4168
+ error: 'Invalid env in payload',
4169
+ };
4170
+ }
4171
+ const cwd = typeof payload.cwd === 'string' && payload.cwd.trim().length > 0
4172
+ ? payload.cwd.trim()
4173
+ : null;
4174
+ const requestedSessionId = typeof payload.session_id === 'string' && payload.session_id.trim().length > 0
4175
+ ? payload.session_id.trim()
4176
+ : null;
4177
+ let sessionId = requestedSessionId ?? randomUUID();
4178
+ if (requestedSessionId && activeShellSessions.has(requestedSessionId)) {
4179
+ return {
4180
+ ok: false,
4181
+ result: {
4182
+ message: 'Shell session ID already exists',
4183
+ session_id: requestedSessionId,
4184
+ },
4185
+ error: 'Shell session ID already exists',
4186
+ };
4187
+ }
4188
+ if (!requestedSessionId) {
4189
+ let retries = 0;
4190
+ while (activeShellSessions.has(sessionId) && retries < 5) {
4191
+ sessionId = randomUUID();
4192
+ retries += 1;
4193
+ }
4194
+ if (activeShellSessions.has(sessionId)) {
4195
+ return {
4196
+ ok: false,
4197
+ result: {
4198
+ message: 'Failed to allocate unique shell session ID',
4199
+ },
4200
+ error: 'Failed to allocate unique shell session ID',
4201
+ };
4202
+ }
4203
+ }
4204
+ const nowIso = new Date().toISOString();
4205
+ const session = {
4206
+ id: sessionId,
4207
+ teamId: config.teamId,
4208
+ gatewayId: config.gatewayId,
4209
+ resourceId,
4210
+ shell,
4211
+ shellArgs,
4212
+ cwd,
4213
+ status: 'running',
4214
+ createdAt: nowIso,
4215
+ updatedAt: nowIso,
4216
+ exitCode: null,
4217
+ error: null,
4218
+ child: undefined,
4219
+ events: [],
4220
+ nextSequence: 1,
4221
+ bufferedBytes: 0,
4222
+ };
4223
+ const sessionEnv = {
4224
+ ...env,
4225
+ };
4226
+ if (!Object.hasOwn(sessionEnv, 'TERM')) {
4227
+ sessionEnv.TERM = 'xterm-256color';
4228
+ }
4229
+ if (!Object.hasOwn(sessionEnv, 'COLUMNS')) {
4230
+ sessionEnv.COLUMNS = String(DEFAULT_SHELL_COLUMNS);
4231
+ }
4232
+ if (!Object.hasOwn(sessionEnv, 'LINES')) {
4233
+ sessionEnv.LINES = String(DEFAULT_SHELL_ROWS);
4234
+ }
4235
+ try {
4236
+ const child = spawn(shell, shellArgs, {
4237
+ cwd: cwd ?? undefined,
4238
+ env: resolveChildProcessEnv(sessionEnv),
4239
+ stdio: ['pipe', 'pipe', 'pipe'],
4240
+ });
4241
+ child.on('error', (error) => {
4242
+ const now = new Date().toISOString();
4243
+ session.status = 'failed';
4244
+ session.error = error.message;
4245
+ session.updatedAt = now;
4246
+ session.child = undefined;
4247
+ appendShellSessionEvent(session, 'system', `Session error: ${error.message}`, now);
4248
+ });
4249
+ child.on('close', (code) => {
4250
+ const now = new Date().toISOString();
4251
+ session.exitCode = code ?? null;
4252
+ if (session.status === 'running') {
4253
+ session.status = 'exited';
4254
+ }
4255
+ session.updatedAt = now;
4256
+ session.child = undefined;
4257
+ appendShellSessionEvent(session, 'system', `Session exited with code ${code ?? 'unknown'}`, now);
4258
+ });
4259
+ child.stdout?.on('data', (chunk) => {
4260
+ appendShellSessionEvent(session, 'stdout', chunk.toString('utf-8'));
4261
+ });
4262
+ child.stderr?.on('data', (chunk) => {
4263
+ appendShellSessionEvent(session, 'stderr', chunk.toString('utf-8'));
4264
+ });
4265
+ await waitForChildStartup(child);
4266
+ if (session.status !== 'running') {
4267
+ const message = session.error ??
4268
+ (session.exitCode !== null
4269
+ ? `Shell session exited during startup with code ${session.exitCode}`
4270
+ : 'Shell session exited during startup');
4271
+ return {
4272
+ ok: false,
4273
+ result: {
4274
+ message,
4275
+ ...buildShellSessionSummary(session),
4276
+ },
4277
+ error: message,
4278
+ };
4279
+ }
4280
+ session.child = child;
4281
+ activeShellSessions.set(sessionId, session);
4282
+ appendShellSessionEvent(session, 'system', `Session opened (${shell}${shellArgs.length > 0 ? ` ${shellArgs.join(' ')}` : ''})`);
4283
+ return {
4284
+ ok: true,
4285
+ result: {
4286
+ message: 'Shell session opened',
4287
+ ...buildShellSessionSummary(session),
4288
+ default_columns: DEFAULT_SHELL_COLUMNS,
4289
+ default_rows: DEFAULT_SHELL_ROWS,
4290
+ },
4291
+ };
4292
+ }
4293
+ catch (error) {
4294
+ activeShellSessions.delete(sessionId);
4295
+ const message = error instanceof Error ? error.message : String(error);
4296
+ return {
4297
+ ok: false,
4298
+ result: {
4299
+ message: `Failed to open shell session: ${message}`,
4300
+ },
4301
+ error: `Failed to open shell session: ${message}`,
4302
+ };
4303
+ }
4304
+ }
4305
+ const sessionId = typeof payload.session_id === 'string' && payload.session_id.trim().length > 0
4306
+ ? payload.session_id.trim()
4307
+ : null;
4308
+ if (!sessionId) {
4309
+ return {
4310
+ ok: false,
4311
+ result: { message: 'Missing session_id in payload' },
4312
+ error: 'Missing session_id in payload',
4313
+ };
4314
+ }
4315
+ const session = activeShellSessions.get(sessionId);
4316
+ if (!session) {
4317
+ return {
4318
+ ok: false,
4319
+ result: { message: 'Shell session not found or expired', session_id: sessionId },
4320
+ error: 'Shell session not found or expired',
4321
+ };
4322
+ }
4323
+ if (session.teamId !== config.teamId || session.gatewayId !== config.gatewayId) {
4324
+ return {
4325
+ ok: false,
4326
+ result: { message: 'Shell session belongs to another gateway context', session_id: sessionId },
4327
+ error: 'Shell session belongs to another gateway context',
4328
+ };
4329
+ }
4330
+ if (resourceId !== session.resourceId) {
4331
+ return {
4332
+ ok: false,
4333
+ result: { message: 'Shell session resource mismatch', session_id: sessionId },
4334
+ error: 'Shell session resource mismatch',
4335
+ };
4336
+ }
4337
+ const requiredAccess = action === 'read' || action === 'status' ? 'read' : 'write';
4338
+ const resourceValidation = await loadValidatedRemoteShellResource(supabase, config, resourceId, requiredAccess);
4339
+ if (!resourceValidation.ok) {
4340
+ revokeShellSession(session, resourceValidation.message);
4341
+ return {
4342
+ ok: false,
4343
+ result: {
4344
+ message: resourceValidation.message,
4345
+ ...buildShellSessionSummary(session),
4346
+ },
4347
+ error: resourceValidation.message,
4348
+ };
4349
+ }
4350
+ if (action === 'status') {
4351
+ return {
4352
+ ok: true,
4353
+ result: {
4354
+ message: 'Shell session status',
4355
+ ...buildShellSessionSummary(session),
4356
+ },
4357
+ };
4358
+ }
4359
+ if (action === 'read') {
4360
+ const sinceSequence = coerceNonNegativeInteger(payload.since_sequence, 0);
4361
+ const maxEvents = coercePositiveInteger(payload.max_events, 200, {
4362
+ min: 1,
4363
+ max: 1000,
4364
+ });
4365
+ const readResult = readShellSessionEvents(session, { sinceSequence, maxEvents });
4366
+ return {
4367
+ ok: true,
4368
+ result: {
4369
+ message: 'Shell session output',
4370
+ ...buildShellSessionSummary(session),
4371
+ since_sequence: sinceSequence,
4372
+ max_events: maxEvents,
4373
+ has_more: readResult.hasMore,
4374
+ events: readResult.events,
4375
+ latest_sequence: readResult.latestSequence,
4376
+ },
4377
+ };
4378
+ }
4379
+ if (action === 'write') {
4380
+ const input = typeof payload.input === 'string' ? payload.input : null;
4381
+ if (input === null) {
4382
+ return {
4383
+ ok: false,
4384
+ result: { message: 'Missing input in payload', session_id: sessionId },
4385
+ error: 'Missing input in payload',
4386
+ };
4387
+ }
4388
+ if (session.status !== 'running' || !session.child || !session.child.stdin) {
4389
+ const message = `Shell session is not writable (status: ${session.status})`;
4390
+ return {
4391
+ ok: false,
4392
+ result: {
4393
+ message,
4394
+ ...buildShellSessionSummary(session),
4395
+ },
4396
+ error: message,
4397
+ };
4398
+ }
4399
+ const appendNewline = payload.append_newline === true;
4400
+ const toWrite = appendNewline ? `${input}\n` : input;
4401
+ const bytesWritten = Buffer.byteLength(toWrite, 'utf-8');
4402
+ try {
4403
+ await new Promise((resolve, reject) => {
4404
+ session.child?.stdin?.write(toWrite, (error) => {
4405
+ if (error) {
4406
+ reject(error);
4407
+ return;
4408
+ }
4409
+ resolve();
4410
+ });
4411
+ });
4412
+ session.updatedAt = new Date().toISOString();
4413
+ return {
4414
+ ok: true,
4415
+ result: {
4416
+ message: 'Shell session input sent',
4417
+ ...buildShellSessionSummary(session),
4418
+ bytes_written: bytesWritten,
4419
+ append_newline: appendNewline,
4420
+ },
4421
+ };
4422
+ }
4423
+ catch (error) {
4424
+ const message = error instanceof Error ? error.message : String(error);
4425
+ return {
4426
+ ok: false,
4427
+ result: {
4428
+ message: `Failed to write to shell session: ${message}`,
4429
+ ...buildShellSessionSummary(session),
4430
+ },
4431
+ error: `Failed to write to shell session: ${message}`,
4432
+ };
4433
+ }
4434
+ }
4435
+ if (action === 'close') {
4436
+ if (session.status === 'running') {
4437
+ cleanupShellSession(session.id, 'shell_session_close');
4438
+ appendShellSessionEvent(session, 'system', 'Session closed by request');
4439
+ }
4440
+ session.status = 'closed';
4441
+ session.updatedAt = new Date().toISOString();
4442
+ return {
4443
+ ok: true,
4444
+ result: {
4445
+ message: 'Shell session closed',
4446
+ ...buildShellSessionSummary(session),
4447
+ },
4448
+ };
4449
+ }
4450
+ return {
4451
+ ok: false,
4452
+ result: { message: `Unsupported shell session action: ${action}` },
4453
+ error: `Unsupported shell session action: ${action}`,
4454
+ };
4455
+ }
3222
4456
  async function failSubagentRun(supabase, job, errorMessage) {
3223
4457
  const payload = job.payload ?? {};
3224
4458
  const subagentId = typeof payload.subagent_id === 'string'
@@ -3256,15 +4490,7 @@ async function failSubagentRun(supabase, job, errorMessage) {
3256
4490
  .limit(1)
3257
4491
  .maybeSingle();
3258
4492
  if (runningRun?.id) {
3259
- const { data: lastEvent } = await supabase
3260
- .from('subagent_run_events')
3261
- .select('sequence')
3262
- .eq('run_id', runningRun.id)
3263
- .order('sequence', { ascending: false })
3264
- .limit(1)
3265
- .maybeSingle();
3266
- const nextSequence = lastEvent && typeof lastEvent.sequence === 'number' ? lastEvent.sequence + 1 : 0;
3267
- await supabase
4493
+ const { error: runFailUpdateError } = await supabase
3268
4494
  .from('subagent_runs')
3269
4495
  .update({
3270
4496
  status: 'failed',
@@ -3274,15 +4500,24 @@ async function failSubagentRun(supabase, job, errorMessage) {
3274
4500
  metadata: asJson(mergedMetadata),
3275
4501
  })
3276
4502
  .eq('id', runningRun.id);
4503
+ if (runFailUpdateError) {
4504
+ logError('Failed to mark running subagent run as failed after gateway error', {
4505
+ subagentId,
4506
+ runId: runningRun.id,
4507
+ error: runFailUpdateError.message,
4508
+ });
4509
+ }
3277
4510
  if (subagent.gateway_id) {
3278
- await supabase.from('subagent_run_events').insert({
3279
- run_id: runningRun.id,
3280
- subagent_id: subagent.id,
3281
- team_id: subagent.team_id,
3282
- gateway_id: subagent.gateway_id,
4511
+ const nextSequence = await getNextSubagentRunEventSequence(supabase, runningRun.id);
4512
+ await insertSubagentRunEvent({
4513
+ supabase,
4514
+ runId: runningRun.id,
4515
+ subagentId: subagent.id,
4516
+ teamId: subagent.team_id,
4517
+ gatewayId: subagent.gateway_id,
3283
4518
  sequence: nextSequence,
3284
- event_type: 'run_failed',
3285
- payload: asJson({ error: errorMessage }),
4519
+ eventType: 'run_failed',
4520
+ payload: { reason: errorMessage, source: 'gateway_job_failure' },
3286
4521
  });
3287
4522
  }
3288
4523
  }
@@ -3328,6 +4563,7 @@ async function processJob(supabase, config, job) {
3328
4563
  provider: jobDetails.provider ?? undefined,
3329
4564
  model: jobDetails.model ?? undefined,
3330
4565
  subagentType: jobDetails.subagent_type ?? undefined,
4566
+ shellAction: jobDetails.shell_action ?? undefined,
3331
4567
  });
3332
4568
  if (claimed.job_type === 'diagnostic') {
3333
4569
  const diagnostic = await handleDiagnosticJob();
@@ -3340,12 +4576,15 @@ async function processJob(supabase, config, job) {
3340
4576
  return;
3341
4577
  }
3342
4578
  if (claimed.job_type === 'subagent_run') {
3343
- const result = await handleSubagentRunJob(supabase, claimed);
4579
+ const result = await handleSubagentRunJob(supabase, config, claimed);
3344
4580
  if (result.ok) {
3345
4581
  await completeJob(supabase, claimed.id, 'completed', result.result);
3346
4582
  }
3347
4583
  else {
3348
- await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
4584
+ await completeJob(supabase, claimed.id, result.jobStatus ?? 'failed', result.result, result.error);
4585
+ if (!result.skipSubagentFailure) {
4586
+ await failSubagentRun(supabase, claimed, result.error ?? 'Subagent run failed');
4587
+ }
3349
4588
  }
3350
4589
  return;
3351
4590
  }
@@ -3360,7 +4599,17 @@ async function processJob(supabase, config, job) {
3360
4599
  return;
3361
4600
  }
3362
4601
  if (claimed.job_type === 'subagent_cancel') {
3363
- const result = await handleSubagentCancelJob(supabase, claimed);
4602
+ const result = await handleSubagentCancelJob(supabase, config, claimed);
4603
+ if (result.ok) {
4604
+ await completeJob(supabase, claimed.id, 'completed', result.result);
4605
+ }
4606
+ else {
4607
+ await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
4608
+ }
4609
+ return;
4610
+ }
4611
+ if (claimed.job_type === 'shell_session') {
4612
+ const result = await handleShellSessionJob(supabase, config, claimed);
3364
4613
  if (result.ok) {
3365
4614
  await completeJob(supabase, claimed.id, 'completed', result.result);
3366
4615
  }
@@ -3431,6 +4680,7 @@ async function startGateway(options, configResolution) {
3431
4680
  }
3432
4681
  }
3433
4682
  cliInfo('Starting gateway...', options);
4683
+ CURRENT_RUNTIME_OPTIONS = options;
3434
4684
  let config;
3435
4685
  try {
3436
4686
  config = await loadConfig(options);
@@ -3506,10 +4756,11 @@ async function startGateway(options, configResolution) {
3506
4756
  await ensureGatewayTmpDir();
3507
4757
  await discoverGatewayCliCommands(options);
3508
4758
  cliInfo('Validating providers...', options);
3509
- const { capabilities: initialCapabilities, providerHealth, anyProviderReady } = await buildCapabilities({
4759
+ const { capabilities: initialCapabilities, providerHealth } = await buildCapabilities({
3510
4760
  validationMode: 'auto',
3511
4761
  previousHealth: config.providerHealth,
3512
4762
  deviceName,
4763
+ installFullControlEnabled: config.full_control_enabled,
3513
4764
  onProgress: (event) => {
3514
4765
  if (event.stage === 'start') {
3515
4766
  cliInfo(` ${formatProviderLabel(event.providerId)}...`, options);
@@ -3526,8 +4777,16 @@ async function startGateway(options, configResolution) {
3526
4777
  let currentProviderHealth = providerHealth;
3527
4778
  CURRENT_CAPABILITIES = initialCapabilities;
3528
4779
  CURRENT_PROVIDER_HEALTH = providerHealth;
4780
+ CURRENT_JOB_INGRESS_STATE = {
4781
+ pending: true,
4782
+ ready: false,
4783
+ };
3529
4784
  const providerStatus = currentCapabilities.providers;
3530
- const initialStatus = anyProviderReady ? 'ready' : 'error';
4785
+ const initialStatus = computeGatewayOperationalStatus({
4786
+ capabilities: currentCapabilities,
4787
+ jobIngressState: CURRENT_JOB_INGRESS_STATE,
4788
+ acceptingJobs: CURRENT_ACCEPTING_JOBS,
4789
+ });
3531
4790
  logInfo('Gateway capabilities loaded', {
3532
4791
  providers: Object.fromEntries(Object.entries(providerStatus).map(([id, entry]) => [
3533
4792
  id,
@@ -3555,19 +4814,32 @@ async function startGateway(options, configResolution) {
3555
4814
  if (ranFullValidation) {
3556
4815
  await emitProviderHealthEvents(supabase, config, providerHealth, 'startup');
3557
4816
  }
3558
- await sendHeartbeat(supabase, initialStatus, currentCapabilities, deviceName);
3559
4817
  let heartbeatStatus = initialStatus;
4818
+ await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
3560
4819
  await writePidFile(process.pid, options);
3561
4820
  let heartbeatInFlight = false;
4821
+ const syncOperationalHeartbeatIfChanged = async () => {
4822
+ const nextStatus = computeGatewayOperationalStatus({
4823
+ capabilities: currentCapabilities,
4824
+ jobIngressState: CURRENT_JOB_INGRESS_STATE,
4825
+ acceptingJobs: CURRENT_ACCEPTING_JOBS,
4826
+ });
4827
+ if (nextStatus === heartbeatStatus)
4828
+ return;
4829
+ heartbeatStatus = nextStatus;
4830
+ await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
4831
+ };
3562
4832
  const refreshHeartbeat = async () => {
3563
4833
  if (heartbeatInFlight)
3564
4834
  return;
3565
4835
  heartbeatInFlight = true;
3566
4836
  try {
3567
- const { capabilities: nextCapabilities, providerHealth: nextProviderHealth, anyProviderReady: nextReady, } = await buildCapabilities({
4837
+ const installFullControlEnabled = await refreshRuntimeInstallFullControl(config);
4838
+ const { capabilities: nextCapabilities, providerHealth: nextProviderHealth, } = await buildCapabilities({
3568
4839
  validationMode: 'light',
3569
4840
  previousHealth: CURRENT_PROVIDER_HEALTH ?? currentProviderHealth,
3570
4841
  deviceName,
4842
+ installFullControlEnabled,
3571
4843
  onProgress: (event) => {
3572
4844
  if (event.stage === 'start')
3573
4845
  return;
@@ -3584,7 +4856,11 @@ async function startGateway(options, configResolution) {
3584
4856
  config.providerHealth = nextProviderHealth;
3585
4857
  CURRENT_CAPABILITIES = nextCapabilities;
3586
4858
  CURRENT_PROVIDER_HEALTH = nextProviderHealth;
3587
- heartbeatStatus = nextReady ? 'ready' : 'error';
4859
+ heartbeatStatus = computeGatewayOperationalStatus({
4860
+ capabilities: nextCapabilities,
4861
+ jobIngressState: CURRENT_JOB_INGRESS_STATE,
4862
+ acceptingJobs: CURRENT_ACCEPTING_JOBS,
4863
+ });
3588
4864
  await sendHeartbeat(supabase, heartbeatStatus, currentCapabilities, deviceName);
3589
4865
  }
3590
4866
  catch (error) {
@@ -3602,12 +4878,63 @@ async function startGateway(options, configResolution) {
3602
4878
  void refreshHeartbeat();
3603
4879
  }, DEFAULT_HEARTBEAT_INTERVAL_MS);
3604
4880
  let acceptingJobs = true;
4881
+ CURRENT_ACCEPTING_JOBS = true;
3605
4882
  let restartRequested = false;
3606
4883
  let restartDetectedAt = null;
3607
4884
  let restartTimer = null;
3608
- const pendingJobs = [];
3609
- const queuedJobIds = new Set();
3610
- const activeJobIds = new Set();
4885
+ let reconcileTimer = null;
4886
+ let channelReconnectTimer = null;
4887
+ const jobQueue = new GatewayJobQueue(resolveJobSubagentId);
4888
+ const activeClaimedJobs = new Map();
4889
+ let lastJobIngressSuccessAtMs = null;
4890
+ let lastJobIngressError = null;
4891
+ let reconcileInFlight = false;
4892
+ let shutdownInProgress = false;
4893
+ const updateJobIngressState = () => {
4894
+ const ready = lastJobIngressSuccessAtMs !== null &&
4895
+ Date.now() - lastJobIngressSuccessAtMs <= DEFAULT_JOB_RECONCILE_INTERVAL_MS * 3;
4896
+ const pending = lastJobIngressSuccessAtMs === null && lastJobIngressError === null;
4897
+ CURRENT_JOB_INGRESS_STATE = {
4898
+ pending,
4899
+ ready,
4900
+ };
4901
+ };
4902
+ const reconcileQueuedJobs = async (reason) => {
4903
+ if (reconcileInFlight || shutdownInProgress)
4904
+ return;
4905
+ reconcileInFlight = true;
4906
+ try {
4907
+ const { data, error } = await supabase
4908
+ .from('gateway_jobs')
4909
+ .select('*')
4910
+ .eq('team_id', config.teamId)
4911
+ .eq('status', 'queued')
4912
+ .eq('gateway_id', config.gatewayId)
4913
+ .order('created_at', { ascending: true });
4914
+ if (error) {
4915
+ throw error;
4916
+ }
4917
+ lastJobIngressSuccessAtMs = Date.now();
4918
+ lastJobIngressError = null;
4919
+ updateJobIngressState();
4920
+ for (const row of data ?? []) {
4921
+ enqueueJob(row);
4922
+ }
4923
+ await syncOperationalHeartbeatIfChanged();
4924
+ }
4925
+ catch (error) {
4926
+ lastJobIngressError = error instanceof Error ? error.message : String(error);
4927
+ updateJobIngressState();
4928
+ logError('Failed to reconcile queued gateway jobs', {
4929
+ error: lastJobIngressError,
4930
+ reason,
4931
+ });
4932
+ await syncOperationalHeartbeatIfChanged();
4933
+ }
4934
+ finally {
4935
+ reconcileInFlight = false;
4936
+ }
4937
+ };
3611
4938
  const entryPath = process.argv[1] || '';
3612
4939
  let modulePath = '';
3613
4940
  try {
@@ -3635,7 +4962,10 @@ async function startGateway(options, configResolution) {
3635
4962
  const maybeRestart = async () => {
3636
4963
  if (!restartRequested)
3637
4964
  return;
3638
- if (activeJobIds.size > 0 || activeSubagentRuns.size > 0) {
4965
+ if (jobQueue.activeCount() > 0 ||
4966
+ activeSubagentRuns.size > 0 ||
4967
+ activeModelRuns.size > 0 ||
4968
+ getRunningShellSessionCount() > 0) {
3639
4969
  return;
3640
4970
  }
3641
4971
  await restartGateway('all jobs complete');
@@ -3646,13 +4976,16 @@ async function startGateway(options, configResolution) {
3646
4976
  restartRequested = true;
3647
4977
  restartDetectedAt = Date.now();
3648
4978
  acceptingJobs = false;
3649
- pendingJobs.length = 0;
3650
- queuedJobIds.clear();
4979
+ CURRENT_ACCEPTING_JOBS = false;
4980
+ jobQueue.clearPending();
3651
4981
  logInfo('Gateway update detected; draining before restart', {
3652
4982
  reason,
3653
- active_jobs: activeJobIds.size,
4983
+ active_jobs: jobQueue.activeCount(),
3654
4984
  active_subagent_runs: activeSubagentRuns.size,
4985
+ active_model_runs: activeModelRuns.size,
4986
+ active_shell_sessions: getRunningShellSessionCount(),
3655
4987
  });
4988
+ void syncOperationalHeartbeatIfChanged();
3656
4989
  void maybeRestart();
3657
4990
  };
3658
4991
  const checkForUpgrade = () => {
@@ -3680,7 +5013,7 @@ async function startGateway(options, configResolution) {
3680
5013
  const claimed = await claimJob(supabase, config, job);
3681
5014
  if (!claimed)
3682
5015
  return;
3683
- const result = await handleSubagentCancelJob(supabase, claimed);
5016
+ const result = await handleSubagentCancelJob(supabase, config, claimed);
3684
5017
  if (result.ok) {
3685
5018
  await completeJob(supabase, claimed.id, 'completed', result.result);
3686
5019
  }
@@ -3688,7 +5021,8 @@ async function startGateway(options, configResolution) {
3688
5021
  await completeJob(supabase, claimed.id, 'failed', result.result, result.error);
3689
5022
  }
3690
5023
  };
3691
- const processJobWithHandling = async (nextJob) => {
5024
+ const processJobWithHandling = async (nextJob, reservedSubagentId) => {
5025
+ activeClaimedJobs.set(nextJob.id, nextJob);
3692
5026
  try {
3693
5027
  await processJob(supabase, config, nextJob);
3694
5028
  }
@@ -3704,7 +5038,8 @@ async function startGateway(options, configResolution) {
3704
5038
  }
3705
5039
  }
3706
5040
  finally {
3707
- activeJobIds.delete(nextJob.id);
5041
+ activeClaimedJobs.delete(nextJob.id);
5042
+ jobQueue.markDone(nextJob.id, reservedSubagentId);
3708
5043
  void processQueue();
3709
5044
  void maybeRestart();
3710
5045
  }
@@ -3712,15 +5047,9 @@ async function startGateway(options, configResolution) {
3712
5047
  const processQueue = async () => {
3713
5048
  if (!acceptingJobs)
3714
5049
  return;
3715
- while (activeJobIds.size < GATEWAY_CONCURRENCY && pendingJobs.length > 0) {
3716
- const nextJob = pendingJobs.shift();
3717
- if (!nextJob)
3718
- break;
3719
- queuedJobIds.delete(nextJob.id);
3720
- if (activeJobIds.has(nextJob.id))
3721
- continue;
3722
- activeJobIds.add(nextJob.id);
3723
- void processJobWithHandling(nextJob);
5050
+ const readyJobs = jobQueue.takeAvailable(GATEWAY_CONCURRENCY);
5051
+ for (const nextJob of readyJobs) {
5052
+ void processJobWithHandling(nextJob.job, nextJob.reservedSubagentId);
3724
5053
  }
3725
5054
  };
3726
5055
  const enqueueJob = (job) => {
@@ -3733,73 +5062,275 @@ async function startGateway(options, configResolution) {
3733
5062
  if (!acceptingJobs) {
3734
5063
  return;
3735
5064
  }
3736
- if (queuedJobIds.has(job.id) || activeJobIds.has(job.id)) {
3737
- return;
5065
+ if (jobQueue.enqueue(job)) {
5066
+ void processQueue();
3738
5067
  }
3739
- pendingJobs.push(job);
3740
- queuedJobIds.add(job.id);
3741
- void processQueue();
3742
5068
  };
3743
5069
  let channel = null;
3744
- const { data: backlog, error: backlogError } = await supabase
3745
- .from('gateway_jobs')
3746
- .select('*')
3747
- .eq('team_id', config.teamId)
3748
- .eq('status', 'queued')
3749
- .or(`gateway_id.is.null,gateway_id.eq.${config.gatewayId}`)
3750
- .order('created_at', { ascending: true });
3751
- if (backlogError) {
3752
- logError('Failed to fetch queued gateway jobs', { error: backlogError.message });
3753
- }
3754
- else if (backlog && backlog.length > 0) {
3755
- backlog.forEach((job) => enqueueJob(job));
3756
- }
3757
- channel = supabase
3758
- .channel(`gateway_jobs_${config.gatewayId}`)
3759
- .on('postgres_changes', {
3760
- event: 'INSERT',
3761
- schema: 'public',
3762
- table: 'gateway_jobs',
3763
- filter: `team_id=eq.${config.teamId}`,
3764
- }, (payload) => {
3765
- const job = payload.new;
3766
- if (job.status !== 'queued')
5070
+ const scheduleChannelReconnect = (reason) => {
5071
+ if (shutdownInProgress || channelReconnectTimer)
3767
5072
  return;
3768
- enqueueJob(job);
3769
- })
3770
- .subscribe((status) => {
3771
- if (status === 'SUBSCRIBED') {
3772
- logInfo('Gateway subscribed to job queue');
5073
+ channelReconnectTimer = setTimeout(() => {
5074
+ channelReconnectTimer = null;
5075
+ void connectJobChannel(`reconnect:${reason}`);
5076
+ }, DEFAULT_JOB_CHANNEL_RECONNECT_MS);
5077
+ };
5078
+ const connectJobChannel = async (reason) => {
5079
+ if (shutdownInProgress)
5080
+ return;
5081
+ if (channel) {
5082
+ try {
5083
+ await channel.unsubscribe();
5084
+ }
5085
+ catch {
5086
+ // ignore unsubscribe failures during reconnect
5087
+ }
3773
5088
  }
3774
- if (status === 'CHANNEL_ERROR') {
3775
- logError('Gateway realtime channel error');
3776
- heartbeatStatus = 'error';
5089
+ channel = supabase
5090
+ .channel(`gateway_jobs_${config.gatewayId}`)
5091
+ .on('postgres_changes', {
5092
+ event: 'INSERT',
5093
+ schema: 'public',
5094
+ table: 'gateway_jobs',
5095
+ filter: `team_id=eq.${config.teamId}`,
5096
+ }, (payload) => {
5097
+ const job = payload.new;
5098
+ if (job.status !== 'queued')
5099
+ return;
5100
+ enqueueJob(job);
5101
+ })
5102
+ .subscribe((status) => {
5103
+ if (status === 'SUBSCRIBED') {
5104
+ logInfo('Gateway subscribed to job queue', { reason });
5105
+ void reconcileQueuedJobs(`channel_subscribed:${reason}`);
5106
+ return;
5107
+ }
5108
+ if (status === 'CHANNEL_ERROR' ||
5109
+ status === 'TIMED_OUT' ||
5110
+ status === 'CLOSED') {
5111
+ logError('Gateway realtime channel unavailable', { status, reason });
5112
+ scheduleChannelReconnect(status.toLowerCase());
5113
+ }
5114
+ });
5115
+ };
5116
+ const requestActiveJobShutdown = (reason) => {
5117
+ for (const activeRun of activeSubagentRuns.values()) {
5118
+ activeRun.cancelled = true;
5119
+ activeRun.cancelReason = reason;
5120
+ requestChildTerminationWithLogging(activeRun.child, {
5121
+ reason,
5122
+ graceMs: SUBAGENT_CANCEL_KILL_TIMEOUT_MS,
5123
+ context: 'gateway_shutdown_subagent',
5124
+ });
3777
5125
  }
3778
- });
3779
- let shutdownInProgress = false;
5126
+ for (const activeModelRun of activeModelRuns.values()) {
5127
+ activeModelRun.shutdownRequested = true;
5128
+ activeModelRun.shutdownReason = reason;
5129
+ requestChildTerminationWithLogging(activeModelRun.child, {
5130
+ reason,
5131
+ graceMs: PROCESS_KILL_GRACE_MS,
5132
+ context: 'gateway_shutdown_model_run',
5133
+ });
5134
+ }
5135
+ };
5136
+ const waitForActiveJobsToDrain = async (graceMs) => {
5137
+ const deadline = Date.now() + graceMs;
5138
+ while (Date.now() < deadline) {
5139
+ if (activeClaimedJobs.size === 0 &&
5140
+ activeSubagentRuns.size === 0 &&
5141
+ activeModelRuns.size === 0) {
5142
+ return;
5143
+ }
5144
+ await sleep(100);
5145
+ }
5146
+ };
5147
+ const cancelSubagentRunOnShutdown = async (subagentId, runId, reason) => {
5148
+ const nowIso = new Date().toISOString();
5149
+ const cancelledOutput = buildErrorOutput(reason);
5150
+ const { data: subagentRow, error: subagentLoadError } = await supabase
5151
+ .from('subagents')
5152
+ .select('team_id, gateway_id, metadata')
5153
+ .eq('id', subagentId)
5154
+ .maybeSingle();
5155
+ if (subagentLoadError) {
5156
+ logError('Failed to load subagent during shutdown cancellation', {
5157
+ subagentId,
5158
+ runId,
5159
+ error: subagentLoadError.message,
5160
+ });
5161
+ return;
5162
+ }
5163
+ const { data: runRow, error: runLoadError } = await supabase
5164
+ .from('subagent_runs')
5165
+ .select('team_id, gateway_id, metadata')
5166
+ .eq('id', runId)
5167
+ .maybeSingle();
5168
+ if (runLoadError) {
5169
+ logError('Failed to load subagent run during shutdown cancellation', {
5170
+ subagentId,
5171
+ runId,
5172
+ error: runLoadError.message,
5173
+ });
5174
+ return;
5175
+ }
5176
+ const updatedRunMetadata = {
5177
+ ...buildRequestedCancelMetadata({
5178
+ base: (runRow?.metadata ?? {}),
5179
+ run_id: runId,
5180
+ reason,
5181
+ requested_at: nowIso,
5182
+ cancel_support: 'gateway',
5183
+ cancellation_scope: 'gateway',
5184
+ }),
5185
+ cancel_state: 'acknowledged',
5186
+ cancelled: true,
5187
+ last_run_at: nowIso,
5188
+ };
5189
+ const { data: updatedRun, error: runUpdateError } = await supabase
5190
+ .from('subagent_runs')
5191
+ .update({
5192
+ status: 'cancelled',
5193
+ output: asJson(cancelledOutput),
5194
+ error: reason,
5195
+ completed_at: nowIso,
5196
+ metadata: asJson(updatedRunMetadata),
5197
+ })
5198
+ .eq('id', runId)
5199
+ .eq('status', 'running')
5200
+ .select('id')
5201
+ .maybeSingle();
5202
+ if (runUpdateError) {
5203
+ logError('Failed to cancel subagent run during shutdown', {
5204
+ subagentId,
5205
+ runId,
5206
+ error: runUpdateError.message,
5207
+ });
5208
+ }
5209
+ const updatedSubagentMetadata = mergeMetadata((subagentRow?.metadata ?? {}), {
5210
+ cancel_run_id: runId,
5211
+ cancel_requested_at: nowIso,
5212
+ cancel_reason: reason,
5213
+ cancel_state: 'acknowledged',
5214
+ cancel_support: 'gateway',
5215
+ cancellation_scope: 'gateway',
5216
+ cancelled: true,
5217
+ last_run_at: nowIso,
5218
+ });
5219
+ const { error: subagentUpdateError } = await supabase
5220
+ .from('subagents')
5221
+ .update({
5222
+ status: 'cancelled',
5223
+ output: asJson(cancelledOutput),
5224
+ error: reason,
5225
+ completed_at: nowIso,
5226
+ metadata: asJson(updatedSubagentMetadata),
5227
+ })
5228
+ .eq('id', subagentId)
5229
+ .eq('status', 'running');
5230
+ if (subagentUpdateError) {
5231
+ logError('Failed to cancel subagent during shutdown', {
5232
+ subagentId,
5233
+ runId,
5234
+ error: subagentUpdateError.message,
5235
+ });
5236
+ }
5237
+ if (updatedRun?.id && subagentRow?.team_id) {
5238
+ const nextSequence = await getNextSubagentRunEventSequence(supabase, runId);
5239
+ await insertSubagentRunEvent({
5240
+ supabase,
5241
+ runId,
5242
+ subagentId,
5243
+ teamId: subagentRow.team_id,
5244
+ gatewayId: subagentRow.gateway_id,
5245
+ sequence: nextSequence,
5246
+ eventType: 'cancellation_acknowledged',
5247
+ payload: {
5248
+ reason,
5249
+ source: 'gateway_shutdown',
5250
+ cancellation_scope: 'gateway',
5251
+ },
5252
+ });
5253
+ await insertSubagentRunEvent({
5254
+ supabase,
5255
+ runId,
5256
+ subagentId,
5257
+ teamId: subagentRow.team_id,
5258
+ gatewayId: subagentRow.gateway_id,
5259
+ sequence: nextSequence + 1,
5260
+ eventType: 'run_cancelled',
5261
+ payload: {
5262
+ reason,
5263
+ source: 'gateway_shutdown',
5264
+ cancellation_scope: 'gateway',
5265
+ },
5266
+ });
5267
+ }
5268
+ };
5269
+ const finalizeResidualGatewayJobs = async (reason) => {
5270
+ for (const [jobId, jobRow] of activeClaimedJobs.entries()) {
5271
+ if (jobRow.job_type === 'subagent_run') {
5272
+ const subagentId = resolveJobSubagentId(jobRow);
5273
+ const activeRun = subagentId ? activeSubagentRuns.get(subagentId) : null;
5274
+ if (subagentId && activeRun) {
5275
+ await cancelSubagentRunOnShutdown(subagentId, activeRun.runId, reason);
5276
+ }
5277
+ await completeJob(supabase, jobId, 'cancelled', { message: reason, cancelled: true }, reason);
5278
+ continue;
5279
+ }
5280
+ await completeJob(supabase, jobId, 'cancelled', { message: reason, cancelled: true }, reason);
5281
+ }
5282
+ };
5283
+ await connectJobChannel('startup');
5284
+ await reconcileQueuedJobs('startup');
5285
+ reconcileTimer = setInterval(() => {
5286
+ void reconcileQueuedJobs('interval');
5287
+ }, DEFAULT_JOB_RECONCILE_INTERVAL_MS);
3780
5288
  const performShutdown = async (signal) => {
3781
5289
  if (shutdownInProgress)
3782
5290
  return;
3783
5291
  shutdownInProgress = true;
5292
+ acceptingJobs = false;
5293
+ CURRENT_ACCEPTING_JOBS = false;
5294
+ jobQueue.clearPending();
3784
5295
  logInfo('Gateway shutting down', { signal });
3785
5296
  clearInterval(heartbeatTimer);
5297
+ if (reconcileTimer) {
5298
+ clearInterval(reconcileTimer);
5299
+ }
5300
+ if (channelReconnectTimer) {
5301
+ clearTimeout(channelReconnectTimer);
5302
+ }
3786
5303
  if (restartTimer) {
3787
5304
  clearInterval(restartTimer);
3788
5305
  }
5306
+ await syncOperationalHeartbeatIfChanged();
5307
+ requestActiveJobShutdown(`Gateway shutting down (${signal})`);
5308
+ await waitForActiveJobsToDrain(DEFAULT_GATEWAY_SHUTDOWN_GRACE_MS);
5309
+ await finalizeResidualGatewayJobs(`Gateway shutting down (${signal})`);
5310
+ for (const sessionId of activeShellSessions.keys()) {
5311
+ cleanupShellSession(sessionId, `gateway_shutdown_${signal}`);
5312
+ }
3789
5313
  heartbeatStatus = 'offline';
5314
+ CURRENT_JOB_INGRESS_STATE = {
5315
+ pending: false,
5316
+ ready: false,
5317
+ };
3790
5318
  await sendHeartbeat(supabase, 'offline', currentCapabilities, deviceName);
3791
5319
  if (channel) {
3792
5320
  await channel.unsubscribe();
3793
5321
  }
3794
5322
  await removePidFile(options);
5323
+ CURRENT_RUNTIME_OPTIONS = null;
3795
5324
  };
3796
5325
  async function restartGateway(reason) {
3797
5326
  if (shutdownInProgress)
3798
5327
  return;
3799
5328
  logInfo('Gateway restarting', {
3800
5329
  reason,
3801
- active_jobs: activeJobIds.size,
5330
+ active_jobs: jobQueue.activeCount(),
3802
5331
  active_subagent_runs: activeSubagentRuns.size,
5332
+ active_model_runs: activeModelRuns.size,
5333
+ active_shell_sessions: getRunningShellSessionCount(),
3803
5334
  });
3804
5335
  const child = spawn(process.execPath, process.argv.slice(1), {
3805
5336
  stdio: 'inherit',
@@ -3827,8 +5358,10 @@ async function startGateway(options, configResolution) {
3827
5358
  if (elapsedMs >= restartMaxWaitMs) {
3828
5359
  logInfo('Gateway restart max wait reached; restarting anyway', {
3829
5360
  wait_ms: elapsedMs,
3830
- active_jobs: activeJobIds.size,
5361
+ active_jobs: jobQueue.activeCount(),
3831
5362
  active_subagent_runs: activeSubagentRuns.size,
5363
+ active_model_runs: activeModelRuns.size,
5364
+ active_shell_sessions: getRunningShellSessionCount(),
3832
5365
  });
3833
5366
  void restartGateway('restart max wait exceeded');
3834
5367
  }
@@ -3848,12 +5381,14 @@ async function startGateway(options, configResolution) {
3848
5381
  gatewayId: config.gatewayId,
3849
5382
  teamId: config.teamId,
3850
5383
  deviceName,
5384
+ installFullControlEnabled: config.full_control_enabled,
3851
5385
  heartbeatIntervalMs: DEFAULT_HEARTBEAT_INTERVAL_MS,
3852
5386
  concurrency: GATEWAY_CONCURRENCY,
3853
5387
  });
3854
5388
  cliSuccess('Gateway running', options, [
3855
5389
  { label: 'Mode', value: 'Foreground' },
3856
5390
  { label: 'Device', value: deviceName },
5391
+ { label: 'Install full control', value: config.full_control_enabled ? 'Enabled' : 'Disabled' },
3857
5392
  { label: 'Gateway ID', value: config.gatewayId, verboseOnly: true },
3858
5393
  { label: 'Team ID', value: config.teamId, verboseOnly: true },
3859
5394
  ]);
@@ -3911,6 +5446,7 @@ async function statusGateway(options) {
3911
5446
  gatewayId: config.gatewayId,
3912
5447
  teamId: config.teamId,
3913
5448
  deviceName: config.deviceName ?? undefined,
5449
+ fullControlEnabled: config.full_control_enabled === true,
3914
5450
  };
3915
5451
  }
3916
5452
  catch {
@@ -3925,6 +5461,14 @@ async function statusGateway(options) {
3925
5461
  { label: 'Version', value: version ?? undefined },
3926
5462
  { label: 'Uptime', value: uptime ?? undefined },
3927
5463
  { label: 'Paired', value: configExists ? 'Yes' : 'No' },
5464
+ {
5465
+ label: 'Install full control',
5466
+ value: configSummary == null
5467
+ ? undefined
5468
+ : configSummary.fullControlEnabled
5469
+ ? 'Enabled'
5470
+ : 'Disabled',
5471
+ },
3928
5472
  { label: 'Device', value: configSummary?.deviceName ?? undefined, verboseOnly: true },
3929
5473
  { label: 'Gateway ID', value: configSummary?.gatewayId ?? undefined, verboseOnly: true },
3930
5474
  { label: 'Team ID', value: configSummary?.teamId ?? undefined, verboseOnly: true },
@@ -3935,6 +5479,40 @@ async function statusGateway(options) {
3935
5479
  { label: 'PID Path', value: formatPathForDisplay(pidPath), verboseOnly: true },
3936
5480
  ]);
3937
5481
  }
5482
+ async function fullControlGateway(options, positional) {
5483
+ const action = (positional[0] ?? 'status').trim().toLowerCase();
5484
+ let config;
5485
+ try {
5486
+ config = await loadConfig(options);
5487
+ }
5488
+ catch (error) {
5489
+ const details = error instanceof Error ? error.message : String(error);
5490
+ throw new GatewayCliError('Gateway config not found. Run "pair" first.', details);
5491
+ }
5492
+ if (action === 'status') {
5493
+ cliInfo('Gateway install full control', options, [
5494
+ { label: 'Enabled', value: config.full_control_enabled ? 'Yes' : 'No' },
5495
+ { label: 'Gateway ID', value: config.gatewayId, verboseOnly: true },
5496
+ ]);
5497
+ return;
5498
+ }
5499
+ const enabled = action === 'enable' || action === 'on' || action === 'true';
5500
+ const disabled = action === 'disable' || action === 'off' || action === 'false';
5501
+ if (!enabled && !disabled) {
5502
+ throw new GatewayCliError('Unknown full-control command.', 'Use one of: enable, disable, status');
5503
+ }
5504
+ const nextValue = enabled;
5505
+ if (config.full_control_enabled === nextValue) {
5506
+ cliInfo(`Gateway install full control already ${nextValue ? 'enabled' : 'disabled'}.`, options);
5507
+ return;
5508
+ }
5509
+ config.full_control_enabled = nextValue;
5510
+ await saveConfig(config, options);
5511
+ if (CURRENT_CONFIG && CURRENT_CONFIG.gatewayId === config.gatewayId) {
5512
+ CURRENT_CONFIG.full_control_enabled = nextValue;
5513
+ }
5514
+ cliSuccess(`Gateway install full control ${nextValue ? 'enabled' : 'disabled'}.`, options, [{ label: 'Gateway ID', value: config.gatewayId, verboseOnly: true }]);
5515
+ }
3938
5516
  async function doctorGateway(options, configResolution) {
3939
5517
  const { configPath, configDir } = resolveGatewayPaths(options);
3940
5518
  const configExists = fsSync.existsSync(configPath);
@@ -3953,6 +5531,10 @@ async function doctorGateway(options, configResolution) {
3953
5531
  cliInfo('Gateway doctor report', options, [
3954
5532
  { label: 'Config dir', value: formatPathForDisplay(configDir) },
3955
5533
  { label: 'Paired', value: configExists ? 'Yes' : 'No' },
5534
+ {
5535
+ label: 'Install full control',
5536
+ value: config == null ? undefined : config.full_control_enabled ? 'Enabled' : 'Disabled',
5537
+ },
3956
5538
  { label: 'Gateway ID', value: config?.gatewayId ?? undefined, verboseOnly: true },
3957
5539
  { label: 'Team ID', value: config?.teamId ?? undefined, verboseOnly: true },
3958
5540
  ]);
@@ -3961,6 +5543,7 @@ async function doctorGateway(options, configResolution) {
3961
5543
  const { providerHealth, anyProviderReady } = await buildCapabilities({
3962
5544
  validationMode: 'full',
3963
5545
  previousHealth: config?.providerHealth,
5546
+ installFullControlEnabled: config?.full_control_enabled === true,
3964
5547
  onProgress: (event) => {
3965
5548
  if (event.stage === 'start') {
3966
5549
  cliInfo(` ${formatProviderLabel(event.providerId)}...`, options);
@@ -4101,6 +5684,10 @@ async function run() {
4101
5684
  await startGateway(parsed.options, configResolution);
4102
5685
  return;
4103
5686
  }
5687
+ if (parsed.command === 'full-control') {
5688
+ await fullControlGateway(parsed.options, parsed.positional);
5689
+ return;
5690
+ }
4104
5691
  if (parsed.command === 'stop') {
4105
5692
  await stopGateway(parsed.options);
4106
5693
  return;