@amaster.ai/employee-runtime-connector 0.1.0-beta.22 → 0.1.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/amaster-runtime-daemon.mjs +212 -44
- package/dist/amaster-runtime.mjs +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -38,7 +38,7 @@ The daemon reports its connector contract, exact package/build, platform/archite
|
|
|
38
38
|
|
|
39
39
|
`AMASTER_RUNTIME_ATTESTATION_MODE` defaults to `shadow`. Set it to `enforce` only after the expected version and build marker are configured and shadow diagnostics are clean; enforce mode refuses to create or lease Runtime V2 mutation commands without a current exact attestation. `AMASTER_RUNTIME_ATTESTATION_TTL_SECONDS` defaults to `300` and accepts `60`–`3600`. Invalid mode or TTL values fail server startup visibly.
|
|
40
40
|
|
|
41
|
-
Threat model: these facts are self-reported by a connector authenticated with its existing connector credential. There is no hardware-backed signature, remote attestation, or independent trust root. The proof detects accidental version/schema/executor drift by an honest connector and correlates commands to the observed facts; it does not prove that a compromised or malicious connector is running the claimed code. Keep the default `shadow` mode observational
|
|
41
|
+
Threat model: these facts are self-reported by a connector authenticated with its existing connector credential. There is no hardware-backed signature, remote attestation, or independent trust root. The proof detects accidental version/schema/executor drift by an honest connector and correlates commands to the observed facts; it does not prove that a compromised or malicious connector is running the claimed code. Proof identity is stable for the exact connector credential and fact set, create/lease revalidate it under the connector transaction lock, stale commands are isolated within a poll batch, and an active connector can recover after restoring exact facts. Keep the default `shadow` mode observational, and do not treat this mechanism as a standalone production security boundary or enable `enforce` before expected markers are configured and shadow diagnostics are clean.
|
|
42
42
|
|
|
43
43
|
The AMaster remote stack derives its runtime image from the Pi base image with
|
|
44
44
|
`docker/Dockerfile.amaster-employee-pi-cli-runtime`. The build installs one exact
|
|
@@ -2144,6 +2144,27 @@ function selectExecutor(config, command) {
|
|
|
2144
2144
|
function processGroupIdForChild(child, processPlatform = process.platform) {
|
|
2145
2145
|
return processPlatform === "win32" || typeof child.pid !== "number" || child.pid <= 0 ? null : child.pid;
|
|
2146
2146
|
}
|
|
2147
|
+
function isExecutorProcessAlive(child, processGroupId = processGroupIdForChild(child), processApi = process) {
|
|
2148
|
+
if (processGroupId !== null) {
|
|
2149
|
+
try {
|
|
2150
|
+
processApi.kill(-processGroupId, 0);
|
|
2151
|
+
return true;
|
|
2152
|
+
} catch (error) {
|
|
2153
|
+
if (error?.code === "ESRCH") return false;
|
|
2154
|
+
return true;
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
return child.exitCode === null && child.signalCode === null;
|
|
2158
|
+
}
|
|
2159
|
+
function isProcessIdAlive(pid, processApi = process) {
|
|
2160
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
2161
|
+
try {
|
|
2162
|
+
processApi.kill(pid, 0);
|
|
2163
|
+
return true;
|
|
2164
|
+
} catch (error) {
|
|
2165
|
+
return error?.code !== "ESRCH";
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2147
2168
|
function signalExecutorProcess(child, signal, processGroupId = processGroupIdForChild(child), processApi = process) {
|
|
2148
2169
|
if (processGroupId !== null) {
|
|
2149
2170
|
try {
|
|
@@ -2156,7 +2177,7 @@ function signalExecutorProcess(child, signal, processGroupId = processGroupIdFor
|
|
|
2156
2177
|
}
|
|
2157
2178
|
}
|
|
2158
2179
|
}
|
|
2159
|
-
if (
|
|
2180
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
2160
2181
|
try {
|
|
2161
2182
|
return child.kill(signal);
|
|
2162
2183
|
} catch {
|
|
@@ -3759,14 +3780,15 @@ function buildRuntimeConnectorAuthHeaders(config) {
|
|
|
3759
3780
|
if (config.token) return { Authorization: `Bearer ${config.token}` };
|
|
3760
3781
|
return {};
|
|
3761
3782
|
}
|
|
3762
|
-
async function postRuntimeConnectorJson(config, path, payload) {
|
|
3783
|
+
async function postRuntimeConnectorJson(config, path, payload, options = {}) {
|
|
3763
3784
|
const res = await fetch(`${config.serverUrl}${path}`, {
|
|
3764
3785
|
method: "POST",
|
|
3765
3786
|
headers: {
|
|
3766
3787
|
"content-type": "application/json",
|
|
3767
3788
|
...buildRuntimeConnectorAuthHeaders(config)
|
|
3768
3789
|
},
|
|
3769
|
-
body: JSON.stringify(payload)
|
|
3790
|
+
body: JSON.stringify(payload),
|
|
3791
|
+
signal: options.signal
|
|
3770
3792
|
});
|
|
3771
3793
|
const text = await res.text();
|
|
3772
3794
|
const body = text ? JSON.parse(text) : null;
|
|
@@ -3797,6 +3819,26 @@ async function postRuntimeConnectorBytes(config, path, body, headers = {}) {
|
|
|
3797
3819
|
}
|
|
3798
3820
|
return response;
|
|
3799
3821
|
}
|
|
3822
|
+
function retryableRuntimeConnectorPost(error) {
|
|
3823
|
+
const status = Number(error?.httpStatus);
|
|
3824
|
+
return !Number.isInteger(status) || status === 408 || status === 429 || status >= 500;
|
|
3825
|
+
}
|
|
3826
|
+
async function postRuntimeConnectorJsonWithRetry(config, path, payload, options = {}) {
|
|
3827
|
+
const maxAttempts = Math.max(1, Math.min(5, Number(options.maxAttempts ?? 3)));
|
|
3828
|
+
const timeoutMs = Math.max(1, Number(options.timeoutMs ?? 5e3));
|
|
3829
|
+
const delayMs = Math.max(0, Number(options.delayMs ?? 100));
|
|
3830
|
+
let lastError;
|
|
3831
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
3832
|
+
try {
|
|
3833
|
+
return await postRuntimeConnectorJson(config, path, payload, { signal: AbortSignal.timeout(timeoutMs) });
|
|
3834
|
+
} catch (error) {
|
|
3835
|
+
lastError = error;
|
|
3836
|
+
if (attempt >= maxAttempts || !retryableRuntimeConnectorPost(error)) throw error;
|
|
3837
|
+
if (delayMs > 0) await new Promise((resolve9) => setTimeout(resolve9, delayMs));
|
|
3838
|
+
}
|
|
3839
|
+
}
|
|
3840
|
+
throw lastError;
|
|
3841
|
+
}
|
|
3800
3842
|
async function bestEffortPostRuntimeConnectorJson(config, path, payload, options = {}) {
|
|
3801
3843
|
try {
|
|
3802
3844
|
return await postRuntimeConnectorJson(config, path, payload);
|
|
@@ -3812,6 +3854,7 @@ async function bestEffortPostRuntimeConnectorJson(config, path, payload, options
|
|
|
3812
3854
|
}
|
|
3813
3855
|
}
|
|
3814
3856
|
var postJson = postRuntimeConnectorJson;
|
|
3857
|
+
var postJsonWithRetry = postRuntimeConnectorJsonWithRetry;
|
|
3815
3858
|
var bestEffortPostJson = bestEffortPostRuntimeConnectorJson;
|
|
3816
3859
|
|
|
3817
3860
|
// src/amaster-runtime-daemon/runtime-artifact-upload.mjs
|
|
@@ -3928,6 +3971,7 @@ function createWorkspaceManifest(workspace, input = {}) {
|
|
|
3928
3971
|
executorKind: input.executorKind ?? null,
|
|
3929
3972
|
executorHome: workspace.executorHome ?? null,
|
|
3930
3973
|
commandId: input.commandId ?? null,
|
|
3974
|
+
generation: input.generation ?? null,
|
|
3931
3975
|
runId: input.runId ?? null,
|
|
3932
3976
|
issueId: input.issueId ?? null,
|
|
3933
3977
|
workspaceKey: workspace.workspaceKey ?? null,
|
|
@@ -4041,6 +4085,7 @@ function resolveExecutionWorkspace(config, command, opts = {}) {
|
|
|
4041
4085
|
createWorkspaceManifest(workspace, {
|
|
4042
4086
|
executorKind,
|
|
4043
4087
|
commandId: readString(command.commandId) ?? null,
|
|
4088
|
+
generation: Number.isInteger(command.generation) ? command.generation : null,
|
|
4044
4089
|
runId,
|
|
4045
4090
|
issueId
|
|
4046
4091
|
});
|
|
@@ -4618,7 +4663,7 @@ function readWorkspaceStatus(cwd, opts = {}) {
|
|
|
4618
4663
|
}
|
|
4619
4664
|
|
|
4620
4665
|
// src/amaster-runtime-daemon.mjs
|
|
4621
|
-
var CONNECTOR_VERSION = "0.1.0-beta.
|
|
4666
|
+
var CONNECTOR_VERSION = "0.1.0-beta.23";
|
|
4622
4667
|
var CONNECTOR_CONTRACT_VERSION = "2026-06-04.v1";
|
|
4623
4668
|
var MAX_CHECKPOINT_BYTES = 20 * 1024 * 1024;
|
|
4624
4669
|
var CHECKPOINT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
@@ -4699,6 +4744,40 @@ function resultOutboxActiveRunCommands(config) {
|
|
|
4699
4744
|
}
|
|
4700
4745
|
return entries;
|
|
4701
4746
|
}
|
|
4747
|
+
function resultOutboxFailedRunCommands(config) {
|
|
4748
|
+
const dir = resultOutboxInvalidDir(config);
|
|
4749
|
+
if (!existsSync9(dir)) return [];
|
|
4750
|
+
const entries = [];
|
|
4751
|
+
for (const file of readdirSync6(dir).filter((name) => name.endsWith(".json")).sort().slice(-100)) {
|
|
4752
|
+
let entry;
|
|
4753
|
+
try {
|
|
4754
|
+
entry = asRecord(JSON.parse(readFileSync7(join10(dir, file), "utf8")));
|
|
4755
|
+
} catch {
|
|
4756
|
+
continue;
|
|
4757
|
+
}
|
|
4758
|
+
const failureReason = readString(entry.invalidReason);
|
|
4759
|
+
if (!failureReason || !["server_rejected_terminal", "retry_exhausted"].includes(failureReason)) continue;
|
|
4760
|
+
const activeRun = asRecord(entry.activeRun);
|
|
4761
|
+
const commandId = readString(activeRun.commandId) ?? readString(entry.commandId);
|
|
4762
|
+
if (!commandId) continue;
|
|
4763
|
+
entries.push({
|
|
4764
|
+
commandId,
|
|
4765
|
+
...readString(activeRun.runId) ? { runId: readString(activeRun.runId) } : {},
|
|
4766
|
+
...readString(activeRun.issueId) ? { issueId: readString(activeRun.issueId) } : {},
|
|
4767
|
+
...readString(activeRun.executorKind) ? { executorKind: readString(activeRun.executorKind) } : {},
|
|
4768
|
+
...readString(activeRun.workspacePath) ? { workspacePath: readString(activeRun.workspacePath) } : {},
|
|
4769
|
+
...readString(activeRun.startedAt) ? { startedAt: readString(activeRun.startedAt) } : {},
|
|
4770
|
+
...readString(activeRun.executorCompletedAt) ? { executorCompletedAt: readString(activeRun.executorCompletedAt) } : {},
|
|
4771
|
+
phase: "result_outbox_failed",
|
|
4772
|
+
outboxPending: 0,
|
|
4773
|
+
evidenceDeliveryStatus: "failed",
|
|
4774
|
+
evidenceFailureReason: failureReason,
|
|
4775
|
+
...typeof entry.httpStatus === "number" ? { evidenceHttpStatus: entry.httpStatus } : {},
|
|
4776
|
+
...readString(entry.invalidAt) ? { evidenceFailedAt: readString(entry.invalidAt) } : {}
|
|
4777
|
+
});
|
|
4778
|
+
}
|
|
4779
|
+
return entries;
|
|
4780
|
+
}
|
|
4702
4781
|
function clearDeliveredResultOutboxCommand(config, entry) {
|
|
4703
4782
|
const commandId = readString(entry?.commandId);
|
|
4704
4783
|
if (commandId) pendingResultOutboxRunCommands.delete(commandId);
|
|
@@ -5001,8 +5080,10 @@ function buildHeartbeatPayload(config, options = {}) {
|
|
|
5001
5080
|
const orphanReaperSummary = asRecord(options.orphanReaper ?? lastOrphanReaperSummary);
|
|
5002
5081
|
const activeRunCommandById = /* @__PURE__ */ new Map();
|
|
5003
5082
|
for (const entry of [
|
|
5083
|
+
...Array.from(localDispatchCommandIds, (commandId) => ({ commandId, phase: "preparing" })),
|
|
5004
5084
|
...Array.from(activeRunCommands.values()),
|
|
5005
5085
|
...Array.from(pendingResultOutboxRunCommands.values()),
|
|
5086
|
+
...resultOutboxFailedRunCommands(config),
|
|
5006
5087
|
...resultOutboxActiveRunCommands(config)
|
|
5007
5088
|
]) {
|
|
5008
5089
|
const commandId = readString(entry.commandId);
|
|
@@ -5993,7 +6074,7 @@ function buildExecutorInvocation(executor, command = {}, workspace = null) {
|
|
|
5993
6074
|
}
|
|
5994
6075
|
return { command: executor.command, args: [], stdin: "prompt" };
|
|
5995
6076
|
}
|
|
5996
|
-
async function executeModelCallCommand(config, command) {
|
|
6077
|
+
async function executeModelCallCommand(config, command, signal) {
|
|
5997
6078
|
const executor = selectExecutor(config, command);
|
|
5998
6079
|
const payload = asRecord(command.payload);
|
|
5999
6080
|
const prompt = readString(payload.prompt) ?? "";
|
|
@@ -6009,6 +6090,7 @@ async function executeModelCallCommand(config, command) {
|
|
|
6009
6090
|
config.executorMaxOutputBytes,
|
|
6010
6091
|
readNumber(payload.maxOutputBytes, 512 * 1024)
|
|
6011
6092
|
));
|
|
6093
|
+
await ackCommand(config, command, "spawned");
|
|
6012
6094
|
await ingestLog(config, command, "system", "info", `Starting ${executor.kind} runtime model call`, {
|
|
6013
6095
|
executorKind: executor.kind,
|
|
6014
6096
|
args: invocation.args,
|
|
@@ -6021,7 +6103,8 @@ async function executeModelCallCommand(config, command) {
|
|
|
6021
6103
|
stdin: invocation.stdin === "prompt" ? prompt : "",
|
|
6022
6104
|
timeoutSeconds,
|
|
6023
6105
|
maxOutputBytes,
|
|
6024
|
-
executorKind: executor.kind
|
|
6106
|
+
executorKind: executor.kind,
|
|
6107
|
+
signal
|
|
6025
6108
|
});
|
|
6026
6109
|
if (execution.stdout) {
|
|
6027
6110
|
await ingestLog(config, command, "stdout", "info", truncateText(execution.stdout, 4e3));
|
|
@@ -6059,6 +6142,17 @@ async function executeModelCallCommand(config, command) {
|
|
|
6059
6142
|
} : {}
|
|
6060
6143
|
}, error ?? void 0);
|
|
6061
6144
|
}
|
|
6145
|
+
async function executeTrackedModelCallCommand(config, command) {
|
|
6146
|
+
const abortController = new AbortController();
|
|
6147
|
+
const forgetActiveModelCall = rememberActiveRunCommand(command, {});
|
|
6148
|
+
const stopActiveModelCallHeartbeats = startActiveRunHeartbeats(config, command, abortController);
|
|
6149
|
+
try {
|
|
6150
|
+
return await executeModelCallCommand(config, command, abortController.signal);
|
|
6151
|
+
} finally {
|
|
6152
|
+
stopActiveModelCallHeartbeats();
|
|
6153
|
+
forgetActiveModelCall();
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6062
6156
|
function redactProtectedText(value, protectedValues = []) {
|
|
6063
6157
|
let text = String(value ?? "");
|
|
6064
6158
|
for (const protectedValue of protectedValues) {
|
|
@@ -6305,6 +6399,7 @@ function realOrResolvedPath(value) {
|
|
|
6305
6399
|
return resolve8(value);
|
|
6306
6400
|
}
|
|
6307
6401
|
}
|
|
6402
|
+
var LSOF_COMMAND = process.platform === "darwin" && existsSync9("/usr/sbin/lsof") ? "/usr/sbin/lsof" : "lsof";
|
|
6308
6403
|
function processCwdForPid(pid) {
|
|
6309
6404
|
if (process.platform === "linux") {
|
|
6310
6405
|
try {
|
|
@@ -6314,7 +6409,7 @@ function processCwdForPid(pid) {
|
|
|
6314
6409
|
}
|
|
6315
6410
|
}
|
|
6316
6411
|
if (process.platform === "darwin") {
|
|
6317
|
-
const result2 = spawnSync5(
|
|
6412
|
+
const result2 = spawnSync5(LSOF_COMMAND, ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], {
|
|
6318
6413
|
encoding: "utf8",
|
|
6319
6414
|
stdio: ["ignore", "pipe", "ignore"]
|
|
6320
6415
|
});
|
|
@@ -6344,7 +6439,7 @@ function allProcessCwdsByPid() {
|
|
|
6344
6439
|
return cwds;
|
|
6345
6440
|
}
|
|
6346
6441
|
if (process.platform === "darwin") {
|
|
6347
|
-
const result2 = spawnSync5(
|
|
6442
|
+
const result2 = spawnSync5(LSOF_COMMAND, ["-nP", "-d", "cwd", "-Fp", "-Fn"], {
|
|
6348
6443
|
encoding: "utf8",
|
|
6349
6444
|
stdio: ["ignore", "pipe", "ignore"]
|
|
6350
6445
|
});
|
|
@@ -6562,8 +6657,19 @@ function runExecutor(command, args, options) {
|
|
|
6562
6657
|
let timer = null;
|
|
6563
6658
|
let rssTimer = null;
|
|
6564
6659
|
let stopKillTimer = null;
|
|
6565
|
-
let
|
|
6660
|
+
let quiescenceTimer = null;
|
|
6661
|
+
let stoppedResidentTimer = null;
|
|
6566
6662
|
let completionOutputDrainTimer = null;
|
|
6663
|
+
let stopReason = null;
|
|
6664
|
+
let childExited = false;
|
|
6665
|
+
let childClosed = false;
|
|
6666
|
+
let closeCode = null;
|
|
6667
|
+
let closeSignal = null;
|
|
6668
|
+
let spawnError = null;
|
|
6669
|
+
let outputDrainForcedClosed = false;
|
|
6670
|
+
let killedWorkspaceResidents = [];
|
|
6671
|
+
let stoppedResidentDrainUntil = 0;
|
|
6672
|
+
const residentSignalledAt = /* @__PURE__ */ new Map();
|
|
6567
6673
|
let piCompletionOutputLineBuffer = "";
|
|
6568
6674
|
const outputBytes = { stdout: 0, stderr: 0 };
|
|
6569
6675
|
const maxOutputBytes = parsePositiveInteger(options.maxOutputBytes, 50 * 1024 * 1024);
|
|
@@ -6582,11 +6688,10 @@ function runExecutor(command, args, options) {
|
|
|
6582
6688
|
if (timer) clearTimeout(timer);
|
|
6583
6689
|
if (rssTimer) clearInterval(rssTimer);
|
|
6584
6690
|
if (stopKillTimer) clearTimeout(stopKillTimer);
|
|
6585
|
-
if (
|
|
6691
|
+
if (quiescenceTimer) clearTimeout(quiescenceTimer);
|
|
6692
|
+
if (stoppedResidentTimer) clearTimeout(stoppedResidentTimer);
|
|
6586
6693
|
if (completionOutputDrainTimer) clearTimeout(completionOutputDrainTimer);
|
|
6587
6694
|
options.signal?.removeEventListener?.("abort", abort);
|
|
6588
|
-
const shouldCleanupWorkspaceResidents = Boolean(aborted || outputFlood || memoryLimit || result2.timedOut || completionOutputType);
|
|
6589
|
-
const killedWorkspaceResidents = shouldCleanupWorkspaceResidents ? killWorkspaceResidentProcesses(options.cwd, processGroupId) : [];
|
|
6590
6695
|
resolveRun({
|
|
6591
6696
|
stdout,
|
|
6592
6697
|
stderr,
|
|
@@ -6595,25 +6700,77 @@ function runExecutor(command, args, options) {
|
|
|
6595
6700
|
memoryLimit,
|
|
6596
6701
|
completionOutputType,
|
|
6597
6702
|
killedWorkspaceResidents,
|
|
6703
|
+
...outputDrainForcedClosed ? { outputDrainForcedClosed: true } : {},
|
|
6598
6704
|
...result2
|
|
6599
6705
|
});
|
|
6600
6706
|
};
|
|
6601
6707
|
const scheduleStopKill = () => {
|
|
6602
|
-
if (stopKillTimer)
|
|
6708
|
+
if (stopKillTimer) return;
|
|
6603
6709
|
stopKillTimer = setTimeout(() => signalExecutorProcess(child, "SIGKILL", processGroupId), 2e3);
|
|
6604
|
-
stopKillTimer.unref?.();
|
|
6605
6710
|
};
|
|
6606
|
-
const
|
|
6607
|
-
|
|
6608
|
-
|
|
6711
|
+
const requestStop = (reason) => {
|
|
6712
|
+
if (settled || stopReason) return;
|
|
6713
|
+
stopReason = reason;
|
|
6714
|
+
signalExecutorProcess(child, "SIGTERM", processGroupId);
|
|
6715
|
+
scheduleStopKill();
|
|
6609
6716
|
};
|
|
6610
|
-
const
|
|
6611
|
-
if (settled || !
|
|
6612
|
-
|
|
6613
|
-
|
|
6614
|
-
|
|
6615
|
-
}
|
|
6616
|
-
|
|
6717
|
+
const reapStoppedWorkspaceResidents = () => {
|
|
6718
|
+
if (settled || !stopReason || stopReason === "completion_cleanup") return;
|
|
6719
|
+
const residents = listWorkspaceResidentProcesses(options.cwd, processGroupId, {
|
|
6720
|
+
processRows: allProcessRows(),
|
|
6721
|
+
processCwdsByPid: allProcessCwdsByPid()
|
|
6722
|
+
});
|
|
6723
|
+
const seen = new Set(killedWorkspaceResidents.map((resident) => resident.pid));
|
|
6724
|
+
const now = Date.now();
|
|
6725
|
+
for (const resident of residents) {
|
|
6726
|
+
if (!seen.has(resident.pid)) killedWorkspaceResidents.push(resident);
|
|
6727
|
+
if (!residentSignalledAt.has(resident.pid)) {
|
|
6728
|
+
residentSignalledAt.set(resident.pid, now);
|
|
6729
|
+
try {
|
|
6730
|
+
process.kill(resident.pid, "SIGTERM");
|
|
6731
|
+
} catch {
|
|
6732
|
+
}
|
|
6733
|
+
}
|
|
6734
|
+
}
|
|
6735
|
+
for (const resident of killedWorkspaceResidents) {
|
|
6736
|
+
const signalledAt = residentSignalledAt.get(resident.pid) ?? now;
|
|
6737
|
+
if (now - signalledAt < 2e3 || !isProcessIdAlive(resident.pid)) continue;
|
|
6738
|
+
try {
|
|
6739
|
+
process.kill(resident.pid, "SIGKILL");
|
|
6740
|
+
} catch {
|
|
6741
|
+
}
|
|
6742
|
+
}
|
|
6743
|
+
if (isExecutorProcessAlive(child, processGroupId) || now < stoppedResidentDrainUntil || killedWorkspaceResidents.some((resident) => isProcessIdAlive(resident.pid))) {
|
|
6744
|
+
stoppedResidentTimer = setTimeout(reapStoppedWorkspaceResidents, 100);
|
|
6745
|
+
} else {
|
|
6746
|
+
stoppedResidentTimer = null;
|
|
6747
|
+
}
|
|
6748
|
+
maybeFinishQuiescent();
|
|
6749
|
+
};
|
|
6750
|
+
const maybeFinishQuiescent = () => {
|
|
6751
|
+
if (settled || !childExited && !childClosed) return;
|
|
6752
|
+
if (isExecutorProcessAlive(child, processGroupId) || !childClosed && Date.now() < stoppedResidentDrainUntil || killedWorkspaceResidents.some((resident) => isProcessIdAlive(resident.pid))) {
|
|
6753
|
+
requestStop("completion_cleanup");
|
|
6754
|
+
if (!quiescenceTimer) {
|
|
6755
|
+
quiescenceTimer = setTimeout(() => {
|
|
6756
|
+
quiescenceTimer = null;
|
|
6757
|
+
maybeFinishQuiescent();
|
|
6758
|
+
}, 25);
|
|
6759
|
+
}
|
|
6760
|
+
return;
|
|
6761
|
+
}
|
|
6762
|
+
if (!childClosed) {
|
|
6763
|
+
outputDrainForcedClosed = true;
|
|
6764
|
+
child.stdout.destroy();
|
|
6765
|
+
child.stderr.destroy();
|
|
6766
|
+
}
|
|
6767
|
+
finish({
|
|
6768
|
+
exitCode: closeCode,
|
|
6769
|
+
signal: closeSignal,
|
|
6770
|
+
timedOut: stopReason === "timeout",
|
|
6771
|
+
spawnError,
|
|
6772
|
+
cancelled: aborted
|
|
6773
|
+
});
|
|
6617
6774
|
};
|
|
6618
6775
|
const maybeSchedulePiCompletionDrain = (text) => {
|
|
6619
6776
|
if (settled || outputFlood || options.executorKind !== "pi" || completionOutputType) return;
|
|
@@ -6638,8 +6795,7 @@ function runExecutor(command, args, options) {
|
|
|
6638
6795
|
completionOutputType = stopType;
|
|
6639
6796
|
completionOutputDrainTimer = setTimeout(() => {
|
|
6640
6797
|
if (settled) return;
|
|
6641
|
-
|
|
6642
|
-
scheduleStopKill();
|
|
6798
|
+
requestStop("completion_output");
|
|
6643
6799
|
}, PI_COMPLETION_OUTPUT_GRACE_MS);
|
|
6644
6800
|
completionOutputDrainTimer.unref?.();
|
|
6645
6801
|
return;
|
|
@@ -6655,8 +6811,7 @@ function runExecutor(command, args, options) {
|
|
|
6655
6811
|
bytes: outputBytes[stream],
|
|
6656
6812
|
limitBytes: maxOutputBytes
|
|
6657
6813
|
};
|
|
6658
|
-
|
|
6659
|
-
scheduleStopKill();
|
|
6814
|
+
requestStop("output_flood");
|
|
6660
6815
|
};
|
|
6661
6816
|
const killForMemoryLimit = (rssBytes) => {
|
|
6662
6817
|
if (settled || memoryLimit) return;
|
|
@@ -6664,23 +6819,17 @@ function runExecutor(command, args, options) {
|
|
|
6664
6819
|
rssBytes,
|
|
6665
6820
|
limitBytes: maxRssBytes
|
|
6666
6821
|
};
|
|
6667
|
-
|
|
6668
|
-
scheduleStopKill();
|
|
6822
|
+
requestStop("memory_limit");
|
|
6669
6823
|
};
|
|
6670
6824
|
const abort = () => {
|
|
6671
6825
|
if (settled) return;
|
|
6672
6826
|
aborted = true;
|
|
6673
|
-
|
|
6674
|
-
scheduleStopKill();
|
|
6827
|
+
requestStop("cancel");
|
|
6675
6828
|
};
|
|
6676
6829
|
options.signal?.addEventListener?.("abort", abort, { once: true });
|
|
6677
6830
|
if (options.signal?.aborted) abort();
|
|
6678
6831
|
timer = setTimeout(() => {
|
|
6679
|
-
|
|
6680
|
-
const timeoutKillTimer = setTimeout(() => signalExecutorProcess(child, "SIGKILL", processGroupId), 2e3);
|
|
6681
|
-
timeoutKillTimer.unref?.();
|
|
6682
|
-
destroyExecutorOutputPipes();
|
|
6683
|
-
finish({ exitCode: null, signal: "SIGTERM", timedOut: true, spawnError: null });
|
|
6832
|
+
requestStop("timeout");
|
|
6684
6833
|
}, options.timeoutSeconds * 1e3);
|
|
6685
6834
|
if (maxRssMb > 0 && processGroupId !== null) {
|
|
6686
6835
|
rssTimer = setInterval(() => {
|
|
@@ -6707,13 +6856,27 @@ function runExecutor(command, args, options) {
|
|
|
6707
6856
|
if (outputBytes.stderr > maxOutputBytes) killForOutputFlood("stderr");
|
|
6708
6857
|
});
|
|
6709
6858
|
child.on("error", (err) => {
|
|
6710
|
-
|
|
6859
|
+
spawnError = err.message;
|
|
6711
6860
|
});
|
|
6712
6861
|
child.on("exit", (code, signal) => {
|
|
6713
|
-
|
|
6862
|
+
childExited = true;
|
|
6863
|
+
closeCode = code;
|
|
6864
|
+
closeSignal = signal;
|
|
6865
|
+
stoppedResidentDrainUntil = Date.now() + 100;
|
|
6866
|
+
if (!stopReason && isExecutorProcessAlive(child, processGroupId)) {
|
|
6867
|
+
requestStop("completion_cleanup");
|
|
6868
|
+
}
|
|
6869
|
+
if (stopReason && stopReason !== "completion_cleanup") {
|
|
6870
|
+
stoppedResidentDrainUntil = Date.now() + 100;
|
|
6871
|
+
reapStoppedWorkspaceResidents();
|
|
6872
|
+
}
|
|
6873
|
+
maybeFinishQuiescent();
|
|
6714
6874
|
});
|
|
6715
6875
|
child.on("close", (code, signal) => {
|
|
6716
|
-
|
|
6876
|
+
childClosed = true;
|
|
6877
|
+
closeCode = code;
|
|
6878
|
+
closeSignal = signal;
|
|
6879
|
+
maybeFinishQuiescent();
|
|
6717
6880
|
});
|
|
6718
6881
|
if (options.stdin) child.stdin.end(options.stdin);
|
|
6719
6882
|
else child.stdin.end();
|
|
@@ -7516,10 +7679,12 @@ function piOutputValidationError(parsed, options = {}) {
|
|
|
7516
7679
|
}
|
|
7517
7680
|
return null;
|
|
7518
7681
|
}
|
|
7519
|
-
async function ackCommand(config, command) {
|
|
7682
|
+
async function ackCommand(config, command, phase = "preparing") {
|
|
7520
7683
|
const connectorId = requireConnectorId(config);
|
|
7521
|
-
return await
|
|
7522
|
-
leaseId: command.leaseId
|
|
7684
|
+
return await postJsonWithRetry(config, `/api/amaster/runtime-connectors/${connectorId}/commands/${command.commandId}/ack`, {
|
|
7685
|
+
leaseId: command.leaseId,
|
|
7686
|
+
generation: command.generation,
|
|
7687
|
+
phase
|
|
7523
7688
|
});
|
|
7524
7689
|
}
|
|
7525
7690
|
async function executeRunCommand(config, command) {
|
|
@@ -7608,6 +7773,7 @@ async function executeRunCommand(config, command) {
|
|
|
7608
7773
|
presentationKind: "context_manifest",
|
|
7609
7774
|
contextManifest
|
|
7610
7775
|
});
|
|
7776
|
+
await ackCommand(config, command, "spawned");
|
|
7611
7777
|
await ingestLog(config, command, "system", "info", `Starting ${executor.kind} executor`, {
|
|
7612
7778
|
executorKind: executor.kind,
|
|
7613
7779
|
cwd,
|
|
@@ -7796,6 +7962,7 @@ async function executeRunCommand(config, command) {
|
|
|
7796
7962
|
});
|
|
7797
7963
|
}
|
|
7798
7964
|
const result2 = {
|
|
7965
|
+
evidenceContract: { version: 1 },
|
|
7799
7966
|
executorKind: executor.kind,
|
|
7800
7967
|
command: invocation.command,
|
|
7801
7968
|
args: invocation.args,
|
|
@@ -7805,6 +7972,7 @@ async function executeRunCommand(config, command) {
|
|
|
7805
7972
|
exitCode: execution.exitCode,
|
|
7806
7973
|
signal: execution.signal,
|
|
7807
7974
|
timedOut: execution.timedOut,
|
|
7975
|
+
...execution.outputDrainForcedClosed ? { outputDrainForcedClosed: true } : {},
|
|
7808
7976
|
...readString(execution.completionOutputType) ? { completionOutputType: readString(execution.completionOutputType) } : {},
|
|
7809
7977
|
...cancelled ? { cancelledByControlPlane: true } : {},
|
|
7810
7978
|
...invocation.nativeSession ? { nativeSession: invocation.nativeSession } : {},
|
|
@@ -7900,7 +8068,7 @@ async function processCommand(config, command) {
|
|
|
7900
8068
|
return;
|
|
7901
8069
|
}
|
|
7902
8070
|
if (command.commandType === "model_call") {
|
|
7903
|
-
await
|
|
8071
|
+
await executeTrackedModelCallCommand(config, command);
|
|
7904
8072
|
return;
|
|
7905
8073
|
}
|
|
7906
8074
|
await completeCommand(config, command, "succeeded", {
|
package/dist/amaster-runtime.mjs
CHANGED
|
@@ -5,7 +5,7 @@ import { dirname, join, resolve } from "node:path";
|
|
|
5
5
|
import { homedir, hostname } from "node:os";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
|
|
8
|
-
const CONNECTOR_VERSION = "0.1.0-beta.
|
|
8
|
+
const CONNECTOR_VERSION = "0.1.0-beta.23";
|
|
9
9
|
|
|
10
10
|
const CAPABILITIES = [
|
|
11
11
|
"remote_registration",
|