@stigmer/runner 3.14.0 → 3.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/__test-utils__/hermetic-activity.d.ts +245 -0
- package/dist/__test-utils__/hermetic-activity.js +369 -0
- package/dist/__test-utils__/hermetic-activity.js.map +1 -0
- package/dist/__test-utils__/mock-client.d.ts +13 -0
- package/dist/__test-utils__/mock-client.js +45 -0
- package/dist/__test-utils__/mock-client.js.map +1 -0
- package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.d.ts +172 -0
- package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.js +331 -0
- package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.js.map +1 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-agent.d.ts +167 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-agent.js +239 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-agent.js.map +1 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.d.ts +97 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.js +132 -0
- package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.js.map +1 -0
- package/dist/harness/capabilities.d.ts +71 -0
- package/dist/harness/capabilities.js +36 -0
- package/dist/harness/capabilities.js.map +1 -0
- package/dist/harness/registry.d.ts +67 -0
- package/dist/harness/registry.js +112 -0
- package/dist/harness/registry.js.map +1 -0
- package/dist/harness/types.d.ts +268 -0
- package/dist/harness/types.js +55 -0
- package/dist/harness/types.js.map +1 -0
- package/package.json +4 -4
- package/src/__test-utils__/__tests__/harness-contract-self-check.test.ts +229 -0
- package/src/__test-utils__/config-fixture.ts +63 -0
- package/src/__test-utils__/harness-contract/contract.ts +536 -0
- package/src/__test-utils__/harness-contract/recording-sink.ts +96 -0
- package/src/__test-utils__/harness-contract/scripted-adapter.ts +289 -0
- package/src/__test-utils__/harness-contract/types.ts +100 -0
- package/src/__test-utils__/hermetic-activity.ts +477 -0
- package/src/__test-utils__/proto-helpers.ts +25 -0
- package/src/__tests__/harness-contract.test.ts +25 -0
- package/src/activities/execute-cursor/__test-utils__/hermetic-cursor.ts +422 -0
- package/src/activities/execute-cursor/__test-utils__/scripted-agent.ts +342 -0
- package/src/activities/execute-cursor/__test-utils__/scripted-sdk.ts +166 -0
- package/src/activities/execute-cursor/__tests__/hermetic/deny-and-retry.test.ts +228 -0
- package/src/activities/execute-cursor/__tests__/hermetic/file-review-capture.test.ts +180 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/deny-and-retry.turn1.status.json +55 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/deny-and-retry.turn2.status.json +77 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/file-review-capture.status.json +126 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/pause.status.json +45 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/plain-turn.status.json +48 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/recovery-fresh-agent.status.json +53 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/tool-call.status.json +68 -0
- package/src/activities/execute-cursor/__tests__/hermetic/goldens/worker-shutdown.status.json +47 -0
- package/src/activities/execute-cursor/__tests__/hermetic/pause-vs-shutdown.test.ts +201 -0
- package/src/activities/execute-cursor/__tests__/hermetic/plain-turn.test.ts +171 -0
- package/src/activities/execute-cursor/__tests__/hermetic/recovery-fresh-agent.test.ts +156 -0
- package/src/activities/execute-cursor/__tests__/hermetic/tool-call.test.ts +137 -0
- package/src/harness/__tests__/registry.test.ts +167 -0
- package/src/harness/capabilities.ts +75 -0
- package/src/harness/registry.ts +123 -0
- package/src/harness/types.ts +278 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hermetic golden: POISONED-HANDLE RECOVERY — a resumed agent whose run fails
|
|
3
|
+
* with a transport error is replaced by a fresh agent, once, and the turn
|
|
4
|
+
* completes on the replacement.
|
|
5
|
+
*
|
|
6
|
+
* Invariant pinned: when `run.wait()` reports `error` on an agent that was
|
|
7
|
+
* RESUMED (`resolution.reason === "resumed_successfully"`) and the classifier
|
|
8
|
+
* files the error as `network` or `agent-stale`, the activity closes the stale
|
|
9
|
+
* handle, calls `Agent.create` for a fresh one, writes the fresh id to the
|
|
10
|
+
* session's `harness_state_id`, re-runs the turn on the fresh agent through the
|
|
11
|
+
* IDENTICAL stream loop and boundary, and ends COMPLETED. The retry happens at
|
|
12
|
+
* most once (`alreadyRetriedWithFreshAgent`). The golden
|
|
13
|
+
* (`goldens/recovery-fresh-agent.status.json`) pins what the user sees after a
|
|
14
|
+
* recovered turn.
|
|
15
|
+
*
|
|
16
|
+
* Parent phase rows exercised beyond the earlier scenarios: `Agent.resume`
|
|
17
|
+
* through `resolveAgentWithTransportRecovery` (a second execution in a session:
|
|
18
|
+
* `thread_id` set, no parked agent); the `status: ERROR` stream event feeding
|
|
19
|
+
* `streamErrorMessage`; `run.wait()` error mapping; the error classifier
|
|
20
|
+
* (`error-classifier.ts` NETWORK_PATTERNS, the same shapes its own tests pin);
|
|
21
|
+
* the fresh-agent recovery spine (`runRecoveryStream`); the second
|
|
22
|
+
* `harness_state_id` write-back.
|
|
23
|
+
*
|
|
24
|
+
* Regenerate ONLY after a deliberate behavior change:
|
|
25
|
+
* npx vitest run src/activities/execute-cursor/__tests__/hermetic -u
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
29
|
+
import { toJson } from "@bufbuild/protobuf";
|
|
30
|
+
import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
|
|
31
|
+
import { ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
32
|
+
|
|
33
|
+
vi.mock("@cursor/sdk", async () =>
|
|
34
|
+
(await import("../../__test-utils__/scripted-sdk.js")).scriptedCursorSdkModule(),
|
|
35
|
+
);
|
|
36
|
+
vi.mock("../../../../client/stigmer-client.js", async () =>
|
|
37
|
+
(await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
import {
|
|
41
|
+
ScriptedClock,
|
|
42
|
+
createHermeticEnvironment,
|
|
43
|
+
type HermeticEnvironment,
|
|
44
|
+
} from "../../../../__test-utils__/hermetic-activity.js";
|
|
45
|
+
import { ScriptedCursorAgent, sdkEvents, step } from "../../__test-utils__/scripted-agent.js";
|
|
46
|
+
import {
|
|
47
|
+
FIXTURE,
|
|
48
|
+
SDK_CATALOG,
|
|
49
|
+
beginCursorScenario,
|
|
50
|
+
cursorExecutionRecord,
|
|
51
|
+
runCursorTurn,
|
|
52
|
+
stubRegistryFetch,
|
|
53
|
+
} from "../../__test-utils__/hermetic-cursor.js";
|
|
54
|
+
|
|
55
|
+
const STALE_AGENT_ID = "agent-hermetic-stale-0001";
|
|
56
|
+
const FRESH_AGENT_ID = "agent-hermetic-fresh-0002";
|
|
57
|
+
const RUN_STALE = "run-hermetic-stale-0001";
|
|
58
|
+
const RUN_FRESH = "run-hermetic-fresh-0001";
|
|
59
|
+
const USER_MESSAGE = "Summarize what changed since yesterday.";
|
|
60
|
+
// A real network-class failure shape (error-classifier NETWORK_PATTERNS:
|
|
61
|
+
// "unavailable", "econnreset", "fetch failed").
|
|
62
|
+
const TRANSPORT_ERROR = "UNAVAILABLE: fetch failed (ECONNRESET) while streaming the run";
|
|
63
|
+
const FINAL_TEXT = "Two files changed since yesterday: notes.md and README.md.";
|
|
64
|
+
|
|
65
|
+
describe("ExecuteCursor hermetic — poisoned-handle recovery on a fresh agent", () => {
|
|
66
|
+
let env: HermeticEnvironment;
|
|
67
|
+
let registry: ReturnType<typeof stubRegistryFetch>;
|
|
68
|
+
const clock = new ScriptedClock();
|
|
69
|
+
|
|
70
|
+
beforeAll(() => {
|
|
71
|
+
env = createHermeticEnvironment();
|
|
72
|
+
registry = stubRegistryFetch();
|
|
73
|
+
clock.install();
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
afterAll(() => {
|
|
77
|
+
clock.uninstall();
|
|
78
|
+
registry.restore();
|
|
79
|
+
env.dispose();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("replaces the stale resumed agent once and completes on the fresh one", async () => {
|
|
83
|
+
// ── Arrange ──────────────────────────────────────────────────────────────
|
|
84
|
+
const evStale = sdkEvents(STALE_AGENT_ID, RUN_STALE);
|
|
85
|
+
const evFresh = sdkEvents(FRESH_AGENT_ID, RUN_FRESH);
|
|
86
|
+
const stale = new ScriptedCursorAgent({
|
|
87
|
+
agentId: STALE_AGENT_ID,
|
|
88
|
+
runIds: [RUN_STALE],
|
|
89
|
+
observeStep: () => clock.tick(),
|
|
90
|
+
turns: [
|
|
91
|
+
[
|
|
92
|
+
step.event(evStale.init()),
|
|
93
|
+
step.event(evStale.assistant("Let me look at the recent changes.")),
|
|
94
|
+
step.event(evStale.status("ERROR", TRANSPORT_ERROR)),
|
|
95
|
+
step.errored({ result: TRANSPORT_ERROR }),
|
|
96
|
+
],
|
|
97
|
+
],
|
|
98
|
+
});
|
|
99
|
+
const fresh = new ScriptedCursorAgent({
|
|
100
|
+
agentId: FRESH_AGENT_ID,
|
|
101
|
+
runIds: [RUN_FRESH],
|
|
102
|
+
observeStep: () => clock.tick(),
|
|
103
|
+
turns: [
|
|
104
|
+
[
|
|
105
|
+
step.event(evFresh.init()),
|
|
106
|
+
step.event(evFresh.assistant(FINAL_TEXT)),
|
|
107
|
+
step.turnEnded({ inputTokens: 2_800, outputTokens: 55 }),
|
|
108
|
+
step.finished({ result: FINAL_TEXT, model: { id: FIXTURE.model, params: [] } }),
|
|
109
|
+
],
|
|
110
|
+
],
|
|
111
|
+
});
|
|
112
|
+
const record = cursorExecutionRecord({ message: USER_MESSAGE });
|
|
113
|
+
const scenario = beginCursorScenario({
|
|
114
|
+
env,
|
|
115
|
+
clock,
|
|
116
|
+
record,
|
|
117
|
+
// `Agent.create` hands out `fresh`; `Agent.resume(STALE)` finds `stale`,
|
|
118
|
+
// the previous turn's agent the session knows only by harness_state_id.
|
|
119
|
+
sdk: { agents: [fresh], resumableAgents: [stale], catalog: SDK_CATALOG },
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
// ── Act: the second execution in the session resumes the stale handle ────
|
|
123
|
+
const invocation = await runCursorTurn(scenario, { threadId: STALE_AGENT_ID, turnSeq: 0 });
|
|
124
|
+
|
|
125
|
+
// ── Assert: outcome ──────────────────────────────────────────────────────
|
|
126
|
+
expect(invocation.outcome.kind, "a recovered turn RETURNS like any completion").toBe("returned");
|
|
127
|
+
const slim = (invocation.outcome as { value: Record<string, unknown> }).value;
|
|
128
|
+
expect(slim.phase).toBe("EXECUTION_COMPLETED");
|
|
129
|
+
expect(slim.final_text).toBe(FINAL_TEXT);
|
|
130
|
+
expect(record.persistedPhases).toEqual([
|
|
131
|
+
ExecutionPhase.EXECUTION_IN_PROGRESS,
|
|
132
|
+
ExecutionPhase.EXECUTION_COMPLETED,
|
|
133
|
+
]);
|
|
134
|
+
|
|
135
|
+
// ── Assert: the recovery spine ───────────────────────────────────────────
|
|
136
|
+
expect(scenario.sdk.resolutions.map((r) => [r.kind, r.agentId])).toEqual([
|
|
137
|
+
["resume", STALE_AGENT_ID],
|
|
138
|
+
["create", FRESH_AGENT_ID],
|
|
139
|
+
]);
|
|
140
|
+
expect(stale.sends, "the stale handle ran exactly once").toHaveLength(1);
|
|
141
|
+
expect(stale.closeCalls, "the poisoned handle is disposed").toBe(1);
|
|
142
|
+
expect(fresh.sends, "the fresh agent ran the turn exactly once — no second retry").toHaveLength(1);
|
|
143
|
+
expect(String(fresh.sends[0].message), "the rebuilt prompt carries the user's message").toContain(USER_MESSAGE);
|
|
144
|
+
|
|
145
|
+
// The session now points at the fresh agent (the stale id is superseded).
|
|
146
|
+
const lastSessionWrite = record.sessionUpdates.at(-1);
|
|
147
|
+
expect(lastSessionWrite?.spec?.harnessStateId).toBe(FRESH_AGENT_ID);
|
|
148
|
+
|
|
149
|
+
// ── Assert: hermeticity ──────────────────────────────────────────────────
|
|
150
|
+
expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
|
|
151
|
+
|
|
152
|
+
// ── Assert: the golden ───────────────────────────────────────────────────
|
|
153
|
+
const json = JSON.stringify(toJson(AgentExecutionStatusSchema, record.lastFullStatus!), null, 2) + "\n";
|
|
154
|
+
await expect(json).toMatchFileSnapshot("./goldens/recovery-fresh-agent.status.json");
|
|
155
|
+
});
|
|
156
|
+
});
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hermetic golden: a turn with an UNGATED TOOL CALL through the whole
|
|
3
|
+
* `ExecuteCursor` activity.
|
|
4
|
+
*
|
|
5
|
+
* Invariant pinned: a `tool_call` the SDK streams as `running` and then
|
|
6
|
+
* `completed` folds into exactly one `ToolCall` row on the AI message — id from
|
|
7
|
+
* `call_id`, name from the stream taxonomy, `args` + `argsPreview` from the
|
|
8
|
+
* event's args, `result` from the completed event, `startedAt`/`completedAt`
|
|
9
|
+
* from the two events' instants, `toolKind` classified, no approval fields set
|
|
10
|
+
* (a read-only built-in is not gated) — and the turn ends COMPLETED. The golden
|
|
11
|
+
* (`goldens/tool-call.status.json`) is the row shape S4's canonical transcript
|
|
12
|
+
* builder must reproduce byte for byte.
|
|
13
|
+
*
|
|
14
|
+
* Parent phase rows exercised beyond `plain-turn`: run the turn and consume
|
|
15
|
+
* the engine stream (tool-call folding in `message-translator.ts`); the turn
|
|
16
|
+
* boundary with nothing to gate or capture (a read tool writes no file; the
|
|
17
|
+
* per-session workspace is unchanged, so the capture finds no candidate).
|
|
18
|
+
*
|
|
19
|
+
* Regenerate ONLY after a deliberate behavior change:
|
|
20
|
+
* npx vitest run src/activities/execute-cursor/__tests__/hermetic -u
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
24
|
+
import { toJson } from "@bufbuild/protobuf";
|
|
25
|
+
import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
|
|
26
|
+
import { ExecutionPhase, ToolCallStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
27
|
+
|
|
28
|
+
vi.mock("@cursor/sdk", async () =>
|
|
29
|
+
(await import("../../__test-utils__/scripted-sdk.js")).scriptedCursorSdkModule(),
|
|
30
|
+
);
|
|
31
|
+
vi.mock("../../../../client/stigmer-client.js", async () =>
|
|
32
|
+
(await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
ScriptedClock,
|
|
37
|
+
createHermeticEnvironment,
|
|
38
|
+
type HermeticEnvironment,
|
|
39
|
+
} from "../../../../__test-utils__/hermetic-activity.js";
|
|
40
|
+
import { ScriptedCursorAgent, sdkEvents, step } from "../../__test-utils__/scripted-agent.js";
|
|
41
|
+
import {
|
|
42
|
+
FIXTURE,
|
|
43
|
+
SDK_CATALOG,
|
|
44
|
+
beginCursorScenario,
|
|
45
|
+
cursorExecutionRecord,
|
|
46
|
+
runCursorTurn,
|
|
47
|
+
stubRegistryFetch,
|
|
48
|
+
} from "../../__test-utils__/hermetic-cursor.js";
|
|
49
|
+
|
|
50
|
+
const AGENT_ID = "agent-hermetic-tool-0001";
|
|
51
|
+
const RUN_ID = "run-hermetic-tool-0001";
|
|
52
|
+
const CALL_ID = "call-hermetic-read-0001";
|
|
53
|
+
const USER_MESSAGE = "What is in README.md?";
|
|
54
|
+
const READ_ARGS = { path: "README.md" };
|
|
55
|
+
const READ_RESULT = "# Hermetic\n\nA fixture readme.\n";
|
|
56
|
+
const ASSISTANT_TEXT = "README.md holds a one-line fixture description.";
|
|
57
|
+
|
|
58
|
+
describe("ExecuteCursor hermetic — ungated tool call", () => {
|
|
59
|
+
let env: HermeticEnvironment;
|
|
60
|
+
let registry: ReturnType<typeof stubRegistryFetch>;
|
|
61
|
+
const clock = new ScriptedClock();
|
|
62
|
+
|
|
63
|
+
beforeAll(() => {
|
|
64
|
+
env = createHermeticEnvironment();
|
|
65
|
+
registry = stubRegistryFetch();
|
|
66
|
+
clock.install();
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
afterAll(() => {
|
|
70
|
+
clock.uninstall();
|
|
71
|
+
registry.restore();
|
|
72
|
+
env.dispose();
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("folds running -> completed into one COMPLETED tool-call row and completes", async () => {
|
|
76
|
+
// ── Arrange ──────────────────────────────────────────────────────────────
|
|
77
|
+
const ev = sdkEvents(AGENT_ID, RUN_ID);
|
|
78
|
+
const agent = new ScriptedCursorAgent({
|
|
79
|
+
agentId: AGENT_ID,
|
|
80
|
+
runIds: [RUN_ID],
|
|
81
|
+
observeStep: () => clock.tick(),
|
|
82
|
+
turns: [
|
|
83
|
+
[
|
|
84
|
+
step.event(ev.init()),
|
|
85
|
+
step.event(ev.assistant("Let me read it.")),
|
|
86
|
+
step.event(ev.toolCall(CALL_ID, "read", "running", READ_ARGS)),
|
|
87
|
+
step.event(ev.toolCall(CALL_ID, "read", "completed", READ_ARGS, READ_RESULT)),
|
|
88
|
+
step.event(ev.assistant(ASSISTANT_TEXT)),
|
|
89
|
+
step.turnEnded({ inputTokens: 2_000, outputTokens: 90 }),
|
|
90
|
+
step.finished({ result: ASSISTANT_TEXT, model: { id: FIXTURE.model, params: [] } }),
|
|
91
|
+
],
|
|
92
|
+
],
|
|
93
|
+
});
|
|
94
|
+
const record = cursorExecutionRecord({ message: USER_MESSAGE });
|
|
95
|
+
const scenario = beginCursorScenario({
|
|
96
|
+
env,
|
|
97
|
+
clock,
|
|
98
|
+
record,
|
|
99
|
+
sdk: { agents: [agent], catalog: SDK_CATALOG },
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// ── Act ──────────────────────────────────────────────────────────────────
|
|
103
|
+
const invocation = await runCursorTurn(scenario);
|
|
104
|
+
|
|
105
|
+
// ── Assert: outcome and phases ───────────────────────────────────────────
|
|
106
|
+
expect(invocation.outcome.kind).toBe("returned");
|
|
107
|
+
const slim = (invocation.outcome as { value: Record<string, unknown> }).value;
|
|
108
|
+
expect(slim.phase).toBe("EXECUTION_COMPLETED");
|
|
109
|
+
expect(slim.final_text).toBe(ASSISTANT_TEXT);
|
|
110
|
+
expect(record.persistedPhases).toEqual([
|
|
111
|
+
ExecutionPhase.EXECUTION_IN_PROGRESS,
|
|
112
|
+
ExecutionPhase.EXECUTION_COMPLETED,
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
// ── Assert: the one tool-call row ────────────────────────────────────────
|
|
116
|
+
const rows = record.toolCalls();
|
|
117
|
+
expect(rows, "running + completed fold into ONE row, never two").toHaveLength(1);
|
|
118
|
+
const row = rows[0];
|
|
119
|
+
expect(row.id).toBe(CALL_ID);
|
|
120
|
+
expect(row.name).toBe("read");
|
|
121
|
+
expect(row.status).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
|
|
122
|
+
expect(row.args).toEqual(READ_ARGS);
|
|
123
|
+
expect(row.result).toBe(READ_RESULT);
|
|
124
|
+
expect(row.requiresApproval, "a read-only built-in is not gated").toBe(false);
|
|
125
|
+
expect(row.startedAt < row.completedAt, "started before completed on the scripted clock").toBe(true);
|
|
126
|
+
|
|
127
|
+
// ── Assert: hermeticity ──────────────────────────────────────────────────
|
|
128
|
+
expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
|
|
129
|
+
expect(agent.runs[0].cancelCalls, "an ungated turn is never cancelled").toHaveLength(0);
|
|
130
|
+
|
|
131
|
+
// ── Assert: the golden ───────────────────────────────────────────────────
|
|
132
|
+
const finalStatus = record.lastFullStatus;
|
|
133
|
+
expect(finalStatus).toBeDefined();
|
|
134
|
+
const json = JSON.stringify(toJson(AgentExecutionStatusSchema, finalStatus!), null, 2) + "\n";
|
|
135
|
+
await expect(json).toMatchFileSnapshot("./goldens/tool-call.status.json");
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pins the harness registry's two load-bearing properties — ORDER and ERROR
|
|
3
|
+
* POSTURE — and the byte-pinned activity names.
|
|
4
|
+
*
|
|
5
|
+
* Order: boot in declaration order, one at a time (the Cursor interceptors
|
|
6
|
+
* must precede anything that dials the control plane); shutdown in reverse.
|
|
7
|
+
* Posture: boot validates the table before touching any adapter and fails
|
|
8
|
+
* fast at the first rejection; shutdown and release continue past a failing
|
|
9
|
+
* adapter and surface every failure in one AggregateError. Names: the map
|
|
10
|
+
* must equal the server's constants byte for byte; the runner cannot import
|
|
11
|
+
* the server, so the literals are asserted here and the server's file is
|
|
12
|
+
* cited beside them.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, it, expect } from "vitest";
|
|
16
|
+
|
|
17
|
+
import { testConfig } from "../../__test-utils__/config-fixture.js";
|
|
18
|
+
import { DEEP_AGENT_VISION_PROFILE } from "../../shared/attachment-vision.js";
|
|
19
|
+
import { HARNESS_ACTIVITY_NAMES, bootHarnesses, releaseHarnessSession, shutdownHarnesses } from "../registry.js";
|
|
20
|
+
import type { HarnessAdapter } from "../types.js";
|
|
21
|
+
|
|
22
|
+
interface StubBehaviour {
|
|
23
|
+
readonly bootRejects?: Error;
|
|
24
|
+
readonly shutdownRejects?: Error;
|
|
25
|
+
readonly releaseRejects?: Error;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** An adapter that only records lifecycle calls into a shared, ordered log. */
|
|
29
|
+
function stubAdapter(name: string, log: string[], behaviour: StubBehaviour = {}): HarnessAdapter {
|
|
30
|
+
return {
|
|
31
|
+
name,
|
|
32
|
+
capabilities: {
|
|
33
|
+
pausePrimitive: "interrupt",
|
|
34
|
+
stateIdSource: "deterministic",
|
|
35
|
+
systemPrompt: true,
|
|
36
|
+
subAgents: false,
|
|
37
|
+
toolRestriction: true,
|
|
38
|
+
visionProfile: DEEP_AGENT_VISION_PROFILE,
|
|
39
|
+
},
|
|
40
|
+
async boot() {
|
|
41
|
+
log.push(`boot:${name}`);
|
|
42
|
+
if (behaviour.bootRejects) throw behaviour.bootRejects;
|
|
43
|
+
},
|
|
44
|
+
async shutdown() {
|
|
45
|
+
log.push(`shutdown:${name}`);
|
|
46
|
+
if (behaviour.shutdownRejects) throw behaviour.shutdownRejects;
|
|
47
|
+
},
|
|
48
|
+
async releaseSession(sessionId) {
|
|
49
|
+
log.push(`release:${name}:${sessionId}`);
|
|
50
|
+
if (behaviour.releaseRejects) throw behaviour.releaseRejects;
|
|
51
|
+
},
|
|
52
|
+
async runTurn() {
|
|
53
|
+
throw new Error(`${name}: runTurn is not under test here`);
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
describe("HARNESS_ACTIVITY_NAMES", () => {
|
|
59
|
+
it("equals the server's byte-pinned activity names (stigmer-server temporal/agentexecution/names.ts)", () => {
|
|
60
|
+
expect(HARNESS_ACTIVITY_NAMES.cursor).toBe("ExecuteCursor");
|
|
61
|
+
expect(HARNESS_ACTIVITY_NAMES["deep-agent"]).toBe("ExecuteDeepAgent");
|
|
62
|
+
expect(Object.keys(HARNESS_ACTIVITY_NAMES).sort()).toEqual(["cursor", "deep-agent"]);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
describe("bootHarnesses", () => {
|
|
67
|
+
it("boots adapters in declaration order, one at a time", async () => {
|
|
68
|
+
const log: string[] = [];
|
|
69
|
+
await bootHarnesses([stubAdapter("first", log), stubAdapter("second", log), stubAdapter("third", log)], testConfig());
|
|
70
|
+
expect(log).toEqual(["boot:first", "boot:second", "boot:third"]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("hands every adapter the same config", async () => {
|
|
74
|
+
const seen: unknown[] = [];
|
|
75
|
+
const config = testConfig({ taskQueue: "registry-test-queue" });
|
|
76
|
+
const adapters = ["a", "b"].map((name) => ({
|
|
77
|
+
...stubAdapter(name, []),
|
|
78
|
+
async boot(c: unknown) {
|
|
79
|
+
seen.push(c);
|
|
80
|
+
},
|
|
81
|
+
}));
|
|
82
|
+
await bootHarnesses(adapters, config);
|
|
83
|
+
expect(seen, "each adapter must receive the one worker config").toEqual([config, config]);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("refuses duplicate adapter names before booting anything", async () => {
|
|
87
|
+
const log: string[] = [];
|
|
88
|
+
const adapters = [stubAdapter("cursor", log), stubAdapter("other", log), stubAdapter("cursor", log)];
|
|
89
|
+
await expect(bootHarnesses(adapters, testConfig())).rejects.toThrow(/duplicate adapter name 'cursor'/);
|
|
90
|
+
expect(log, "a table with a duplicate must not boot even its first adapter").toEqual([]);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("fails fast: the first rejection propagates and later adapters never boot", async () => {
|
|
94
|
+
const log: string[] = [];
|
|
95
|
+
const adapters = [
|
|
96
|
+
stubAdapter("first", log),
|
|
97
|
+
stubAdapter("broken", log, { bootRejects: new Error("interceptor install failed") }),
|
|
98
|
+
stubAdapter("third", log),
|
|
99
|
+
];
|
|
100
|
+
await expect(bootHarnesses(adapters, testConfig())).rejects.toThrow("interceptor install failed");
|
|
101
|
+
expect(log).toEqual(["boot:first", "boot:broken"]);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("boots an empty table as a no-op", async () => {
|
|
105
|
+
await expect(bootHarnesses([], testConfig())).resolves.toBeUndefined();
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
describe("shutdownHarnesses", () => {
|
|
110
|
+
it("shuts adapters down in reverse declaration order", async () => {
|
|
111
|
+
const log: string[] = [];
|
|
112
|
+
await shutdownHarnesses([stubAdapter("first", log), stubAdapter("second", log), stubAdapter("third", log)]);
|
|
113
|
+
expect(log).toEqual(["shutdown:third", "shutdown:second", "shutdown:first"]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("continues past a failing adapter and reports every failure in one AggregateError", async () => {
|
|
117
|
+
const log: string[] = [];
|
|
118
|
+
const adapters = [
|
|
119
|
+
stubAdapter("first", log, { shutdownRejects: new Error("first leaked") }),
|
|
120
|
+
stubAdapter("second", log),
|
|
121
|
+
stubAdapter("third", log, { shutdownRejects: new Error("third leaked") }),
|
|
122
|
+
];
|
|
123
|
+
const failure = await shutdownHarnesses(adapters).catch((e: unknown) => e);
|
|
124
|
+
expect(failure).toBeInstanceOf(AggregateError);
|
|
125
|
+
const aggregate = failure as AggregateError;
|
|
126
|
+
expect(aggregate.message).toContain("third: shutdown rejected: third leaked");
|
|
127
|
+
expect(aggregate.message).toContain("first: shutdown rejected: first leaked");
|
|
128
|
+
expect(aggregate.errors.map((e: Error) => e.message)).toEqual([
|
|
129
|
+
"third: shutdown rejected: third leaked",
|
|
130
|
+
"first: shutdown rejected: first leaked",
|
|
131
|
+
]);
|
|
132
|
+
expect(log, "every adapter must be shut down even when an earlier one failed").toEqual([
|
|
133
|
+
"shutdown:third",
|
|
134
|
+
"shutdown:second",
|
|
135
|
+
"shutdown:first",
|
|
136
|
+
]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it("preserves the original error as the cause of each reported failure", async () => {
|
|
140
|
+
const original = new Error("socket already closed");
|
|
141
|
+
const failure = await shutdownHarnesses([stubAdapter("only", [], { shutdownRejects: original })]).catch((e: unknown) => e);
|
|
142
|
+
expect((failure as AggregateError).errors[0]?.cause).toBe(original);
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
describe("releaseHarnessSession", () => {
|
|
147
|
+
it("tells every adapter, in declaration order, with the session id", async () => {
|
|
148
|
+
const log: string[] = [];
|
|
149
|
+
await releaseHarnessSession([stubAdapter("first", log), stubAdapter("second", log)], "ses-42");
|
|
150
|
+
expect(log).toEqual(["release:first:ses-42", "release:second:ses-42"]);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("continues past a failing adapter and reports the failure with the session id", async () => {
|
|
154
|
+
const log: string[] = [];
|
|
155
|
+
const adapters = [
|
|
156
|
+
stubAdapter("first", log, { releaseRejects: new Error("executor busy") }),
|
|
157
|
+
stubAdapter("second", log),
|
|
158
|
+
];
|
|
159
|
+
const failure = await releaseHarnessSession(adapters, "ses-7").catch((e: unknown) => e);
|
|
160
|
+
expect(failure).toBeInstanceOf(AggregateError);
|
|
161
|
+
expect((failure as AggregateError).message).toContain("releaseSession('ses-7') failed");
|
|
162
|
+
expect((failure as AggregateError).errors.map((e: Error) => e.message)).toEqual([
|
|
163
|
+
"first: releaseSession('ses-7') rejected: executor busy",
|
|
164
|
+
]);
|
|
165
|
+
expect(log).toEqual(["release:first:ses-7", "release:second:ses-7"]);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harness capability flags — the facts about an engine the runtime branches
|
|
3
|
+
* on, declared once per adapter and read nowhere else.
|
|
4
|
+
*
|
|
5
|
+
* The runtime is written against the flags, not against harness names: a
|
|
6
|
+
* phase that must differ per engine asks "does this harness accept a system
|
|
7
|
+
* prompt?" rather than "is this Cursor?". That is what keeps a new harness a
|
|
8
|
+
* registry row and an SDK slice instead of a new branch in every phase.
|
|
9
|
+
* Every row of the matrix below is either a flag here or internal to one
|
|
10
|
+
* adapter; nothing in it needed a third kind of thing.
|
|
11
|
+
*
|
|
12
|
+
* The matrix the contract was designed against (2026-09; Claude and Codex are
|
|
13
|
+
* the surveyed SDKs, not built harnesses):
|
|
14
|
+
*
|
|
15
|
+
* | Capability | Native (LangGraph) | Cursor (`@cursor/sdk`) | Claude (`@anthropic-ai/claude-agent-sdk`) | Codex (`@openai/codex-sdk`) |
|
|
16
|
+
* |-------------------|-----------------------------|-----------------------------------------|-----------------------------------------------|------------------------------------------------|
|
|
17
|
+
* | `pausePrimitive` | `interrupt` (checkpoint) | `deny-and-retry` (hook, ledger, cancel) | `callback` (`canUseTool`, in-process) | `none` (approvalPolicy only) → capture-only |
|
|
18
|
+
* | `stateIdSource` | `deterministic` (thread id) | `engine-minted` (agent id) | either (caller-supplied or minted) | `engine-minted` (thread id) |
|
|
19
|
+
* | `systemPrompt` | yes | no (rides the first message) | yes | no (`AGENTS.md` or first message) |
|
|
20
|
+
* | `subAgents` | yes (compiled sub-graphs) | yes (`agents` option) | yes (`AgentDefinition`) | no |
|
|
21
|
+
* | `toolRestriction` | yes (tool list) | no (the hook enforces) | yes (`disallowedTools`, `tools`) | per-server config |
|
|
22
|
+
* | `visionProfile` | PNG, JPEG, WebP, GIF | PNG, JPEG (transport re-sniffs) | (surveyed later) | (surveyed later) |
|
|
23
|
+
*
|
|
24
|
+
* Adapter-internal, deliberately NOT flags: how MCP servers are bound, how
|
|
25
|
+
* metering is routed, how the cost cap is applied inside the engine, how a
|
|
26
|
+
* run is cancelled. Those differ per engine but the runtime never needs to
|
|
27
|
+
* know.
|
|
28
|
+
*
|
|
29
|
+
* Two flags matter to the contract kit directly: `pausePrimitive` (the kit
|
|
30
|
+
* proves the two real primitives are indistinguishable above the contract
|
|
31
|
+
* line — both end a turn `awaiting_approval` and both take the decisions on
|
|
32
|
+
* reinvocation) and `stateIdSource` (an `engine-minted` adapter must bind its
|
|
33
|
+
* id before its first persist; a `deterministic` one must never bind).
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import type { VisionProfile } from "../shared/attachment-vision.js";
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* How the engine can be made to stop before a gated side effect.
|
|
40
|
+
*
|
|
41
|
+
* - `interrupt`: the engine checkpoints and stops at the gate (LangGraph
|
|
42
|
+
* `interrupt`); the adapter resumes it with the decisions.
|
|
43
|
+
* - `deny-and-retry`: an out-of-process hook denies the tool, the adapter
|
|
44
|
+
* records the denial, cancels the run, and re-runs with grants on the next
|
|
45
|
+
* invocation.
|
|
46
|
+
* - `callback`: an in-process callback decides per tool; the runtime still
|
|
47
|
+
* returns `awaiting_approval` and reinvokes, so the adapter answers the
|
|
48
|
+
* callback with "deny, stop" and resumes with the decisions.
|
|
49
|
+
* - `none`: the engine offers no gate; the runtime confines it to
|
|
50
|
+
* capture-only work (a read-only sandbox) and gated actions never reach
|
|
51
|
+
* the engine.
|
|
52
|
+
*/
|
|
53
|
+
export type PausePrimitive = "interrupt" | "deny-and-retry" | "callback" | "none";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Who mints the engine's state id. `deterministic`: the runtime derives it
|
|
57
|
+
* from the session before the first turn (`EnsureThread`), so it is known
|
|
58
|
+
* before any engine exists. `engine-minted`: the engine issues it on first
|
|
59
|
+
* use and the adapter must hand it to the runtime at once
|
|
60
|
+
* (`TurnSink.bindHarnessState`) so a crash mid-turn still resumes.
|
|
61
|
+
*/
|
|
62
|
+
export type StateIdSource = "deterministic" | "engine-minted";
|
|
63
|
+
|
|
64
|
+
export interface HarnessCapabilities {
|
|
65
|
+
readonly pausePrimitive: PausePrimitive;
|
|
66
|
+
readonly stateIdSource: StateIdSource;
|
|
67
|
+
/** The engine accepts a system prompt; otherwise instructions ride the first user message. */
|
|
68
|
+
readonly systemPrompt: boolean;
|
|
69
|
+
/** The engine runs delegated sub-agents from a definition map. */
|
|
70
|
+
readonly subAgents: boolean;
|
|
71
|
+
/** The engine can hide or deny tools by name; otherwise the gate enforces `enabledTools`. */
|
|
72
|
+
readonly toolRestriction: boolean;
|
|
73
|
+
/** Which image types the engine can display inline; the runtime degrades the rest before the turn. */
|
|
74
|
+
readonly visionProfile: VisionProfile;
|
|
75
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The harness registry — the lifecycle fan-out over the adapters a worker
|
|
3
|
+
* runs, and the byte-pinned binding from harness to Temporal activity name.
|
|
4
|
+
*
|
|
5
|
+
* Plain functions over an `adapters` argument, no module state and no DI
|
|
6
|
+
* (the composition roots are staged plain functions; a missing adapter is a
|
|
7
|
+
* compile error or a loud boot throw, never a silent no-op). ORDER IS
|
|
8
|
+
* LOAD-BEARING: `bootHarnesses` runs adapters in declaration order because
|
|
9
|
+
* the Cursor harness's interceptors must patch `node:http2` before anything
|
|
10
|
+
* dials the control plane, and `shutdownHarnesses` runs them in reverse so
|
|
11
|
+
* what was set up last is torn down first.
|
|
12
|
+
*
|
|
13
|
+
* Error posture, ruled at the entry's gate (Q-S1-10):
|
|
14
|
+
* - Boot validates the whole table BEFORE booting anything (a duplicate name
|
|
15
|
+
* is a configuration defect, and a half-booted worker is the worst state to
|
|
16
|
+
* discover it in), then fails fast at the first adapter that rejects. A
|
|
17
|
+
* worker that cannot boot a harness must not start.
|
|
18
|
+
* - Shutdown and session release CONTINUE past a failing adapter and throw
|
|
19
|
+
* one `AggregateError` at the end naming each failure, so one bad teardown
|
|
20
|
+
* never leaks the others' resources.
|
|
21
|
+
*
|
|
22
|
+
* What is NOT here yet: `createHarnessActivities` (its body is the turn
|
|
23
|
+
* runtime, which does not exist until the extraction entry) and the table of
|
|
24
|
+
* real adapters (they exist once the Cursor and native harnesses implement
|
|
25
|
+
* the contract). Both land with the runtime; no empty table sits on `main`
|
|
26
|
+
* between the two.
|
|
27
|
+
*
|
|
28
|
+
* `HarnessName` lives here and not in `types.ts` on purpose: the wire
|
|
29
|
+
* vocabulary is the registry's concern. An adapter never declares the
|
|
30
|
+
* activity it is bound to (its `name` is a diagnostic identity); the registry
|
|
31
|
+
* row does.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import type { Config } from "../config.js";
|
|
35
|
+
import type { HarnessAdapter } from "./types.js";
|
|
36
|
+
|
|
37
|
+
/** The harnesses the control plane can dispatch to, as the registry knows them. */
|
|
38
|
+
export type HarnessName = "cursor" | "deep-agent";
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Byte-pinned Temporal activity names, one per harness. The server side of
|
|
42
|
+
* the pin is `stigmer-server/src/temporal/agentexecution/names.ts`
|
|
43
|
+
* (`EXECUTE_CURSOR_ACTIVITY_NAME`, `EXECUTE_DEEP_AGENT_ACTIVITY_NAME`); the
|
|
44
|
+
* two must stay byte-identical or the workflow schedules an activity no
|
|
45
|
+
* worker registers. Never "cleaned up".
|
|
46
|
+
*/
|
|
47
|
+
export const HARNESS_ACTIVITY_NAMES = {
|
|
48
|
+
cursor: "ExecuteCursor",
|
|
49
|
+
"deep-agent": "ExecuteDeepAgent",
|
|
50
|
+
} as const satisfies Record<HarnessName, string>;
|
|
51
|
+
|
|
52
|
+
export type HarnessActivityName = (typeof HARNESS_ACTIVITY_NAMES)[HarnessName];
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Refuse a table two of whose adapters share a name. Names are the registry's
|
|
56
|
+
* identity for diagnostics and for this check; a duplicate means two adapters
|
|
57
|
+
* would be indistinguishable in every log line and kit message.
|
|
58
|
+
*/
|
|
59
|
+
function assertUniqueNames(adapters: readonly HarnessAdapter[]): void {
|
|
60
|
+
const seen = new Set<string>();
|
|
61
|
+
for (const adapter of adapters) {
|
|
62
|
+
if (seen.has(adapter.name)) {
|
|
63
|
+
throw new Error(`harness registry: duplicate adapter name '${adapter.name}'; every adapter must have a unique name`);
|
|
64
|
+
}
|
|
65
|
+
seen.add(adapter.name);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Boot every adapter in declaration order, one at a time, awaiting each. The
|
|
71
|
+
* table is validated first; the first rejection stops the boot and propagates
|
|
72
|
+
* (adapters after it are never booted; adapters before it stay booted for the
|
|
73
|
+
* caller's shutdown path to release).
|
|
74
|
+
*/
|
|
75
|
+
export async function bootHarnesses(adapters: readonly HarnessAdapter[], config: Config): Promise<void> {
|
|
76
|
+
assertUniqueNames(adapters);
|
|
77
|
+
for (const adapter of adapters) {
|
|
78
|
+
await adapter.boot(config);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Shut every adapter down in reverse declaration order, continuing past
|
|
84
|
+
* failures. Rejects with one `AggregateError` carrying every failure once all
|
|
85
|
+
* adapters have been given their chance.
|
|
86
|
+
*/
|
|
87
|
+
export async function shutdownHarnesses(adapters: readonly HarnessAdapter[]): Promise<void> {
|
|
88
|
+
const failures: Error[] = [];
|
|
89
|
+
for (const adapter of [...adapters].reverse()) {
|
|
90
|
+
try {
|
|
91
|
+
await adapter.shutdown();
|
|
92
|
+
} catch (err) {
|
|
93
|
+
failures.push(describeFailure(adapter, "shutdown", err));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
throwIfAny(failures, "harness registry: shutdown failed for one or more adapters");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Tell every adapter the session is done on this host, in declaration order,
|
|
101
|
+
* continuing past failures. An adapter that parks nothing per session
|
|
102
|
+
* resolves as a no-op; the registry does not know which do.
|
|
103
|
+
*/
|
|
104
|
+
export async function releaseHarnessSession(adapters: readonly HarnessAdapter[], sessionId: string): Promise<void> {
|
|
105
|
+
const failures: Error[] = [];
|
|
106
|
+
for (const adapter of adapters) {
|
|
107
|
+
try {
|
|
108
|
+
await adapter.releaseSession(sessionId);
|
|
109
|
+
} catch (err) {
|
|
110
|
+
failures.push(describeFailure(adapter, `releaseSession('${sessionId}')`, err));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
throwIfAny(failures, `harness registry: releaseSession('${sessionId}') failed for one or more adapters`);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function describeFailure(adapter: HarnessAdapter, call: string, err: unknown): Error {
|
|
117
|
+
const cause = err instanceof Error ? err : new Error(String(err));
|
|
118
|
+
return new Error(`${adapter.name}: ${call} rejected: ${cause.message}`, { cause });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function throwIfAny(failures: readonly Error[], message: string): void {
|
|
122
|
+
if (failures.length > 0) throw new AggregateError(failures, `${message}: ${failures.map((f) => f.message).join("; ")}`);
|
|
123
|
+
}
|