@stigmer/runner 3.29.0 → 3.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/call-eval.d.ts +16 -0
- package/dist/activities/call-eval.js +54 -1
- package/dist/activities/call-eval.js.map +1 -1
- package/dist/activities/hydrate-workflow-execution.d.ts +1 -1
- package/dist/activities/workflow-event-activities.d.ts +2 -2
- package/dist/activities/workflow-event-activities.js +2 -2
- package/dist/harness/transcript/builder.d.ts +3 -2
- package/dist/harness/transcript/builder.js +28 -8
- package/dist/harness/transcript/builder.js.map +1 -1
- package/dist/shared/approval-policy.d.ts +5 -0
- package/dist/shared/approval-policy.js +8 -0
- package/dist/shared/approval-policy.js.map +1 -1
- package/dist/shared/args-preview.d.ts +12 -5
- package/dist/shared/args-preview.js +12 -5
- package/dist/shared/args-preview.js.map +1 -1
- package/dist/worker.d.ts +2 -4
- package/dist/worker.js +2 -4
- package/dist/worker.js.map +1 -1
- package/dist/workflow-engine/tasks/run.d.ts +13 -1
- package/dist/workflow-engine/tasks/run.js +16 -1
- package/dist/workflow-engine/tasks/run.js.map +1 -1
- package/dist/workflows/call-agent-orchestrator.d.ts +1 -1
- package/dist/workflows/call-agent-orchestrator.js +1 -1
- package/dist/workflows/engine-core.d.ts +28 -8
- package/dist/workflows/engine-core.js +9 -7
- package/dist/workflows/engine-core.js.map +1 -1
- package/dist/workflows/index.d.ts +22 -7
- package/dist/workflows/index.js +22 -7
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/workflow-signals.d.ts +3 -3
- package/dist/workflows/workflow-signals.js +3 -3
- package/package.json +5 -5
- package/src/__test-utils__/workflows/index.ts +14 -0
- package/src/__test-utils__/workflows/inline-model.ts +32 -0
- package/src/__tests__/dispatch-surface-e2e.test.ts +212 -0
- package/src/__tests__/golden-e2e.test.ts +4 -3
- package/src/__tests__/history-encryption-e2e.test.ts +1 -1
- package/src/activities/__tests__/call-eval.test.ts +115 -0
- package/src/activities/call-eval.ts +59 -1
- package/src/activities/hydrate-workflow-execution.ts +1 -1
- package/src/activities/workflow-event-activities.ts +2 -2
- package/src/harness/transcript/__tests__/builder.test.ts +56 -1
- package/src/harness/transcript/builder.ts +29 -8
- package/src/shared/__tests__/approval-policy.test.ts +10 -0
- package/src/shared/__tests__/args-preview.test.ts +50 -1
- package/src/shared/approval-policy.ts +7 -0
- package/src/shared/args-preview.ts +12 -5
- package/src/worker.ts +2 -4
- package/src/workflow-engine/__tests__/tasks/run.test.ts +25 -0
- package/src/workflow-engine/tasks/run.ts +19 -1
- package/src/workflows/__tests__/barrel.test.ts +41 -0
- package/src/workflows/__tests__/{execute-serverless-workflow.test.ts → engine-inline-model.test.ts} +16 -4
- package/src/workflows/call-agent-orchestrator.ts +1 -1
- package/src/workflows/engine-core.ts +34 -9
- package/src/workflows/index.ts +22 -10
- package/src/workflows/workflow-signals.ts +3 -3
- package/dist/workflows/execute-serverless-workflow.d.ts +0 -32
- package/dist/workflows/execute-serverless-workflow.js +0 -36
- package/dist/workflows/execute-serverless-workflow.js.map +0 -1
- package/src/workflows/execute-serverless-workflow.ts +0 -60
|
@@ -26,7 +26,7 @@ import { fileURLToPath } from "node:url";
|
|
|
26
26
|
import { loadWorkflowFromYaml } from "../workflow-engine/loader.js";
|
|
27
27
|
import { evaluateExpressionBatch } from "../workflow-engine/expression.js";
|
|
28
28
|
import { EncryptionPayloadCodec } from "@stigmer/temporal-codecs";
|
|
29
|
-
import type { ExecuteServerlessWorkflowInput } from "../workflows/
|
|
29
|
+
import type { ExecuteServerlessWorkflowInput } from "../workflows/engine-core.js";
|
|
30
30
|
|
|
31
31
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
32
32
|
const WORKFLOWS_PATH = join(__dirname, "../workflows/index.ts");
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// Unit arms for the eval task's verdict schema: the structured-output shape
|
|
2
|
+
// the judge is bound to, per scoring mode (stigmer#1293).
|
|
3
|
+
//
|
|
4
|
+
// Pinned: each mode declares exactly the fields its parser reads, all
|
|
5
|
+
// required, and a multi-criteria `name` is limited to the configured
|
|
6
|
+
// criteria's names; the schema reaches the LLM call in place of the empty
|
|
7
|
+
// object it replaced; converted the way the LLM call converts it, the schema
|
|
8
|
+
// accepts a well-formed verdict and refuses a nameless criterion, an
|
|
9
|
+
// unconfigured name and a missing field, the shapes a real judge returned
|
|
10
|
+
// when the schema declared no fields; a structured multi-criteria verdict
|
|
11
|
+
// comes back as named criteria with their weighted score.
|
|
12
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
13
|
+
import { jsonSchemaToZod } from "../../shared/json-schema-to-zod.js";
|
|
14
|
+
|
|
15
|
+
const callLlmAction = vi.fn();
|
|
16
|
+
vi.mock("../call-llm.js", () => ({ callLlmAction: (...args: unknown[]) => callLlmAction(...args) }));
|
|
17
|
+
|
|
18
|
+
const { callEvalAction, verdictSchema } = await import("../call-eval.js");
|
|
19
|
+
|
|
20
|
+
const CRITERIA = [
|
|
21
|
+
{ name: "accurate", description: "matches the source", weight: 3 },
|
|
22
|
+
{ name: "concise", description: "short", weight: 1 },
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
beforeEach(() => {
|
|
26
|
+
callLlmAction.mockReset();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("verdictSchema", () => {
|
|
30
|
+
it("declares the fields each mode's parser reads, all required", () => {
|
|
31
|
+
expect(verdictSchema("EVAL_PASS_FAIL")).toEqual({
|
|
32
|
+
type: "object",
|
|
33
|
+
properties: { pass: { type: "boolean" }, reasoning: { type: "string" } },
|
|
34
|
+
required: ["pass", "reasoning"],
|
|
35
|
+
});
|
|
36
|
+
expect(verdictSchema("EVAL_NUMERIC_SCORE")).toEqual({
|
|
37
|
+
type: "object",
|
|
38
|
+
properties: { score: { type: "number" }, reasoning: { type: "string" } },
|
|
39
|
+
required: ["score", "reasoning"],
|
|
40
|
+
});
|
|
41
|
+
expect(verdictSchema("EVAL_MULTI_CRITERIA", CRITERIA)).toEqual({
|
|
42
|
+
type: "object",
|
|
43
|
+
properties: {
|
|
44
|
+
criteria: {
|
|
45
|
+
type: "array",
|
|
46
|
+
items: {
|
|
47
|
+
type: "object",
|
|
48
|
+
properties: {
|
|
49
|
+
name: { type: "string", enum: ["accurate", "concise"] },
|
|
50
|
+
score: { type: "number" },
|
|
51
|
+
reasoning: { type: "string" },
|
|
52
|
+
},
|
|
53
|
+
required: ["name", "score", "reasoning"],
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
required: ["criteria"],
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("refuses an unknown scoring mode", () => {
|
|
62
|
+
expect(() => verdictSchema("EVAL_VIBES")).toThrow("Unknown scoring_mode 'EVAL_VIBES'");
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("converted as the LLM call converts it, binds the multi-criteria verdict's shape", () => {
|
|
66
|
+
const schema = jsonSchemaToZod(verdictSchema("EVAL_MULTI_CRITERIA", CRITERIA));
|
|
67
|
+
const good = {
|
|
68
|
+
criteria: [
|
|
69
|
+
{ name: "accurate", score: 1, reasoning: "right" },
|
|
70
|
+
{ name: "concise", score: 0.5, reasoning: "long" },
|
|
71
|
+
],
|
|
72
|
+
};
|
|
73
|
+
expect(schema.safeParse(good).success).toBe(true);
|
|
74
|
+
expect(schema.safeParse({}).success, "no criteria at all").toBe(false);
|
|
75
|
+
expect(schema.safeParse({ criteria: [{ criterion: "accurate", score: 1, reasoning: "r" }] }).success, "a nameless criterion").toBe(false);
|
|
76
|
+
expect(schema.safeParse({ criteria: [{ name: "style", score: 1, reasoning: "r" }] }).success, "an unconfigured name").toBe(false);
|
|
77
|
+
expect(schema.safeParse({ criteria: [{ name: "accurate", reasoning: "r" }] }).success, "a missing score").toBe(false);
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
describe("callEvalAction", () => {
|
|
82
|
+
function reply(result: unknown): { result: unknown; model: string; input_tokens: number; output_tokens: number } {
|
|
83
|
+
return { result, model: "claude-sonnet-4-6", input_tokens: 100, output_tokens: 20 };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
it("binds the judge to its mode's verdict schema, never an empty object", async () => {
|
|
87
|
+
for (const [mode, result] of [
|
|
88
|
+
["EVAL_PASS_FAIL", { pass: true, reasoning: "ok" }],
|
|
89
|
+
["EVAL_NUMERIC_SCORE", { score: 0.5, reasoning: "ok" }],
|
|
90
|
+
] as const) {
|
|
91
|
+
callLlmAction.mockResolvedValueOnce(reply(result));
|
|
92
|
+
await callEvalAction({ model: "claude-sonnet-4-6", subject: "s", rubric: "r", scoring_mode: mode }, {}, "exec-1");
|
|
93
|
+
expect(callLlmAction.mock.lastCall?.[0].response_schema).toEqual(verdictSchema(mode));
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("reads a structured multi-criteria verdict as named criteria with their weighted score", async () => {
|
|
98
|
+
callLlmAction.mockResolvedValueOnce(
|
|
99
|
+
reply({
|
|
100
|
+
criteria: [
|
|
101
|
+
{ name: "accurate", score: 1, reasoning: "right" },
|
|
102
|
+
{ name: "concise", score: 0.5, reasoning: "long" },
|
|
103
|
+
],
|
|
104
|
+
}),
|
|
105
|
+
);
|
|
106
|
+
const result = await callEvalAction(
|
|
107
|
+
{ model: "claude-sonnet-4-6", subject: "s", rubric: "r", scoring_mode: "EVAL_MULTI_CRITERIA", criteria: CRITERIA, threshold: 0 },
|
|
108
|
+
{},
|
|
109
|
+
"exec-2",
|
|
110
|
+
);
|
|
111
|
+
expect(callLlmAction.mock.lastCall?.[0].response_schema).toEqual(verdictSchema("EVAL_MULTI_CRITERIA", CRITERIA));
|
|
112
|
+
expect(result.criteria?.map((c) => c.name)).toEqual(["accurate", "concise"]);
|
|
113
|
+
expect(result.score).toBeCloseTo(0.875, 5);
|
|
114
|
+
});
|
|
115
|
+
});
|
|
@@ -11,6 +11,16 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Uses callLlmAction for the underlying LLM call, keeping proxy/auth
|
|
13
13
|
* handling in one place.
|
|
14
|
+
*
|
|
15
|
+
* The judge answers through structured output, and the schema it is given is
|
|
16
|
+
* the verdict's own shape for its scoring mode (verdictSchema): the forced
|
|
17
|
+
* `extract` tool a provider binds describes exactly the fields the parsers
|
|
18
|
+
* below read, with a multi-criteria `name` limited to the configured
|
|
19
|
+
* criteria. An object schema with no fields left the model to guess the
|
|
20
|
+
* shape from the prompt's prose alone, and on a real judge the multi-criteria
|
|
21
|
+
* mode came back with no criteria or with nameless ones, read as a score of
|
|
22
|
+
* 0 or an average over "unknown" (stigmer#1293). The prompts still spell the
|
|
23
|
+
* format out; the schema is what binds it.
|
|
14
24
|
*/
|
|
15
25
|
|
|
16
26
|
import { ApplicationFailure } from "@temporalio/activity";
|
|
@@ -54,6 +64,54 @@ export interface EvalResult {
|
|
|
54
64
|
output_tokens: number;
|
|
55
65
|
}
|
|
56
66
|
|
|
67
|
+
// ─── Verdict Schemas ─────────────────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The structured-output schema of a judge's verdict, per scoring mode: the
|
|
71
|
+
* fields the mode's parser reads, all required. For EVAL_MULTI_CRITERIA each
|
|
72
|
+
* returned criterion's `name` is one of the configured criteria's names.
|
|
73
|
+
*/
|
|
74
|
+
export function verdictSchema(scoringMode: string, criteria: readonly EvalCriterion[] = []): Record<string, unknown> {
|
|
75
|
+
switch (scoringMode) {
|
|
76
|
+
case "EVAL_PASS_FAIL":
|
|
77
|
+
return {
|
|
78
|
+
type: "object",
|
|
79
|
+
properties: { pass: { type: "boolean" }, reasoning: { type: "string" } },
|
|
80
|
+
required: ["pass", "reasoning"],
|
|
81
|
+
};
|
|
82
|
+
case "EVAL_NUMERIC_SCORE":
|
|
83
|
+
return {
|
|
84
|
+
type: "object",
|
|
85
|
+
properties: { score: { type: "number" }, reasoning: { type: "string" } },
|
|
86
|
+
required: ["score", "reasoning"],
|
|
87
|
+
};
|
|
88
|
+
case "EVAL_MULTI_CRITERIA":
|
|
89
|
+
return {
|
|
90
|
+
type: "object",
|
|
91
|
+
properties: {
|
|
92
|
+
criteria: {
|
|
93
|
+
type: "array",
|
|
94
|
+
items: {
|
|
95
|
+
type: "object",
|
|
96
|
+
properties: {
|
|
97
|
+
name: { type: "string", enum: criteria.map((c) => c.name) },
|
|
98
|
+
score: { type: "number" },
|
|
99
|
+
reasoning: { type: "string" },
|
|
100
|
+
},
|
|
101
|
+
required: ["name", "score", "reasoning"],
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
required: ["criteria"],
|
|
106
|
+
};
|
|
107
|
+
default:
|
|
108
|
+
throw ApplicationFailure.nonRetryable(
|
|
109
|
+
`Unknown scoring_mode '${scoringMode}'. Supported: EVAL_PASS_FAIL, EVAL_NUMERIC_SCORE, EVAL_MULTI_CRITERIA`,
|
|
110
|
+
"EVAL_UNKNOWN_SCORING_MODE",
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
57
115
|
// ─── Judge Prompt Builders ───────────────────────────────────────────────
|
|
58
116
|
|
|
59
117
|
function buildPassFailPrompt(rubric: string, subject: string): string {
|
|
@@ -272,7 +330,7 @@ export async function callEvalAction(
|
|
|
272
330
|
model: config.model,
|
|
273
331
|
prompt,
|
|
274
332
|
system_prompt: config.system_prompt,
|
|
275
|
-
response_schema:
|
|
333
|
+
response_schema: verdictSchema(scoringMode, config.criteria),
|
|
276
334
|
},
|
|
277
335
|
runtimeEnv,
|
|
278
336
|
executionId,
|
|
@@ -19,7 +19,7 @@ import { StigmerClient } from "../client/stigmer-client.js";
|
|
|
19
19
|
import { loadWorkflowFromYaml } from "../workflow-engine/loader.js";
|
|
20
20
|
import { ValidationState } from "@stigmer/protos/ai/stigmer/agentic/workflow/v1/serverless/validation_pb";
|
|
21
21
|
import type { Config } from "../config.js";
|
|
22
|
-
import type { ExecuteServerlessWorkflowInput } from "../workflows/
|
|
22
|
+
import type { ExecuteServerlessWorkflowInput } from "../workflows/engine-core.js";
|
|
23
23
|
|
|
24
24
|
export interface HydrateInput {
|
|
25
25
|
readonly execution_id: string;
|
|
@@ -167,8 +167,8 @@ function nextSequence(): bigint {
|
|
|
167
167
|
* failed run) returns the highest persisted sequence_number, so new
|
|
168
168
|
* events continue from N+1.
|
|
169
169
|
*
|
|
170
|
-
* When executionId is empty (
|
|
171
|
-
*
|
|
170
|
+
* When executionId is empty (an engine run with no persisted execution,
|
|
171
|
+
* as the test harness's inline entry makes), returns 0 as a safe fallback.
|
|
172
172
|
*
|
|
173
173
|
* Returns a plain number — the value crosses back into the workflow
|
|
174
174
|
* through Temporal's JSON payload converter, which cannot carry BigInt.
|
|
@@ -174,7 +174,7 @@ describe("TranscriptBuilder — tool_finished and tool_error are upserts", () =>
|
|
|
174
174
|
});
|
|
175
175
|
});
|
|
176
176
|
|
|
177
|
-
describe("TranscriptBuilder — every row with args carries an elided, redacted argsPreview", () => {
|
|
177
|
+
describe("TranscriptBuilder — every row with args carries them redacted, and an elided, redacted argsPreview", () => {
|
|
178
178
|
it("an ungated row's preview is its args, secret keys redacted", () => {
|
|
179
179
|
const { status } = feed(builder(), [
|
|
180
180
|
{ kind: "tool_started", callId: "c-1", name: "connect", input: { host: "localhost", password: "super-secret" }, mcpServerSlug: "db" },
|
|
@@ -182,6 +182,61 @@ describe("TranscriptBuilder — every row with args carries an elided, redacted
|
|
|
182
182
|
expect(JSON.parse(rowOf(status, "c-1").argsPreview)).toEqual({ host: "localhost", password: "[REDACTED]" });
|
|
183
183
|
});
|
|
184
184
|
|
|
185
|
+
// stigmer#1119: the row's own args were stored raw, so a settled row's
|
|
186
|
+
// detail view showed what the waiting row's preview hid. Every path that
|
|
187
|
+
// writes a row's args redacts them; every other key stays verbatim.
|
|
188
|
+
describe("the row's args never carry a secret-keyed value, on every path that writes them", () => {
|
|
189
|
+
const input = { host: "localhost", password: "super-secret" };
|
|
190
|
+
const redacted = { host: "localhost", password: "[REDACTED]" };
|
|
191
|
+
|
|
192
|
+
it("a started call", () => {
|
|
193
|
+
const { status } = feed(builder(), [{ kind: "tool_started", callId: "c-1", name: "connect", input, mcpServerSlug: "db" }]);
|
|
194
|
+
expect(rowOf(status, "c-1").args).toEqual(redacted);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it("a re-emitted start filling the args the row lacked", () => {
|
|
198
|
+
const { status } = feed(builder(), [
|
|
199
|
+
{ kind: "tool_started", callId: "c-1", name: "connect", input: {}, mcpServerSlug: "db" },
|
|
200
|
+
{ kind: "tool_started", callId: "c-1", name: "connect", input, mcpServerSlug: "db" },
|
|
201
|
+
]);
|
|
202
|
+
expect(rowOf(status, "c-1").args).toEqual(redacted);
|
|
203
|
+
expect(JSON.parse(rowOf(status, "c-1").argsPreview)).toEqual(redacted);
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it("arguments streamed in chunks, once they parse", () => {
|
|
207
|
+
const { status } = feed(builder(), [
|
|
208
|
+
{ kind: "tool_started", callId: "c-1", name: "connect", input: {}, mcpServerSlug: "db" },
|
|
209
|
+
{ kind: "tool_arg_delta", callId: "c-1", argsChunk: '{"host":"localhost",' },
|
|
210
|
+
{ kind: "tool_arg_delta", callId: "c-1", argsChunk: '"password":"super-secret"}' },
|
|
211
|
+
]);
|
|
212
|
+
expect(rowOf(status, "c-1").args).toEqual(redacted);
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
it("streamed arguments that parse to something other than an object are not args", () => {
|
|
216
|
+
const { status } = feed(builder(), [
|
|
217
|
+
{ kind: "tool_started", callId: "c-1", name: "connect", input: {}, mcpServerSlug: "db" },
|
|
218
|
+
{ kind: "tool_arg_delta", callId: "c-1", argsChunk: '"super-secret"' },
|
|
219
|
+
]);
|
|
220
|
+
expect(rowOf(status, "c-1").args).toBeUndefined();
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("an approval that reopens a streamed row with the proposal's args", () => {
|
|
224
|
+
const { status } = feed(builder(), [
|
|
225
|
+
{ kind: "tool_started", callId: "c-1", name: "connect", input: { host: "localhost" }, mcpServerSlug: "db" },
|
|
226
|
+
{ kind: "approval_proposed", callId: "c-1", name: "connect", mcpServerSlug: "db", message: "Connect?", args: input },
|
|
227
|
+
]);
|
|
228
|
+
expect(rowOf(status, "c-1").args).toEqual(redacted);
|
|
229
|
+
expect(JSON.parse(rowOf(status, "c-1").argsPreview)).toEqual(redacted);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it("an approval for a call the stream never showed", () => {
|
|
233
|
+
const { status } = feed(builder(), [
|
|
234
|
+
{ kind: "approval_proposed", callId: "c-1", name: "connect", mcpServerSlug: "db", message: "Connect?", args: input },
|
|
235
|
+
]);
|
|
236
|
+
expect(rowOf(status, "c-1").args).toEqual(redacted);
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
|
|
185
240
|
it("a long non-salient value is left out of the preview; a salient one (the command) survives verbatim", () => {
|
|
186
241
|
const command = "echo " + "x".repeat(400);
|
|
187
242
|
const { status } = feed(builder(), [
|
|
@@ -39,8 +39,9 @@
|
|
|
39
39
|
* RUNNING (a WAITING row on native's durable-checkpoint resume; an
|
|
40
40
|
* INTERRUPTED row on a recovery replay, the enum's supersede rule) and
|
|
41
41
|
* takes the args and preview it lacked — never duplicated. Every row
|
|
42
|
-
* with args carries
|
|
43
|
-
*
|
|
42
|
+
* with args carries them with secret-keyed values redacted
|
|
43
|
+
* (stigmer#1119), and an `argsPreview` elided and redacted, salient
|
|
44
|
+
* fields verbatim. A finish or an error is an upsert: status
|
|
44
45
|
* monotonic, `completedAt` once, a non-empty result or message
|
|
45
46
|
* overwrites, an empty one never clears; a re-emit that
|
|
46
47
|
* changes nothing forces no persist; a fact the harness
|
|
@@ -124,7 +125,7 @@ import {
|
|
|
124
125
|
ToolKind,
|
|
125
126
|
} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
126
127
|
import { POLICY_ENGINE_VERSION, toProtoPolicySource } from "../../shared/approval-policy.js";
|
|
127
|
-
import { SALIENT_ARG_FIELDS, buildElidedArgsPreview } from "../../shared/args-preview.js";
|
|
128
|
+
import { SALIENT_ARG_FIELDS, buildElidedArgsPreview, redactSensitiveArgs } from "../../shared/args-preview.js";
|
|
128
129
|
import { classifyTool } from "../../shared/tool-kind.js";
|
|
129
130
|
import { applyTodoUpdate } from "../../shared/todos.js";
|
|
130
131
|
import { utcTimestamp } from "../../shared/status.js";
|
|
@@ -453,7 +454,7 @@ export class TranscriptBuilder {
|
|
|
453
454
|
existing.status = ToolCallStatus.TOOL_CALL_RUNNING;
|
|
454
455
|
}
|
|
455
456
|
if (Object.keys(input).length > 0) {
|
|
456
|
-
if (!existing.args) existing.args = input
|
|
457
|
+
if (!existing.args) existing.args = rowArgs(input);
|
|
457
458
|
if (!existing.argsPreview) stampArgsPreview(existing, input);
|
|
458
459
|
}
|
|
459
460
|
this._dirty = true;
|
|
@@ -479,7 +480,7 @@ export class TranscriptBuilder {
|
|
|
479
480
|
});
|
|
480
481
|
|
|
481
482
|
if (Object.keys(input).length > 0) {
|
|
482
|
-
tc.args = input
|
|
483
|
+
tc.args = rowArgs(input);
|
|
483
484
|
stampArgsPreview(tc, input);
|
|
484
485
|
}
|
|
485
486
|
|
|
@@ -596,10 +597,16 @@ export class TranscriptBuilder {
|
|
|
596
597
|
const buffer = (scope.argBuffers.get(callId) ?? "") + argsChunk;
|
|
597
598
|
scope.argBuffers.set(callId, buffer);
|
|
598
599
|
|
|
600
|
+
let parsed: unknown;
|
|
599
601
|
try {
|
|
600
|
-
|
|
602
|
+
parsed = JSON.parse(buffer);
|
|
601
603
|
} catch {
|
|
602
604
|
// Partial JSON — will parse on next chunk or tool-finished
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
// A tool's arguments are an object (a proto Struct); anything else is not args.
|
|
608
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
|
609
|
+
tc.args = rowArgs(parsed as Record<string, unknown>);
|
|
603
610
|
}
|
|
604
611
|
}
|
|
605
612
|
|
|
@@ -641,7 +648,7 @@ export class TranscriptBuilder {
|
|
|
641
648
|
// and they re-derive the preview. Absent, the row
|
|
642
649
|
// keeps what it had.
|
|
643
650
|
if (event.args && Object.keys(event.args).length > 0) {
|
|
644
|
-
existing.args = event.args
|
|
651
|
+
existing.args = rowArgs(event.args);
|
|
645
652
|
stampArgsPreview(existing, event.args);
|
|
646
653
|
}
|
|
647
654
|
if (event.contentDigest) existing.approvalContentDigest = event.contentDigest;
|
|
@@ -671,7 +678,7 @@ export class TranscriptBuilder {
|
|
|
671
678
|
tc.policyEngineVersion = POLICY_ENGINE_VERSION;
|
|
672
679
|
}
|
|
673
680
|
if (event.args && Object.keys(event.args).length > 0) {
|
|
674
|
-
tc.args = event.args
|
|
681
|
+
tc.args = rowArgs(event.args);
|
|
675
682
|
stampArgsPreview(tc, event.args);
|
|
676
683
|
}
|
|
677
684
|
if (event.contentDigest) tc.approvalContentDigest = event.contentDigest;
|
|
@@ -754,6 +761,20 @@ function finalizeStreaming(transcript: Transcript): void {
|
|
|
754
761
|
}
|
|
755
762
|
}
|
|
756
763
|
|
|
764
|
+
/**
|
|
765
|
+
* The row's `args`, on every row with args: the call's arguments with every
|
|
766
|
+
* secret-keyed value redacted (`redactSensitiveArgs`, the preview's own
|
|
767
|
+
* first step). Until stigmer#1119 the row stored them raw and only the
|
|
768
|
+
* preview was redacted, so a settled row's detail view showed a token the
|
|
769
|
+
* waiting row had hidden. Nothing that reads a row's args back needs a
|
|
770
|
+
* secret: exact-apply, the approval identity and the content digest read
|
|
771
|
+
* path, content, edit and command fields, which
|
|
772
|
+
* `shared/__tests__/args-preview.test.ts` pins disjoint from the secret keys.
|
|
773
|
+
*/
|
|
774
|
+
function rowArgs(args: Record<string, unknown>): JsonObject {
|
|
775
|
+
return redactSensitiveArgs(args) as JsonObject;
|
|
776
|
+
}
|
|
777
|
+
|
|
757
778
|
/**
|
|
758
779
|
* The row's `args_preview`, on EVERY row with args (since #1097):
|
|
759
780
|
* `message.proto` promises it sanitized, redacted, at creation, for inline
|
|
@@ -418,6 +418,16 @@ describe("resolveApprovalMessage", () => {
|
|
|
418
418
|
)).toBe("Set 42");
|
|
419
419
|
});
|
|
420
420
|
|
|
421
|
+
// stigmer#1119: the message is stored on the row and shown on the approval
|
|
422
|
+
// card, so a secret-keyed placeholder never resolves to the secret.
|
|
423
|
+
it("redacts a secret-keyed placeholder, whatever the key's case, and keeps the rest", () => {
|
|
424
|
+
expect(resolveApprovalMessage(
|
|
425
|
+
"Push to {{args.repo}} with {{args.token}} ({{args.Api_Key}})?",
|
|
426
|
+
"push",
|
|
427
|
+
{ repo: "acme/x", token: "s3cret", Api_Key: "sk-123" },
|
|
428
|
+
)).toBe("Push to acme/x with [REDACTED] ([REDACTED])?");
|
|
429
|
+
});
|
|
430
|
+
|
|
421
431
|
it("handles null arg values", () => {
|
|
422
432
|
expect(resolveApprovalMessage(
|
|
423
433
|
"Value: {{args.val}}",
|
|
@@ -12,8 +12,25 @@
|
|
|
12
12
|
* caller left.)
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
+
import { readFileSync } from "node:fs";
|
|
16
|
+
import { dirname, resolve } from "node:path";
|
|
17
|
+
import { fileURLToPath } from "node:url";
|
|
15
18
|
import { describe, it, expect } from "vitest";
|
|
16
|
-
import { buildElidedArgsPreview, redactSensitiveArgs, SALIENT_ARG_FIELDS } from "../args-preview.js";
|
|
19
|
+
import { buildElidedArgsPreview, redactSensitiveArgs, SALIENT_ARG_FIELDS, SENSITIVE_ARG_KEYS } from "../args-preview.js";
|
|
20
|
+
import { EDIT_NEW_FIELDS, EDIT_OLD_FIELDS, WRITE_CONTENT_FIELDS, extractFilePath } from "../file-tools.js";
|
|
21
|
+
|
|
22
|
+
interface SecretArgsFixture {
|
|
23
|
+
keys: string[];
|
|
24
|
+
marker: string;
|
|
25
|
+
cases: { name: string; args: Record<string, unknown>; redacted: Record<string, unknown> }[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// The cross-surface contract the @stigmer/sdk reader asserts too
|
|
29
|
+
// (sdk/typescript/src/execution/__tests__/tool-view.fixtures.test.ts).
|
|
30
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
31
|
+
const secretArgs = JSON.parse(
|
|
32
|
+
readFileSync(resolve(here, "../../../../../../test/fixtures/tool-view/secret-args.json"), "utf8"),
|
|
33
|
+
) as SecretArgsFixture;
|
|
17
34
|
|
|
18
35
|
describe("SALIENT_ARG_FIELDS", () => {
|
|
19
36
|
it("spans both harnesses' shapes for a file and a shell command, in priority order", () => {
|
|
@@ -27,6 +44,38 @@ describe("redactSensitiveArgs", () => {
|
|
|
27
44
|
});
|
|
28
45
|
});
|
|
29
46
|
|
|
47
|
+
describe("the secret-args wire contract (test/fixtures/tool-view/secret-args.json)", () => {
|
|
48
|
+
it("SENSITIVE_ARG_KEYS is the fixture's key set, the one the SDK reader mirrors", () => {
|
|
49
|
+
expect([...SENSITIVE_ARG_KEYS].sort()).toEqual([...secretArgs.keys].sort());
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
for (const c of secretArgs.cases) {
|
|
53
|
+
it(`redacts ${c.name}`, () => {
|
|
54
|
+
expect(redactSensitiveArgs(c.args)).toEqual(c.redacted);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
// A row's args are redacted at creation (stigmer#1119), and three readers act
|
|
60
|
+
// on them afterwards: exact-apply writes an approved file from the path and
|
|
61
|
+
// content, the Cursor approval identity keys on the salient field, and the
|
|
62
|
+
// content digest hashes the write or edit content. Widening the secret set into
|
|
63
|
+
// one of those fields would make an approved write land as the marker on disk.
|
|
64
|
+
describe("secret keys never overlap the fields a row's args are acted on by", () => {
|
|
65
|
+
const actedOn = [...SALIENT_ARG_FIELDS, ...WRITE_CONTENT_FIELDS, ...EDIT_OLD_FIELDS, ...EDIT_NEW_FIELDS];
|
|
66
|
+
|
|
67
|
+
it("no salient, write-content or edit field is a secret key", () => {
|
|
68
|
+
const overlap = actedOn.filter((field) => SENSITIVE_ARG_KEYS.has(field.toLowerCase()));
|
|
69
|
+
expect(overlap).toEqual([]);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("no secret-keyed argument resolves as a file path", () => {
|
|
73
|
+
for (const key of SENSITIVE_ARG_KEYS) {
|
|
74
|
+
expect(extractFilePath({ [key]: "/w/a.txt" })).toBeNull();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
|
|
30
79
|
describe("buildElidedArgsPreview", () => {
|
|
31
80
|
const SALIENT = SALIENT_ARG_FIELDS;
|
|
32
81
|
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import type { ToolApprovalPolicy } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/spec_pb";
|
|
19
19
|
import type { AgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
|
|
20
20
|
import { ApprovalAction, ApprovalMode, ApprovalPolicySource } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
21
|
+
import { SENSITIVE_ARG_KEYS } from "./args-preview.js";
|
|
21
22
|
import { toolApprovalCategory, type ToolApprovalCategory } from "./tool-kind.js";
|
|
22
23
|
import type { ResolvedMcpServer } from "./mcp-resolver.js";
|
|
23
24
|
|
|
@@ -451,6 +452,11 @@ export function lookupMcpToolPolicy(
|
|
|
451
452
|
* - {{args.field_name}} — replaced with the argument value
|
|
452
453
|
* - {{tool_name}} — replaced with the tool name
|
|
453
454
|
* - Missing fields are replaced with "<unknown>"
|
|
455
|
+
* - A secret-keyed field ({@link SENSITIVE_ARG_KEYS}) is replaced with
|
|
456
|
+
* "[REDACTED]" (stigmer#1119): the message is stored on the row and shown on
|
|
457
|
+
* the approval card, which must carry no more than the redacted preview.
|
|
458
|
+
* Every approval message in the runner, native and Cursor alike, resolves
|
|
459
|
+
* here, so this is the one place the rule applies to it.
|
|
454
460
|
*/
|
|
455
461
|
export function resolveApprovalMessage(
|
|
456
462
|
template: string,
|
|
@@ -460,6 +466,7 @@ export function resolveApprovalMessage(
|
|
|
460
466
|
return template
|
|
461
467
|
.replace(/\{\{tool_name\}\}/g, toolName)
|
|
462
468
|
.replace(/\{\{args\.(\w+)\}\}/g, (_match, field: string) => {
|
|
469
|
+
if (SENSITIVE_ARG_KEYS.has(field.toLowerCase())) return "[REDACTED]";
|
|
463
470
|
const value = args[field];
|
|
464
471
|
if (value === undefined || value === null) return "<unknown>";
|
|
465
472
|
if (typeof value === "string") return value;
|
|
@@ -34,8 +34,13 @@
|
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
|
-
* Argument keys whose values are secrets and must never appear in a preview
|
|
38
|
-
*
|
|
37
|
+
* Argument keys whose values are secrets and must never appear in a preview,
|
|
38
|
+
* on a row's `args`, or in an approval message (stigmer#1119). Matched
|
|
39
|
+
* case-insensitively against the top-level key name. Clients mirror this set
|
|
40
|
+
* to hide the values on rows persisted before the runner redacted `args`
|
|
41
|
+
* (`@stigmer/sdk`'s `isSecretArgKey`); the shared, machine-checked contract is
|
|
42
|
+
* `test/fixtures/tool-view/secret-args.json` — keep this set and that fixture
|
|
43
|
+
* in lockstep.
|
|
39
44
|
*/
|
|
40
45
|
export const SENSITIVE_ARG_KEYS: ReadonlySet<string> = new Set([
|
|
41
46
|
"password", "token", "secret", "api_key", "apikey",
|
|
@@ -60,9 +65,11 @@ export const SALIENT_ARG_FIELDS = ["file_path", "path", "target_notebook", "comm
|
|
|
60
65
|
/**
|
|
61
66
|
* Redact secret-keyed values (see {@link SENSITIVE_ARG_KEYS}), preserving every
|
|
62
67
|
* other entry verbatim. The first step of the preview builder, and the shape
|
|
63
|
-
* stamped as `args` on
|
|
64
|
-
*
|
|
65
|
-
*
|
|
68
|
+
* stamped as `args` on every row: by the transcript builder on every row with
|
|
69
|
+
* args (since stigmer#1119), and on a proposed row the harness recovered from
|
|
70
|
+
* its engine's state (issue #754's header fix). Full enough for the UI's
|
|
71
|
+
* path/primary-arg extraction and for exact-apply, never carrying a secret
|
|
72
|
+
* value.
|
|
66
73
|
*/
|
|
67
74
|
export function redactSensitiveArgs(
|
|
68
75
|
args: Record<string, unknown>,
|
package/src/worker.ts
CHANGED
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Registers both activities and workflows on a single queue.
|
|
5
5
|
*
|
|
6
|
-
* Workflows:
|
|
7
|
-
*
|
|
8
|
-
* - "stigmer/mcp-server/discover" — Legacy discover-only workflow
|
|
9
|
-
* - "stigmer/workflow/execute" — CNCF Serverless Workflow execution
|
|
6
|
+
* Workflows: exactly the types in `workflows/index.ts`, each started by
|
|
7
|
+
* the server with ids (that file's header is the list and the rule).
|
|
10
8
|
*
|
|
11
9
|
* Architecture:
|
|
12
10
|
* - Java backend starts connect workflows on the runner's task queue
|
|
@@ -79,6 +79,31 @@ describe("RunTaskBuilder", () => {
|
|
|
79
79
|
const builder = new RunTaskBuilder("task", taskDef);
|
|
80
80
|
expect(() => builder.build()).toThrow("workflow.name is required");
|
|
81
81
|
});
|
|
82
|
+
|
|
83
|
+
it.each([
|
|
84
|
+
"stigmer/workflow/execute",
|
|
85
|
+
"stigmer/workflow/execute-from-execution",
|
|
86
|
+
"stigmer/mcp-server/connect",
|
|
87
|
+
"stigmer/mcp-server/discover",
|
|
88
|
+
"stigmer/agent-execution/invoke",
|
|
89
|
+
])("refuses the platform workflow type %s", (name) => {
|
|
90
|
+
const taskDef: RunTaskDef = {
|
|
91
|
+
kind: "run",
|
|
92
|
+
run: { workflow: { name, namespace: "default", version: "1.0.0" } },
|
|
93
|
+
};
|
|
94
|
+
const builder = new RunTaskBuilder("reachIn", taskDef);
|
|
95
|
+
expect(() => builder.build()).toThrow(
|
|
96
|
+
`Run task 'reachIn': '${name}' is a platform workflow type, and a workflow cannot start one`,
|
|
97
|
+
);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("admits a name that only mentions the platform outside the prefix", () => {
|
|
101
|
+
const taskDef: RunTaskDef = {
|
|
102
|
+
kind: "run",
|
|
103
|
+
run: { workflow: { name: "my-stigmer/child", namespace: "default", version: "1.0.0" } },
|
|
104
|
+
};
|
|
105
|
+
expect(() => new RunTaskBuilder("child", taskDef).build()).not.toThrow();
|
|
106
|
+
});
|
|
82
107
|
});
|
|
83
108
|
|
|
84
109
|
describe("script execution", () => {
|
|
@@ -7,7 +7,9 @@
|
|
|
7
7
|
* - `run.workflow`: Child Temporal workflow (await or fire-and-forget)
|
|
8
8
|
*
|
|
9
9
|
* The kernel validates configuration and delegates to `ctx.runCommand`
|
|
10
|
-
* (for script/shell) or `ctx.runWorkflow` (for child workflows).
|
|
10
|
+
* (for script/shell) or `ctx.runWorkflow` (for child workflows). A child
|
|
11
|
+
* workflow's name may not be a platform workflow type
|
|
12
|
+
* ({@link PLATFORM_WORKFLOW_TYPE_PREFIX} says why).
|
|
11
13
|
*
|
|
12
14
|
* YAML shapes:
|
|
13
15
|
* - runScript:
|
|
@@ -42,6 +44,17 @@ import type {
|
|
|
42
44
|
|
|
43
45
|
const SUPPORTED_LANGUAGES = ["js", "python"];
|
|
44
46
|
|
|
47
|
+
/**
|
|
48
|
+
* The namespace of the platform's own Temporal workflow types: every type a
|
|
49
|
+
* runner registers (`workflows/index.ts`) and every type the server starts.
|
|
50
|
+
* A child started by `run.workflow` runs on the parent's task queue, so a
|
|
51
|
+
* name in this namespace would reach the runner's own types directly: work
|
|
52
|
+
* the server never dispatched, outside the run it belongs to. A workflow
|
|
53
|
+
* therefore cannot name one. `workflows/__tests__/barrel.test.ts` pins that
|
|
54
|
+
* every registered type carries this prefix, so the refusal covers each.
|
|
55
|
+
*/
|
|
56
|
+
export const PLATFORM_WORKFLOW_TYPE_PREFIX = "stigmer/";
|
|
57
|
+
|
|
45
58
|
export class RunTaskBuilder implements TaskBuilder {
|
|
46
59
|
readonly taskName: string;
|
|
47
60
|
readonly taskDef: TaskDef;
|
|
@@ -138,5 +151,10 @@ function validateRunConfig(config: RunConfig, taskName: string): void {
|
|
|
138
151
|
if (!config.workflow.name) {
|
|
139
152
|
throw new Error(`Run task '${taskName}': workflow.name is required`);
|
|
140
153
|
}
|
|
154
|
+
if (config.workflow.name.startsWith(PLATFORM_WORKFLOW_TYPE_PREFIX)) {
|
|
155
|
+
throw new Error(
|
|
156
|
+
`Run task '${taskName}': '${config.workflow.name}' is a platform workflow type, and a workflow cannot start one`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
141
159
|
}
|
|
142
160
|
}
|