@lnilluv/pi-ralph-loop 0.1.4-dev.0 → 0.1.4-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -12
- package/package.json +1 -1
- package/src/index.ts +1034 -168
- package/src/ralph-draft-llm.ts +35 -7
- package/src/ralph-draft.ts +1 -1
- package/src/ralph.ts +708 -51
- package/src/runner-rpc.ts +434 -0
- package/src/runner-state.ts +822 -0
- package/src/runner.ts +957 -0
- package/tests/fixtures/parity/migrate/OPEN_QUESTIONS.md +3 -0
- package/tests/fixtures/parity/migrate/RALPH.md +27 -0
- package/tests/fixtures/parity/migrate/golden/MIGRATED.md +15 -0
- package/tests/fixtures/parity/migrate/legacy/source.md +6 -0
- package/tests/fixtures/parity/migrate/legacy/source.yaml +3 -0
- package/tests/fixtures/parity/migrate/scripts/show-legacy.sh +10 -0
- package/tests/fixtures/parity/migrate/scripts/verify.sh +15 -0
- package/tests/fixtures/parity/research/OPEN_QUESTIONS.md +3 -0
- package/tests/fixtures/parity/research/RALPH.md +45 -0
- package/tests/fixtures/parity/research/claim-evidence-checklist.md +15 -0
- package/tests/fixtures/parity/research/expected-outputs.md +22 -0
- package/tests/fixtures/parity/research/scripts/show-snapshots.sh +13 -0
- package/tests/fixtures/parity/research/scripts/verify.sh +55 -0
- package/tests/fixtures/parity/research/snapshots/app-factory-ai-cli.md +11 -0
- package/tests/fixtures/parity/research/snapshots/docs-factory-ai-cli-features-missions.md +11 -0
- package/tests/fixtures/parity/research/snapshots/factory-ai-news-missions.md +11 -0
- package/tests/fixtures/parity/research/source-manifest.md +20 -0
- package/tests/index.test.ts +3169 -104
- package/tests/parity/README.md +9 -0
- package/tests/parity/harness.py +526 -0
- package/tests/parity-harness.test.ts +42 -0
- package/tests/parity-research-fixture.test.ts +34 -0
- package/tests/ralph-draft-llm.test.ts +82 -9
- package/tests/ralph-draft.test.ts +1 -1
- package/tests/ralph.test.ts +1265 -36
- package/tests/runner-event-contract.test.ts +235 -0
- package/tests/runner-rpc.test.ts +358 -0
- package/tests/runner-state.test.ts +553 -0
- package/tests/runner.test.ts +1347 -0
package/tests/ralph.test.ts
CHANGED
|
@@ -13,7 +13,9 @@ import {
|
|
|
13
13
|
extractDraftMetadata,
|
|
14
14
|
generateDraft,
|
|
15
15
|
isWeakStrengthenedDraft,
|
|
16
|
+
acceptStrengthenedDraft,
|
|
16
17
|
normalizeStrengthenedDraft,
|
|
18
|
+
inspectDraftContent,
|
|
17
19
|
inspectExistingTarget,
|
|
18
20
|
inspectRepo,
|
|
19
21
|
looksLikePath,
|
|
@@ -24,6 +26,8 @@ import {
|
|
|
24
26
|
renderIterationPrompt,
|
|
25
27
|
renderRalphBody,
|
|
26
28
|
resolvePlaceholders,
|
|
29
|
+
resolveCommandRun,
|
|
30
|
+
runtimeArgEntriesToMap,
|
|
27
31
|
slugifyTask,
|
|
28
32
|
shouldValidateExistingDraft,
|
|
29
33
|
validateDraftContent,
|
|
@@ -70,6 +74,34 @@ function assertMetadataSource(metadata: ReturnType<typeof extractDraftMetadata>,
|
|
|
70
74
|
assert.equal(metadata.source, expected);
|
|
71
75
|
}
|
|
72
76
|
|
|
77
|
+
function makeFixRequest() {
|
|
78
|
+
return buildDraftRequest(
|
|
79
|
+
"Fix flaky auth tests",
|
|
80
|
+
{ slug: "fix-flaky-auth-tests", dirPath: "/repo/fix-flaky-auth-tests", ralphPath: "/repo/fix-flaky-auth-tests/RALPH.md" },
|
|
81
|
+
{ packageManager: "npm", testCommand: "npm test", lintCommand: "npm run lint", hasGit: true, topLevelDirs: ["src"], topLevelFiles: ["package.json"] },
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function makeFixRequestWithArgs(args: readonly string[]) {
|
|
86
|
+
const request = makeFixRequest();
|
|
87
|
+
return {
|
|
88
|
+
...request,
|
|
89
|
+
baselineDraft: request.baselineDraft.replace(`commands:\n`, `args:\n - ${args.join("\n - ")}\ncommands:\n`),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function makeFixRequestWithCompletionPromise(completionPromise: string) {
|
|
94
|
+
const request = makeFixRequest();
|
|
95
|
+
return {
|
|
96
|
+
...request,
|
|
97
|
+
baselineDraft: request.baselineDraft.replace("timeout: 300\n", `completion_promise: ${completionPromise}\ntimeout: 300\n`),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function makeStrengthenedDraft(frontmatterLines: readonly string[], body: string, task = "Fix flaky auth tests") {
|
|
102
|
+
return `${encodeMetadata({ generator: "pi-ralph-loop", version: 2, source: "llm-strengthened", task, mode: "fix" })}\n---\n${frontmatterLines.join("\n")}\n---\n${body}`;
|
|
103
|
+
}
|
|
104
|
+
|
|
73
105
|
test("parseRalphMarkdown falls back to default frontmatter when no frontmatter is present", () => {
|
|
74
106
|
const parsed = parseRalphMarkdown("hello\nworld");
|
|
75
107
|
|
|
@@ -79,55 +111,349 @@ test("parseRalphMarkdown falls back to default frontmatter when no frontmatter i
|
|
|
79
111
|
|
|
80
112
|
test("parseRalphMarkdown parses frontmatter and normalizes line endings", () => {
|
|
81
113
|
const parsed = parseRalphMarkdown(
|
|
82
|
-
"\uFEFF---\r\ncommands:\r\n - name: build\r\n run: npm test\r\n timeout: 15\r\nmax_iterations: 3\r\ntimeout: 12.5\r\ncompletion_promise: done\r\nguardrails:\r\n block_commands:\r\n - rm .*\r\n protected_files:\r\n - src/**\r\n---\r\nBody\r\n",
|
|
114
|
+
"\uFEFF---\r\ncommands:\r\n - name: build\r\n run: npm test\r\n timeout: 15\r\nmax_iterations: 3\r\ninter_iteration_delay: 7\r\ntimeout: 12.5\r\nrequired_outputs:\r\n - docs/ARCHITECTURE.md\r\ncompletion_promise: done\r\nguardrails:\r\n block_commands:\r\n - rm .*\r\n protected_files:\r\n - src/**\r\n---\r\nBody\r\n",
|
|
83
115
|
);
|
|
84
116
|
|
|
85
117
|
assert.deepEqual(parsed.frontmatter, {
|
|
86
118
|
commands: [{ name: "build", run: "npm test", timeout: 15 }],
|
|
87
119
|
maxIterations: 3,
|
|
120
|
+
interIterationDelay: 7,
|
|
88
121
|
timeout: 12.5,
|
|
89
122
|
completionPromise: "done",
|
|
123
|
+
requiredOutputs: ["docs/ARCHITECTURE.md"],
|
|
90
124
|
guardrails: { blockCommands: ["rm .*"], protectedFiles: ["src/**"] },
|
|
91
125
|
invalidCommandEntries: undefined,
|
|
92
126
|
});
|
|
93
127
|
assert.equal(parsed.body, "Body\n");
|
|
94
128
|
});
|
|
95
129
|
|
|
96
|
-
test("
|
|
130
|
+
test("parseRalphMarkdown parses declared args as runtime parameters", () => {
|
|
131
|
+
const parsed = parseRalphMarkdown(
|
|
132
|
+
"---\nargs:\n - owner\n - mode\ncommands: []\nmax_iterations: 1\ntimeout: 1\nguardrails:\n block_commands: []\n protected_files: []\n---\nBody\n",
|
|
133
|
+
);
|
|
134
|
+
|
|
135
|
+
assert.deepEqual(parsed.frontmatter.args, ["owner", "mode"]);
|
|
136
|
+
assert.equal(validateFrontmatter(parsed.frontmatter), null);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("validateFrontmatter accepts valid input and rejects invalid bounds, names, args, and globs", () => {
|
|
97
140
|
assert.equal(validateFrontmatter(defaultFrontmatter()), null);
|
|
141
|
+
assert.equal(
|
|
142
|
+
validateFrontmatter({ ...defaultFrontmatter(), args: ["owner"] }),
|
|
143
|
+
null,
|
|
144
|
+
);
|
|
145
|
+
assert.equal(
|
|
146
|
+
validateFrontmatter({ ...defaultFrontmatter(), args: ["owner", "owner"] }),
|
|
147
|
+
"Invalid args: names must be unique",
|
|
148
|
+
);
|
|
149
|
+
assert.equal(
|
|
150
|
+
validateFrontmatter({ ...defaultFrontmatter(), args: ["build now"] }),
|
|
151
|
+
"Invalid arg name: build now must match ^\\w[\\w-]*$",
|
|
152
|
+
);
|
|
98
153
|
assert.equal(
|
|
99
154
|
validateFrontmatter({ ...defaultFrontmatter(), maxIterations: 0 }),
|
|
100
|
-
"Invalid max_iterations: must be
|
|
155
|
+
"Invalid max_iterations: must be between 1 and 50",
|
|
156
|
+
);
|
|
157
|
+
assert.equal(
|
|
158
|
+
validateFrontmatter({ ...defaultFrontmatter(), maxIterations: 51 }),
|
|
159
|
+
"Invalid max_iterations: must be between 1 and 50",
|
|
160
|
+
);
|
|
161
|
+
assert.equal(
|
|
162
|
+
validateFrontmatter({ ...defaultFrontmatter(), interIterationDelay: 3 }),
|
|
163
|
+
null,
|
|
164
|
+
);
|
|
165
|
+
assert.equal(
|
|
166
|
+
validateFrontmatter({ ...defaultFrontmatter(), interIterationDelay: -1 }),
|
|
167
|
+
"Invalid inter_iteration_delay: must be a non-negative integer",
|
|
168
|
+
);
|
|
169
|
+
assert.equal(
|
|
170
|
+
validateFrontmatter({ ...defaultFrontmatter(), interIterationDelay: 1.5 }),
|
|
171
|
+
"Invalid inter_iteration_delay: must be a non-negative integer",
|
|
101
172
|
);
|
|
102
173
|
assert.equal(
|
|
103
174
|
validateFrontmatter({ ...defaultFrontmatter(), timeout: 0 }),
|
|
104
|
-
"Invalid timeout: must be
|
|
175
|
+
"Invalid timeout: must be greater than 0 and at most 300",
|
|
176
|
+
);
|
|
177
|
+
assert.equal(
|
|
178
|
+
validateFrontmatter({ ...defaultFrontmatter(), timeout: 301 }),
|
|
179
|
+
"Invalid timeout: must be greater than 0 and at most 300",
|
|
105
180
|
);
|
|
106
181
|
assert.equal(
|
|
107
182
|
validateFrontmatter({ ...defaultFrontmatter(), guardrails: { blockCommands: ["["], protectedFiles: [] } }),
|
|
108
183
|
"Invalid block_commands regex: [",
|
|
109
184
|
);
|
|
185
|
+
assert.equal(
|
|
186
|
+
validateFrontmatter({ ...defaultFrontmatter(), guardrails: { blockCommands: [], protectedFiles: ["**/*"] } }),
|
|
187
|
+
"Invalid protected_files glob: **/*",
|
|
188
|
+
);
|
|
110
189
|
assert.equal(
|
|
111
190
|
validateFrontmatter({ ...defaultFrontmatter(), commands: [{ name: "", run: "echo ok", timeout: 1 }] }),
|
|
112
191
|
"Invalid command: name is required",
|
|
113
192
|
);
|
|
193
|
+
assert.equal(
|
|
194
|
+
validateFrontmatter({ ...defaultFrontmatter(), commands: [{ name: "build now", run: "echo ok", timeout: 1 }] }),
|
|
195
|
+
"Invalid command name: build now must match ^\\w[\\w-]*$",
|
|
196
|
+
);
|
|
114
197
|
assert.equal(
|
|
115
198
|
validateFrontmatter({ ...defaultFrontmatter(), commands: [{ name: "build", run: "", timeout: 1 }] }),
|
|
116
199
|
"Invalid command build: run is required",
|
|
117
200
|
);
|
|
118
201
|
assert.equal(
|
|
119
202
|
validateFrontmatter({ ...defaultFrontmatter(), commands: [{ name: "build", run: "echo ok", timeout: 0 }] }),
|
|
120
|
-
"Invalid command build: timeout must be
|
|
203
|
+
"Invalid command build: timeout must be greater than 0 and at most 300",
|
|
204
|
+
);
|
|
205
|
+
assert.equal(
|
|
206
|
+
validateFrontmatter({ ...defaultFrontmatter(), commands: [{ name: "build", run: "echo ok", timeout: 301 }] }),
|
|
207
|
+
"Invalid command build: timeout must be greater than 0 and at most 300",
|
|
208
|
+
);
|
|
209
|
+
assert.equal(
|
|
210
|
+
validateFrontmatter({ ...defaultFrontmatter(), timeout: 20, commands: [{ name: "build", run: "echo ok", timeout: 21 }] }),
|
|
211
|
+
"Invalid command build: timeout must not exceed top-level timeout",
|
|
121
212
|
);
|
|
122
213
|
assert.equal(
|
|
123
214
|
validateFrontmatter(parseRalphMarkdown("---\ncommands:\n - nope\n - null\n---\nbody").frontmatter),
|
|
124
215
|
"Invalid command entry at index 0",
|
|
125
216
|
);
|
|
217
|
+
assert.equal(
|
|
218
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: [""] }),
|
|
219
|
+
"Invalid required_outputs entry: must be a relative file path",
|
|
220
|
+
);
|
|
221
|
+
assert.equal(
|
|
222
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["/abs.md"] }),
|
|
223
|
+
"Invalid required_outputs entry: /abs.md must be a relative file path",
|
|
224
|
+
);
|
|
225
|
+
assert.equal(
|
|
226
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["../oops.md"] }),
|
|
227
|
+
"Invalid required_outputs entry: ../oops.md must be a relative file path",
|
|
228
|
+
);
|
|
229
|
+
assert.equal(
|
|
230
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["docs/"] }),
|
|
231
|
+
"Invalid required_outputs entry: docs/ must be a relative file path",
|
|
232
|
+
);
|
|
233
|
+
assert.equal(
|
|
234
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["./file.md"] }),
|
|
235
|
+
"Invalid required_outputs entry: ./file.md must be a relative file path",
|
|
236
|
+
);
|
|
237
|
+
assert.equal(
|
|
238
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["docs/./guide.md"] }),
|
|
239
|
+
"Invalid required_outputs entry: docs/./guide.md must be a relative file path",
|
|
240
|
+
);
|
|
241
|
+
assert.equal(
|
|
242
|
+
validateFrontmatter({ ...defaultFrontmatter(), requiredOutputs: ["docs/\nREADME.md"] }),
|
|
243
|
+
"Invalid required_outputs entry: docs/\nREADME.md must be a relative file path",
|
|
244
|
+
);
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
test("validateFrontmatter rejects unsafe completion_promise values and Mission Brief fails closed", () => {
|
|
248
|
+
assert.equal(
|
|
249
|
+
validateFrontmatter({ ...defaultFrontmatter(), completionPromise: "ready\nnow" }),
|
|
250
|
+
"Invalid completion_promise: must be a single-line string without line breaks or angle brackets",
|
|
251
|
+
);
|
|
252
|
+
assert.equal(
|
|
253
|
+
validateFrontmatter({ ...defaultFrontmatter(), completionPromise: "<promise>ready</promise>" }),
|
|
254
|
+
"Invalid completion_promise: must be a single-line string without line breaks or angle brackets",
|
|
255
|
+
);
|
|
256
|
+
|
|
257
|
+
const plan = generateDraft(
|
|
258
|
+
"Fix flaky auth tests",
|
|
259
|
+
{ slug: "fix-flaky-auth-tests", dirPath: "/repo/fix-flaky-auth-tests", ralphPath: "/repo/fix-flaky-auth-tests/RALPH.md" },
|
|
260
|
+
{ packageManager: "npm", testCommand: "npm test", lintCommand: "npm run lint", hasGit: true, topLevelDirs: ["src"], topLevelFiles: ["package.json"] },
|
|
261
|
+
);
|
|
262
|
+
const brief = buildMissionBrief({
|
|
263
|
+
...plan,
|
|
264
|
+
content: plan.content.replace(
|
|
265
|
+
"timeout: 300\n",
|
|
266
|
+
"timeout: 45\ncompletion_promise: |\n ready\n now\n",
|
|
267
|
+
),
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
assert.match(brief, /^Mission Brief/m);
|
|
271
|
+
assert.match(brief, /Invalid RALPH\.md: Invalid completion_promise: must be a single-line string without line breaks or angle brackets/);
|
|
272
|
+
assert.match(brief, /^Draft status$/m);
|
|
273
|
+
assert.doesNotMatch(brief, /Finish behavior/);
|
|
274
|
+
assert.doesNotMatch(brief, /<promise>/);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test("inspectDraftContent, validateDraftContent, and Mission Brief fail closed on raw invalid completion_promise values", () => {
|
|
278
|
+
const plan = generateDraft(
|
|
279
|
+
"Fix flaky auth tests",
|
|
280
|
+
{ slug: "fix-flaky-auth-tests", dirPath: "/repo/fix-flaky-auth-tests", ralphPath: "/repo/fix-flaky-auth-tests/RALPH.md" },
|
|
281
|
+
{ packageManager: "npm", testCommand: "npm test", lintCommand: "npm run lint", hasGit: true, topLevelDirs: ["src"], topLevelFiles: ["package.json"] },
|
|
282
|
+
);
|
|
283
|
+
|
|
284
|
+
for (const [label, rawValue] of [
|
|
285
|
+
["array", "completion_promise: [oops]"],
|
|
286
|
+
["number", "completion_promise: 7"],
|
|
287
|
+
["blank string", 'completion_promise: ""'],
|
|
288
|
+
] as const) {
|
|
289
|
+
const raw = plan.content.replace("timeout: 300\n", `${rawValue}\ntimeout: 300\n`);
|
|
290
|
+
|
|
291
|
+
assert.equal(
|
|
292
|
+
inspectDraftContent(raw).error,
|
|
293
|
+
"Invalid completion_promise: must be a single-line string without line breaks or angle brackets",
|
|
294
|
+
label,
|
|
295
|
+
);
|
|
296
|
+
assert.equal(
|
|
297
|
+
validateDraftContent(raw),
|
|
298
|
+
"Invalid completion_promise: must be a single-line string without line breaks or angle brackets",
|
|
299
|
+
label,
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
const brief = buildMissionBrief({ ...plan, content: raw });
|
|
303
|
+
assert.match(brief, /^Mission Brief/m, label);
|
|
304
|
+
assert.match(brief, /Invalid RALPH\.md: Invalid completion_promise: must be a single-line string without line breaks or angle brackets/, label);
|
|
305
|
+
assert.doesNotMatch(brief, /Finish behavior/, label);
|
|
306
|
+
assert.doesNotMatch(brief, /<promise>/, label);
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
test("acceptStrengthenedDraft rejects required_outputs changes", () => {
|
|
311
|
+
const request = makeFixRequest();
|
|
312
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
313
|
+
[
|
|
314
|
+
"commands:",
|
|
315
|
+
" - name: tests",
|
|
316
|
+
" run: npm test",
|
|
317
|
+
" timeout: 20",
|
|
318
|
+
"max_iterations: 20",
|
|
319
|
+
"timeout: 120",
|
|
320
|
+
"required_outputs:",
|
|
321
|
+
" - ARCHITECTURE.md",
|
|
322
|
+
"guardrails:",
|
|
323
|
+
" block_commands:",
|
|
324
|
+
" - 'git\\s+push'",
|
|
325
|
+
" protected_files:",
|
|
326
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
327
|
+
],
|
|
328
|
+
"Task: Fix flaky auth tests\n\nKeep the change small.",
|
|
329
|
+
);
|
|
330
|
+
|
|
331
|
+
assert.equal(acceptStrengthenedDraft(request, strengthenedDraft), null);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
test("inspectDraftContent, validateDraftContent, and Mission Brief fail closed on raw malformed guardrails values", () => {
|
|
335
|
+
const plan = generateDraft(
|
|
336
|
+
"Fix flaky auth tests",
|
|
337
|
+
{ slug: "fix-flaky-auth-tests", dirPath: "/repo/fix-flaky-auth-tests", ralphPath: "/repo/fix-flaky-auth-tests/RALPH.md" },
|
|
338
|
+
{ packageManager: "npm", testCommand: "npm test", lintCommand: "npm run lint", hasGit: true, topLevelDirs: ["src"], topLevelFiles: ["package.json"] },
|
|
339
|
+
);
|
|
340
|
+
|
|
341
|
+
for (const { label, frontmatterLines, expectedError, briefError } of [
|
|
342
|
+
{
|
|
343
|
+
label: "block_commands scalar",
|
|
344
|
+
frontmatterLines: [
|
|
345
|
+
"commands: []",
|
|
346
|
+
"max_iterations: 25",
|
|
347
|
+
"timeout: 300",
|
|
348
|
+
"guardrails:",
|
|
349
|
+
" block_commands: 'git\\s+push'",
|
|
350
|
+
" protected_files: []",
|
|
351
|
+
],
|
|
352
|
+
expectedError: "Invalid RALPH frontmatter: guardrails.block_commands must be a YAML sequence",
|
|
353
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: guardrails\.block_commands must be a YAML sequence/,
|
|
354
|
+
},
|
|
355
|
+
{
|
|
356
|
+
label: "block_commands null",
|
|
357
|
+
frontmatterLines: [
|
|
358
|
+
"commands: []",
|
|
359
|
+
"max_iterations: 25",
|
|
360
|
+
"timeout: 300",
|
|
361
|
+
"guardrails:",
|
|
362
|
+
" block_commands: null",
|
|
363
|
+
" protected_files: []",
|
|
364
|
+
],
|
|
365
|
+
expectedError: "Invalid RALPH frontmatter: guardrails.block_commands must be a YAML sequence",
|
|
366
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: guardrails\.block_commands must be a YAML sequence/,
|
|
367
|
+
},
|
|
368
|
+
{
|
|
369
|
+
label: "protected_files scalar",
|
|
370
|
+
frontmatterLines: [
|
|
371
|
+
"commands: []",
|
|
372
|
+
"max_iterations: 25",
|
|
373
|
+
"timeout: 300",
|
|
374
|
+
"guardrails:",
|
|
375
|
+
" block_commands: []",
|
|
376
|
+
" protected_files: 'src/generated/**'",
|
|
377
|
+
],
|
|
378
|
+
expectedError: "Invalid RALPH frontmatter: guardrails.protected_files must be a YAML sequence",
|
|
379
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: guardrails\.protected_files must be a YAML sequence/,
|
|
380
|
+
},
|
|
381
|
+
{
|
|
382
|
+
label: "protected_files null",
|
|
383
|
+
frontmatterLines: [
|
|
384
|
+
"commands: []",
|
|
385
|
+
"max_iterations: 25",
|
|
386
|
+
"timeout: 300",
|
|
387
|
+
"guardrails:",
|
|
388
|
+
" block_commands: []",
|
|
389
|
+
" protected_files: null",
|
|
390
|
+
],
|
|
391
|
+
expectedError: "Invalid RALPH frontmatter: guardrails.protected_files must be a YAML sequence",
|
|
392
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: guardrails\.protected_files must be a YAML sequence/,
|
|
393
|
+
},
|
|
394
|
+
] as const) {
|
|
395
|
+
const raw = makeStrengthenedDraft(frontmatterLines, "Task: Fix flaky auth tests\n\nKeep the change small.");
|
|
396
|
+
const inspection = inspectDraftContent(raw);
|
|
397
|
+
|
|
398
|
+
assert.equal(inspection.error, expectedError, label);
|
|
399
|
+
assert.equal(validateDraftContent(raw), expectedError, label);
|
|
400
|
+
|
|
401
|
+
const brief = buildMissionBrief({ ...plan, content: raw });
|
|
402
|
+
assert.match(brief, /^Mission Brief/m, label);
|
|
403
|
+
assert.match(brief, briefError, label);
|
|
404
|
+
assert.doesNotMatch(brief, /Finish behavior/, label);
|
|
405
|
+
assert.doesNotMatch(brief, /<promise>/, label);
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test("acceptStrengthenedDraft rejects raw malformed guardrails shapes", () => {
|
|
410
|
+
const request = makeFixRequest();
|
|
411
|
+
|
|
412
|
+
for (const { label, frontmatterLines } of [
|
|
413
|
+
{
|
|
414
|
+
label: "block_commands scalar",
|
|
415
|
+
frontmatterLines: [
|
|
416
|
+
"commands:",
|
|
417
|
+
" - name: tests",
|
|
418
|
+
" run: npm test",
|
|
419
|
+
" timeout: 20",
|
|
420
|
+
"max_iterations: 20",
|
|
421
|
+
"timeout: 120",
|
|
422
|
+
"guardrails:",
|
|
423
|
+
" block_commands: 'git\\s+push'",
|
|
424
|
+
" protected_files:",
|
|
425
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
426
|
+
],
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
label: "protected_files scalar",
|
|
430
|
+
frontmatterLines: [
|
|
431
|
+
"commands:",
|
|
432
|
+
" - name: tests",
|
|
433
|
+
" run: npm test",
|
|
434
|
+
" timeout: 20",
|
|
435
|
+
"max_iterations: 20",
|
|
436
|
+
"timeout: 120",
|
|
437
|
+
"guardrails:",
|
|
438
|
+
" block_commands:",
|
|
439
|
+
" - 'git\\s+push'",
|
|
440
|
+
" protected_files: 'src/generated/**'",
|
|
441
|
+
],
|
|
442
|
+
},
|
|
443
|
+
] as const) {
|
|
444
|
+
const strengthenedDraft = makeStrengthenedDraft(frontmatterLines, "Task: Fix flaky auth tests\n\nKeep the change small.");
|
|
445
|
+
|
|
446
|
+
assert.equal(acceptStrengthenedDraft(request, strengthenedDraft), null, label);
|
|
447
|
+
}
|
|
126
448
|
});
|
|
127
449
|
|
|
128
450
|
test("runCommands skips blocked commands before shelling out", async () => {
|
|
129
451
|
const calls: string[] = [];
|
|
452
|
+
const proofEntries: Array<{ customType: string; data: any }> = [];
|
|
130
453
|
const pi = {
|
|
454
|
+
appendEntry: (customType: string, data: any) => {
|
|
455
|
+
proofEntries.push({ customType, data });
|
|
456
|
+
},
|
|
131
457
|
exec: async (_tool: string, args: string[]) => {
|
|
132
458
|
calls.push(args.join(" "));
|
|
133
459
|
return { killed: false, stdout: "allowed", stderr: "" };
|
|
@@ -148,6 +474,46 @@ test("runCommands skips blocked commands before shelling out", async () => {
|
|
|
148
474
|
{ name: "allowed", output: "allowed" },
|
|
149
475
|
]);
|
|
150
476
|
assert.deepEqual(calls, ["-c echo ok"]);
|
|
477
|
+
assert.equal(proofEntries.length, 1);
|
|
478
|
+
assert.equal(proofEntries[0].customType, "ralph-blocked-command");
|
|
479
|
+
assert.equal(proofEntries[0].data.command, "git push origin main");
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
test("runCommands resolves args before shelling out", async () => {
|
|
483
|
+
const calls: string[] = [];
|
|
484
|
+
const pi = {
|
|
485
|
+
exec: async (_tool: string, args: string[]) => {
|
|
486
|
+
calls.push(args.join(" "));
|
|
487
|
+
return { killed: false, stdout: "ok", stderr: "" };
|
|
488
|
+
},
|
|
489
|
+
} as any;
|
|
490
|
+
|
|
491
|
+
const outputs = await runCommands(
|
|
492
|
+
[{ name: "greet", run: "echo {{ args.owner }}", timeout: 1 }],
|
|
493
|
+
[],
|
|
494
|
+
pi,
|
|
495
|
+
{ owner: "Ada" },
|
|
496
|
+
);
|
|
497
|
+
|
|
498
|
+
assert.deepEqual(outputs, [{ name: "greet", output: "ok" }]);
|
|
499
|
+
assert.deepEqual(calls, ["-c echo 'Ada'"]);
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
test("runCommands blocks anchored guardrails even when the first token comes from an arg", async () => {
|
|
503
|
+
const pi = {
|
|
504
|
+
exec: async () => {
|
|
505
|
+
throw new Error("should not execute blocked command");
|
|
506
|
+
},
|
|
507
|
+
} as any;
|
|
508
|
+
|
|
509
|
+
const outputs = await runCommands(
|
|
510
|
+
[{ name: "blocked", run: "{{ args.tool }} hello", timeout: 1 }],
|
|
511
|
+
["^printf\\b"],
|
|
512
|
+
pi,
|
|
513
|
+
{ tool: "printf" },
|
|
514
|
+
);
|
|
515
|
+
|
|
516
|
+
assert.deepEqual(outputs, [{ name: "blocked", output: "[blocked by guardrail: ^printf\\b]" }]);
|
|
151
517
|
});
|
|
152
518
|
|
|
153
519
|
test("legacy RALPH.md drafts bypass the generated-draft validation gate", () => {
|
|
@@ -161,25 +527,170 @@ test("legacy RALPH.md drafts bypass the generated-draft validation gate", () =>
|
|
|
161
527
|
assert.equal(shouldValidateExistingDraft(draft.content), true);
|
|
162
528
|
});
|
|
163
529
|
|
|
164
|
-
test("
|
|
530
|
+
test("inspectDraftContent rejects malformed args frontmatter shapes", () => {
|
|
531
|
+
const invalid = inspectDraftContent(["---", "args: owner", "commands: []", "max_iterations: 1", "timeout: 1", "guardrails:", " block_commands: []", " protected_files: []", "---", "Body"].join("\n"));
|
|
532
|
+
|
|
533
|
+
assert.equal(invalid.error, "Invalid RALPH frontmatter: args must be a YAML sequence");
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
test("inspectDraftContent rejects malformed inter_iteration_delay frontmatter shapes", () => {
|
|
537
|
+
const invalid = inspectDraftContent(["---", "commands: []", "max_iterations: 1", "inter_iteration_delay: true", "timeout: 1", "guardrails:", " block_commands: []", " protected_files: []", "---", "Body"].join("\n"));
|
|
538
|
+
|
|
539
|
+
assert.equal(invalid.error, "Invalid RALPH frontmatter: inter_iteration_delay must be a YAML number");
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
test("render helpers expand placeholders, keep body text plain, and shell-quote command args", () => {
|
|
165
543
|
const outputs = [{ name: "build", output: "done" }];
|
|
166
544
|
|
|
167
545
|
assert.equal(
|
|
168
|
-
resolvePlaceholders("{{ commands.build }} {{ ralph.iteration }} {{ ralph.name }} {{ commands.missing }}", outputs, {
|
|
546
|
+
resolvePlaceholders("{{ commands.build }} {{ ralph.iteration }} {{ ralph.name }} {{ ralph.max_iterations }} {{ args.owner }} {{ commands.missing }}", outputs, {
|
|
169
547
|
iteration: 7,
|
|
170
548
|
name: "ralph",
|
|
171
|
-
|
|
172
|
-
|
|
549
|
+
maxIterations: 12,
|
|
550
|
+
}, { owner: "Ada" }),
|
|
551
|
+
"done 7 ralph 12 Ada ",
|
|
173
552
|
);
|
|
174
|
-
assert.equal(
|
|
553
|
+
assert.equal(
|
|
554
|
+
renderRalphBody("keep<!-- hidden -->{{ args.owner }}{{ ralph.name }}", [], { iteration: 1, name: "ralph", maxIterations: 1 }, { owner: "Ada; echo injected" }),
|
|
555
|
+
"keepAda; echo injectedralph",
|
|
556
|
+
);
|
|
557
|
+
assert.equal(resolveCommandRun("npm run {{ args.script }}", { script: "test" }), "npm run 'test'");
|
|
558
|
+
assert.equal(resolveCommandRun("echo {{ args.owner }}", { owner: "Ada; echo injected" }), "echo 'Ada; echo injected'");
|
|
559
|
+
assert.throws(() => resolveCommandRun("npm run {{ args.missing }}", { script: "test" }), /Missing required arg: missing/);
|
|
175
560
|
assert.equal(renderIterationPrompt("Body", 2, 5), "[ralph: iteration 2/5]\n\nBody");
|
|
176
561
|
});
|
|
177
562
|
|
|
178
|
-
test("
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
assert.
|
|
182
|
-
|
|
563
|
+
test("resolvePlaceholders leaves command output placeholders literal", () => {
|
|
564
|
+
const outputs = [{ name: "build", output: "echo {{ args.owner }}" }];
|
|
565
|
+
|
|
566
|
+
assert.equal(
|
|
567
|
+
resolvePlaceholders("{{ commands.build }} {{ args.owner }}", outputs, {
|
|
568
|
+
iteration: 7,
|
|
569
|
+
name: "ralph",
|
|
570
|
+
maxIterations: 12,
|
|
571
|
+
}, { owner: "Ada" }),
|
|
572
|
+
"echo {{ args.owner }} Ada",
|
|
573
|
+
);
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
test("renderIterationPrompt includes completion-gate reminders and previous failure reasons", () => {
|
|
577
|
+
const prompt = renderIterationPrompt("Body", 2, 5, {
|
|
578
|
+
completionPromise: "DONE",
|
|
579
|
+
requiredOutputs: ["ARCHITECTURE.md", "OPEN_QUESTIONS.md"],
|
|
580
|
+
failureReasons: ["Missing required output: ARCHITECTURE.md", "OPEN_QUESTIONS.md still has P0 items"],
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
assert.match(prompt, /Required outputs must exist before stopping: ARCHITECTURE\.md, OPEN_QUESTIONS\.md/);
|
|
584
|
+
assert.match(prompt, /OPEN_QUESTIONS\.md must have no remaining P0\/P1 items before stopping\./);
|
|
585
|
+
assert.match(prompt, /Label inferred claims as HYPOTHESIS\./);
|
|
586
|
+
assert.match(prompt, /Previous gate failures: Missing required output: ARCHITECTURE\.md; OPEN_QUESTIONS\.md still has P0 items/);
|
|
587
|
+
assert.match(prompt, /Emit <promise>DONE<\/promise> only when the gate is truly satisfied\./);
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
test("renderIterationPrompt includes a rejection section when durable progress is still missing", () => {
|
|
591
|
+
const prompt = renderIterationPrompt("Body", 2, 5, {
|
|
592
|
+
completionPromise: "DONE",
|
|
593
|
+
requiredOutputs: ["ARCHITECTURE.md"],
|
|
594
|
+
rejectionReasons: ["durable progress"],
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
assert.match(prompt, /\[completion gate rejection\]/);
|
|
598
|
+
assert.match(prompt, /Still missing: durable progress/);
|
|
599
|
+
assert.match(prompt, /Emit <promise>DONE<\/promise> only when the gate is truly satisfied\./);
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
test("parseCommandArgs handles explicit path args, leaves task text alone, and rejects task args", () => {
|
|
603
|
+
assert.deepEqual(parseCommandArgs("--path my-task"), { mode: "path", value: "my-task", runtimeArgs: [], error: undefined });
|
|
604
|
+
assert.deepEqual(parseCommandArgs("--path my-task --arg owner=Ada --arg mode=fix"), {
|
|
605
|
+
mode: "path",
|
|
606
|
+
value: "my-task",
|
|
607
|
+
runtimeArgs: [
|
|
608
|
+
{ name: "owner", value: "Ada" },
|
|
609
|
+
{ name: "mode", value: "fix" },
|
|
610
|
+
],
|
|
611
|
+
error: undefined,
|
|
612
|
+
});
|
|
613
|
+
assert.deepEqual(parseCommandArgs(" reverse engineer this app "), { mode: "auto", value: "reverse engineer this app", runtimeArgs: [], error: undefined });
|
|
614
|
+
assert.deepEqual(parseCommandArgs("reverse engineer --arg name=value literally"), {
|
|
615
|
+
mode: "auto",
|
|
616
|
+
value: "reverse engineer --arg name=value literally",
|
|
617
|
+
runtimeArgs: [],
|
|
618
|
+
error: undefined,
|
|
619
|
+
});
|
|
620
|
+
assert.deepEqual(parseCommandArgs("--path my --argument task"), {
|
|
621
|
+
mode: "path",
|
|
622
|
+
value: "my --argument task",
|
|
623
|
+
runtimeArgs: [],
|
|
624
|
+
error: undefined,
|
|
625
|
+
});
|
|
626
|
+
assert.equal(parseCommandArgs("--task reverse engineer auth --arg owner=Ada").error, "--arg is only supported with /ralph --path");
|
|
627
|
+
});
|
|
628
|
+
|
|
629
|
+
test("parseCommandArgs parses quoted explicit-path args and preserves literal equals", () => {
|
|
630
|
+
assert.deepEqual(parseCommandArgs('--path my-task --arg owner="Ada Lovelace"'), {
|
|
631
|
+
mode: "path",
|
|
632
|
+
value: "my-task",
|
|
633
|
+
runtimeArgs: [{ name: "owner", value: "Ada Lovelace" }],
|
|
634
|
+
error: undefined,
|
|
635
|
+
});
|
|
636
|
+
assert.deepEqual(parseCommandArgs("--path my-task --arg team='core infra' --arg note='a=b=c'"), {
|
|
637
|
+
mode: "path",
|
|
638
|
+
value: "my-task",
|
|
639
|
+
runtimeArgs: [
|
|
640
|
+
{ name: "team", value: "core infra" },
|
|
641
|
+
{ name: "note", value: "a=b=c" },
|
|
642
|
+
],
|
|
643
|
+
error: undefined,
|
|
644
|
+
});
|
|
645
|
+
assert.equal(parseCommandArgs('--path my-task --arg owner="Ada Lovelace" --arg owner=\'Ada Smith\'').error, "Duplicate --arg: owner");
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
test("parseCommandArgs rejects malformed explicit-path args", () => {
|
|
649
|
+
assert.equal(parseCommandArgs("--path my-task --arg owner=").error, "Invalid --arg entry: value is required");
|
|
650
|
+
assert.equal(parseCommandArgs("--path my-task --arg =Ada").error, "Invalid --arg entry: name is required");
|
|
651
|
+
assert.equal(
|
|
652
|
+
parseCommandArgs("--path my-task --arg owner=Ada Lovelace").error,
|
|
653
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
654
|
+
);
|
|
655
|
+
assert.equal(
|
|
656
|
+
parseCommandArgs("--path my-task --arg owner=Ada extra text").error,
|
|
657
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
658
|
+
);
|
|
659
|
+
assert.equal(
|
|
660
|
+
parseCommandArgs("--path my-task --arg=name=value").error,
|
|
661
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
662
|
+
);
|
|
663
|
+
assert.equal(
|
|
664
|
+
parseCommandArgs("--path my-task --argowner=Ada").error,
|
|
665
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
666
|
+
);
|
|
667
|
+
assert.equal(
|
|
668
|
+
parseCommandArgs('--path my-task --arg owner="Ada"extra').error,
|
|
669
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
670
|
+
);
|
|
671
|
+
assert.equal(
|
|
672
|
+
parseCommandArgs('--path my-task --arg owner="Ada"--arg team=core').error,
|
|
673
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
674
|
+
);
|
|
675
|
+
assert.equal(
|
|
676
|
+
parseCommandArgs('--path my-task --arg owner=pre"Ada Lovelace"post').error,
|
|
677
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
678
|
+
);
|
|
679
|
+
assert.equal(
|
|
680
|
+
parseCommandArgs("--path my-task --arg owner='Ada'--arg team=core").error,
|
|
681
|
+
"Invalid --arg syntax: values must be a single token and no trailing text is allowed",
|
|
682
|
+
);
|
|
683
|
+
});
|
|
684
|
+
|
|
685
|
+
test("runtimeArgEntriesToMap preserves special arg names and command substitution can read them", () => {
|
|
686
|
+
const parsed = parseCommandArgs("--path my-task --arg __proto__=safe");
|
|
687
|
+
assert.equal(parsed.error, undefined);
|
|
688
|
+
|
|
689
|
+
const mapped = runtimeArgEntriesToMap(parsed.runtimeArgs);
|
|
690
|
+
assert.equal(mapped.error, undefined);
|
|
691
|
+
assert.equal(Object.getPrototypeOf(mapped.runtimeArgs), null);
|
|
692
|
+
assert.deepEqual(Object.keys(mapped.runtimeArgs), ["__proto__"]);
|
|
693
|
+
assert.equal(resolveCommandRun("echo {{ args.__proto__ }}", mapped.runtimeArgs), "echo 'safe'");
|
|
183
694
|
});
|
|
184
695
|
|
|
185
696
|
test("explicit path mode stays path-centric and does not offer task fallback", async () => {
|
|
@@ -262,8 +773,140 @@ test("validateDraftContent rejects missing and malformed frontmatter", () => {
|
|
|
262
773
|
assert.equal(validateDraftContent("Task body"), "Missing RALPH frontmatter");
|
|
263
774
|
assert.equal(
|
|
264
775
|
validateDraftContent("---\nmax_iterations: 0\n---\nBody"),
|
|
265
|
-
"Invalid max_iterations: must be
|
|
776
|
+
"Invalid max_iterations: must be between 1 and 50",
|
|
777
|
+
);
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
test("validateDraftContent fails closed on YAML frontmatter that is not a mapping", () => {
|
|
781
|
+
assert.equal(validateDraftContent("---\n- nope\n---\nBody"), "Invalid RALPH frontmatter: Frontmatter must be a YAML mapping");
|
|
782
|
+
});
|
|
783
|
+
|
|
784
|
+
test("inspectDraftContent and validateDraftContent fail closed on raw malformed frontmatter shapes and scalars", () => {
|
|
785
|
+
const makeRawDraft = (frontmatterLines: readonly string[]) => `---\n${frontmatterLines.join("\n")}\n---\nTask: Fix flaky auth tests\n\nKeep the change small.`;
|
|
786
|
+
|
|
787
|
+
for (const { label, raw, expectedError } of [
|
|
788
|
+
{
|
|
789
|
+
label: "commands mapping",
|
|
790
|
+
raw: makeRawDraft([
|
|
791
|
+
"commands:",
|
|
792
|
+
" name: tests",
|
|
793
|
+
" run: npm test",
|
|
794
|
+
" timeout: 20",
|
|
795
|
+
"max_iterations: 2",
|
|
796
|
+
"timeout: 300",
|
|
797
|
+
]),
|
|
798
|
+
expectedError: "Invalid RALPH frontmatter: commands must be a YAML sequence",
|
|
799
|
+
},
|
|
800
|
+
{
|
|
801
|
+
label: "max_iterations boolean",
|
|
802
|
+
raw: makeRawDraft(["commands: []", "max_iterations: true", "timeout: 300"]),
|
|
803
|
+
expectedError: "Invalid RALPH frontmatter: max_iterations must be a YAML number",
|
|
804
|
+
},
|
|
805
|
+
{
|
|
806
|
+
label: "timeout boolean",
|
|
807
|
+
raw: makeRawDraft(["commands: []", "max_iterations: 2", "timeout: true"]),
|
|
808
|
+
expectedError: "Invalid RALPH frontmatter: timeout must be a YAML number",
|
|
809
|
+
},
|
|
810
|
+
{
|
|
811
|
+
label: "command name array",
|
|
812
|
+
raw: makeRawDraft([
|
|
813
|
+
"commands:",
|
|
814
|
+
" - name:",
|
|
815
|
+
" - build",
|
|
816
|
+
" run: npm test",
|
|
817
|
+
" timeout: 20",
|
|
818
|
+
"max_iterations: 2",
|
|
819
|
+
"timeout: 300",
|
|
820
|
+
]),
|
|
821
|
+
expectedError: "Invalid RALPH frontmatter: commands[0].name must be a YAML string",
|
|
822
|
+
},
|
|
823
|
+
{
|
|
824
|
+
label: "command run array",
|
|
825
|
+
raw: makeRawDraft([
|
|
826
|
+
"commands:",
|
|
827
|
+
" - name: build",
|
|
828
|
+
" run:",
|
|
829
|
+
" - npm test",
|
|
830
|
+
" timeout: 20",
|
|
831
|
+
"max_iterations: 2",
|
|
832
|
+
"timeout: 300",
|
|
833
|
+
]),
|
|
834
|
+
expectedError: "Invalid RALPH frontmatter: commands[0].run must be a YAML string",
|
|
835
|
+
},
|
|
836
|
+
{
|
|
837
|
+
label: "command timeout array",
|
|
838
|
+
raw: makeRawDraft([
|
|
839
|
+
"commands:",
|
|
840
|
+
" - name: build",
|
|
841
|
+
" run: npm test",
|
|
842
|
+
" timeout:",
|
|
843
|
+
" - 20",
|
|
844
|
+
"max_iterations: 2",
|
|
845
|
+
"timeout: 300",
|
|
846
|
+
]),
|
|
847
|
+
expectedError: "Invalid RALPH frontmatter: commands[0].timeout must be a YAML number",
|
|
848
|
+
},
|
|
849
|
+
] as const) {
|
|
850
|
+
assert.equal(inspectDraftContent(raw).error, expectedError, label);
|
|
851
|
+
assert.equal(validateDraftContent(raw), expectedError, label);
|
|
852
|
+
}
|
|
853
|
+
});
|
|
854
|
+
|
|
855
|
+
test("inspectDraftContent, validateDraftContent, and Mission Brief fail closed on raw malformed required_outputs values", () => {
|
|
856
|
+
const plan = generateDraft(
|
|
857
|
+
"Fix flaky auth tests",
|
|
858
|
+
{ slug: "fix-flaky-auth-tests", dirPath: "/repo/fix-flaky-auth-tests", ralphPath: "/repo/fix-flaky-auth-tests/RALPH.md" },
|
|
859
|
+
{ packageManager: "npm", testCommand: "npm test", lintCommand: "npm run lint", hasGit: true, topLevelDirs: ["src"], topLevelFiles: ["package.json"] },
|
|
266
860
|
);
|
|
861
|
+
|
|
862
|
+
for (const { label, rawValue, expectedError, briefError } of [
|
|
863
|
+
{
|
|
864
|
+
label: "required_outputs scalar",
|
|
865
|
+
rawValue: "required_outputs: docs/ARCHITECTURE.md",
|
|
866
|
+
expectedError: "Invalid RALPH frontmatter: required_outputs must be a YAML sequence",
|
|
867
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: required_outputs must be a YAML sequence/,
|
|
868
|
+
},
|
|
869
|
+
{
|
|
870
|
+
label: "required_outputs entry number",
|
|
871
|
+
rawValue: "required_outputs:\n - 123",
|
|
872
|
+
expectedError: "Invalid RALPH frontmatter: required_outputs[0] must be a YAML string",
|
|
873
|
+
briefError: /Invalid RALPH\.md: Invalid RALPH frontmatter: required_outputs\[0\] must be a YAML string/,
|
|
874
|
+
},
|
|
875
|
+
] as const) {
|
|
876
|
+
const raw = plan.content.replace("timeout: 300\n", `${rawValue}\ntimeout: 300\n`);
|
|
877
|
+
|
|
878
|
+
assert.equal(inspectDraftContent(raw).error, expectedError, label);
|
|
879
|
+
assert.equal(validateDraftContent(raw), expectedError, label);
|
|
880
|
+
|
|
881
|
+
const brief = buildMissionBrief({ ...plan, content: raw });
|
|
882
|
+
assert.match(brief, /^Mission Brief/m, label);
|
|
883
|
+
assert.match(brief, briefError, label);
|
|
884
|
+
assert.doesNotMatch(brief, /Finish behavior/, label);
|
|
885
|
+
assert.doesNotMatch(brief, /<promise>/, label);
|
|
886
|
+
}
|
|
887
|
+
});
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
test("inspectDraftContent and validateDraftContent reject metadata-tagged generated drafts with malformed commands mappings", () => {
|
|
891
|
+
const raw = makeStrengthenedDraft(
|
|
892
|
+
[
|
|
893
|
+
"commands:",
|
|
894
|
+
" name: tests",
|
|
895
|
+
" run: npm test",
|
|
896
|
+
" timeout: 20",
|
|
897
|
+
"max_iterations: 20",
|
|
898
|
+
"timeout: 120",
|
|
899
|
+
"guardrails:",
|
|
900
|
+
" block_commands:",
|
|
901
|
+
" - 'git\\s+push'",
|
|
902
|
+
" protected_files:",
|
|
903
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
904
|
+
],
|
|
905
|
+
"Task: Fix flaky auth tests\n\nKeep the change small.",
|
|
906
|
+
);
|
|
907
|
+
|
|
908
|
+
assert.equal(inspectDraftContent(raw).error, "Invalid RALPH frontmatter: commands must be a YAML sequence");
|
|
909
|
+
assert.equal(validateDraftContent(raw), "Invalid RALPH frontmatter: commands must be a YAML sequence");
|
|
267
910
|
});
|
|
268
911
|
|
|
269
912
|
test("buildMissionBrief fails closed when the current draft content is invalid", () => {
|
|
@@ -363,8 +1006,10 @@ test("generated drafts reparse as valid RALPH files", () => {
|
|
|
363
1006
|
{ name: "repo-map", run: "find . -maxdepth 2 -type f | sort | head -n 120", timeout: 20 },
|
|
364
1007
|
],
|
|
365
1008
|
maxIterations: 12,
|
|
1009
|
+
interIterationDelay: 0,
|
|
366
1010
|
timeout: 300,
|
|
367
1011
|
completionPromise: undefined,
|
|
1012
|
+
requiredOutputs: [],
|
|
368
1013
|
guardrails: { blockCommands: ["git\\s+push"], protectedFiles: [] },
|
|
369
1014
|
invalidCommandEntries: undefined,
|
|
370
1015
|
});
|
|
@@ -445,29 +1090,606 @@ test("normalizeStrengthenedDraft keeps deterministic frontmatter in body-only mo
|
|
|
445
1090
|
});
|
|
446
1091
|
});
|
|
447
1092
|
|
|
1093
|
+
test("normalizeStrengthenedDraft keeps declared args placeholders in body-only mode", () => {
|
|
1094
|
+
const request = makeFixRequestWithArgs(["owner"]);
|
|
1095
|
+
const baseline = parseRalphMarkdown(request.baselineDraft);
|
|
1096
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
1097
|
+
[
|
|
1098
|
+
"args:",
|
|
1099
|
+
" - mode",
|
|
1100
|
+
"commands:",
|
|
1101
|
+
" - name: tests",
|
|
1102
|
+
" run: npm test",
|
|
1103
|
+
" timeout: 20",
|
|
1104
|
+
"max_iterations: 25",
|
|
1105
|
+
"timeout: 300",
|
|
1106
|
+
"guardrails:",
|
|
1107
|
+
" block_commands:",
|
|
1108
|
+
" - 'git\\s+push'",
|
|
1109
|
+
" protected_files:",
|
|
1110
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1111
|
+
],
|
|
1112
|
+
"Task: Fix flaky auth tests\n\nUse {{ args.owner }} to scope the fix.",
|
|
1113
|
+
);
|
|
1114
|
+
|
|
1115
|
+
const normalized = normalizeStrengthenedDraft(request, strengthenedDraft, "body-only");
|
|
1116
|
+
const reparsed = parseRalphMarkdown(normalized.content);
|
|
1117
|
+
|
|
1118
|
+
const { args: _args, ...baselineFrontmatter } = baseline.frontmatter;
|
|
1119
|
+
assert.deepEqual(reparsed.frontmatter, baselineFrontmatter);
|
|
1120
|
+
assert.equal(reparsed.body.trimStart(), "Task: Fix flaky auth tests\n\nUse {{ args.owner }} to scope the fix.");
|
|
1121
|
+
});
|
|
1122
|
+
|
|
1123
|
+
test("normalizeStrengthenedDraft falls back to the baseline body when body-only args placeholders are undeclared", () => {
|
|
1124
|
+
const request = makeFixRequestWithArgs(["owner"]);
|
|
1125
|
+
const baseline = parseRalphMarkdown(request.baselineDraft);
|
|
1126
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
1127
|
+
[
|
|
1128
|
+
"args:",
|
|
1129
|
+
" - mode",
|
|
1130
|
+
"commands:",
|
|
1131
|
+
" - name: tests",
|
|
1132
|
+
" run: npm test",
|
|
1133
|
+
" timeout: 20",
|
|
1134
|
+
"max_iterations: 25",
|
|
1135
|
+
"timeout: 300",
|
|
1136
|
+
"guardrails:",
|
|
1137
|
+
" block_commands:",
|
|
1138
|
+
" - 'git\\s+push'",
|
|
1139
|
+
" protected_files:",
|
|
1140
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1141
|
+
],
|
|
1142
|
+
"Task: Fix flaky auth tests\n\nUse {{ args.mode }} to scope the fix.",
|
|
1143
|
+
);
|
|
1144
|
+
|
|
1145
|
+
const normalized = normalizeStrengthenedDraft(request, strengthenedDraft, "body-only");
|
|
1146
|
+
const reparsed = parseRalphMarkdown(normalized.content);
|
|
1147
|
+
|
|
1148
|
+
const { args: _args, ...baselineFrontmatter } = baseline.frontmatter;
|
|
1149
|
+
assert.deepEqual(reparsed.frontmatter, baselineFrontmatter);
|
|
1150
|
+
assert.equal(reparsed.body.trimStart(), baseline.body.trimStart());
|
|
1151
|
+
});
|
|
1152
|
+
|
|
1153
|
+
test("normalizeStrengthenedDraft falls back to the baseline body when body-only frontmatter is invalid", () => {
|
|
1154
|
+
const request = makeFixRequest();
|
|
1155
|
+
const baseline = parseRalphMarkdown(request.baselineDraft);
|
|
1156
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
1157
|
+
[
|
|
1158
|
+
"commands:",
|
|
1159
|
+
" name: rogue",
|
|
1160
|
+
" run: rm -rf /",
|
|
1161
|
+
" timeout: 1",
|
|
1162
|
+
"max_iterations: 20",
|
|
1163
|
+
"timeout: 120",
|
|
1164
|
+
"guardrails:",
|
|
1165
|
+
" block_commands:",
|
|
1166
|
+
" - 'git\\s+push'",
|
|
1167
|
+
" protected_files:",
|
|
1168
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1169
|
+
],
|
|
1170
|
+
"Task: Fix flaky auth tests\n\nRead-only enforced and write protection is enforced.",
|
|
1171
|
+
);
|
|
1172
|
+
|
|
1173
|
+
const normalized = normalizeStrengthenedDraft(request, strengthenedDraft, "body-only");
|
|
1174
|
+
const reparsed = parseRalphMarkdown(normalized.content);
|
|
1175
|
+
|
|
1176
|
+
assert.deepEqual(reparsed.frontmatter, baseline.frontmatter);
|
|
1177
|
+
assert.equal(reparsed.body.trimStart(), baseline.body.trimStart());
|
|
1178
|
+
});
|
|
1179
|
+
|
|
1180
|
+
test("normalizeStrengthenedDraft falls back to the baseline body when body-only YAML syntax is malformed", () => {
|
|
1181
|
+
const request = makeFixRequest();
|
|
1182
|
+
const baseline = parseRalphMarkdown(request.baselineDraft);
|
|
1183
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
1184
|
+
[
|
|
1185
|
+
"commands: [",
|
|
1186
|
+
"max_iterations: 20",
|
|
1187
|
+
"timeout: 120",
|
|
1188
|
+
"guardrails:",
|
|
1189
|
+
" block_commands:",
|
|
1190
|
+
" - 'git\\s+push'",
|
|
1191
|
+
" protected_files:",
|
|
1192
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1193
|
+
],
|
|
1194
|
+
"Task: Fix flaky auth tests\n\nRead-only enforced and write protection is enforced.",
|
|
1195
|
+
);
|
|
1196
|
+
|
|
1197
|
+
const normalized = normalizeStrengthenedDraft(request, strengthenedDraft, "body-only");
|
|
1198
|
+
const reparsed = parseRalphMarkdown(normalized.content);
|
|
1199
|
+
|
|
1200
|
+
assert.deepEqual(reparsed.frontmatter, baseline.frontmatter);
|
|
1201
|
+
assert.equal(reparsed.body.trimStart(), baseline.body.trimStart());
|
|
1202
|
+
});
|
|
1203
|
+
|
|
448
1204
|
test("normalizeStrengthenedDraft applies strengthened commands in body-and-commands mode", () => {
|
|
449
|
-
const request =
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
1205
|
+
const request = makeFixRequest();
|
|
1206
|
+
const baseline = parseRalphMarkdown(request.baselineDraft);
|
|
1207
|
+
const strengthenedDraft = makeStrengthenedDraft(
|
|
1208
|
+
[
|
|
1209
|
+
"commands:",
|
|
1210
|
+
" - name: git-log",
|
|
1211
|
+
" run: git log --oneline -10",
|
|
1212
|
+
" timeout: 15",
|
|
1213
|
+
" - name: tests",
|
|
1214
|
+
" run: npm test",
|
|
1215
|
+
" timeout: 45",
|
|
1216
|
+
"max_iterations: 20",
|
|
1217
|
+
"timeout: 120",
|
|
1218
|
+
"guardrails:",
|
|
1219
|
+
" block_commands:",
|
|
1220
|
+
" - 'git\\s+push'",
|
|
1221
|
+
" protected_files:",
|
|
1222
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1223
|
+
],
|
|
1224
|
+
"Task: Fix flaky auth tests\n\nUse {{ commands.tests }} and {{ commands.git-log }}.\n\nIteration {{ ralph.iteration }} of {{ ralph.name }}.",
|
|
454
1225
|
);
|
|
455
|
-
|
|
1226
|
+
|
|
1227
|
+
const accepted = acceptStrengthenedDraft(request, strengthenedDraft);
|
|
1228
|
+
if (!accepted) {
|
|
1229
|
+
assert.fail("expected strengthened draft to be accepted");
|
|
1230
|
+
}
|
|
456
1231
|
|
|
457
1232
|
const normalized = normalizeStrengthenedDraft(request, strengthenedDraft, "body-and-commands");
|
|
1233
|
+
assert.equal(normalized.content, accepted.content);
|
|
458
1234
|
const reparsed = parseRalphMarkdown(normalized.content);
|
|
459
1235
|
|
|
460
|
-
assert.deepEqual(reparsed.frontmatter.commands, [
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
assert.
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
1236
|
+
assert.deepEqual(reparsed.frontmatter.commands, [
|
|
1237
|
+
{ name: "git-log", run: "git log --oneline -10", timeout: 15 },
|
|
1238
|
+
{ name: "tests", run: "npm test", timeout: 45 },
|
|
1239
|
+
]);
|
|
1240
|
+
assert.equal(reparsed.frontmatter.maxIterations, 20);
|
|
1241
|
+
assert.equal(reparsed.frontmatter.timeout, 120);
|
|
1242
|
+
assert.deepEqual(reparsed.frontmatter.guardrails, baseline.frontmatter.guardrails);
|
|
1243
|
+
assert.equal(validateFrontmatter(reparsed.frontmatter), null);
|
|
1244
|
+
assert.match(reparsed.body, /Use \{\{ commands\.tests \}\} and \{\{ commands\.git-log \}\}\./);
|
|
1245
|
+
});
|
|
1246
|
+
|
|
1247
|
+
test("acceptStrengthenedDraft rejects malformed commands frontmatter shapes", () => {
|
|
1248
|
+
const request = makeFixRequest();
|
|
1249
|
+
const body = "Task: Fix flaky auth tests\n\nKeep the change small.";
|
|
1250
|
+
|
|
1251
|
+
assert.equal(
|
|
1252
|
+
acceptStrengthenedDraft(
|
|
1253
|
+
request,
|
|
1254
|
+
makeStrengthenedDraft(
|
|
1255
|
+
[
|
|
1256
|
+
"commands:",
|
|
1257
|
+
" name: tests",
|
|
1258
|
+
" run: npm test",
|
|
1259
|
+
" timeout: 20",
|
|
1260
|
+
"max_iterations: 20",
|
|
1261
|
+
"timeout: 120",
|
|
1262
|
+
"guardrails:",
|
|
1263
|
+
" block_commands:",
|
|
1264
|
+
" - 'git\\s+push'",
|
|
1265
|
+
" protected_files:",
|
|
1266
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1267
|
+
],
|
|
1268
|
+
body,
|
|
1269
|
+
),
|
|
1270
|
+
),
|
|
1271
|
+
null,
|
|
1272
|
+
);
|
|
1273
|
+
});
|
|
1274
|
+
|
|
1275
|
+
test("acceptStrengthenedDraft rejects invented, renamed, swapped, and duplicate commands", () => {
|
|
1276
|
+
const request = makeFixRequest();
|
|
1277
|
+
const body = "Task: Fix flaky auth tests\n\nKeep the change small.";
|
|
1278
|
+
|
|
1279
|
+
assert.equal(
|
|
1280
|
+
acceptStrengthenedDraft(
|
|
1281
|
+
request,
|
|
1282
|
+
makeStrengthenedDraft(
|
|
1283
|
+
[
|
|
1284
|
+
"commands:",
|
|
1285
|
+
" - name: smoke",
|
|
1286
|
+
" run: npm run smoke",
|
|
1287
|
+
" timeout: 20",
|
|
1288
|
+
"max_iterations: 20",
|
|
1289
|
+
"timeout: 120",
|
|
1290
|
+
"guardrails:",
|
|
1291
|
+
" block_commands:",
|
|
1292
|
+
" - 'git\\s+push'",
|
|
1293
|
+
" protected_files:",
|
|
1294
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1295
|
+
],
|
|
1296
|
+
body,
|
|
1297
|
+
),
|
|
1298
|
+
),
|
|
1299
|
+
null,
|
|
1300
|
+
);
|
|
1301
|
+
|
|
1302
|
+
assert.equal(
|
|
1303
|
+
acceptStrengthenedDraft(
|
|
1304
|
+
request,
|
|
1305
|
+
makeStrengthenedDraft(
|
|
1306
|
+
[
|
|
1307
|
+
"commands:",
|
|
1308
|
+
" - name: unit-tests",
|
|
1309
|
+
" run: npm test",
|
|
1310
|
+
" timeout: 20",
|
|
1311
|
+
"max_iterations: 20",
|
|
1312
|
+
"timeout: 120",
|
|
1313
|
+
"guardrails:",
|
|
1314
|
+
" block_commands:",
|
|
1315
|
+
" - 'git\\s+push'",
|
|
1316
|
+
" protected_files:",
|
|
1317
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1318
|
+
],
|
|
1319
|
+
body,
|
|
1320
|
+
),
|
|
1321
|
+
),
|
|
1322
|
+
null,
|
|
1323
|
+
);
|
|
1324
|
+
|
|
1325
|
+
assert.equal(
|
|
1326
|
+
acceptStrengthenedDraft(
|
|
1327
|
+
request,
|
|
1328
|
+
makeStrengthenedDraft(
|
|
1329
|
+
[
|
|
1330
|
+
"commands:",
|
|
1331
|
+
" - name: tests",
|
|
1332
|
+
" run: git log --oneline -10",
|
|
1333
|
+
" timeout: 20",
|
|
1334
|
+
" - name: git-log",
|
|
1335
|
+
" run: npm test",
|
|
1336
|
+
" timeout: 20",
|
|
1337
|
+
"max_iterations: 20",
|
|
1338
|
+
"timeout: 120",
|
|
1339
|
+
"guardrails:",
|
|
1340
|
+
" block_commands:",
|
|
1341
|
+
" - 'git\\s+push'",
|
|
1342
|
+
" protected_files:",
|
|
1343
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1344
|
+
],
|
|
1345
|
+
body,
|
|
1346
|
+
),
|
|
1347
|
+
),
|
|
1348
|
+
null,
|
|
1349
|
+
);
|
|
1350
|
+
|
|
1351
|
+
assert.equal(
|
|
1352
|
+
acceptStrengthenedDraft(
|
|
1353
|
+
request,
|
|
1354
|
+
makeStrengthenedDraft(
|
|
1355
|
+
[
|
|
1356
|
+
"commands:",
|
|
1357
|
+
" - name: tests",
|
|
1358
|
+
" run: npm test",
|
|
1359
|
+
" timeout: 20",
|
|
1360
|
+
" - name: tests",
|
|
1361
|
+
" run: npm test",
|
|
1362
|
+
" timeout: 20",
|
|
1363
|
+
"max_iterations: 20",
|
|
1364
|
+
"timeout: 120",
|
|
1365
|
+
"guardrails:",
|
|
1366
|
+
" block_commands:",
|
|
1367
|
+
" - 'git\\s+push'",
|
|
1368
|
+
" protected_files:",
|
|
1369
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1370
|
+
],
|
|
1371
|
+
body,
|
|
1372
|
+
),
|
|
1373
|
+
),
|
|
1374
|
+
null,
|
|
1375
|
+
);
|
|
1376
|
+
});
|
|
1377
|
+
|
|
1378
|
+
test("acceptStrengthenedDraft rejects placeholder drift, increased limits, and command-timeout overflow", () => {
|
|
1379
|
+
const request = makeFixRequest();
|
|
1380
|
+
|
|
1381
|
+
assert.equal(
|
|
1382
|
+
acceptStrengthenedDraft(
|
|
1383
|
+
request,
|
|
1384
|
+
makeStrengthenedDraft(
|
|
1385
|
+
[
|
|
1386
|
+
"commands:",
|
|
1387
|
+
" - name: git-log",
|
|
1388
|
+
" run: git log --oneline -10",
|
|
1389
|
+
" timeout: 20",
|
|
1390
|
+
" - name: tests",
|
|
1391
|
+
" run: npm test",
|
|
1392
|
+
" timeout: 20",
|
|
1393
|
+
"max_iterations: 26",
|
|
1394
|
+
"timeout: 120",
|
|
1395
|
+
"guardrails:",
|
|
1396
|
+
" block_commands:",
|
|
1397
|
+
" - 'git\\s+push'",
|
|
1398
|
+
" protected_files:",
|
|
1399
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1400
|
+
],
|
|
1401
|
+
"Task: Fix flaky auth tests\n\nUse {{ commands.tests }} and {{ commands.lint }}.",
|
|
1402
|
+
),
|
|
1403
|
+
),
|
|
1404
|
+
null,
|
|
1405
|
+
);
|
|
1406
|
+
|
|
1407
|
+
assert.equal(
|
|
1408
|
+
acceptStrengthenedDraft(
|
|
1409
|
+
request,
|
|
1410
|
+
makeStrengthenedDraft(
|
|
1411
|
+
[
|
|
1412
|
+
"commands:",
|
|
1413
|
+
" - name: git-log",
|
|
1414
|
+
" run: git log --oneline -10",
|
|
1415
|
+
" timeout: 20",
|
|
1416
|
+
" - name: tests",
|
|
1417
|
+
" run: npm test",
|
|
1418
|
+
" timeout: 21",
|
|
1419
|
+
"max_iterations: 20",
|
|
1420
|
+
"timeout: 20",
|
|
1421
|
+
"guardrails:",
|
|
1422
|
+
" block_commands:",
|
|
1423
|
+
" - 'git\\s+push'",
|
|
1424
|
+
" protected_files:",
|
|
1425
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1426
|
+
],
|
|
1427
|
+
"Task: Fix flaky auth tests\n\nUse {{ commands.tests }} and {{ commands.git-log }}.",
|
|
1428
|
+
),
|
|
1429
|
+
),
|
|
1430
|
+
null,
|
|
1431
|
+
);
|
|
1432
|
+
|
|
1433
|
+
assert.equal(
|
|
1434
|
+
acceptStrengthenedDraft(
|
|
1435
|
+
request,
|
|
1436
|
+
makeStrengthenedDraft(
|
|
1437
|
+
[
|
|
1438
|
+
"commands:",
|
|
1439
|
+
" - name: git-log",
|
|
1440
|
+
" run: git log --oneline -10",
|
|
1441
|
+
" timeout: 20",
|
|
1442
|
+
" - name: tests",
|
|
1443
|
+
" run: npm test",
|
|
1444
|
+
" timeout: 20",
|
|
1445
|
+
"max_iterations: 20",
|
|
1446
|
+
"timeout: 301",
|
|
1447
|
+
"guardrails:",
|
|
1448
|
+
" block_commands:",
|
|
1449
|
+
" - 'git\\s+push'",
|
|
1450
|
+
" protected_files:",
|
|
1451
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1452
|
+
],
|
|
1453
|
+
"Task: Fix flaky auth tests\n\nUse {{ commands.tests }} and {{ commands.git-log }}.",
|
|
1454
|
+
),
|
|
1455
|
+
),
|
|
1456
|
+
null,
|
|
1457
|
+
);
|
|
1458
|
+
});
|
|
1459
|
+
|
|
1460
|
+
test("acceptStrengthenedDraft rejects args placeholders in strengthened bodies", () => {
|
|
1461
|
+
const request = makeFixRequest();
|
|
1462
|
+
|
|
1463
|
+
assert.equal(
|
|
1464
|
+
acceptStrengthenedDraft(
|
|
1465
|
+
request,
|
|
1466
|
+
makeStrengthenedDraft(
|
|
1467
|
+
[
|
|
1468
|
+
"args:",
|
|
1469
|
+
" - owner",
|
|
1470
|
+
"commands:",
|
|
1471
|
+
" - name: git-log",
|
|
1472
|
+
" run: git log --oneline -10",
|
|
1473
|
+
" timeout: 20",
|
|
1474
|
+
" - name: tests",
|
|
1475
|
+
" run: npm test",
|
|
1476
|
+
" timeout: 20",
|
|
1477
|
+
"max_iterations: 20",
|
|
1478
|
+
"timeout: 120",
|
|
1479
|
+
"guardrails:",
|
|
1480
|
+
" block_commands:",
|
|
1481
|
+
" - 'git\\s+push'",
|
|
1482
|
+
" protected_files:",
|
|
1483
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1484
|
+
],
|
|
1485
|
+
"Task: Fix flaky auth tests\n\nUse {{ args.owner }} and {{ commands.tests }}.",
|
|
1486
|
+
),
|
|
1487
|
+
),
|
|
1488
|
+
null,
|
|
1489
|
+
);
|
|
1490
|
+
});
|
|
1491
|
+
|
|
1492
|
+
test("acceptStrengthenedDraft rejects args frontmatter drift", () => {
|
|
1493
|
+
const request = makeFixRequestWithArgs(["owner"]);
|
|
1494
|
+
const body = "Task: Fix flaky auth tests\n\nKeep the change small.";
|
|
1495
|
+
|
|
1496
|
+
assert.equal(
|
|
1497
|
+
acceptStrengthenedDraft(
|
|
1498
|
+
request,
|
|
1499
|
+
makeStrengthenedDraft(
|
|
1500
|
+
[
|
|
1501
|
+
"args:",
|
|
1502
|
+
" - owner",
|
|
1503
|
+
" - mode",
|
|
1504
|
+
"commands:",
|
|
1505
|
+
" - name: git-log",
|
|
1506
|
+
" run: git log --oneline -10",
|
|
1507
|
+
" timeout: 20",
|
|
1508
|
+
" - name: tests",
|
|
1509
|
+
" run: npm test",
|
|
1510
|
+
" timeout: 20",
|
|
1511
|
+
"max_iterations: 20",
|
|
1512
|
+
"timeout: 120",
|
|
1513
|
+
"guardrails:",
|
|
1514
|
+
" block_commands:",
|
|
1515
|
+
" - 'git\\s+push'",
|
|
1516
|
+
" protected_files:",
|
|
1517
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1518
|
+
],
|
|
1519
|
+
body,
|
|
1520
|
+
),
|
|
1521
|
+
),
|
|
1522
|
+
null,
|
|
1523
|
+
);
|
|
1524
|
+
});
|
|
1525
|
+
|
|
1526
|
+
test("acceptStrengthenedDraft rejects changed guardrails and missing secret-path protection", () => {
|
|
1527
|
+
const request = makeFixRequest();
|
|
1528
|
+
const body = "Task: Fix flaky auth tests\n\nKeep the change small.";
|
|
1529
|
+
|
|
1530
|
+
assert.equal(
|
|
1531
|
+
acceptStrengthenedDraft(
|
|
1532
|
+
request,
|
|
1533
|
+
makeStrengthenedDraft(
|
|
1534
|
+
[
|
|
1535
|
+
"commands:",
|
|
1536
|
+
" - name: git-log",
|
|
1537
|
+
" run: git log --oneline -10",
|
|
1538
|
+
" timeout: 20",
|
|
1539
|
+
" - name: tests",
|
|
1540
|
+
" run: npm test",
|
|
1541
|
+
" timeout: 20",
|
|
1542
|
+
"max_iterations: 20",
|
|
1543
|
+
"timeout: 120",
|
|
1544
|
+
"guardrails:",
|
|
1545
|
+
" block_commands:",
|
|
1546
|
+
" - 'git\\s+push'",
|
|
1547
|
+
" protected_files:",
|
|
1548
|
+
" - .env*",
|
|
1549
|
+
],
|
|
1550
|
+
body,
|
|
1551
|
+
),
|
|
1552
|
+
),
|
|
1553
|
+
null,
|
|
1554
|
+
);
|
|
1555
|
+
|
|
1556
|
+
assert.equal(
|
|
1557
|
+
acceptStrengthenedDraft(
|
|
1558
|
+
request,
|
|
1559
|
+
makeStrengthenedDraft(
|
|
1560
|
+
[
|
|
1561
|
+
"commands:",
|
|
1562
|
+
" - name: git-log",
|
|
1563
|
+
" run: git log --oneline -10",
|
|
1564
|
+
" timeout: 20",
|
|
1565
|
+
" - name: tests",
|
|
1566
|
+
" run: npm test",
|
|
1567
|
+
" timeout: 20",
|
|
1568
|
+
"max_iterations: 20",
|
|
1569
|
+
"timeout: 120",
|
|
1570
|
+
"guardrails:",
|
|
1571
|
+
" block_commands:",
|
|
1572
|
+
" - 'git\\s+push'",
|
|
1573
|
+
" protected_files: [],",
|
|
1574
|
+
],
|
|
1575
|
+
body,
|
|
1576
|
+
),
|
|
1577
|
+
),
|
|
1578
|
+
null,
|
|
1579
|
+
);
|
|
1580
|
+
});
|
|
1581
|
+
|
|
1582
|
+
test("acceptStrengthenedDraft accepts unchanged completion_promise and rejects new or invalid ones", () => {
|
|
1583
|
+
const request = makeFixRequest();
|
|
1584
|
+
const body = "Task: Fix flaky auth tests\n\nKeep the change small.";
|
|
1585
|
+
|
|
1586
|
+
assert.equal(
|
|
1587
|
+
acceptStrengthenedDraft(
|
|
1588
|
+
request,
|
|
1589
|
+
makeStrengthenedDraft(
|
|
1590
|
+
[
|
|
1591
|
+
"commands:",
|
|
1592
|
+
" - name: git-log",
|
|
1593
|
+
" run: git log --oneline -10",
|
|
1594
|
+
" timeout: 20",
|
|
1595
|
+
" - name: tests",
|
|
1596
|
+
" run: npm test",
|
|
1597
|
+
" timeout: 20",
|
|
1598
|
+
"max_iterations: 20",
|
|
1599
|
+
"timeout: 120",
|
|
1600
|
+
"guardrails:",
|
|
1601
|
+
" block_commands:",
|
|
1602
|
+
" - 'git\\s+push'",
|
|
1603
|
+
" protected_files:",
|
|
1604
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1605
|
+
"completion_promise: ship-it",
|
|
1606
|
+
],
|
|
1607
|
+
body,
|
|
1608
|
+
),
|
|
1609
|
+
),
|
|
1610
|
+
null,
|
|
1611
|
+
);
|
|
1612
|
+
|
|
1613
|
+
const promisedRequest = makeFixRequestWithCompletionPromise("ship-it");
|
|
1614
|
+
const promisedDraft = makeStrengthenedDraft(
|
|
1615
|
+
[
|
|
1616
|
+
"commands:",
|
|
1617
|
+
" - name: git-log",
|
|
1618
|
+
" run: git log --oneline -10",
|
|
1619
|
+
" timeout: 20",
|
|
1620
|
+
" - name: tests",
|
|
1621
|
+
" run: npm test",
|
|
1622
|
+
" timeout: 20",
|
|
1623
|
+
"max_iterations: 20",
|
|
1624
|
+
"timeout: 120",
|
|
1625
|
+
"guardrails:",
|
|
1626
|
+
" block_commands:",
|
|
1627
|
+
" - 'git\\s+push'",
|
|
1628
|
+
" protected_files:",
|
|
1629
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1630
|
+
"completion_promise: ship-it",
|
|
1631
|
+
],
|
|
1632
|
+
body,
|
|
1633
|
+
);
|
|
1634
|
+
const accepted = acceptStrengthenedDraft(promisedRequest, promisedDraft);
|
|
1635
|
+
if (!accepted) {
|
|
1636
|
+
assert.fail("expected unchanged completion_promise to be accepted");
|
|
1637
|
+
}
|
|
1638
|
+
assert.equal(parseRalphMarkdown(accepted.content).frontmatter.completionPromise, "ship-it");
|
|
1639
|
+
|
|
1640
|
+
assert.equal(
|
|
1641
|
+
acceptStrengthenedDraft(
|
|
1642
|
+
promisedRequest,
|
|
1643
|
+
makeStrengthenedDraft(
|
|
1644
|
+
[
|
|
1645
|
+
"commands:",
|
|
1646
|
+
" - name: git-log",
|
|
1647
|
+
" run: git log --oneline -10",
|
|
1648
|
+
" timeout: 20",
|
|
1649
|
+
" - name: tests",
|
|
1650
|
+
" run: npm test",
|
|
1651
|
+
" timeout: 20",
|
|
1652
|
+
"max_iterations: 20",
|
|
1653
|
+
"timeout: 120",
|
|
1654
|
+
"guardrails:",
|
|
1655
|
+
" block_commands:",
|
|
1656
|
+
" - 'git\\s+push'",
|
|
1657
|
+
" protected_files:",
|
|
1658
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1659
|
+
"completion_promise: ship-it-too",
|
|
1660
|
+
],
|
|
1661
|
+
body,
|
|
1662
|
+
),
|
|
1663
|
+
),
|
|
1664
|
+
null,
|
|
1665
|
+
);
|
|
1666
|
+
|
|
1667
|
+
assert.equal(
|
|
1668
|
+
acceptStrengthenedDraft(
|
|
1669
|
+
request,
|
|
1670
|
+
makeStrengthenedDraft(
|
|
1671
|
+
[
|
|
1672
|
+
"commands:",
|
|
1673
|
+
" - name: git-log",
|
|
1674
|
+
" run: git log --oneline -10",
|
|
1675
|
+
" timeout: 20",
|
|
1676
|
+
" - name: tests",
|
|
1677
|
+
" run: npm test",
|
|
1678
|
+
" timeout: 20",
|
|
1679
|
+
"max_iterations: 20",
|
|
1680
|
+
"timeout: 120",
|
|
1681
|
+
"guardrails:",
|
|
1682
|
+
" block_commands:",
|
|
1683
|
+
" - 'git\\s+push'",
|
|
1684
|
+
" protected_files:",
|
|
1685
|
+
` - '${SECRET_PATH_POLICY_TOKEN}'`,
|
|
1686
|
+
"completion_promise: [oops]",
|
|
1687
|
+
],
|
|
1688
|
+
body,
|
|
1689
|
+
),
|
|
1690
|
+
),
|
|
1691
|
+
null,
|
|
1692
|
+
);
|
|
471
1693
|
});
|
|
472
1694
|
|
|
473
1695
|
test("isWeakStrengthenedDraft rejects unchanged bodies and fake runtime enforcement claims", () => {
|
|
@@ -525,7 +1747,7 @@ test("generated draft starts fail closed when validation no longer passes", asyn
|
|
|
525
1747
|
|
|
526
1748
|
await handler(`--path ${ralphPath}`, ctx);
|
|
527
1749
|
|
|
528
|
-
assert.deepEqual(notifications, [{ level: "error", message: "Invalid RALPH.md: Invalid max_iterations: must be
|
|
1750
|
+
assert.deepEqual(notifications, [{ level: "error", message: "Invalid RALPH.md: Invalid max_iterations: must be between 1 and 50" }]);
|
|
529
1751
|
});
|
|
530
1752
|
|
|
531
1753
|
test("generateDraft creates metadata-rich analysis and fix drafts", () => {
|
|
@@ -547,6 +1769,8 @@ test("generateDraft creates metadata-rich analysis and fix drafts", () => {
|
|
|
547
1769
|
const analysisBrief = buildMissionBrief(analysisDraft);
|
|
548
1770
|
assert.match(analysisBrief, /- blocks git push/);
|
|
549
1771
|
assert.doesNotMatch(analysisBrief, /read-only/);
|
|
1772
|
+
assert.doesNotMatch(analysisBrief, /Required outputs must exist before stopping/);
|
|
1773
|
+
assert.doesNotMatch(analysisBrief, /OPEN_QUESTIONS\.md must have no remaining P0\/P1 items before stopping\./);
|
|
550
1774
|
|
|
551
1775
|
const fixDraft = generateDraft(
|
|
552
1776
|
"Fix flaky auth tests",
|
|
@@ -582,7 +1806,7 @@ test("generated draft metadata survives task text containing HTML comment marker
|
|
|
582
1806
|
assert.equal(validateDraftContent(draft.content), null);
|
|
583
1807
|
assert.match(draft.content, /Task: Reverse engineer the parser <!-- tricky --> and document the edge case/);
|
|
584
1808
|
assert.match(parsed.body, /Task: Reverse engineer the parser <!-- tricky --> and document the edge case/);
|
|
585
|
-
const rendered = renderRalphBody(parsed.body, [], { iteration: 1, name: "ralph" });
|
|
1809
|
+
const rendered = renderRalphBody(parsed.body, [], { iteration: 1, name: "ralph", maxIterations: 1 });
|
|
586
1810
|
assert.match(rendered, /Task: Reverse engineer the parser <!-- tricky --> and document the edge case/);
|
|
587
1811
|
assert.doesNotMatch(rendered, /<!-- tricky -->/);
|
|
588
1812
|
});
|
|
@@ -598,7 +1822,8 @@ test("buildMissionBrief refreshes after draft edits", () => {
|
|
|
598
1822
|
content: plan.content
|
|
599
1823
|
.replace("Task: Fix flaky auth tests", "Task: Fix flaky auth regressions")
|
|
600
1824
|
.replace("name: tests\n run: npm test\n timeout: 120", "name: smoke\n run: npm run smoke\n timeout: 45")
|
|
601
|
-
.replace("max_iterations: 25", "max_iterations: 7")
|
|
1825
|
+
.replace("max_iterations: 25", "max_iterations: 7")
|
|
1826
|
+
.replace("timeout: 300\n", "timeout: 90\ncompletion_promise: deploy-ready\n"),
|
|
602
1827
|
};
|
|
603
1828
|
|
|
604
1829
|
const brief = buildMissionBrief(editedPlan);
|
|
@@ -607,5 +1832,9 @@ test("buildMissionBrief refreshes after draft edits", () => {
|
|
|
607
1832
|
assert.doesNotMatch(brief, /Fix flaky auth tests/);
|
|
608
1833
|
assert.match(brief, /smoke: npm run smoke/);
|
|
609
1834
|
assert.match(brief, /Stop after 7 iterations or \/ralph-stop/);
|
|
1835
|
+
assert.match(brief, /Stop if an iteration exceeds 90s/);
|
|
1836
|
+
assert.match(brief, /OPEN_QUESTIONS\.md must have no remaining P0\/P1 items before stopping\./);
|
|
1837
|
+
assert.match(brief, /Stop early on <promise>deploy-ready<\/promise>/);
|
|
1838
|
+
assert.match(brief, /OPEN_QUESTIONS\.md must have no remaining P0\/P1 items before stopping\./);
|
|
610
1839
|
assert.doesNotMatch(brief, /tests: npm test/);
|
|
611
1840
|
});
|