@dailephd/my-dev-kit-lab 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +6 -1
  2. package/README.md +7 -5
  3. package/dist/src/agents/adapters/codexAdapter.js +4 -1
  4. package/dist/src/cli/help.js +38 -2
  5. package/dist/src/cli/runLabCli.js +9 -1
  6. package/dist/src/commands/buildGalleryCommand.js +16 -3
  7. package/dist/src/commands/runComparePromptsCommand.js +109 -0
  8. package/dist/src/commands/runExperimentRunCommand.js +47 -2
  9. package/dist/src/core/processTree.js +41 -17
  10. package/dist/src/core/runMeasuredCommand.js +14 -8
  11. package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +23 -1
  12. package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +7 -2
  13. package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +29 -4
  14. package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +1 -1
  15. package/dist/src/experiments/plugins/agentSuccessRate/config.js +47 -2
  16. package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +4 -2
  17. package/dist/src/experiments/plugins/agentSuccessRate/promptContract.js +121 -0
  18. package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +32 -2
  19. package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +105 -11
  20. package/dist/src/experiments/plugins/agentSuccessRate/strictDiffResponse.js +312 -0
  21. package/dist/src/gallery/galleryPathSafety.js +111 -0
  22. package/dist/src/gallery/readGallerySources.js +542 -0
  23. package/dist/src/gallery/writeGalleryManifest.js +57 -3
  24. package/dist/src/report/experiments/buildPluginExperimentReport.js +4 -16
  25. package/dist/src/report/experiments/htmlSafety.js +32 -0
  26. package/dist/src/report/experiments/metricEvidence.js +389 -0
  27. package/dist/src/report/experiments/metricGlossary.js +227 -0
  28. package/dist/src/report/experiments/metricGlossaryCatalog.js +199 -0
  29. package/dist/src/report/experiments/promptTemplateComparisonEvidence.js +377 -0
  30. package/dist/src/report/experiments/promptTemplateComparisonModel.js +436 -0
  31. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +158 -64
  32. package/dist/src/report/experiments/renderPluginExperimentReportText.js +22 -677
  33. package/dist/src/report/experiments/renderPromptTemplateComparison.js +192 -0
  34. package/dist/src/report/experiments/renderSpecializedTextSections.js +633 -0
  35. package/dist/src/report/experiments/reportSectionRegistry.js +79 -0
  36. package/dist/src/report/experiments/specializedReportSections.js +142 -0
  37. package/dist/src/report/experiments/writePluginExperimentReports.js +4 -2
  38. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +146 -133
  39. package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +54 -36
  40. package/dist/src/securityValidation/commandRunner.js +13 -6
  41. package/docs/METRICS.md +11 -1
  42. package/package.json +1 -1
@@ -0,0 +1,312 @@
1
+ /**
2
+ * Application-level output validation for the real-agent prompt contract. It runs on the final answer text that a
3
+ * provider adapter has ALREADY extracted from its transport (Codex JSONL / Claude result envelope) and before the
4
+ * Lab-owned patch pipeline. It never parses a provider transport envelope and never falls back to legacy extraction
5
+ * for an invalid strict response.
6
+ */
7
+ export const STRICT_DIFF_RESPONSE_SCHEMA_VERSION = "1.0.0";
8
+ export const STRICT_DIFF_MAX_PATCH_BYTES = 1_048_576;
9
+ // Worst case JSON escaping (\u00XX) expands one patch byte to six response bytes, plus the fixed envelope.
10
+ const MAX_RESPONSE_BYTES = STRICT_DIFF_MAX_PATCH_BYTES * 6 + 1024;
11
+ const MAX_DEPTH = 32;
12
+ export const OUTPUT_CONTRACT_CODES = [
13
+ "STRICT_EMPTY_RESPONSE",
14
+ "STRICT_RESPONSE_TOO_LARGE",
15
+ "STRICT_INVALID_UNICODE",
16
+ "STRICT_MARKDOWN_FENCE",
17
+ "STRICT_NOT_JSON",
18
+ "STRICT_TOO_DEEP",
19
+ "STRICT_DUPLICATE_KEY",
20
+ "STRICT_MULTIPLE_DOCUMENTS",
21
+ "STRICT_TRAILING_CONTENT",
22
+ "STRICT_NOT_OBJECT",
23
+ "STRICT_MISSING_FIELD",
24
+ "STRICT_UNKNOWN_FIELD",
25
+ "STRICT_FIELD_TYPE",
26
+ "STRICT_SCHEMA_VERSION",
27
+ "STRICT_EMPTY_PATCH",
28
+ "STRICT_PATCH_TOO_LARGE",
29
+ "STRICT_PATCH_NOT_RAW_DIFF",
30
+ "EXPLANATION_EMPTY",
31
+ "EXPLANATION_CODE_FENCE",
32
+ "EXPLANATION_PROSE_BEFORE",
33
+ "EXPLANATION_PROSE_IN_DIFF",
34
+ "EXPLANATION_MALFORMED_DIFF"
35
+ ];
36
+ class JsonProblem extends Error {
37
+ code;
38
+ constructor(code) {
39
+ super(code);
40
+ this.code = code;
41
+ }
42
+ }
43
+ const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
44
+ const isWellFormedUnicode = (value) => !LONE_SURROGATE.test(value);
45
+ const isWhitespace = (char) => char === " " || char === "\t" || char === "\n" || char === "\r";
46
+ /** Small bounded RFC 8259 parser. It exists because JSON.parse silently keeps the last of duplicate object keys. */
47
+ function parseJsonStrict(text) {
48
+ let index = 0;
49
+ const skip = () => {
50
+ while (index < text.length && isWhitespace(text[index]))
51
+ index += 1;
52
+ };
53
+ const bad = () => {
54
+ throw new JsonProblem("STRICT_NOT_JSON");
55
+ };
56
+ const parseString = () => {
57
+ index += 1; // opening quote
58
+ let out = "";
59
+ for (;;) {
60
+ if (index >= text.length)
61
+ bad();
62
+ const char = text[index];
63
+ if (char === '"') {
64
+ index += 1;
65
+ return out;
66
+ }
67
+ if (char.charCodeAt(0) < 0x20)
68
+ bad();
69
+ if (char === "\\") {
70
+ const next = text[index + 1];
71
+ index += 2;
72
+ switch (next) {
73
+ case '"':
74
+ out += '"';
75
+ break;
76
+ case "\\":
77
+ out += "\\";
78
+ break;
79
+ case "/":
80
+ out += "/";
81
+ break;
82
+ case "b":
83
+ out += "\b";
84
+ break;
85
+ case "f":
86
+ out += "\f";
87
+ break;
88
+ case "n":
89
+ out += "\n";
90
+ break;
91
+ case "r":
92
+ out += "\r";
93
+ break;
94
+ case "t":
95
+ out += "\t";
96
+ break;
97
+ case "u": {
98
+ const hex = text.slice(index, index + 4);
99
+ if (!/^[0-9a-fA-F]{4}$/.test(hex))
100
+ bad();
101
+ out += String.fromCharCode(parseInt(hex, 16));
102
+ index += 4;
103
+ break;
104
+ }
105
+ default:
106
+ bad();
107
+ }
108
+ continue;
109
+ }
110
+ out += char;
111
+ index += 1;
112
+ }
113
+ };
114
+ const parseValue = (depth) => {
115
+ if (depth > MAX_DEPTH)
116
+ throw new JsonProblem("STRICT_TOO_DEEP");
117
+ skip();
118
+ const char = text[index];
119
+ if (char === undefined)
120
+ return bad();
121
+ if (char === "{") {
122
+ index += 1;
123
+ const object = new Map();
124
+ skip();
125
+ if (text[index] === "}") {
126
+ index += 1;
127
+ return object;
128
+ }
129
+ for (;;) {
130
+ skip();
131
+ if (text[index] !== '"')
132
+ bad();
133
+ const key = parseString();
134
+ if (object.has(key))
135
+ throw new JsonProblem("STRICT_DUPLICATE_KEY");
136
+ skip();
137
+ if (text[index] !== ":")
138
+ bad();
139
+ index += 1;
140
+ object.set(key, parseValue(depth + 1));
141
+ skip();
142
+ if (text[index] === ",") {
143
+ index += 1;
144
+ continue;
145
+ }
146
+ if (text[index] === "}") {
147
+ index += 1;
148
+ return object;
149
+ }
150
+ bad();
151
+ }
152
+ }
153
+ if (char === "[") {
154
+ index += 1;
155
+ const array = [];
156
+ skip();
157
+ if (text[index] === "]") {
158
+ index += 1;
159
+ return array;
160
+ }
161
+ for (;;) {
162
+ array.push(parseValue(depth + 1));
163
+ skip();
164
+ if (text[index] === ",") {
165
+ index += 1;
166
+ continue;
167
+ }
168
+ if (text[index] === "]") {
169
+ index += 1;
170
+ return array;
171
+ }
172
+ bad();
173
+ }
174
+ }
175
+ if (char === '"')
176
+ return parseString();
177
+ for (const [literal, value] of [["true", true], ["false", false], ["null", null]]) {
178
+ if (text.startsWith(literal, index)) {
179
+ index += literal.length;
180
+ return value;
181
+ }
182
+ }
183
+ const number = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/.exec(text.slice(index, index + 64));
184
+ if (number) {
185
+ index += number[0].length;
186
+ return Number(number[0]);
187
+ }
188
+ return bad();
189
+ };
190
+ const value = parseValue(0);
191
+ skip();
192
+ return { value, end: index };
193
+ }
194
+ const HUNK_HEADER = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
195
+ const HEADER_PREFIXES = [
196
+ "diff --git ", "index ", "--- ", "+++ ", "new file mode ", "deleted file mode ", "old mode ", "new mode ",
197
+ "similarity index ", "dissimilarity index ", "rename from ", "rename to ", "copy from ", "copy to ", "Binary files "
198
+ ];
199
+ /**
200
+ * Raw-diff-only grammar check: every line must be a git diff header, a hunk header, or a hunk body line whose count
201
+ * is consumed exactly by the preceding hunk header. Prose before, between or after the diff is a violation.
202
+ */
203
+ function checkRawDiffOnly(text) {
204
+ if (text.trim() === "")
205
+ return "EXPLANATION_EMPTY";
206
+ if (/^[ \t]*```/m.test(text))
207
+ return "EXPLANATION_CODE_FENCE";
208
+ const trimmed = text.trimStart();
209
+ if (!trimmed.startsWith("diff --git ") && !trimmed.startsWith("--- "))
210
+ return "EXPLANATION_PROSE_BEFORE";
211
+ const lines = trimmed.split(/\r?\n/);
212
+ while (lines.length > 0 && lines[lines.length - 1] === "")
213
+ lines.pop();
214
+ let oldRemaining = 0;
215
+ let newRemaining = 0;
216
+ for (const line of lines) {
217
+ if (oldRemaining > 0 || newRemaining > 0) {
218
+ if (line.startsWith("\\"))
219
+ continue;
220
+ const marker = line.length === 0 ? " " : line[0];
221
+ if (marker === " ") {
222
+ oldRemaining -= 1;
223
+ newRemaining -= 1;
224
+ }
225
+ else if (marker === "-")
226
+ oldRemaining -= 1;
227
+ else if (marker === "+")
228
+ newRemaining -= 1;
229
+ else
230
+ return "EXPLANATION_PROSE_IN_DIFF";
231
+ if (oldRemaining < 0 || newRemaining < 0)
232
+ return "EXPLANATION_PROSE_IN_DIFF";
233
+ continue;
234
+ }
235
+ if (line.startsWith("\\"))
236
+ continue;
237
+ const hunk = HUNK_HEADER.exec(line);
238
+ if (hunk) {
239
+ oldRemaining = hunk[2] === undefined ? 1 : Number(hunk[2]);
240
+ newRemaining = hunk[4] === undefined ? 1 : Number(hunk[4]);
241
+ continue;
242
+ }
243
+ if (HEADER_PREFIXES.some((prefix) => line.startsWith(prefix)))
244
+ continue;
245
+ return "EXPLANATION_PROSE_IN_DIFF";
246
+ }
247
+ return oldRemaining === 0 && newRemaining === 0 ? null : "EXPLANATION_MALFORMED_DIFF";
248
+ }
249
+ /** Validates one strict-diff-v1 final answer. Returns the decoded patch only for an exactly-conforming response. */
250
+ export function parseStrictDiffResponse(text) {
251
+ if (typeof text !== "string" || text.trim() === "")
252
+ return { ok: false, code: "STRICT_EMPTY_RESPONSE" };
253
+ if (Buffer.byteLength(text, "utf8") > MAX_RESPONSE_BYTES)
254
+ return { ok: false, code: "STRICT_RESPONSE_TOO_LARGE" };
255
+ if (!isWellFormedUnicode(text))
256
+ return { ok: false, code: "STRICT_INVALID_UNICODE" };
257
+ if (/^\s*```/.test(text))
258
+ return { ok: false, code: "STRICT_MARKDOWN_FENCE" };
259
+ let parsed;
260
+ try {
261
+ parsed = parseJsonStrict(text);
262
+ }
263
+ catch (error) {
264
+ return { ok: false, code: error instanceof JsonProblem ? error.code : "STRICT_NOT_JSON" };
265
+ }
266
+ const rest = text.slice(parsed.end);
267
+ if (rest !== "") {
268
+ const next = rest.trimStart()[0];
269
+ return { ok: false, code: next === "{" || next === "[" ? "STRICT_MULTIPLE_DOCUMENTS" : "STRICT_TRAILING_CONTENT" };
270
+ }
271
+ const value = parsed.value;
272
+ if (!(value instanceof Map))
273
+ return { ok: false, code: "STRICT_NOT_OBJECT" };
274
+ for (const key of value.keys()) {
275
+ if (key !== "schemaVersion" && key !== "patch")
276
+ return { ok: false, code: "STRICT_UNKNOWN_FIELD" };
277
+ }
278
+ if (!value.has("schemaVersion") || !value.has("patch"))
279
+ return { ok: false, code: "STRICT_MISSING_FIELD" };
280
+ const schemaVersion = value.get("schemaVersion");
281
+ const patch = value.get("patch");
282
+ if (typeof schemaVersion !== "string" || typeof patch !== "string")
283
+ return { ok: false, code: "STRICT_FIELD_TYPE" };
284
+ if (schemaVersion !== STRICT_DIFF_RESPONSE_SCHEMA_VERSION)
285
+ return { ok: false, code: "STRICT_SCHEMA_VERSION" };
286
+ if (!isWellFormedUnicode(patch))
287
+ return { ok: false, code: "STRICT_INVALID_UNICODE" };
288
+ if (patch.trim() === "")
289
+ return { ok: false, code: "STRICT_EMPTY_PATCH" };
290
+ if (Buffer.byteLength(patch, "utf8") > STRICT_DIFF_MAX_PATCH_BYTES)
291
+ return { ok: false, code: "STRICT_PATCH_TOO_LARGE" };
292
+ if (checkRawDiffOnly(patch) !== null)
293
+ return { ok: false, code: "STRICT_PATCH_NOT_RAW_DIFF" };
294
+ return { ok: true, response: { schemaVersion: STRICT_DIFF_RESPONSE_SCHEMA_VERSION, patch } };
295
+ }
296
+ /**
297
+ * Applies the effective output contract to an extracted provider final answer.
298
+ * - strict-diff-v1: exactly StrictDiffResponseV1; the decoded patch is returned.
299
+ * - legacy-diff-v1 + explanationMode none: a raw unified diff with no fence or prose; the text is returned unchanged.
300
+ * - legacy-diff-v1 + explanationMode legacy: historical behavior, text returned unchanged.
301
+ */
302
+ export function validateAgentSuccessFinalAnswer(settings, finalAnswerText) {
303
+ if (settings.templateId === "strict-diff-v1") {
304
+ const parsed = parseStrictDiffResponse(finalAnswerText);
305
+ return parsed.ok ? { ok: true, patchText: parsed.response.patch } : { ok: false, code: parsed.code };
306
+ }
307
+ if (settings.explanationMode === "none") {
308
+ const violation = checkRawDiffOnly(finalAnswerText);
309
+ return violation === null ? { ok: true, patchText: finalAnswerText } : { ok: false, code: violation };
310
+ }
311
+ return { ok: true, patchText: finalAnswerText };
312
+ }
@@ -0,0 +1,111 @@
1
+ import { open, realpath, stat } from "node:fs/promises";
2
+ import path from "node:path";
3
+ // Fail-closed path and media validation for gallery source artifacts. Source artifacts are untrusted
4
+ // filesystem inputs: a reference is accepted only when it is a safe run-root-relative POSIX path whose
5
+ // physically resolved target is an ordinary supported file inside the verified source root.
6
+ const SCHEME_PATTERN = /^[a-zA-Z][a-zA-Z0-9+.-]*:/;
7
+ // eslint-disable-next-line no-control-regex
8
+ const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
9
+ export const GALLERY_MEDIA_EXTENSIONS = {
10
+ video: [".webm"],
11
+ screenshot: [".png", ".jpg", ".jpeg", ".webp"],
12
+ markdown: [".md"],
13
+ text: [".txt", ".srt", ".vtt", ".html"],
14
+ json: [".json"]
15
+ };
16
+ /** True only for a non-empty relative POSIX path with no traversal, drive, UNC, scheme, NUL or encoding tricks. */
17
+ export function isSafeRelativeArtifactPath(value) {
18
+ if (typeof value !== "string" || value.length === 0 || value.length > 1024)
19
+ return false;
20
+ if (CONTROL_PATTERN.test(value) || value.includes("\\") || value.includes("%"))
21
+ return false;
22
+ if (value.startsWith("/") || SCHEME_PATTERN.test(value))
23
+ return false;
24
+ return value.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..");
25
+ }
26
+ /**
27
+ * Validates a link value for the gallery HTML. A gallery link is output-relative, so a leading "../" run is
28
+ * legitimate (the artifact lives in a verified source root outside the gallery directory); any other
29
+ * traversal, absolute form, scheme or malformed segment is rejected. Returns the percent-encoded href or null.
30
+ */
31
+ export function safeGalleryHref(value, allowedExtensions) {
32
+ if (typeof value !== "string" || value.length === 0 || value.length > 2048)
33
+ return null;
34
+ if (CONTROL_PATTERN.test(value) || value.includes("\\") || value.includes("%"))
35
+ return null;
36
+ if (value.startsWith("/") || SCHEME_PATTERN.test(value))
37
+ return null;
38
+ const segments = value.split("/");
39
+ let index = 0;
40
+ while (segments[index] === "..")
41
+ index += 1;
42
+ const rest = segments.slice(index);
43
+ if (rest.length === 0 || rest.some((segment) => segment === "" || segment === "." || segment === ".."))
44
+ return null;
45
+ if (allowedExtensions) {
46
+ const ext = path.posix.extname(rest[rest.length - 1]).toLowerCase();
47
+ if (!allowedExtensions.includes(ext))
48
+ return null;
49
+ }
50
+ return [...segments.slice(0, index), ...rest.map((segment) => encodeURIComponent(segment))].join("/");
51
+ }
52
+ function within(root, candidate) {
53
+ const relative = path.relative(root, candidate);
54
+ return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
55
+ }
56
+ async function sniff(file, kind, ext) {
57
+ if (kind === "markdown" || kind === "text" || kind === "json")
58
+ return true;
59
+ const handle = await open(file, "r");
60
+ try {
61
+ const buffer = Buffer.alloc(12);
62
+ const { bytesRead } = await handle.read(buffer, 0, 12, 0);
63
+ if (bytesRead < 4)
64
+ return false;
65
+ if (kind === "video")
66
+ return buffer[0] === 0x1a && buffer[1] === 0x45 && buffer[2] === 0xdf && buffer[3] === 0xa3;
67
+ if (ext === ".png")
68
+ return buffer.subarray(0, 4).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47]));
69
+ if (ext === ".jpg" || ext === ".jpeg")
70
+ return buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff;
71
+ if (ext === ".webp")
72
+ return bytesRead >= 12 && buffer.toString("latin1", 0, 4) === "RIFF" && buffer.toString("latin1", 8, 12) === "WEBP";
73
+ return false;
74
+ }
75
+ finally {
76
+ await handle.close();
77
+ }
78
+ }
79
+ /** Resolves and verifies one run-root-relative artifact reference. Never throws; failure carries a reason. */
80
+ export async function verifyArtifactFile(realRoot, reference, kind) {
81
+ if (!isSafeRelativeArtifactPath(reference))
82
+ return { ok: false, reason: "the recorded path is not a safe run-root-relative path" };
83
+ const ext = path.posix.extname(reference).toLowerCase();
84
+ if (!GALLERY_MEDIA_EXTENSIONS[kind].includes(ext))
85
+ return { ok: false, reason: `the file type ${ext || "(none)"} is not supported for ${kind}` };
86
+ try {
87
+ const real = await realpath(path.resolve(realRoot, ...reference.split("/")));
88
+ if (!within(realRoot, real))
89
+ return { ok: false, reason: "the file resolves outside the source root" };
90
+ const info = await stat(real);
91
+ if (!info.isFile())
92
+ return { ok: false, reason: "the path is not an ordinary file" };
93
+ if (!(await sniff(real, kind, ext)))
94
+ return { ok: false, reason: `the file content is not a valid ${ext.slice(1)} ${kind}` };
95
+ return { ok: true, absolutePath: real };
96
+ }
97
+ catch {
98
+ return { ok: false, reason: "the file is missing or unreadable" };
99
+ }
100
+ }
101
+ /** Output-relative POSIX link text for a verified file, or null when no relative path exists (for example another drive). */
102
+ export function outputRelativePath(realOutDir, absoluteFile) {
103
+ const relative = path.relative(realOutDir, absoluteFile);
104
+ if (relative === "" || path.isAbsolute(relative))
105
+ return null;
106
+ return relative.split(path.sep).join("/");
107
+ }
108
+ /** Explicit user-supplied source text must be a local filesystem path, never a URL. */
109
+ export function looksLikeUrl(value) {
110
+ return /^[a-zA-Z][a-zA-Z0-9+.-]+:/.test(value) && !/^[a-zA-Z]:[\/]/.test(value);
111
+ }