@dailephd/my-dev-kit-lab 0.9.2 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -1
- package/README.md +9 -5
- package/dist/src/agents/adapters/codexAdapter.js +4 -1
- package/dist/src/cli/help.js +38 -2
- package/dist/src/cli/runLabCli.js +9 -1
- package/dist/src/commands/buildGalleryCommand.js +16 -3
- package/dist/src/commands/runComparePromptsCommand.js +109 -0
- package/dist/src/commands/runExperimentRunCommand.js +47 -2
- package/dist/src/core/processTree.js +41 -17
- package/dist/src/core/runMeasuredCommand.js +14 -8
- package/dist/src/evaluation/affectedNeighborhood.js +140 -32
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +23 -1
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +7 -2
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +29 -4
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +1 -1
- package/dist/src/experiments/plugins/agentSuccessRate/config.js +47 -2
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +4 -2
- package/dist/src/experiments/plugins/agentSuccessRate/promptContract.js +121 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +32 -2
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +105 -11
- package/dist/src/experiments/plugins/agentSuccessRate/strictDiffResponse.js +312 -0
- package/dist/src/gallery/galleryPathSafety.js +111 -0
- package/dist/src/gallery/readGallerySources.js +542 -0
- package/dist/src/gallery/writeGalleryManifest.js +57 -3
- package/dist/src/report/experiments/buildPluginExperimentReport.js +4 -16
- package/dist/src/report/experiments/htmlSafety.js +32 -0
- package/dist/src/report/experiments/metricEvidence.js +389 -0
- package/dist/src/report/experiments/metricGlossary.js +227 -0
- package/dist/src/report/experiments/metricGlossaryCatalog.js +199 -0
- package/dist/src/report/experiments/promptTemplateComparisonEvidence.js +377 -0
- package/dist/src/report/experiments/promptTemplateComparisonModel.js +436 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +158 -64
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +22 -677
- package/dist/src/report/experiments/renderPromptTemplateComparison.js +192 -0
- package/dist/src/report/experiments/renderSpecializedTextSections.js +633 -0
- package/dist/src/report/experiments/reportSectionRegistry.js +79 -0
- package/dist/src/report/experiments/specializedReportSections.js +142 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -2
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +146 -133
- package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +54 -36
- package/dist/src/securityValidation/commandRunner.js +13 -6
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +46 -21
- package/docs/METRICS.md +11 -1
- package/package.json +3 -3
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { open, realpath, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
// Fail-closed path and media validation for gallery source artifacts. Source artifacts are untrusted
|
|
4
|
+
// filesystem inputs: a reference is accepted only when it is a safe run-root-relative POSIX path whose
|
|
5
|
+
// physically resolved target is an ordinary supported file inside the verified source root.
|
|
6
|
+
const SCHEME_PATTERN = /^[a-zA-Z][a-zA-Z0-9+.-]*:/;
|
|
7
|
+
// eslint-disable-next-line no-control-regex
|
|
8
|
+
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
|
9
|
+
export const GALLERY_MEDIA_EXTENSIONS = {
|
|
10
|
+
video: [".webm"],
|
|
11
|
+
screenshot: [".png", ".jpg", ".jpeg", ".webp"],
|
|
12
|
+
markdown: [".md"],
|
|
13
|
+
text: [".txt", ".srt", ".vtt", ".html"],
|
|
14
|
+
json: [".json"]
|
|
15
|
+
};
|
|
16
|
+
/** True only for a non-empty relative POSIX path with no traversal, drive, UNC, scheme, NUL or encoding tricks. */
|
|
17
|
+
export function isSafeRelativeArtifactPath(value) {
|
|
18
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 1024)
|
|
19
|
+
return false;
|
|
20
|
+
if (CONTROL_PATTERN.test(value) || value.includes("\\") || value.includes("%"))
|
|
21
|
+
return false;
|
|
22
|
+
if (value.startsWith("/") || SCHEME_PATTERN.test(value))
|
|
23
|
+
return false;
|
|
24
|
+
return value.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..");
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Validates a link value for the gallery HTML. A gallery link is output-relative, so a leading "../" run is
|
|
28
|
+
* legitimate (the artifact lives in a verified source root outside the gallery directory); any other
|
|
29
|
+
* traversal, absolute form, scheme or malformed segment is rejected. Returns the percent-encoded href or null.
|
|
30
|
+
*/
|
|
31
|
+
export function safeGalleryHref(value, allowedExtensions) {
|
|
32
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 2048)
|
|
33
|
+
return null;
|
|
34
|
+
if (CONTROL_PATTERN.test(value) || value.includes("\\") || value.includes("%"))
|
|
35
|
+
return null;
|
|
36
|
+
if (value.startsWith("/") || SCHEME_PATTERN.test(value))
|
|
37
|
+
return null;
|
|
38
|
+
const segments = value.split("/");
|
|
39
|
+
let index = 0;
|
|
40
|
+
while (segments[index] === "..")
|
|
41
|
+
index += 1;
|
|
42
|
+
const rest = segments.slice(index);
|
|
43
|
+
if (rest.length === 0 || rest.some((segment) => segment === "" || segment === "." || segment === ".."))
|
|
44
|
+
return null;
|
|
45
|
+
if (allowedExtensions) {
|
|
46
|
+
const ext = path.posix.extname(rest[rest.length - 1]).toLowerCase();
|
|
47
|
+
if (!allowedExtensions.includes(ext))
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
return [...segments.slice(0, index), ...rest.map((segment) => encodeURIComponent(segment))].join("/");
|
|
51
|
+
}
|
|
52
|
+
function within(root, candidate) {
|
|
53
|
+
const relative = path.relative(root, candidate);
|
|
54
|
+
return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
55
|
+
}
|
|
56
|
+
async function sniff(file, kind, ext) {
|
|
57
|
+
if (kind === "markdown" || kind === "text" || kind === "json")
|
|
58
|
+
return true;
|
|
59
|
+
const handle = await open(file, "r");
|
|
60
|
+
try {
|
|
61
|
+
const buffer = Buffer.alloc(12);
|
|
62
|
+
const { bytesRead } = await handle.read(buffer, 0, 12, 0);
|
|
63
|
+
if (bytesRead < 4)
|
|
64
|
+
return false;
|
|
65
|
+
if (kind === "video")
|
|
66
|
+
return buffer[0] === 0x1a && buffer[1] === 0x45 && buffer[2] === 0xdf && buffer[3] === 0xa3;
|
|
67
|
+
if (ext === ".png")
|
|
68
|
+
return buffer.subarray(0, 4).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
|
69
|
+
if (ext === ".jpg" || ext === ".jpeg")
|
|
70
|
+
return buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff;
|
|
71
|
+
if (ext === ".webp")
|
|
72
|
+
return bytesRead >= 12 && buffer.toString("latin1", 0, 4) === "RIFF" && buffer.toString("latin1", 8, 12) === "WEBP";
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
await handle.close();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** Resolves and verifies one run-root-relative artifact reference. Never throws; failure carries a reason. */
|
|
80
|
+
export async function verifyArtifactFile(realRoot, reference, kind) {
|
|
81
|
+
if (!isSafeRelativeArtifactPath(reference))
|
|
82
|
+
return { ok: false, reason: "the recorded path is not a safe run-root-relative path" };
|
|
83
|
+
const ext = path.posix.extname(reference).toLowerCase();
|
|
84
|
+
if (!GALLERY_MEDIA_EXTENSIONS[kind].includes(ext))
|
|
85
|
+
return { ok: false, reason: `the file type ${ext || "(none)"} is not supported for ${kind}` };
|
|
86
|
+
try {
|
|
87
|
+
const real = await realpath(path.resolve(realRoot, ...reference.split("/")));
|
|
88
|
+
if (!within(realRoot, real))
|
|
89
|
+
return { ok: false, reason: "the file resolves outside the source root" };
|
|
90
|
+
const info = await stat(real);
|
|
91
|
+
if (!info.isFile())
|
|
92
|
+
return { ok: false, reason: "the path is not an ordinary file" };
|
|
93
|
+
if (!(await sniff(real, kind, ext)))
|
|
94
|
+
return { ok: false, reason: `the file content is not a valid ${ext.slice(1)} ${kind}` };
|
|
95
|
+
return { ok: true, absolutePath: real };
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return { ok: false, reason: "the file is missing or unreadable" };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** Output-relative POSIX link text for a verified file, or null when no relative path exists (for example another drive). */
|
|
102
|
+
export function outputRelativePath(realOutDir, absoluteFile) {
|
|
103
|
+
const relative = path.relative(realOutDir, absoluteFile);
|
|
104
|
+
if (relative === "" || path.isAbsolute(relative))
|
|
105
|
+
return null;
|
|
106
|
+
return relative.split(path.sep).join("/");
|
|
107
|
+
}
|
|
108
|
+
/** Explicit user-supplied source text must be a local filesystem path, never a URL. */
|
|
109
|
+
export function looksLikeUrl(value) {
|
|
110
|
+
return /^[a-zA-Z][a-zA-Z0-9+.-]+:/.test(value) && !/^[a-zA-Z]:[\/]/.test(value);
|
|
111
|
+
}
|
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdir, readFile, realpath, stat } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { TUTORIAL_MANIFEST_SCHEMA_VERSION } from "../tutorial/tutorialManifest.js";
|
|
5
|
+
import { looksLikeUrl, outputRelativePath, verifyArtifactFile } from "./galleryPathSafety.js";
|
|
6
|
+
// Trusted, read-only artifact-family readers for `gallery build --source`. Each reader recognizes a family
|
|
7
|
+
// from an existing producer schema/signature, reads only the bounded metadata needed to identify and display
|
|
8
|
+
// it, and never executes artifact content. Gallery items only ever carry statuses the producer recorded.
|
|
9
|
+
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
|
10
|
+
const TUTORIAL_MANIFEST_RELATIVE = "artifacts/tutorial-manifest.json";
|
|
11
|
+
const SUPPORTED_AUDIT_SCHEMA = "1.0";
|
|
12
|
+
const SUPPORTED_SECURITY_SCHEMA = 1;
|
|
13
|
+
const PROMPT_COMPARISON_SCHEMA = "prompt-template-comparison/1.0.0";
|
|
14
|
+
function isObject(value) {
|
|
15
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16
|
+
}
|
|
17
|
+
function str(value) {
|
|
18
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
19
|
+
}
|
|
20
|
+
function num(value) {
|
|
21
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
22
|
+
}
|
|
23
|
+
function bounded(value, limit = 300) {
|
|
24
|
+
return value.length > limit ? `${value.slice(0, limit)}...` : value;
|
|
25
|
+
}
|
|
26
|
+
function slug(value) {
|
|
27
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "item";
|
|
28
|
+
}
|
|
29
|
+
function itemId(family, identity, file) {
|
|
30
|
+
return `${family}-${slug(identity)}-${createHash("sha256").update(file).digest("hex").slice(0, 8)}`;
|
|
31
|
+
}
|
|
32
|
+
async function readJson(file, label) {
|
|
33
|
+
let size;
|
|
34
|
+
try {
|
|
35
|
+
size = (await stat(file)).size;
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw new Error(`Gallery source ${label}: cannot read ${path.basename(file)} (${error instanceof Error ? error.message : String(error)}).`);
|
|
39
|
+
}
|
|
40
|
+
if (size > MAX_JSON_BYTES)
|
|
41
|
+
throw new Error(`Gallery source ${label}: ${path.basename(file)} exceeds the ${MAX_JSON_BYTES}-byte limit.`);
|
|
42
|
+
let parsed;
|
|
43
|
+
try {
|
|
44
|
+
parsed = JSON.parse(await readFile(file, "utf8"));
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
throw new Error(`Gallery source ${label}: ${path.basename(file)} is not valid JSON.`);
|
|
48
|
+
}
|
|
49
|
+
if (!isObject(parsed))
|
|
50
|
+
throw new Error(`Gallery source ${label}: ${path.basename(file)} is not a JSON object.`);
|
|
51
|
+
return parsed;
|
|
52
|
+
}
|
|
53
|
+
/** Signature tests against existing producer schemas. More than one match is an ambiguity error. */
|
|
54
|
+
function matchingFamilies(data) {
|
|
55
|
+
const matches = [];
|
|
56
|
+
const plugin = data.plugin;
|
|
57
|
+
const metadata = data.metadata;
|
|
58
|
+
if (isObject(data.scenario) && isObject(data.run) && Array.isArray(data.steps) && Array.isArray(data.artifacts)) {
|
|
59
|
+
matches.push("tutorial-run");
|
|
60
|
+
}
|
|
61
|
+
if (isObject(plugin) && str(plugin.id) !== null && isObject(metadata) && str(metadata.runId) !== null && Array.isArray(data.variants)) {
|
|
62
|
+
const warm = isObject(data.warmIndexReuse) && isObject(data.warmIndexReuse.agentCampaign);
|
|
63
|
+
matches.push(warm ? "warm-index-campaign" : "plugin-experiment-report");
|
|
64
|
+
}
|
|
65
|
+
if (isObject(metadata) && metadata.reportType === "code-rot-audit" && str(data.schemaVersion) !== null)
|
|
66
|
+
matches.push("audit-report");
|
|
67
|
+
if (Array.isArray(data.checks) && Array.isArray(data.findings) && isObject(data.summary) && str(data.verdict) !== null) {
|
|
68
|
+
matches.push("security-validation-report");
|
|
69
|
+
}
|
|
70
|
+
if (str(data.executiveSummary) !== null && isObject(data.androidProjectSummary) && str(data.verdict) !== null) {
|
|
71
|
+
matches.push("android-validation-report");
|
|
72
|
+
}
|
|
73
|
+
if (num(data.totalRuns) !== null && num(data.completedRuns) !== null && Array.isArray(data.strategies))
|
|
74
|
+
matches.push("controlled-experiment");
|
|
75
|
+
if (typeof data.schemaVersion === "string" && data.schemaVersion.startsWith("prompt-template-comparison/"))
|
|
76
|
+
matches.push("prompt-template-comparison");
|
|
77
|
+
return matches;
|
|
78
|
+
}
|
|
79
|
+
function classify(data, label, file) {
|
|
80
|
+
const matches = matchingFamilies(data);
|
|
81
|
+
if (matches.length > 1) {
|
|
82
|
+
throw new Error(`Gallery source ${label}: ${path.basename(file)} matches more than one artifact family (${matches.join(", ")}); refusing ambiguous recognition.`);
|
|
83
|
+
}
|
|
84
|
+
return matches[0] ?? null;
|
|
85
|
+
}
|
|
86
|
+
const KNOWN_NAME = /(^code-rot-audit\.json$|^experiment-summary\.json$|-android-security-validation\.json$|-security-validation\.json$|^report\.json$|^tutorial-manifest\.json$|^prompt-template-comparison\.json$)/;
|
|
87
|
+
async function collect(sourceArg) {
|
|
88
|
+
const label = sourceArg;
|
|
89
|
+
let real;
|
|
90
|
+
try {
|
|
91
|
+
real = await realpath(sourceArg);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
throw new Error(`Gallery source ${label} does not exist or is unreadable.`);
|
|
95
|
+
}
|
|
96
|
+
const info = await stat(real);
|
|
97
|
+
const found = [];
|
|
98
|
+
const addFile = async (file, root, explicit) => {
|
|
99
|
+
const data = await readJson(file, label);
|
|
100
|
+
// The canonical tutorial manifest is recognized by its fixed name so a malformed one gets a specific error.
|
|
101
|
+
const family = path.basename(file) === "tutorial-manifest.json" ? "tutorial-run" : classify(data, label, file);
|
|
102
|
+
if (family === null) {
|
|
103
|
+
// A generic report.json that is not a plugin report is simply not ours; a named family file must match.
|
|
104
|
+
if (!explicit && path.basename(file) === "report.json")
|
|
105
|
+
return;
|
|
106
|
+
throw new Error(`Gallery source ${label}: ${path.basename(file)} is not a recognized or valid artifact family document.`);
|
|
107
|
+
}
|
|
108
|
+
found.push({ family, file, root, data });
|
|
109
|
+
};
|
|
110
|
+
if (info.isFile()) {
|
|
111
|
+
const name = path.basename(real);
|
|
112
|
+
if (!KNOWN_NAME.test(name))
|
|
113
|
+
throw new Error(`Gallery source ${label}: unsupported source type (${name}).`);
|
|
114
|
+
let root = path.dirname(real);
|
|
115
|
+
if (name === "tutorial-manifest.json") {
|
|
116
|
+
if (path.basename(root) !== "artifacts") {
|
|
117
|
+
throw new Error(`Gallery source ${label}: a tutorial manifest must be <run-root>/artifacts/tutorial-manifest.json.`);
|
|
118
|
+
}
|
|
119
|
+
root = path.dirname(root);
|
|
120
|
+
}
|
|
121
|
+
await addFile(real, root, true);
|
|
122
|
+
return found;
|
|
123
|
+
}
|
|
124
|
+
if (!info.isDirectory())
|
|
125
|
+
throw new Error(`Gallery source ${label} is neither a file nor a directory.`);
|
|
126
|
+
const tutorialManifest = path.join(real, ...TUTORIAL_MANIFEST_RELATIVE.split("/"));
|
|
127
|
+
let hasTutorial = false;
|
|
128
|
+
try {
|
|
129
|
+
hasTutorial = (await stat(tutorialManifest)).isFile();
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
hasTutorial = false;
|
|
133
|
+
}
|
|
134
|
+
if (hasTutorial)
|
|
135
|
+
await addFile(tutorialManifest, real, true);
|
|
136
|
+
const names = (await readdir(real)).filter((name) => name !== "tutorial-manifest.json" && KNOWN_NAME.test(name)).sort();
|
|
137
|
+
for (const name of names) {
|
|
138
|
+
const file = path.join(real, name);
|
|
139
|
+
if ((await stat(file)).isFile())
|
|
140
|
+
await addFile(file, real, false);
|
|
141
|
+
}
|
|
142
|
+
if (found.length === 0) {
|
|
143
|
+
throw new Error(`Gallery source ${label}: no supported artifact family was recognized at this root.`);
|
|
144
|
+
}
|
|
145
|
+
return found;
|
|
146
|
+
}
|
|
147
|
+
async function linkIfPresent(root, realOutDir, name, kind) {
|
|
148
|
+
const verified = await verifyArtifactFile(root, name, kind);
|
|
149
|
+
return verified.ok ? outputRelativePath(realOutDir, verified.absolutePath) : null;
|
|
150
|
+
}
|
|
151
|
+
async function readPluginFamily(found, realOutDir) {
|
|
152
|
+
const json = outputRelativePath(realOutDir, found.file);
|
|
153
|
+
if (json === null)
|
|
154
|
+
throw new Error("Gallery source report has no relative link from the gallery directory.");
|
|
155
|
+
const html = await linkIfPresent(found.root, realOutDir, "report.html", "text");
|
|
156
|
+
const text = await linkIfPresent(found.root, realOutDir, "report.txt", "text");
|
|
157
|
+
const execution = await linkIfPresent(found.root, realOutDir, "warm-index-execution.json", "json");
|
|
158
|
+
const plugin = found.data.plugin;
|
|
159
|
+
const metadata = found.data.metadata;
|
|
160
|
+
const target = isObject(found.data.target) ? found.data.target : {};
|
|
161
|
+
const runStatus = str(metadata.status) ?? "unknown";
|
|
162
|
+
const status = runStatus === "completed" ? "pass" : runStatus === "skipped" ? "skipped" : "warning";
|
|
163
|
+
const warnings = [];
|
|
164
|
+
if (status === "warning")
|
|
165
|
+
warnings.push(`Experiment run status is ${runStatus}.`);
|
|
166
|
+
if (html === null)
|
|
167
|
+
warnings.push("HTML report unavailable: report.html was not found at the source root.");
|
|
168
|
+
const metrics = [{ id: "run-status", label: "Run status", value: runStatus }];
|
|
169
|
+
if (Array.isArray(found.data.metrics))
|
|
170
|
+
metrics.push({ id: "metric-count", label: "Metric count", value: found.data.metrics.length });
|
|
171
|
+
const campaign = found.family === "warm-index-campaign";
|
|
172
|
+
const tags = [campaign ? "warm-index-campaign" : "experiment-report"];
|
|
173
|
+
if (target.privacyProjection === "external-local-redacted")
|
|
174
|
+
tags.push("external-local-redacted");
|
|
175
|
+
const artifactPaths = [text, execution].filter((value) => value !== null);
|
|
176
|
+
return {
|
|
177
|
+
id: itemId(found.family, str(metadata.runId), found.file),
|
|
178
|
+
title: `${str(plugin.name) ?? str(plugin.id)} report`,
|
|
179
|
+
description: `Plugin experiment report for run ${str(metadata.runId)} (plugin ${str(plugin.id)}).`,
|
|
180
|
+
kind: campaign ? "warm-index-campaign-report" : "plugin-experiment-report",
|
|
181
|
+
status,
|
|
182
|
+
htmlPath: html ?? "",
|
|
183
|
+
summaryPath: json,
|
|
184
|
+
...(artifactPaths.length > 0 ? { artifactPaths } : {}),
|
|
185
|
+
sourceFamily: found.family,
|
|
186
|
+
sourceId: `${str(plugin.id)}/${str(metadata.runId)}`,
|
|
187
|
+
tags,
|
|
188
|
+
metrics,
|
|
189
|
+
warnings
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function simpleItem(found, realOutDir, fields) {
|
|
193
|
+
const summary = outputRelativePath(realOutDir, found.file);
|
|
194
|
+
if (summary === null)
|
|
195
|
+
throw new Error(`Gallery source ${path.basename(found.file)} has no relative link from the gallery directory.`);
|
|
196
|
+
const { identity, ...rest } = fields;
|
|
197
|
+
return { ...rest, id: itemId(found.family, identity, found.file), htmlPath: "", summaryPath: summary, sourceFamily: found.family, sourceId: identity };
|
|
198
|
+
}
|
|
199
|
+
function readAudit(found, realOutDir) {
|
|
200
|
+
const data = found.data;
|
|
201
|
+
if (data.schemaVersion !== SUPPORTED_AUDIT_SCHEMA) {
|
|
202
|
+
throw new Error(`Gallery source ${path.basename(found.file)}: unsupported audit report schemaVersion ${JSON.stringify(data.schemaVersion)}.`);
|
|
203
|
+
}
|
|
204
|
+
const exit = isObject(data.exit) ? data.exit : null;
|
|
205
|
+
const summary = isObject(data.summary) ? data.summary : {};
|
|
206
|
+
const metadata = data.metadata;
|
|
207
|
+
const breached = exit?.failOnBreached;
|
|
208
|
+
const warnings = [];
|
|
209
|
+
let status = "pass";
|
|
210
|
+
if (exit === null || typeof breached !== "boolean") {
|
|
211
|
+
status = "warning";
|
|
212
|
+
warnings.push("Audit exit status is not recorded in this report.");
|
|
213
|
+
}
|
|
214
|
+
else if (breached) {
|
|
215
|
+
status = "warning";
|
|
216
|
+
warnings.push("The audit fail-on threshold was breached.");
|
|
217
|
+
}
|
|
218
|
+
const metrics = [];
|
|
219
|
+
const total = num(summary.totalIssues);
|
|
220
|
+
if (total !== null)
|
|
221
|
+
metrics.push({ id: "total-issues", label: "Total issues", value: total });
|
|
222
|
+
const label = str(exit?.finalVerdictLabel);
|
|
223
|
+
if (label)
|
|
224
|
+
metrics.push({ id: "verdict", label: "Audit verdict", value: label });
|
|
225
|
+
const types = Array.isArray(metadata.auditTypes) ? metadata.auditTypes.filter((t) => typeof t === "string").join(", ") : "audit";
|
|
226
|
+
return simpleItem(found, realOutDir, {
|
|
227
|
+
identity: `code-rot-audit/${str(metadata.generatedAt) ?? "unknown"}`,
|
|
228
|
+
title: "Code-rot audit report",
|
|
229
|
+
description: `Standalone audit report (${types}).`,
|
|
230
|
+
kind: "audit-report",
|
|
231
|
+
status,
|
|
232
|
+
tags: ["audit"],
|
|
233
|
+
metrics,
|
|
234
|
+
warnings
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
function verdictStatus(verdict) {
|
|
238
|
+
return verdict === "ready-for-release-preparation" ? "pass" : "warning";
|
|
239
|
+
}
|
|
240
|
+
function readSecurity(found, realOutDir) {
|
|
241
|
+
const data = found.data;
|
|
242
|
+
if (data.schemaVersion !== SUPPORTED_SECURITY_SCHEMA) {
|
|
243
|
+
throw new Error(`Gallery source ${path.basename(found.file)}: unsupported security report schemaVersion ${JSON.stringify(data.schemaVersion)}.`);
|
|
244
|
+
}
|
|
245
|
+
const verdict = str(data.verdict);
|
|
246
|
+
const summary = data.summary;
|
|
247
|
+
const metadata = isObject(data.metadata) ? data.metadata : {};
|
|
248
|
+
const metrics = [{ id: "verdict", label: "Verdict", value: verdict }];
|
|
249
|
+
for (const [id, label] of [["totalChecks", "Total checks"], ["failed", "Failed checks"], ["totalFindings", "Total findings"], ["blockerFindings", "Blocker findings"]]) {
|
|
250
|
+
const value = num(summary[id]);
|
|
251
|
+
if (value !== null)
|
|
252
|
+
metrics.push({ id, label, value });
|
|
253
|
+
}
|
|
254
|
+
const status = verdictStatus(verdict);
|
|
255
|
+
return simpleItem(found, realOutDir, {
|
|
256
|
+
identity: `${path.basename(found.file, ".json")}/${str(metadata.generatedAt) ?? "unknown"}`,
|
|
257
|
+
title: "Security validation report",
|
|
258
|
+
description: "Standalone security-validation report.",
|
|
259
|
+
kind: "security-validation-report",
|
|
260
|
+
status,
|
|
261
|
+
tags: ["security"],
|
|
262
|
+
metrics,
|
|
263
|
+
warnings: status === "pass" ? [] : [`Security verdict is ${verdict}.`]
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
function readAndroid(found, realOutDir) {
|
|
267
|
+
const data = found.data;
|
|
268
|
+
const verdict = str(data.verdict);
|
|
269
|
+
const status = verdictStatus(verdict);
|
|
270
|
+
const metadata = isObject(data.metadata) ? data.metadata : {};
|
|
271
|
+
return simpleItem(found, realOutDir, {
|
|
272
|
+
identity: `${path.basename(found.file, ".json")}/${str(metadata.generatedAt) ?? "unknown"}`,
|
|
273
|
+
title: "Android validation report",
|
|
274
|
+
description: "Static Android/mobile validation report.",
|
|
275
|
+
kind: "android-validation-report",
|
|
276
|
+
status,
|
|
277
|
+
tags: ["android", "security"],
|
|
278
|
+
metrics: [{ id: "verdict", label: "Verdict", value: verdict }],
|
|
279
|
+
warnings: status === "pass" ? [] : [`Android verdict is ${verdict}.`]
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
function readExperiment(found, realOutDir) {
|
|
283
|
+
const data = found.data;
|
|
284
|
+
const completed = num(data.completedRuns);
|
|
285
|
+
const total = num(data.totalRuns);
|
|
286
|
+
const problem = ["failedRuns", "skippedRuns", "unavailableRuns", "limitReachedRuns", "timeoutRuns", "invalidOutputRuns"].reduce((sum, key) => sum + (num(data[key]) ?? 0), 0);
|
|
287
|
+
const status = completed === 0 ? "skipped" : problem > 0 ? "warning" : "pass";
|
|
288
|
+
return simpleItem(found, realOutDir, {
|
|
289
|
+
identity: `experiment/${str(data.generatedAt) ?? "unknown"}`,
|
|
290
|
+
title: "Controlled experiment summary",
|
|
291
|
+
description: "Controlled experiment summary read from experiment-summary.json.",
|
|
292
|
+
kind: "controlled-experiment",
|
|
293
|
+
status,
|
|
294
|
+
tags: ["experiment"],
|
|
295
|
+
metrics: [
|
|
296
|
+
{ id: "total-runs", label: "Total runs", value: total },
|
|
297
|
+
{ id: "completed-runs", label: "Completed runs", value: completed }
|
|
298
|
+
],
|
|
299
|
+
warnings: status === "pass" ? [] : [`${completed} of ${total} controlled runs completed.`]
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
// ---------------------------------------------------------------------------
|
|
303
|
+
// Prompt-template comparison (v0.9.3)
|
|
304
|
+
// ---------------------------------------------------------------------------
|
|
305
|
+
async function readPromptComparison(found, realOutDir) {
|
|
306
|
+
const data = found.data;
|
|
307
|
+
const name = path.basename(found.file);
|
|
308
|
+
if (data.schemaVersion !== PROMPT_COMPARISON_SCHEMA) {
|
|
309
|
+
throw new Error(`Gallery source ${name}: unsupported prompt comparison schemaVersion ${JSON.stringify(data.schemaVersion)}.`);
|
|
310
|
+
}
|
|
311
|
+
const comparison = data.comparison;
|
|
312
|
+
const baseline = data.baseline;
|
|
313
|
+
const candidate = data.candidate;
|
|
314
|
+
const cohort = data.cohort;
|
|
315
|
+
const invalid = data.invalidOutput;
|
|
316
|
+
const improvement = data.behavioralImprovement;
|
|
317
|
+
const bad = (what) => {
|
|
318
|
+
throw new Error(`Gallery source ${name}: malformed prompt comparison (${what}).`);
|
|
319
|
+
};
|
|
320
|
+
if (!isObject(comparison) || (comparison.status !== "COMPARABLE" && comparison.status !== "INCOMPARABLE"))
|
|
321
|
+
bad("comparison");
|
|
322
|
+
if (!isObject(baseline) || !isObject(candidate))
|
|
323
|
+
bad("baseline/candidate");
|
|
324
|
+
if (!isObject(cohort) || num(cohort.matchedUnitCount) === null)
|
|
325
|
+
bad("cohort");
|
|
326
|
+
if (!isObject(invalid) || !Array.isArray(invalid.strata))
|
|
327
|
+
bad("invalidOutput");
|
|
328
|
+
if (!isObject(improvement) || (improvement.status !== "VERIFIED" && improvement.status !== "NOT_VERIFIED"))
|
|
329
|
+
bad("behavioralImprovement");
|
|
330
|
+
const comparisonObj = comparison;
|
|
331
|
+
const improvementObj = improvement;
|
|
332
|
+
const status = comparisonObj.status;
|
|
333
|
+
const base = baseline;
|
|
334
|
+
const cand = candidate;
|
|
335
|
+
const coh = cohort;
|
|
336
|
+
const overall = isObject(invalid.overall) ? invalid.overall : null;
|
|
337
|
+
const metrics = [
|
|
338
|
+
{ id: "comparison-status", label: "Comparison status", value: status },
|
|
339
|
+
{ id: "baseline-template", label: "Baseline template", value: str(base.templateId) ?? "unverified" },
|
|
340
|
+
{ id: "candidate-template", label: "Candidate template", value: str(cand.templateId) ?? "unverified" },
|
|
341
|
+
{ id: "matched-units", label: "Matched units", value: num(coh.matchedUnitCount) },
|
|
342
|
+
{ id: "eligible-units", label: "Eligible matched units", value: num(coh.eligibleMatchedUnitCount) ?? 0 },
|
|
343
|
+
{ id: "behavioral-improvement", label: "Behavioral improvement", value: String(improvement.status) }
|
|
344
|
+
];
|
|
345
|
+
const rateOf = (side) => (isObject(side) ? num(side.ratePercent) : null);
|
|
346
|
+
if (overall) {
|
|
347
|
+
const b = rateOf(overall.baseline);
|
|
348
|
+
const c = rateOf(overall.candidate);
|
|
349
|
+
if (b !== null)
|
|
350
|
+
metrics.push({ id: "baseline-invalid-output-rate", label: "Baseline invalid-output rate", value: Number(b.toFixed(2)), unit: "%" });
|
|
351
|
+
if (c !== null)
|
|
352
|
+
metrics.push({ id: "candidate-invalid-output-rate", label: "Candidate invalid-output rate", value: Number(c.toFixed(2)), unit: "%" });
|
|
353
|
+
const diff = num(overall.percentagePointDifference);
|
|
354
|
+
if (diff !== null)
|
|
355
|
+
metrics.push({ id: "invalid-output-difference", label: "Invalid-output difference (baseline minus candidate)", value: Number(diff.toFixed(2)), unit: "pp" });
|
|
356
|
+
}
|
|
357
|
+
const warnings = [];
|
|
358
|
+
if (status === "INCOMPARABLE") {
|
|
359
|
+
const reasons = Array.isArray(comparisonObj.incomparabilityReasons) ? comparisonObj.incomparabilityReasons.filter((r) => typeof r === "string") : [];
|
|
360
|
+
for (const reason of reasons.slice(0, 5))
|
|
361
|
+
warnings.push(`Incomparable: ${bounded(reason)}`);
|
|
362
|
+
if (reasons.length === 0)
|
|
363
|
+
warnings.push("Incomparable: no reason recorded.");
|
|
364
|
+
}
|
|
365
|
+
if (improvementObj.status !== "VERIFIED")
|
|
366
|
+
warnings.push(`Behavioral improvement is ${String(improvementObj.status)}: ${bounded(str(improvementObj.reason) ?? "no reason recorded")}`);
|
|
367
|
+
if (Array.isArray(data.limitations)) {
|
|
368
|
+
for (const limitation of data.limitations.filter((l) => typeof l === "string").slice(0, 3))
|
|
369
|
+
warnings.push(`Limitation: ${bounded(limitation)}`);
|
|
370
|
+
}
|
|
371
|
+
const html = await linkIfPresent(found.root, realOutDir, "prompt-template-comparison.html", "text");
|
|
372
|
+
const text = await linkIfPresent(found.root, realOutDir, "prompt-template-comparison.txt", "text");
|
|
373
|
+
if (html === null)
|
|
374
|
+
warnings.push("HTML comparison report unavailable: prompt-template-comparison.html was not found.");
|
|
375
|
+
const json = outputRelativePath(realOutDir, found.file);
|
|
376
|
+
if (json === null)
|
|
377
|
+
throw new Error(`Gallery source ${name} has no relative link from the gallery directory.`);
|
|
378
|
+
const identity = `${str(base.runId) ?? "baseline"}-vs-${str(cand.runId) ?? "candidate"}`;
|
|
379
|
+
return {
|
|
380
|
+
id: itemId("prompt-template-comparison", identity, found.file),
|
|
381
|
+
title: "Prompt template comparison",
|
|
382
|
+
description: `Baseline ${str(base.templateId) ?? "unverified"} versus candidate ${str(cand.templateId) ?? "unverified"} (${status}).`,
|
|
383
|
+
kind: "prompt-template-comparison",
|
|
384
|
+
status: status === "COMPARABLE" ? "pass" : "warning",
|
|
385
|
+
htmlPath: html ?? "",
|
|
386
|
+
summaryPath: json,
|
|
387
|
+
...(text ? { artifactPaths: [text] } : {}),
|
|
388
|
+
sourceFamily: "prompt-template-comparison",
|
|
389
|
+
sourceId: identity,
|
|
390
|
+
tags: ["prompt-comparison"],
|
|
391
|
+
metrics,
|
|
392
|
+
warnings
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
// ---------------------------------------------------------------------------
|
|
396
|
+
// Tutorial runs
|
|
397
|
+
// ---------------------------------------------------------------------------
|
|
398
|
+
function validateTutorialManifest(data, label) {
|
|
399
|
+
if (data.schemaVersion !== TUTORIAL_MANIFEST_SCHEMA_VERSION) {
|
|
400
|
+
throw new Error(`Gallery source ${label}: unsupported tutorial manifest schemaVersion ${JSON.stringify(data.schemaVersion)}.`);
|
|
401
|
+
}
|
|
402
|
+
const bad = (what) => {
|
|
403
|
+
throw new Error(`Gallery source ${label}: malformed tutorial manifest (${what}).`);
|
|
404
|
+
};
|
|
405
|
+
const scenario = data.scenario;
|
|
406
|
+
const run = data.run;
|
|
407
|
+
if (!isObject(scenario) || !str(scenario.id) || !str(scenario.title))
|
|
408
|
+
bad("scenario");
|
|
409
|
+
if (!isObject(run) || !str(run.id) || !str(run.status) || !str(run.startedAt) || !str(run.endedAt))
|
|
410
|
+
bad("run");
|
|
411
|
+
if (!Array.isArray(data.steps))
|
|
412
|
+
bad("steps");
|
|
413
|
+
if (!Array.isArray(data.artifacts))
|
|
414
|
+
bad("artifacts");
|
|
415
|
+
for (const key of ["warnings", "cleanupErrors"]) {
|
|
416
|
+
const value = data[key];
|
|
417
|
+
if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string"))
|
|
418
|
+
bad(key);
|
|
419
|
+
}
|
|
420
|
+
return data;
|
|
421
|
+
}
|
|
422
|
+
async function readTutorial(found, realOutDir) {
|
|
423
|
+
const manifest = validateTutorialManifest(found.data, path.basename(found.root));
|
|
424
|
+
const warnings = [
|
|
425
|
+
...manifest.warnings.map((warning) => bounded(warning)),
|
|
426
|
+
...manifest.cleanupErrors.map((error) => `Cleanup error: ${bounded(error)}`)
|
|
427
|
+
];
|
|
428
|
+
const records = manifest.artifacts.filter(isObject);
|
|
429
|
+
const resolveKind = async (kind, media, human) => {
|
|
430
|
+
const matching = records.filter((record) => record.kind === kind);
|
|
431
|
+
if (matching.length === 0) {
|
|
432
|
+
warnings.push(`${human} unavailable: not recorded in the tutorial manifest.`);
|
|
433
|
+
return [];
|
|
434
|
+
}
|
|
435
|
+
const links = [];
|
|
436
|
+
for (const record of matching) {
|
|
437
|
+
if (record.status !== "written") {
|
|
438
|
+
warnings.push(`${human} unavailable: artifact status is ${record.status ?? "unknown"}${record.error ? ` (${bounded(record.error)})` : ""}.`);
|
|
439
|
+
if (record.status === "failed")
|
|
440
|
+
degraded = true;
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
const verified = await verifyArtifactFile(found.root, record.path, media);
|
|
444
|
+
if (!verified.ok) {
|
|
445
|
+
warnings.push(`${human} unavailable: ${verified.reason}.`);
|
|
446
|
+
degraded = true;
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
const link = outputRelativePath(realOutDir, verified.absolutePath);
|
|
450
|
+
if (link === null) {
|
|
451
|
+
warnings.push(`${human} unavailable: no relative link exists from the gallery directory.`);
|
|
452
|
+
degraded = true;
|
|
453
|
+
}
|
|
454
|
+
else
|
|
455
|
+
links.push(link);
|
|
456
|
+
}
|
|
457
|
+
return links;
|
|
458
|
+
};
|
|
459
|
+
let degraded = false;
|
|
460
|
+
const videos = await resolveKind("video", "video", "Video");
|
|
461
|
+
const screenshots = await resolveKind("screenshot", "screenshot", "Screenshot");
|
|
462
|
+
const markdown = await resolveKind("markdown", "markdown", "Markdown");
|
|
463
|
+
const subtitles = [...(await resolveKind("srt", "text", "SRT subtitles")), ...(await resolveKind("vtt", "text", "VTT subtitles"))];
|
|
464
|
+
const runStatus = manifest.run.status;
|
|
465
|
+
// Media the manifest truthfully records as absent does not downgrade a passed run; a recorded artifact that
|
|
466
|
+
// failed or cannot be verified does.
|
|
467
|
+
const status = runStatus === "passed" && !degraded && manifest.cleanupErrors.length === 0 ? "pass" : "warning";
|
|
468
|
+
if (runStatus !== "passed")
|
|
469
|
+
warnings.unshift(`Tutorial run status is ${runStatus}; the run did not pass.`);
|
|
470
|
+
const summary = outputRelativePath(realOutDir, found.file);
|
|
471
|
+
const duration = num(manifest.run.durationMs);
|
|
472
|
+
return {
|
|
473
|
+
id: itemId("tutorial-run", `${manifest.scenario.id}-${manifest.run.id}`, found.file),
|
|
474
|
+
title: manifest.scenario.title,
|
|
475
|
+
description: manifest.scenario.description ?? `Tutorial run ${manifest.run.id}.`,
|
|
476
|
+
kind: "tutorial-run",
|
|
477
|
+
status,
|
|
478
|
+
htmlPath: "",
|
|
479
|
+
...(summary === null ? {} : { summaryPath: summary }),
|
|
480
|
+
...(screenshots[0] ? { screenshotPath: screenshots[0] } : {}),
|
|
481
|
+
artifactPaths: subtitles,
|
|
482
|
+
sourceFamily: "tutorial-run",
|
|
483
|
+
sourceId: `${manifest.scenario.id}/${manifest.run.id}`,
|
|
484
|
+
...(videos[0] ? { videoPath: videos[0] } : {}),
|
|
485
|
+
...(markdown[0] ? { markdownPath: markdown[0] } : {}),
|
|
486
|
+
...(screenshots.length > 0 ? { screenshotPaths: screenshots } : {}),
|
|
487
|
+
tags: ["tutorial"],
|
|
488
|
+
metrics: [
|
|
489
|
+
{ id: "run-status", label: "Run status", value: runStatus },
|
|
490
|
+
{ id: "step-count", label: "Step count", value: manifest.steps.length },
|
|
491
|
+
...(duration !== null ? [{ id: "duration-ms", label: "Duration", value: duration, unit: "ms" }] : [])
|
|
492
|
+
],
|
|
493
|
+
warnings
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
// ---------------------------------------------------------------------------
|
|
497
|
+
/**
|
|
498
|
+
* Reads every explicitly supplied source, in the supplied order. Repeated sources and repeated artifacts
|
|
499
|
+
* (same physical file and family) produce one item. Any unreadable, malformed, unsupported or ambiguous
|
|
500
|
+
* source fails the whole build; no placeholder success item is ever produced.
|
|
501
|
+
*/
|
|
502
|
+
export async function readGallerySources(options) {
|
|
503
|
+
const items = [];
|
|
504
|
+
const seen = new Set();
|
|
505
|
+
for (const source of options.sources) {
|
|
506
|
+
if (source.trim() === "")
|
|
507
|
+
throw new Error("Gallery source path must not be empty.");
|
|
508
|
+
if (looksLikeUrl(source))
|
|
509
|
+
throw new Error(`Gallery source ${source}: URLs are not supported; supply a local path.`);
|
|
510
|
+
for (const entry of await collect(source)) {
|
|
511
|
+
const key = `${entry.family}|${await realpath(entry.file)}`;
|
|
512
|
+
if (seen.has(key))
|
|
513
|
+
continue;
|
|
514
|
+
seen.add(key);
|
|
515
|
+
switch (entry.family) {
|
|
516
|
+
case "tutorial-run":
|
|
517
|
+
items.push(await readTutorial(entry, options.realOutDir));
|
|
518
|
+
break;
|
|
519
|
+
case "plugin-experiment-report":
|
|
520
|
+
case "warm-index-campaign":
|
|
521
|
+
items.push(await readPluginFamily(entry, options.realOutDir));
|
|
522
|
+
break;
|
|
523
|
+
case "audit-report":
|
|
524
|
+
items.push(readAudit(entry, options.realOutDir));
|
|
525
|
+
break;
|
|
526
|
+
case "security-validation-report":
|
|
527
|
+
items.push(readSecurity(entry, options.realOutDir));
|
|
528
|
+
break;
|
|
529
|
+
case "android-validation-report":
|
|
530
|
+
items.push(readAndroid(entry, options.realOutDir));
|
|
531
|
+
break;
|
|
532
|
+
case "controlled-experiment":
|
|
533
|
+
items.push(readExperiment(entry, options.realOutDir));
|
|
534
|
+
break;
|
|
535
|
+
case "prompt-template-comparison":
|
|
536
|
+
items.push(await readPromptComparison(entry, options.realOutDir));
|
|
537
|
+
break;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return { items };
|
|
542
|
+
}
|