@farming-labs/docs 0.2.53 → 0.2.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-BXvi0uhS.mjs +623 -0
- package/dist/{agent-BMxMpEa0.mjs → agent-DXkXi-pS.mjs} +590 -19
- package/dist/agent-evals-BD17jOqL.mjs +1166 -0
- package/dist/{agent-export--X9Go-wj.mjs → agent-export-n-a0GbeU.mjs} +9 -8
- package/dist/agent-scope-B8lptqCd.mjs +283 -0
- package/dist/agent-surface-drift-LS8zQgbq.mjs +1429 -0
- package/dist/{agents-CW-zTNKP.mjs → agents-BJaRQimo.mjs} +7 -6
- package/dist/{analytics-BZEwTK-8.mjs → analytics-Bx44lg6d.mjs} +1 -1
- package/dist/cli/index.mjs +29 -29
- package/dist/client/react.d.mts +1 -1
- package/dist/client/react.mjs +1 -1
- package/dist/{cloud-zs0qAnKJ.mjs → cloud-HWFlbfLJ.mjs} +4 -4
- package/dist/{cloud-ask-ai-BFlx8VMO.mjs → cloud-ask-ai-1k0q6OAH.mjs} +1 -1
- package/dist/{cloud-ask-ai-CMzYVzVX.d.mts → cloud-ask-ai-466g6uAi.d.mts} +1 -1
- package/dist/code-blocks-C9awWzEQ.mjs +871 -0
- package/dist/codeblocks-BzfkREEC.mjs +250 -0
- package/dist/{config-DhIyFsy9.mjs → config-CydaZ5PB.mjs} +52 -11
- package/dist/{dev-DFeVU2Ui.mjs → dev-BA2kRuCn.mjs} +3 -3
- package/dist/docs-cloud-server.d.mts +2 -2
- package/dist/docs-cloud-server.mjs +2 -2
- package/dist/{doctor-CkM5tKdD.mjs → doctor-CQpQ2zZm.mjs} +150 -32
- package/dist/{downgrade-Dj_KXV11.mjs → downgrade-BStNhyCc.mjs} +2 -2
- package/dist/index.d.mts +36 -17
- package/dist/index.mjs +6 -7
- package/dist/{init-Cmz-oqnp.mjs → init-gnHQ_Pz9.mjs} +3 -3
- package/dist/{mcp-B4Q1PN93.mjs → mcp-DnxZZWpY.mjs} +6 -5
- package/dist/mcp.d.mts +22 -4
- package/dist/mcp.mjs +520 -43
- package/dist/{package-version-DxnTZH4m.mjs → package-version-DRIc08EU.mjs} +1 -1
- package/dist/{reading-time-BuK2YL23.mjs → reading-time-BrTd3DIh.mjs} +2 -3
- package/dist/{review-BEsbm0l2.mjs → review-B6gyEFkD.mjs} +149 -14
- package/dist/{robots-BOwkq9du.mjs → robots-4BUJxlvV.mjs} +4 -5
- package/dist/{robots-C6uQG-fS.mjs → robots-DAptQnkx.mjs} +1 -1
- package/dist/{search-CJNLkdjX.mjs → search-CJIC1Cqo.mjs} +1 -2
- package/dist/{search-CtAXkk_J.mjs → search-CbPm2x5g.mjs} +6 -5
- package/dist/{search-CTu6Bvq-.d.mts → search-CfWvmVYA.d.mts} +1 -1
- package/dist/server.d.mts +108 -5
- package/dist/server.mjs +9 -9
- package/dist/{sitemap-OPb8EdNe.mjs → sitemap-D6nP3J1Q.mjs} +8 -7
- package/dist/{sitemap-server-DnilMEiZ.mjs → sitemap-server-wsNLyVkb.mjs} +1 -1
- package/dist/{types-CdKu8KmO.d.mts → types-BVgucdVm.d.mts} +77 -1
- package/dist/{upgrade-BNdBB9tQ.mjs → upgrade-CXYRNV0C.mjs} +2 -2
- package/package.json +1 -1
- package/dist/agent-2t4kZLGo.mjs +0 -10
- package/dist/agent-contract-b0akaPcg.mjs +0 -573
- package/dist/codeblocks-DWM4IJPL.mjs +0 -1710
- package/dist/review-BHdy75TE.mjs +0 -541
- /package/dist/{cloud-analytics-Dlk_byos.mjs → cloud-analytics-CSyFE6SS.mjs} +0 -0
- /package/dist/{errors-BxfYhp1L.mjs → errors-BFUtdZfC.mjs} +0 -0
- /package/dist/{templates-Bty9M6lA.mjs → templates-1Cod8KrJ.mjs} +0 -0
- /package/dist/{utils-CMZTjRQT.mjs → utils-DBCCkkJS.mjs} +0 -0
|
@@ -0,0 +1,1166 @@
|
|
|
1
|
+
import { a as normalizeAgentScopeValues, i as normalizeAgentLocale, n as agentVersionConstraintsOverlap, o as normalizeAgentVersion, r as normalizeAgentFramework, t as agentVersionConstraintMatches } from "./agent-scope-B8lptqCd.mjs";
|
|
2
|
+
import { buildDocsMcpContext } from "./mcp.mjs";
|
|
3
|
+
import { n as planCodeBlockTargets, r as resolveDocsCodeBlocksValidateConfig, t as extractCodeBlocksFromMarkdown } from "./code-blocks-C9awWzEQ.mjs";
|
|
4
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { dirname, isAbsolute, join, relative } from "node:path";
|
|
6
|
+
import { createJiti } from "jiti";
|
|
7
|
+
|
|
8
|
+
//#region src/review.ts
|
|
9
|
+
const DEFAULT_DOCS_REVIEW_WORKFLOW_PATH = ".github/workflows/docs-review.yml";
|
|
10
|
+
const DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW_PATH = ".github/workflows/docs-review-reusable.yml";
|
|
11
|
+
const DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW = "farming-labs/docs/.github/workflows/docs-review-reusable.yml@main";
|
|
12
|
+
const LOCAL_DOCS_REVIEW_REUSABLE_WORKFLOW = "./.github/workflows/docs-review-reusable.yml";
|
|
13
|
+
const DEFAULT_DOCS_REVIEW_PNPM_VERSION = "10";
|
|
14
|
+
const DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD = 80;
|
|
15
|
+
const DEFAULT_DOCS_REVIEW_CI_NAME = "docs-review";
|
|
16
|
+
const DEFAULT_REVIEW_RULES = {
|
|
17
|
+
brokenLinks: "error",
|
|
18
|
+
frontmatter: "error",
|
|
19
|
+
duplicateSlugs: "error",
|
|
20
|
+
invalidMdx: "error",
|
|
21
|
+
configExamples: "warn",
|
|
22
|
+
codeFenceMetadata: "warn",
|
|
23
|
+
runnableMetadata: "warn",
|
|
24
|
+
agentContext: "warn",
|
|
25
|
+
commandHealth: "warn",
|
|
26
|
+
relatedCoverage: "suggestion",
|
|
27
|
+
configConfidence: "warn",
|
|
28
|
+
agentSurfaceDrift: "error",
|
|
29
|
+
goldenTasks: "warn"
|
|
30
|
+
};
|
|
31
|
+
const DEFAULT_REVIEW_WEIGHTS = {
|
|
32
|
+
error: 20,
|
|
33
|
+
warn: 8,
|
|
34
|
+
suggestion: 2
|
|
35
|
+
};
|
|
36
|
+
const REVIEW_SEVERITIES = new Set([
|
|
37
|
+
"off",
|
|
38
|
+
"suggestion",
|
|
39
|
+
"warn",
|
|
40
|
+
"error"
|
|
41
|
+
]);
|
|
42
|
+
const REVIEW_CI_MODES = new Set([
|
|
43
|
+
"off",
|
|
44
|
+
"warn",
|
|
45
|
+
"block"
|
|
46
|
+
]);
|
|
47
|
+
const FILE_EXTS = [
|
|
48
|
+
"tsx",
|
|
49
|
+
"ts",
|
|
50
|
+
"jsx",
|
|
51
|
+
"js"
|
|
52
|
+
];
|
|
53
|
+
function resolveDocsReviewConfig(review) {
|
|
54
|
+
if (review === false) return {
|
|
55
|
+
enabled: false,
|
|
56
|
+
ci: {
|
|
57
|
+
enabled: false,
|
|
58
|
+
name: DEFAULT_DOCS_REVIEW_CI_NAME,
|
|
59
|
+
mode: "off",
|
|
60
|
+
annotations: false,
|
|
61
|
+
comment: false
|
|
62
|
+
},
|
|
63
|
+
score: {
|
|
64
|
+
threshold: DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD,
|
|
65
|
+
weights: DEFAULT_REVIEW_WEIGHTS
|
|
66
|
+
},
|
|
67
|
+
rules: DEFAULT_REVIEW_RULES
|
|
68
|
+
};
|
|
69
|
+
const objectConfig = review && typeof review === "object" ? review : {};
|
|
70
|
+
const ciConfig = objectConfig.ci;
|
|
71
|
+
const ciObject = ciConfig && typeof ciConfig === "object" ? ciConfig : {};
|
|
72
|
+
const configuredMode = ciObject.mode;
|
|
73
|
+
const mode = configuredMode && REVIEW_CI_MODES.has(configuredMode) ? configuredMode : "warn";
|
|
74
|
+
const enabled = objectConfig.enabled !== false;
|
|
75
|
+
const ciEnabled = enabled && ciConfig !== false && ciObject.enabled !== false && mode !== "off";
|
|
76
|
+
return {
|
|
77
|
+
enabled,
|
|
78
|
+
ci: {
|
|
79
|
+
enabled: ciEnabled,
|
|
80
|
+
name: normalizeCiName(ciObject.name),
|
|
81
|
+
mode: ciEnabled ? mode : "off",
|
|
82
|
+
annotations: ciObject.annotations !== false,
|
|
83
|
+
comment: ciObject.comment !== false
|
|
84
|
+
},
|
|
85
|
+
score: {
|
|
86
|
+
threshold: clampScoreThreshold(objectConfig.score?.threshold),
|
|
87
|
+
weights: {
|
|
88
|
+
error: normalizeWeight(objectConfig.score?.weights?.error, DEFAULT_REVIEW_WEIGHTS.error),
|
|
89
|
+
warn: normalizeWeight(objectConfig.score?.weights?.warn, DEFAULT_REVIEW_WEIGHTS.warn),
|
|
90
|
+
suggestion: normalizeWeight(objectConfig.score?.weights?.suggestion, DEFAULT_REVIEW_WEIGHTS.suggestion)
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
rules: {
|
|
94
|
+
...DEFAULT_REVIEW_RULES,
|
|
95
|
+
...normalizeRules(objectConfig.rules)
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function buildDocsReviewWorkflow(options = {}) {
|
|
100
|
+
const packageManager = options.packageManager ?? "npm";
|
|
101
|
+
const ciName = normalizeCiName(options.ciName);
|
|
102
|
+
const projectDir = normalizeProjectDir(options.projectDir);
|
|
103
|
+
const configPath = options.configPath ?? "docs.config.ts";
|
|
104
|
+
const reusableWorkflow = options.reusableWorkflow ?? DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW;
|
|
105
|
+
const filters = normalizePathFilters(options.pathFilters);
|
|
106
|
+
const workflowInputs = [
|
|
107
|
+
["check-name", ciName],
|
|
108
|
+
["config", configPath],
|
|
109
|
+
["working-directory", projectDir],
|
|
110
|
+
["package-manager", packageManager],
|
|
111
|
+
options.pnpmVersion ? ["pnpm-version", options.pnpmVersion] : void 0,
|
|
112
|
+
options.buildCommand ? ["build-command", options.buildCommand] : void 0,
|
|
113
|
+
options.reviewCommand ? ["review-command", options.reviewCommand] : void 0
|
|
114
|
+
].filter((input) => Boolean(input)).map(([key, value]) => ` ${key}: ${JSON.stringify(value)}`).join("\n");
|
|
115
|
+
return `# Generated by @farming-labs/docs. You can edit this file.
|
|
116
|
+
name: Docs Review
|
|
117
|
+
|
|
118
|
+
on:
|
|
119
|
+
pull_request:
|
|
120
|
+
paths:
|
|
121
|
+
${filters.map((filter) => ` - ${JSON.stringify(filter)}`).join("\n")}
|
|
122
|
+
|
|
123
|
+
permissions:
|
|
124
|
+
contents: read
|
|
125
|
+
checks: write
|
|
126
|
+
pull-requests: write
|
|
127
|
+
|
|
128
|
+
jobs:
|
|
129
|
+
docs-review:
|
|
130
|
+
name: ${JSON.stringify(ciName)}
|
|
131
|
+
uses: ${reusableWorkflow}
|
|
132
|
+
with:
|
|
133
|
+
${workflowInputs}
|
|
134
|
+
permissions:
|
|
135
|
+
contents: read
|
|
136
|
+
checks: write
|
|
137
|
+
pull-requests: write
|
|
138
|
+
`;
|
|
139
|
+
}
|
|
140
|
+
function ensureDocsReviewWorkflow(options) {
|
|
141
|
+
const rootDir = options.rootDir;
|
|
142
|
+
const repoRoot = findGitRoot(rootDir) ?? rootDir;
|
|
143
|
+
const workflowRelativePath = options.workflowPath ?? DEFAULT_DOCS_REVIEW_WORKFLOW_PATH;
|
|
144
|
+
const workflowPath = isAbsolute(workflowRelativePath) ? workflowRelativePath : join(repoRoot, workflowRelativePath);
|
|
145
|
+
const resultRelativePath = toPosixPath(relative(repoRoot, workflowPath));
|
|
146
|
+
const review = resolveDocsReviewConfig(options.config?.review ?? readDocsReviewConfigFromSource(options.configContent ?? readConfig(rootDir)));
|
|
147
|
+
if (!review.enabled || !review.ci.enabled) return {
|
|
148
|
+
status: "disabled",
|
|
149
|
+
path: workflowPath,
|
|
150
|
+
relativePath: resultRelativePath
|
|
151
|
+
};
|
|
152
|
+
if (existsSync(workflowPath)) return {
|
|
153
|
+
status: "exists",
|
|
154
|
+
path: workflowPath,
|
|
155
|
+
relativePath: resultRelativePath
|
|
156
|
+
};
|
|
157
|
+
const configPath = options.configPath ?? findDocsConfigPath(rootDir);
|
|
158
|
+
const configContent = options.configContent ?? readConfig(rootDir, configPath);
|
|
159
|
+
const projectDir = toPosixPath(relative(repoRoot, rootDir)) || ".";
|
|
160
|
+
const packageManager = detectPackageManager(repoRoot);
|
|
161
|
+
const workflow = buildDocsReviewWorkflow({
|
|
162
|
+
packageManager,
|
|
163
|
+
ciName: review.ci.name,
|
|
164
|
+
reusableWorkflow: hasLocalDocsWorkspacePackage(repoRoot) ? LOCAL_DOCS_REVIEW_REUSABLE_WORKFLOW : DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW,
|
|
165
|
+
pnpmVersion: detectPnpmVersionFallback(repoRoot, packageManager),
|
|
166
|
+
projectDir,
|
|
167
|
+
configPath,
|
|
168
|
+
buildCommand: detectLocalDocsCliBuildCommand(repoRoot, packageManager),
|
|
169
|
+
reviewCommand: detectLocalDocsCliReviewCommand(repoRoot, rootDir),
|
|
170
|
+
pathFilters: buildDocsReviewWorkflowPathFilters({
|
|
171
|
+
rootDir,
|
|
172
|
+
repoRoot,
|
|
173
|
+
config: options.config,
|
|
174
|
+
configPath,
|
|
175
|
+
configContent
|
|
176
|
+
})
|
|
177
|
+
});
|
|
178
|
+
mkdirSync(dirname(workflowPath), { recursive: true });
|
|
179
|
+
writeFileSync(workflowPath, workflow, "utf-8");
|
|
180
|
+
options.log?.(`[docs] Created ${resultRelativePath} for Docs Review CI.`);
|
|
181
|
+
return {
|
|
182
|
+
status: "created",
|
|
183
|
+
path: workflowPath,
|
|
184
|
+
relativePath: resultRelativePath
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function readDocsReviewConfigFromSource(content) {
|
|
188
|
+
if (!content) return void 0;
|
|
189
|
+
const rootObject = extractRootObjectLiteral(content) ?? content;
|
|
190
|
+
const reviewCursor = findTopLevelPropertyValueIndex(rootObject, "review");
|
|
191
|
+
if (reviewCursor === void 0) return void 0;
|
|
192
|
+
if (rootObject.startsWith("true", reviewCursor)) return true;
|
|
193
|
+
if (rootObject.startsWith("false", reviewCursor)) return false;
|
|
194
|
+
if (rootObject[reviewCursor] !== "{") return void 0;
|
|
195
|
+
const reviewEnd = findMatchingObjectEnd(rootObject, reviewCursor);
|
|
196
|
+
if (reviewEnd === void 0) return void 0;
|
|
197
|
+
const reviewBlock = rootObject.slice(reviewCursor + 1, reviewEnd);
|
|
198
|
+
const scoreBlock = extractTopLevelObjectLiteral(reviewBlock, "score");
|
|
199
|
+
const ciCursor = findTopLevelPropertyValueIndex(reviewBlock, "ci");
|
|
200
|
+
const rulesBlock = extractTopLevelObjectLiteral(reviewBlock, "rules");
|
|
201
|
+
const ci = ciCursor === void 0 ? void 0 : reviewBlock.startsWith("true", ciCursor) ? true : reviewBlock.startsWith("false", ciCursor) ? false : reviewBlock[ciCursor] === "{" ? readReviewCiObject(reviewBlock, ciCursor) : void 0;
|
|
202
|
+
return {
|
|
203
|
+
enabled: readTopLevelBoolean(reviewBlock, "enabled"),
|
|
204
|
+
score: scoreBlock ? {
|
|
205
|
+
threshold: readTopLevelNumber(scoreBlock, "threshold"),
|
|
206
|
+
weights: readReviewWeights(scoreBlock)
|
|
207
|
+
} : void 0,
|
|
208
|
+
ci,
|
|
209
|
+
rules: rulesBlock ? readReviewRules(rulesBlock) : void 0
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function buildDocsReviewWorkflowPathFilters(options) {
|
|
213
|
+
const repoRoot = options.repoRoot ?? findGitRoot(options.rootDir) ?? options.rootDir;
|
|
214
|
+
const configPath = options.configPath ?? findDocsConfigPath(options.rootDir);
|
|
215
|
+
const configContent = options.configContent ?? readConfig(options.rootDir, configPath);
|
|
216
|
+
const entry = options.config?.entry ?? readTopLevelString(configContent, "entry") ?? "docs";
|
|
217
|
+
const contentDir = options.config?.contentDir ?? readTopLevelString(configContent, "contentDir");
|
|
218
|
+
const projectDir = toPosixPath(relative(repoRoot, options.rootDir));
|
|
219
|
+
const projectPrefix = projectDir && projectDir !== "." ? `${projectDir}/` : "";
|
|
220
|
+
const candidates = [
|
|
221
|
+
prefixPath(projectPrefix, configPath),
|
|
222
|
+
prefixPath(projectPrefix, `${entry}/**`),
|
|
223
|
+
prefixPath(projectPrefix, `app/${entry}/**`),
|
|
224
|
+
prefixPath(projectPrefix, `src/app/${entry}/**`),
|
|
225
|
+
contentDir ? prefixPath(projectPrefix, `${trimSlashes(contentDir)}/**`) : void 0,
|
|
226
|
+
prefixPath(projectPrefix, "content/docs/**"),
|
|
227
|
+
prefixPath(projectPrefix, "src/content/docs/**"),
|
|
228
|
+
prefixPath(projectPrefix, "src/lib/docs.config.*"),
|
|
229
|
+
DEFAULT_DOCS_REVIEW_WORKFLOW_PATH,
|
|
230
|
+
DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW_PATH
|
|
231
|
+
];
|
|
232
|
+
return Array.from(new Set(candidates.filter((candidate) => Boolean(candidate))));
|
|
233
|
+
}
|
|
234
|
+
function normalizeRules(rules) {
|
|
235
|
+
if (!rules) return {};
|
|
236
|
+
const knownRules = new Set(Object.keys(DEFAULT_REVIEW_RULES));
|
|
237
|
+
return Object.fromEntries(Object.entries(rules).filter((entry) => knownRules.has(entry[0]) && REVIEW_SEVERITIES.has(entry[1])));
|
|
238
|
+
}
|
|
239
|
+
function normalizeWeight(value, fallback) {
|
|
240
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
241
|
+
}
|
|
242
|
+
function normalizeCiName(value) {
|
|
243
|
+
return value?.trim() || DEFAULT_DOCS_REVIEW_CI_NAME;
|
|
244
|
+
}
|
|
245
|
+
function clampScoreThreshold(value) {
|
|
246
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD;
|
|
247
|
+
return Math.min(100, Math.max(0, Math.round(value)));
|
|
248
|
+
}
|
|
249
|
+
function readReviewCiObject(source, cursor) {
|
|
250
|
+
const end = findMatchingObjectEnd(source, cursor);
|
|
251
|
+
if (end === void 0) return void 0;
|
|
252
|
+
const block = source.slice(cursor + 1, end);
|
|
253
|
+
const mode = readTopLevelString(block, "mode");
|
|
254
|
+
return {
|
|
255
|
+
enabled: readTopLevelBoolean(block, "enabled"),
|
|
256
|
+
name: readTopLevelString(block, "name"),
|
|
257
|
+
mode: mode && REVIEW_CI_MODES.has(mode) ? mode : void 0,
|
|
258
|
+
annotations: readTopLevelBoolean(block, "annotations"),
|
|
259
|
+
comment: readTopLevelBoolean(block, "comment")
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
function readReviewRules(block) {
|
|
263
|
+
const rules = {};
|
|
264
|
+
for (const key of Object.keys(DEFAULT_REVIEW_RULES)) {
|
|
265
|
+
const value = readTopLevelString(block, key);
|
|
266
|
+
if (value && REVIEW_SEVERITIES.has(value)) rules[key] = value;
|
|
267
|
+
}
|
|
268
|
+
return rules;
|
|
269
|
+
}
|
|
270
|
+
function readReviewWeights(block) {
|
|
271
|
+
const weightsBlock = extractTopLevelObjectLiteral(block, "weights");
|
|
272
|
+
if (!weightsBlock) return void 0;
|
|
273
|
+
return {
|
|
274
|
+
error: readTopLevelNumber(weightsBlock, "error"),
|
|
275
|
+
warn: readTopLevelNumber(weightsBlock, "warn"),
|
|
276
|
+
suggestion: readTopLevelNumber(weightsBlock, "suggestion")
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
function detectPackageManager(rootDir) {
|
|
280
|
+
if (existsSync(join(rootDir, "pnpm-lock.yaml"))) return "pnpm";
|
|
281
|
+
if (existsSync(join(rootDir, "yarn.lock"))) return "yarn";
|
|
282
|
+
if (existsSync(join(rootDir, "bun.lock")) || existsSync(join(rootDir, "bun.lockb"))) return "bun";
|
|
283
|
+
return "npm";
|
|
284
|
+
}
|
|
285
|
+
function detectLocalDocsCliBuildCommand(repoRoot, packageManager) {
|
|
286
|
+
if (!hasLocalDocsWorkspacePackage(repoRoot)) return void 0;
|
|
287
|
+
if (packageManager === "pnpm") return "pnpm --filter @farming-labs/docs run build";
|
|
288
|
+
if (packageManager === "yarn") return "yarn workspace @farming-labs/docs build";
|
|
289
|
+
if (packageManager === "bun") return "bun run --filter @farming-labs/docs build";
|
|
290
|
+
return "npm run build --workspace=@farming-labs/docs";
|
|
291
|
+
}
|
|
292
|
+
function detectPnpmVersionFallback(repoRoot, packageManager) {
|
|
293
|
+
if (packageManager !== "pnpm") return void 0;
|
|
294
|
+
try {
|
|
295
|
+
if (JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf-8")).packageManager?.startsWith("pnpm@")) return void 0;
|
|
296
|
+
} catch {}
|
|
297
|
+
return DEFAULT_DOCS_REVIEW_PNPM_VERSION;
|
|
298
|
+
}
|
|
299
|
+
function detectLocalDocsCliReviewCommand(repoRoot, rootDir) {
|
|
300
|
+
if (!hasLocalDocsWorkspacePackage(repoRoot)) return void 0;
|
|
301
|
+
return `node ${shellQuote(toPosixPath(relative(rootDir, join(repoRoot, "packages", "docs", "dist", "cli", "index.mjs"))) || "./packages/docs/dist/cli/index.mjs")}`;
|
|
302
|
+
}
|
|
303
|
+
function hasLocalDocsWorkspacePackage(repoRoot) {
|
|
304
|
+
const packageJsonPath = join(repoRoot, "packages", "docs", "package.json");
|
|
305
|
+
if (!existsSync(packageJsonPath)) return false;
|
|
306
|
+
try {
|
|
307
|
+
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
308
|
+
return packageJson.name === "@farming-labs/docs" && Boolean(packageJson.scripts?.build);
|
|
309
|
+
} catch {
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
function shellQuote(value) {
|
|
314
|
+
if (/^[A-Za-z0-9_./:-]+$/.test(value)) return value;
|
|
315
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
316
|
+
}
|
|
317
|
+
function normalizeProjectDir(value) {
|
|
318
|
+
if (!value || value === "") return ".";
|
|
319
|
+
return toPosixPath(value).replace(/\/+$/, "") || ".";
|
|
320
|
+
}
|
|
321
|
+
function normalizePathFilters(filters) {
|
|
322
|
+
const normalized = (filters && filters.length > 0 ? filters : ["docs.config.*", "app/docs/**"]).map((filter) => toPosixPath(filter).replace(/^\.\/+/, "")).filter(Boolean);
|
|
323
|
+
return Array.from(new Set(normalized));
|
|
324
|
+
}
|
|
325
|
+
function findGitRoot(start) {
|
|
326
|
+
let current = start;
|
|
327
|
+
while (true) {
|
|
328
|
+
if (existsSync(join(current, ".git"))) return current;
|
|
329
|
+
const parent = dirname(current);
|
|
330
|
+
if (parent === current) return void 0;
|
|
331
|
+
current = parent;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
function findDocsConfigPath(rootDir) {
|
|
335
|
+
for (const ext of FILE_EXTS) {
|
|
336
|
+
const path = `docs.config.${ext}`;
|
|
337
|
+
if (existsSync(join(rootDir, path))) return path;
|
|
338
|
+
}
|
|
339
|
+
return "docs.config.ts";
|
|
340
|
+
}
|
|
341
|
+
function readConfig(rootDir, configPath = findDocsConfigPath(rootDir)) {
|
|
342
|
+
const fullPath = join(rootDir, configPath);
|
|
343
|
+
if (!existsSync(fullPath)) return void 0;
|
|
344
|
+
try {
|
|
345
|
+
return readFileSync(fullPath, "utf-8");
|
|
346
|
+
} catch {
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
function prefixPath(prefix, value) {
|
|
351
|
+
return `${prefix}${toPosixPath(value).replace(/^\.\/+/, "")}`;
|
|
352
|
+
}
|
|
353
|
+
function trimSlashes(value) {
|
|
354
|
+
return toPosixPath(value).replace(/^\/+|\/+$/g, "");
|
|
355
|
+
}
|
|
356
|
+
function toPosixPath(value) {
|
|
357
|
+
return value.replaceAll("\\", "/");
|
|
358
|
+
}
|
|
359
|
+
function extractRootObjectLiteral(content) {
|
|
360
|
+
const candidateIndexes = [content.search(/\bdefineDocs\s*\(/), content.search(/\bexport\s+default\b/)].filter((value) => value !== -1);
|
|
361
|
+
for (const startIndex of candidateIndexes) {
|
|
362
|
+
const braceStart = content.indexOf("{", startIndex);
|
|
363
|
+
if (braceStart === -1) continue;
|
|
364
|
+
const braceEnd = findMatchingObjectEnd(content, braceStart);
|
|
365
|
+
if (braceEnd !== void 0) return content.slice(braceStart + 1, braceEnd);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
function extractTopLevelObjectLiteral(content, key) {
|
|
369
|
+
const cursor = findTopLevelPropertyValueIndex(content, key);
|
|
370
|
+
if (cursor === void 0 || content[cursor] !== "{") return void 0;
|
|
371
|
+
const end = findMatchingObjectEnd(content, cursor);
|
|
372
|
+
return end === void 0 ? void 0 : content.slice(cursor + 1, end);
|
|
373
|
+
}
|
|
374
|
+
function readTopLevelBoolean(content, key) {
|
|
375
|
+
const cursor = findTopLevelPropertyValueIndex(content, key);
|
|
376
|
+
if (cursor === void 0) return void 0;
|
|
377
|
+
if (content.startsWith("true", cursor)) return true;
|
|
378
|
+
if (content.startsWith("false", cursor)) return false;
|
|
379
|
+
}
|
|
380
|
+
function readTopLevelNumber(content, key) {
|
|
381
|
+
const cursor = findTopLevelPropertyValueIndex(content, key);
|
|
382
|
+
if (cursor === void 0) return void 0;
|
|
383
|
+
const match = content.slice(cursor).match(/^-?\d+(?:\.\d+)?/);
|
|
384
|
+
if (!match) return void 0;
|
|
385
|
+
const value = Number(match[0]);
|
|
386
|
+
return Number.isFinite(value) ? value : void 0;
|
|
387
|
+
}
|
|
388
|
+
function readTopLevelString(content, key) {
|
|
389
|
+
if (!content) return void 0;
|
|
390
|
+
const cursor = findTopLevelPropertyValueIndex(content, key);
|
|
391
|
+
if (cursor === void 0) return void 0;
|
|
392
|
+
const quote = content[cursor];
|
|
393
|
+
if (quote !== "\"" && quote !== "'") return void 0;
|
|
394
|
+
let value = "";
|
|
395
|
+
for (let index = cursor + 1; index < content.length; index += 1) {
|
|
396
|
+
const char = content[index];
|
|
397
|
+
if (char === "\\") {
|
|
398
|
+
const escaped = content[index + 1];
|
|
399
|
+
if (escaped) {
|
|
400
|
+
value += escaped;
|
|
401
|
+
index += 1;
|
|
402
|
+
}
|
|
403
|
+
continue;
|
|
404
|
+
}
|
|
405
|
+
if (char === quote) return value;
|
|
406
|
+
value += char;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
function findTopLevelPropertyValueIndex(block, key) {
|
|
410
|
+
let objectDepth = 0;
|
|
411
|
+
let arrayDepth = 0;
|
|
412
|
+
let parenDepth = 0;
|
|
413
|
+
let inString = null;
|
|
414
|
+
let inLineComment = false;
|
|
415
|
+
let inBlockComment = false;
|
|
416
|
+
let escaped = false;
|
|
417
|
+
for (let index = 0; index < block.length; index += 1) {
|
|
418
|
+
const char = block[index];
|
|
419
|
+
const next = block[index + 1];
|
|
420
|
+
if (inLineComment) {
|
|
421
|
+
if (char === "\n") inLineComment = false;
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
if (inBlockComment) {
|
|
425
|
+
if (char === "*" && next === "/") {
|
|
426
|
+
inBlockComment = false;
|
|
427
|
+
index += 1;
|
|
428
|
+
}
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
if (inString) {
|
|
432
|
+
if (escaped) {
|
|
433
|
+
escaped = false;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
if (char === "\\") {
|
|
437
|
+
escaped = true;
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
if (char === inString) inString = null;
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
if (char === "/" && next === "/") {
|
|
444
|
+
inLineComment = true;
|
|
445
|
+
index += 1;
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
if (char === "/" && next === "*") {
|
|
449
|
+
inBlockComment = true;
|
|
450
|
+
index += 1;
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
if (char === "\"" || char === "'" || char === "`") {
|
|
454
|
+
inString = char;
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
if (char === "{") {
|
|
458
|
+
objectDepth += 1;
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
if (char === "}") {
|
|
462
|
+
objectDepth = Math.max(0, objectDepth - 1);
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (char === "[") {
|
|
466
|
+
arrayDepth += 1;
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
if (char === "]") {
|
|
470
|
+
arrayDepth = Math.max(0, arrayDepth - 1);
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
if (char === "(") {
|
|
474
|
+
parenDepth += 1;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
if (char === ")") {
|
|
478
|
+
parenDepth = Math.max(0, parenDepth - 1);
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
if (objectDepth !== 0 || arrayDepth !== 0 || parenDepth !== 0) continue;
|
|
482
|
+
if (!block.startsWith(key, index)) continue;
|
|
483
|
+
const before = block[index - 1] ?? "";
|
|
484
|
+
const after = block[index + key.length] ?? "";
|
|
485
|
+
if (/[A-Za-z0-9_$]/.test(before) || /[A-Za-z0-9_$]/.test(after)) continue;
|
|
486
|
+
let cursor = index + key.length;
|
|
487
|
+
while (/\s/.test(block[cursor] ?? "")) cursor += 1;
|
|
488
|
+
if (block[cursor] !== ":") continue;
|
|
489
|
+
cursor += 1;
|
|
490
|
+
while (/\s/.test(block[cursor] ?? "")) cursor += 1;
|
|
491
|
+
return cursor;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
function findMatchingObjectEnd(block, start) {
|
|
495
|
+
let depth = 0;
|
|
496
|
+
let inString = null;
|
|
497
|
+
let inLineComment = false;
|
|
498
|
+
let inBlockComment = false;
|
|
499
|
+
let escaped = false;
|
|
500
|
+
for (let index = start; index < block.length; index += 1) {
|
|
501
|
+
const char = block[index];
|
|
502
|
+
const next = block[index + 1];
|
|
503
|
+
if (inLineComment) {
|
|
504
|
+
if (char === "\n") inLineComment = false;
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
if (inBlockComment) {
|
|
508
|
+
if (char === "*" && next === "/") {
|
|
509
|
+
inBlockComment = false;
|
|
510
|
+
index += 1;
|
|
511
|
+
}
|
|
512
|
+
continue;
|
|
513
|
+
}
|
|
514
|
+
if (inString) {
|
|
515
|
+
if (escaped) {
|
|
516
|
+
escaped = false;
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (char === "\\") {
|
|
520
|
+
escaped = true;
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (char === inString) inString = null;
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
if (char === "/" && next === "/") {
|
|
527
|
+
inLineComment = true;
|
|
528
|
+
index += 1;
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
if (char === "/" && next === "*") {
|
|
532
|
+
inBlockComment = true;
|
|
533
|
+
index += 1;
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
if (char === "\"" || char === "'" || char === "`") {
|
|
537
|
+
inString = char;
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
if (char === "{") {
|
|
541
|
+
depth += 1;
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
if (char !== "}") continue;
|
|
545
|
+
depth -= 1;
|
|
546
|
+
if (depth === 0) return index;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
//#endregion
|
|
551
|
+
//#region src/agent-evals.ts
|
|
552
|
+
const DEFAULT_TOP_K = 5;
|
|
553
|
+
const DEFAULT_TOKEN_BUDGET = 4e3;
|
|
554
|
+
const CONTEXT_SEPARATOR = "\n\n---\n\n";
|
|
555
|
+
const STATIC_CODE_PLAN_CONFIG = resolveDocsCodeBlocksValidateConfig(true);
|
|
556
|
+
const STATIC_JITI = createJiti(import.meta.url, {
|
|
557
|
+
fsCache: false,
|
|
558
|
+
moduleCache: false
|
|
559
|
+
});
|
|
560
|
+
const EXECUTABLE_LANGUAGES = new Set([
|
|
561
|
+
"bash",
|
|
562
|
+
"curl",
|
|
563
|
+
"javascript",
|
|
564
|
+
"js",
|
|
565
|
+
"jsx",
|
|
566
|
+
"json",
|
|
567
|
+
"node",
|
|
568
|
+
"py",
|
|
569
|
+
"python",
|
|
570
|
+
"rb",
|
|
571
|
+
"ruby",
|
|
572
|
+
"sh",
|
|
573
|
+
"shell",
|
|
574
|
+
"ts",
|
|
575
|
+
"tsx",
|
|
576
|
+
"typescript",
|
|
577
|
+
"zsh"
|
|
578
|
+
]);
|
|
579
|
+
function isRecord(value) {
|
|
580
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
581
|
+
}
|
|
582
|
+
function normalizeRuntimeString(value, path, issues, required = false) {
|
|
583
|
+
if (value === void 0) {
|
|
584
|
+
if (required) issues.push(`${path} must be a non-empty string.`);
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
588
|
+
issues.push(`${path} must be a non-empty string.`);
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
return value.trim();
|
|
592
|
+
}
|
|
593
|
+
function normalizeRuntimeStringList(value, path, issues, options = {}) {
|
|
594
|
+
if (value === void 0) {
|
|
595
|
+
if (options.required) issues.push(`${path} must be a non-empty string array.`);
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
if (!Array.isArray(value)) {
|
|
599
|
+
issues.push(`${path} must be a string array.`);
|
|
600
|
+
return [];
|
|
601
|
+
}
|
|
602
|
+
const normalized = [];
|
|
603
|
+
let hasInvalidItem = false;
|
|
604
|
+
for (const item of value) {
|
|
605
|
+
if (typeof item !== "string" || !item.trim()) {
|
|
606
|
+
hasInvalidItem = true;
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
if (!normalized.includes(item.trim())) normalized.push(item.trim());
|
|
610
|
+
}
|
|
611
|
+
if (hasInvalidItem) issues.push(`${path} must contain only non-empty strings.`);
|
|
612
|
+
if (options.required && normalized.length === 0) issues.push(`${path} must contain at least one source.`);
|
|
613
|
+
return normalized;
|
|
614
|
+
}
|
|
615
|
+
function normalizeRuntimeNumber(value, path, issues, options = {}) {
|
|
616
|
+
if (value === void 0) return void 0;
|
|
617
|
+
if (typeof value !== "number" || !Number.isFinite(value) || options.integer && !Number.isInteger(value) || options.minimum !== void 0 && value < options.minimum || options.maximum !== void 0 && value > options.maximum) {
|
|
618
|
+
const range = options.minimum !== void 0 || options.maximum !== void 0 ? ` between ${options.minimum ?? "-Infinity"} and ${options.maximum ?? "Infinity"}` : "";
|
|
619
|
+
issues.push(`${path} must be a finite${options.integer ? " integer" : " number"}${range}.`);
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
return value;
|
|
623
|
+
}
|
|
624
|
+
function normalizeRuntimeFilters(value, path, issues) {
|
|
625
|
+
if (value === void 0) return void 0;
|
|
626
|
+
if (!isRecord(value)) {
|
|
627
|
+
issues.push(`${path} must be an object.`);
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
const framework = normalizeRuntimeString(value.framework, `${path}.framework`, issues);
|
|
631
|
+
const version = normalizeRuntimeString(value.version, `${path}.version`, issues);
|
|
632
|
+
const locale = normalizeRuntimeString(value.locale, `${path}.locale`, issues);
|
|
633
|
+
return framework || version || locale ? {
|
|
634
|
+
framework,
|
|
635
|
+
version,
|
|
636
|
+
locale
|
|
637
|
+
} : void 0;
|
|
638
|
+
}
|
|
639
|
+
function normalizeRuntimeExamples(value, path, issues) {
|
|
640
|
+
if (value === void 0) return void 0;
|
|
641
|
+
if (!Array.isArray(value)) {
|
|
642
|
+
issues.push(`${path} must be an array.`);
|
|
643
|
+
return [];
|
|
644
|
+
}
|
|
645
|
+
return value.flatMap((item, index) => {
|
|
646
|
+
const itemPath = `${path}[${index}]`;
|
|
647
|
+
if (!isRecord(item)) {
|
|
648
|
+
issues.push(`${itemPath} must be an object.`);
|
|
649
|
+
return [];
|
|
650
|
+
}
|
|
651
|
+
const source = normalizeRuntimeString(item.source, `${itemPath}.source`, issues);
|
|
652
|
+
const language = normalizeRuntimeString(item.language, `${itemPath}.language`, issues);
|
|
653
|
+
const framework = normalizeRuntimeString(item.framework, `${itemPath}.framework`, issues);
|
|
654
|
+
const packageManager = normalizeRuntimeString(item.packageManager, `${itemPath}.packageManager`, issues);
|
|
655
|
+
const title = normalizeRuntimeString(item.title, `${itemPath}.title`, issues);
|
|
656
|
+
const includes = normalizeRuntimeStringList(item.includes, `${itemPath}.includes`, issues);
|
|
657
|
+
let runnable;
|
|
658
|
+
if (item.runnable !== void 0) if (typeof item.runnable === "boolean") runnable = item.runnable;
|
|
659
|
+
else issues.push(`${itemPath}.runnable must be a boolean.`);
|
|
660
|
+
return [{
|
|
661
|
+
source,
|
|
662
|
+
language,
|
|
663
|
+
framework,
|
|
664
|
+
packageManager,
|
|
665
|
+
title,
|
|
666
|
+
runnable,
|
|
667
|
+
includes
|
|
668
|
+
}];
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
function normalizeGoldenTaskInput(value, index, initialIssues = []) {
|
|
672
|
+
const issues = [...initialIssues];
|
|
673
|
+
const path = `tasks[${index}]`;
|
|
674
|
+
const input = isRecord(value) ? value : {};
|
|
675
|
+
if (!isRecord(value)) issues.push(`${path} must be an object.`);
|
|
676
|
+
const id = normalizeRuntimeString(input.id, `${path}.id`, issues, true) ?? `invalid-task-${index + 1}`;
|
|
677
|
+
const query = normalizeRuntimeString(input.query, `${path}.query`, issues, true) ?? "";
|
|
678
|
+
const filters = normalizeRuntimeFilters(input.filters, `${path}.filters`, issues);
|
|
679
|
+
const tokenBudget = normalizeRuntimeNumber(input.tokenBudget, `${path}.tokenBudget`, issues, {
|
|
680
|
+
minimum: 1,
|
|
681
|
+
integer: true
|
|
682
|
+
});
|
|
683
|
+
const topK = normalizeRuntimeNumber(input.topK, `${path}.topK`, issues, {
|
|
684
|
+
minimum: 1,
|
|
685
|
+
integer: true
|
|
686
|
+
});
|
|
687
|
+
const expectPath = `${path}.expect`;
|
|
688
|
+
const expectation = isRecord(input.expect) ? input.expect : {};
|
|
689
|
+
if (!isRecord(input.expect)) issues.push(`${expectPath} must be an object.`);
|
|
690
|
+
const relevantSources = normalizeRuntimeStringList(expectation.relevantSources, `${expectPath}.relevantSources`, issues, { required: true }) ?? [];
|
|
691
|
+
const allowedSources = normalizeRuntimeStringList(expectation.allowedSources, `${expectPath}.allowedSources`, issues);
|
|
692
|
+
const forbiddenSources = normalizeRuntimeStringList(expectation.forbiddenSources, `${expectPath}.forbiddenSources`, issues);
|
|
693
|
+
const requiredCitations = normalizeRuntimeStringList(expectation.requiredCitations, `${expectPath}.requiredCitations`, issues);
|
|
694
|
+
const minRecallAtK = normalizeRuntimeNumber(expectation.minRecallAtK, `${expectPath}.minRecallAtK`, issues, {
|
|
695
|
+
minimum: 0,
|
|
696
|
+
maximum: 1
|
|
697
|
+
});
|
|
698
|
+
const maxFirstRelevantRank = normalizeRuntimeNumber(expectation.maxFirstRelevantRank, `${expectPath}.maxFirstRelevantRank`, issues, {
|
|
699
|
+
minimum: 1,
|
|
700
|
+
integer: true
|
|
701
|
+
});
|
|
702
|
+
const minUsefulByteRatio = normalizeRuntimeNumber(expectation.minUsefulByteRatio, `${expectPath}.minUsefulByteRatio`, issues, {
|
|
703
|
+
minimum: 0,
|
|
704
|
+
maximum: 1
|
|
705
|
+
});
|
|
706
|
+
const examples = normalizeRuntimeExamples(expectation.examples, `${expectPath}.examples`, issues);
|
|
707
|
+
const forbidden = forbiddenSources ?? [];
|
|
708
|
+
for (const [name, sources] of [
|
|
709
|
+
["relevantSources", relevantSources],
|
|
710
|
+
["allowedSources", allowedSources ?? []],
|
|
711
|
+
["requiredCitations", requiredCitations ?? []]
|
|
712
|
+
]) {
|
|
713
|
+
const overlaps = sources.filter((source) => forbidden.some((forbiddenSource) => sourceMatches(source, forbiddenSource) || sourceMatches(forbiddenSource, source)));
|
|
714
|
+
if (overlaps.length > 0) issues.push(`${expectPath}.${name} must not overlap ${expectPath}.forbiddenSources (${overlaps.join(", ")}).`);
|
|
715
|
+
}
|
|
716
|
+
return {
|
|
717
|
+
task: {
|
|
718
|
+
id,
|
|
719
|
+
query,
|
|
720
|
+
filters,
|
|
721
|
+
tokenBudget,
|
|
722
|
+
topK,
|
|
723
|
+
expect: {
|
|
724
|
+
relevantSources,
|
|
725
|
+
allowedSources,
|
|
726
|
+
forbiddenSources,
|
|
727
|
+
requiredCitations,
|
|
728
|
+
minRecallAtK,
|
|
729
|
+
maxFirstRelevantRank,
|
|
730
|
+
minUsefulByteRatio,
|
|
731
|
+
examples
|
|
732
|
+
}
|
|
733
|
+
},
|
|
734
|
+
issues
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
function round(value) {
|
|
738
|
+
return Math.round((value + Number.EPSILON) * 1e4) / 1e4;
|
|
739
|
+
}
|
|
740
|
+
function clamp(value, minimum, maximum) {
|
|
741
|
+
return Math.min(maximum, Math.max(minimum, value));
|
|
742
|
+
}
|
|
743
|
+
function normalizePositiveInteger(value, fallback, maximum) {
|
|
744
|
+
if (!Number.isFinite(value)) return fallback;
|
|
745
|
+
return clamp(Math.floor(value ?? fallback), 1, maximum);
|
|
746
|
+
}
|
|
747
|
+
function safeDecode(value) {
|
|
748
|
+
try {
|
|
749
|
+
return decodeURIComponent(value);
|
|
750
|
+
} catch {
|
|
751
|
+
return value;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
function canonicalizeSource(value) {
|
|
755
|
+
let pathname;
|
|
756
|
+
let hash;
|
|
757
|
+
try {
|
|
758
|
+
const parsed = new URL(value, "https://docs.local");
|
|
759
|
+
pathname = parsed.pathname;
|
|
760
|
+
hash = parsed.hash;
|
|
761
|
+
} catch {
|
|
762
|
+
const [pathAndQuery = "/", fragment = ""] = value.split("#", 2);
|
|
763
|
+
pathname = pathAndQuery.split("?", 1)[0] || "/";
|
|
764
|
+
hash = fragment ? `#${fragment}` : "";
|
|
765
|
+
}
|
|
766
|
+
pathname = pathname.replace(/\/{2,}/gu, "/").replace(/\.md$/iu, "");
|
|
767
|
+
if (pathname !== "/") pathname = pathname.replace(/\/+$/gu, "");
|
|
768
|
+
if (!pathname.startsWith("/")) pathname = `/${pathname}`;
|
|
769
|
+
const normalizedHash = hash ? `#${safeDecode(hash.slice(1)).trim().replace(/^#+/u, "").toLowerCase()}` : "";
|
|
770
|
+
return `${pathname || "/"}${normalizedHash === "#" ? "" : normalizedHash}`;
|
|
771
|
+
}
|
|
772
|
+
function sourcePage(value) {
|
|
773
|
+
return canonicalizeSource(value).split("#", 1)[0] ?? "/";
|
|
774
|
+
}
|
|
775
|
+
function sourceMatches(actual, expected) {
|
|
776
|
+
const canonicalActual = canonicalizeSource(actual);
|
|
777
|
+
const canonicalExpected = canonicalizeSource(expected);
|
|
778
|
+
return canonicalExpected.includes("#") ? canonicalActual === canonicalExpected : sourcePage(canonicalActual) === sourcePage(canonicalExpected);
|
|
779
|
+
}
|
|
780
|
+
function uniqueCanonicalSources(values) {
|
|
781
|
+
return Array.from(new Set(values.map(canonicalizeSource)));
|
|
782
|
+
}
|
|
783
|
+
function valuesOverlap(left, right, matches) {
|
|
784
|
+
return left.some((leftValue) => right.some((rightValue) => matches(leftValue, rightValue)));
|
|
785
|
+
}
|
|
786
|
+
function getPageScope(page) {
|
|
787
|
+
const topFramework = normalizeAgentScopeValues(page.framework).map(normalizeAgentFramework);
|
|
788
|
+
const contractFramework = normalizeAgentScopeValues(page.agent?.appliesTo?.framework).map(normalizeAgentFramework);
|
|
789
|
+
const topVersion = normalizeAgentScopeValues(page.version).map(normalizeAgentVersion);
|
|
790
|
+
const contractVersion = normalizeAgentScopeValues(page.agent?.appliesTo?.version).map(normalizeAgentVersion);
|
|
791
|
+
const locale = normalizeAgentScopeValues(page.locale).map(normalizeAgentLocale);
|
|
792
|
+
return {
|
|
793
|
+
framework: Array.from(new Set([...topFramework, ...contractFramework])),
|
|
794
|
+
version: Array.from(new Set([...topVersion, ...contractVersion])),
|
|
795
|
+
locale,
|
|
796
|
+
frameworkAmbiguous: topFramework.length > 0 && contractFramework.length > 0 && !valuesOverlap(topFramework, contractFramework, (left, right) => left === right),
|
|
797
|
+
versionAmbiguous: topVersion.length > 0 && contractVersion.length > 0 && !valuesOverlap(topVersion, contractVersion, agentVersionConstraintsOverlap)
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
function compareCodePoints(left, right) {
|
|
801
|
+
if (left === right) return 0;
|
|
802
|
+
return left < right ? -1 : 1;
|
|
803
|
+
}
|
|
804
|
+
function toGoldenSources(sources, relevantSources) {
|
|
805
|
+
return sources.map((source, index) => ({
|
|
806
|
+
rank: index + 1,
|
|
807
|
+
url: canonicalizeSource(source.url),
|
|
808
|
+
title: source.section ?? source.title,
|
|
809
|
+
framework: source.framework,
|
|
810
|
+
version: source.version,
|
|
811
|
+
utf8Bytes: source.utf8Bytes,
|
|
812
|
+
relevant: relevantSources.some((expected) => sourceMatches(source.url, expected)),
|
|
813
|
+
truncated: source.truncated
|
|
814
|
+
}));
|
|
815
|
+
}
|
|
816
|
+
function toContextCandidates(sources, pagesByUrl) {
|
|
817
|
+
return sources.flatMap((source) => {
|
|
818
|
+
const page = pagesByUrl.get(sourcePage(source.pageUrl));
|
|
819
|
+
if (!page) return [];
|
|
820
|
+
return [{
|
|
821
|
+
page,
|
|
822
|
+
source: canonicalizeSource(source.url),
|
|
823
|
+
content: source.content,
|
|
824
|
+
scope: getPageScope(page)
|
|
825
|
+
}];
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
function extractRenderedCitations(context, sources) {
|
|
829
|
+
const actual = [];
|
|
830
|
+
const blockUtf8Bytes = [];
|
|
831
|
+
let cursor = 0;
|
|
832
|
+
let layoutIntegrity = true;
|
|
833
|
+
for (let index = 0; index < sources.length; index += 1) {
|
|
834
|
+
const attributableStart = cursor;
|
|
835
|
+
if (index > 0) {
|
|
836
|
+
if (!context.startsWith(CONTEXT_SEPARATOR, cursor)) {
|
|
837
|
+
layoutIntegrity = false;
|
|
838
|
+
break;
|
|
839
|
+
}
|
|
840
|
+
cursor += 7;
|
|
841
|
+
}
|
|
842
|
+
const contentMarker = `\n\n${sources[index].content}`;
|
|
843
|
+
const contentStart = context.indexOf(contentMarker, cursor);
|
|
844
|
+
if (contentStart < cursor) {
|
|
845
|
+
layoutIntegrity = false;
|
|
846
|
+
break;
|
|
847
|
+
}
|
|
848
|
+
const sourceLines = context.slice(cursor, contentStart).split(/\r?\n/u).map((line) => line.match(/^Source:\s+(.+)$/u)?.[1]?.trim()).filter((source) => Boolean(source));
|
|
849
|
+
if (sourceLines.length !== 1) layoutIntegrity = false;
|
|
850
|
+
actual.push(...sourceLines.map(canonicalizeSource));
|
|
851
|
+
cursor = contentStart + contentMarker.length;
|
|
852
|
+
blockUtf8Bytes.push(Buffer.byteLength(context.slice(attributableStart, cursor), "utf8"));
|
|
853
|
+
}
|
|
854
|
+
if (cursor !== context.length) layoutIntegrity = false;
|
|
855
|
+
return {
|
|
856
|
+
actual,
|
|
857
|
+
layoutIntegrity,
|
|
858
|
+
blockUtf8Bytes
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
function hasValidStaticSyntax(target) {
|
|
862
|
+
const language = target.language?.toLowerCase();
|
|
863
|
+
if (language === "json") try {
|
|
864
|
+
JSON.parse(target.code);
|
|
865
|
+
return true;
|
|
866
|
+
} catch {
|
|
867
|
+
return false;
|
|
868
|
+
}
|
|
869
|
+
if (language === "javascript" || language === "js" || language === "jsx" || language === "node" || language === "typescript" || language === "ts" || language === "tsx") try {
|
|
870
|
+
const typescript = language === "typescript" || language === "ts" || language === "tsx";
|
|
871
|
+
const jsx = language === "jsx" || language === "tsx";
|
|
872
|
+
const transformed = STATIC_JITI.transform({
|
|
873
|
+
source: target.code,
|
|
874
|
+
filename: `golden-example.${language === "typescript" ? "ts" : language}`,
|
|
875
|
+
ts: typescript,
|
|
876
|
+
jsx,
|
|
877
|
+
async: true
|
|
878
|
+
});
|
|
879
|
+
return !/\bexports\.__JITI_ERROR__\s*=/u.test(transformed);
|
|
880
|
+
} catch {
|
|
881
|
+
return false;
|
|
882
|
+
}
|
|
883
|
+
return true;
|
|
884
|
+
}
|
|
885
|
+
function isStaticExecutable(target, plan) {
|
|
886
|
+
const language = target.language?.toLowerCase();
|
|
887
|
+
if (!target.runnable || !language || !EXECUTABLE_LANGUAGES.has(language)) return false;
|
|
888
|
+
if (!plan || plan.action === "skip") return false;
|
|
889
|
+
if (!target.code.trim()) return false;
|
|
890
|
+
if (target.code.split("\n").some((line) => /^\s*(?:(?:\/\/|#)\s*)?\.\.\.\s*$/u.test(line))) return false;
|
|
891
|
+
return hasValidStaticSyntax(target);
|
|
892
|
+
}
|
|
893
|
+
async function collectRetrievedExamples(candidates) {
|
|
894
|
+
const examples = [];
|
|
895
|
+
for (const candidate of candidates) {
|
|
896
|
+
const targets = extractCodeBlocksFromMarkdown({
|
|
897
|
+
source: candidate.content,
|
|
898
|
+
filePath: candidate.page.sourcePath ?? candidate.page.slug,
|
|
899
|
+
relativePath: candidate.page.slug
|
|
900
|
+
});
|
|
901
|
+
for (const target of targets) examples.push({
|
|
902
|
+
source: candidate.source,
|
|
903
|
+
target
|
|
904
|
+
});
|
|
905
|
+
}
|
|
906
|
+
const plans = await planCodeBlockTargets(examples.map((example) => example.target), STATIC_CODE_PLAN_CONFIG);
|
|
907
|
+
return examples.map((example, index) => ({
|
|
908
|
+
...example,
|
|
909
|
+
executable: isStaticExecutable(example.target, plans[index])
|
|
910
|
+
}));
|
|
911
|
+
}
|
|
912
|
+
function normalizedEqual(left, right) {
|
|
913
|
+
return (left ?? "").trim().toLowerCase() === (right ?? "").trim().toLowerCase();
|
|
914
|
+
}
|
|
915
|
+
function exampleMatches(expected, actual) {
|
|
916
|
+
if (expected.source && !sourceMatches(actual.source, expected.source)) return false;
|
|
917
|
+
if (expected.language && !normalizedEqual(actual.target.language, expected.language)) return false;
|
|
918
|
+
if (expected.framework && !normalizedEqual(actual.target.framework, expected.framework)) return false;
|
|
919
|
+
if (expected.packageManager && !normalizedEqual(actual.target.packageManager, expected.packageManager)) return false;
|
|
920
|
+
if (expected.title && !normalizedEqual(actual.target.title, expected.title)) return false;
|
|
921
|
+
if (expected.includes?.some((fragment) => !actual.target.code.includes(fragment))) return false;
|
|
922
|
+
if (expected.runnable === false) return !actual.target.runnable;
|
|
923
|
+
return actual.executable;
|
|
924
|
+
}
|
|
925
|
+
async function evaluateExamples(expectedExamples, candidates) {
|
|
926
|
+
const actualExamples = await collectRetrievedExamples(candidates);
|
|
927
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
928
|
+
const results = expectedExamples.map((expected) => {
|
|
929
|
+
const index = actualExamples.findIndex((example, candidateIndex) => !claimed.has(candidateIndex) && exampleMatches(expected, example));
|
|
930
|
+
if (index < 0) return {
|
|
931
|
+
expected,
|
|
932
|
+
executable: false,
|
|
933
|
+
passed: false
|
|
934
|
+
};
|
|
935
|
+
claimed.add(index);
|
|
936
|
+
const matched = actualExamples[index];
|
|
937
|
+
return {
|
|
938
|
+
expected,
|
|
939
|
+
matchedId: matched.target.id,
|
|
940
|
+
source: matched.source,
|
|
941
|
+
executable: matched.executable,
|
|
942
|
+
passed: true
|
|
943
|
+
};
|
|
944
|
+
});
|
|
945
|
+
const matched = results.filter((result) => result.passed).length;
|
|
946
|
+
return {
|
|
947
|
+
expected: expectedExamples.length,
|
|
948
|
+
matched,
|
|
949
|
+
executable: results.filter((result) => result.executable).length,
|
|
950
|
+
results,
|
|
951
|
+
passed: matched === expectedExamples.length
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
function buildRetrievalMetrics(task, sources, topK) {
|
|
955
|
+
const expected = uniqueCanonicalSources(task.expect.relevantSources);
|
|
956
|
+
const forbidden = uniqueCanonicalSources(task.expect.forbiddenSources ?? []);
|
|
957
|
+
const retrievedRelevant = expected.filter((source) => sources.some((actual) => sourceMatches(actual.url, source))).length;
|
|
958
|
+
const recallAtK = expected.length > 0 ? retrievedRelevant / expected.length : 0;
|
|
959
|
+
const firstRelevant = sources.find((source) => source.relevant)?.rank ?? null;
|
|
960
|
+
const forbiddenSources = sources.filter((source) => forbidden.some((forbiddenSource) => sourceMatches(source.url, forbiddenSource))).map((source) => source.url);
|
|
961
|
+
const minimumRecall = clamp(task.expect.minRecallAtK ?? 1, 0, 1);
|
|
962
|
+
const maximumRank = normalizePositiveInteger(task.expect.maxFirstRelevantRank, topK, topK);
|
|
963
|
+
return {
|
|
964
|
+
expectedRelevant: expected.length,
|
|
965
|
+
retrievedRelevant,
|
|
966
|
+
recallAtK: round(recallAtK),
|
|
967
|
+
firstRelevantRank: firstRelevant,
|
|
968
|
+
reciprocalRank: firstRelevant ? round(1 / firstRelevant) : 0,
|
|
969
|
+
forbiddenSources,
|
|
970
|
+
passed: expected.length > 0 && recallAtK >= minimumRecall && firstRelevant !== null && firstRelevant <= maximumRank && forbiddenSources.length === 0
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
function buildCitationMetrics(task, context, sources) {
|
|
974
|
+
const expected = uniqueCanonicalSources(task.expect.requiredCitations ?? task.expect.relevantSources);
|
|
975
|
+
const allowed = uniqueCanonicalSources([
|
|
976
|
+
...task.expect.relevantSources,
|
|
977
|
+
...task.expect.requiredCitations ?? [],
|
|
978
|
+
...task.expect.allowedSources ?? []
|
|
979
|
+
]);
|
|
980
|
+
const extracted = extractRenderedCitations(context, sources);
|
|
981
|
+
const actual = extracted.actual;
|
|
982
|
+
const sourceRecords = sources.map((source) => canonicalizeSource(source.url));
|
|
983
|
+
const missing = expected.filter((expectedSource) => !actual.some((actualSource) => sourceMatches(actualSource, expectedSource)));
|
|
984
|
+
const unexpected = actual.filter((actualSource) => !allowed.some((allowedSource) => sourceMatches(actualSource, allowedSource)));
|
|
985
|
+
const correct = actual.filter((actualSource) => allowed.some((allowedSource) => sourceMatches(actualSource, allowedSource))).length;
|
|
986
|
+
const matchedExpected = expected.filter((expectedSource) => actual.some((actualSource) => sourceMatches(actualSource, expectedSource))).length;
|
|
987
|
+
const precision = actual.length > 0 ? correct / actual.length : expected.length === 0 ? 1 : 0;
|
|
988
|
+
const recall = expected.length > 0 ? matchedExpected / expected.length : 1;
|
|
989
|
+
const integrity = extracted.layoutIntegrity && actual.length === sourceRecords.length && actual.every((source, index) => source === sourceRecords[index]);
|
|
990
|
+
return {
|
|
991
|
+
expected,
|
|
992
|
+
actual,
|
|
993
|
+
missing,
|
|
994
|
+
unexpected,
|
|
995
|
+
precision: round(precision),
|
|
996
|
+
recall: round(recall),
|
|
997
|
+
integrity,
|
|
998
|
+
passed: missing.length === 0 && unexpected.length === 0 && integrity
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
function buildSelectionMetrics(filters, candidates) {
|
|
1002
|
+
const requestedFramework = filters?.framework;
|
|
1003
|
+
const requestedVersion = filters?.version;
|
|
1004
|
+
const framework = requestedFramework ? normalizeAgentFramework(requestedFramework) : void 0;
|
|
1005
|
+
const firstFrameworkMatchRank = framework ? candidates.findIndex((candidate) => candidate.scope.framework.includes(framework)) + 1 || null : null;
|
|
1006
|
+
const firstVersionMatchRank = requestedVersion ? candidates.findIndex((candidate) => candidate.scope.version.some((version) => agentVersionConstraintMatches(requestedVersion, version))) + 1 || null : null;
|
|
1007
|
+
const conflictingSources = candidates.filter((candidate) => framework && candidate.scope.framework.length > 0 && !candidate.scope.framework.includes(framework) || requestedVersion && candidate.scope.version.length > 0 && !candidate.scope.version.some((version) => agentVersionConstraintMatches(requestedVersion, version))).map((candidate) => candidate.source);
|
|
1008
|
+
const ambiguousSources = candidates.filter((candidate) => candidate.scope.frameworkAmbiguous || candidate.scope.versionAmbiguous || Boolean(framework && candidate.scope.framework.length === 0) || Boolean(requestedVersion && candidate.scope.version.length === 0)).map((candidate) => candidate.source);
|
|
1009
|
+
return {
|
|
1010
|
+
requestedFramework,
|
|
1011
|
+
requestedVersion,
|
|
1012
|
+
firstFrameworkMatchRank,
|
|
1013
|
+
firstVersionMatchRank,
|
|
1014
|
+
conflictingSources,
|
|
1015
|
+
ambiguousSources,
|
|
1016
|
+
passed: (!framework || firstFrameworkMatchRank !== null) && (!requestedVersion || firstVersionMatchRank !== null) && conflictingSources.length === 0 && ambiguousSources.length === 0
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
function buildUsageMetrics(task, tokenBudget, context, relevantSources) {
|
|
1020
|
+
const usedUtf8Bytes = context.budget.usedUtf8Bytes;
|
|
1021
|
+
const rendered = extractRenderedCitations(context.context, context.sources);
|
|
1022
|
+
const usefulUtf8Bytes = context.sources.reduce((total, source, index) => relevantSources.some((expected) => sourceMatches(source.url, expected)) ? total + (rendered.blockUtf8Bytes[index] ?? 0) : total, 0);
|
|
1023
|
+
const usefulByteRatio = usedUtf8Bytes > 0 ? usefulUtf8Bytes / usedUtf8Bytes : 0;
|
|
1024
|
+
const minimumUsefulRatio = clamp(task.expect.minUsefulByteRatio ?? 0, 0, 1);
|
|
1025
|
+
const withinBudget = usedUtf8Bytes <= tokenBudget;
|
|
1026
|
+
return {
|
|
1027
|
+
budgetUnit: "utf8-bytes",
|
|
1028
|
+
tokenBudget,
|
|
1029
|
+
usedUtf8Bytes,
|
|
1030
|
+
remainingUtf8Bytes: context.budget.remainingUtf8Bytes,
|
|
1031
|
+
estimatedTokens: Math.ceil(usedUtf8Bytes / 4),
|
|
1032
|
+
conservativeTokenUpperBound: usedUtf8Bytes,
|
|
1033
|
+
usefulUtf8Bytes,
|
|
1034
|
+
usefulByteRatio: round(usefulByteRatio),
|
|
1035
|
+
truncated: context.budget.truncated,
|
|
1036
|
+
withinBudget,
|
|
1037
|
+
passed: withinBudget && usefulByteRatio >= minimumUsefulRatio
|
|
1038
|
+
};
|
|
1039
|
+
}
|
|
1040
|
+
function calculateTaskScore(report) {
|
|
1041
|
+
const dimensions = [
|
|
1042
|
+
{
|
|
1043
|
+
weight: 30,
|
|
1044
|
+
score: report.retrieval.forbiddenSources.length > 0 ? 0 : (report.retrieval.recallAtK + report.retrieval.reciprocalRank) / 2
|
|
1045
|
+
},
|
|
1046
|
+
{
|
|
1047
|
+
weight: 25,
|
|
1048
|
+
score: report.citations.integrity ? (report.citations.precision + report.citations.recall) / 2 : 0
|
|
1049
|
+
},
|
|
1050
|
+
...report.hasSelectionExpectation ? [{
|
|
1051
|
+
weight: 20,
|
|
1052
|
+
score: report.selection.passed ? 1 : 0
|
|
1053
|
+
}] : [],
|
|
1054
|
+
...report.hasExampleExpectation ? [{
|
|
1055
|
+
weight: 15,
|
|
1056
|
+
score: report.examples.expected > 0 ? report.examples.matched / report.examples.expected : 1
|
|
1057
|
+
}] : [],
|
|
1058
|
+
{
|
|
1059
|
+
weight: 10,
|
|
1060
|
+
score: report.usage.passed ? 1 : 0
|
|
1061
|
+
}
|
|
1062
|
+
];
|
|
1063
|
+
const totalWeight = dimensions.reduce((sum, dimension) => sum + dimension.weight, 0);
|
|
1064
|
+
return round(dimensions.reduce((sum, dimension) => sum + dimension.weight * dimension.score, 0) / totalWeight * 100);
|
|
1065
|
+
}
|
|
1066
|
+
async function evaluateTask(pages, task, configurationIssues = []) {
|
|
1067
|
+
const topK = normalizePositiveInteger(task.topK, DEFAULT_TOP_K, 50);
|
|
1068
|
+
const tokenBudget = normalizePositiveInteger(task.tokenBudget, DEFAULT_TOKEN_BUDGET, 1e6);
|
|
1069
|
+
const relevantSources = uniqueCanonicalSources(task.expect.relevantSources);
|
|
1070
|
+
const orderedPages = pages.slice().sort((left, right) => compareCodePoints(canonicalizeSource(left.url), canonicalizeSource(right.url)));
|
|
1071
|
+
const pagesByUrl = new Map(orderedPages.map((page) => [sourcePage(page.url), page]));
|
|
1072
|
+
const contextOptions = {
|
|
1073
|
+
pages: orderedPages,
|
|
1074
|
+
query: task.query,
|
|
1075
|
+
framework: task.filters?.framework,
|
|
1076
|
+
version: task.filters?.version,
|
|
1077
|
+
locale: task.filters?.locale,
|
|
1078
|
+
maxResults: topK
|
|
1079
|
+
};
|
|
1080
|
+
const [rankedContext, budgetedContext] = await Promise.all([buildDocsMcpContext({
|
|
1081
|
+
...contextOptions,
|
|
1082
|
+
tokenBudget: Number.MAX_SAFE_INTEGER
|
|
1083
|
+
}), buildDocsMcpContext({
|
|
1084
|
+
...contextOptions,
|
|
1085
|
+
tokenBudget
|
|
1086
|
+
})]);
|
|
1087
|
+
const rankedSources = toGoldenSources(rankedContext.sources, relevantSources);
|
|
1088
|
+
const budgetedSources = toGoldenSources(budgetedContext.sources, relevantSources);
|
|
1089
|
+
const rankedCandidates = toContextCandidates(rankedContext.sources, pagesByUrl);
|
|
1090
|
+
const budgetedCandidates = toContextCandidates(budgetedContext.sources, pagesByUrl);
|
|
1091
|
+
const retrieval = buildRetrievalMetrics(task, rankedSources, topK);
|
|
1092
|
+
const citations = buildCitationMetrics(task, budgetedContext.context, budgetedContext.sources);
|
|
1093
|
+
const selection = buildSelectionMetrics(task.filters, rankedCandidates);
|
|
1094
|
+
const examples = await evaluateExamples(task.expect.examples ?? [], budgetedCandidates);
|
|
1095
|
+
const usage = buildUsageMetrics(task, tokenBudget, budgetedContext, relevantSources);
|
|
1096
|
+
const issues = configurationIssues.map((issue) => `Invalid golden task configuration: ${issue}`);
|
|
1097
|
+
if (relevantSources.length === 0) issues.push("No relevantSources are configured for this task.");
|
|
1098
|
+
if (!retrieval.passed) issues.push("Retrieval did not satisfy the expected recall, rank, or exclusion rules.");
|
|
1099
|
+
if (!citations.passed) issues.push("Rendered citations are missing, unexpected, or do not match source records.");
|
|
1100
|
+
if (!selection.passed) issues.push("Retrieved context is ambiguous or does not explicitly match the requested framework/version.");
|
|
1101
|
+
if (!examples.passed) issues.push("One or more expected runnable examples were not retrieved or are not executable.");
|
|
1102
|
+
if (!usage.passed) issues.push("Context exceeded its budget or did not meet the configured useful-byte ratio.");
|
|
1103
|
+
const passed = configurationIssues.length === 0 && relevantSources.length > 0 && retrieval.passed && citations.passed && selection.passed && examples.passed && usage.passed;
|
|
1104
|
+
const calculatedScore = configurationIssues.length > 0 ? 0 : calculateTaskScore({
|
|
1105
|
+
retrieval,
|
|
1106
|
+
citations,
|
|
1107
|
+
selection,
|
|
1108
|
+
examples,
|
|
1109
|
+
usage,
|
|
1110
|
+
hasSelectionExpectation: Boolean(task.filters?.framework || task.filters?.version),
|
|
1111
|
+
hasExampleExpectation: (task.expect.examples?.length ?? 0) > 0
|
|
1112
|
+
});
|
|
1113
|
+
const score = passed ? calculatedScore : Math.min(calculatedScore, 99);
|
|
1114
|
+
return {
|
|
1115
|
+
id: task.id,
|
|
1116
|
+
query: task.query,
|
|
1117
|
+
status: passed ? "passed" : "failed",
|
|
1118
|
+
passed,
|
|
1119
|
+
score,
|
|
1120
|
+
context: budgetedContext.context,
|
|
1121
|
+
sources: budgetedSources,
|
|
1122
|
+
retrieval,
|
|
1123
|
+
citations,
|
|
1124
|
+
selection,
|
|
1125
|
+
examples,
|
|
1126
|
+
usage,
|
|
1127
|
+
issues
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
/**
|
|
1131
|
+
* Run deterministic, offline golden-task evaluations against MCP-ready docs pages.
|
|
1132
|
+
* An empty task list is intentionally unmeasured so CI cannot turn absent coverage into a pass.
|
|
1133
|
+
*/
|
|
1134
|
+
async function runDocsGoldenTasks(pages, tasks) {
|
|
1135
|
+
const runtimeTasks = tasks;
|
|
1136
|
+
if (runtimeTasks === void 0 || Array.isArray(runtimeTasks) && runtimeTasks.length === 0) return {
|
|
1137
|
+
status: "unmeasured",
|
|
1138
|
+
passed: null,
|
|
1139
|
+
score: null,
|
|
1140
|
+
taskCount: 0,
|
|
1141
|
+
passedTaskCount: 0,
|
|
1142
|
+
failedTaskCount: 0,
|
|
1143
|
+
tasks: []
|
|
1144
|
+
};
|
|
1145
|
+
const normalizedTasks = Array.isArray(runtimeTasks) ? runtimeTasks.map((task, index) => normalizeGoldenTaskInput(task, index)) : [normalizeGoldenTaskInput(runtimeTasks, 0, ["agent.evaluations.tasks must be an array."])];
|
|
1146
|
+
const idCounts = normalizedTasks.reduce((counts, input) => {
|
|
1147
|
+
counts.set(input.task.id, (counts.get(input.task.id) ?? 0) + 1);
|
|
1148
|
+
return counts;
|
|
1149
|
+
}, /* @__PURE__ */ new Map());
|
|
1150
|
+
for (const input of normalizedTasks) if ((idCounts.get(input.task.id) ?? 0) > 1) input.issues.push(`task id ${JSON.stringify(input.task.id)} is duplicated.`);
|
|
1151
|
+
const reports = await Promise.all(normalizedTasks.map(({ task, issues }) => evaluateTask(pages, task, issues)));
|
|
1152
|
+
const passedTaskCount = reports.filter((task) => task.passed).length;
|
|
1153
|
+
const failedTaskCount = reports.length - passedTaskCount;
|
|
1154
|
+
return {
|
|
1155
|
+
status: failedTaskCount === 0 ? "passed" : "failed",
|
|
1156
|
+
passed: failedTaskCount === 0,
|
|
1157
|
+
score: round(reports.reduce((sum, task) => sum + task.score, 0) / reports.length),
|
|
1158
|
+
taskCount: reports.length,
|
|
1159
|
+
passedTaskCount,
|
|
1160
|
+
failedTaskCount,
|
|
1161
|
+
tasks: reports
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
//#endregion
|
|
1166
|
+
export { buildDocsReviewWorkflowPathFilters as a, resolveDocsReviewConfig as c, buildDocsReviewWorkflow as i, DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD as n, ensureDocsReviewWorkflow as o, DEFAULT_DOCS_REVIEW_WORKFLOW_PATH as r, readDocsReviewConfigFromSource as s, runDocsGoldenTasks as t };
|