@farming-labs/docs 0.2.57 → 0.2.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/{agent-CxDzPqtl.mjs → agent-BFqyqEnC.mjs} +557 -71
  2. package/dist/{agent-v6R0V3lm.mjs → agent-DlxriaTs.mjs} +3 -3
  3. package/dist/agent-evals-kJs2Y9xR.mjs +2144 -0
  4. package/dist/{agent-export-CZR20g0e.mjs → agent-export-BgUaiW8f.mjs} +20 -10
  5. package/dist/{agents-BPonjolo.mjs → agents-Djh-HXih.mjs} +15 -7
  6. package/dist/cli/index.mjs +29 -29
  7. package/dist/client/react.d.mts +1 -1
  8. package/dist/{cloud-C45jpBtp.mjs → cloud-BH_sHX64.mjs} +3 -3
  9. package/dist/{cloud-ask-ai-CvRVnwdK.d.mts → cloud-ask-ai-B2WnG4fF.d.mts} +1 -1
  10. package/dist/{code-blocks-C9awWzEQ.mjs → code-blocks-qe0T8-xe.mjs} +1 -1
  11. package/dist/{codeblocks-CSqgqbg-.mjs → codeblocks-Bq67u32v.mjs} +2 -2
  12. package/dist/{dev-BA2kRuCn.mjs → dev-CAQlbguk.mjs} +2 -2
  13. package/dist/docs-cloud-server.d.mts +2 -2
  14. package/dist/docs-cloud-server.mjs +1 -1
  15. package/dist/{doctor-svA9Flsx.mjs → doctor-CO1VMcF_.mjs} +208 -41
  16. package/dist/{downgrade-BStNhyCc.mjs → downgrade-BZs86NVr.mjs} +2 -2
  17. package/dist/{agent-surface-drift-rGFS38Ak.mjs → golden-evaluations-BN9u2wxw.mjs} +60 -4
  18. package/dist/index.d.mts +98 -20
  19. package/dist/index.mjs +6 -5
  20. package/dist/{init-Dm0yDz6v.mjs → init-B_9ENq8Z.mjs} +3 -3
  21. package/dist/{mcp-BYIgKsPn.mjs → mcp-BAJr3wC2.mjs} +6 -6
  22. package/dist/mcp.d.mts +1 -1
  23. package/dist/mcp.mjs +149 -7
  24. package/dist/metadata-BDuewuzq.mjs +237 -0
  25. package/dist/{package-version-DRIc08EU.mjs → package-version-qik_4J6C.mjs} +1 -1
  26. package/dist/{reading-time-W_Id2ayC.mjs → reading-time-BkEft6SD.mjs} +307 -250
  27. package/dist/{review-DZ81wm7h.mjs → review-NC-sOdXn.mjs} +44 -32
  28. package/dist/{robots-mSc98PMp.mjs → robots-DskPvGPw.mjs} +4 -4
  29. package/dist/{robots-DRpoiYAV.mjs → robots-ltltiLJF.mjs} +4 -1
  30. package/dist/{search-9uEaC23e.d.mts → search-C1JitPwi.d.mts} +153 -6
  31. package/dist/{search-OHp36znm.mjs → search-D57JXQLj.mjs} +150 -39
  32. package/dist/{search-AMEoabYU.mjs → search-o4Ud6OXv.mjs} +6 -6
  33. package/dist/server.d.mts +108 -6
  34. package/dist/server.mjs +6 -6
  35. package/dist/{sitemap-Cyc0Bbbn.mjs → sitemap-Cq-Yj_iA.mjs} +7 -7
  36. package/dist/{sitemap-server-zFYctvCa.mjs → sitemap-server-C1ibVKOy.mjs} +2 -1
  37. package/dist/{types-R-DN23yw.d.mts → types-XHABMh_f.d.mts} +96 -6
  38. package/dist/{upgrade-CXYRNV0C.mjs → upgrade-BCJTCW3O.mjs} +2 -2
  39. package/package.json +1 -1
  40. package/dist/agent-evals-BD17jOqL.mjs +0 -1166
  41. /package/dist/{agent-scope-B8lptqCd.mjs → agent-scope-CCaIY1aK.mjs} +0 -0
  42. /package/dist/{cloud-ask-ai-1k0q6OAH.mjs → cloud-ask-ai-hnJfj8-X.mjs} +0 -0
  43. /package/dist/{config-DZbZiYEa.mjs → config-DASewQ0x.mjs} +0 -0
  44. /package/dist/{errors-BFUtdZfC.mjs → errors-CVqZ3kOO.mjs} +0 -0
  45. /package/dist/{templates-1Cod8KrJ.mjs → templates-CfQjpDWW.mjs} +0 -0
  46. /package/dist/{utils-DBCCkkJS.mjs → utils-6UCLxv4B.mjs} +0 -0
@@ -1,1166 +0,0 @@
1
- import { a as normalizeAgentScopeValues, i as normalizeAgentLocale, n as agentVersionConstraintsOverlap, o as normalizeAgentVersion, r as normalizeAgentFramework, t as agentVersionConstraintMatches } from "./agent-scope-B8lptqCd.mjs";
2
- import { buildDocsMcpContext } from "./mcp.mjs";
3
- import { n as planCodeBlockTargets, r as resolveDocsCodeBlocksValidateConfig, t as extractCodeBlocksFromMarkdown } from "./code-blocks-C9awWzEQ.mjs";
4
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
5
- import { dirname, isAbsolute, join, relative } from "node:path";
6
- import { createJiti } from "jiti";
7
-
8
- //#region src/review.ts
9
- const DEFAULT_DOCS_REVIEW_WORKFLOW_PATH = ".github/workflows/docs-review.yml";
10
- const DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW_PATH = ".github/workflows/docs-review-reusable.yml";
11
- const DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW = "farming-labs/docs/.github/workflows/docs-review-reusable.yml@main";
12
- const LOCAL_DOCS_REVIEW_REUSABLE_WORKFLOW = "./.github/workflows/docs-review-reusable.yml";
13
- const DEFAULT_DOCS_REVIEW_PNPM_VERSION = "10";
14
- const DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD = 80;
15
- const DEFAULT_DOCS_REVIEW_CI_NAME = "docs-review";
16
- const DEFAULT_REVIEW_RULES = {
17
- brokenLinks: "error",
18
- frontmatter: "error",
19
- duplicateSlugs: "error",
20
- invalidMdx: "error",
21
- configExamples: "warn",
22
- codeFenceMetadata: "warn",
23
- runnableMetadata: "warn",
24
- agentContext: "warn",
25
- commandHealth: "warn",
26
- relatedCoverage: "suggestion",
27
- configConfidence: "warn",
28
- agentSurfaceDrift: "error",
29
- goldenTasks: "warn"
30
- };
31
- const DEFAULT_REVIEW_WEIGHTS = {
32
- error: 20,
33
- warn: 8,
34
- suggestion: 2
35
- };
36
- const REVIEW_SEVERITIES = new Set([
37
- "off",
38
- "suggestion",
39
- "warn",
40
- "error"
41
- ]);
42
- const REVIEW_CI_MODES = new Set([
43
- "off",
44
- "warn",
45
- "block"
46
- ]);
47
- const FILE_EXTS = [
48
- "tsx",
49
- "ts",
50
- "jsx",
51
- "js"
52
- ];
53
- function resolveDocsReviewConfig(review) {
54
- if (review === false) return {
55
- enabled: false,
56
- ci: {
57
- enabled: false,
58
- name: DEFAULT_DOCS_REVIEW_CI_NAME,
59
- mode: "off",
60
- annotations: false,
61
- comment: false
62
- },
63
- score: {
64
- threshold: DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD,
65
- weights: DEFAULT_REVIEW_WEIGHTS
66
- },
67
- rules: DEFAULT_REVIEW_RULES
68
- };
69
- const objectConfig = review && typeof review === "object" ? review : {};
70
- const ciConfig = objectConfig.ci;
71
- const ciObject = ciConfig && typeof ciConfig === "object" ? ciConfig : {};
72
- const configuredMode = ciObject.mode;
73
- const mode = configuredMode && REVIEW_CI_MODES.has(configuredMode) ? configuredMode : "warn";
74
- const enabled = objectConfig.enabled !== false;
75
- const ciEnabled = enabled && ciConfig !== false && ciObject.enabled !== false && mode !== "off";
76
- return {
77
- enabled,
78
- ci: {
79
- enabled: ciEnabled,
80
- name: normalizeCiName(ciObject.name),
81
- mode: ciEnabled ? mode : "off",
82
- annotations: ciObject.annotations !== false,
83
- comment: ciObject.comment !== false
84
- },
85
- score: {
86
- threshold: clampScoreThreshold(objectConfig.score?.threshold),
87
- weights: {
88
- error: normalizeWeight(objectConfig.score?.weights?.error, DEFAULT_REVIEW_WEIGHTS.error),
89
- warn: normalizeWeight(objectConfig.score?.weights?.warn, DEFAULT_REVIEW_WEIGHTS.warn),
90
- suggestion: normalizeWeight(objectConfig.score?.weights?.suggestion, DEFAULT_REVIEW_WEIGHTS.suggestion)
91
- }
92
- },
93
- rules: {
94
- ...DEFAULT_REVIEW_RULES,
95
- ...normalizeRules(objectConfig.rules)
96
- }
97
- };
98
- }
99
- function buildDocsReviewWorkflow(options = {}) {
100
- const packageManager = options.packageManager ?? "npm";
101
- const ciName = normalizeCiName(options.ciName);
102
- const projectDir = normalizeProjectDir(options.projectDir);
103
- const configPath = options.configPath ?? "docs.config.ts";
104
- const reusableWorkflow = options.reusableWorkflow ?? DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW;
105
- const filters = normalizePathFilters(options.pathFilters);
106
- const workflowInputs = [
107
- ["check-name", ciName],
108
- ["config", configPath],
109
- ["working-directory", projectDir],
110
- ["package-manager", packageManager],
111
- options.pnpmVersion ? ["pnpm-version", options.pnpmVersion] : void 0,
112
- options.buildCommand ? ["build-command", options.buildCommand] : void 0,
113
- options.reviewCommand ? ["review-command", options.reviewCommand] : void 0
114
- ].filter((input) => Boolean(input)).map(([key, value]) => ` ${key}: ${JSON.stringify(value)}`).join("\n");
115
- return `# Generated by @farming-labs/docs. You can edit this file.
116
- name: Docs Review
117
-
118
- on:
119
- pull_request:
120
- paths:
121
- ${filters.map((filter) => ` - ${JSON.stringify(filter)}`).join("\n")}
122
-
123
- permissions:
124
- contents: read
125
- checks: write
126
- pull-requests: write
127
-
128
- jobs:
129
- docs-review:
130
- name: ${JSON.stringify(ciName)}
131
- uses: ${reusableWorkflow}
132
- with:
133
- ${workflowInputs}
134
- permissions:
135
- contents: read
136
- checks: write
137
- pull-requests: write
138
- `;
139
- }
140
- function ensureDocsReviewWorkflow(options) {
141
- const rootDir = options.rootDir;
142
- const repoRoot = findGitRoot(rootDir) ?? rootDir;
143
- const workflowRelativePath = options.workflowPath ?? DEFAULT_DOCS_REVIEW_WORKFLOW_PATH;
144
- const workflowPath = isAbsolute(workflowRelativePath) ? workflowRelativePath : join(repoRoot, workflowRelativePath);
145
- const resultRelativePath = toPosixPath(relative(repoRoot, workflowPath));
146
- const review = resolveDocsReviewConfig(options.config?.review ?? readDocsReviewConfigFromSource(options.configContent ?? readConfig(rootDir)));
147
- if (!review.enabled || !review.ci.enabled) return {
148
- status: "disabled",
149
- path: workflowPath,
150
- relativePath: resultRelativePath
151
- };
152
- if (existsSync(workflowPath)) return {
153
- status: "exists",
154
- path: workflowPath,
155
- relativePath: resultRelativePath
156
- };
157
- const configPath = options.configPath ?? findDocsConfigPath(rootDir);
158
- const configContent = options.configContent ?? readConfig(rootDir, configPath);
159
- const projectDir = toPosixPath(relative(repoRoot, rootDir)) || ".";
160
- const packageManager = detectPackageManager(repoRoot);
161
- const workflow = buildDocsReviewWorkflow({
162
- packageManager,
163
- ciName: review.ci.name,
164
- reusableWorkflow: hasLocalDocsWorkspacePackage(repoRoot) ? LOCAL_DOCS_REVIEW_REUSABLE_WORKFLOW : DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW,
165
- pnpmVersion: detectPnpmVersionFallback(repoRoot, packageManager),
166
- projectDir,
167
- configPath,
168
- buildCommand: detectLocalDocsCliBuildCommand(repoRoot, packageManager),
169
- reviewCommand: detectLocalDocsCliReviewCommand(repoRoot, rootDir),
170
- pathFilters: buildDocsReviewWorkflowPathFilters({
171
- rootDir,
172
- repoRoot,
173
- config: options.config,
174
- configPath,
175
- configContent
176
- })
177
- });
178
- mkdirSync(dirname(workflowPath), { recursive: true });
179
- writeFileSync(workflowPath, workflow, "utf-8");
180
- options.log?.(`[docs] Created ${resultRelativePath} for Docs Review CI.`);
181
- return {
182
- status: "created",
183
- path: workflowPath,
184
- relativePath: resultRelativePath
185
- };
186
- }
187
- function readDocsReviewConfigFromSource(content) {
188
- if (!content) return void 0;
189
- const rootObject = extractRootObjectLiteral(content) ?? content;
190
- const reviewCursor = findTopLevelPropertyValueIndex(rootObject, "review");
191
- if (reviewCursor === void 0) return void 0;
192
- if (rootObject.startsWith("true", reviewCursor)) return true;
193
- if (rootObject.startsWith("false", reviewCursor)) return false;
194
- if (rootObject[reviewCursor] !== "{") return void 0;
195
- const reviewEnd = findMatchingObjectEnd(rootObject, reviewCursor);
196
- if (reviewEnd === void 0) return void 0;
197
- const reviewBlock = rootObject.slice(reviewCursor + 1, reviewEnd);
198
- const scoreBlock = extractTopLevelObjectLiteral(reviewBlock, "score");
199
- const ciCursor = findTopLevelPropertyValueIndex(reviewBlock, "ci");
200
- const rulesBlock = extractTopLevelObjectLiteral(reviewBlock, "rules");
201
- const ci = ciCursor === void 0 ? void 0 : reviewBlock.startsWith("true", ciCursor) ? true : reviewBlock.startsWith("false", ciCursor) ? false : reviewBlock[ciCursor] === "{" ? readReviewCiObject(reviewBlock, ciCursor) : void 0;
202
- return {
203
- enabled: readTopLevelBoolean(reviewBlock, "enabled"),
204
- score: scoreBlock ? {
205
- threshold: readTopLevelNumber(scoreBlock, "threshold"),
206
- weights: readReviewWeights(scoreBlock)
207
- } : void 0,
208
- ci,
209
- rules: rulesBlock ? readReviewRules(rulesBlock) : void 0
210
- };
211
- }
212
- function buildDocsReviewWorkflowPathFilters(options) {
213
- const repoRoot = options.repoRoot ?? findGitRoot(options.rootDir) ?? options.rootDir;
214
- const configPath = options.configPath ?? findDocsConfigPath(options.rootDir);
215
- const configContent = options.configContent ?? readConfig(options.rootDir, configPath);
216
- const entry = options.config?.entry ?? readTopLevelString(configContent, "entry") ?? "docs";
217
- const contentDir = options.config?.contentDir ?? readTopLevelString(configContent, "contentDir");
218
- const projectDir = toPosixPath(relative(repoRoot, options.rootDir));
219
- const projectPrefix = projectDir && projectDir !== "." ? `${projectDir}/` : "";
220
- const candidates = [
221
- prefixPath(projectPrefix, configPath),
222
- prefixPath(projectPrefix, `${entry}/**`),
223
- prefixPath(projectPrefix, `app/${entry}/**`),
224
- prefixPath(projectPrefix, `src/app/${entry}/**`),
225
- contentDir ? prefixPath(projectPrefix, `${trimSlashes(contentDir)}/**`) : void 0,
226
- prefixPath(projectPrefix, "content/docs/**"),
227
- prefixPath(projectPrefix, "src/content/docs/**"),
228
- prefixPath(projectPrefix, "src/lib/docs.config.*"),
229
- DEFAULT_DOCS_REVIEW_WORKFLOW_PATH,
230
- DEFAULT_DOCS_REVIEW_REUSABLE_WORKFLOW_PATH
231
- ];
232
- return Array.from(new Set(candidates.filter((candidate) => Boolean(candidate))));
233
- }
234
- function normalizeRules(rules) {
235
- if (!rules) return {};
236
- const knownRules = new Set(Object.keys(DEFAULT_REVIEW_RULES));
237
- return Object.fromEntries(Object.entries(rules).filter((entry) => knownRules.has(entry[0]) && REVIEW_SEVERITIES.has(entry[1])));
238
- }
239
- function normalizeWeight(value, fallback) {
240
- return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
241
- }
242
- function normalizeCiName(value) {
243
- return value?.trim() || DEFAULT_DOCS_REVIEW_CI_NAME;
244
- }
245
- function clampScoreThreshold(value) {
246
- if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD;
247
- return Math.min(100, Math.max(0, Math.round(value)));
248
- }
249
- function readReviewCiObject(source, cursor) {
250
- const end = findMatchingObjectEnd(source, cursor);
251
- if (end === void 0) return void 0;
252
- const block = source.slice(cursor + 1, end);
253
- const mode = readTopLevelString(block, "mode");
254
- return {
255
- enabled: readTopLevelBoolean(block, "enabled"),
256
- name: readTopLevelString(block, "name"),
257
- mode: mode && REVIEW_CI_MODES.has(mode) ? mode : void 0,
258
- annotations: readTopLevelBoolean(block, "annotations"),
259
- comment: readTopLevelBoolean(block, "comment")
260
- };
261
- }
262
- function readReviewRules(block) {
263
- const rules = {};
264
- for (const key of Object.keys(DEFAULT_REVIEW_RULES)) {
265
- const value = readTopLevelString(block, key);
266
- if (value && REVIEW_SEVERITIES.has(value)) rules[key] = value;
267
- }
268
- return rules;
269
- }
270
- function readReviewWeights(block) {
271
- const weightsBlock = extractTopLevelObjectLiteral(block, "weights");
272
- if (!weightsBlock) return void 0;
273
- return {
274
- error: readTopLevelNumber(weightsBlock, "error"),
275
- warn: readTopLevelNumber(weightsBlock, "warn"),
276
- suggestion: readTopLevelNumber(weightsBlock, "suggestion")
277
- };
278
- }
279
- function detectPackageManager(rootDir) {
280
- if (existsSync(join(rootDir, "pnpm-lock.yaml"))) return "pnpm";
281
- if (existsSync(join(rootDir, "yarn.lock"))) return "yarn";
282
- if (existsSync(join(rootDir, "bun.lock")) || existsSync(join(rootDir, "bun.lockb"))) return "bun";
283
- return "npm";
284
- }
285
- function detectLocalDocsCliBuildCommand(repoRoot, packageManager) {
286
- if (!hasLocalDocsWorkspacePackage(repoRoot)) return void 0;
287
- if (packageManager === "pnpm") return "pnpm --filter @farming-labs/docs run build";
288
- if (packageManager === "yarn") return "yarn workspace @farming-labs/docs build";
289
- if (packageManager === "bun") return "bun run --filter @farming-labs/docs build";
290
- return "npm run build --workspace=@farming-labs/docs";
291
- }
292
- function detectPnpmVersionFallback(repoRoot, packageManager) {
293
- if (packageManager !== "pnpm") return void 0;
294
- try {
295
- if (JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf-8")).packageManager?.startsWith("pnpm@")) return void 0;
296
- } catch {}
297
- return DEFAULT_DOCS_REVIEW_PNPM_VERSION;
298
- }
299
- function detectLocalDocsCliReviewCommand(repoRoot, rootDir) {
300
- if (!hasLocalDocsWorkspacePackage(repoRoot)) return void 0;
301
- return `node ${shellQuote(toPosixPath(relative(rootDir, join(repoRoot, "packages", "docs", "dist", "cli", "index.mjs"))) || "./packages/docs/dist/cli/index.mjs")}`;
302
- }
303
- function hasLocalDocsWorkspacePackage(repoRoot) {
304
- const packageJsonPath = join(repoRoot, "packages", "docs", "package.json");
305
- if (!existsSync(packageJsonPath)) return false;
306
- try {
307
- const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
308
- return packageJson.name === "@farming-labs/docs" && Boolean(packageJson.scripts?.build);
309
- } catch {
310
- return false;
311
- }
312
- }
313
- function shellQuote(value) {
314
- if (/^[A-Za-z0-9_./:-]+$/.test(value)) return value;
315
- return `'${value.replaceAll("'", "'\\''")}'`;
316
- }
317
- function normalizeProjectDir(value) {
318
- if (!value || value === "") return ".";
319
- return toPosixPath(value).replace(/\/+$/, "") || ".";
320
- }
321
- function normalizePathFilters(filters) {
322
- const normalized = (filters && filters.length > 0 ? filters : ["docs.config.*", "app/docs/**"]).map((filter) => toPosixPath(filter).replace(/^\.\/+/, "")).filter(Boolean);
323
- return Array.from(new Set(normalized));
324
- }
325
- function findGitRoot(start) {
326
- let current = start;
327
- while (true) {
328
- if (existsSync(join(current, ".git"))) return current;
329
- const parent = dirname(current);
330
- if (parent === current) return void 0;
331
- current = parent;
332
- }
333
- }
334
- function findDocsConfigPath(rootDir) {
335
- for (const ext of FILE_EXTS) {
336
- const path = `docs.config.${ext}`;
337
- if (existsSync(join(rootDir, path))) return path;
338
- }
339
- return "docs.config.ts";
340
- }
341
- function readConfig(rootDir, configPath = findDocsConfigPath(rootDir)) {
342
- const fullPath = join(rootDir, configPath);
343
- if (!existsSync(fullPath)) return void 0;
344
- try {
345
- return readFileSync(fullPath, "utf-8");
346
- } catch {
347
- return;
348
- }
349
- }
350
- function prefixPath(prefix, value) {
351
- return `${prefix}${toPosixPath(value).replace(/^\.\/+/, "")}`;
352
- }
353
- function trimSlashes(value) {
354
- return toPosixPath(value).replace(/^\/+|\/+$/g, "");
355
- }
356
- function toPosixPath(value) {
357
- return value.replaceAll("\\", "/");
358
- }
359
- function extractRootObjectLiteral(content) {
360
- const candidateIndexes = [content.search(/\bdefineDocs\s*\(/), content.search(/\bexport\s+default\b/)].filter((value) => value !== -1);
361
- for (const startIndex of candidateIndexes) {
362
- const braceStart = content.indexOf("{", startIndex);
363
- if (braceStart === -1) continue;
364
- const braceEnd = findMatchingObjectEnd(content, braceStart);
365
- if (braceEnd !== void 0) return content.slice(braceStart + 1, braceEnd);
366
- }
367
- }
368
- function extractTopLevelObjectLiteral(content, key) {
369
- const cursor = findTopLevelPropertyValueIndex(content, key);
370
- if (cursor === void 0 || content[cursor] !== "{") return void 0;
371
- const end = findMatchingObjectEnd(content, cursor);
372
- return end === void 0 ? void 0 : content.slice(cursor + 1, end);
373
- }
374
- function readTopLevelBoolean(content, key) {
375
- const cursor = findTopLevelPropertyValueIndex(content, key);
376
- if (cursor === void 0) return void 0;
377
- if (content.startsWith("true", cursor)) return true;
378
- if (content.startsWith("false", cursor)) return false;
379
- }
380
- function readTopLevelNumber(content, key) {
381
- const cursor = findTopLevelPropertyValueIndex(content, key);
382
- if (cursor === void 0) return void 0;
383
- const match = content.slice(cursor).match(/^-?\d+(?:\.\d+)?/);
384
- if (!match) return void 0;
385
- const value = Number(match[0]);
386
- return Number.isFinite(value) ? value : void 0;
387
- }
388
- function readTopLevelString(content, key) {
389
- if (!content) return void 0;
390
- const cursor = findTopLevelPropertyValueIndex(content, key);
391
- if (cursor === void 0) return void 0;
392
- const quote = content[cursor];
393
- if (quote !== "\"" && quote !== "'") return void 0;
394
- let value = "";
395
- for (let index = cursor + 1; index < content.length; index += 1) {
396
- const char = content[index];
397
- if (char === "\\") {
398
- const escaped = content[index + 1];
399
- if (escaped) {
400
- value += escaped;
401
- index += 1;
402
- }
403
- continue;
404
- }
405
- if (char === quote) return value;
406
- value += char;
407
- }
408
- }
409
- function findTopLevelPropertyValueIndex(block, key) {
410
- let objectDepth = 0;
411
- let arrayDepth = 0;
412
- let parenDepth = 0;
413
- let inString = null;
414
- let inLineComment = false;
415
- let inBlockComment = false;
416
- let escaped = false;
417
- for (let index = 0; index < block.length; index += 1) {
418
- const char = block[index];
419
- const next = block[index + 1];
420
- if (inLineComment) {
421
- if (char === "\n") inLineComment = false;
422
- continue;
423
- }
424
- if (inBlockComment) {
425
- if (char === "*" && next === "/") {
426
- inBlockComment = false;
427
- index += 1;
428
- }
429
- continue;
430
- }
431
- if (inString) {
432
- if (escaped) {
433
- escaped = false;
434
- continue;
435
- }
436
- if (char === "\\") {
437
- escaped = true;
438
- continue;
439
- }
440
- if (char === inString) inString = null;
441
- continue;
442
- }
443
- if (char === "/" && next === "/") {
444
- inLineComment = true;
445
- index += 1;
446
- continue;
447
- }
448
- if (char === "/" && next === "*") {
449
- inBlockComment = true;
450
- index += 1;
451
- continue;
452
- }
453
- if (char === "\"" || char === "'" || char === "`") {
454
- inString = char;
455
- continue;
456
- }
457
- if (char === "{") {
458
- objectDepth += 1;
459
- continue;
460
- }
461
- if (char === "}") {
462
- objectDepth = Math.max(0, objectDepth - 1);
463
- continue;
464
- }
465
- if (char === "[") {
466
- arrayDepth += 1;
467
- continue;
468
- }
469
- if (char === "]") {
470
- arrayDepth = Math.max(0, arrayDepth - 1);
471
- continue;
472
- }
473
- if (char === "(") {
474
- parenDepth += 1;
475
- continue;
476
- }
477
- if (char === ")") {
478
- parenDepth = Math.max(0, parenDepth - 1);
479
- continue;
480
- }
481
- if (objectDepth !== 0 || arrayDepth !== 0 || parenDepth !== 0) continue;
482
- if (!block.startsWith(key, index)) continue;
483
- const before = block[index - 1] ?? "";
484
- const after = block[index + key.length] ?? "";
485
- if (/[A-Za-z0-9_$]/.test(before) || /[A-Za-z0-9_$]/.test(after)) continue;
486
- let cursor = index + key.length;
487
- while (/\s/.test(block[cursor] ?? "")) cursor += 1;
488
- if (block[cursor] !== ":") continue;
489
- cursor += 1;
490
- while (/\s/.test(block[cursor] ?? "")) cursor += 1;
491
- return cursor;
492
- }
493
- }
494
- function findMatchingObjectEnd(block, start) {
495
- let depth = 0;
496
- let inString = null;
497
- let inLineComment = false;
498
- let inBlockComment = false;
499
- let escaped = false;
500
- for (let index = start; index < block.length; index += 1) {
501
- const char = block[index];
502
- const next = block[index + 1];
503
- if (inLineComment) {
504
- if (char === "\n") inLineComment = false;
505
- continue;
506
- }
507
- if (inBlockComment) {
508
- if (char === "*" && next === "/") {
509
- inBlockComment = false;
510
- index += 1;
511
- }
512
- continue;
513
- }
514
- if (inString) {
515
- if (escaped) {
516
- escaped = false;
517
- continue;
518
- }
519
- if (char === "\\") {
520
- escaped = true;
521
- continue;
522
- }
523
- if (char === inString) inString = null;
524
- continue;
525
- }
526
- if (char === "/" && next === "/") {
527
- inLineComment = true;
528
- index += 1;
529
- continue;
530
- }
531
- if (char === "/" && next === "*") {
532
- inBlockComment = true;
533
- index += 1;
534
- continue;
535
- }
536
- if (char === "\"" || char === "'" || char === "`") {
537
- inString = char;
538
- continue;
539
- }
540
- if (char === "{") {
541
- depth += 1;
542
- continue;
543
- }
544
- if (char !== "}") continue;
545
- depth -= 1;
546
- if (depth === 0) return index;
547
- }
548
- }
549
-
550
- //#endregion
551
- //#region src/agent-evals.ts
552
- const DEFAULT_TOP_K = 5;
553
- const DEFAULT_TOKEN_BUDGET = 4e3;
554
- const CONTEXT_SEPARATOR = "\n\n---\n\n";
555
- const STATIC_CODE_PLAN_CONFIG = resolveDocsCodeBlocksValidateConfig(true);
556
- const STATIC_JITI = createJiti(import.meta.url, {
557
- fsCache: false,
558
- moduleCache: false
559
- });
560
- const EXECUTABLE_LANGUAGES = new Set([
561
- "bash",
562
- "curl",
563
- "javascript",
564
- "js",
565
- "jsx",
566
- "json",
567
- "node",
568
- "py",
569
- "python",
570
- "rb",
571
- "ruby",
572
- "sh",
573
- "shell",
574
- "ts",
575
- "tsx",
576
- "typescript",
577
- "zsh"
578
- ]);
579
- function isRecord(value) {
580
- return typeof value === "object" && value !== null && !Array.isArray(value);
581
- }
582
- function normalizeRuntimeString(value, path, issues, required = false) {
583
- if (value === void 0) {
584
- if (required) issues.push(`${path} must be a non-empty string.`);
585
- return;
586
- }
587
- if (typeof value !== "string" || !value.trim()) {
588
- issues.push(`${path} must be a non-empty string.`);
589
- return;
590
- }
591
- return value.trim();
592
- }
593
- function normalizeRuntimeStringList(value, path, issues, options = {}) {
594
- if (value === void 0) {
595
- if (options.required) issues.push(`${path} must be a non-empty string array.`);
596
- return;
597
- }
598
- if (!Array.isArray(value)) {
599
- issues.push(`${path} must be a string array.`);
600
- return [];
601
- }
602
- const normalized = [];
603
- let hasInvalidItem = false;
604
- for (const item of value) {
605
- if (typeof item !== "string" || !item.trim()) {
606
- hasInvalidItem = true;
607
- continue;
608
- }
609
- if (!normalized.includes(item.trim())) normalized.push(item.trim());
610
- }
611
- if (hasInvalidItem) issues.push(`${path} must contain only non-empty strings.`);
612
- if (options.required && normalized.length === 0) issues.push(`${path} must contain at least one source.`);
613
- return normalized;
614
- }
615
- function normalizeRuntimeNumber(value, path, issues, options = {}) {
616
- if (value === void 0) return void 0;
617
- if (typeof value !== "number" || !Number.isFinite(value) || options.integer && !Number.isInteger(value) || options.minimum !== void 0 && value < options.minimum || options.maximum !== void 0 && value > options.maximum) {
618
- const range = options.minimum !== void 0 || options.maximum !== void 0 ? ` between ${options.minimum ?? "-Infinity"} and ${options.maximum ?? "Infinity"}` : "";
619
- issues.push(`${path} must be a finite${options.integer ? " integer" : " number"}${range}.`);
620
- return;
621
- }
622
- return value;
623
- }
624
- function normalizeRuntimeFilters(value, path, issues) {
625
- if (value === void 0) return void 0;
626
- if (!isRecord(value)) {
627
- issues.push(`${path} must be an object.`);
628
- return;
629
- }
630
- const framework = normalizeRuntimeString(value.framework, `${path}.framework`, issues);
631
- const version = normalizeRuntimeString(value.version, `${path}.version`, issues);
632
- const locale = normalizeRuntimeString(value.locale, `${path}.locale`, issues);
633
- return framework || version || locale ? {
634
- framework,
635
- version,
636
- locale
637
- } : void 0;
638
- }
639
- function normalizeRuntimeExamples(value, path, issues) {
640
- if (value === void 0) return void 0;
641
- if (!Array.isArray(value)) {
642
- issues.push(`${path} must be an array.`);
643
- return [];
644
- }
645
- return value.flatMap((item, index) => {
646
- const itemPath = `${path}[${index}]`;
647
- if (!isRecord(item)) {
648
- issues.push(`${itemPath} must be an object.`);
649
- return [];
650
- }
651
- const source = normalizeRuntimeString(item.source, `${itemPath}.source`, issues);
652
- const language = normalizeRuntimeString(item.language, `${itemPath}.language`, issues);
653
- const framework = normalizeRuntimeString(item.framework, `${itemPath}.framework`, issues);
654
- const packageManager = normalizeRuntimeString(item.packageManager, `${itemPath}.packageManager`, issues);
655
- const title = normalizeRuntimeString(item.title, `${itemPath}.title`, issues);
656
- const includes = normalizeRuntimeStringList(item.includes, `${itemPath}.includes`, issues);
657
- let runnable;
658
- if (item.runnable !== void 0) if (typeof item.runnable === "boolean") runnable = item.runnable;
659
- else issues.push(`${itemPath}.runnable must be a boolean.`);
660
- return [{
661
- source,
662
- language,
663
- framework,
664
- packageManager,
665
- title,
666
- runnable,
667
- includes
668
- }];
669
- });
670
- }
671
- function normalizeGoldenTaskInput(value, index, initialIssues = []) {
672
- const issues = [...initialIssues];
673
- const path = `tasks[${index}]`;
674
- const input = isRecord(value) ? value : {};
675
- if (!isRecord(value)) issues.push(`${path} must be an object.`);
676
- const id = normalizeRuntimeString(input.id, `${path}.id`, issues, true) ?? `invalid-task-${index + 1}`;
677
- const query = normalizeRuntimeString(input.query, `${path}.query`, issues, true) ?? "";
678
- const filters = normalizeRuntimeFilters(input.filters, `${path}.filters`, issues);
679
- const tokenBudget = normalizeRuntimeNumber(input.tokenBudget, `${path}.tokenBudget`, issues, {
680
- minimum: 1,
681
- integer: true
682
- });
683
- const topK = normalizeRuntimeNumber(input.topK, `${path}.topK`, issues, {
684
- minimum: 1,
685
- integer: true
686
- });
687
- const expectPath = `${path}.expect`;
688
- const expectation = isRecord(input.expect) ? input.expect : {};
689
- if (!isRecord(input.expect)) issues.push(`${expectPath} must be an object.`);
690
- const relevantSources = normalizeRuntimeStringList(expectation.relevantSources, `${expectPath}.relevantSources`, issues, { required: true }) ?? [];
691
- const allowedSources = normalizeRuntimeStringList(expectation.allowedSources, `${expectPath}.allowedSources`, issues);
692
- const forbiddenSources = normalizeRuntimeStringList(expectation.forbiddenSources, `${expectPath}.forbiddenSources`, issues);
693
- const requiredCitations = normalizeRuntimeStringList(expectation.requiredCitations, `${expectPath}.requiredCitations`, issues);
694
- const minRecallAtK = normalizeRuntimeNumber(expectation.minRecallAtK, `${expectPath}.minRecallAtK`, issues, {
695
- minimum: 0,
696
- maximum: 1
697
- });
698
- const maxFirstRelevantRank = normalizeRuntimeNumber(expectation.maxFirstRelevantRank, `${expectPath}.maxFirstRelevantRank`, issues, {
699
- minimum: 1,
700
- integer: true
701
- });
702
- const minUsefulByteRatio = normalizeRuntimeNumber(expectation.minUsefulByteRatio, `${expectPath}.minUsefulByteRatio`, issues, {
703
- minimum: 0,
704
- maximum: 1
705
- });
706
- const examples = normalizeRuntimeExamples(expectation.examples, `${expectPath}.examples`, issues);
707
- const forbidden = forbiddenSources ?? [];
708
- for (const [name, sources] of [
709
- ["relevantSources", relevantSources],
710
- ["allowedSources", allowedSources ?? []],
711
- ["requiredCitations", requiredCitations ?? []]
712
- ]) {
713
- const overlaps = sources.filter((source) => forbidden.some((forbiddenSource) => sourceMatches(source, forbiddenSource) || sourceMatches(forbiddenSource, source)));
714
- if (overlaps.length > 0) issues.push(`${expectPath}.${name} must not overlap ${expectPath}.forbiddenSources (${overlaps.join(", ")}).`);
715
- }
716
- return {
717
- task: {
718
- id,
719
- query,
720
- filters,
721
- tokenBudget,
722
- topK,
723
- expect: {
724
- relevantSources,
725
- allowedSources,
726
- forbiddenSources,
727
- requiredCitations,
728
- minRecallAtK,
729
- maxFirstRelevantRank,
730
- minUsefulByteRatio,
731
- examples
732
- }
733
- },
734
- issues
735
- };
736
- }
737
- function round(value) {
738
- return Math.round((value + Number.EPSILON) * 1e4) / 1e4;
739
- }
740
- function clamp(value, minimum, maximum) {
741
- return Math.min(maximum, Math.max(minimum, value));
742
- }
743
- function normalizePositiveInteger(value, fallback, maximum) {
744
- if (!Number.isFinite(value)) return fallback;
745
- return clamp(Math.floor(value ?? fallback), 1, maximum);
746
- }
747
- function safeDecode(value) {
748
- try {
749
- return decodeURIComponent(value);
750
- } catch {
751
- return value;
752
- }
753
- }
754
- function canonicalizeSource(value) {
755
- let pathname;
756
- let hash;
757
- try {
758
- const parsed = new URL(value, "https://docs.local");
759
- pathname = parsed.pathname;
760
- hash = parsed.hash;
761
- } catch {
762
- const [pathAndQuery = "/", fragment = ""] = value.split("#", 2);
763
- pathname = pathAndQuery.split("?", 1)[0] || "/";
764
- hash = fragment ? `#${fragment}` : "";
765
- }
766
- pathname = pathname.replace(/\/{2,}/gu, "/").replace(/\.md$/iu, "");
767
- if (pathname !== "/") pathname = pathname.replace(/\/+$/gu, "");
768
- if (!pathname.startsWith("/")) pathname = `/${pathname}`;
769
- const normalizedHash = hash ? `#${safeDecode(hash.slice(1)).trim().replace(/^#+/u, "").toLowerCase()}` : "";
770
- return `${pathname || "/"}${normalizedHash === "#" ? "" : normalizedHash}`;
771
- }
772
- function sourcePage(value) {
773
- return canonicalizeSource(value).split("#", 1)[0] ?? "/";
774
- }
775
- function sourceMatches(actual, expected) {
776
- const canonicalActual = canonicalizeSource(actual);
777
- const canonicalExpected = canonicalizeSource(expected);
778
- return canonicalExpected.includes("#") ? canonicalActual === canonicalExpected : sourcePage(canonicalActual) === sourcePage(canonicalExpected);
779
- }
780
- function uniqueCanonicalSources(values) {
781
- return Array.from(new Set(values.map(canonicalizeSource)));
782
- }
783
- function valuesOverlap(left, right, matches) {
784
- return left.some((leftValue) => right.some((rightValue) => matches(leftValue, rightValue)));
785
- }
786
- function getPageScope(page) {
787
- const topFramework = normalizeAgentScopeValues(page.framework).map(normalizeAgentFramework);
788
- const contractFramework = normalizeAgentScopeValues(page.agent?.appliesTo?.framework).map(normalizeAgentFramework);
789
- const topVersion = normalizeAgentScopeValues(page.version).map(normalizeAgentVersion);
790
- const contractVersion = normalizeAgentScopeValues(page.agent?.appliesTo?.version).map(normalizeAgentVersion);
791
- const locale = normalizeAgentScopeValues(page.locale).map(normalizeAgentLocale);
792
- return {
793
- framework: Array.from(new Set([...topFramework, ...contractFramework])),
794
- version: Array.from(new Set([...topVersion, ...contractVersion])),
795
- locale,
796
- frameworkAmbiguous: topFramework.length > 0 && contractFramework.length > 0 && !valuesOverlap(topFramework, contractFramework, (left, right) => left === right),
797
- versionAmbiguous: topVersion.length > 0 && contractVersion.length > 0 && !valuesOverlap(topVersion, contractVersion, agentVersionConstraintsOverlap)
798
- };
799
- }
800
- function compareCodePoints(left, right) {
801
- if (left === right) return 0;
802
- return left < right ? -1 : 1;
803
- }
804
- function toGoldenSources(sources, relevantSources) {
805
- return sources.map((source, index) => ({
806
- rank: index + 1,
807
- url: canonicalizeSource(source.url),
808
- title: source.section ?? source.title,
809
- framework: source.framework,
810
- version: source.version,
811
- utf8Bytes: source.utf8Bytes,
812
- relevant: relevantSources.some((expected) => sourceMatches(source.url, expected)),
813
- truncated: source.truncated
814
- }));
815
- }
816
- function toContextCandidates(sources, pagesByUrl) {
817
- return sources.flatMap((source) => {
818
- const page = pagesByUrl.get(sourcePage(source.pageUrl));
819
- if (!page) return [];
820
- return [{
821
- page,
822
- source: canonicalizeSource(source.url),
823
- content: source.content,
824
- scope: getPageScope(page)
825
- }];
826
- });
827
- }
828
- function extractRenderedCitations(context, sources) {
829
- const actual = [];
830
- const blockUtf8Bytes = [];
831
- let cursor = 0;
832
- let layoutIntegrity = true;
833
- for (let index = 0; index < sources.length; index += 1) {
834
- const attributableStart = cursor;
835
- if (index > 0) {
836
- if (!context.startsWith(CONTEXT_SEPARATOR, cursor)) {
837
- layoutIntegrity = false;
838
- break;
839
- }
840
- cursor += 7;
841
- }
842
- const contentMarker = `\n\n${sources[index].content}`;
843
- const contentStart = context.indexOf(contentMarker, cursor);
844
- if (contentStart < cursor) {
845
- layoutIntegrity = false;
846
- break;
847
- }
848
- const sourceLines = context.slice(cursor, contentStart).split(/\r?\n/u).map((line) => line.match(/^Source:\s+(.+)$/u)?.[1]?.trim()).filter((source) => Boolean(source));
849
- if (sourceLines.length !== 1) layoutIntegrity = false;
850
- actual.push(...sourceLines.map(canonicalizeSource));
851
- cursor = contentStart + contentMarker.length;
852
- blockUtf8Bytes.push(Buffer.byteLength(context.slice(attributableStart, cursor), "utf8"));
853
- }
854
- if (cursor !== context.length) layoutIntegrity = false;
855
- return {
856
- actual,
857
- layoutIntegrity,
858
- blockUtf8Bytes
859
- };
860
- }
861
- function hasValidStaticSyntax(target) {
862
- const language = target.language?.toLowerCase();
863
- if (language === "json") try {
864
- JSON.parse(target.code);
865
- return true;
866
- } catch {
867
- return false;
868
- }
869
- if (language === "javascript" || language === "js" || language === "jsx" || language === "node" || language === "typescript" || language === "ts" || language === "tsx") try {
870
- const typescript = language === "typescript" || language === "ts" || language === "tsx";
871
- const jsx = language === "jsx" || language === "tsx";
872
- const transformed = STATIC_JITI.transform({
873
- source: target.code,
874
- filename: `golden-example.${language === "typescript" ? "ts" : language}`,
875
- ts: typescript,
876
- jsx,
877
- async: true
878
- });
879
- return !/\bexports\.__JITI_ERROR__\s*=/u.test(transformed);
880
- } catch {
881
- return false;
882
- }
883
- return true;
884
- }
885
- function isStaticExecutable(target, plan) {
886
- const language = target.language?.toLowerCase();
887
- if (!target.runnable || !language || !EXECUTABLE_LANGUAGES.has(language)) return false;
888
- if (!plan || plan.action === "skip") return false;
889
- if (!target.code.trim()) return false;
890
- if (target.code.split("\n").some((line) => /^\s*(?:(?:\/\/|#)\s*)?\.\.\.\s*$/u.test(line))) return false;
891
- return hasValidStaticSyntax(target);
892
- }
893
- async function collectRetrievedExamples(candidates) {
894
- const examples = [];
895
- for (const candidate of candidates) {
896
- const targets = extractCodeBlocksFromMarkdown({
897
- source: candidate.content,
898
- filePath: candidate.page.sourcePath ?? candidate.page.slug,
899
- relativePath: candidate.page.slug
900
- });
901
- for (const target of targets) examples.push({
902
- source: candidate.source,
903
- target
904
- });
905
- }
906
- const plans = await planCodeBlockTargets(examples.map((example) => example.target), STATIC_CODE_PLAN_CONFIG);
907
- return examples.map((example, index) => ({
908
- ...example,
909
- executable: isStaticExecutable(example.target, plans[index])
910
- }));
911
- }
912
- function normalizedEqual(left, right) {
913
- return (left ?? "").trim().toLowerCase() === (right ?? "").trim().toLowerCase();
914
- }
915
- function exampleMatches(expected, actual) {
916
- if (expected.source && !sourceMatches(actual.source, expected.source)) return false;
917
- if (expected.language && !normalizedEqual(actual.target.language, expected.language)) return false;
918
- if (expected.framework && !normalizedEqual(actual.target.framework, expected.framework)) return false;
919
- if (expected.packageManager && !normalizedEqual(actual.target.packageManager, expected.packageManager)) return false;
920
- if (expected.title && !normalizedEqual(actual.target.title, expected.title)) return false;
921
- if (expected.includes?.some((fragment) => !actual.target.code.includes(fragment))) return false;
922
- if (expected.runnable === false) return !actual.target.runnable;
923
- return actual.executable;
924
- }
925
- async function evaluateExamples(expectedExamples, candidates) {
926
- const actualExamples = await collectRetrievedExamples(candidates);
927
- const claimed = /* @__PURE__ */ new Set();
928
- const results = expectedExamples.map((expected) => {
929
- const index = actualExamples.findIndex((example, candidateIndex) => !claimed.has(candidateIndex) && exampleMatches(expected, example));
930
- if (index < 0) return {
931
- expected,
932
- executable: false,
933
- passed: false
934
- };
935
- claimed.add(index);
936
- const matched = actualExamples[index];
937
- return {
938
- expected,
939
- matchedId: matched.target.id,
940
- source: matched.source,
941
- executable: matched.executable,
942
- passed: true
943
- };
944
- });
945
- const matched = results.filter((result) => result.passed).length;
946
- return {
947
- expected: expectedExamples.length,
948
- matched,
949
- executable: results.filter((result) => result.executable).length,
950
- results,
951
- passed: matched === expectedExamples.length
952
- };
953
- }
954
- function buildRetrievalMetrics(task, sources, topK) {
955
- const expected = uniqueCanonicalSources(task.expect.relevantSources);
956
- const forbidden = uniqueCanonicalSources(task.expect.forbiddenSources ?? []);
957
- const retrievedRelevant = expected.filter((source) => sources.some((actual) => sourceMatches(actual.url, source))).length;
958
- const recallAtK = expected.length > 0 ? retrievedRelevant / expected.length : 0;
959
- const firstRelevant = sources.find((source) => source.relevant)?.rank ?? null;
960
- const forbiddenSources = sources.filter((source) => forbidden.some((forbiddenSource) => sourceMatches(source.url, forbiddenSource))).map((source) => source.url);
961
- const minimumRecall = clamp(task.expect.minRecallAtK ?? 1, 0, 1);
962
- const maximumRank = normalizePositiveInteger(task.expect.maxFirstRelevantRank, topK, topK);
963
- return {
964
- expectedRelevant: expected.length,
965
- retrievedRelevant,
966
- recallAtK: round(recallAtK),
967
- firstRelevantRank: firstRelevant,
968
- reciprocalRank: firstRelevant ? round(1 / firstRelevant) : 0,
969
- forbiddenSources,
970
- passed: expected.length > 0 && recallAtK >= minimumRecall && firstRelevant !== null && firstRelevant <= maximumRank && forbiddenSources.length === 0
971
- };
972
- }
973
- function buildCitationMetrics(task, context, sources) {
974
- const expected = uniqueCanonicalSources(task.expect.requiredCitations ?? task.expect.relevantSources);
975
- const allowed = uniqueCanonicalSources([
976
- ...task.expect.relevantSources,
977
- ...task.expect.requiredCitations ?? [],
978
- ...task.expect.allowedSources ?? []
979
- ]);
980
- const extracted = extractRenderedCitations(context, sources);
981
- const actual = extracted.actual;
982
- const sourceRecords = sources.map((source) => canonicalizeSource(source.url));
983
- const missing = expected.filter((expectedSource) => !actual.some((actualSource) => sourceMatches(actualSource, expectedSource)));
984
- const unexpected = actual.filter((actualSource) => !allowed.some((allowedSource) => sourceMatches(actualSource, allowedSource)));
985
- const correct = actual.filter((actualSource) => allowed.some((allowedSource) => sourceMatches(actualSource, allowedSource))).length;
986
- const matchedExpected = expected.filter((expectedSource) => actual.some((actualSource) => sourceMatches(actualSource, expectedSource))).length;
987
- const precision = actual.length > 0 ? correct / actual.length : expected.length === 0 ? 1 : 0;
988
- const recall = expected.length > 0 ? matchedExpected / expected.length : 1;
989
- const integrity = extracted.layoutIntegrity && actual.length === sourceRecords.length && actual.every((source, index) => source === sourceRecords[index]);
990
- return {
991
- expected,
992
- actual,
993
- missing,
994
- unexpected,
995
- precision: round(precision),
996
- recall: round(recall),
997
- integrity,
998
- passed: missing.length === 0 && unexpected.length === 0 && integrity
999
- };
1000
- }
1001
- function buildSelectionMetrics(filters, candidates) {
1002
- const requestedFramework = filters?.framework;
1003
- const requestedVersion = filters?.version;
1004
- const framework = requestedFramework ? normalizeAgentFramework(requestedFramework) : void 0;
1005
- const firstFrameworkMatchRank = framework ? candidates.findIndex((candidate) => candidate.scope.framework.includes(framework)) + 1 || null : null;
1006
- const firstVersionMatchRank = requestedVersion ? candidates.findIndex((candidate) => candidate.scope.version.some((version) => agentVersionConstraintMatches(requestedVersion, version))) + 1 || null : null;
1007
- const conflictingSources = candidates.filter((candidate) => framework && candidate.scope.framework.length > 0 && !candidate.scope.framework.includes(framework) || requestedVersion && candidate.scope.version.length > 0 && !candidate.scope.version.some((version) => agentVersionConstraintMatches(requestedVersion, version))).map((candidate) => candidate.source);
1008
- const ambiguousSources = candidates.filter((candidate) => candidate.scope.frameworkAmbiguous || candidate.scope.versionAmbiguous || Boolean(framework && candidate.scope.framework.length === 0) || Boolean(requestedVersion && candidate.scope.version.length === 0)).map((candidate) => candidate.source);
1009
- return {
1010
- requestedFramework,
1011
- requestedVersion,
1012
- firstFrameworkMatchRank,
1013
- firstVersionMatchRank,
1014
- conflictingSources,
1015
- ambiguousSources,
1016
- passed: (!framework || firstFrameworkMatchRank !== null) && (!requestedVersion || firstVersionMatchRank !== null) && conflictingSources.length === 0 && ambiguousSources.length === 0
1017
- };
1018
- }
1019
- function buildUsageMetrics(task, tokenBudget, context, relevantSources) {
1020
- const usedUtf8Bytes = context.budget.usedUtf8Bytes;
1021
- const rendered = extractRenderedCitations(context.context, context.sources);
1022
- const usefulUtf8Bytes = context.sources.reduce((total, source, index) => relevantSources.some((expected) => sourceMatches(source.url, expected)) ? total + (rendered.blockUtf8Bytes[index] ?? 0) : total, 0);
1023
- const usefulByteRatio = usedUtf8Bytes > 0 ? usefulUtf8Bytes / usedUtf8Bytes : 0;
1024
- const minimumUsefulRatio = clamp(task.expect.minUsefulByteRatio ?? 0, 0, 1);
1025
- const withinBudget = usedUtf8Bytes <= tokenBudget;
1026
- return {
1027
- budgetUnit: "utf8-bytes",
1028
- tokenBudget,
1029
- usedUtf8Bytes,
1030
- remainingUtf8Bytes: context.budget.remainingUtf8Bytes,
1031
- estimatedTokens: Math.ceil(usedUtf8Bytes / 4),
1032
- conservativeTokenUpperBound: usedUtf8Bytes,
1033
- usefulUtf8Bytes,
1034
- usefulByteRatio: round(usefulByteRatio),
1035
- truncated: context.budget.truncated,
1036
- withinBudget,
1037
- passed: withinBudget && usefulByteRatio >= minimumUsefulRatio
1038
- };
1039
- }
1040
- function calculateTaskScore(report) {
1041
- const dimensions = [
1042
- {
1043
- weight: 30,
1044
- score: report.retrieval.forbiddenSources.length > 0 ? 0 : (report.retrieval.recallAtK + report.retrieval.reciprocalRank) / 2
1045
- },
1046
- {
1047
- weight: 25,
1048
- score: report.citations.integrity ? (report.citations.precision + report.citations.recall) / 2 : 0
1049
- },
1050
- ...report.hasSelectionExpectation ? [{
1051
- weight: 20,
1052
- score: report.selection.passed ? 1 : 0
1053
- }] : [],
1054
- ...report.hasExampleExpectation ? [{
1055
- weight: 15,
1056
- score: report.examples.expected > 0 ? report.examples.matched / report.examples.expected : 1
1057
- }] : [],
1058
- {
1059
- weight: 10,
1060
- score: report.usage.passed ? 1 : 0
1061
- }
1062
- ];
1063
- const totalWeight = dimensions.reduce((sum, dimension) => sum + dimension.weight, 0);
1064
- return round(dimensions.reduce((sum, dimension) => sum + dimension.weight * dimension.score, 0) / totalWeight * 100);
1065
- }
1066
- async function evaluateTask(pages, task, configurationIssues = []) {
1067
- const topK = normalizePositiveInteger(task.topK, DEFAULT_TOP_K, 50);
1068
- const tokenBudget = normalizePositiveInteger(task.tokenBudget, DEFAULT_TOKEN_BUDGET, 1e6);
1069
- const relevantSources = uniqueCanonicalSources(task.expect.relevantSources);
1070
- const orderedPages = pages.slice().sort((left, right) => compareCodePoints(canonicalizeSource(left.url), canonicalizeSource(right.url)));
1071
- const pagesByUrl = new Map(orderedPages.map((page) => [sourcePage(page.url), page]));
1072
- const contextOptions = {
1073
- pages: orderedPages,
1074
- query: task.query,
1075
- framework: task.filters?.framework,
1076
- version: task.filters?.version,
1077
- locale: task.filters?.locale,
1078
- maxResults: topK
1079
- };
1080
- const [rankedContext, budgetedContext] = await Promise.all([buildDocsMcpContext({
1081
- ...contextOptions,
1082
- tokenBudget: Number.MAX_SAFE_INTEGER
1083
- }), buildDocsMcpContext({
1084
- ...contextOptions,
1085
- tokenBudget
1086
- })]);
1087
- const rankedSources = toGoldenSources(rankedContext.sources, relevantSources);
1088
- const budgetedSources = toGoldenSources(budgetedContext.sources, relevantSources);
1089
- const rankedCandidates = toContextCandidates(rankedContext.sources, pagesByUrl);
1090
- const budgetedCandidates = toContextCandidates(budgetedContext.sources, pagesByUrl);
1091
- const retrieval = buildRetrievalMetrics(task, rankedSources, topK);
1092
- const citations = buildCitationMetrics(task, budgetedContext.context, budgetedContext.sources);
1093
- const selection = buildSelectionMetrics(task.filters, rankedCandidates);
1094
- const examples = await evaluateExamples(task.expect.examples ?? [], budgetedCandidates);
1095
- const usage = buildUsageMetrics(task, tokenBudget, budgetedContext, relevantSources);
1096
- const issues = configurationIssues.map((issue) => `Invalid golden task configuration: ${issue}`);
1097
- if (relevantSources.length === 0) issues.push("No relevantSources are configured for this task.");
1098
- if (!retrieval.passed) issues.push("Retrieval did not satisfy the expected recall, rank, or exclusion rules.");
1099
- if (!citations.passed) issues.push("Rendered citations are missing, unexpected, or do not match source records.");
1100
- if (!selection.passed) issues.push("Retrieved context is ambiguous or does not explicitly match the requested framework/version.");
1101
- if (!examples.passed) issues.push("One or more expected runnable examples were not retrieved or are not executable.");
1102
- if (!usage.passed) issues.push("Context exceeded its budget or did not meet the configured useful-byte ratio.");
1103
- const passed = configurationIssues.length === 0 && relevantSources.length > 0 && retrieval.passed && citations.passed && selection.passed && examples.passed && usage.passed;
1104
- const calculatedScore = configurationIssues.length > 0 ? 0 : calculateTaskScore({
1105
- retrieval,
1106
- citations,
1107
- selection,
1108
- examples,
1109
- usage,
1110
- hasSelectionExpectation: Boolean(task.filters?.framework || task.filters?.version),
1111
- hasExampleExpectation: (task.expect.examples?.length ?? 0) > 0
1112
- });
1113
- const score = passed ? calculatedScore : Math.min(calculatedScore, 99);
1114
- return {
1115
- id: task.id,
1116
- query: task.query,
1117
- status: passed ? "passed" : "failed",
1118
- passed,
1119
- score,
1120
- context: budgetedContext.context,
1121
- sources: budgetedSources,
1122
- retrieval,
1123
- citations,
1124
- selection,
1125
- examples,
1126
- usage,
1127
- issues
1128
- };
1129
- }
1130
- /**
1131
- * Run deterministic, offline golden-task evaluations against MCP-ready docs pages.
1132
- * An empty task list is intentionally unmeasured so CI cannot turn absent coverage into a pass.
1133
- */
1134
- async function runDocsGoldenTasks(pages, tasks) {
1135
- const runtimeTasks = tasks;
1136
- if (runtimeTasks === void 0 || Array.isArray(runtimeTasks) && runtimeTasks.length === 0) return {
1137
- status: "unmeasured",
1138
- passed: null,
1139
- score: null,
1140
- taskCount: 0,
1141
- passedTaskCount: 0,
1142
- failedTaskCount: 0,
1143
- tasks: []
1144
- };
1145
- const normalizedTasks = Array.isArray(runtimeTasks) ? runtimeTasks.map((task, index) => normalizeGoldenTaskInput(task, index)) : [normalizeGoldenTaskInput(runtimeTasks, 0, ["agent.evaluations.tasks must be an array."])];
1146
- const idCounts = normalizedTasks.reduce((counts, input) => {
1147
- counts.set(input.task.id, (counts.get(input.task.id) ?? 0) + 1);
1148
- return counts;
1149
- }, /* @__PURE__ */ new Map());
1150
- for (const input of normalizedTasks) if ((idCounts.get(input.task.id) ?? 0) > 1) input.issues.push(`task id ${JSON.stringify(input.task.id)} is duplicated.`);
1151
- const reports = await Promise.all(normalizedTasks.map(({ task, issues }) => evaluateTask(pages, task, issues)));
1152
- const passedTaskCount = reports.filter((task) => task.passed).length;
1153
- const failedTaskCount = reports.length - passedTaskCount;
1154
- return {
1155
- status: failedTaskCount === 0 ? "passed" : "failed",
1156
- passed: failedTaskCount === 0,
1157
- score: round(reports.reduce((sum, task) => sum + task.score, 0) / reports.length),
1158
- taskCount: reports.length,
1159
- passedTaskCount,
1160
- failedTaskCount,
1161
- tasks: reports
1162
- };
1163
- }
1164
-
1165
- //#endregion
1166
- export { buildDocsReviewWorkflowPathFilters as a, resolveDocsReviewConfig as c, buildDocsReviewWorkflow as i, DEFAULT_DOCS_REVIEW_SCORE_THRESHOLD as n, ensureDocsReviewWorkflow as o, DEFAULT_DOCS_REVIEW_WORKFLOW_PATH as r, readDocsReviewConfigFromSource as s, runDocsGoldenTasks as t };