@agentskit/doc-bridge 1.6.4 → 1.7.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +249 -0
  2. package/CONTRIBUTING.md +6 -4
  3. package/action.yml +1 -1
  4. package/dist/cli/program.js +1139 -294
  5. package/dist/cli/program.js.map +1 -1
  6. package/dist/config/index.d.ts +1 -1
  7. package/dist/config/index.js +43 -5
  8. package/dist/config/index.js.map +1 -1
  9. package/dist/index-BUL0q7s8.d.ts +660 -0
  10. package/dist/index.d.ts +817 -2134
  11. package/dist/index.js +1154 -244
  12. package/dist/index.js.map +1 -1
  13. package/docs/PRD-enterprise-hardening.md +288 -0
  14. package/docs/RELEASE.md +22 -8
  15. package/docs/adr/0001-enterprise-verification-contract.md +35 -0
  16. package/docs/agent-corpus/INDEX.md +2 -2
  17. package/docs/agent-corpus/chat.md +2 -2
  18. package/docs/agent-corpus/cli.md +2 -2
  19. package/docs/agent-corpus/conformance.md +2 -2
  20. package/docs/agent-corpus/doc-bridge.md +1 -1
  21. package/docs/agent-corpus/doctor.md +2 -2
  22. package/docs/agent-corpus/gates.md +2 -2
  23. package/docs/agent-corpus/mcp.md +2 -2
  24. package/docs/agent-corpus/memory.md +2 -2
  25. package/docs/agent-corpus/query.md +2 -2
  26. package/docs/knowledge-engine-runbook.md +30 -2
  27. package/docs/spec/analyzer-plugin-v1.md +24 -0
  28. package/docs/spec/benchmark-v1.md +36 -0
  29. package/docs/spec/config-v1.md +156 -0
  30. package/docs/validation-cycle-plan.md +255 -0
  31. package/docs/verification-harness.md +37 -4
  32. package/mcpb/manifest.json +1 -1
  33. package/package.json +68 -70
  34. package/scripts/check-ecosystem-upstream.mjs +3 -2
  35. package/scripts/report-visual-check.mjs +64 -12
  36. package/scripts/verification-harness.mjs +216 -14
  37. package/skills/doc-bridge-handoff/scripts/resolve-handoff.mjs +1 -1
  38. package/src/agents/registry-adapter.ts +31 -7
  39. package/src/cli/demo.ts +2 -2
  40. package/src/cli/program.ts +59 -16
  41. package/src/config/index.ts +2 -0
  42. package/src/config/load-config.ts +7 -1
  43. package/src/config/schema.ts +60 -2
  44. package/src/conformance/documentation-standard-v1.ts +14 -8
  45. package/src/discovery/documentation.ts +90 -23
  46. package/src/discovery/repository.ts +147 -19
  47. package/src/doctor/run-doctor.ts +2 -15
  48. package/src/federation/llms.ts +72 -20
  49. package/src/fixes/proposals.ts +4 -3
  50. package/src/index-builder/human-adapters/fumadocs.ts +1 -1
  51. package/src/index-builder/watch-index.ts +1 -1
  52. package/src/index.ts +29 -0
  53. package/src/lib/bounded-text.ts +15 -10
  54. package/src/metrics/benchmark.ts +176 -0
  55. package/src/plugins/contract.ts +89 -0
  56. package/src/reconciliation/reconcile.ts +181 -5
  57. package/src/report/html.ts +318 -88
  58. package/src/rules/engine.ts +15 -2
  59. package/src/safety/repository.ts +1 -1
  60. package/src/schemas/knowledge.ts +21 -3
  61. package/src/validate.ts +7 -1
  62. package/src/version.ts +1 -1
  63. package/src/workflow/engine.ts +65 -9
  64. package/dist/index-DudNuwI5.d.ts +0 -2060
@@ -52,17 +52,6 @@ const gradeForScore = (score: number): DoctorReport['grade'] => {
52
52
  return 'F'
53
53
  }
54
54
 
55
- const agentDocPaths = (index: DocBridgeIndexV1): Set<string> => {
56
- const paths = new Set<string>()
57
- for (const owner of Object.values(index.lookup?.ownership ?? {})) {
58
- if (owner.agentDoc) paths.add(owner.agentDoc)
59
- }
60
- for (const handoff of Object.values(index.handoffs ?? {})) {
61
- if (handoff.startHere) paths.add(handoff.startHere)
62
- }
63
- return paths
64
- }
65
-
66
55
  const computeScore = (coverage: DoctorCoverage): number => {
67
56
  let score = 0
68
57
 
@@ -166,7 +155,7 @@ export const runDoctor = (root: string, config: DocBridgeConfigV1): DoctorReport
166
155
  let index: DocBridgeIndexV1 | undefined
167
156
  let hasIndex = true
168
157
  let freshnessOk = false
169
- let freshnessMessage = 'Index is fresh'
158
+ let freshnessMessage: string
170
159
 
171
160
  try {
172
161
  index = loadDocBridgeIndex(root, config)
@@ -191,8 +180,6 @@ export const runDoctor = (root: string, config: DocBridgeConfigV1): DoctorReport
191
180
  const missingHumanDoc = ownership.filter(([, owner]) => !owner.humanDoc).map(([id]) => id)
192
181
 
193
182
  const indexedPaths = new Set(index.knowledge.map((entry) => entry.path))
194
- const expectedAgentDocs = agentDocPaths(index)
195
- const unindexed = [...expectedAgentDocs].filter((path) => !indexedPaths.has(path))
196
183
 
197
184
  const corpusDocs = scanAgentCorpus(root, config).filter(
198
185
  (doc) => doc.path !== config.corpus.agent.index,
@@ -283,4 +270,4 @@ export const formatDoctorText = (report: DoctorReport): string[] => {
283
270
 
284
271
  lines.push('', 'Next actions', ...report.nextActions.map((action) => ` → ${action}`))
285
272
  return lines
286
- }
273
+ }
@@ -31,13 +31,25 @@ const defaultFetchText: FetchText = async (url) => {
31
31
  return res.text()
32
32
  }
33
33
 
34
+ const httpUrl = (value: string): string | undefined => {
35
+ try {
36
+ const parsed = new URL(value)
37
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return undefined
38
+ if (parsed.username || parsed.password) return undefined
39
+ return parsed.href
40
+ } catch {
41
+ return undefined
42
+ }
43
+ }
44
+
34
45
  const sourceText = async (
35
46
  root: string,
36
47
  source: string,
37
48
  fetchText: FetchText,
38
49
  ): Promise<string | null> => {
39
50
  try {
40
- if (/^https?:\/\//.test(source)) return await fetchText(source)
51
+ const remote = httpUrl(source)
52
+ if (remote) return await fetchText(remote)
41
53
  const path = resolve(root, source)
42
54
  if (!existsSync(path)) return null
43
55
  return readFileSync(path, 'utf8')
@@ -52,34 +64,72 @@ const sameOrigin = (base: string, target: string): boolean => {
52
64
  }
53
65
 
54
66
  export const parseLlmsTxtLinks = (raw: string): { title: string; url: string; description?: string }[] => {
55
- const links = [...raw.matchAll(/\[([^\]]+)\]\(([^)]+)\)(?::\s*([^\n]+))?/g)].map((match) => ({
56
- title: match[1] ?? match[2] ?? 'link',
57
- url: match[2] ?? '',
58
- ...(match[3]?.trim() ? { description: match[3].trim() } : {}),
59
- })).filter((link) => link.url)
60
-
61
- for (const match of raw.matchAll(/(?:^|\s)(?:Raw|llms\.txt|Full bundle|ZIP bundle)?:?\s*(https?:\/\/\S+)/gi)) {
62
- const url = match[1]?.replace(/[),.;]+$/, '')
63
- if (url && !links.some((link) => link.url === url)) links.push({ title: slugFromPath(url), url })
67
+ const links: { title: string; url: string; description?: string }[] = []
68
+ for (const line of raw.split(/\r?\n/)) {
69
+ let cursor = 0
70
+ while (cursor < line.length) {
71
+ const open = line.indexOf('[', cursor)
72
+ if (open < 0) break
73
+ const titleEnd = line.indexOf(']', open + 1)
74
+ const urlStart = titleEnd < 0 ? -1 : line.indexOf('(', titleEnd + 1)
75
+ const urlEnd = urlStart < 0 ? -1 : line.indexOf(')', urlStart + 1)
76
+ if (titleEnd < 0 || urlStart !== titleEnd + 1 || urlEnd < 0) {
77
+ cursor = open + 1
78
+ continue
79
+ }
80
+ const title = line.slice(open + 1, titleEnd).trim()
81
+ const url = line.slice(urlStart + 1, urlEnd).trim()
82
+ const description = line.slice(urlEnd + 1).trim().replace(/^:\s*/, '')
83
+ if (url) links.push({ title: title || url, url, ...(description ? { description } : {}) })
84
+ cursor = urlEnd + 1
85
+ }
86
+
87
+ for (const token of line.split(/\s+/)) {
88
+ let end = token.length
89
+ while (end > 0 && '),.;:'.includes(token[end - 1] ?? '')) end -= 1
90
+ const url = token.slice(0, end)
91
+ if (!/^https?:\/\//i.test(url) || links.some((link) => link.url === url)) continue
92
+ links.push({ title: slugFromPath(url), url })
93
+ }
64
94
  }
65
95
  return links
66
96
  }
67
97
 
98
+ const firstMatchingLine = (section: string, predicate: (line: string) => boolean): string | undefined =>
99
+ section.split(/\r?\n/).find((line) => predicate(line.trim()))?.trim()
100
+
101
+ const sectionTitle = (section: string, sourceUrl: string): string => {
102
+ const titleLine = firstMatchingLine(section, (line) => line.startsWith('title:'))
103
+ if (titleLine) return titleLine.slice('title:'.length).trim()
104
+ const urlLine = firstMatchingLine(section, (line) => /^https?:\/\//i.test(line))
105
+ if (urlLine) {
106
+ try {
107
+ return new URL(urlLine).pathname.split('/').filter(Boolean).at(-1) ?? slugFromPath(sourceUrl)
108
+ } catch {
109
+ return slugFromPath(sourceUrl)
110
+ }
111
+ }
112
+ const heading = firstMatchingLine(section, (line) => line.startsWith('#'))
113
+ if (heading) return heading.replace(/^#+\s*/, '').trim()
114
+ return slugFromPath(sourceUrl)
115
+ }
116
+
68
117
  export const chunksFromMarkdown = (
69
118
  property: string,
70
119
  raw: string,
71
120
  sourceUrl: string,
72
121
  ): DocBridgeRetrievedChunk[] => {
73
- const searchable = raw.includes('\n==== ') ? raw : raw.replace(/^---\n[\s\S]*?\n---\n?/, '')
122
+ const frontmatterEnd = raw.startsWith('---\n') ? raw.indexOf('\n---', 4) : -1
123
+ const searchable = raw.includes('\n==== ')
124
+ ? raw
125
+ : frontmatterEnd >= 0
126
+ ? raw.slice(frontmatterEnd + '\n---'.length).replace(/^\n/, '')
127
+ : raw
74
128
  const sections = searchable.includes('\n==== ')
75
129
  ? searchable.split(/\n====\s+/).filter((section) => section.trim())
76
130
  : searchable.split(/\n(?=##?\s+)/)
77
131
  return sections.map((section, index) => {
78
- const title =
79
- /^title:\s*(.+)$/m.exec(section)?.[1]?.trim() ??
80
- /^https?:\/\/\S+\/([^/\s]+)$/m.exec(section)?.[1]?.trim() ??
81
- /^#+\s+(.+)$/m.exec(section)?.[1]?.trim() ??
82
- slugFromPath(sourceUrl)
132
+ const title = sectionTitle(section, sourceUrl)
83
133
  const id = slugFromPath(title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')) || `${index}`
84
134
  return {
85
135
  chunkKey: `${property}:federated:${id}`,
@@ -112,10 +162,12 @@ export const loadFederatedChunks = async (
112
162
  chunks.push(...chunksFromMarkdown(source.id, llms, source.llmsTxt))
113
163
  const links = parseLlmsTxtLinks(llms)
114
164
  for (const link of links) {
115
- const url = !/^https?:\/\//.test(link.url) && source.rawBaseUrl
116
- ? `${source.rawBaseUrl.replace(/\/$/, '')}/${link.url.replace(/^\//, '')}`
117
- : link.url
118
- if (!/\.(md|txt)(?:$|\?)/.test(url)) continue
165
+ const linkUrl = httpUrl(link.url)
166
+ const baseUrl = source.rawBaseUrl ? httpUrl(source.rawBaseUrl) : undefined
167
+ const url = linkUrl ?? (baseUrl ? new URL(link.url, baseUrl).href : link.url)
168
+ let pathname = url
169
+ try { pathname = new URL(url).pathname } catch { /* local source */ }
170
+ if (!/\.(md|txt)$/i.test(pathname)) continue
119
171
  if (!sameOrigin(source.llmsTxt, url)) continue
120
172
  const raw = await sourceText(root, url, fetchText)
121
173
  if (raw) chunks.push(...chunksFromMarkdown(source.id, raw, url))
@@ -1,4 +1,4 @@
1
- import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
1
+ import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
2
2
  import { basename, dirname, extname, join, relative, resolve, sep } from 'node:path'
3
3
 
4
4
  import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
@@ -99,8 +99,9 @@ export const createArtifactNormalizationProposal = (root: string, artifactPath:
99
99
  const projectRoot = realpathSync.native(resolve(root))
100
100
  const path = artifactPath.split(sep).join('/')
101
101
  const absolute = containedPath(projectRoot, path)
102
- if (!absolute || !existsSync(absolute) || !statSync(absolute).isFile()) return undefined
103
- const before = readFileSync(absolute, 'utf8')
102
+ if (!absolute) return undefined
103
+ let before: string
104
+ try { before = readFileSync(absolute, 'utf8') } catch { return undefined }
104
105
  let after: string
105
106
  try { after = `${JSON.stringify(sortJson(JSON.parse(before) as unknown), null, 2)}\n` } catch { return undefined }
106
107
  return after === before ? undefined : makeProposal(projectRoot, options, [{ path: relative(projectRoot, absolute).split(sep).join('/'), before, after }], ['The artifact contains valid JSON.'], ['The artifact is valid canonical JSON with one trailing newline.'])
@@ -1,5 +1,5 @@
1
1
  import { existsSync, readFileSync } from 'node:fs'
2
- import { dirname, join } from 'node:path'
2
+ import { join } from 'node:path'
3
3
 
4
4
  import { optionString, scanMarkdownDocs, type HumanAdapter } from './core.js'
5
5
 
@@ -1,5 +1,5 @@
1
1
  import { existsSync, watch } from 'node:fs'
2
- import { dirname, join, resolve } from 'node:path'
2
+ import { dirname, resolve } from 'node:path'
3
3
 
4
4
  import type { DocBridgeConfigV1 } from '../config/schema.js'
5
5
  import { buildDocBridgeIndex } from './build-index.js'
package/src/index.ts CHANGED
@@ -20,11 +20,13 @@ export {
20
20
  RulesConfigSchema,
21
21
  WorkflowConfigSchema,
22
22
  RepositorySafetyConfigSchema,
23
+ ReportConfigSchema,
23
24
  type RuleId,
24
25
  type RuleSeverity,
25
26
  type RulesConfig,
26
27
  type WorkflowConfig,
27
28
  type RepositorySafetyConfig,
29
+ type ReportConfig,
28
30
  } from './config/schema.js'
29
31
 
30
32
  export {
@@ -180,6 +182,33 @@ export {
180
182
  type GithubPrResult,
181
183
  } from './memory/github-pr.js'
182
184
  export { canonicalJsonV1, contentHashForArtifactV1, sha256NormalizedV1 } from './index-builder/content-hash.js'
185
+ export {
186
+ ANALYZER_PLUGIN_CONTRACT_VERSION,
187
+ AnalyzerPluginManifestSchema,
188
+ AnalyzerPluginOutputSchema,
189
+ createAnalyzerRegistry,
190
+ type AnalyzerPlugin,
191
+ type AnalyzerPluginInput,
192
+ type AnalyzerPluginManifest,
193
+ type AnalyzerPluginOutput,
194
+ type AnalyzerRegistry,
195
+ } from './plugins/contract.js'
196
+ export {
197
+ BENCHMARK_SCHEMA_VERSION,
198
+ BenchmarkFixtureV1Schema,
199
+ benchmarkFixture,
200
+ compareBenchmarkSnapshots,
201
+ formatBenchmarkText,
202
+ measureAgentEfficiency,
203
+ measureBenchmark,
204
+ type AgentEfficiencyObservation,
205
+ type BenchmarkFixtureV1,
206
+ type BenchmarkObservation,
207
+ type BenchmarkResult,
208
+ type BenchmarkSetMetrics,
209
+ type BenchmarkSnapshot,
210
+ type BenchmarkSnapshotDiff,
211
+ } from './metrics/benchmark.js'
183
212
  export {
184
213
  AgentProposalV1Schema,
185
214
  AffectedFileSchema,
@@ -1,4 +1,4 @@
1
- import { readFileSync, statSync } from 'node:fs'
1
+ import { closeSync, fstatSync, openSync, readFileSync } from 'node:fs'
2
2
 
3
3
  export const MAX_DOCUMENT_BYTES = 4 * 1_024 * 1_024
4
4
  export const MAX_CORPUS_BYTES = 64 * 1_024 * 1_024
@@ -12,14 +12,19 @@ export const readBoundedText = (
12
12
  ): string => {
13
13
  const maxFileBytes = limits?.maxFileBytes ?? MAX_DOCUMENT_BYTES
14
14
  const maxCorpusBytes = limits?.maxCorpusBytes ?? MAX_CORPUS_BYTES
15
- const stat = statSync(path)
16
- if (!stat.isFile()) throw new Error(`Documentation path is not a regular file: ${path}`)
17
- if (stat.size > maxFileBytes) {
18
- throw new Error(`Documentation file exceeds the ${maxFileBytes} byte limit: ${path}`)
15
+ const fd = openSync(path, 'r')
16
+ try {
17
+ const stat = fstatSync(fd)
18
+ if (!stat.isFile()) throw new Error(`Documentation path is not a regular file: ${path}`)
19
+ if (stat.size > maxFileBytes) {
20
+ throw new Error(`Documentation file exceeds the ${maxFileBytes} byte limit: ${path}`)
21
+ }
22
+ if (budget.used + stat.size > maxCorpusBytes) {
23
+ throw new Error(`Documentation corpus exceeds the ${maxCorpusBytes} byte read budget.`)
24
+ }
25
+ budget.used += stat.size
26
+ return readFileSync(fd, 'utf8')
27
+ } finally {
28
+ closeSync(fd)
19
29
  }
20
- if (budget.used + stat.size > maxCorpusBytes) {
21
- throw new Error(`Documentation corpus exceeds the ${maxCorpusBytes} byte read budget.`)
22
- }
23
- budget.used += stat.size
24
- return readFileSync(path, 'utf8')
25
30
  }
@@ -0,0 +1,176 @@
1
+ import { z } from 'zod'
2
+
3
+ export const BENCHMARK_SCHEMA_VERSION = 1 as const
4
+
5
+ const stringList = z.array(z.string().min(1).max(512)).max(100_000)
6
+ const benchmarkSets = z.object({
7
+ entities: stringList.default([]),
8
+ relations: stringList.default([]),
9
+ findings: stringList.default([]),
10
+ }).strict()
11
+
12
+ export const BenchmarkFixtureV1Schema = z.object({
13
+ schemaVersion: z.literal(BENCHMARK_SCHEMA_VERSION),
14
+ supported: benchmarkSets,
15
+ excluded: benchmarkSets.optional(),
16
+ }).strict()
17
+
18
+ export type BenchmarkFixtureV1 = z.infer<typeof BenchmarkFixtureV1Schema>
19
+
20
+ export type BenchmarkObservation = {
21
+ readonly entities: readonly string[]
22
+ readonly relations: readonly string[]
23
+ readonly findings: readonly string[]
24
+ readonly evidenced?: readonly string[]
25
+ readonly findingCategories?: Readonly<Record<string, number>>
26
+ }
27
+
28
+ export type BenchmarkSetMetrics = {
29
+ readonly truePositives: number
30
+ readonly falsePositives: number
31
+ readonly falseNegatives: number
32
+ readonly precision: number
33
+ readonly recall: number
34
+ readonly duplicateCount: number
35
+ }
36
+
37
+ export type BenchmarkResult = {
38
+ readonly schemaVersion: typeof BENCHMARK_SCHEMA_VERSION
39
+ readonly quality: {
40
+ readonly entities: BenchmarkSetMetrics
41
+ readonly relations: BenchmarkSetMetrics
42
+ readonly findings: BenchmarkSetMetrics
43
+ }
44
+ readonly evidenceRatio: number
45
+ readonly findingDensity: number
46
+ readonly findingCategoryDistribution: Readonly<Record<string, number>>
47
+ readonly excludedCaseCount: number
48
+ readonly excludedCaseIds: Readonly<{ entities: number; relations: number; findings: number }>
49
+ readonly thresholds: { readonly precision: number; readonly recall: number }
50
+ readonly regressions: readonly string[]
51
+ }
52
+
53
+ const unique = (values: readonly string[]): Set<string> => new Set(values)
54
+ const ratio = (numerator: number, denominator: number): number => denominator === 0 ? 1 : numerator / denominator
55
+
56
+ const setMetrics = (
57
+ actualValues: readonly string[],
58
+ expectedValues: readonly string[],
59
+ excludedValues: readonly string[],
60
+ ): BenchmarkSetMetrics => {
61
+ const excluded = unique(excludedValues)
62
+ const activeActualValues = actualValues.filter((value) => !excluded.has(value))
63
+ const actual = unique(activeActualValues)
64
+ const expected = unique(expectedValues.filter((value) => !excluded.has(value)))
65
+ const truePositives = [...actual].filter((value) => expected.has(value)).length
66
+ const falsePositives = actual.size - truePositives
67
+ const falseNegatives = expected.size - truePositives
68
+ return {
69
+ truePositives,
70
+ falsePositives,
71
+ falseNegatives,
72
+ precision: ratio(truePositives, actual.size),
73
+ recall: ratio(truePositives, expected.size),
74
+ duplicateCount: activeActualValues.length - actual.size,
75
+ }
76
+ }
77
+
78
+ export const benchmarkFixture = (fixture: unknown): BenchmarkFixtureV1 => BenchmarkFixtureV1Schema.parse(fixture)
79
+
80
+ export const measureBenchmark = (
81
+ observation: BenchmarkObservation,
82
+ fixture: BenchmarkFixtureV1,
83
+ thresholds: { readonly precision?: number; readonly recall?: number } = {},
84
+ ): BenchmarkResult => {
85
+ const excluded = fixture.excluded ?? { entities: [], relations: [], findings: [] }
86
+ const quality = {
87
+ entities: setMetrics(observation.entities, fixture.supported.entities, excluded.entities),
88
+ relations: setMetrics(observation.relations, fixture.supported.relations, excluded.relations),
89
+ findings: setMetrics(observation.findings, fixture.supported.findings, excluded.findings),
90
+ }
91
+ const actualEntities = unique(observation.entities)
92
+ const evidenced = unique(observation.evidenced ?? [])
93
+ const evidenceRatio = ratio([...actualEntities].filter((id) => evidenced.has(id)).length, actualEntities.size)
94
+ const findingDensity = ratio(unique(observation.findings).size, actualEntities.size)
95
+ const precisionThreshold = thresholds.precision ?? 0.95
96
+ const recallThreshold = thresholds.recall ?? 1
97
+ const regressions: string[] = []
98
+ for (const [name, metrics] of Object.entries(quality)) {
99
+ if (metrics.precision < precisionThreshold) regressions.push(`${name}.precision below ${precisionThreshold}`)
100
+ if (metrics.recall < recallThreshold) regressions.push(`${name}.recall below ${recallThreshold}`)
101
+ }
102
+ return {
103
+ schemaVersion: BENCHMARK_SCHEMA_VERSION,
104
+ quality,
105
+ evidenceRatio,
106
+ findingDensity,
107
+ findingCategoryDistribution: Object.fromEntries(Object.entries(observation.findingCategories ?? {}).sort(([a], [b]) => a.localeCompare(b))),
108
+ excludedCaseCount: excluded.entities.length + excluded.relations.length + excluded.findings.length,
109
+ excludedCaseIds: { entities: excluded.entities.length, relations: excluded.relations.length, findings: excluded.findings.length },
110
+ thresholds: { precision: precisionThreshold, recall: recallThreshold },
111
+ regressions,
112
+ }
113
+ }
114
+
115
+ export type BenchmarkSnapshot = Readonly<Record<string, string>>
116
+
117
+ export type BenchmarkSnapshotDiff = {
118
+ readonly added: number
119
+ readonly removed: number
120
+ readonly unchanged: number
121
+ readonly reclassified: number
122
+ readonly unchangedEvidence: number
123
+ }
124
+
125
+ export const compareBenchmarkSnapshots = (
126
+ previous: BenchmarkSnapshot,
127
+ current: BenchmarkSnapshot,
128
+ ): BenchmarkSnapshotDiff => {
129
+ const ids = new Set([...Object.keys(previous), ...Object.keys(current)])
130
+ let added = 0
131
+ let removed = 0
132
+ let unchanged = 0
133
+ let reclassified = 0
134
+ let unchangedEvidence = 0
135
+ for (const id of ids) {
136
+ if (!(id in previous)) { added += 1; continue }
137
+ if (!(id in current)) { removed += 1; continue }
138
+ if (previous[id] === current[id]) unchanged += 1
139
+ else reclassified += 1
140
+ if (previous[id] === current[id]) unchangedEvidence += 1
141
+ }
142
+ return { added, removed, unchanged, reclassified, unchangedEvidence }
143
+ }
144
+
145
+ export type AgentEfficiencyObservation = {
146
+ readonly hits: number
147
+ readonly queries: number
148
+ readonly latencyMs: readonly number[]
149
+ readonly responseBytes: readonly number[]
150
+ readonly estimatedTokens: readonly number[]
151
+ readonly corpusBytes: number
152
+ }
153
+
154
+ const percentile95 = (values: readonly number[]): number => {
155
+ if (!values.length) return 0
156
+ const sorted = [...values].sort((a, b) => a - b)
157
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
158
+ }
159
+
160
+ export const measureAgentEfficiency = (observation: AgentEfficiencyObservation) => ({
161
+ hitRate: ratio(observation.hits, observation.queries),
162
+ latencyP95Ms: percentile95(observation.latencyMs),
163
+ responseBytesP95: percentile95(observation.responseBytes),
164
+ estimatedTokensP95: percentile95(observation.estimatedTokens),
165
+ corpusBytes: observation.corpusBytes,
166
+ contextReduction: observation.corpusBytes > 0 ? 1 - (percentile95(observation.responseBytes) / observation.corpusBytes) : 0,
167
+ })
168
+
169
+ export const formatBenchmarkText = (result: BenchmarkResult): string => [
170
+ `Precision: entities ${result.quality.entities.precision.toFixed(3)}, relations ${result.quality.relations.precision.toFixed(3)}, findings ${result.quality.findings.precision.toFixed(3)}`,
171
+ `Recall: entities ${result.quality.entities.recall.toFixed(3)}, relations ${result.quality.relations.recall.toFixed(3)}, findings ${result.quality.findings.recall.toFixed(3)}`,
172
+ `Evidence ratio: ${result.evidenceRatio.toFixed(3)}`,
173
+ `Finding density: ${result.findingDensity.toFixed(3)}`,
174
+ `Excluded cases: ${result.excludedCaseCount}`,
175
+ `Regressions: ${result.regressions.length ? result.regressions.join('; ') : 'none'}`,
176
+ ].join('\n')
@@ -0,0 +1,89 @@
1
+ import { z } from 'zod'
2
+
3
+ import { CoverageSchema, DiagnosticSchema, EntitySchema, RelationSchema, type Coverage, type KnowledgeDiagnostic, type KnowledgeEntity, type KnowledgeRelation } from '../schemas/knowledge.js'
4
+
5
+ export const ANALYZER_PLUGIN_CONTRACT_VERSION = 1 as const
6
+
7
+ export const AnalyzerPluginManifestSchema = z.object({
8
+ id: z.string().regex(/^[a-z][a-z0-9-]*$/).max(128),
9
+ version: z.string().min(1).max(64),
10
+ languages: z.array(z.string().min(1).max(64)).min(1).max(32),
11
+ frameworks: z.array(z.string().min(1).max(128)).max(32).default([]),
12
+ capabilities: z.array(z.string().min(1).max(128)).min(1).max(32),
13
+ knowledgeSchemaVersion: z.literal(1),
14
+ compatibility: z.object({ pipelineMajor: z.number().int().nonnegative() }).strict(),
15
+ unsupportedConstructs: z.array(z.string().min(1).max(256)).max(128).default([]),
16
+ resourceLimits: z.object({ maxFiles: z.number().int().positive().optional(), maxBytes: z.number().int().positive().optional() }).strict().default({}),
17
+ }).strict()
18
+
19
+ export type AnalyzerPluginManifest = z.infer<typeof AnalyzerPluginManifestSchema>
20
+
21
+ export const AnalyzerPluginOutputSchema = z.object({
22
+ entities: z.array(EntitySchema).max(50_000).default([]),
23
+ relations: z.array(RelationSchema).max(100_000).default([]),
24
+ coverage: z.array(CoverageSchema).max(1_000).default([]),
25
+ diagnostics: z.array(DiagnosticSchema).max(100_000).default([]),
26
+ }).strict()
27
+
28
+ export type AnalyzerPluginOutput = z.infer<typeof AnalyzerPluginOutputSchema>
29
+
30
+ export type AnalyzerPluginInput = {
31
+ readonly language: string
32
+ readonly framework?: string
33
+ readonly files: readonly { readonly path: string; readonly bytes: number }[]
34
+ readonly value?: unknown
35
+ }
36
+
37
+ export type AnalyzerPlugin = {
38
+ readonly manifest: AnalyzerPluginManifest
39
+ readonly analyze: (input: AnalyzerPluginInput) => Promise<unknown> | unknown
40
+ }
41
+
42
+ export type AnalyzerRegistry = {
43
+ readonly register: (plugin: AnalyzerPlugin) => void
44
+ readonly list: () => readonly AnalyzerPluginManifest[]
45
+ readonly analyze: (id: string, input: AnalyzerPluginInput) => Promise<AnalyzerPluginOutput>
46
+ }
47
+
48
+ const pipelineMajor = (version: string): number => Number.parseInt(version.split('.')[0] ?? '', 10)
49
+
50
+ export const createAnalyzerRegistry = (options: { readonly pipelineVersion?: string; readonly maxPlugins?: number } = {}): AnalyzerRegistry => {
51
+ const plugins = new Map<string, AnalyzerPlugin>()
52
+ const pipeline = pipelineMajor(options.pipelineVersion ?? '1.0.0')
53
+ return {
54
+ register(plugin) {
55
+ const manifest = AnalyzerPluginManifestSchema.parse(plugin.manifest)
56
+ if (manifest.compatibility.pipelineMajor !== pipeline) throw new Error(`Analyzer plugin "${manifest.id}" requires pipeline major ${manifest.compatibility.pipelineMajor}; current pipeline is ${pipeline}.`)
57
+ if (plugins.has(manifest.id)) throw new Error(`Analyzer plugin "${manifest.id}" is already registered.`)
58
+ if (options.maxPlugins !== undefined && plugins.size >= options.maxPlugins) throw new Error(`Analyzer plugin limit ${options.maxPlugins} exceeded.`)
59
+ plugins.set(manifest.id, { ...plugin, manifest })
60
+ },
61
+ list() {
62
+ return [...plugins.values()].map((plugin) => plugin.manifest).sort((a, b) => a.id.localeCompare(b.id))
63
+ },
64
+ async analyze(id, input) {
65
+ const plugin = plugins.get(id)
66
+ if (!plugin) throw new Error(`Analyzer plugin "${id}" is not registered.`)
67
+ const { maxFiles, maxBytes } = plugin.manifest.resourceLimits
68
+ const bytes = input.files.reduce((total, file) => total + file.bytes, 0)
69
+ if (maxFiles !== undefined && input.files.length > maxFiles) throw new Error(`Analyzer plugin "${id}" file limit ${maxFiles} exceeded.`)
70
+ if (maxBytes !== undefined && bytes > maxBytes) throw new Error(`Analyzer plugin "${id}" byte limit ${maxBytes} exceeded.`)
71
+ try {
72
+ const output = AnalyzerPluginOutputSchema.parse(await plugin.analyze(input))
73
+ return {
74
+ ...output,
75
+ coverage: output.coverage.map((entry) => ({ ...entry, analyzer: plugin.manifest.id, analyzerVersion: plugin.manifest.version })),
76
+ }
77
+ } catch (error) {
78
+ return {
79
+ entities: [],
80
+ relations: [],
81
+ diagnostics: [],
82
+ coverage: [{ analyzer: plugin.manifest.id, analyzerVersion: plugin.manifest.version, scope: 'plugin', status: 'not-analyzed', reason: `Plugin failed safely: ${error instanceof Error ? error.message : String(error)}` }],
83
+ }
84
+ }
85
+ },
86
+ }
87
+ }
88
+
89
+ export type { Coverage, KnowledgeDiagnostic, KnowledgeEntity, KnowledgeRelation }