@agentskit/doc-bridge 1.6.4 → 1.7.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +249 -0
- package/CONTRIBUTING.md +6 -4
- package/action.yml +1 -1
- package/dist/cli/program.js +1139 -294
- package/dist/cli/program.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +43 -5
- package/dist/config/index.js.map +1 -1
- package/dist/index-BUL0q7s8.d.ts +660 -0
- package/dist/index.d.ts +817 -2134
- package/dist/index.js +1154 -244
- package/dist/index.js.map +1 -1
- package/docs/PRD-enterprise-hardening.md +288 -0
- package/docs/RELEASE.md +22 -8
- package/docs/adr/0001-enterprise-verification-contract.md +35 -0
- package/docs/agent-corpus/INDEX.md +2 -2
- package/docs/agent-corpus/chat.md +2 -2
- package/docs/agent-corpus/cli.md +2 -2
- package/docs/agent-corpus/conformance.md +2 -2
- package/docs/agent-corpus/doc-bridge.md +1 -1
- package/docs/agent-corpus/doctor.md +2 -2
- package/docs/agent-corpus/gates.md +2 -2
- package/docs/agent-corpus/mcp.md +2 -2
- package/docs/agent-corpus/memory.md +2 -2
- package/docs/agent-corpus/query.md +2 -2
- package/docs/knowledge-engine-runbook.md +30 -2
- package/docs/spec/analyzer-plugin-v1.md +24 -0
- package/docs/spec/benchmark-v1.md +36 -0
- package/docs/spec/config-v1.md +156 -0
- package/docs/validation-cycle-plan.md +255 -0
- package/docs/verification-harness.md +37 -4
- package/mcpb/manifest.json +1 -1
- package/package.json +68 -70
- package/scripts/check-ecosystem-upstream.mjs +3 -2
- package/scripts/report-visual-check.mjs +64 -12
- package/scripts/verification-harness.mjs +216 -14
- package/skills/doc-bridge-handoff/scripts/resolve-handoff.mjs +1 -1
- package/src/agents/registry-adapter.ts +31 -7
- package/src/cli/demo.ts +2 -2
- package/src/cli/program.ts +59 -16
- package/src/config/index.ts +2 -0
- package/src/config/load-config.ts +7 -1
- package/src/config/schema.ts +60 -2
- package/src/conformance/documentation-standard-v1.ts +14 -8
- package/src/discovery/documentation.ts +90 -23
- package/src/discovery/repository.ts +147 -19
- package/src/doctor/run-doctor.ts +2 -15
- package/src/federation/llms.ts +72 -20
- package/src/fixes/proposals.ts +4 -3
- package/src/index-builder/human-adapters/fumadocs.ts +1 -1
- package/src/index-builder/watch-index.ts +1 -1
- package/src/index.ts +29 -0
- package/src/lib/bounded-text.ts +15 -10
- package/src/metrics/benchmark.ts +176 -0
- package/src/plugins/contract.ts +89 -0
- package/src/reconciliation/reconcile.ts +181 -5
- package/src/report/html.ts +318 -88
- package/src/rules/engine.ts +15 -2
- package/src/safety/repository.ts +1 -1
- package/src/schemas/knowledge.ts +21 -3
- package/src/validate.ts +7 -1
- package/src/version.ts +1 -1
- package/src/workflow/engine.ts +65 -9
- package/dist/index-DudNuwI5.d.ts +0 -2060
package/src/doctor/run-doctor.ts
CHANGED
|
@@ -52,17 +52,6 @@ const gradeForScore = (score: number): DoctorReport['grade'] => {
|
|
|
52
52
|
return 'F'
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
const agentDocPaths = (index: DocBridgeIndexV1): Set<string> => {
|
|
56
|
-
const paths = new Set<string>()
|
|
57
|
-
for (const owner of Object.values(index.lookup?.ownership ?? {})) {
|
|
58
|
-
if (owner.agentDoc) paths.add(owner.agentDoc)
|
|
59
|
-
}
|
|
60
|
-
for (const handoff of Object.values(index.handoffs ?? {})) {
|
|
61
|
-
if (handoff.startHere) paths.add(handoff.startHere)
|
|
62
|
-
}
|
|
63
|
-
return paths
|
|
64
|
-
}
|
|
65
|
-
|
|
66
55
|
const computeScore = (coverage: DoctorCoverage): number => {
|
|
67
56
|
let score = 0
|
|
68
57
|
|
|
@@ -166,7 +155,7 @@ export const runDoctor = (root: string, config: DocBridgeConfigV1): DoctorReport
|
|
|
166
155
|
let index: DocBridgeIndexV1 | undefined
|
|
167
156
|
let hasIndex = true
|
|
168
157
|
let freshnessOk = false
|
|
169
|
-
let freshnessMessage
|
|
158
|
+
let freshnessMessage: string
|
|
170
159
|
|
|
171
160
|
try {
|
|
172
161
|
index = loadDocBridgeIndex(root, config)
|
|
@@ -191,8 +180,6 @@ export const runDoctor = (root: string, config: DocBridgeConfigV1): DoctorReport
|
|
|
191
180
|
const missingHumanDoc = ownership.filter(([, owner]) => !owner.humanDoc).map(([id]) => id)
|
|
192
181
|
|
|
193
182
|
const indexedPaths = new Set(index.knowledge.map((entry) => entry.path))
|
|
194
|
-
const expectedAgentDocs = agentDocPaths(index)
|
|
195
|
-
const unindexed = [...expectedAgentDocs].filter((path) => !indexedPaths.has(path))
|
|
196
183
|
|
|
197
184
|
const corpusDocs = scanAgentCorpus(root, config).filter(
|
|
198
185
|
(doc) => doc.path !== config.corpus.agent.index,
|
|
@@ -283,4 +270,4 @@ export const formatDoctorText = (report: DoctorReport): string[] => {
|
|
|
283
270
|
|
|
284
271
|
lines.push('', 'Next actions', ...report.nextActions.map((action) => ` → ${action}`))
|
|
285
272
|
return lines
|
|
286
|
-
}
|
|
273
|
+
}
|
package/src/federation/llms.ts
CHANGED
|
@@ -31,13 +31,25 @@ const defaultFetchText: FetchText = async (url) => {
|
|
|
31
31
|
return res.text()
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
const httpUrl = (value: string): string | undefined => {
|
|
35
|
+
try {
|
|
36
|
+
const parsed = new URL(value)
|
|
37
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return undefined
|
|
38
|
+
if (parsed.username || parsed.password) return undefined
|
|
39
|
+
return parsed.href
|
|
40
|
+
} catch {
|
|
41
|
+
return undefined
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
34
45
|
const sourceText = async (
|
|
35
46
|
root: string,
|
|
36
47
|
source: string,
|
|
37
48
|
fetchText: FetchText,
|
|
38
49
|
): Promise<string | null> => {
|
|
39
50
|
try {
|
|
40
|
-
|
|
51
|
+
const remote = httpUrl(source)
|
|
52
|
+
if (remote) return await fetchText(remote)
|
|
41
53
|
const path = resolve(root, source)
|
|
42
54
|
if (!existsSync(path)) return null
|
|
43
55
|
return readFileSync(path, 'utf8')
|
|
@@ -52,34 +64,72 @@ const sameOrigin = (base: string, target: string): boolean => {
|
|
|
52
64
|
}
|
|
53
65
|
|
|
54
66
|
export const parseLlmsTxtLinks = (raw: string): { title: string; url: string; description?: string }[] => {
|
|
55
|
-
const links
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
67
|
+
const links: { title: string; url: string; description?: string }[] = []
|
|
68
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
69
|
+
let cursor = 0
|
|
70
|
+
while (cursor < line.length) {
|
|
71
|
+
const open = line.indexOf('[', cursor)
|
|
72
|
+
if (open < 0) break
|
|
73
|
+
const titleEnd = line.indexOf(']', open + 1)
|
|
74
|
+
const urlStart = titleEnd < 0 ? -1 : line.indexOf('(', titleEnd + 1)
|
|
75
|
+
const urlEnd = urlStart < 0 ? -1 : line.indexOf(')', urlStart + 1)
|
|
76
|
+
if (titleEnd < 0 || urlStart !== titleEnd + 1 || urlEnd < 0) {
|
|
77
|
+
cursor = open + 1
|
|
78
|
+
continue
|
|
79
|
+
}
|
|
80
|
+
const title = line.slice(open + 1, titleEnd).trim()
|
|
81
|
+
const url = line.slice(urlStart + 1, urlEnd).trim()
|
|
82
|
+
const description = line.slice(urlEnd + 1).trim().replace(/^:\s*/, '')
|
|
83
|
+
if (url) links.push({ title: title || url, url, ...(description ? { description } : {}) })
|
|
84
|
+
cursor = urlEnd + 1
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
for (const token of line.split(/\s+/)) {
|
|
88
|
+
let end = token.length
|
|
89
|
+
while (end > 0 && '),.;:'.includes(token[end - 1] ?? '')) end -= 1
|
|
90
|
+
const url = token.slice(0, end)
|
|
91
|
+
if (!/^https?:\/\//i.test(url) || links.some((link) => link.url === url)) continue
|
|
92
|
+
links.push({ title: slugFromPath(url), url })
|
|
93
|
+
}
|
|
64
94
|
}
|
|
65
95
|
return links
|
|
66
96
|
}
|
|
67
97
|
|
|
98
|
+
const firstMatchingLine = (section: string, predicate: (line: string) => boolean): string | undefined =>
|
|
99
|
+
section.split(/\r?\n/).find((line) => predicate(line.trim()))?.trim()
|
|
100
|
+
|
|
101
|
+
const sectionTitle = (section: string, sourceUrl: string): string => {
|
|
102
|
+
const titleLine = firstMatchingLine(section, (line) => line.startsWith('title:'))
|
|
103
|
+
if (titleLine) return titleLine.slice('title:'.length).trim()
|
|
104
|
+
const urlLine = firstMatchingLine(section, (line) => /^https?:\/\//i.test(line))
|
|
105
|
+
if (urlLine) {
|
|
106
|
+
try {
|
|
107
|
+
return new URL(urlLine).pathname.split('/').filter(Boolean).at(-1) ?? slugFromPath(sourceUrl)
|
|
108
|
+
} catch {
|
|
109
|
+
return slugFromPath(sourceUrl)
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const heading = firstMatchingLine(section, (line) => line.startsWith('#'))
|
|
113
|
+
if (heading) return heading.replace(/^#+\s*/, '').trim()
|
|
114
|
+
return slugFromPath(sourceUrl)
|
|
115
|
+
}
|
|
116
|
+
|
|
68
117
|
export const chunksFromMarkdown = (
|
|
69
118
|
property: string,
|
|
70
119
|
raw: string,
|
|
71
120
|
sourceUrl: string,
|
|
72
121
|
): DocBridgeRetrievedChunk[] => {
|
|
73
|
-
const
|
|
122
|
+
const frontmatterEnd = raw.startsWith('---\n') ? raw.indexOf('\n---', 4) : -1
|
|
123
|
+
const searchable = raw.includes('\n==== ')
|
|
124
|
+
? raw
|
|
125
|
+
: frontmatterEnd >= 0
|
|
126
|
+
? raw.slice(frontmatterEnd + '\n---'.length).replace(/^\n/, '')
|
|
127
|
+
: raw
|
|
74
128
|
const sections = searchable.includes('\n==== ')
|
|
75
129
|
? searchable.split(/\n====\s+/).filter((section) => section.trim())
|
|
76
130
|
: searchable.split(/\n(?=##?\s+)/)
|
|
77
131
|
return sections.map((section, index) => {
|
|
78
|
-
const title =
|
|
79
|
-
/^title:\s*(.+)$/m.exec(section)?.[1]?.trim() ??
|
|
80
|
-
/^https?:\/\/\S+\/([^/\s]+)$/m.exec(section)?.[1]?.trim() ??
|
|
81
|
-
/^#+\s+(.+)$/m.exec(section)?.[1]?.trim() ??
|
|
82
|
-
slugFromPath(sourceUrl)
|
|
132
|
+
const title = sectionTitle(section, sourceUrl)
|
|
83
133
|
const id = slugFromPath(title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')) || `${index}`
|
|
84
134
|
return {
|
|
85
135
|
chunkKey: `${property}:federated:${id}`,
|
|
@@ -112,10 +162,12 @@ export const loadFederatedChunks = async (
|
|
|
112
162
|
chunks.push(...chunksFromMarkdown(source.id, llms, source.llmsTxt))
|
|
113
163
|
const links = parseLlmsTxtLinks(llms)
|
|
114
164
|
for (const link of links) {
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
165
|
+
const linkUrl = httpUrl(link.url)
|
|
166
|
+
const baseUrl = source.rawBaseUrl ? httpUrl(source.rawBaseUrl) : undefined
|
|
167
|
+
const url = linkUrl ?? (baseUrl ? new URL(link.url, baseUrl).href : link.url)
|
|
168
|
+
let pathname = url
|
|
169
|
+
try { pathname = new URL(url).pathname } catch { /* local source */ }
|
|
170
|
+
if (!/\.(md|txt)$/i.test(pathname)) continue
|
|
119
171
|
if (!sameOrigin(source.llmsTxt, url)) continue
|
|
120
172
|
const raw = await sourceText(root, url, fetchText)
|
|
121
173
|
if (raw) chunks.push(...chunksFromMarkdown(source.id, raw, url))
|
package/src/fixes/proposals.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync, readdirSync, readFileSync, realpathSync, renameSync,
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
2
2
|
import { basename, dirname, extname, join, relative, resolve, sep } from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
|
|
@@ -99,8 +99,9 @@ export const createArtifactNormalizationProposal = (root: string, artifactPath:
|
|
|
99
99
|
const projectRoot = realpathSync.native(resolve(root))
|
|
100
100
|
const path = artifactPath.split(sep).join('/')
|
|
101
101
|
const absolute = containedPath(projectRoot, path)
|
|
102
|
-
if (!absolute
|
|
103
|
-
|
|
102
|
+
if (!absolute) return undefined
|
|
103
|
+
let before: string
|
|
104
|
+
try { before = readFileSync(absolute, 'utf8') } catch { return undefined }
|
|
104
105
|
let after: string
|
|
105
106
|
try { after = `${JSON.stringify(sortJson(JSON.parse(before) as unknown), null, 2)}\n` } catch { return undefined }
|
|
106
107
|
return after === before ? undefined : makeProposal(projectRoot, options, [{ path: relative(projectRoot, absolute).split(sep).join('/'), before, after }], ['The artifact contains valid JSON.'], ['The artifact is valid canonical JSON with one trailing newline.'])
|
package/src/index.ts
CHANGED
|
@@ -20,11 +20,13 @@ export {
|
|
|
20
20
|
RulesConfigSchema,
|
|
21
21
|
WorkflowConfigSchema,
|
|
22
22
|
RepositorySafetyConfigSchema,
|
|
23
|
+
ReportConfigSchema,
|
|
23
24
|
type RuleId,
|
|
24
25
|
type RuleSeverity,
|
|
25
26
|
type RulesConfig,
|
|
26
27
|
type WorkflowConfig,
|
|
27
28
|
type RepositorySafetyConfig,
|
|
29
|
+
type ReportConfig,
|
|
28
30
|
} from './config/schema.js'
|
|
29
31
|
|
|
30
32
|
export {
|
|
@@ -180,6 +182,33 @@ export {
|
|
|
180
182
|
type GithubPrResult,
|
|
181
183
|
} from './memory/github-pr.js'
|
|
182
184
|
export { canonicalJsonV1, contentHashForArtifactV1, sha256NormalizedV1 } from './index-builder/content-hash.js'
|
|
185
|
+
export {
|
|
186
|
+
ANALYZER_PLUGIN_CONTRACT_VERSION,
|
|
187
|
+
AnalyzerPluginManifestSchema,
|
|
188
|
+
AnalyzerPluginOutputSchema,
|
|
189
|
+
createAnalyzerRegistry,
|
|
190
|
+
type AnalyzerPlugin,
|
|
191
|
+
type AnalyzerPluginInput,
|
|
192
|
+
type AnalyzerPluginManifest,
|
|
193
|
+
type AnalyzerPluginOutput,
|
|
194
|
+
type AnalyzerRegistry,
|
|
195
|
+
} from './plugins/contract.js'
|
|
196
|
+
export {
|
|
197
|
+
BENCHMARK_SCHEMA_VERSION,
|
|
198
|
+
BenchmarkFixtureV1Schema,
|
|
199
|
+
benchmarkFixture,
|
|
200
|
+
compareBenchmarkSnapshots,
|
|
201
|
+
formatBenchmarkText,
|
|
202
|
+
measureAgentEfficiency,
|
|
203
|
+
measureBenchmark,
|
|
204
|
+
type AgentEfficiencyObservation,
|
|
205
|
+
type BenchmarkFixtureV1,
|
|
206
|
+
type BenchmarkObservation,
|
|
207
|
+
type BenchmarkResult,
|
|
208
|
+
type BenchmarkSetMetrics,
|
|
209
|
+
type BenchmarkSnapshot,
|
|
210
|
+
type BenchmarkSnapshotDiff,
|
|
211
|
+
} from './metrics/benchmark.js'
|
|
183
212
|
export {
|
|
184
213
|
AgentProposalV1Schema,
|
|
185
214
|
AffectedFileSchema,
|
package/src/lib/bounded-text.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { closeSync, fstatSync, openSync, readFileSync } from 'node:fs'
|
|
2
2
|
|
|
3
3
|
export const MAX_DOCUMENT_BYTES = 4 * 1_024 * 1_024
|
|
4
4
|
export const MAX_CORPUS_BYTES = 64 * 1_024 * 1_024
|
|
@@ -12,14 +12,19 @@ export const readBoundedText = (
|
|
|
12
12
|
): string => {
|
|
13
13
|
const maxFileBytes = limits?.maxFileBytes ?? MAX_DOCUMENT_BYTES
|
|
14
14
|
const maxCorpusBytes = limits?.maxCorpusBytes ?? MAX_CORPUS_BYTES
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
throw new Error(`Documentation
|
|
15
|
+
const fd = openSync(path, 'r')
|
|
16
|
+
try {
|
|
17
|
+
const stat = fstatSync(fd)
|
|
18
|
+
if (!stat.isFile()) throw new Error(`Documentation path is not a regular file: ${path}`)
|
|
19
|
+
if (stat.size > maxFileBytes) {
|
|
20
|
+
throw new Error(`Documentation file exceeds the ${maxFileBytes} byte limit: ${path}`)
|
|
21
|
+
}
|
|
22
|
+
if (budget.used + stat.size > maxCorpusBytes) {
|
|
23
|
+
throw new Error(`Documentation corpus exceeds the ${maxCorpusBytes} byte read budget.`)
|
|
24
|
+
}
|
|
25
|
+
budget.used += stat.size
|
|
26
|
+
return readFileSync(fd, 'utf8')
|
|
27
|
+
} finally {
|
|
28
|
+
closeSync(fd)
|
|
19
29
|
}
|
|
20
|
-
if (budget.used + stat.size > maxCorpusBytes) {
|
|
21
|
-
throw new Error(`Documentation corpus exceeds the ${maxCorpusBytes} byte read budget.`)
|
|
22
|
-
}
|
|
23
|
-
budget.used += stat.size
|
|
24
|
-
return readFileSync(path, 'utf8')
|
|
25
30
|
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
export const BENCHMARK_SCHEMA_VERSION = 1 as const
|
|
4
|
+
|
|
5
|
+
const stringList = z.array(z.string().min(1).max(512)).max(100_000)
|
|
6
|
+
const benchmarkSets = z.object({
|
|
7
|
+
entities: stringList.default([]),
|
|
8
|
+
relations: stringList.default([]),
|
|
9
|
+
findings: stringList.default([]),
|
|
10
|
+
}).strict()
|
|
11
|
+
|
|
12
|
+
export const BenchmarkFixtureV1Schema = z.object({
|
|
13
|
+
schemaVersion: z.literal(BENCHMARK_SCHEMA_VERSION),
|
|
14
|
+
supported: benchmarkSets,
|
|
15
|
+
excluded: benchmarkSets.optional(),
|
|
16
|
+
}).strict()
|
|
17
|
+
|
|
18
|
+
export type BenchmarkFixtureV1 = z.infer<typeof BenchmarkFixtureV1Schema>
|
|
19
|
+
|
|
20
|
+
export type BenchmarkObservation = {
|
|
21
|
+
readonly entities: readonly string[]
|
|
22
|
+
readonly relations: readonly string[]
|
|
23
|
+
readonly findings: readonly string[]
|
|
24
|
+
readonly evidenced?: readonly string[]
|
|
25
|
+
readonly findingCategories?: Readonly<Record<string, number>>
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type BenchmarkSetMetrics = {
|
|
29
|
+
readonly truePositives: number
|
|
30
|
+
readonly falsePositives: number
|
|
31
|
+
readonly falseNegatives: number
|
|
32
|
+
readonly precision: number
|
|
33
|
+
readonly recall: number
|
|
34
|
+
readonly duplicateCount: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type BenchmarkResult = {
|
|
38
|
+
readonly schemaVersion: typeof BENCHMARK_SCHEMA_VERSION
|
|
39
|
+
readonly quality: {
|
|
40
|
+
readonly entities: BenchmarkSetMetrics
|
|
41
|
+
readonly relations: BenchmarkSetMetrics
|
|
42
|
+
readonly findings: BenchmarkSetMetrics
|
|
43
|
+
}
|
|
44
|
+
readonly evidenceRatio: number
|
|
45
|
+
readonly findingDensity: number
|
|
46
|
+
readonly findingCategoryDistribution: Readonly<Record<string, number>>
|
|
47
|
+
readonly excludedCaseCount: number
|
|
48
|
+
readonly excludedCaseIds: Readonly<{ entities: number; relations: number; findings: number }>
|
|
49
|
+
readonly thresholds: { readonly precision: number; readonly recall: number }
|
|
50
|
+
readonly regressions: readonly string[]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const unique = (values: readonly string[]): Set<string> => new Set(values)
|
|
54
|
+
const ratio = (numerator: number, denominator: number): number => denominator === 0 ? 1 : numerator / denominator
|
|
55
|
+
|
|
56
|
+
const setMetrics = (
|
|
57
|
+
actualValues: readonly string[],
|
|
58
|
+
expectedValues: readonly string[],
|
|
59
|
+
excludedValues: readonly string[],
|
|
60
|
+
): BenchmarkSetMetrics => {
|
|
61
|
+
const excluded = unique(excludedValues)
|
|
62
|
+
const activeActualValues = actualValues.filter((value) => !excluded.has(value))
|
|
63
|
+
const actual = unique(activeActualValues)
|
|
64
|
+
const expected = unique(expectedValues.filter((value) => !excluded.has(value)))
|
|
65
|
+
const truePositives = [...actual].filter((value) => expected.has(value)).length
|
|
66
|
+
const falsePositives = actual.size - truePositives
|
|
67
|
+
const falseNegatives = expected.size - truePositives
|
|
68
|
+
return {
|
|
69
|
+
truePositives,
|
|
70
|
+
falsePositives,
|
|
71
|
+
falseNegatives,
|
|
72
|
+
precision: ratio(truePositives, actual.size),
|
|
73
|
+
recall: ratio(truePositives, expected.size),
|
|
74
|
+
duplicateCount: activeActualValues.length - actual.size,
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export const benchmarkFixture = (fixture: unknown): BenchmarkFixtureV1 => BenchmarkFixtureV1Schema.parse(fixture)
|
|
79
|
+
|
|
80
|
+
export const measureBenchmark = (
|
|
81
|
+
observation: BenchmarkObservation,
|
|
82
|
+
fixture: BenchmarkFixtureV1,
|
|
83
|
+
thresholds: { readonly precision?: number; readonly recall?: number } = {},
|
|
84
|
+
): BenchmarkResult => {
|
|
85
|
+
const excluded = fixture.excluded ?? { entities: [], relations: [], findings: [] }
|
|
86
|
+
const quality = {
|
|
87
|
+
entities: setMetrics(observation.entities, fixture.supported.entities, excluded.entities),
|
|
88
|
+
relations: setMetrics(observation.relations, fixture.supported.relations, excluded.relations),
|
|
89
|
+
findings: setMetrics(observation.findings, fixture.supported.findings, excluded.findings),
|
|
90
|
+
}
|
|
91
|
+
const actualEntities = unique(observation.entities)
|
|
92
|
+
const evidenced = unique(observation.evidenced ?? [])
|
|
93
|
+
const evidenceRatio = ratio([...actualEntities].filter((id) => evidenced.has(id)).length, actualEntities.size)
|
|
94
|
+
const findingDensity = ratio(unique(observation.findings).size, actualEntities.size)
|
|
95
|
+
const precisionThreshold = thresholds.precision ?? 0.95
|
|
96
|
+
const recallThreshold = thresholds.recall ?? 1
|
|
97
|
+
const regressions: string[] = []
|
|
98
|
+
for (const [name, metrics] of Object.entries(quality)) {
|
|
99
|
+
if (metrics.precision < precisionThreshold) regressions.push(`${name}.precision below ${precisionThreshold}`)
|
|
100
|
+
if (metrics.recall < recallThreshold) regressions.push(`${name}.recall below ${recallThreshold}`)
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
schemaVersion: BENCHMARK_SCHEMA_VERSION,
|
|
104
|
+
quality,
|
|
105
|
+
evidenceRatio,
|
|
106
|
+
findingDensity,
|
|
107
|
+
findingCategoryDistribution: Object.fromEntries(Object.entries(observation.findingCategories ?? {}).sort(([a], [b]) => a.localeCompare(b))),
|
|
108
|
+
excludedCaseCount: excluded.entities.length + excluded.relations.length + excluded.findings.length,
|
|
109
|
+
excludedCaseIds: { entities: excluded.entities.length, relations: excluded.relations.length, findings: excluded.findings.length },
|
|
110
|
+
thresholds: { precision: precisionThreshold, recall: recallThreshold },
|
|
111
|
+
regressions,
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export type BenchmarkSnapshot = Readonly<Record<string, string>>
|
|
116
|
+
|
|
117
|
+
export type BenchmarkSnapshotDiff = {
|
|
118
|
+
readonly added: number
|
|
119
|
+
readonly removed: number
|
|
120
|
+
readonly unchanged: number
|
|
121
|
+
readonly reclassified: number
|
|
122
|
+
readonly unchangedEvidence: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export const compareBenchmarkSnapshots = (
|
|
126
|
+
previous: BenchmarkSnapshot,
|
|
127
|
+
current: BenchmarkSnapshot,
|
|
128
|
+
): BenchmarkSnapshotDiff => {
|
|
129
|
+
const ids = new Set([...Object.keys(previous), ...Object.keys(current)])
|
|
130
|
+
let added = 0
|
|
131
|
+
let removed = 0
|
|
132
|
+
let unchanged = 0
|
|
133
|
+
let reclassified = 0
|
|
134
|
+
let unchangedEvidence = 0
|
|
135
|
+
for (const id of ids) {
|
|
136
|
+
if (!(id in previous)) { added += 1; continue }
|
|
137
|
+
if (!(id in current)) { removed += 1; continue }
|
|
138
|
+
if (previous[id] === current[id]) unchanged += 1
|
|
139
|
+
else reclassified += 1
|
|
140
|
+
if (previous[id] === current[id]) unchangedEvidence += 1
|
|
141
|
+
}
|
|
142
|
+
return { added, removed, unchanged, reclassified, unchangedEvidence }
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type AgentEfficiencyObservation = {
|
|
146
|
+
readonly hits: number
|
|
147
|
+
readonly queries: number
|
|
148
|
+
readonly latencyMs: readonly number[]
|
|
149
|
+
readonly responseBytes: readonly number[]
|
|
150
|
+
readonly estimatedTokens: readonly number[]
|
|
151
|
+
readonly corpusBytes: number
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const percentile95 = (values: readonly number[]): number => {
|
|
155
|
+
if (!values.length) return 0
|
|
156
|
+
const sorted = [...values].sort((a, b) => a - b)
|
|
157
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export const measureAgentEfficiency = (observation: AgentEfficiencyObservation) => ({
|
|
161
|
+
hitRate: ratio(observation.hits, observation.queries),
|
|
162
|
+
latencyP95Ms: percentile95(observation.latencyMs),
|
|
163
|
+
responseBytesP95: percentile95(observation.responseBytes),
|
|
164
|
+
estimatedTokensP95: percentile95(observation.estimatedTokens),
|
|
165
|
+
corpusBytes: observation.corpusBytes,
|
|
166
|
+
contextReduction: observation.corpusBytes > 0 ? 1 - (percentile95(observation.responseBytes) / observation.corpusBytes) : 0,
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
export const formatBenchmarkText = (result: BenchmarkResult): string => [
|
|
170
|
+
`Precision: entities ${result.quality.entities.precision.toFixed(3)}, relations ${result.quality.relations.precision.toFixed(3)}, findings ${result.quality.findings.precision.toFixed(3)}`,
|
|
171
|
+
`Recall: entities ${result.quality.entities.recall.toFixed(3)}, relations ${result.quality.relations.recall.toFixed(3)}, findings ${result.quality.findings.recall.toFixed(3)}`,
|
|
172
|
+
`Evidence ratio: ${result.evidenceRatio.toFixed(3)}`,
|
|
173
|
+
`Finding density: ${result.findingDensity.toFixed(3)}`,
|
|
174
|
+
`Excluded cases: ${result.excludedCaseCount}`,
|
|
175
|
+
`Regressions: ${result.regressions.length ? result.regressions.join('; ') : 'none'}`,
|
|
176
|
+
].join('\n')
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
import { CoverageSchema, DiagnosticSchema, EntitySchema, RelationSchema, type Coverage, type KnowledgeDiagnostic, type KnowledgeEntity, type KnowledgeRelation } from '../schemas/knowledge.js'
|
|
4
|
+
|
|
5
|
+
export const ANALYZER_PLUGIN_CONTRACT_VERSION = 1 as const
|
|
6
|
+
|
|
7
|
+
export const AnalyzerPluginManifestSchema = z.object({
|
|
8
|
+
id: z.string().regex(/^[a-z][a-z0-9-]*$/).max(128),
|
|
9
|
+
version: z.string().min(1).max(64),
|
|
10
|
+
languages: z.array(z.string().min(1).max(64)).min(1).max(32),
|
|
11
|
+
frameworks: z.array(z.string().min(1).max(128)).max(32).default([]),
|
|
12
|
+
capabilities: z.array(z.string().min(1).max(128)).min(1).max(32),
|
|
13
|
+
knowledgeSchemaVersion: z.literal(1),
|
|
14
|
+
compatibility: z.object({ pipelineMajor: z.number().int().nonnegative() }).strict(),
|
|
15
|
+
unsupportedConstructs: z.array(z.string().min(1).max(256)).max(128).default([]),
|
|
16
|
+
resourceLimits: z.object({ maxFiles: z.number().int().positive().optional(), maxBytes: z.number().int().positive().optional() }).strict().default({}),
|
|
17
|
+
}).strict()
|
|
18
|
+
|
|
19
|
+
export type AnalyzerPluginManifest = z.infer<typeof AnalyzerPluginManifestSchema>
|
|
20
|
+
|
|
21
|
+
export const AnalyzerPluginOutputSchema = z.object({
|
|
22
|
+
entities: z.array(EntitySchema).max(50_000).default([]),
|
|
23
|
+
relations: z.array(RelationSchema).max(100_000).default([]),
|
|
24
|
+
coverage: z.array(CoverageSchema).max(1_000).default([]),
|
|
25
|
+
diagnostics: z.array(DiagnosticSchema).max(100_000).default([]),
|
|
26
|
+
}).strict()
|
|
27
|
+
|
|
28
|
+
export type AnalyzerPluginOutput = z.infer<typeof AnalyzerPluginOutputSchema>
|
|
29
|
+
|
|
30
|
+
export type AnalyzerPluginInput = {
|
|
31
|
+
readonly language: string
|
|
32
|
+
readonly framework?: string
|
|
33
|
+
readonly files: readonly { readonly path: string; readonly bytes: number }[]
|
|
34
|
+
readonly value?: unknown
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type AnalyzerPlugin = {
|
|
38
|
+
readonly manifest: AnalyzerPluginManifest
|
|
39
|
+
readonly analyze: (input: AnalyzerPluginInput) => Promise<unknown> | unknown
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type AnalyzerRegistry = {
|
|
43
|
+
readonly register: (plugin: AnalyzerPlugin) => void
|
|
44
|
+
readonly list: () => readonly AnalyzerPluginManifest[]
|
|
45
|
+
readonly analyze: (id: string, input: AnalyzerPluginInput) => Promise<AnalyzerPluginOutput>
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const pipelineMajor = (version: string): number => Number.parseInt(version.split('.')[0] ?? '', 10)
|
|
49
|
+
|
|
50
|
+
export const createAnalyzerRegistry = (options: { readonly pipelineVersion?: string; readonly maxPlugins?: number } = {}): AnalyzerRegistry => {
|
|
51
|
+
const plugins = new Map<string, AnalyzerPlugin>()
|
|
52
|
+
const pipeline = pipelineMajor(options.pipelineVersion ?? '1.0.0')
|
|
53
|
+
return {
|
|
54
|
+
register(plugin) {
|
|
55
|
+
const manifest = AnalyzerPluginManifestSchema.parse(plugin.manifest)
|
|
56
|
+
if (manifest.compatibility.pipelineMajor !== pipeline) throw new Error(`Analyzer plugin "${manifest.id}" requires pipeline major ${manifest.compatibility.pipelineMajor}; current pipeline is ${pipeline}.`)
|
|
57
|
+
if (plugins.has(manifest.id)) throw new Error(`Analyzer plugin "${manifest.id}" is already registered.`)
|
|
58
|
+
if (options.maxPlugins !== undefined && plugins.size >= options.maxPlugins) throw new Error(`Analyzer plugin limit ${options.maxPlugins} exceeded.`)
|
|
59
|
+
plugins.set(manifest.id, { ...plugin, manifest })
|
|
60
|
+
},
|
|
61
|
+
list() {
|
|
62
|
+
return [...plugins.values()].map((plugin) => plugin.manifest).sort((a, b) => a.id.localeCompare(b.id))
|
|
63
|
+
},
|
|
64
|
+
async analyze(id, input) {
|
|
65
|
+
const plugin = plugins.get(id)
|
|
66
|
+
if (!plugin) throw new Error(`Analyzer plugin "${id}" is not registered.`)
|
|
67
|
+
const { maxFiles, maxBytes } = plugin.manifest.resourceLimits
|
|
68
|
+
const bytes = input.files.reduce((total, file) => total + file.bytes, 0)
|
|
69
|
+
if (maxFiles !== undefined && input.files.length > maxFiles) throw new Error(`Analyzer plugin "${id}" file limit ${maxFiles} exceeded.`)
|
|
70
|
+
if (maxBytes !== undefined && bytes > maxBytes) throw new Error(`Analyzer plugin "${id}" byte limit ${maxBytes} exceeded.`)
|
|
71
|
+
try {
|
|
72
|
+
const output = AnalyzerPluginOutputSchema.parse(await plugin.analyze(input))
|
|
73
|
+
return {
|
|
74
|
+
...output,
|
|
75
|
+
coverage: output.coverage.map((entry) => ({ ...entry, analyzer: plugin.manifest.id, analyzerVersion: plugin.manifest.version })),
|
|
76
|
+
}
|
|
77
|
+
} catch (error) {
|
|
78
|
+
return {
|
|
79
|
+
entities: [],
|
|
80
|
+
relations: [],
|
|
81
|
+
diagnostics: [],
|
|
82
|
+
coverage: [{ analyzer: plugin.manifest.id, analyzerVersion: plugin.manifest.version, scope: 'plugin', status: 'not-analyzed', reason: `Plugin failed safely: ${error instanceof Error ? error.message : String(error)}` }],
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export type { Coverage, KnowledgeDiagnostic, KnowledgeEntity, KnowledgeRelation }
|