@agentskit/doc-bridge 1.6.3 → 1.7.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +249 -0
- package/action.yml +1 -1
- package/dist/cli/program.js +793 -137
- package/dist/cli/program.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +38 -2
- package/dist/config/index.js.map +1 -1
- package/dist/{index-DudNuwI5.d.ts → index-C2PCQSrB.d.ts} +216 -25
- package/dist/index.d.ts +837 -67
- package/dist/index.js +858 -127
- package/dist/index.js.map +1 -1
- package/docs/PRD-enterprise-hardening.md +288 -0
- package/docs/adr/0001-enterprise-verification-contract.md +35 -0
- package/docs/knowledge-engine-runbook.md +18 -2
- package/docs/spec/analyzer-plugin-v1.md +24 -0
- package/docs/spec/benchmark-v1.md +30 -0
- package/docs/spec/config-v1.md +111 -0
- package/docs/validation-cycle-plan.md +236 -0
- package/docs/verification-harness.md +33 -4
- package/mcpb/manifest.json +1 -1
- package/package.json +1 -1
- package/scripts/report-visual-check.mjs +63 -17
- package/scripts/verification-harness.mjs +216 -13
- package/skills/doc-bridge-handoff/scripts/resolve-handoff.mjs +1 -1
- package/src/agents/registry-adapter.ts +31 -7
- package/src/cli/program.ts +44 -11
- package/src/config/index.ts +2 -0
- package/src/config/schema.ts +56 -0
- package/src/discovery/documentation.ts +46 -5
- package/src/discovery/repository.ts +95 -16
- package/src/index.ts +29 -0
- package/src/metrics/benchmark.ts +176 -0
- package/src/plugins/contract.ts +89 -0
- package/src/reconciliation/reconcile.ts +137 -3
- package/src/report/html.ts +302 -78
- package/src/schemas/knowledge.ts +16 -1
- package/src/version.ts +1 -1
- package/src/workflow/engine.ts +65 -9
package/src/config/schema.ts
CHANGED
|
@@ -181,8 +181,48 @@ export const RulesConfigSchema = z
|
|
|
181
181
|
|
|
182
182
|
export const ReconciliationConfigSchema = z
|
|
183
183
|
.object({
|
|
184
|
+
/** Semantic comparison level. Raw discovery always keeps file-level relations. */
|
|
185
|
+
scope: z.enum(['file', 'module', 'package']).optional(),
|
|
184
186
|
/** Relation kinds that must have documentation declarations. Omit to require all observed kinds; [] disables this signal. */
|
|
185
187
|
requiredRelationKinds: z.array(z.string().min(1).max(128)).max(128).optional(),
|
|
188
|
+
includeOrphanedDocuments: z.boolean().optional(),
|
|
189
|
+
})
|
|
190
|
+
.strict()
|
|
191
|
+
|
|
192
|
+
export const AnalysisConfigSchema = z
|
|
193
|
+
.object({
|
|
194
|
+
plugins: z
|
|
195
|
+
.array(
|
|
196
|
+
z
|
|
197
|
+
.object({
|
|
198
|
+
id: z.string().regex(/^[a-z][a-z0-9-]*$/).max(128),
|
|
199
|
+
enabled: z.boolean().optional(),
|
|
200
|
+
order: z.number().int().nonnegative().optional(),
|
|
201
|
+
options: z.record(z.string(), z.unknown()).optional(),
|
|
202
|
+
reason: z.string().min(1).max(1_024).optional(),
|
|
203
|
+
})
|
|
204
|
+
.strict(),
|
|
205
|
+
)
|
|
206
|
+
.max(128)
|
|
207
|
+
.optional(),
|
|
208
|
+
jsTs: z
|
|
209
|
+
.object({
|
|
210
|
+
runtimeWiringMethods: z.array(z.string().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/).max(64)).max(64).optional(),
|
|
211
|
+
runtimeWiringAdapters: z
|
|
212
|
+
.array(
|
|
213
|
+
z
|
|
214
|
+
.object({
|
|
215
|
+
id: z.string().min(1).max(128),
|
|
216
|
+
methods: z.array(z.string().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/).max(64)).min(1).max(64),
|
|
217
|
+
})
|
|
218
|
+
.strict(),
|
|
219
|
+
)
|
|
220
|
+
.max(32)
|
|
221
|
+
.optional(),
|
|
222
|
+
includeTestRuntimeWiring: z.boolean().optional(),
|
|
223
|
+
})
|
|
224
|
+
.strict()
|
|
225
|
+
.optional(),
|
|
186
226
|
})
|
|
187
227
|
.strict()
|
|
188
228
|
|
|
@@ -203,6 +243,13 @@ export const RepositorySafetyConfigSchema = z
|
|
|
203
243
|
})
|
|
204
244
|
.strict()
|
|
205
245
|
|
|
246
|
+
export const ReportConfigSchema = z
|
|
247
|
+
.object({
|
|
248
|
+
/** Public report privacy mode. Private is the default; anonymized preserves topology without project identity. */
|
|
249
|
+
privacy: z.enum(['private', 'anonymized']).optional(),
|
|
250
|
+
})
|
|
251
|
+
.strict()
|
|
252
|
+
|
|
206
253
|
export const SurfacesConfigSchema = z
|
|
207
254
|
.object({
|
|
208
255
|
cli: z
|
|
@@ -310,6 +357,11 @@ export const IntelligenceConfigSchema = z
|
|
|
310
357
|
agentId: z.string().min(1).max(256).optional(),
|
|
311
358
|
agentRoot: z.string().min(1).max(512).optional(),
|
|
312
359
|
runnerModule: z.string().min(1).max(512).optional(),
|
|
360
|
+
deterministic: z.boolean().optional(),
|
|
361
|
+
timeoutMs: z.number().int().positive().max(600_000).optional(),
|
|
362
|
+
maxTokens: z.number().int().positive().max(1_000_000).optional(),
|
|
363
|
+
maxResponseBytes: z.number().int().positive().max(10_000_000).optional(),
|
|
364
|
+
maxConcurrency: z.number().int().positive().max(64).optional(),
|
|
313
365
|
})
|
|
314
366
|
.strict()
|
|
315
367
|
.optional(),
|
|
@@ -442,9 +494,11 @@ export const DocBridgeConfigV1Schema = z
|
|
|
442
494
|
routing: RoutingConfigSchema.optional(),
|
|
443
495
|
gates: GatesConfigSchema.optional(),
|
|
444
496
|
reconciliation: ReconciliationConfigSchema.optional(),
|
|
497
|
+
analysis: AnalysisConfigSchema.optional(),
|
|
445
498
|
rules: RulesConfigSchema.optional(),
|
|
446
499
|
workflow: WorkflowConfigSchema.optional(),
|
|
447
500
|
safety: RepositorySafetyConfigSchema.optional(),
|
|
501
|
+
report: ReportConfigSchema.optional(),
|
|
448
502
|
surfaces: SurfacesConfigSchema.optional(),
|
|
449
503
|
intelligence: IntelligenceConfigSchema.optional(),
|
|
450
504
|
federation: FederationConfigSchema.optional(),
|
|
@@ -458,8 +512,10 @@ export type HumanCorpusConfig = z.infer<typeof HumanCorpusConfigSchema>
|
|
|
458
512
|
export type DocumentationStandardV1Config = z.infer<typeof DocumentationStandardV1ConfigSchema>
|
|
459
513
|
export type DocumentationStandardRuleId = z.infer<typeof DocumentationStandardRuleIdSchema>
|
|
460
514
|
export type ReconciliationConfig = z.infer<typeof ReconciliationConfigSchema>
|
|
515
|
+
export type AnalysisConfig = z.infer<typeof AnalysisConfigSchema>
|
|
461
516
|
export type RuleId = z.infer<typeof RuleIdSchema>
|
|
462
517
|
export type RuleSeverity = z.infer<typeof RuleSeveritySchema>
|
|
463
518
|
export type RulesConfig = z.infer<typeof RulesConfigSchema>
|
|
464
519
|
export type WorkflowConfig = z.infer<typeof WorkflowConfigSchema>
|
|
465
520
|
export type RepositorySafetyConfig = z.infer<typeof RepositorySafetyConfigSchema>
|
|
521
|
+
export type ReportConfig = z.infer<typeof ReportConfigSchema>
|
|
@@ -22,6 +22,8 @@ export type DocumentationDeclarationInput = {
|
|
|
22
22
|
export type DocumentationDeclarationOptions = {
|
|
23
23
|
readonly snapshot: Pick<DiscoverySnapshotV1, 'entities'>
|
|
24
24
|
readonly documentId?: string
|
|
25
|
+
/** Agent corpus root used for conservative package/app path inference. */
|
|
26
|
+
readonly agentRoot?: string
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
export type DocumentationDeclarationResult = {
|
|
@@ -71,6 +73,15 @@ const scalar = (value: string): string => {
|
|
|
71
73
|
return trimmed
|
|
72
74
|
}
|
|
73
75
|
|
|
76
|
+
const conventionalPackageReference = (path: string, agentRoot: string): string | undefined => {
|
|
77
|
+
const prefix = `${agentRoot.replace(/\/$/, '')}/`
|
|
78
|
+
if (!path.startsWith(prefix)) return undefined
|
|
79
|
+
const relative = path.slice(prefix.length)
|
|
80
|
+
const [scope, file] = relative.split('/')
|
|
81
|
+
if ((scope !== 'packages' && scope !== 'apps') || !file) return undefined
|
|
82
|
+
return file.replace(/\.mdx?$/, '')
|
|
83
|
+
}
|
|
84
|
+
|
|
74
85
|
const list = (value: string): string[] | undefined => {
|
|
75
86
|
const trimmed = value.trim()
|
|
76
87
|
if (!trimmed.startsWith('[') || !trimmed.endsWith(']')) return undefined
|
|
@@ -103,7 +114,15 @@ const resolveEntity = (
|
|
|
103
114
|
lineStart: number,
|
|
104
115
|
unresolved: Map<string, KnowledgeEntity>,
|
|
105
116
|
): KnowledgeEntity => {
|
|
106
|
-
const
|
|
117
|
+
const direct = entities.find((entity) => entity.id === reference || entity.aliases?.includes(reference))
|
|
118
|
+
if (direct) return direct
|
|
119
|
+
const packageReference = reference.replace(/^package:/, '')
|
|
120
|
+
const packageCandidates = entities.filter((entity) => {
|
|
121
|
+
if (entity.kind !== 'package') return false
|
|
122
|
+
const pathName = entity.path?.split('/').pop()
|
|
123
|
+
return entity.name === reference || entity.path === reference || pathName === packageReference || entity.name.endsWith(`/${packageReference}`)
|
|
124
|
+
})
|
|
125
|
+
const resolved = packageCandidates.length === 1 ? packageCandidates[0] : undefined
|
|
107
126
|
if (resolved) return resolved
|
|
108
127
|
const id = `unresolved:${reference}`
|
|
109
128
|
const existing = unresolved.get(id)
|
|
@@ -123,9 +142,14 @@ const relationKey = (from: string, to: string, kind: string): string => `${from}
|
|
|
123
142
|
|
|
124
143
|
const parseBlock = (
|
|
125
144
|
input: DocumentationDeclarationInput,
|
|
145
|
+
options: Pick<DocumentationDeclarationOptions, 'agentRoot'> = {},
|
|
126
146
|
): { readonly covers: readonly { value: string; line: number }[]; readonly relations: readonly RelationFields[]; readonly diagnostics: readonly DocumentationDiagnostic[]; readonly hasDocbridge: boolean } => {
|
|
147
|
+
const conventionalPath = conventionalPackageReference(input.path, options.agentRoot ?? 'docs/for-agents')
|
|
127
148
|
const frontmatter = findFrontmatter(input.content)
|
|
128
149
|
if (!frontmatter) {
|
|
150
|
+
if (conventionalPath && !input.content.replace(/^\uFEFF/, '').startsWith('---')) {
|
|
151
|
+
return { covers: [{ value: conventionalPath, line: 1 }], relations: [], diagnostics: [], hasDocbridge: true }
|
|
152
|
+
}
|
|
129
153
|
if (input.content.replace(/^\uFEFF/, '').startsWith('---')) {
|
|
130
154
|
return {
|
|
131
155
|
covers: [],
|
|
@@ -138,8 +162,24 @@ const parseBlock = (
|
|
|
138
162
|
}
|
|
139
163
|
|
|
140
164
|
const { lines, end } = frontmatter
|
|
141
|
-
|
|
142
|
-
|
|
165
|
+
let docbridgeLine = -1
|
|
166
|
+
let typeLine = -1
|
|
167
|
+
let packageLine = -1
|
|
168
|
+
let humanDocLine = -1
|
|
169
|
+
for (let index = 1; index < end; index += 1) {
|
|
170
|
+
const line = lines[index] ?? ''
|
|
171
|
+
if (docbridgeLine < 0 && /^docbridge\s*:/.test(line)) docbridgeLine = index
|
|
172
|
+
if (typeLine < 0 && /^type\s*:/.test(line)) typeLine = index
|
|
173
|
+
if (packageLine < 0 && /^package\s*:/.test(line)) packageLine = index
|
|
174
|
+
if (humanDocLine < 0 && /^humanDoc\s*:/.test(line)) humanDocLine = index
|
|
175
|
+
}
|
|
176
|
+
if (docbridgeLine < 0) {
|
|
177
|
+
const type = typeLine >= 0 ? scalar(lines[typeLine]?.slice('type:'.length) ?? '') : ''
|
|
178
|
+
const packageReference = packageLine >= 0 ? scalar(lines[packageLine]?.slice('package:'.length) ?? '') : conventionalPath ?? ''
|
|
179
|
+
if (type === 'package' && packageReference) return { covers: [{ value: packageReference, line: packageLine >= 0 ? packageLine + 1 : typeLine + 1 }], relations: [], diagnostics: [], hasDocbridge: true }
|
|
180
|
+
if (conventionalPath) return { covers: [{ value: packageReference, line: humanDocLine >= 0 ? humanDocLine + 1 : 1 }], relations: [], diagnostics: [], hasDocbridge: true }
|
|
181
|
+
return { covers: [], relations: [], diagnostics: [], hasDocbridge: false }
|
|
182
|
+
}
|
|
143
183
|
|
|
144
184
|
const diagnostics: DocumentationDiagnostic[] = []
|
|
145
185
|
const covers: { value: string; line: number }[] = []
|
|
@@ -231,7 +271,7 @@ export const parseDocumentationDeclarations = (
|
|
|
231
271
|
input: DocumentationDeclarationInput,
|
|
232
272
|
options: DocumentationDeclarationOptions,
|
|
233
273
|
): DocumentationDeclarationResult => {
|
|
234
|
-
const parsed = parseBlock(input)
|
|
274
|
+
const parsed = parseBlock(input, options)
|
|
235
275
|
if (!parsed.hasDocbridge) return { hasDocbridge: false, entities: [], relations: [], diagnostics: [] }
|
|
236
276
|
const diagnostics = [...parsed.diagnostics]
|
|
237
277
|
const unresolved = new Map<string, KnowledgeEntity>()
|
|
@@ -300,13 +340,14 @@ export const parseDocumentationDeclarations = (
|
|
|
300
340
|
export const applyDocumentationDeclarations = (
|
|
301
341
|
snapshot: DiscoverySnapshotV1,
|
|
302
342
|
documents: readonly DocumentationDeclarationInput[],
|
|
343
|
+
options: Pick<DocumentationDeclarationOptions, 'agentRoot'> = {},
|
|
303
344
|
): DocumentationAnalysisResult => {
|
|
304
345
|
const entities = new Map(snapshot.entities.map((entity) => [entity.id, entity]))
|
|
305
346
|
const relations = new Map(snapshot.relations.map((relation) => [relation.id, relation]))
|
|
306
347
|
const diagnostics: DocumentationDiagnostic[] = []
|
|
307
348
|
|
|
308
349
|
for (const document of documents) {
|
|
309
|
-
const result = parseDocumentationDeclarations(document, { snapshot })
|
|
350
|
+
const result = parseDocumentationDeclarations(document, { snapshot, ...options })
|
|
310
351
|
diagnostics.push(...result.diagnostics)
|
|
311
352
|
for (const entity of result.entities) entities.set(entity.id, entity)
|
|
312
353
|
for (const relation of result.relations) relations.set(relation.id, relation)
|
|
@@ -21,6 +21,8 @@ const SOURCE_EXTENSIONS = ['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts',
|
|
|
21
21
|
const DOCUMENT_EXTENSIONS = ['.md', '.mdx'] as const
|
|
22
22
|
const DEFAULT_MAX_FILES = 10_000
|
|
23
23
|
const EMPTY_HASH = '0'.repeat(64)
|
|
24
|
+
const DEFAULT_RUNTIME_WIRING_METHODS = ['register', 'use', 'mount', 'attach'] as const
|
|
25
|
+
const TEST_MODULE_PATTERN = /(?:\.test|\.spec|__tests__)/
|
|
24
26
|
|
|
25
27
|
type JsonRecord = Record<string, unknown>
|
|
26
28
|
|
|
@@ -42,8 +44,9 @@ type ModuleInfo = {
|
|
|
42
44
|
|
|
43
45
|
type ImportReference = {
|
|
44
46
|
readonly specifier: string
|
|
45
|
-
readonly kind: 'imports' | 're-exports'
|
|
47
|
+
readonly kind: 'imports' | 're-exports' | 'runtime-wiring'
|
|
46
48
|
readonly evidence: Evidence
|
|
49
|
+
readonly detection?: 'dynamic-literal' | 'runtime-wiring-static'
|
|
47
50
|
}
|
|
48
51
|
|
|
49
52
|
type DiscoveryOptions = {
|
|
@@ -97,6 +100,13 @@ const firstLineContaining = (text: string, pattern: string): number | undefined
|
|
|
97
100
|
return line >= 0 ? line + 1 : undefined
|
|
98
101
|
}
|
|
99
102
|
|
|
103
|
+
const documentClassification = (path: string): string => {
|
|
104
|
+
if (/(^|\/)docs\/for-agents(?:\/|$)/.test(path)) return 'agent'
|
|
105
|
+
if (/(^|\/)docs(?:\/|$)/.test(path)) return 'human'
|
|
106
|
+
if (/(^|\/)(README|CONTRIBUTING|SECURITY|CHANGELOG)(?:\.|$)/i.test(path)) return 'project'
|
|
107
|
+
return 'unclassified'
|
|
108
|
+
}
|
|
109
|
+
|
|
100
110
|
const packageName = (manifest: JsonRecord, fallback: string): string | undefined =>
|
|
101
111
|
typeof manifest.name === 'string' && manifest.name.length > 0 ? manifest.name : fallback || undefined
|
|
102
112
|
|
|
@@ -262,30 +272,69 @@ const moduleReferences = (
|
|
|
262
272
|
root: string,
|
|
263
273
|
path: string,
|
|
264
274
|
sourceFile: ts.SourceFile,
|
|
265
|
-
|
|
275
|
+
runtimeWiringMethods: ReadonlySet<string>,
|
|
276
|
+
): { readonly references: readonly ImportReference[]; readonly exports: readonly string[]; readonly hasDynamic: boolean; readonly hasLiteralDynamic: boolean; readonly hasRuntimeWiring: boolean; readonly hasUnresolvedRuntimeWiring: boolean } => {
|
|
266
277
|
const references: ImportReference[] = []
|
|
267
278
|
let hasDynamic = false
|
|
279
|
+
let hasLiteralDynamic = false
|
|
268
280
|
let hasRuntimeWiring = false
|
|
269
|
-
|
|
270
|
-
|
|
281
|
+
let hasUnresolvedRuntimeWiring = false
|
|
282
|
+
const importedBindings = new Map<string, string>()
|
|
283
|
+
const staticStringBindings = new Map<string, string | undefined>()
|
|
284
|
+
const collectStaticStringBindings = (node: ts.Node): void => {
|
|
285
|
+
if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer && ts.isStringLiteralLike(node.initializer) && ts.isVariableDeclarationList(node.parent) && (node.parent.flags & ts.NodeFlags.Const) !== 0) {
|
|
286
|
+
const previous = staticStringBindings.get(node.name.text)
|
|
287
|
+
staticStringBindings.set(node.name.text, !staticStringBindings.has(node.name.text) || previous === node.initializer.text ? node.initializer.text : undefined)
|
|
288
|
+
}
|
|
289
|
+
ts.forEachChild(node, collectStaticStringBindings)
|
|
290
|
+
}
|
|
291
|
+
collectStaticStringBindings(sourceFile)
|
|
292
|
+
const addReference = (specifier: ts.StringLiteralLike, kind: ImportReference['kind'], node: ts.Node, detection?: ImportReference['detection']): void => {
|
|
293
|
+
references.push({ specifier: specifier.text, kind, evidence: nodeEvidence(root, path, sourceFile, node), ...(detection ? { detection } : {}) })
|
|
271
294
|
}
|
|
272
295
|
|
|
273
296
|
const visit = (node: ts.Node): void => {
|
|
274
297
|
if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) {
|
|
275
298
|
addReference(node.moduleSpecifier, 'imports', node)
|
|
299
|
+
const clause = node.importClause
|
|
300
|
+
if (clause?.name) importedBindings.set(clause.name.text, node.moduleSpecifier.text)
|
|
301
|
+
if (clause?.namedBindings && ts.isNamespaceImport(clause.namedBindings)) importedBindings.set(clause.namedBindings.name.text, node.moduleSpecifier.text)
|
|
302
|
+
if (clause?.namedBindings && ts.isNamedImports(clause.namedBindings)) {
|
|
303
|
+
for (const element of clause.namedBindings.elements) importedBindings.set((element.name ?? element.propertyName)?.text ?? '', node.moduleSpecifier.text)
|
|
304
|
+
}
|
|
276
305
|
} else if (ts.isExportDeclaration(node) && node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) {
|
|
277
306
|
addReference(node.moduleSpecifier, 're-exports', node)
|
|
278
307
|
} else if (ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference) && ts.isStringLiteral(node.moduleReference.expression)) {
|
|
279
308
|
addReference(node.moduleReference.expression, 'imports', node)
|
|
309
|
+
importedBindings.set(node.name.text, node.moduleReference.expression.text)
|
|
280
310
|
} else if (ts.isCallExpression(node)) {
|
|
281
311
|
if (node.expression.kind === ts.SyntaxKind.ImportKeyword) {
|
|
282
|
-
if (
|
|
312
|
+
if (node.arguments[0] && ts.isStringLiteralLike(node.arguments[0])) {
|
|
313
|
+
hasLiteralDynamic = true
|
|
314
|
+
addReference(node.arguments[0], 'imports', node, 'dynamic-literal')
|
|
315
|
+
} else if (node.arguments[0] && ts.isIdentifier(node.arguments[0]) && staticStringBindings.get(node.arguments[0].text)) {
|
|
316
|
+
hasLiteralDynamic = true
|
|
317
|
+
addReference({ text: staticStringBindings.get(node.arguments[0].text)! } as ts.StringLiteralLike, 'imports', node, 'dynamic-literal')
|
|
318
|
+
} else hasDynamic = true
|
|
283
319
|
} else if (ts.isIdentifier(node.expression) && node.expression.text === 'require') {
|
|
284
320
|
const argument = node.arguments[0]
|
|
285
321
|
if (argument && ts.isStringLiteralLike(argument)) addReference(argument, 'imports', node)
|
|
322
|
+
else if (argument && ts.isIdentifier(argument) && staticStringBindings.get(argument.text)) addReference({ text: staticStringBindings.get(argument.text)! } as ts.StringLiteralLike, 'imports', node)
|
|
286
323
|
else hasDynamic = true
|
|
287
|
-
} else if (ts.isPropertyAccessExpression(node.expression) && node.expression.name.text
|
|
324
|
+
} else if (ts.isPropertyAccessExpression(node.expression) && runtimeWiringMethods.has(node.expression.name.text)) {
|
|
288
325
|
hasRuntimeWiring = true
|
|
326
|
+
const before = references.length
|
|
327
|
+
let hasPotentialTargetArgument = false
|
|
328
|
+
for (const argument of node.arguments) {
|
|
329
|
+
if (ts.isIdentifier(argument)) {
|
|
330
|
+
hasPotentialTargetArgument = true
|
|
331
|
+
const specifier = importedBindings.get(argument.text)
|
|
332
|
+
if (specifier) addReference({ text: specifier } as ts.StringLiteralLike, 'runtime-wiring', node, 'runtime-wiring-static')
|
|
333
|
+
} else if (ts.isPropertyAccessExpression(argument) || ts.isCallExpression(argument)) {
|
|
334
|
+
hasPotentialTargetArgument = true
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (hasPotentialTargetArgument && references.length === before) hasUnresolvedRuntimeWiring = true
|
|
289
338
|
}
|
|
290
339
|
}
|
|
291
340
|
ts.forEachChild(node, visit)
|
|
@@ -295,7 +344,9 @@ const moduleReferences = (
|
|
|
295
344
|
references,
|
|
296
345
|
exports: exportedNames(sourceFile),
|
|
297
346
|
hasDynamic,
|
|
347
|
+
hasLiteralDynamic,
|
|
298
348
|
hasRuntimeWiring,
|
|
349
|
+
hasUnresolvedRuntimeWiring,
|
|
299
350
|
}
|
|
300
351
|
}
|
|
301
352
|
|
|
@@ -395,10 +446,10 @@ const artifact = (root: string, config: DocBridgeConfigV1 | undefined, files: re
|
|
|
395
446
|
sourceRevisionKind: revision.kind,
|
|
396
447
|
configurationHash: sha256NormalizedV1(config ?? {}),
|
|
397
448
|
pipelineVersion: '1.0.0',
|
|
398
|
-
analyzerVersions: { repository: '1.0.0', 'js-ts': '1.
|
|
449
|
+
analyzerVersions: { repository: '1.0.0', 'js-ts': '1.3.0' },
|
|
399
450
|
entities: [...entities].sort((a, b) => a.id.localeCompare(b.id)),
|
|
400
451
|
relations: [...relations].sort((a, b) => a.id.localeCompare(b.id)),
|
|
401
|
-
coverage: [
|
|
452
|
+
coverage: coverage.map((entry) => ({ ...entry, analyzerVersion: entry.analyzerVersion ?? ({ repository: '1.0.0', 'js-ts': '1.3.0' }[entry.analyzer] ?? '1.0.0') })),
|
|
402
453
|
}
|
|
403
454
|
return DiscoverySnapshotV1Schema.parse({ ...base, contentHash: contentHashForArtifactV1(base) })
|
|
404
455
|
}
|
|
@@ -463,13 +514,13 @@ export const discoverRepository = (opts: DiscoveryOptions = {}): DiscoverySnapsh
|
|
|
463
514
|
const sourceFile = ts.createSourceFile(absPath, text, ts.ScriptTarget.Latest, true, scriptKind(absPath))
|
|
464
515
|
const exports = exportedNames(sourceFile)
|
|
465
516
|
modules.set(resolve(absPath), { absPath, path, entityId: id, ...(pkg ? { packageId: pkg.id } : {}) })
|
|
466
|
-
addEntity({ id, kind: 'module', name: basename(absPath), path, provenance: 'observed', evidence: [lineEvidence('code', root, absPath, 1, sourceFile.getLineAndCharacterOfPosition(sourceFile.getEnd()).line + 1)], ...(exports.length ? { metadata: { exports, test:
|
|
517
|
+
addEntity({ id, kind: 'module', name: basename(absPath), path, provenance: 'observed', evidence: [lineEvidence('code', root, absPath, 1, sourceFile.getLineAndCharacterOfPosition(sourceFile.getEnd()).line + 1)], ...(exports.length ? { metadata: { exports, test: TEST_MODULE_PATTERN.test(path) } } : {}) })
|
|
467
518
|
if (pkg) addRelation({ id: entityId('relation', `${pkg.id}:contains:${id}`), kind: 'contains', from: pkg.id, to: id, provenance: 'observed', evidence: [lineEvidence('code', root, absPath, 1)] })
|
|
468
519
|
}
|
|
469
520
|
|
|
470
521
|
for (const absPath of documentPaths) {
|
|
471
522
|
const path = relativePath(root, absPath)
|
|
472
|
-
addEntity({ id: entityId('document', path), kind: 'document', name: basename(absPath), path, provenance: 'observed', evidence: [lineEvidence('documentation', root, absPath, 1)] })
|
|
523
|
+
addEntity({ id: entityId('document', path), kind: 'document', name: basename(absPath), path, provenance: 'observed', evidence: [lineEvidence('documentation', root, absPath, 1)], metadata: { classification: documentClassification(path) } })
|
|
473
524
|
}
|
|
474
525
|
|
|
475
526
|
const compiler = readCompilerOptions(root)
|
|
@@ -478,8 +529,8 @@ export const discoverRepository = (opts: DiscoveryOptions = {}): DiscoverySnapsh
|
|
|
478
529
|
{ analyzer: 'repository', scope: 'package-manager', status: hasPackageManagerMetadata(root, rootManifest) ? 'complete' : 'partial', ...(!hasPackageManagerMetadata(root, rootManifest) ? { reason: `No package manager metadata found; default helper would fall back to ${detectPackageManager(root)}.` } : {}) },
|
|
479
530
|
{ analyzer: 'repository', scope: 'workspace-packages', status: packageResult.coverage.some((item) => item.status === 'partial') ? 'partial' : 'complete', ...(packageResult.coverage.find((item) => item.reason)?.reason ? { reason: packageResult.coverage.find((item) => item.reason)?.reason } : {}) },
|
|
480
531
|
{ analyzer: 'js-ts', scope: 'static-imports-and-exports', status: compiler.error ? 'partial' : 'complete', ...(compiler.error ? { reason: compiler.error } : {}) },
|
|
481
|
-
{ analyzer: 'js-ts', scope: 'dynamic-imports', status: 'not-
|
|
482
|
-
{ analyzer: 'js-ts', scope: 'runtime-wiring', status: 'not-
|
|
532
|
+
{ analyzer: 'js-ts', scope: 'dynamic-imports', status: 'not-applicable', reason: 'No dynamic loading expression was observed.' },
|
|
533
|
+
{ analyzer: 'js-ts', scope: 'runtime-wiring', status: 'not-applicable', reason: 'No configured runtime-wiring call was observed.' },
|
|
483
534
|
{ analyzer: 'js-ts', scope: 'generated-code', status: 'not-analyzed', reason: 'Generated code is not interpreted as source architecture.' },
|
|
484
535
|
]
|
|
485
536
|
|
|
@@ -492,10 +543,26 @@ export const discoverRepository = (opts: DiscoveryOptions = {}): DiscoverySnapsh
|
|
|
492
543
|
}
|
|
493
544
|
}
|
|
494
545
|
|
|
546
|
+
const dynamicCoverageIndex = coverage.findIndex((entry) => entry.scope === 'dynamic-imports')
|
|
547
|
+
const configuredRuntimeWiringMethods = new Set([
|
|
548
|
+
...(opts.config?.analysis?.jsTs?.runtimeWiringMethods ?? []),
|
|
549
|
+
...(opts.config?.analysis?.jsTs?.runtimeWiringAdapters?.flatMap((adapter) => adapter.methods) ?? []),
|
|
550
|
+
])
|
|
551
|
+
if (!configuredRuntimeWiringMethods.size) for (const method of DEFAULT_RUNTIME_WIRING_METHODS) configuredRuntimeWiringMethods.add(method)
|
|
552
|
+
const includeTestRuntimeWiring = opts.config?.analysis?.jsTs?.includeTestRuntimeWiring ?? false
|
|
553
|
+
let observedLiteralDynamic = false
|
|
554
|
+
let observedUnresolvedDynamic = false
|
|
555
|
+
let observedRuntimeWiring = false
|
|
556
|
+
let observedUnresolvedRuntimeWiring = false
|
|
495
557
|
for (const module of modules.values()) {
|
|
496
558
|
const text = readFileSync(module.absPath, 'utf8')
|
|
497
559
|
const sourceFile = ts.createSourceFile(module.absPath, text, ts.ScriptTarget.Latest, true, scriptKind(module.absPath))
|
|
498
|
-
const
|
|
560
|
+
const runtimeWiringMethods = includeTestRuntimeWiring || !TEST_MODULE_PATTERN.test(module.path) ? configuredRuntimeWiringMethods : new Set<string>()
|
|
561
|
+
const references = moduleReferences(root, module.absPath, sourceFile, runtimeWiringMethods)
|
|
562
|
+
observedLiteralDynamic ||= references.hasLiteralDynamic
|
|
563
|
+
observedUnresolvedDynamic ||= references.hasDynamic
|
|
564
|
+
observedRuntimeWiring ||= references.hasRuntimeWiring
|
|
565
|
+
observedUnresolvedRuntimeWiring ||= references.hasUnresolvedRuntimeWiring
|
|
499
566
|
for (const reference of references.references) {
|
|
500
567
|
const target = resolveReference(reference, module.absPath, modules, packageResult.packages, compiler.options)
|
|
501
568
|
if (!target) continue
|
|
@@ -503,12 +570,24 @@ export const discoverRepository = (opts: DiscoveryOptions = {}): DiscoverySnapsh
|
|
|
503
570
|
const externalName = target.targetId.replace(/^external:/, '')
|
|
504
571
|
addEntity({ id: target.targetId, kind: 'external', name: externalName, provenance: 'observed', evidence: [reference.evidence] })
|
|
505
572
|
}
|
|
506
|
-
addRelation({ id: entityId('relation', `${module.entityId}:${reference.kind}:${target.targetId}`), kind: reference.kind, from: module.entityId, to: target.targetId, provenance: 'observed', evidence: [reference.evidence] })
|
|
573
|
+
addRelation({ id: entityId('relation', `${module.entityId}:${reference.kind}:${target.targetId}`), kind: reference.kind, from: module.entityId, to: target.targetId, provenance: 'observed', evidence: [reference.evidence], ...(reference.detection ? { metadata: { detection: reference.detection } } : {}) })
|
|
507
574
|
}
|
|
508
|
-
if (references.hasDynamic) coverage.push({ analyzer: 'js-ts', scope: `dynamic-imports:${module.path}`, status: 'not-analyzed', reason: 'A dynamic import
|
|
509
|
-
if (references.
|
|
575
|
+
if (references.hasLiteralDynamic || references.hasDynamic) coverage.push({ analyzer: 'js-ts', scope: `dynamic-imports:${module.path}`, status: references.hasDynamic ? 'not-analyzed' : 'complete', reason: references.hasDynamic ? 'A non-literal dynamic import was found; the target is unresolved.' : 'Literal dynamic imports were resolved.', evidence: [lineEvidence('code', root, module.absPath)] })
|
|
576
|
+
if (references.hasUnresolvedRuntimeWiring) coverage.push({ analyzer: 'js-ts', scope: `runtime-wiring:${module.path}`, status: 'not-analyzed', reason: 'A runtime registration/wiring call was found without a statically imported target.', evidence: [lineEvidence('code', root, module.absPath)] })
|
|
510
577
|
}
|
|
511
578
|
|
|
579
|
+
if (dynamicCoverageIndex >= 0) coverage[dynamicCoverageIndex] = observedUnresolvedDynamic
|
|
580
|
+
? { analyzer: 'js-ts', scope: 'dynamic-imports', status: 'partial', reason: 'Literal dynamic imports are resolved; non-literal import expressions and require calls remain unresolved.' }
|
|
581
|
+
: observedLiteralDynamic
|
|
582
|
+
? { analyzer: 'js-ts', scope: 'dynamic-imports', status: 'complete', reason: 'All observed dynamic imports used literal targets and were resolved.' }
|
|
583
|
+
: { analyzer: 'js-ts', scope: 'dynamic-imports', status: 'not-applicable', reason: 'No dynamic loading expression was observed.' }
|
|
584
|
+
const runtimeCoverageIndex = coverage.findIndex((entry) => entry.scope === 'runtime-wiring')
|
|
585
|
+
if (runtimeCoverageIndex >= 0) coverage[runtimeCoverageIndex] = observedUnresolvedRuntimeWiring
|
|
586
|
+
? { analyzer: 'js-ts', scope: 'runtime-wiring', status: 'partial', reason: 'Some configured runtime-wiring calls remain unresolved after static binding analysis.' }
|
|
587
|
+
: observedRuntimeWiring
|
|
588
|
+
? { analyzer: 'js-ts', scope: 'runtime-wiring', status: 'complete', reason: 'All observed configured runtime-wiring calls resolved to static bindings.' }
|
|
589
|
+
: { analyzer: 'js-ts', scope: 'runtime-wiring', status: 'not-applicable', reason: 'No configured runtime-wiring call was observed.' }
|
|
590
|
+
|
|
512
591
|
return artifact(root, opts.config, allFiles, [...entities.values()], [...relations.values()], coverage)
|
|
513
592
|
}
|
|
514
593
|
|
package/src/index.ts
CHANGED
|
@@ -20,11 +20,13 @@ export {
|
|
|
20
20
|
RulesConfigSchema,
|
|
21
21
|
WorkflowConfigSchema,
|
|
22
22
|
RepositorySafetyConfigSchema,
|
|
23
|
+
ReportConfigSchema,
|
|
23
24
|
type RuleId,
|
|
24
25
|
type RuleSeverity,
|
|
25
26
|
type RulesConfig,
|
|
26
27
|
type WorkflowConfig,
|
|
27
28
|
type RepositorySafetyConfig,
|
|
29
|
+
type ReportConfig,
|
|
28
30
|
} from './config/schema.js'
|
|
29
31
|
|
|
30
32
|
export {
|
|
@@ -180,6 +182,33 @@ export {
|
|
|
180
182
|
type GithubPrResult,
|
|
181
183
|
} from './memory/github-pr.js'
|
|
182
184
|
export { canonicalJsonV1, contentHashForArtifactV1, sha256NormalizedV1 } from './index-builder/content-hash.js'
|
|
185
|
+
export {
|
|
186
|
+
ANALYZER_PLUGIN_CONTRACT_VERSION,
|
|
187
|
+
AnalyzerPluginManifestSchema,
|
|
188
|
+
AnalyzerPluginOutputSchema,
|
|
189
|
+
createAnalyzerRegistry,
|
|
190
|
+
type AnalyzerPlugin,
|
|
191
|
+
type AnalyzerPluginInput,
|
|
192
|
+
type AnalyzerPluginManifest,
|
|
193
|
+
type AnalyzerPluginOutput,
|
|
194
|
+
type AnalyzerRegistry,
|
|
195
|
+
} from './plugins/contract.js'
|
|
196
|
+
export {
|
|
197
|
+
BENCHMARK_SCHEMA_VERSION,
|
|
198
|
+
BenchmarkFixtureV1Schema,
|
|
199
|
+
benchmarkFixture,
|
|
200
|
+
compareBenchmarkSnapshots,
|
|
201
|
+
formatBenchmarkText,
|
|
202
|
+
measureAgentEfficiency,
|
|
203
|
+
measureBenchmark,
|
|
204
|
+
type AgentEfficiencyObservation,
|
|
205
|
+
type BenchmarkFixtureV1,
|
|
206
|
+
type BenchmarkObservation,
|
|
207
|
+
type BenchmarkResult,
|
|
208
|
+
type BenchmarkSetMetrics,
|
|
209
|
+
type BenchmarkSnapshot,
|
|
210
|
+
type BenchmarkSnapshotDiff,
|
|
211
|
+
} from './metrics/benchmark.js'
|
|
183
212
|
export {
|
|
184
213
|
AgentProposalV1Schema,
|
|
185
214
|
AffectedFileSchema,
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
export const BENCHMARK_SCHEMA_VERSION = 1 as const
|
|
4
|
+
|
|
5
|
+
const stringList = z.array(z.string().min(1).max(512)).max(100_000)
|
|
6
|
+
const benchmarkSets = z.object({
|
|
7
|
+
entities: stringList.default([]),
|
|
8
|
+
relations: stringList.default([]),
|
|
9
|
+
findings: stringList.default([]),
|
|
10
|
+
}).strict()
|
|
11
|
+
|
|
12
|
+
export const BenchmarkFixtureV1Schema = z.object({
|
|
13
|
+
schemaVersion: z.literal(BENCHMARK_SCHEMA_VERSION),
|
|
14
|
+
supported: benchmarkSets,
|
|
15
|
+
excluded: benchmarkSets.optional(),
|
|
16
|
+
}).strict()
|
|
17
|
+
|
|
18
|
+
export type BenchmarkFixtureV1 = z.infer<typeof BenchmarkFixtureV1Schema>
|
|
19
|
+
|
|
20
|
+
export type BenchmarkObservation = {
|
|
21
|
+
readonly entities: readonly string[]
|
|
22
|
+
readonly relations: readonly string[]
|
|
23
|
+
readonly findings: readonly string[]
|
|
24
|
+
readonly evidenced?: readonly string[]
|
|
25
|
+
readonly findingCategories?: Readonly<Record<string, number>>
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type BenchmarkSetMetrics = {
|
|
29
|
+
readonly truePositives: number
|
|
30
|
+
readonly falsePositives: number
|
|
31
|
+
readonly falseNegatives: number
|
|
32
|
+
readonly precision: number
|
|
33
|
+
readonly recall: number
|
|
34
|
+
readonly duplicateCount: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type BenchmarkResult = {
|
|
38
|
+
readonly schemaVersion: typeof BENCHMARK_SCHEMA_VERSION
|
|
39
|
+
readonly quality: {
|
|
40
|
+
readonly entities: BenchmarkSetMetrics
|
|
41
|
+
readonly relations: BenchmarkSetMetrics
|
|
42
|
+
readonly findings: BenchmarkSetMetrics
|
|
43
|
+
}
|
|
44
|
+
readonly evidenceRatio: number
|
|
45
|
+
readonly findingDensity: number
|
|
46
|
+
readonly findingCategoryDistribution: Readonly<Record<string, number>>
|
|
47
|
+
readonly excludedCaseCount: number
|
|
48
|
+
readonly excludedCaseIds: Readonly<{ entities: number; relations: number; findings: number }>
|
|
49
|
+
readonly thresholds: { readonly precision: number; readonly recall: number }
|
|
50
|
+
readonly regressions: readonly string[]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const unique = (values: readonly string[]): Set<string> => new Set(values)
|
|
54
|
+
const ratio = (numerator: number, denominator: number): number => denominator === 0 ? 1 : numerator / denominator
|
|
55
|
+
|
|
56
|
+
const setMetrics = (
|
|
57
|
+
actualValues: readonly string[],
|
|
58
|
+
expectedValues: readonly string[],
|
|
59
|
+
excludedValues: readonly string[],
|
|
60
|
+
): BenchmarkSetMetrics => {
|
|
61
|
+
const excluded = unique(excludedValues)
|
|
62
|
+
const activeActualValues = actualValues.filter((value) => !excluded.has(value))
|
|
63
|
+
const actual = unique(activeActualValues)
|
|
64
|
+
const expected = unique(expectedValues.filter((value) => !excluded.has(value)))
|
|
65
|
+
const truePositives = [...actual].filter((value) => expected.has(value)).length
|
|
66
|
+
const falsePositives = actual.size - truePositives
|
|
67
|
+
const falseNegatives = expected.size - truePositives
|
|
68
|
+
return {
|
|
69
|
+
truePositives,
|
|
70
|
+
falsePositives,
|
|
71
|
+
falseNegatives,
|
|
72
|
+
precision: ratio(truePositives, actual.size),
|
|
73
|
+
recall: ratio(truePositives, expected.size),
|
|
74
|
+
duplicateCount: activeActualValues.length - actual.size,
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export const benchmarkFixture = (fixture: unknown): BenchmarkFixtureV1 => BenchmarkFixtureV1Schema.parse(fixture)
|
|
79
|
+
|
|
80
|
+
export const measureBenchmark = (
|
|
81
|
+
observation: BenchmarkObservation,
|
|
82
|
+
fixture: BenchmarkFixtureV1,
|
|
83
|
+
thresholds: { readonly precision?: number; readonly recall?: number } = {},
|
|
84
|
+
): BenchmarkResult => {
|
|
85
|
+
const excluded = fixture.excluded ?? { entities: [], relations: [], findings: [] }
|
|
86
|
+
const quality = {
|
|
87
|
+
entities: setMetrics(observation.entities, fixture.supported.entities, excluded.entities),
|
|
88
|
+
relations: setMetrics(observation.relations, fixture.supported.relations, excluded.relations),
|
|
89
|
+
findings: setMetrics(observation.findings, fixture.supported.findings, excluded.findings),
|
|
90
|
+
}
|
|
91
|
+
const actualEntities = unique(observation.entities)
|
|
92
|
+
const evidenced = unique(observation.evidenced ?? [])
|
|
93
|
+
const evidenceRatio = ratio([...actualEntities].filter((id) => evidenced.has(id)).length, actualEntities.size)
|
|
94
|
+
const findingDensity = ratio(unique(observation.findings).size, actualEntities.size)
|
|
95
|
+
const precisionThreshold = thresholds.precision ?? 0.95
|
|
96
|
+
const recallThreshold = thresholds.recall ?? 1
|
|
97
|
+
const regressions: string[] = []
|
|
98
|
+
for (const [name, metrics] of Object.entries(quality)) {
|
|
99
|
+
if (metrics.precision < precisionThreshold) regressions.push(`${name}.precision below ${precisionThreshold}`)
|
|
100
|
+
if (metrics.recall < recallThreshold) regressions.push(`${name}.recall below ${recallThreshold}`)
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
schemaVersion: BENCHMARK_SCHEMA_VERSION,
|
|
104
|
+
quality,
|
|
105
|
+
evidenceRatio,
|
|
106
|
+
findingDensity,
|
|
107
|
+
findingCategoryDistribution: Object.fromEntries(Object.entries(observation.findingCategories ?? {}).sort(([a], [b]) => a.localeCompare(b))),
|
|
108
|
+
excludedCaseCount: excluded.entities.length + excluded.relations.length + excluded.findings.length,
|
|
109
|
+
excludedCaseIds: { entities: excluded.entities.length, relations: excluded.relations.length, findings: excluded.findings.length },
|
|
110
|
+
thresholds: { precision: precisionThreshold, recall: recallThreshold },
|
|
111
|
+
regressions,
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export type BenchmarkSnapshot = Readonly<Record<string, string>>
|
|
116
|
+
|
|
117
|
+
export type BenchmarkSnapshotDiff = {
|
|
118
|
+
readonly added: number
|
|
119
|
+
readonly removed: number
|
|
120
|
+
readonly unchanged: number
|
|
121
|
+
readonly reclassified: number
|
|
122
|
+
readonly unchangedEvidence: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export const compareBenchmarkSnapshots = (
|
|
126
|
+
previous: BenchmarkSnapshot,
|
|
127
|
+
current: BenchmarkSnapshot,
|
|
128
|
+
): BenchmarkSnapshotDiff => {
|
|
129
|
+
const ids = new Set([...Object.keys(previous), ...Object.keys(current)])
|
|
130
|
+
let added = 0
|
|
131
|
+
let removed = 0
|
|
132
|
+
let unchanged = 0
|
|
133
|
+
let reclassified = 0
|
|
134
|
+
let unchangedEvidence = 0
|
|
135
|
+
for (const id of ids) {
|
|
136
|
+
if (!(id in previous)) { added += 1; continue }
|
|
137
|
+
if (!(id in current)) { removed += 1; continue }
|
|
138
|
+
if (previous[id] === current[id]) unchanged += 1
|
|
139
|
+
else reclassified += 1
|
|
140
|
+
if (previous[id] === current[id]) unchangedEvidence += 1
|
|
141
|
+
}
|
|
142
|
+
return { added, removed, unchanged, reclassified, unchangedEvidence }
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type AgentEfficiencyObservation = {
|
|
146
|
+
readonly hits: number
|
|
147
|
+
readonly queries: number
|
|
148
|
+
readonly latencyMs: readonly number[]
|
|
149
|
+
readonly responseBytes: readonly number[]
|
|
150
|
+
readonly estimatedTokens: readonly number[]
|
|
151
|
+
readonly corpusBytes: number
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const percentile95 = (values: readonly number[]): number => {
|
|
155
|
+
if (!values.length) return 0
|
|
156
|
+
const sorted = [...values].sort((a, b) => a - b)
|
|
157
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export const measureAgentEfficiency = (observation: AgentEfficiencyObservation) => ({
|
|
161
|
+
hitRate: ratio(observation.hits, observation.queries),
|
|
162
|
+
latencyP95Ms: percentile95(observation.latencyMs),
|
|
163
|
+
responseBytesP95: percentile95(observation.responseBytes),
|
|
164
|
+
estimatedTokensP95: percentile95(observation.estimatedTokens),
|
|
165
|
+
corpusBytes: observation.corpusBytes,
|
|
166
|
+
contextReduction: observation.corpusBytes > 0 ? 1 - (percentile95(observation.responseBytes) / observation.corpusBytes) : 0,
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
export const formatBenchmarkText = (result: BenchmarkResult): string => [
|
|
170
|
+
`Precision: entities ${result.quality.entities.precision.toFixed(3)}, relations ${result.quality.relations.precision.toFixed(3)}, findings ${result.quality.findings.precision.toFixed(3)}`,
|
|
171
|
+
`Recall: entities ${result.quality.entities.recall.toFixed(3)}, relations ${result.quality.relations.recall.toFixed(3)}, findings ${result.quality.findings.recall.toFixed(3)}`,
|
|
172
|
+
`Evidence ratio: ${result.evidenceRatio.toFixed(3)}`,
|
|
173
|
+
`Finding density: ${result.findingDensity.toFixed(3)}`,
|
|
174
|
+
`Excluded cases: ${result.excludedCaseCount}`,
|
|
175
|
+
`Regressions: ${result.regressions.length ? result.regressions.join('; ') : 'none'}`,
|
|
176
|
+
].join('\n')
|