agentic-scorecard 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_PROMPT.md +68 -34
- package/FEEDBACK.md +53 -0
- package/README.md +120 -39
- package/benchmark/v0.2/adapter-contract.md +25 -0
- package/benchmark/v0.2/agent-evidence-schema.json +52 -0
- package/benchmark/v0.2/attestation-schema.json +26 -0
- package/benchmark/v0.2/benchmark.yaml +127 -0
- package/benchmark/v0.2/controls/context.yaml +51 -0
- package/benchmark/v0.2/controls/environment.yaml +70 -0
- package/benchmark/v0.2/controls/governance.yaml +60 -0
- package/benchmark/v0.2/controls/learning.yaml +48 -0
- package/benchmark/v0.2/controls/observability.yaml +76 -0
- package/benchmark/v0.2/controls/resilience.yaml +60 -0
- package/benchmark/v0.2/controls/security.yaml +71 -0
- package/benchmark/v0.2/controls/specification.yaml +62 -0
- package/benchmark/v0.2/controls/testing.yaml +60 -0
- package/benchmark/v0.2/controls/tooling.yaml +58 -0
- package/benchmark/v0.2/report-schema.json +47 -0
- package/benchmark/v0.2/scoring-policy.md +71 -0
- package/benchmark/v0.3/adapter-contract.md +44 -0
- package/benchmark/v0.3/adapters/known-agent-harnesses.yaml +177 -0
- package/benchmark/v0.3/adapters/known-python-tooling.yaml +18 -0
- package/benchmark/v0.3/adapters/known-security-tools.yaml +6 -0
- package/benchmark/v0.3/agent-evidence-schema.json +67 -0
- package/benchmark/v0.3/attestation-schema.json +26 -0
- package/benchmark/v0.3/benchmark.yaml +127 -0
- package/benchmark/v0.3/controls/context.yaml +79 -0
- package/benchmark/v0.3/controls/environment.yaml +93 -0
- package/benchmark/v0.3/controls/governance.yaml +97 -0
- package/benchmark/v0.3/controls/learning.yaml +80 -0
- package/benchmark/v0.3/controls/observability.yaml +91 -0
- package/benchmark/v0.3/controls/resilience.yaml +107 -0
- package/benchmark/v0.3/controls/security.yaml +129 -0
- package/benchmark/v0.3/controls/specification.yaml +83 -0
- package/benchmark/v0.3/controls/testing.yaml +85 -0
- package/benchmark/v0.3/controls/tooling.yaml +69 -0
- package/benchmark/v0.3/report-schema.json +89 -0
- package/benchmark/v0.3/scoring-policy.md +100 -0
- package/dist/cli.js +1043 -111
- package/package.json +5 -3
- package/templates/agent-evidence.yaml +11 -0
- package/templates/attestations.yaml +4 -4
- package/templates/baseline.csv +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/cli.ts
|
|
4
|
-
import { appendFile, mkdir, readFile as
|
|
5
|
-
import { dirname as dirname2, join as
|
|
4
|
+
import { appendFile, mkdir, readFile as readFile4, writeFile } from "fs/promises";
|
|
5
|
+
import { dirname as dirname2, join as join3, resolve as resolve4 } from "path";
|
|
6
6
|
import { Command } from "commander";
|
|
7
7
|
import { stringify } from "yaml";
|
|
8
8
|
|
|
@@ -29,31 +29,52 @@ var dimensionIds = [
|
|
|
29
29
|
];
|
|
30
30
|
var DimensionIdSchema = z.enum(dimensionIds);
|
|
31
31
|
var LevelSchema = z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3), z.literal(4)]);
|
|
32
|
+
var EvidenceScopeSchema = z.enum(["repository", "platform", "organization", "outcome"]);
|
|
33
|
+
var ManualEvidenceScopeSchema = z.enum(["platform", "organization", "outcome"]);
|
|
34
|
+
var AssessmentScopeSchema = z.enum(["tracked", "workspace"]);
|
|
32
35
|
var PathAnySchema = z.object({
|
|
33
36
|
type: z.literal("path_any"),
|
|
34
|
-
|
|
37
|
+
scope: z.literal("repository").default("repository"),
|
|
38
|
+
patterns: z.array(z.string().min(1)).min(1),
|
|
39
|
+
min_bytes: z.number().int().positive().default(1)
|
|
35
40
|
});
|
|
36
41
|
var PathAllSchema = z.object({
|
|
37
42
|
type: z.literal("path_all"),
|
|
38
|
-
|
|
43
|
+
scope: z.literal("repository").default("repository"),
|
|
44
|
+
patterns: z.array(z.string().min(1)).min(1),
|
|
45
|
+
min_bytes: z.number().int().positive().default(1)
|
|
39
46
|
});
|
|
40
47
|
var ContentAnySchema = z.object({
|
|
41
48
|
type: z.literal("content_any"),
|
|
49
|
+
scope: z.literal("repository").default("repository"),
|
|
42
50
|
files: z.array(z.string().min(1)).min(1),
|
|
43
51
|
needles: z.array(z.string().min(1)).min(1)
|
|
44
52
|
});
|
|
45
53
|
var ContentAllSchema = z.object({
|
|
46
54
|
type: z.literal("content_all"),
|
|
55
|
+
scope: z.literal("repository").default("repository"),
|
|
47
56
|
files: z.array(z.string().min(1)).min(1),
|
|
48
57
|
needles: z.array(z.string().min(1)).min(1)
|
|
49
58
|
});
|
|
59
|
+
var ContentTermsSchema = z.object({
|
|
60
|
+
type: z.literal("content_terms"),
|
|
61
|
+
scope: z.literal("repository").default("repository"),
|
|
62
|
+
files: z.array(z.string().min(1)).min(1),
|
|
63
|
+
terms: z.array(z.string().min(1)).min(1),
|
|
64
|
+
min_terms: z.number().int().positive(),
|
|
65
|
+
required_any_terms: z.array(z.string().min(1)).min(1).optional(),
|
|
66
|
+
max_span_lines: z.number().int().positive().max(200).optional(),
|
|
67
|
+
max_files_per_pattern: z.number().int().positive().max(250).optional()
|
|
68
|
+
});
|
|
50
69
|
var MaxBytesSchema = z.object({
|
|
51
70
|
type: z.literal("max_bytes"),
|
|
71
|
+
scope: z.literal("repository").default("repository"),
|
|
52
72
|
patterns: z.array(z.string().min(1)).min(1),
|
|
53
73
|
max_bytes: z.number().int().positive()
|
|
54
74
|
});
|
|
55
75
|
var ManualSchema = z.object({
|
|
56
76
|
type: z.literal("manual"),
|
|
77
|
+
scope: ManualEvidenceScopeSchema.default("organization"),
|
|
57
78
|
prompt: z.string().min(1)
|
|
58
79
|
});
|
|
59
80
|
var EvidenceCheckSchema = z.discriminatedUnion("type", [
|
|
@@ -61,6 +82,7 @@ var EvidenceCheckSchema = z.discriminatedUnion("type", [
|
|
|
61
82
|
PathAllSchema,
|
|
62
83
|
ContentAnySchema,
|
|
63
84
|
ContentAllSchema,
|
|
85
|
+
ContentTermsSchema,
|
|
64
86
|
MaxBytesSchema,
|
|
65
87
|
ManualSchema
|
|
66
88
|
]);
|
|
@@ -73,13 +95,48 @@ var RawControlSchema = z.object({
|
|
|
73
95
|
evidence: z.array(EvidenceCheckSchema).min(1),
|
|
74
96
|
remediation: z.string().min(1),
|
|
75
97
|
references: z.array(z.string().min(1)).default([]),
|
|
76
|
-
allow_attestation: z.boolean().default(
|
|
77
|
-
allow_not_applicable: z.boolean().default(false)
|
|
98
|
+
allow_attestation: z.boolean().default(false),
|
|
99
|
+
allow_not_applicable: z.boolean().default(false),
|
|
100
|
+
allow_agent_evidence: z.boolean().default(false),
|
|
101
|
+
agent_evidence_scopes: z.array(EvidenceScopeSchema).default([])
|
|
102
|
+
});
|
|
103
|
+
var LegacyRawControlSchema = RawControlSchema.extend({
|
|
104
|
+
allow_attestation: z.boolean().default(true)
|
|
78
105
|
});
|
|
79
106
|
var ControlFileSchema = z.object({
|
|
80
107
|
dimension: DimensionIdSchema,
|
|
81
108
|
controls: z.array(RawControlSchema).min(1)
|
|
82
109
|
});
|
|
110
|
+
var LegacyControlFileSchema = z.object({
|
|
111
|
+
dimension: DimensionIdSchema,
|
|
112
|
+
controls: z.array(LegacyRawControlSchema).min(1)
|
|
113
|
+
});
|
|
114
|
+
var DetectorAdapterExtensionSchema = z.object({
|
|
115
|
+
control_id: z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/),
|
|
116
|
+
evidence_index: z.number().int().nonnegative(),
|
|
117
|
+
patterns: z.array(z.string().min(1)).min(1).optional(),
|
|
118
|
+
files: z.array(z.string().min(1)).min(1).optional(),
|
|
119
|
+
terms: z.array(z.string().min(1)).min(1).optional(),
|
|
120
|
+
required_any_terms: z.array(z.string().min(1)).min(1).optional()
|
|
121
|
+
}).strict().superRefine((extension, context) => {
|
|
122
|
+
const extensionKinds = [
|
|
123
|
+
extension.patterns,
|
|
124
|
+
extension.files,
|
|
125
|
+
extension.terms,
|
|
126
|
+
extension.required_any_terms
|
|
127
|
+
].filter(Boolean).length;
|
|
128
|
+
if (extensionKinds !== 1) {
|
|
129
|
+
context.addIssue({
|
|
130
|
+
code: z.ZodIssueCode.custom,
|
|
131
|
+
message: "A detector extension must declare exactly one of patterns, files, terms, or required_any_terms"
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
var DetectorAdapterSchema = z.object({
|
|
136
|
+
id: z.string().regex(/^[a-z0-9-]+$/),
|
|
137
|
+
benchmark_version: z.string().regex(/^\d+\.\d+\.\d+$/),
|
|
138
|
+
extensions: z.array(DetectorAdapterExtensionSchema).min(1)
|
|
139
|
+
}).strict();
|
|
83
140
|
var DimensionSchema = z.object({
|
|
84
141
|
id: DimensionIdSchema,
|
|
85
142
|
title: z.string().min(1),
|
|
@@ -116,16 +173,90 @@ var AttestationSchema = z.object({
|
|
|
116
173
|
evidence: z.string().min(1),
|
|
117
174
|
owner: z.string().min(1),
|
|
118
175
|
reviewed_at: z.string().date(),
|
|
176
|
+
expires_at: z.string().date()
|
|
177
|
+
});
|
|
178
|
+
var LegacyAttestationSchema = AttestationSchema.extend({
|
|
119
179
|
expires_at: z.string().date().nullable().default(null)
|
|
120
180
|
});
|
|
121
181
|
var AttestationFileSchema = z.object({
|
|
122
182
|
benchmark_version: z.string().min(1),
|
|
123
183
|
attestations: z.record(z.string(), AttestationSchema).default({})
|
|
124
184
|
});
|
|
185
|
+
var LegacyAttestationFileSchema = z.object({
|
|
186
|
+
benchmark_version: z.string().min(1),
|
|
187
|
+
attestations: z.record(z.string(), LegacyAttestationSchema).default({})
|
|
188
|
+
});
|
|
189
|
+
var AgentEvidenceClaimSchema = z.object({
|
|
190
|
+
status: z.enum(["met", "not_met", "unknown"]),
|
|
191
|
+
scope: z.enum(["platform", "organization", "outcome"]),
|
|
192
|
+
summary: z.string().min(1),
|
|
193
|
+
references: z.array(z.string().min(1)).min(1),
|
|
194
|
+
collected_at: z.string().datetime(),
|
|
195
|
+
expires_at: z.string().datetime(),
|
|
196
|
+
error: z.string().min(1).nullable().default(null)
|
|
197
|
+
}).strict().superRefine((claim, context) => {
|
|
198
|
+
if (claim.error && claim.status !== "unknown") {
|
|
199
|
+
context.addIssue({
|
|
200
|
+
code: z.ZodIssueCode.custom,
|
|
201
|
+
message: "A claim with an error must have unknown status",
|
|
202
|
+
path: ["status"]
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
var AgentEvidenceFileSchema = z.object({
|
|
207
|
+
schema_version: z.literal("0.2.0"),
|
|
208
|
+
benchmark_version: z.literal("0.2.0"),
|
|
209
|
+
target: z.object({
|
|
210
|
+
repository: z.string().min(1),
|
|
211
|
+
git_head: z.string().min(1).nullable()
|
|
212
|
+
}).strict(),
|
|
213
|
+
collector: z.object({
|
|
214
|
+
name: z.string().min(1),
|
|
215
|
+
version: z.string().min(1)
|
|
216
|
+
}).strict(),
|
|
217
|
+
claims: z.record(z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/), AgentEvidenceClaimSchema)
|
|
218
|
+
}).strict();
|
|
219
|
+
var AgentEvidenceClaimV03Schema = z.object({
|
|
220
|
+
status: z.enum(["met", "not_met", "unknown"]),
|
|
221
|
+
scope: EvidenceScopeSchema,
|
|
222
|
+
summary: z.string().min(1),
|
|
223
|
+
references: z.array(z.string().min(1)).min(1),
|
|
224
|
+
collected_at: z.string().datetime(),
|
|
225
|
+
expires_at: z.string().datetime(),
|
|
226
|
+
error: z.string().min(1).nullable().default(null)
|
|
227
|
+
}).strict().superRefine((claim, context) => {
|
|
228
|
+
if (claim.error && claim.status !== "unknown") {
|
|
229
|
+
context.addIssue({
|
|
230
|
+
code: z.ZodIssueCode.custom,
|
|
231
|
+
message: "A claim with an error must have unknown status",
|
|
232
|
+
path: ["status"]
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
if (claim.scope === "repository" && claim.references.some((reference) => !reference.startsWith("repo:"))) {
|
|
236
|
+
context.addIssue({
|
|
237
|
+
code: z.ZodIssueCode.custom,
|
|
238
|
+
message: "Repository claims must use repo:<tracked-path>[#Lx-Ly] references",
|
|
239
|
+
path: ["references"]
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
var AgentEvidenceFileV03Schema = z.object({
|
|
244
|
+
schema_version: z.literal("0.3.0"),
|
|
245
|
+
benchmark_version: z.literal("0.3.0"),
|
|
246
|
+
target: z.object({
|
|
247
|
+
repository: z.string().min(1),
|
|
248
|
+
git_head: z.string().min(1)
|
|
249
|
+
}).strict(),
|
|
250
|
+
collector: z.object({
|
|
251
|
+
name: z.string().min(1),
|
|
252
|
+
version: z.string().min(1)
|
|
253
|
+
}).strict(),
|
|
254
|
+
claims: z.record(z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/), AgentEvidenceClaimV03Schema)
|
|
255
|
+
}).strict();
|
|
125
256
|
|
|
126
257
|
// src/load.ts
|
|
127
258
|
var packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
128
|
-
var defaultBenchmarkRoot = join(packageRoot, "benchmark", "v0.
|
|
259
|
+
var defaultBenchmarkRoot = join(packageRoot, "benchmark", "v0.3");
|
|
129
260
|
async function readYaml(path) {
|
|
130
261
|
return parse(await readFile(path, "utf8"));
|
|
131
262
|
}
|
|
@@ -134,15 +265,89 @@ async function loadBenchmark(root = defaultBenchmarkRoot) {
|
|
|
134
265
|
const controlPaths = await fg("controls/*.yaml", { cwd: root, absolute: true, onlyFiles: true });
|
|
135
266
|
const controls = [];
|
|
136
267
|
for (const path of controlPaths.sort()) {
|
|
137
|
-
const file = ControlFileSchema.parse(await readYaml(path));
|
|
268
|
+
const file = benchmark.version === "0.1.0" ? LegacyControlFileSchema.parse(await readYaml(path)) : ControlFileSchema.parse(await readYaml(path));
|
|
138
269
|
controls.push(...file.controls.map((control) => ({ ...control, dimension: file.dimension })));
|
|
139
270
|
}
|
|
271
|
+
const adapterPaths = await fg("adapters/*.yaml", {
|
|
272
|
+
cwd: root,
|
|
273
|
+
absolute: true,
|
|
274
|
+
onlyFiles: true
|
|
275
|
+
});
|
|
276
|
+
for (const path of adapterPaths.sort()) {
|
|
277
|
+
const adapter = DetectorAdapterSchema.parse(await readYaml(path));
|
|
278
|
+
applyDetectorAdapter(benchmark, controls, adapter);
|
|
279
|
+
}
|
|
140
280
|
validateCatalog(benchmark, controls);
|
|
141
281
|
return { benchmark, controls };
|
|
142
282
|
}
|
|
143
|
-
|
|
283
|
+
function applyDetectorAdapter(benchmark, controls, adapter) {
|
|
284
|
+
if (adapter.benchmark_version !== benchmark.version) {
|
|
285
|
+
throw new Error(
|
|
286
|
+
`Detector adapter ${adapter.id} targets ${adapter.benchmark_version}, not ${benchmark.version}`
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
for (const extension of adapter.extensions) {
|
|
290
|
+
const control = controls.find(({ id }) => id === extension.control_id);
|
|
291
|
+
if (!control) {
|
|
292
|
+
throw new Error(`Detector adapter ${adapter.id} references unknown ${extension.control_id}`);
|
|
293
|
+
}
|
|
294
|
+
const check = control.evidence[extension.evidence_index];
|
|
295
|
+
if (!check) {
|
|
296
|
+
throw new Error(
|
|
297
|
+
`Detector adapter ${adapter.id} references missing evidence index ${extension.evidence_index} on ${control.id}`
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
if (extension.patterns) {
|
|
301
|
+
if (!("patterns" in check)) {
|
|
302
|
+
throw new Error(
|
|
303
|
+
`Detector adapter ${adapter.id} cannot add patterns to ${control.id} evidence ${extension.evidence_index}`
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
check.patterns = [.../* @__PURE__ */ new Set([...check.patterns, ...extension.patterns])];
|
|
307
|
+
}
|
|
308
|
+
if (extension.files) {
|
|
309
|
+
if (!("files" in check)) {
|
|
310
|
+
throw new Error(
|
|
311
|
+
`Detector adapter ${adapter.id} cannot add files to ${control.id} evidence ${extension.evidence_index}`
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
check.files = [.../* @__PURE__ */ new Set([...check.files, ...extension.files])];
|
|
315
|
+
}
|
|
316
|
+
if (extension.terms) {
|
|
317
|
+
if (check.type !== "content_terms") {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`Detector adapter ${adapter.id} cannot add terms to ${control.id} evidence ${extension.evidence_index}`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
check.terms = [.../* @__PURE__ */ new Set([...check.terms, ...extension.terms])];
|
|
323
|
+
}
|
|
324
|
+
if (extension.required_any_terms) {
|
|
325
|
+
if (check.type !== "content_terms") {
|
|
326
|
+
throw new Error(
|
|
327
|
+
`Detector adapter ${adapter.id} cannot add required terms to ${control.id} evidence ${extension.evidence_index}`
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
check.required_any_terms = [
|
|
331
|
+
.../* @__PURE__ */ new Set([...check.required_any_terms ?? [], ...extension.required_any_terms])
|
|
332
|
+
];
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
async function loadAttestations(path, benchmarkVersion, options = {}) {
|
|
144
337
|
try {
|
|
145
|
-
const
|
|
338
|
+
const rawFile = await readYaml(path);
|
|
339
|
+
const artifactVersion = versionField(rawFile, "benchmark_version");
|
|
340
|
+
if (artifactVersion && handleVersionMismatch(
|
|
341
|
+
"Attestation",
|
|
342
|
+
path,
|
|
343
|
+
artifactVersion,
|
|
344
|
+
benchmarkVersion,
|
|
345
|
+
"init --force",
|
|
346
|
+
options
|
|
347
|
+
)) {
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
const file = benchmarkVersion === "0.1.0" ? LegacyAttestationFileSchema.parse(rawFile) : AttestationFileSchema.parse(rawFile);
|
|
146
351
|
if (file.benchmark_version !== benchmarkVersion) {
|
|
147
352
|
throw new Error(
|
|
148
353
|
`Attestation benchmark version ${file.benchmark_version} does not match ${benchmarkVersion}`
|
|
@@ -154,11 +359,87 @@ async function loadAttestations(path, benchmarkVersion) {
|
|
|
154
359
|
throw error;
|
|
155
360
|
}
|
|
156
361
|
}
|
|
362
|
+
async function loadAgentEvidence(path, benchmarkVersion, options = {}) {
|
|
363
|
+
try {
|
|
364
|
+
const rawFile = await readYaml(path);
|
|
365
|
+
const artifactVersion = versionField(rawFile, "benchmark_version");
|
|
366
|
+
const schemaVersion = versionField(rawFile, "schema_version");
|
|
367
|
+
const mismatchedVersion = [artifactVersion, schemaVersion].find(
|
|
368
|
+
(version) => version && version !== benchmarkVersion
|
|
369
|
+
);
|
|
370
|
+
if (mismatchedVersion && handleVersionMismatch(
|
|
371
|
+
"Agent evidence",
|
|
372
|
+
path,
|
|
373
|
+
mismatchedVersion,
|
|
374
|
+
benchmarkVersion,
|
|
375
|
+
"init-evidence --force",
|
|
376
|
+
options
|
|
377
|
+
)) {
|
|
378
|
+
return null;
|
|
379
|
+
}
|
|
380
|
+
const file = (() => {
|
|
381
|
+
if (benchmarkVersion === "0.2.0") return AgentEvidenceFileSchema.parse(rawFile);
|
|
382
|
+
if (benchmarkVersion === "0.3.0") return AgentEvidenceFileV03Schema.parse(rawFile);
|
|
383
|
+
throw new Error(`Agent evidence bundles are unsupported for benchmark ${benchmarkVersion}`);
|
|
384
|
+
})();
|
|
385
|
+
if (file.benchmark_version !== benchmarkVersion) {
|
|
386
|
+
throw new Error(
|
|
387
|
+
`Agent evidence benchmark version ${file.benchmark_version} does not match ${benchmarkVersion}`
|
|
388
|
+
);
|
|
389
|
+
}
|
|
390
|
+
return file;
|
|
391
|
+
} catch (error) {
|
|
392
|
+
if (error.code === "ENOENT") return null;
|
|
393
|
+
throw error;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
function versionField(value, field) {
|
|
397
|
+
if (!value || typeof value !== "object" || !(field in value)) return null;
|
|
398
|
+
const version = value[field];
|
|
399
|
+
return typeof version === "string" ? version : null;
|
|
400
|
+
}
|
|
401
|
+
function handleVersionMismatch(label, path, artifactVersion, benchmarkVersion, regenerateCommand, options) {
|
|
402
|
+
if (artifactVersion === benchmarkVersion) return false;
|
|
403
|
+
const mismatch = `${label} file ${path} targets ADRB v${artifactVersion}, not v${benchmarkVersion}`;
|
|
404
|
+
if (!options.ignoreVersionMismatch) {
|
|
405
|
+
throw new Error(
|
|
406
|
+
`${mismatch}. Regenerate it with \`agentic-scorecard ${regenerateCommand}\` or pass a v${benchmarkVersion} file.`
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
options.onWarning?.(
|
|
410
|
+
`Ignored auto-loaded ${label.toLowerCase()} file ${path} because it targets ADRB v${artifactVersion}, not v${benchmarkVersion}. Regenerate it with \`agentic-scorecard ${regenerateCommand}\` before relying on its claims.`
|
|
411
|
+
);
|
|
412
|
+
return true;
|
|
413
|
+
}
|
|
157
414
|
function validateCatalog(benchmark, controls) {
|
|
158
415
|
const ids = /* @__PURE__ */ new Set();
|
|
159
416
|
for (const control of controls) {
|
|
160
417
|
if (ids.has(control.id)) throw new Error(`Duplicate control id: ${control.id}`);
|
|
161
418
|
ids.add(control.id);
|
|
419
|
+
if (["0.2.0", "0.3.0"].includes(benchmark.version) && control.evidence.some(({ type }) => type === "content_any" || type === "content_all")) {
|
|
420
|
+
throw new Error(
|
|
421
|
+
`${control.id} uses a legacy broad content collector in benchmark ${benchmark.version}`
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
const manualScopes = control.evidence.filter(({ type }) => type === "manual").map(({ scope }) => scope);
|
|
425
|
+
const allowedAgentScopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : manualScopes;
|
|
426
|
+
if (control.allow_agent_evidence && allowedAgentScopes.length === 0) {
|
|
427
|
+
throw new Error(`${control.id} allows agent evidence without an eligible evidence scope`);
|
|
428
|
+
}
|
|
429
|
+
if (!control.allow_agent_evidence && control.agent_evidence_scopes.length > 0) {
|
|
430
|
+
throw new Error(`${control.id} declares agent evidence scopes but does not allow it`);
|
|
431
|
+
}
|
|
432
|
+
if (benchmark.version === "0.2.0" && control.agent_evidence_scopes.some((scope) => scope === "repository")) {
|
|
433
|
+
throw new Error(`${control.id} changes immutable v0.2 repository evidence semantics`);
|
|
434
|
+
}
|
|
435
|
+
if (["0.2.0", "0.3.0"].includes(benchmark.version) && control.allow_attestation && !control.evidence.some(({ type }) => type === "manual")) {
|
|
436
|
+
throw new Error(`${control.id} allows attestation for repository-detected evidence`);
|
|
437
|
+
}
|
|
438
|
+
for (const check of control.evidence) {
|
|
439
|
+
if (check.type === "content_terms" && check.min_terms > check.terms.length) {
|
|
440
|
+
throw new Error(`${control.id} requires more content terms than it defines`);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
162
443
|
}
|
|
163
444
|
const benchmarkDimensions = benchmark.dimensions.map(({ id }) => id);
|
|
164
445
|
if (benchmarkDimensions.join(",") !== dimensionIds.join(",")) {
|
|
@@ -181,26 +462,114 @@ var statusIcon = {
|
|
|
181
462
|
unknown: "UNKNOWN",
|
|
182
463
|
not_applicable: "N/A"
|
|
183
464
|
};
|
|
465
|
+
function controlScope(control) {
|
|
466
|
+
return control.evidence.find(({ scope }) => scope !== "repository")?.scope ?? "repository";
|
|
467
|
+
}
|
|
468
|
+
function safeText(value) {
|
|
469
|
+
return value.replace(/\s+/g, " ").replace(/([\\`*_[\]<>|])/g, "\\$1");
|
|
470
|
+
}
|
|
471
|
+
function sentence(value) {
|
|
472
|
+
const safe = safeText(value);
|
|
473
|
+
return /[.!?]$/.test(safe) ? safe : `${safe}.`;
|
|
474
|
+
}
|
|
475
|
+
function references(values) {
|
|
476
|
+
return values.map((value) => `\`${value.replaceAll("`", "'").replace(/\s+/g, " ")}\``).join(", ");
|
|
477
|
+
}
|
|
478
|
+
function evidenceLines(control) {
|
|
479
|
+
const lines = control.evidence.flatMap((evidence) => {
|
|
480
|
+
const evidenceReferences = evidence.references.length > 0 ? ` References: ${references(evidence.references)}.` : "";
|
|
481
|
+
return `- **${evidence.scope}/${evidence.type}:** ${evidence.status} \u2014 ${sentence(evidence.summary)}${evidenceReferences}`;
|
|
482
|
+
});
|
|
483
|
+
if (control.agent_evidence) {
|
|
484
|
+
lines.push(
|
|
485
|
+
`- **Agent-collected:** ${control.agent_evidence.status} \u2014 ${sentence(control.agent_evidence.summary)} References: ${references(control.agent_evidence.references)}.`
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
if (control.attestation) {
|
|
489
|
+
lines.push(
|
|
490
|
+
`- **Human-attested:** ${control.attestation.status} \u2014 ${references([control.attestation.evidence])} (owner: ${safeText(control.attestation.owner)}; reviewed: ${control.attestation.reviewed_at}).`
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
if (control.agent_evidence && control.attestation && control.agent_evidence.status !== "unknown" && control.attestation.status !== "unknown" && control.agent_evidence.status !== control.attestation.status) {
|
|
494
|
+
lines.push(
|
|
495
|
+
"- **Conflict:** agent-collected and human-attested statuses disagree; fail closed."
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
return lines;
|
|
499
|
+
}
|
|
500
|
+
function appendControlDetails(lines, heading, controls) {
|
|
501
|
+
lines.push("", `## ${heading}`, "");
|
|
502
|
+
if (controls.length === 0) {
|
|
503
|
+
lines.push("None.");
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
for (const control of controls) {
|
|
507
|
+
lines.push(
|
|
508
|
+
`### ${statusIcon[control.status]} ${control.id} \u2014 ${control.title}`,
|
|
509
|
+
"",
|
|
510
|
+
`**Risk:** ${control.risk}`,
|
|
511
|
+
"",
|
|
512
|
+
`**Improve:** ${control.remediation}`,
|
|
513
|
+
"",
|
|
514
|
+
`Evidence confidence: ${control.confidence}.`,
|
|
515
|
+
"",
|
|
516
|
+
...evidenceLines(control),
|
|
517
|
+
""
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
184
521
|
function toMarkdown(report) {
|
|
185
522
|
const target = report.profiles.find(({ id }) => id === report.target.profile);
|
|
523
|
+
const targetDependencies = target?.evidence_dependencies;
|
|
524
|
+
const dependencyCount = (targetDependencies?.agent_collected ?? 0) + (targetDependencies?.attested ?? 0);
|
|
525
|
+
const targetProvenance = target?.passed && dependencyCount > 0 ? ` (depends on ${[
|
|
526
|
+
targetDependencies?.agent_collected ? `${targetDependencies.agent_collected} agent-collected` : null,
|
|
527
|
+
targetDependencies?.attested ? `${targetDependencies.attested} human-attested` : null
|
|
528
|
+
].filter(Boolean).join(" and ")} required ${dependencyCount === 1 ? "control" : "controls"})` : "";
|
|
529
|
+
const established = report.controls.filter(
|
|
530
|
+
({ status }) => status === "met" || status === "not_applicable"
|
|
531
|
+
);
|
|
532
|
+
const unresolved = report.controls.filter(
|
|
533
|
+
({ status }) => status === "not_met" || status === "unknown"
|
|
534
|
+
);
|
|
535
|
+
const repositoryGaps = unresolved.filter((control) => controlScope(control) === "repository");
|
|
536
|
+
const externalControls = unresolved.filter(
|
|
537
|
+
(control) => ["platform", "organization"].includes(controlScope(control))
|
|
538
|
+
);
|
|
539
|
+
const outcomeControls = unresolved.filter((control) => controlScope(control) === "outcome");
|
|
186
540
|
const lines = [
|
|
187
541
|
"# Agentic Development Readiness Assessment",
|
|
188
542
|
"",
|
|
189
543
|
`- Benchmark: ${report.benchmark.id} v${report.benchmark.version}`,
|
|
190
544
|
`- Repository: \`${report.target.repository}\``,
|
|
545
|
+
`- Assessment scope: **${report.target.scope}**`,
|
|
546
|
+
`- Git commit: ${report.target.git_head ? `\`${report.target.git_head}\`` : "unavailable"}`,
|
|
547
|
+
`- Working tree dirty: ${report.target.working_tree_dirty === null ? "unknown" : String(report.target.working_tree_dirty)}`,
|
|
191
548
|
`- Assessed: ${report.assessed_at}`,
|
|
192
549
|
`- Score: **${report.score.total}/${report.score.maximum} (${report.score.percentage}%)**`,
|
|
550
|
+
...report.score.repository ? [
|
|
551
|
+
`- Repository-detected progress: **${report.score.repository.achieved}/${report.score.repository.ceiling} (${report.score.repository.percentage}%)** of the maturity levels the offline repository collector can establish`
|
|
552
|
+
] : [],
|
|
193
553
|
`- Highest readiness profile: **${report.readiness.highest_profile ?? "none"}**`,
|
|
194
|
-
`- Target \`${report.target.profile}\`: **${report.readiness.target_passed ?
|
|
195
|
-
`- Evidence: ${report.evidence_summary.
|
|
554
|
+
`- Target \`${report.target.profile}\`: **${report.readiness.target_passed ? `PASS${targetProvenance}` : "FAIL"}**`,
|
|
555
|
+
`- Evidence: ${report.evidence_summary.repository_detected} repository-detected, ${report.evidence_summary.agent_collected} agent-collected, ${report.evidence_summary.attested} human-attested, ${report.evidence_summary.unmet} unmet, ${report.evidence_summary.unknown} unknown${report.evidence_summary.resolved !== void 0 && report.evidence_summary.total !== void 0 ? `; ${report.evidence_summary.resolved}/${report.evidence_summary.total} controls resolved` : ""}`,
|
|
556
|
+
...report.warnings && report.warnings.length > 0 ? [`- Warnings: **${report.warnings.length} \u2014 review before using this assessment**`] : [],
|
|
196
557
|
"",
|
|
558
|
+
...report.warnings && report.warnings.length > 0 ? [
|
|
559
|
+
"## Warnings",
|
|
560
|
+
"",
|
|
561
|
+
...report.warnings.map((warning) => `- WARNING: ${safeText(warning)}`),
|
|
562
|
+
""
|
|
563
|
+
] : [],
|
|
197
564
|
"## Dimensions",
|
|
198
565
|
"",
|
|
199
566
|
"| Dimension | Score | Controls met |",
|
|
200
567
|
"| --- | ---: | ---: |",
|
|
201
568
|
...report.dimensions.map(
|
|
202
569
|
({ title, score, controls_met: met, controls_total: total }) => `| ${title} | ${score}/4 | ${met}/${total} |`
|
|
203
|
-
)
|
|
570
|
+
),
|
|
571
|
+
"",
|
|
572
|
+
"> A dimension earns only consecutive levels. Controls met above the first gap remain visible but do not increase its score."
|
|
204
573
|
];
|
|
205
574
|
if (target && !target.passed) {
|
|
206
575
|
lines.push("", "## Target-profile blockers", "");
|
|
@@ -210,27 +579,22 @@ function toMarkdown(report) {
|
|
|
210
579
|
);
|
|
211
580
|
}
|
|
212
581
|
}
|
|
213
|
-
lines.push("", "##
|
|
214
|
-
|
|
215
|
-
({ status }) => status === "not_met" || status === "unknown"
|
|
216
|
-
);
|
|
217
|
-
if (actionControls.length === 0) {
|
|
582
|
+
lines.push("", "## Established controls", "");
|
|
583
|
+
if (established.length === 0) {
|
|
218
584
|
lines.push("None.");
|
|
219
585
|
} else {
|
|
220
|
-
|
|
586
|
+
lines.push("| Control | Level | Evidence scope | Confidence |", "| --- | ---: | --- | --- |");
|
|
587
|
+
for (const control of established) {
|
|
221
588
|
lines.push(
|
|
222
|
-
|
|
223
|
-
"",
|
|
224
|
-
`**Risk:** ${control.risk}`,
|
|
225
|
-
"",
|
|
226
|
-
`**Improve:** ${control.remediation}`,
|
|
227
|
-
"",
|
|
228
|
-
`Evidence confidence: ${control.confidence}.`,
|
|
229
|
-
""
|
|
589
|
+
`| ${control.id} \u2014 ${control.title.replaceAll("|", "\\|")} | ${control.level} | ${controlScope(control)} | ${control.confidence} |`
|
|
230
590
|
);
|
|
231
591
|
}
|
|
232
592
|
}
|
|
593
|
+
appendControlDetails(lines, "Repository evidence gaps", repositoryGaps);
|
|
594
|
+
appendControlDetails(lines, "External controls not established", externalControls);
|
|
595
|
+
appendControlDetails(lines, "Outcome evidence not established", outcomeControls);
|
|
233
596
|
lines.push(
|
|
597
|
+
"",
|
|
234
598
|
"## Limitations",
|
|
235
599
|
"",
|
|
236
600
|
...report.limitations.map((limitation) => `- ${limitation}`),
|
|
@@ -239,143 +603,401 @@ function toMarkdown(report) {
|
|
|
239
603
|
return lines.join("\n");
|
|
240
604
|
}
|
|
241
605
|
|
|
606
|
+
// src/repository.ts
|
|
607
|
+
import { execFile } from "child_process";
|
|
608
|
+
import { realpath } from "fs/promises";
|
|
609
|
+
import { isAbsolute, relative, resolve as resolve2, sep } from "path";
|
|
610
|
+
import { promisify } from "util";
|
|
611
|
+
var execFileAsync = promisify(execFile);
|
|
612
|
+
var generatedEvidenceIgnores = [
|
|
613
|
+
"**/.git/**",
|
|
614
|
+
"**/node_modules/**",
|
|
615
|
+
"**/dist/**",
|
|
616
|
+
"**/coverage/**",
|
|
617
|
+
"**/.agentic/reports/**",
|
|
618
|
+
"**/.agentic/report*.json",
|
|
619
|
+
"**/.agentic/agentic-readiness*.json",
|
|
620
|
+
"**/.agentic/agentic-readiness*.md",
|
|
621
|
+
"**/.agentic/attestations.*",
|
|
622
|
+
"**/.agentic/agent-evidence.*",
|
|
623
|
+
"**/.agentic/evidence-request.*"
|
|
624
|
+
];
|
|
625
|
+
function relativePathWithin(root, path) {
|
|
626
|
+
const candidate = relative(root, path);
|
|
627
|
+
if (candidate === "") return candidate;
|
|
628
|
+
if (candidate === ".." || candidate.startsWith(`..${sep}`) || isAbsolute(candidate)) {
|
|
629
|
+
return null;
|
|
630
|
+
}
|
|
631
|
+
return candidate;
|
|
632
|
+
}
|
|
633
|
+
function normalizeExcludedPath(path, requestedRoot, canonicalRoot) {
|
|
634
|
+
const absolute = isAbsolute(path) ? resolve2(path) : resolve2(requestedRoot, path);
|
|
635
|
+
return relativePathWithin(requestedRoot, absolute) ?? relativePathWithin(canonicalRoot, absolute);
|
|
636
|
+
}
|
|
637
|
+
function sanitizeRemote(remote) {
|
|
638
|
+
if (!remote) return null;
|
|
639
|
+
try {
|
|
640
|
+
const url = new URL(remote);
|
|
641
|
+
if (url.protocol === "http:" || url.protocol === "https:") {
|
|
642
|
+
url.username = "";
|
|
643
|
+
url.password = "";
|
|
644
|
+
} else if (url.password) {
|
|
645
|
+
url.password = "";
|
|
646
|
+
}
|
|
647
|
+
return url.toString();
|
|
648
|
+
} catch {
|
|
649
|
+
return remote;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
async function git(repo, args) {
|
|
653
|
+
try {
|
|
654
|
+
const { stdout } = await execFileAsync("git", ["-C", repo, ...args], {
|
|
655
|
+
encoding: "utf8",
|
|
656
|
+
maxBuffer: 1e7
|
|
657
|
+
});
|
|
658
|
+
return stdout;
|
|
659
|
+
} catch {
|
|
660
|
+
return null;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
async function createRepositoryContext(repository, scope, excludedPaths = []) {
|
|
664
|
+
const requestedRoot = resolve2(repository);
|
|
665
|
+
const root = await realpath(requestedRoot);
|
|
666
|
+
const [headOutput, remoteOutput, statusOutput, trackedStatusOutput] = await Promise.all([
|
|
667
|
+
git(root, ["rev-parse", "HEAD"]),
|
|
668
|
+
git(root, ["config", "--get", "remote.origin.url"]),
|
|
669
|
+
git(root, ["status", "--porcelain"]),
|
|
670
|
+
git(root, ["status", "--porcelain", "--untracked-files=no"])
|
|
671
|
+
]);
|
|
672
|
+
let includedPaths = null;
|
|
673
|
+
if (scope === "tracked") {
|
|
674
|
+
const tracked = await git(root, ["ls-files", "-z", "--cached"]);
|
|
675
|
+
if (tracked === null) {
|
|
676
|
+
throw new Error(
|
|
677
|
+
"Tracked assessment requires a Git worktree. Use --scope workspace for a provisional filesystem assessment."
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
includedPaths = new Set(tracked.split("\0").filter((path) => path.length > 0));
|
|
681
|
+
}
|
|
682
|
+
return {
|
|
683
|
+
metadata: {
|
|
684
|
+
root,
|
|
685
|
+
scope,
|
|
686
|
+
git_head: headOutput?.trim() || null,
|
|
687
|
+
git_remote: sanitizeRemote(remoteOutput?.trim() || null),
|
|
688
|
+
working_tree_dirty: statusOutput === null ? null : statusOutput.length > 0,
|
|
689
|
+
tracked_tree_dirty: trackedStatusOutput === null ? null : trackedStatusOutput.length > 0
|
|
690
|
+
},
|
|
691
|
+
includedPaths,
|
|
692
|
+
excludedPaths: new Set(
|
|
693
|
+
excludedPaths.flatMap((path) => {
|
|
694
|
+
const normalized = normalizeExcludedPath(path, requestedRoot, root);
|
|
695
|
+
return normalized === null ? [] : [normalized];
|
|
696
|
+
})
|
|
697
|
+
)
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
function repositoryEvidenceTarget(metadata) {
|
|
701
|
+
return {
|
|
702
|
+
repository: metadata.git_remote ?? metadata.root,
|
|
703
|
+
git_head: metadata.git_head
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// src/score.ts
|
|
708
|
+
import { readFile as readFile3 } from "fs/promises";
|
|
709
|
+
import { join as join2 } from "path";
|
|
710
|
+
|
|
242
711
|
// src/evidence.ts
|
|
243
|
-
import { lstat, readFile as readFile2, realpath, stat } from "fs/promises";
|
|
244
|
-
import {
|
|
712
|
+
import { lstat, readFile as readFile2, realpath as realpath2, stat } from "fs/promises";
|
|
713
|
+
import { resolve as resolve3, sep as sep2 } from "path";
|
|
245
714
|
import fg2 from "fast-glob";
|
|
246
|
-
var ignored = ["**/.git/**", "**/node_modules/**", "**/dist/**", "**/coverage/**"];
|
|
247
715
|
var maxContentFileBytes = 512e3;
|
|
248
716
|
var maxContentFiles = 250;
|
|
249
717
|
var maxContentTotalBytes = 5e6;
|
|
250
|
-
async function matches(
|
|
251
|
-
|
|
252
|
-
cwd:
|
|
718
|
+
async function matches(context, patterns) {
|
|
719
|
+
const found = await fg2(patterns, {
|
|
720
|
+
cwd: context.metadata.root,
|
|
253
721
|
dot: true,
|
|
254
|
-
onlyFiles:
|
|
722
|
+
onlyFiles: true,
|
|
255
723
|
unique: true,
|
|
256
724
|
followSymbolicLinks: false,
|
|
257
|
-
ignore:
|
|
258
|
-
})
|
|
725
|
+
ignore: generatedEvidenceIgnores
|
|
726
|
+
});
|
|
727
|
+
return found.filter((path) => context.includedPaths === null || context.includedPaths.has(path)).filter((path) => !context.excludedPaths.has(path)).sort();
|
|
728
|
+
}
|
|
729
|
+
async function safeFileSize(root, path) {
|
|
730
|
+
try {
|
|
731
|
+
const requestedPath = resolve3(root, path);
|
|
732
|
+
if ((await lstat(requestedPath)).isSymbolicLink()) return null;
|
|
733
|
+
const canonicalPath = await realpath2(requestedPath);
|
|
734
|
+
if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep2}`)) return null;
|
|
735
|
+
const metadata = await stat(canonicalPath);
|
|
736
|
+
return metadata.isFile() ? metadata.size : null;
|
|
737
|
+
} catch {
|
|
738
|
+
return null;
|
|
739
|
+
}
|
|
259
740
|
}
|
|
260
|
-
async function
|
|
261
|
-
const found = await matches(
|
|
741
|
+
async function nonEmptyMatches(context, patterns, minBytes) {
|
|
742
|
+
const found = await matches(context, patterns);
|
|
743
|
+
const qualifying = await Promise.all(
|
|
744
|
+
found.map(async (path) => ({ path, size: await safeFileSize(context.metadata.root, path) }))
|
|
745
|
+
);
|
|
746
|
+
return qualifying.filter(({ size }) => size !== null && size >= minBytes).map(({ path }) => path);
|
|
747
|
+
}
|
|
748
|
+
async function evaluatePathAny(context, check) {
|
|
749
|
+
const found = await nonEmptyMatches(context, check.patterns, check.min_bytes);
|
|
262
750
|
return result(
|
|
263
751
|
check.type,
|
|
752
|
+
check.scope,
|
|
264
753
|
found.length > 0 ? "met" : "not_met",
|
|
265
|
-
`${found.length} matching
|
|
754
|
+
`${found.length} safe, non-empty matching file(s)`,
|
|
266
755
|
found
|
|
267
756
|
);
|
|
268
757
|
}
|
|
269
|
-
async function evaluatePathAll(
|
|
270
|
-
const groups = await Promise.all(
|
|
758
|
+
async function evaluatePathAll(context, check) {
|
|
759
|
+
const groups = await Promise.all(
|
|
760
|
+
check.patterns.map(async (pattern) => nonEmptyMatches(context, [pattern], check.min_bytes))
|
|
761
|
+
);
|
|
271
762
|
const missing = check.patterns.filter((_, index) => groups[index]?.length === 0);
|
|
272
763
|
const found = [...new Set(groups.flat())].sort();
|
|
273
764
|
return result(
|
|
274
765
|
check.type,
|
|
766
|
+
check.scope,
|
|
275
767
|
missing.length === 0 ? "met" : "not_met",
|
|
276
|
-
missing.length === 0 ? "Every required
|
|
768
|
+
missing.length === 0 ? "Every required pattern matched a safe, non-empty file" : `Missing non-empty patterns: ${missing.join(", ")}`,
|
|
277
769
|
found
|
|
278
770
|
);
|
|
279
771
|
}
|
|
280
|
-
async function readSearchableFiles(
|
|
281
|
-
const root =
|
|
282
|
-
const paths = (await matches(
|
|
772
|
+
async function readSearchableFiles(context, patterns, maxFilesPerPattern) {
|
|
773
|
+
const root = context.metadata.root;
|
|
774
|
+
const paths = maxFilesPerPattern ? await prioritizedMatches(context, patterns, maxFilesPerPattern) : (await matches(context, patterns)).slice(0, maxContentFiles);
|
|
283
775
|
const files = [];
|
|
284
776
|
let totalBytes = 0;
|
|
285
777
|
for (const path of paths) {
|
|
286
778
|
try {
|
|
287
|
-
const requestedPath =
|
|
779
|
+
const requestedPath = resolve3(root, path);
|
|
288
780
|
if ((await lstat(requestedPath)).isSymbolicLink()) continue;
|
|
289
|
-
const canonicalPath = await
|
|
290
|
-
if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${
|
|
781
|
+
const canonicalPath = await realpath2(requestedPath);
|
|
782
|
+
if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep2}`)) continue;
|
|
291
783
|
const metadata = await stat(canonicalPath);
|
|
292
|
-
if (!metadata.isFile() || metadata.size > maxContentFileBytes || totalBytes + metadata.size > maxContentTotalBytes) {
|
|
784
|
+
if (!metadata.isFile() || metadata.size === 0 || metadata.size > maxContentFileBytes || totalBytes + metadata.size > maxContentTotalBytes) {
|
|
293
785
|
continue;
|
|
294
786
|
}
|
|
787
|
+
const text = (await readFile2(canonicalPath, "utf8")).toLowerCase();
|
|
788
|
+
if (isGeneratedAssessment(text)) continue;
|
|
295
789
|
totalBytes += metadata.size;
|
|
296
|
-
files.push({ path, text
|
|
790
|
+
files.push({ path, text });
|
|
297
791
|
} catch {
|
|
298
792
|
}
|
|
299
793
|
}
|
|
300
794
|
return files;
|
|
301
795
|
}
|
|
302
|
-
async function
|
|
303
|
-
const
|
|
796
|
+
async function prioritizedMatches(context, patterns, maxFilesPerPattern) {
|
|
797
|
+
const selected = [];
|
|
798
|
+
const seen = /* @__PURE__ */ new Set();
|
|
799
|
+
const groups = await Promise.all(
|
|
800
|
+
patterns.map(
|
|
801
|
+
async (pattern) => (await matches(context, [pattern])).slice(0, maxFilesPerPattern)
|
|
802
|
+
)
|
|
803
|
+
);
|
|
804
|
+
for (let candidateIndex = 0; candidateIndex < maxFilesPerPattern; candidateIndex += 1) {
|
|
805
|
+
for (const group of groups) {
|
|
806
|
+
const path = group[candidateIndex];
|
|
807
|
+
if (!path) continue;
|
|
808
|
+
if (seen.has(path)) continue;
|
|
809
|
+
seen.add(path);
|
|
810
|
+
selected.push(path);
|
|
811
|
+
if (selected.length === maxContentFiles) return selected;
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
return selected;
|
|
815
|
+
}
|
|
816
|
+
function isGeneratedAssessment(text) {
|
|
817
|
+
const normalized = text.trimStart();
|
|
818
|
+
if (normalized.startsWith("# agentic development readiness assessment")) return true;
|
|
819
|
+
if (!normalized.startsWith("{")) return false;
|
|
820
|
+
try {
|
|
821
|
+
const candidate = JSON.parse(normalized);
|
|
822
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) return false;
|
|
823
|
+
const report = candidate;
|
|
824
|
+
const benchmark = report.benchmark;
|
|
825
|
+
return Boolean(benchmark) && typeof benchmark === "object" && !Array.isArray(benchmark) && benchmark.id === "agentic-development-readiness" && typeof report.assessed_at === "string" && Array.isArray(report.controls);
|
|
826
|
+
} catch {
|
|
827
|
+
return false;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
async function evaluateLegacyContent(context, check) {
|
|
831
|
+
const files = await readSearchableFiles(context, check.files);
|
|
832
|
+
const matchingFiles = files.filter(
|
|
833
|
+
({ text }) => check.needles.some((needle) => text.includes(needle.toLowerCase()))
|
|
834
|
+
);
|
|
304
835
|
const matchedNeedles = check.needles.filter(
|
|
305
836
|
(needle) => files.some(({ text }) => text.includes(needle.toLowerCase()))
|
|
306
837
|
);
|
|
307
838
|
const passed = check.type === "content_any" ? matchedNeedles.length > 0 : matchedNeedles.length === check.needles.length;
|
|
308
839
|
return result(
|
|
309
840
|
check.type,
|
|
841
|
+
check.scope,
|
|
310
842
|
passed ? "met" : "not_met",
|
|
311
|
-
`Matched ${matchedNeedles.length}/${check.needles.length}
|
|
312
|
-
|
|
843
|
+
`Matched ${matchedNeedles.length}/${check.needles.length} term(s) across ${files.length} candidate file(s)`,
|
|
844
|
+
matchingFiles.map(({ path }) => path)
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
async function evaluateContentTerms(context, check) {
|
|
848
|
+
const files = await readSearchableFiles(context, check.files, check.max_files_per_pattern);
|
|
849
|
+
const matchesByFile = files.map(({ path, text }) => ({
|
|
850
|
+
path,
|
|
851
|
+
...strongestContentMatch(
|
|
852
|
+
text,
|
|
853
|
+
check.terms,
|
|
854
|
+
check.required_any_terms ?? [],
|
|
855
|
+
check.min_terms,
|
|
856
|
+
check.max_span_lines
|
|
857
|
+
)
|
|
858
|
+
}));
|
|
859
|
+
const qualifying = matchesByFile.filter(({ qualifies }) => qualifies);
|
|
860
|
+
const strongest = matchesByFile.reduce((maximum, file) => Math.max(maximum, file.matched), 0);
|
|
861
|
+
const strongestRequired = matchesByFile.reduce(
|
|
862
|
+
(maximum, file) => Math.max(maximum, file.requiredMatched),
|
|
863
|
+
0
|
|
313
864
|
);
|
|
865
|
+
const requiredSummary = check.required_any_terms ? `; strongest required match ${strongestRequired}/${check.required_any_terms.length}` : "";
|
|
866
|
+
const proximitySummary = check.max_span_lines ? ` within ${check.max_span_lines}-line window(s)` : "";
|
|
867
|
+
return result(
|
|
868
|
+
check.type,
|
|
869
|
+
check.scope,
|
|
870
|
+
qualifying.length > 0 ? "met" : "not_met",
|
|
871
|
+
`${qualifying.length} qualifying file(s); strongest co-located match ${strongest}/${check.terms.length} term(s)${requiredSummary}${proximitySummary} across ${files.length} candidate file(s); threshold ${check.min_terms}`,
|
|
872
|
+
qualifying.map(({ path }) => path)
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
function strongestContentMatch(text, terms, requiredTerms, minTerms, maxSpanLines) {
|
|
876
|
+
if (!maxSpanLines) {
|
|
877
|
+
const matched = terms.filter((term) => containsTerm(text, term)).length;
|
|
878
|
+
const requiredMatched = requiredTerms.filter((term) => containsTerm(text, term)).length;
|
|
879
|
+
return {
|
|
880
|
+
matched,
|
|
881
|
+
requiredMatched,
|
|
882
|
+
qualifies: matched >= minTerms && (requiredTerms.length === 0 || requiredMatched > 0)
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
const termCounts = terms.map(() => 0);
|
|
886
|
+
const requiredCounts = requiredTerms.map(() => 0);
|
|
887
|
+
const lines = text.split(/\r?\n/);
|
|
888
|
+
let strongest = 0;
|
|
889
|
+
let strongestRequired = 0;
|
|
890
|
+
let qualifies = false;
|
|
891
|
+
const update = (line, direction) => {
|
|
892
|
+
terms.forEach((term, index) => {
|
|
893
|
+
if (containsTerm(line, term)) termCounts[index] = (termCounts[index] ?? 0) + direction;
|
|
894
|
+
});
|
|
895
|
+
requiredTerms.forEach((term, index) => {
|
|
896
|
+
if (containsTerm(line, term)) {
|
|
897
|
+
requiredCounts[index] = (requiredCounts[index] ?? 0) + direction;
|
|
898
|
+
}
|
|
899
|
+
});
|
|
900
|
+
};
|
|
901
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
902
|
+
update(lines[index] ?? "", 1);
|
|
903
|
+
if (index >= maxSpanLines) update(lines[index - maxSpanLines] ?? "", -1);
|
|
904
|
+
const matched = termCounts.filter((count) => count > 0).length;
|
|
905
|
+
const requiredMatched = requiredCounts.filter((count) => count > 0).length;
|
|
906
|
+
strongest = Math.max(strongest, matched);
|
|
907
|
+
strongestRequired = Math.max(strongestRequired, requiredMatched);
|
|
908
|
+
if (matched >= minTerms && (requiredTerms.length === 0 || requiredMatched > 0)) {
|
|
909
|
+
qualifies = true;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
return { matched: strongest, requiredMatched: strongestRequired, qualifies };
|
|
314
913
|
}
|
|
315
|
-
|
|
316
|
-
const
|
|
317
|
-
|
|
914
|
+
function containsTerm(text, term) {
|
|
915
|
+
const pattern = term.trim().split(/\s+/).map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("[\\s_-]+");
|
|
916
|
+
return new RegExp(`(^|[^a-z0-9])${pattern}(?=$|[^a-z0-9])`, "i").test(text);
|
|
917
|
+
}
|
|
918
|
+
async function evaluateMaxBytes(context, check) {
|
|
919
|
+
const paths = await matches(context, check.patterns);
|
|
318
920
|
let total = 0;
|
|
319
|
-
|
|
921
|
+
const inspected = [];
|
|
320
922
|
for (const path of paths) {
|
|
321
|
-
const
|
|
322
|
-
if (
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
}
|
|
328
|
-
const passed = inspected > 0 && total <= check.max_bytes;
|
|
923
|
+
const size = await safeFileSize(context.metadata.root, path);
|
|
924
|
+
if (size === null) continue;
|
|
925
|
+
total += size;
|
|
926
|
+
inspected.push(path);
|
|
927
|
+
}
|
|
928
|
+
const passed = inspected.length > 0 && total <= check.max_bytes;
|
|
329
929
|
return result(
|
|
330
930
|
check.type,
|
|
931
|
+
check.scope,
|
|
331
932
|
passed ? "met" : "not_met",
|
|
332
|
-
`${total} byte(s) across ${inspected} safe matching file(s); maximum ${check.max_bytes}`,
|
|
333
|
-
|
|
933
|
+
`${total} byte(s) across ${inspected.length} safe matching file(s); maximum ${check.max_bytes}`,
|
|
934
|
+
inspected
|
|
334
935
|
);
|
|
335
936
|
}
|
|
336
|
-
function result(type, status, summary,
|
|
337
|
-
return { type, status, summary, references };
|
|
937
|
+
function result(type, scope, status, summary, references2) {
|
|
938
|
+
return { type, scope, status, summary, references: references2 };
|
|
338
939
|
}
|
|
339
|
-
async function evaluateCheck(
|
|
940
|
+
async function evaluateCheck(context, check) {
|
|
340
941
|
switch (check.type) {
|
|
341
942
|
case "path_any":
|
|
342
|
-
return evaluatePathAny(
|
|
943
|
+
return evaluatePathAny(context, check);
|
|
343
944
|
case "path_all":
|
|
344
|
-
return evaluatePathAll(
|
|
945
|
+
return evaluatePathAll(context, check);
|
|
345
946
|
case "content_any":
|
|
346
947
|
case "content_all":
|
|
347
|
-
return
|
|
948
|
+
return evaluateLegacyContent(context, check);
|
|
949
|
+
case "content_terms":
|
|
950
|
+
return evaluateContentTerms(context, check);
|
|
348
951
|
case "max_bytes":
|
|
349
|
-
return evaluateMaxBytes(
|
|
952
|
+
return evaluateMaxBytes(context, check);
|
|
350
953
|
case "manual":
|
|
351
|
-
return result("manual", "unknown", check.prompt, []);
|
|
954
|
+
return result("manual", check.scope, "unknown", check.prompt, []);
|
|
352
955
|
}
|
|
353
956
|
}
|
|
354
957
|
function activeAttestation(control, attestations, now) {
|
|
958
|
+
if (!control.allow_attestation) return null;
|
|
355
959
|
const attestation = attestations?.attestations[control.id];
|
|
356
960
|
if (!attestation) return null;
|
|
357
961
|
if (attestation.expires_at && new Date(attestation.expires_at) < now) return null;
|
|
358
962
|
if (attestation.status === "not_applicable" && !control.allow_not_applicable) return null;
|
|
359
963
|
return attestation;
|
|
360
964
|
}
|
|
361
|
-
|
|
965
|
+
function activeAgentEvidence(control, claim, now) {
|
|
966
|
+
if (!claim || !control.allow_agent_evidence) return null;
|
|
967
|
+
if (new Date(claim.expires_at) < now) return null;
|
|
968
|
+
return claim;
|
|
969
|
+
}
|
|
970
|
+
async function evaluateControl(context, control, attestations, agentClaim, now = /* @__PURE__ */ new Date()) {
|
|
362
971
|
const evidence = await Promise.all(
|
|
363
|
-
control.evidence.map(async (check) => evaluateCheck(
|
|
972
|
+
control.evidence.map(async (check) => evaluateCheck(context, check))
|
|
364
973
|
);
|
|
365
974
|
const attestation = activeAttestation(control, attestations, now);
|
|
975
|
+
const agentEvidence = activeAgentEvidence(control, agentClaim, now);
|
|
366
976
|
const checksPassed = evidence.every(({ status: status2 }) => status2 === "met");
|
|
367
977
|
const hasManualCheck = control.evidence.some(({ type }) => type === "manual");
|
|
368
978
|
let status = checksPassed ? "met" : "not_met";
|
|
369
|
-
let confidence = checksPassed && !hasManualCheck ? "
|
|
370
|
-
|
|
979
|
+
let confidence = checksPassed && !hasManualCheck ? "repository-detected" : "none";
|
|
980
|
+
const attestationStatus = attestation?.status === "unknown" ? null : attestation?.status ?? null;
|
|
981
|
+
const hasExternalConflict = agentEvidence !== null && agentEvidence.status !== "unknown" && attestationStatus !== null && agentEvidence.status !== attestationStatus;
|
|
982
|
+
if (checksPassed && !hasManualCheck) {
|
|
983
|
+
} else if (hasExternalConflict) {
|
|
984
|
+
status = "unknown";
|
|
985
|
+
confidence = "none";
|
|
986
|
+
} else if (agentEvidence && agentEvidence.status !== "unknown") {
|
|
987
|
+
status = agentEvidence.status;
|
|
988
|
+
confidence = "agent-collected";
|
|
989
|
+
} else if (attestation?.status === "not_applicable") {
|
|
371
990
|
status = "not_applicable";
|
|
372
991
|
confidence = "attested";
|
|
373
|
-
} else if (attestation?.status === "met"
|
|
992
|
+
} else if (attestation?.status === "met") {
|
|
374
993
|
status = "met";
|
|
375
|
-
confidence =
|
|
994
|
+
confidence = "attested";
|
|
376
995
|
} else if (attestation?.status === "not_met" || attestation?.status === "unknown") {
|
|
377
996
|
status = attestation.status;
|
|
378
997
|
confidence = "attested";
|
|
998
|
+
} else if (agentEvidence?.status === "unknown") {
|
|
999
|
+
status = "unknown";
|
|
1000
|
+
confidence = "agent-collected";
|
|
379
1001
|
} else if (evidence.some(({ status: checkStatus }) => checkStatus === "unknown")) {
|
|
380
1002
|
status = "unknown";
|
|
381
1003
|
}
|
|
@@ -389,6 +1011,7 @@ async function evaluateControl(repo, control, attestations, now = /* @__PURE__ *
|
|
|
389
1011
|
status,
|
|
390
1012
|
confidence,
|
|
391
1013
|
evidence,
|
|
1014
|
+
agent_evidence: agentEvidence,
|
|
392
1015
|
attestation,
|
|
393
1016
|
remediation: control.remediation
|
|
394
1017
|
};
|
|
@@ -409,6 +1032,46 @@ function dimensionScore(controls, dimension) {
|
|
|
409
1032
|
}
|
|
410
1033
|
return score;
|
|
411
1034
|
}
|
|
1035
|
+
function repositoryScore(catalog, results, dimensions) {
|
|
1036
|
+
let achieved = 0;
|
|
1037
|
+
let ceiling = 0;
|
|
1038
|
+
const resultsById = new Map(results.map((result2) => [result2.id, result2]));
|
|
1039
|
+
for (const { id: dimension } of dimensions) {
|
|
1040
|
+
let dimensionAchieved = 0;
|
|
1041
|
+
let dimensionCeiling = 0;
|
|
1042
|
+
let achievedOpen = true;
|
|
1043
|
+
let ceilingOpen = true;
|
|
1044
|
+
for (const level of [1, 2, 3, 4]) {
|
|
1045
|
+
const controlsAtLevel = catalog.filter(
|
|
1046
|
+
(control) => control.dimension === dimension && control.level === level
|
|
1047
|
+
);
|
|
1048
|
+
const repositoryDetectable = controlsAtLevel.every(
|
|
1049
|
+
(control) => control.evidence.every((evidence) => evidence.scope === "repository")
|
|
1050
|
+
);
|
|
1051
|
+
if (ceilingOpen && repositoryDetectable) {
|
|
1052
|
+
dimensionCeiling = level;
|
|
1053
|
+
} else {
|
|
1054
|
+
ceilingOpen = false;
|
|
1055
|
+
}
|
|
1056
|
+
const repositoryEstablished = controlsAtLevel.every((control) => {
|
|
1057
|
+
const result2 = resultsById.get(control.id);
|
|
1058
|
+
return result2?.status === "met" && result2.confidence === "repository-detected";
|
|
1059
|
+
});
|
|
1060
|
+
if (achievedOpen && repositoryDetectable && repositoryEstablished) {
|
|
1061
|
+
dimensionAchieved = level;
|
|
1062
|
+
} else {
|
|
1063
|
+
achievedOpen = false;
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
achieved += dimensionAchieved;
|
|
1067
|
+
ceiling += dimensionCeiling;
|
|
1068
|
+
}
|
|
1069
|
+
return {
|
|
1070
|
+
achieved,
|
|
1071
|
+
ceiling,
|
|
1072
|
+
percentage: ceiling === 0 ? 0 : Math.round(achieved / ceiling * 100)
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
412
1075
|
function assessProfiles(benchmark, dimensions, controls) {
|
|
413
1076
|
return benchmark.readiness_profiles.map((profile) => {
|
|
414
1077
|
const blockers = benchmark.dimensions.flatMap(({ id }) => {
|
|
@@ -426,18 +1089,46 @@ function assessProfiles(benchmark, dimensions, controls) {
|
|
|
426
1089
|
}
|
|
427
1090
|
];
|
|
428
1091
|
});
|
|
429
|
-
|
|
1092
|
+
const requiredControls = controls.filter(
|
|
1093
|
+
(control) => control.level <= profile.floors[control.dimension] && controlPasses(control)
|
|
1094
|
+
);
|
|
1095
|
+
return {
|
|
1096
|
+
id: profile.id,
|
|
1097
|
+
title: profile.title,
|
|
1098
|
+
passed: blockers.length === 0,
|
|
1099
|
+
blockers,
|
|
1100
|
+
...benchmark.version === "0.3.0" ? {
|
|
1101
|
+
evidence_dependencies: {
|
|
1102
|
+
agent_collected: requiredControls.filter(
|
|
1103
|
+
({ confidence }) => confidence === "agent-collected"
|
|
1104
|
+
).length,
|
|
1105
|
+
attested: requiredControls.filter(({ confidence }) => confidence === "attested").length
|
|
1106
|
+
}
|
|
1107
|
+
} : {}
|
|
1108
|
+
};
|
|
430
1109
|
});
|
|
431
1110
|
}
|
|
432
|
-
async function assess(repo, benchmark, catalog, profileId,
|
|
1111
|
+
async function assess(repo, benchmark, catalog, profileId, options = {}) {
|
|
433
1112
|
const profile = benchmark.readiness_profiles.find(({ id }) => id === profileId);
|
|
434
1113
|
if (!profile) {
|
|
435
1114
|
throw new Error(
|
|
436
1115
|
`Unknown profile ${profileId}. Choose: ${benchmark.readiness_profiles.map(({ id }) => id).join(", ")}`
|
|
437
1116
|
);
|
|
438
1117
|
}
|
|
1118
|
+
const scope = options.scope ?? "tracked";
|
|
1119
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
1120
|
+
const context = await createRepositoryContext(repo, scope, options.excludedPaths);
|
|
1121
|
+
await validateAgentEvidence(benchmark, catalog, context, options.agentEvidence ?? null, now);
|
|
439
1122
|
const controls = await Promise.all(
|
|
440
|
-
catalog.map(
|
|
1123
|
+
catalog.map(
|
|
1124
|
+
async (control) => evaluateControl(
|
|
1125
|
+
context,
|
|
1126
|
+
control,
|
|
1127
|
+
options.attestations ?? null,
|
|
1128
|
+
options.agentEvidence?.claims[control.id] ?? null,
|
|
1129
|
+
now
|
|
1130
|
+
)
|
|
1131
|
+
)
|
|
441
1132
|
);
|
|
442
1133
|
const dimensions = benchmark.dimensions.map(({ id, title }) => {
|
|
443
1134
|
const dimensionControls = controls.filter((control) => control.dimension === id);
|
|
@@ -451,40 +1142,160 @@ async function assess(repo, benchmark, catalog, profileId, attestations, now = /
|
|
|
451
1142
|
});
|
|
452
1143
|
const profiles = assessProfiles(benchmark, dimensions, controls);
|
|
453
1144
|
const total = dimensions.reduce((sum, { score }) => sum + score, 0);
|
|
1145
|
+
const repository = benchmark.version === "0.3.0" ? repositoryScore(catalog, controls, benchmark.dimensions) : void 0;
|
|
454
1146
|
const highestProfile = [...profiles].reverse().find(({ passed }) => passed)?.id ?? null;
|
|
455
1147
|
const targetPassed = profiles.find(({ id }) => id === profileId)?.passed ?? false;
|
|
456
1148
|
return {
|
|
457
|
-
schema_version: "0.
|
|
1149
|
+
schema_version: benchmark.version === "0.3.0" ? "0.3.0" : "0.2.0",
|
|
458
1150
|
benchmark: { id: benchmark.id, version: benchmark.version },
|
|
459
|
-
target: {
|
|
1151
|
+
target: {
|
|
1152
|
+
repository: repo,
|
|
1153
|
+
profile: profileId,
|
|
1154
|
+
scope,
|
|
1155
|
+
git_head: context.metadata.git_head,
|
|
1156
|
+
git_remote: context.metadata.git_remote,
|
|
1157
|
+
working_tree_dirty: context.metadata.working_tree_dirty
|
|
1158
|
+
},
|
|
460
1159
|
assessed_at: now.toISOString(),
|
|
461
|
-
|
|
1160
|
+
...benchmark.version === "0.3.0" ? { warnings: options.warnings ?? [] } : {},
|
|
1161
|
+
score: {
|
|
1162
|
+
total,
|
|
1163
|
+
maximum: 40,
|
|
1164
|
+
percentage: Math.round(total / 40 * 100),
|
|
1165
|
+
...repository ? { repository } : {}
|
|
1166
|
+
},
|
|
462
1167
|
evidence_summary: {
|
|
463
|
-
|
|
464
|
-
({ confidence, status }) => confidence === "
|
|
1168
|
+
repository_detected: controls.filter(
|
|
1169
|
+
({ confidence, status }) => confidence === "repository-detected" && status === "met"
|
|
1170
|
+
).length,
|
|
1171
|
+
agent_collected: controls.filter(
|
|
1172
|
+
({ confidence, status }) => confidence === "agent-collected" && status === "met"
|
|
465
1173
|
).length,
|
|
466
1174
|
attested: controls.filter(
|
|
467
1175
|
({ confidence, status }) => confidence === "attested" && status === "met"
|
|
468
1176
|
).length,
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
1177
|
+
unmet: controls.filter(({ status }) => status === "not_met").length,
|
|
1178
|
+
unknown: controls.filter(({ status }) => status === "unknown").length,
|
|
1179
|
+
...benchmark.version === "0.3.0" ? {
|
|
1180
|
+
resolved: controls.filter(({ status }) => status !== "unknown").length,
|
|
1181
|
+
total: controls.length
|
|
1182
|
+
} : {}
|
|
472
1183
|
},
|
|
473
1184
|
dimensions,
|
|
474
1185
|
controls,
|
|
475
1186
|
profiles,
|
|
476
1187
|
readiness: { highest_profile: highestProfile, target_passed: targetPassed },
|
|
477
1188
|
limitations: [
|
|
478
|
-
"
|
|
479
|
-
"
|
|
1189
|
+
scope === "tracked" ? "Tracked mode considers only Git-tracked paths, using current working-tree contents; uncommitted edits to tracked files can affect the result." : "Workspace mode includes untracked local files and is provisional; do not compare it directly with tracked-mode reports.",
|
|
1190
|
+
"Repository-detected evidence proves a qualifying artifact match, not consistent practice or external enforcement.",
|
|
1191
|
+
...benchmark.version === "0.3.0" ? [
|
|
1192
|
+
"Repository-detected progress uses only deterministic offline evidence and its attainable ceiling; it is explanatory and does not replace the normative score or readiness floors.",
|
|
1193
|
+
"Agent-collected repository evidence is semantic, target-bound, and source-backed but is not independently verified or relabelled as repository-detected."
|
|
1194
|
+
] : [],
|
|
1195
|
+
"Agent-collected evidence and human attestations are reported separately and are not independently verified.",
|
|
480
1196
|
"This assessment does not grant production access, deployment authority, or certification."
|
|
481
1197
|
]
|
|
482
1198
|
};
|
|
483
1199
|
}
|
|
1200
|
+
async function validateAgentEvidence(benchmark, catalog, context, evidence, now) {
|
|
1201
|
+
if (!evidence) return;
|
|
1202
|
+
if (evidence.benchmark_version !== benchmark.version) {
|
|
1203
|
+
throw new Error(
|
|
1204
|
+
`Agent evidence benchmark ${evidence.benchmark_version} does not match ${benchmark.version}`
|
|
1205
|
+
);
|
|
1206
|
+
}
|
|
1207
|
+
const expectedTarget = repositoryEvidenceTarget(context.metadata);
|
|
1208
|
+
if (evidence.target.repository !== expectedTarget.repository) {
|
|
1209
|
+
throw new Error(
|
|
1210
|
+
`Agent evidence target ${evidence.target.repository} does not match ${expectedTarget.repository}`
|
|
1211
|
+
);
|
|
1212
|
+
}
|
|
1213
|
+
if (evidence.target.git_head !== expectedTarget.git_head) {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Agent evidence commit ${evidence.target.git_head ?? "unavailable"} does not match ${expectedTarget.git_head ?? "an unavailable Git commit"}`
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
if (benchmark.version === "0.3.0" && context.metadata.tracked_tree_dirty) {
|
|
1219
|
+
throw new Error("ADRB v0.3 agent evidence requires tracked files to match the bound commit");
|
|
1220
|
+
}
|
|
1221
|
+
const controls = new Map(catalog.map((control) => [control.id, control]));
|
|
1222
|
+
if (Object.keys(evidence.claims).length > 0 && (evidence.collector.name.startsWith("TODO") || evidence.collector.version.startsWith("TODO"))) {
|
|
1223
|
+
throw new Error("Agent evidence with claims must identify the collector name and version");
|
|
1224
|
+
}
|
|
1225
|
+
for (const [controlId, claim] of Object.entries(evidence.claims)) {
|
|
1226
|
+
const control = controls.get(controlId);
|
|
1227
|
+
if (!control) throw new Error(`Agent evidence references unknown control ${controlId}`);
|
|
1228
|
+
if (!control.allow_agent_evidence) {
|
|
1229
|
+
throw new Error(`${controlId} does not permit agent-collected evidence`);
|
|
1230
|
+
}
|
|
1231
|
+
const manualScopes = control.evidence.filter(({ type }) => type === "manual").map(({ scope: evidenceScope }) => evidenceScope);
|
|
1232
|
+
const allowedScopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : manualScopes;
|
|
1233
|
+
if (!allowedScopes.includes(claim.scope)) {
|
|
1234
|
+
throw new Error(`${controlId} does not accept ${claim.scope} evidence`);
|
|
1235
|
+
}
|
|
1236
|
+
if (claim.scope === "repository") {
|
|
1237
|
+
if (context.includedPaths === null) {
|
|
1238
|
+
throw new Error(
|
|
1239
|
+
`${controlId} uses repository-scoped agent evidence, which requires --scope tracked`
|
|
1240
|
+
);
|
|
1241
|
+
}
|
|
1242
|
+
for (const reference of claim.references) {
|
|
1243
|
+
const parsedReference = repositoryReference(reference);
|
|
1244
|
+
if (!parsedReference || !context.includedPaths.has(parsedReference.path) || context.excludedPaths.has(parsedReference.path)) {
|
|
1245
|
+
throw new Error(`${controlId} references an unavailable tracked path: ${reference}`);
|
|
1246
|
+
}
|
|
1247
|
+
if (parsedReference.lines) {
|
|
1248
|
+
const contents = await readFile3(
|
|
1249
|
+
join2(context.metadata.root, parsedReference.path),
|
|
1250
|
+
"utf8"
|
|
1251
|
+
);
|
|
1252
|
+
const lineCount = countLines(contents);
|
|
1253
|
+
if (parsedReference.lines.start < 1 || parsedReference.lines.end < parsedReference.lines.start) {
|
|
1254
|
+
throw new Error(`${controlId} references an invalid line range: ${reference}`);
|
|
1255
|
+
}
|
|
1256
|
+
if (parsedReference.lines.end > lineCount) {
|
|
1257
|
+
throw new Error(
|
|
1258
|
+
`${controlId} references lines beyond ${parsedReference.path}'s ${lineCount} lines: ${reference}`
|
|
1259
|
+
);
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
if (new Date(claim.collected_at) > new Date(claim.expires_at)) {
|
|
1265
|
+
throw new Error(`${controlId} expires before it was collected`);
|
|
1266
|
+
}
|
|
1267
|
+
if (new Date(claim.collected_at) > now) {
|
|
1268
|
+
throw new Error(`${controlId} has a future collection timestamp`);
|
|
1269
|
+
}
|
|
1270
|
+
if (claim.summary.startsWith("TODO") || claim.references.some((reference) => reference.startsWith("TODO"))) {
|
|
1271
|
+
throw new Error(`${controlId} contains unresolved TODO evidence`);
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
function repositoryReference(reference) {
|
|
1276
|
+
if (!reference.startsWith("repo:")) return null;
|
|
1277
|
+
const withoutPrefix = reference.slice("repo:".length);
|
|
1278
|
+
const lineMatch = withoutPrefix.match(/#L(\d+)(?:-L?(\d+))?$/);
|
|
1279
|
+
const path = lineMatch ? withoutPrefix.slice(0, lineMatch.index) : withoutPrefix;
|
|
1280
|
+
if (path.length === 0 || path.startsWith("/") || path.includes("#") || path.includes("\\") || path.split("/").some((part) => part === ".." || part === ".")) {
|
|
1281
|
+
return null;
|
|
1282
|
+
}
|
|
1283
|
+
const start = lineMatch ? Number(lineMatch[1]) : null;
|
|
1284
|
+
const end = lineMatch ? Number(lineMatch[2] ?? lineMatch[1]) : null;
|
|
1285
|
+
return {
|
|
1286
|
+
path,
|
|
1287
|
+
lines: start === null || end === null ? null : { start, end }
|
|
1288
|
+
};
|
|
1289
|
+
}
|
|
1290
|
+
function countLines(contents) {
|
|
1291
|
+
if (contents.length === 0) return 0;
|
|
1292
|
+
const lines = contents.split("\n").length;
|
|
1293
|
+
return contents.endsWith("\n") ? lines - 1 : lines;
|
|
1294
|
+
}
|
|
484
1295
|
|
|
485
1296
|
// src/cli.ts
|
|
486
1297
|
var program = new Command();
|
|
487
|
-
program.name("agentic-scorecard").description("Evidence-backed readiness assessment for agentic software development harnesses").version("0.
|
|
1298
|
+
program.name("agentic-scorecard").description("Evidence-backed readiness assessment for agentic software development harnesses").version("0.3.0");
|
|
488
1299
|
program.command("validate").description("Validate the bundled benchmark catalog").action(async () => {
|
|
489
1300
|
const { benchmark, controls } = await loadBenchmark();
|
|
490
1301
|
process.stdout.write(
|
|
@@ -510,17 +1321,20 @@ ${control.evidence.map((check) => `- ${stringify(check).trim().replaceAll("\n",
|
|
|
510
1321
|
);
|
|
511
1322
|
});
|
|
512
1323
|
program.command("init").argument("[repository]", "repository to initialize", ".").option("--force", "replace an existing attestation file", false).description("Create a manual-attestation template").action(async (repository, options) => {
|
|
513
|
-
const repo =
|
|
514
|
-
const path =
|
|
1324
|
+
const repo = resolve4(repository);
|
|
1325
|
+
const path = join3(repo, ".agentic", "attestations.yaml");
|
|
515
1326
|
if (!options.force) {
|
|
516
1327
|
try {
|
|
517
|
-
await
|
|
1328
|
+
await readFile4(path, "utf8");
|
|
518
1329
|
throw new Error(`${path} already exists; use --force to replace it`);
|
|
519
1330
|
} catch (error) {
|
|
520
1331
|
if (error.code !== "ENOENT") throw error;
|
|
521
1332
|
}
|
|
522
1333
|
}
|
|
523
1334
|
const { benchmark, controls } = await loadBenchmark();
|
|
1335
|
+
const reviewedAt = /* @__PURE__ */ new Date();
|
|
1336
|
+
const expiresAt = new Date(reviewedAt);
|
|
1337
|
+
expiresAt.setDate(expiresAt.getDate() + 90);
|
|
524
1338
|
const attestations = Object.fromEntries(
|
|
525
1339
|
controls.filter((control) => control.evidence.some(({ type }) => type === "manual")).map((control) => {
|
|
526
1340
|
const manualCheck = control.evidence.find(
|
|
@@ -532,8 +1346,8 @@ program.command("init").argument("[repository]", "repository to initialize", "."
|
|
|
532
1346
|
status: "unknown",
|
|
533
1347
|
evidence: `TODO: ${manualCheck?.prompt ?? control.outcome}`,
|
|
534
1348
|
owner: "TODO",
|
|
535
|
-
reviewed_at:
|
|
536
|
-
expires_at:
|
|
1349
|
+
reviewed_at: reviewedAt.toISOString().slice(0, 10),
|
|
1350
|
+
expires_at: expiresAt.toISOString().slice(0, 10)
|
|
537
1351
|
}
|
|
538
1352
|
];
|
|
539
1353
|
})
|
|
@@ -541,30 +1355,145 @@ program.command("init").argument("[repository]", "repository to initialize", "."
|
|
|
541
1355
|
await mkdir(dirname2(path), { recursive: true });
|
|
542
1356
|
await writeFile(
|
|
543
1357
|
path,
|
|
544
|
-
`# Claims are
|
|
1358
|
+
`# Claims are visibly human-attested. Link durable evidence; do not paste secrets.
|
|
545
1359
|
${stringify({ benchmark_version: benchmark.version, attestations })}`,
|
|
546
1360
|
"utf8"
|
|
547
1361
|
);
|
|
548
1362
|
process.stdout.write(`Created ${path}
|
|
549
1363
|
`);
|
|
550
1364
|
});
|
|
551
|
-
program.command("
|
|
1365
|
+
program.command("init-evidence").argument("[repository]", "repository to prepare external evidence for", ".").option("--output <path>", "agent evidence bundle path").option("--request-output <path>", "human-readable evidence request path").option("--force", "replace an existing agent evidence bundle", false).description("Create a target-bound template for unresolved agent-collected evidence").action(
|
|
1366
|
+
async (repository, options) => {
|
|
1367
|
+
const repo = resolve4(repository);
|
|
1368
|
+
const path = resolve4(options.output ?? join3(repo, ".agentic", "agent-evidence.yaml"));
|
|
1369
|
+
const requestPath = resolve4(
|
|
1370
|
+
options.requestOutput ?? join3(repo, ".agentic", "evidence-request.md")
|
|
1371
|
+
);
|
|
1372
|
+
if (path === requestPath) {
|
|
1373
|
+
throw new Error("Agent evidence bundle and request paths must be different");
|
|
1374
|
+
}
|
|
1375
|
+
if (!options.force) {
|
|
1376
|
+
for (const candidate of [path, requestPath]) {
|
|
1377
|
+
try {
|
|
1378
|
+
await readFile4(candidate, "utf8");
|
|
1379
|
+
throw new Error(`${candidate} already exists; use --force to replace it`);
|
|
1380
|
+
} catch (error) {
|
|
1381
|
+
if (error.code !== "ENOENT") throw error;
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
const { benchmark, controls } = await loadBenchmark();
|
|
1386
|
+
const context = await createRepositoryContext(repo, "tracked").catch((error) => {
|
|
1387
|
+
if (error instanceof Error && error.message.startsWith("Tracked assessment requires a Git worktree")) {
|
|
1388
|
+
throw new Error(
|
|
1389
|
+
"init-evidence requires a Git worktree with a commit so the bundle can be target-bound."
|
|
1390
|
+
);
|
|
1391
|
+
}
|
|
1392
|
+
throw error;
|
|
1393
|
+
});
|
|
1394
|
+
if (!context.metadata.git_head) {
|
|
1395
|
+
throw new Error(
|
|
1396
|
+
"init-evidence requires a Git commit so the bundle can be target-bound. Commit the assessed state and try again."
|
|
1397
|
+
);
|
|
1398
|
+
}
|
|
1399
|
+
if (context.metadata.tracked_tree_dirty) {
|
|
1400
|
+
throw new Error(
|
|
1401
|
+
"init-evidence requires tracked files to match HEAD so every claim binds to the exact assessed commit."
|
|
1402
|
+
);
|
|
1403
|
+
}
|
|
1404
|
+
const baseline = await assess(repo, benchmark, controls, "read-only-analysis", {
|
|
1405
|
+
scope: "tracked"
|
|
1406
|
+
});
|
|
1407
|
+
const unresolved = new Set(
|
|
1408
|
+
baseline.controls.filter(({ status }) => status !== "met" && status !== "not_applicable").map(({ id }) => id)
|
|
1409
|
+
);
|
|
1410
|
+
const eligibleControls = controls.filter(
|
|
1411
|
+
({ allow_agent_evidence: allowed, id }) => allowed && unresolved.has(id)
|
|
1412
|
+
);
|
|
1413
|
+
const bundle = {
|
|
1414
|
+
schema_version: benchmark.version,
|
|
1415
|
+
benchmark_version: benchmark.version,
|
|
1416
|
+
target: repositoryEvidenceTarget(context.metadata),
|
|
1417
|
+
collector: { name: "TODO: agent or adapter name", version: "TODO" },
|
|
1418
|
+
claims: {}
|
|
1419
|
+
};
|
|
1420
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
1421
|
+
await writeFile(
|
|
1422
|
+
path,
|
|
1423
|
+
`# Use authorized read-only tools. Do not paste secrets or raw sensitive content.
|
|
1424
|
+
${stringify(bundle)}`,
|
|
1425
|
+
"utf8"
|
|
1426
|
+
);
|
|
1427
|
+
await mkdir(dirname2(requestPath), { recursive: true });
|
|
1428
|
+
await writeFile(
|
|
1429
|
+
requestPath,
|
|
1430
|
+
[
|
|
1431
|
+
"# ADRB v0.3 evidence request",
|
|
1432
|
+
"",
|
|
1433
|
+
`- Repository: ${bundle.target.repository}`,
|
|
1434
|
+
`- Git commit: ${bundle.target.git_head ?? "unavailable"}`,
|
|
1435
|
+
`- Benchmark: ${benchmark.version}`,
|
|
1436
|
+
"",
|
|
1437
|
+
"Repository claims may cite only tracked paths from the bound commit and remain agent-collected, not repository-detected. Obtain authorization before accessing connected systems. Use read-only, least-privileged tools. Add only attempted claims to the bundle; partial or inconclusive evidence remains `unknown`. Never paste source excerpts, secrets, prompts, personal data, or raw sensitive content.",
|
|
1438
|
+
"",
|
|
1439
|
+
...eligibleControls.flatMap((control) => {
|
|
1440
|
+
const manualCheck = control.evidence.find(
|
|
1441
|
+
(check) => check.type === "manual"
|
|
1442
|
+
);
|
|
1443
|
+
const scopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : [manualCheck?.scope ?? "organization"];
|
|
1444
|
+
return [
|
|
1445
|
+
`## ${control.id} \u2014 ${control.title}`,
|
|
1446
|
+
"",
|
|
1447
|
+
`- Scope: ${scopes.join(", ")}`,
|
|
1448
|
+
`- Request: ${manualCheck?.prompt ?? control.outcome}`,
|
|
1449
|
+
`- Risk: ${control.risk}`,
|
|
1450
|
+
""
|
|
1451
|
+
];
|
|
1452
|
+
})
|
|
1453
|
+
].join("\n"),
|
|
1454
|
+
"utf8"
|
|
1455
|
+
);
|
|
1456
|
+
process.stdout.write(`Created ${path}
|
|
1457
|
+
Created ${requestPath}
|
|
1458
|
+
`);
|
|
1459
|
+
}
|
|
1460
|
+
);
|
|
1461
|
+
program.command("assess").argument("[repository]", "repository to assess", ".").option("--profile <profile>", "target autonomy profile", "pr-creation").option("--format <format>", "json or markdown", "markdown").option("--output <path>", "write the report to a file").option("--attestations <path>", "manual attestation file").option("--agent-evidence <path>", "agent-collected repository or external evidence bundle").option("--scope <scope>", "tracked or workspace", "tracked").option("--enforce", "exit non-zero when the target profile fails", false).option("--github-output", "append summary values to $GITHUB_OUTPUT", false).description("Assess a repository using local, read-only evidence collection").action(
|
|
552
1462
|
async (repository, options) => {
|
|
553
1463
|
if (!["json", "markdown"].includes(options.format)) {
|
|
554
1464
|
throw new Error("--format must be json or markdown");
|
|
555
1465
|
}
|
|
556
|
-
|
|
1466
|
+
if (!["tracked", "workspace"].includes(options.scope)) {
|
|
1467
|
+
throw new Error("--scope must be tracked or workspace");
|
|
1468
|
+
}
|
|
1469
|
+
const repo = resolve4(repository);
|
|
557
1470
|
const { benchmark, controls } = await loadBenchmark();
|
|
558
|
-
const
|
|
559
|
-
|
|
1471
|
+
const warnings = [];
|
|
1472
|
+
const attestationPath = resolve4(
|
|
1473
|
+
options.attestations ?? join3(repo, ".agentic", "attestations.yaml")
|
|
560
1474
|
);
|
|
561
|
-
const attestations = await loadAttestations(attestationPath, benchmark.version
|
|
562
|
-
|
|
1475
|
+
const attestations = await loadAttestations(attestationPath, benchmark.version, {
|
|
1476
|
+
ignoreVersionMismatch: options.attestations === void 0,
|
|
1477
|
+
onWarning: (warning) => warnings.push(warning)
|
|
1478
|
+
});
|
|
1479
|
+
const agentEvidencePath = resolve4(
|
|
1480
|
+
options.agentEvidence ?? join3(repo, ".agentic", "agent-evidence.yaml")
|
|
1481
|
+
);
|
|
1482
|
+
const agentEvidence = await loadAgentEvidence(agentEvidencePath, benchmark.version, {
|
|
1483
|
+
ignoreVersionMismatch: options.agentEvidence === void 0,
|
|
1484
|
+
onWarning: (warning) => warnings.push(warning)
|
|
1485
|
+
});
|
|
1486
|
+
const reportPath = options.output ? resolve4(options.output) : null;
|
|
1487
|
+
const report = await assess(repo, benchmark, controls, options.profile, {
|
|
1488
|
+
scope: options.scope,
|
|
1489
|
+
attestations,
|
|
1490
|
+
agentEvidence,
|
|
1491
|
+
warnings,
|
|
1492
|
+
excludedPaths: [attestationPath, agentEvidencePath, ...reportPath ? [reportPath] : []]
|
|
1493
|
+
});
|
|
563
1494
|
const output = options.format === "json" ? `${JSON.stringify(report, null, 2)}
|
|
564
1495
|
` : toMarkdown(report);
|
|
565
|
-
|
|
566
|
-
if (options.output) {
|
|
567
|
-
reportPath = resolve3(options.output);
|
|
1496
|
+
if (reportPath) {
|
|
568
1497
|
await mkdir(dirname2(reportPath), { recursive: true });
|
|
569
1498
|
await writeFile(reportPath, output, "utf8");
|
|
570
1499
|
process.stdout.write(`Wrote ${reportPath}
|
|
@@ -579,6 +1508,9 @@ program.command("assess").argument("[repository]", "repository to assess", ".").
|
|
|
579
1508
|
githubOutput,
|
|
580
1509
|
`score=${report.score.total}
|
|
581
1510
|
percentage=${report.score.percentage}
|
|
1511
|
+
repository_score=${report.score.repository?.achieved ?? ""}
|
|
1512
|
+
repository_ceiling=${report.score.repository?.ceiling ?? ""}
|
|
1513
|
+
repository_percentage=${report.score.repository?.percentage ?? ""}
|
|
582
1514
|
highest_profile=${report.readiness.highest_profile ?? "none"}
|
|
583
1515
|
target_passed=${String(report.readiness.target_passed)}
|
|
584
1516
|
report_path=${reportPath ?? ""}
|