agentic-scorecard 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_PROMPT.md +64 -0
- package/LICENSE +201 -0
- package/NOTICE +4 -0
- package/README.md +191 -0
- package/benchmark/mappings/nist-ai-rmf.md +22 -0
- package/benchmark/mappings/openssf-scorecard.md +13 -0
- package/benchmark/mappings/owasp-agentic.md +16 -0
- package/benchmark/v0.1/adapter-contract.md +22 -0
- package/benchmark/v0.1/benchmark.yaml +126 -0
- package/benchmark/v0.1/controls/context.yaml +43 -0
- package/benchmark/v0.1/controls/environment.yaml +74 -0
- package/benchmark/v0.1/controls/governance.yaml +53 -0
- package/benchmark/v0.1/controls/learning.yaml +40 -0
- package/benchmark/v0.1/controls/observability.yaml +54 -0
- package/benchmark/v0.1/controls/resilience.yaml +40 -0
- package/benchmark/v0.1/controls/security.yaml +64 -0
- package/benchmark/v0.1/controls/specification.yaml +42 -0
- package/benchmark/v0.1/controls/testing.yaml +42 -0
- package/benchmark/v0.1/controls/tooling.yaml +48 -0
- package/benchmark/v0.1/evidence-schema.json +28 -0
- package/benchmark/v0.1/report-schema.json +32 -0
- package/benchmark/v0.1/scoring-policy.md +59 -0
- package/dist/cli.js +592 -0
- package/package.json +74 -0
- package/templates/attestations.yaml +9 -0
- package/templates/baseline.csv +1 -0
- package/templates/remediation-plan.md +15 -0
- package/templates/security-preflight.md +18 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/cli.ts
|
|
4
|
+
import { appendFile, mkdir, readFile as readFile3, writeFile } from "fs/promises";
|
|
5
|
+
import { dirname as dirname2, join as join2, resolve as resolve3 } from "path";
|
|
6
|
+
import { Command } from "commander";
|
|
7
|
+
import { stringify } from "yaml";
|
|
8
|
+
|
|
9
|
+
// src/load.ts
|
|
10
|
+
import { readFile } from "fs/promises";
|
|
11
|
+
import { dirname, join, resolve } from "path";
|
|
12
|
+
import { fileURLToPath } from "url";
|
|
13
|
+
import fg from "fast-glob";
|
|
14
|
+
import { parse } from "yaml";
|
|
15
|
+
|
|
16
|
+
// src/schema.ts
|
|
17
|
+
import { z } from "zod";
|
|
18
|
+
var dimensionIds = [
|
|
19
|
+
"context",
|
|
20
|
+
"environment",
|
|
21
|
+
"specification",
|
|
22
|
+
"tooling",
|
|
23
|
+
"security",
|
|
24
|
+
"testing",
|
|
25
|
+
"governance",
|
|
26
|
+
"learning",
|
|
27
|
+
"observability",
|
|
28
|
+
"resilience"
|
|
29
|
+
];
|
|
30
|
+
var DimensionIdSchema = z.enum(dimensionIds);
|
|
31
|
+
var LevelSchema = z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3), z.literal(4)]);
|
|
32
|
+
var PathAnySchema = z.object({
|
|
33
|
+
type: z.literal("path_any"),
|
|
34
|
+
patterns: z.array(z.string().min(1)).min(1)
|
|
35
|
+
});
|
|
36
|
+
var PathAllSchema = z.object({
|
|
37
|
+
type: z.literal("path_all"),
|
|
38
|
+
patterns: z.array(z.string().min(1)).min(1)
|
|
39
|
+
});
|
|
40
|
+
var ContentAnySchema = z.object({
|
|
41
|
+
type: z.literal("content_any"),
|
|
42
|
+
files: z.array(z.string().min(1)).min(1),
|
|
43
|
+
needles: z.array(z.string().min(1)).min(1)
|
|
44
|
+
});
|
|
45
|
+
var ContentAllSchema = z.object({
|
|
46
|
+
type: z.literal("content_all"),
|
|
47
|
+
files: z.array(z.string().min(1)).min(1),
|
|
48
|
+
needles: z.array(z.string().min(1)).min(1)
|
|
49
|
+
});
|
|
50
|
+
var MaxBytesSchema = z.object({
|
|
51
|
+
type: z.literal("max_bytes"),
|
|
52
|
+
patterns: z.array(z.string().min(1)).min(1),
|
|
53
|
+
max_bytes: z.number().int().positive()
|
|
54
|
+
});
|
|
55
|
+
var ManualSchema = z.object({
|
|
56
|
+
type: z.literal("manual"),
|
|
57
|
+
prompt: z.string().min(1)
|
|
58
|
+
});
|
|
59
|
+
var EvidenceCheckSchema = z.discriminatedUnion("type", [
|
|
60
|
+
PathAnySchema,
|
|
61
|
+
PathAllSchema,
|
|
62
|
+
ContentAnySchema,
|
|
63
|
+
ContentAllSchema,
|
|
64
|
+
MaxBytesSchema,
|
|
65
|
+
ManualSchema
|
|
66
|
+
]);
|
|
67
|
+
var RawControlSchema = z.object({
|
|
68
|
+
id: z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/),
|
|
69
|
+
level: z.union([z.literal(1), z.literal(2), z.literal(3), z.literal(4)]),
|
|
70
|
+
title: z.string().min(1),
|
|
71
|
+
outcome: z.string().min(1),
|
|
72
|
+
risk: z.string().min(1),
|
|
73
|
+
evidence: z.array(EvidenceCheckSchema).min(1),
|
|
74
|
+
remediation: z.string().min(1),
|
|
75
|
+
references: z.array(z.string().min(1)).default([]),
|
|
76
|
+
allow_attestation: z.boolean().default(true),
|
|
77
|
+
allow_not_applicable: z.boolean().default(false)
|
|
78
|
+
});
|
|
79
|
+
var ControlFileSchema = z.object({
|
|
80
|
+
dimension: DimensionIdSchema,
|
|
81
|
+
controls: z.array(RawControlSchema).min(1)
|
|
82
|
+
});
|
|
83
|
+
var DimensionSchema = z.object({
|
|
84
|
+
id: DimensionIdSchema,
|
|
85
|
+
title: z.string().min(1),
|
|
86
|
+
description: z.string().min(1)
|
|
87
|
+
});
|
|
88
|
+
var FloorsSchema = z.object(
|
|
89
|
+
Object.fromEntries(dimensionIds.map((dimension) => [dimension, LevelSchema]))
|
|
90
|
+
);
|
|
91
|
+
var ReadinessProfileSchema = z.object({
|
|
92
|
+
id: z.string().regex(/^[a-z0-9-]+$/),
|
|
93
|
+
title: z.string().min(1),
|
|
94
|
+
description: z.string().min(1),
|
|
95
|
+
floors: FloorsSchema
|
|
96
|
+
});
|
|
97
|
+
var BenchmarkSchema = z.object({
|
|
98
|
+
id: z.string().min(1),
|
|
99
|
+
version: z.string().regex(/^\d+\.\d+\.\d+$/),
|
|
100
|
+
title: z.string().min(1),
|
|
101
|
+
description: z.string().min(1),
|
|
102
|
+
maturity_levels: z.record(z.string(), z.string()),
|
|
103
|
+
dimensions: z.array(DimensionSchema).length(dimensionIds.length),
|
|
104
|
+
readiness_profiles: z.array(ReadinessProfileSchema).min(1),
|
|
105
|
+
scoring: z.object({
|
|
106
|
+
dimension_method: z.literal("consecutive-levels"),
|
|
107
|
+
overall_method: z.literal("sum"),
|
|
108
|
+
maximum_score: z.literal(40),
|
|
109
|
+
readiness_method: z.literal("non-compensating-floors"),
|
|
110
|
+
attestation_counts_as_verified: z.literal(false)
|
|
111
|
+
})
|
|
112
|
+
});
|
|
113
|
+
var AttestationStatusSchema = z.enum(["met", "not_met", "not_applicable", "unknown"]);
|
|
114
|
+
var AttestationSchema = z.object({
|
|
115
|
+
status: AttestationStatusSchema,
|
|
116
|
+
evidence: z.string().min(1),
|
|
117
|
+
owner: z.string().min(1),
|
|
118
|
+
reviewed_at: z.string().date(),
|
|
119
|
+
expires_at: z.string().date().nullable().default(null)
|
|
120
|
+
});
|
|
121
|
+
var AttestationFileSchema = z.object({
|
|
122
|
+
benchmark_version: z.string().min(1),
|
|
123
|
+
attestations: z.record(z.string(), AttestationSchema).default({})
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
// src/load.ts
|
|
127
|
+
var packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
128
|
+
var defaultBenchmarkRoot = join(packageRoot, "benchmark", "v0.1");
|
|
129
|
+
async function readYaml(path) {
|
|
130
|
+
return parse(await readFile(path, "utf8"));
|
|
131
|
+
}
|
|
132
|
+
async function loadBenchmark(root = defaultBenchmarkRoot) {
|
|
133
|
+
const benchmark = BenchmarkSchema.parse(await readYaml(join(root, "benchmark.yaml")));
|
|
134
|
+
const controlPaths = await fg("controls/*.yaml", { cwd: root, absolute: true, onlyFiles: true });
|
|
135
|
+
const controls = [];
|
|
136
|
+
for (const path of controlPaths.sort()) {
|
|
137
|
+
const file = ControlFileSchema.parse(await readYaml(path));
|
|
138
|
+
controls.push(...file.controls.map((control) => ({ ...control, dimension: file.dimension })));
|
|
139
|
+
}
|
|
140
|
+
validateCatalog(benchmark, controls);
|
|
141
|
+
return { benchmark, controls };
|
|
142
|
+
}
|
|
143
|
+
async function loadAttestations(path, benchmarkVersion) {
|
|
144
|
+
try {
|
|
145
|
+
const file = AttestationFileSchema.parse(await readYaml(path));
|
|
146
|
+
if (file.benchmark_version !== benchmarkVersion) {
|
|
147
|
+
throw new Error(
|
|
148
|
+
`Attestation benchmark version ${file.benchmark_version} does not match ${benchmarkVersion}`
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
return file;
|
|
152
|
+
} catch (error) {
|
|
153
|
+
if (error.code === "ENOENT") return null;
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function validateCatalog(benchmark, controls) {
|
|
158
|
+
const ids = /* @__PURE__ */ new Set();
|
|
159
|
+
for (const control of controls) {
|
|
160
|
+
if (ids.has(control.id)) throw new Error(`Duplicate control id: ${control.id}`);
|
|
161
|
+
ids.add(control.id);
|
|
162
|
+
}
|
|
163
|
+
const benchmarkDimensions = benchmark.dimensions.map(({ id }) => id);
|
|
164
|
+
if (benchmarkDimensions.join(",") !== dimensionIds.join(",")) {
|
|
165
|
+
throw new Error("Benchmark dimensions must use the canonical order and complete dimension set");
|
|
166
|
+
}
|
|
167
|
+
for (const dimension of dimensionIds) {
|
|
168
|
+
for (const level of [1, 2, 3, 4]) {
|
|
169
|
+
const count = controls.filter(
|
|
170
|
+
(control) => control.dimension === dimension && control.level === level
|
|
171
|
+
).length;
|
|
172
|
+
if (count === 0) throw new Error(`Dimension ${dimension} has no level ${level} control`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// src/report.ts
|
|
178
|
+
var statusIcon = {
|
|
179
|
+
met: "PASS",
|
|
180
|
+
not_met: "FAIL",
|
|
181
|
+
unknown: "UNKNOWN",
|
|
182
|
+
not_applicable: "N/A"
|
|
183
|
+
};
|
|
184
|
+
function toMarkdown(report) {
|
|
185
|
+
const target = report.profiles.find(({ id }) => id === report.target.profile);
|
|
186
|
+
const lines = [
|
|
187
|
+
"# Agentic Development Readiness Assessment",
|
|
188
|
+
"",
|
|
189
|
+
`- Benchmark: ${report.benchmark.id} v${report.benchmark.version}`,
|
|
190
|
+
`- Repository: \`${report.target.repository}\``,
|
|
191
|
+
`- Assessed: ${report.assessed_at}`,
|
|
192
|
+
`- Score: **${report.score.total}/${report.score.maximum} (${report.score.percentage}%)**`,
|
|
193
|
+
`- Highest readiness profile: **${report.readiness.highest_profile ?? "none"}**`,
|
|
194
|
+
`- Target \`${report.target.profile}\`: **${report.readiness.target_passed ? "PASS" : "FAIL"}**`,
|
|
195
|
+
`- Evidence: ${report.evidence_summary.verified} verified, ${report.evidence_summary.attested} attested, ${report.evidence_summary.unmet_or_unknown} unmet/unknown`,
|
|
196
|
+
"",
|
|
197
|
+
"## Dimensions",
|
|
198
|
+
"",
|
|
199
|
+
"| Dimension | Score | Controls met |",
|
|
200
|
+
"| --- | ---: | ---: |",
|
|
201
|
+
...report.dimensions.map(
|
|
202
|
+
({ title, score, controls_met: met, controls_total: total }) => `| ${title} | ${score}/4 | ${met}/${total} |`
|
|
203
|
+
)
|
|
204
|
+
];
|
|
205
|
+
if (target && !target.passed) {
|
|
206
|
+
lines.push("", "## Target-profile blockers", "");
|
|
207
|
+
for (const blocker of target.blockers) {
|
|
208
|
+
lines.push(
|
|
209
|
+
`- **${blocker.dimension}:** ${blocker.actual}/4; requires ${blocker.required}/4 (${blocker.control_ids.join(", ") || "lower-level gap"})`
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
lines.push("", "## Controls requiring action", "");
|
|
214
|
+
const actionControls = report.controls.filter(
|
|
215
|
+
({ status }) => status === "not_met" || status === "unknown"
|
|
216
|
+
);
|
|
217
|
+
if (actionControls.length === 0) {
|
|
218
|
+
lines.push("None.");
|
|
219
|
+
} else {
|
|
220
|
+
for (const control of actionControls) {
|
|
221
|
+
lines.push(
|
|
222
|
+
`### ${statusIcon[control.status]} ${control.id} \u2014 ${control.title}`,
|
|
223
|
+
"",
|
|
224
|
+
`**Risk:** ${control.risk}`,
|
|
225
|
+
"",
|
|
226
|
+
`**Improve:** ${control.remediation}`,
|
|
227
|
+
"",
|
|
228
|
+
`Evidence confidence: ${control.confidence}.`,
|
|
229
|
+
""
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
lines.push(
|
|
234
|
+
"## Limitations",
|
|
235
|
+
"",
|
|
236
|
+
...report.limitations.map((limitation) => `- ${limitation}`),
|
|
237
|
+
""
|
|
238
|
+
);
|
|
239
|
+
return lines.join("\n");
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// src/evidence.ts
|
|
243
|
+
import { lstat, readFile as readFile2, realpath, stat } from "fs/promises";
|
|
244
|
+
import { relative, resolve as resolve2, sep } from "path";
|
|
245
|
+
import fg2 from "fast-glob";
|
|
246
|
+
var ignored = ["**/.git/**", "**/node_modules/**", "**/dist/**", "**/coverage/**"];
|
|
247
|
+
var maxContentFileBytes = 512e3;
|
|
248
|
+
var maxContentFiles = 250;
|
|
249
|
+
var maxContentTotalBytes = 5e6;
|
|
250
|
+
async function matches(repo, patterns) {
|
|
251
|
+
return (await fg2(patterns, {
|
|
252
|
+
cwd: repo,
|
|
253
|
+
dot: true,
|
|
254
|
+
onlyFiles: false,
|
|
255
|
+
unique: true,
|
|
256
|
+
followSymbolicLinks: false,
|
|
257
|
+
ignore: ignored
|
|
258
|
+
})).sort();
|
|
259
|
+
}
|
|
260
|
+
async function evaluatePathAny(repo, check) {
|
|
261
|
+
const found = await matches(repo, check.patterns);
|
|
262
|
+
return result(
|
|
263
|
+
check.type,
|
|
264
|
+
found.length > 0 ? "met" : "not_met",
|
|
265
|
+
`${found.length} matching path(s)`,
|
|
266
|
+
found
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
async function evaluatePathAll(repo, check) {
|
|
270
|
+
const groups = await Promise.all(check.patterns.map(async (pattern) => matches(repo, [pattern])));
|
|
271
|
+
const missing = check.patterns.filter((_, index) => groups[index]?.length === 0);
|
|
272
|
+
const found = [...new Set(groups.flat())].sort();
|
|
273
|
+
return result(
|
|
274
|
+
check.type,
|
|
275
|
+
missing.length === 0 ? "met" : "not_met",
|
|
276
|
+
missing.length === 0 ? "Every required path pattern matched" : `Missing patterns: ${missing.join(", ")}`,
|
|
277
|
+
found
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
async function readSearchableFiles(repo, patterns) {
|
|
281
|
+
const root = await realpath(repo);
|
|
282
|
+
const paths = (await matches(repo, patterns)).slice(0, maxContentFiles);
|
|
283
|
+
const files = [];
|
|
284
|
+
let totalBytes = 0;
|
|
285
|
+
for (const path of paths) {
|
|
286
|
+
try {
|
|
287
|
+
const requestedPath = resolve2(root, path);
|
|
288
|
+
if ((await lstat(requestedPath)).isSymbolicLink()) continue;
|
|
289
|
+
const canonicalPath = await realpath(requestedPath);
|
|
290
|
+
if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep}`)) continue;
|
|
291
|
+
const metadata = await stat(canonicalPath);
|
|
292
|
+
if (!metadata.isFile() || metadata.size > maxContentFileBytes || totalBytes + metadata.size > maxContentTotalBytes) {
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
totalBytes += metadata.size;
|
|
296
|
+
files.push({ path, text: (await readFile2(canonicalPath, "utf8")).toLowerCase() });
|
|
297
|
+
} catch {
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return files;
|
|
301
|
+
}
|
|
302
|
+
async function evaluateContent(repo, check) {
|
|
303
|
+
const files = await readSearchableFiles(repo, check.files);
|
|
304
|
+
const matchedNeedles = check.needles.filter(
|
|
305
|
+
(needle) => files.some(({ text }) => text.includes(needle.toLowerCase()))
|
|
306
|
+
);
|
|
307
|
+
const passed = check.type === "content_any" ? matchedNeedles.length > 0 : matchedNeedles.length === check.needles.length;
|
|
308
|
+
return result(
|
|
309
|
+
check.type,
|
|
310
|
+
passed ? "met" : "not_met",
|
|
311
|
+
`Matched ${matchedNeedles.length}/${check.needles.length} required term(s) across ${files.length} file(s)`,
|
|
312
|
+
files.map(({ path }) => path)
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
async function evaluateMaxBytes(repo, check) {
|
|
316
|
+
const paths = await matches(repo, check.patterns);
|
|
317
|
+
const root = await realpath(repo);
|
|
318
|
+
let total = 0;
|
|
319
|
+
let inspected = 0;
|
|
320
|
+
for (const path of paths) {
|
|
321
|
+
const requestedPath = resolve2(root, path);
|
|
322
|
+
if ((await lstat(requestedPath)).isSymbolicLink()) continue;
|
|
323
|
+
const canonicalPath = await realpath(requestedPath);
|
|
324
|
+
if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep}`)) continue;
|
|
325
|
+
total += (await stat(canonicalPath)).size;
|
|
326
|
+
inspected += 1;
|
|
327
|
+
}
|
|
328
|
+
const passed = inspected > 0 && total <= check.max_bytes;
|
|
329
|
+
return result(
|
|
330
|
+
check.type,
|
|
331
|
+
passed ? "met" : "not_met",
|
|
332
|
+
`${total} byte(s) across ${inspected} safe matching file(s); maximum ${check.max_bytes}`,
|
|
333
|
+
paths
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
function result(type, status, summary, references) {
|
|
337
|
+
return { type, status, summary, references };
|
|
338
|
+
}
|
|
339
|
+
async function evaluateCheck(repo, check) {
|
|
340
|
+
switch (check.type) {
|
|
341
|
+
case "path_any":
|
|
342
|
+
return evaluatePathAny(repo, check);
|
|
343
|
+
case "path_all":
|
|
344
|
+
return evaluatePathAll(repo, check);
|
|
345
|
+
case "content_any":
|
|
346
|
+
case "content_all":
|
|
347
|
+
return evaluateContent(repo, check);
|
|
348
|
+
case "max_bytes":
|
|
349
|
+
return evaluateMaxBytes(repo, check);
|
|
350
|
+
case "manual":
|
|
351
|
+
return result("manual", "unknown", check.prompt, []);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
function activeAttestation(control, attestations, now) {
|
|
355
|
+
const attestation = attestations?.attestations[control.id];
|
|
356
|
+
if (!attestation) return null;
|
|
357
|
+
if (attestation.expires_at && new Date(attestation.expires_at) < now) return null;
|
|
358
|
+
if (attestation.status === "not_applicable" && !control.allow_not_applicable) return null;
|
|
359
|
+
return attestation;
|
|
360
|
+
}
|
|
361
|
+
async function evaluateControl(repo, control, attestations, now = /* @__PURE__ */ new Date()) {
|
|
362
|
+
const evidence = await Promise.all(
|
|
363
|
+
control.evidence.map(async (check) => evaluateCheck(repo, check))
|
|
364
|
+
);
|
|
365
|
+
const attestation = activeAttestation(control, attestations, now);
|
|
366
|
+
const checksPassed = evidence.every(({ status: status2 }) => status2 === "met");
|
|
367
|
+
const hasManualCheck = control.evidence.some(({ type }) => type === "manual");
|
|
368
|
+
let status = checksPassed ? "met" : "not_met";
|
|
369
|
+
let confidence = checksPassed && !hasManualCheck ? "verified" : "none";
|
|
370
|
+
if (attestation?.status === "not_applicable") {
|
|
371
|
+
status = "not_applicable";
|
|
372
|
+
confidence = "attested";
|
|
373
|
+
} else if (attestation?.status === "met" && control.allow_attestation) {
|
|
374
|
+
status = "met";
|
|
375
|
+
confidence = checksPassed && !hasManualCheck ? "verified" : "attested";
|
|
376
|
+
} else if (attestation?.status === "not_met" || attestation?.status === "unknown") {
|
|
377
|
+
status = attestation.status;
|
|
378
|
+
confidence = "attested";
|
|
379
|
+
} else if (evidence.some(({ status: checkStatus }) => checkStatus === "unknown")) {
|
|
380
|
+
status = "unknown";
|
|
381
|
+
}
|
|
382
|
+
return {
|
|
383
|
+
id: control.id,
|
|
384
|
+
dimension: control.dimension,
|
|
385
|
+
level: control.level,
|
|
386
|
+
title: control.title,
|
|
387
|
+
outcome: control.outcome,
|
|
388
|
+
risk: control.risk,
|
|
389
|
+
status,
|
|
390
|
+
confidence,
|
|
391
|
+
evidence,
|
|
392
|
+
attestation,
|
|
393
|
+
remediation: control.remediation
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// src/score.ts
|
|
398
|
+
function controlPasses(control) {
|
|
399
|
+
return control.status === "met" || control.status === "not_applicable";
|
|
400
|
+
}
|
|
401
|
+
function dimensionScore(controls, dimension) {
|
|
402
|
+
let score = 0;
|
|
403
|
+
for (const level of [1, 2, 3, 4]) {
|
|
404
|
+
const atLevel = controls.filter(
|
|
405
|
+
(control) => control.dimension === dimension && control.level === level
|
|
406
|
+
);
|
|
407
|
+
if (atLevel.length === 0 || !atLevel.every(controlPasses)) break;
|
|
408
|
+
score = level;
|
|
409
|
+
}
|
|
410
|
+
return score;
|
|
411
|
+
}
|
|
412
|
+
function assessProfiles(benchmark, dimensions, controls) {
|
|
413
|
+
return benchmark.readiness_profiles.map((profile) => {
|
|
414
|
+
const blockers = benchmark.dimensions.flatMap(({ id }) => {
|
|
415
|
+
const actual = dimensions.find((dimension) => dimension.id === id)?.score ?? 0;
|
|
416
|
+
const required = profile.floors[id];
|
|
417
|
+
if (actual >= required) return [];
|
|
418
|
+
return [
|
|
419
|
+
{
|
|
420
|
+
dimension: id,
|
|
421
|
+
actual,
|
|
422
|
+
required,
|
|
423
|
+
control_ids: controls.filter(
|
|
424
|
+
(control) => control.dimension === id && control.level <= required && !controlPasses(control)
|
|
425
|
+
).map(({ id: controlId }) => controlId)
|
|
426
|
+
}
|
|
427
|
+
];
|
|
428
|
+
});
|
|
429
|
+
return { id: profile.id, title: profile.title, passed: blockers.length === 0, blockers };
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
async function assess(repo, benchmark, catalog, profileId, attestations, now = /* @__PURE__ */ new Date()) {
|
|
433
|
+
const profile = benchmark.readiness_profiles.find(({ id }) => id === profileId);
|
|
434
|
+
if (!profile) {
|
|
435
|
+
throw new Error(
|
|
436
|
+
`Unknown profile ${profileId}. Choose: ${benchmark.readiness_profiles.map(({ id }) => id).join(", ")}`
|
|
437
|
+
);
|
|
438
|
+
}
|
|
439
|
+
const controls = await Promise.all(
|
|
440
|
+
catalog.map(async (control) => evaluateControl(repo, control, attestations, now))
|
|
441
|
+
);
|
|
442
|
+
const dimensions = benchmark.dimensions.map(({ id, title }) => {
|
|
443
|
+
const dimensionControls = controls.filter((control) => control.dimension === id);
|
|
444
|
+
return {
|
|
445
|
+
id,
|
|
446
|
+
title,
|
|
447
|
+
score: dimensionScore(controls, id),
|
|
448
|
+
controls_met: dimensionControls.filter(controlPasses).length,
|
|
449
|
+
controls_total: dimensionControls.length
|
|
450
|
+
};
|
|
451
|
+
});
|
|
452
|
+
const profiles = assessProfiles(benchmark, dimensions, controls);
|
|
453
|
+
const total = dimensions.reduce((sum, { score }) => sum + score, 0);
|
|
454
|
+
const highestProfile = [...profiles].reverse().find(({ passed }) => passed)?.id ?? null;
|
|
455
|
+
const targetPassed = profiles.find(({ id }) => id === profileId)?.passed ?? false;
|
|
456
|
+
return {
|
|
457
|
+
schema_version: "0.1.0",
|
|
458
|
+
benchmark: { id: benchmark.id, version: benchmark.version },
|
|
459
|
+
target: { repository: repo, profile: profileId },
|
|
460
|
+
assessed_at: now.toISOString(),
|
|
461
|
+
score: { total, maximum: 40, percentage: Math.round(total / 40 * 100) },
|
|
462
|
+
evidence_summary: {
|
|
463
|
+
verified: controls.filter(
|
|
464
|
+
({ confidence, status }) => confidence === "verified" && status === "met"
|
|
465
|
+
).length,
|
|
466
|
+
attested: controls.filter(
|
|
467
|
+
({ confidence, status }) => confidence === "attested" && status === "met"
|
|
468
|
+
).length,
|
|
469
|
+
unmet_or_unknown: controls.filter(
|
|
470
|
+
({ status }) => status === "not_met" || status === "unknown"
|
|
471
|
+
).length
|
|
472
|
+
},
|
|
473
|
+
dimensions,
|
|
474
|
+
controls,
|
|
475
|
+
profiles,
|
|
476
|
+
readiness: { highest_profile: highestProfile, target_passed: targetPassed },
|
|
477
|
+
limitations: [
|
|
478
|
+
"Repository evidence proves that an artifact exists, not that people consistently follow it.",
|
|
479
|
+
"Self-attestations are reported separately and are not tool-verified evidence.",
|
|
480
|
+
"This assessment does not grant production access, deployment authority, or certification."
|
|
481
|
+
]
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// src/cli.ts
|
|
486
|
+
var program = new Command();
|
|
487
|
+
program.name("agentic-scorecard").description("Evidence-backed readiness assessment for agentic software development harnesses").version("0.1.0");
|
|
488
|
+
program.command("validate").description("Validate the bundled benchmark catalog").action(async () => {
|
|
489
|
+
const { benchmark, controls } = await loadBenchmark();
|
|
490
|
+
process.stdout.write(
|
|
491
|
+
`Valid ${benchmark.id} v${benchmark.version}: ${controls.length} controls across ${benchmark.dimensions.length} dimensions.
|
|
492
|
+
`
|
|
493
|
+
);
|
|
494
|
+
});
|
|
495
|
+
program.command("explain").argument("<control-id>", "ADRB control id").description("Explain one control and its evidence rules").action(async (controlId) => {
|
|
496
|
+
const { controls } = await loadBenchmark();
|
|
497
|
+
const control = controls.find(({ id }) => id === controlId.toUpperCase());
|
|
498
|
+
if (!control) throw new Error(`Unknown control: ${controlId}`);
|
|
499
|
+
process.stdout.write(
|
|
500
|
+
`${control.id} \u2014 ${control.title}
|
|
501
|
+
Dimension: ${control.dimension}; level: ${control.level}
|
|
502
|
+
|
|
503
|
+
Outcome: ${control.outcome}
|
|
504
|
+
Risk: ${control.risk}
|
|
505
|
+
Remediation: ${control.remediation}
|
|
506
|
+
|
|
507
|
+
Evidence:
|
|
508
|
+
${control.evidence.map((check) => `- ${stringify(check).trim().replaceAll("\n", "\n ")}`).join("\n")}
|
|
509
|
+
`
|
|
510
|
+
);
|
|
511
|
+
});
|
|
512
|
+
program.command("init").argument("[repository]", "repository to initialize", ".").option("--force", "replace an existing attestation file", false).description("Create a manual-attestation template").action(async (repository, options) => {
|
|
513
|
+
const repo = resolve3(repository);
|
|
514
|
+
const path = join2(repo, ".agentic", "attestations.yaml");
|
|
515
|
+
if (!options.force) {
|
|
516
|
+
try {
|
|
517
|
+
await readFile3(path, "utf8");
|
|
518
|
+
throw new Error(`${path} already exists; use --force to replace it`);
|
|
519
|
+
} catch (error) {
|
|
520
|
+
if (error.code !== "ENOENT") throw error;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
const { benchmark, controls } = await loadBenchmark();
|
|
524
|
+
const attestations = Object.fromEntries(
|
|
525
|
+
controls.filter((control) => control.evidence.some(({ type }) => type === "manual")).map((control) => {
|
|
526
|
+
const manualCheck = control.evidence.find(
|
|
527
|
+
(check) => check.type === "manual"
|
|
528
|
+
);
|
|
529
|
+
return [
|
|
530
|
+
control.id,
|
|
531
|
+
{
|
|
532
|
+
status: "unknown",
|
|
533
|
+
evidence: `TODO: ${manualCheck?.prompt ?? control.outcome}`,
|
|
534
|
+
owner: "TODO",
|
|
535
|
+
reviewed_at: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
|
|
536
|
+
expires_at: null
|
|
537
|
+
}
|
|
538
|
+
];
|
|
539
|
+
})
|
|
540
|
+
);
|
|
541
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
542
|
+
await writeFile(
|
|
543
|
+
path,
|
|
544
|
+
`# Claims are visible as attested, never tool-verified. Link durable evidence; do not paste secrets.
|
|
545
|
+
${stringify({ benchmark_version: benchmark.version, attestations })}`,
|
|
546
|
+
"utf8"
|
|
547
|
+
);
|
|
548
|
+
process.stdout.write(`Created ${path}
|
|
549
|
+
`);
|
|
550
|
+
});
|
|
551
|
+
program.command("assess").argument("[repository]", "repository to assess", ".").option("--profile <profile>", "target autonomy profile", "pr-creation").option("--format <format>", "json or markdown", "markdown").option("--output <path>", "write the report to a file").option("--attestations <path>", "manual attestation file").option("--enforce", "exit non-zero when the target profile fails", false).option("--github-output", "append summary values to $GITHUB_OUTPUT", false).description("Assess a repository using local, read-only evidence collection").action(
|
|
552
|
+
async (repository, options) => {
|
|
553
|
+
if (!["json", "markdown"].includes(options.format)) {
|
|
554
|
+
throw new Error("--format must be json or markdown");
|
|
555
|
+
}
|
|
556
|
+
const repo = resolve3(repository);
|
|
557
|
+
const { benchmark, controls } = await loadBenchmark();
|
|
558
|
+
const attestationPath = resolve3(
|
|
559
|
+
options.attestations ?? join2(repo, ".agentic", "attestations.yaml")
|
|
560
|
+
);
|
|
561
|
+
const attestations = await loadAttestations(attestationPath, benchmark.version);
|
|
562
|
+
const report = await assess(repo, benchmark, controls, options.profile, attestations);
|
|
563
|
+
const output = options.format === "json" ? `${JSON.stringify(report, null, 2)}
|
|
564
|
+
` : toMarkdown(report);
|
|
565
|
+
let reportPath = null;
|
|
566
|
+
if (options.output) {
|
|
567
|
+
reportPath = resolve3(options.output);
|
|
568
|
+
await mkdir(dirname2(reportPath), { recursive: true });
|
|
569
|
+
await writeFile(reportPath, output, "utf8");
|
|
570
|
+
process.stdout.write(`Wrote ${reportPath}
|
|
571
|
+
`);
|
|
572
|
+
} else {
|
|
573
|
+
process.stdout.write(output);
|
|
574
|
+
}
|
|
575
|
+
if (options.githubOutput) {
|
|
576
|
+
const githubOutput = process.env.GITHUB_OUTPUT;
|
|
577
|
+
if (!githubOutput) throw new Error("$GITHUB_OUTPUT is unavailable");
|
|
578
|
+
await appendFile(
|
|
579
|
+
githubOutput,
|
|
580
|
+
`score=${report.score.total}
|
|
581
|
+
percentage=${report.score.percentage}
|
|
582
|
+
highest_profile=${report.readiness.highest_profile ?? "none"}
|
|
583
|
+
target_passed=${String(report.readiness.target_passed)}
|
|
584
|
+
report_path=${reportPath ?? ""}
|
|
585
|
+
`,
|
|
586
|
+
"utf8"
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
if (options.enforce && !report.readiness.target_passed) process.exitCode = 2;
|
|
590
|
+
}
|
|
591
|
+
);
|
|
592
|
+
await program.parseAsync();
|
package/package.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "agentic-scorecard",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Vendor-neutral, evidence-backed readiness benchmark for agentic software development harnesses.",
|
|
5
|
+
"author": "Planet B2B Inc.",
|
|
6
|
+
"homepage": "https://github.com/Planet-B2B/agentic-readiness#readme",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/Planet-B2B/agentic-readiness.git"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/Planet-B2B/agentic-readiness/issues"
|
|
13
|
+
},
|
|
14
|
+
"type": "module",
|
|
15
|
+
"bin": {
|
|
16
|
+
"agentic-scorecard": "dist/cli.js"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"dist/",
|
|
20
|
+
"benchmark/",
|
|
21
|
+
"templates/",
|
|
22
|
+
"AGENT_PROMPT.md",
|
|
23
|
+
"README.md",
|
|
24
|
+
"LICENSE",
|
|
25
|
+
"NOTICE"
|
|
26
|
+
],
|
|
27
|
+
"scripts": {
|
|
28
|
+
"build": "tsup src/cli.ts --format esm --clean",
|
|
29
|
+
"dev": "tsx src/cli.ts",
|
|
30
|
+
"test": "vitest run --coverage",
|
|
31
|
+
"typecheck": "tsc --noEmit",
|
|
32
|
+
"lint": "eslint .",
|
|
33
|
+
"format": "prettier --write .",
|
|
34
|
+
"format:check": "prettier --check .",
|
|
35
|
+
"validate:benchmark": "tsx src/cli.ts validate",
|
|
36
|
+
"self-assess": "tsx src/cli.ts assess . --profile planning --format markdown",
|
|
37
|
+
"check": "npm run format:check && npm run lint && npm run typecheck && npm test && npm run validate:benchmark && npm run build"
|
|
38
|
+
},
|
|
39
|
+
"engines": {
|
|
40
|
+
"node": "^20.19.0 || ^22.13.0 || >=24"
|
|
41
|
+
},
|
|
42
|
+
"keywords": [
|
|
43
|
+
"agentic-development",
|
|
44
|
+
"ai-governance",
|
|
45
|
+
"benchmark",
|
|
46
|
+
"scorecard",
|
|
47
|
+
"software-engineering"
|
|
48
|
+
],
|
|
49
|
+
"license": "Apache-2.0",
|
|
50
|
+
"publishConfig": {
|
|
51
|
+
"access": "public"
|
|
52
|
+
},
|
|
53
|
+
"dependencies": {
|
|
54
|
+
"commander": "^14.0.0",
|
|
55
|
+
"fast-glob": "^3.3.3",
|
|
56
|
+
"yaml": "^2.8.0",
|
|
57
|
+
"zod": "^3.25.0"
|
|
58
|
+
},
|
|
59
|
+
"devDependencies": {
|
|
60
|
+
"@eslint/js": "^9.31.0",
|
|
61
|
+
"@types/node": "^22.16.0",
|
|
62
|
+
"@vitest/coverage-v8": "^3.2.4",
|
|
63
|
+
"eslint": "^9.31.0",
|
|
64
|
+
"prettier": "^3.6.2",
|
|
65
|
+
"tsup": "^8.5.0",
|
|
66
|
+
"tsx": "^4.20.3",
|
|
67
|
+
"typescript": "^5.8.3",
|
|
68
|
+
"typescript-eslint": "^8.37.0",
|
|
69
|
+
"vitest": "^3.2.4"
|
|
70
|
+
},
|
|
71
|
+
"overrides": {
|
|
72
|
+
"esbuild": "^0.28.1"
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Claims are attested, never tool-verified. Link evidence; do not paste sensitive content.
|
|
2
|
+
benchmark_version: 0.1.0
|
|
3
|
+
attestations:
|
|
4
|
+
ADRB-TOL-003:
|
|
5
|
+
status: unknown
|
|
6
|
+
evidence: TODO link to least-privilege, authorization, audit, and fail-closed evidence
|
|
7
|
+
owner: TODO
|
|
8
|
+
reviewed_at: 2026-01-01
|
|
9
|
+
expires_at: null
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
repository,benchmark_version,target_profile,assessment_date,total_score,highest_profile,verified_controls,attested_controls,owner,report_reference
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Agentic readiness remediation plan
|
|
2
|
+
|
|
3
|
+
- Repository:
|
|
4
|
+
- Benchmark version:
|
|
5
|
+
- Current and target profiles:
|
|
6
|
+
- Assessment date:
|
|
7
|
+
- Accountable owner:
|
|
8
|
+
|
|
9
|
+
| Priority | Control | Gap and evidence | Improvement | Owner | Due | Verification | Status |
|
|
10
|
+
| -------- | -------- | ---------------- | ----------- | ----- | --- | ------------ | ------ |
|
|
11
|
+
| 1 | ADRB-... | | | | | | |
|
|
12
|
+
|
|
13
|
+
## Accepted risks and expiring exceptions
|
|
14
|
+
|
|
15
|
+
## Reassessment date
|