agentic-scorecard 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/AGENT_PROMPT.md +68 -34
  2. package/FEEDBACK.md +53 -0
  3. package/README.md +120 -39
  4. package/benchmark/v0.2/adapter-contract.md +25 -0
  5. package/benchmark/v0.2/agent-evidence-schema.json +52 -0
  6. package/benchmark/v0.2/attestation-schema.json +26 -0
  7. package/benchmark/v0.2/benchmark.yaml +127 -0
  8. package/benchmark/v0.2/controls/context.yaml +51 -0
  9. package/benchmark/v0.2/controls/environment.yaml +70 -0
  10. package/benchmark/v0.2/controls/governance.yaml +60 -0
  11. package/benchmark/v0.2/controls/learning.yaml +48 -0
  12. package/benchmark/v0.2/controls/observability.yaml +76 -0
  13. package/benchmark/v0.2/controls/resilience.yaml +60 -0
  14. package/benchmark/v0.2/controls/security.yaml +71 -0
  15. package/benchmark/v0.2/controls/specification.yaml +62 -0
  16. package/benchmark/v0.2/controls/testing.yaml +60 -0
  17. package/benchmark/v0.2/controls/tooling.yaml +58 -0
  18. package/benchmark/v0.2/report-schema.json +47 -0
  19. package/benchmark/v0.2/scoring-policy.md +71 -0
  20. package/benchmark/v0.3/adapter-contract.md +44 -0
  21. package/benchmark/v0.3/adapters/known-agent-harnesses.yaml +177 -0
  22. package/benchmark/v0.3/adapters/known-python-tooling.yaml +18 -0
  23. package/benchmark/v0.3/adapters/known-security-tools.yaml +6 -0
  24. package/benchmark/v0.3/agent-evidence-schema.json +67 -0
  25. package/benchmark/v0.3/attestation-schema.json +26 -0
  26. package/benchmark/v0.3/benchmark.yaml +127 -0
  27. package/benchmark/v0.3/controls/context.yaml +79 -0
  28. package/benchmark/v0.3/controls/environment.yaml +93 -0
  29. package/benchmark/v0.3/controls/governance.yaml +97 -0
  30. package/benchmark/v0.3/controls/learning.yaml +80 -0
  31. package/benchmark/v0.3/controls/observability.yaml +91 -0
  32. package/benchmark/v0.3/controls/resilience.yaml +107 -0
  33. package/benchmark/v0.3/controls/security.yaml +129 -0
  34. package/benchmark/v0.3/controls/specification.yaml +83 -0
  35. package/benchmark/v0.3/controls/testing.yaml +85 -0
  36. package/benchmark/v0.3/controls/tooling.yaml +69 -0
  37. package/benchmark/v0.3/report-schema.json +89 -0
  38. package/benchmark/v0.3/scoring-policy.md +100 -0
  39. package/dist/cli.js +1043 -111
  40. package/package.json +5 -3
  41. package/templates/agent-evidence.yaml +11 -0
  42. package/templates/attestations.yaml +4 -4
  43. package/templates/baseline.csv +1 -1
package/dist/cli.js CHANGED
@@ -1,8 +1,8 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  // src/cli.ts
4
- import { appendFile, mkdir, readFile as readFile3, writeFile } from "fs/promises";
5
- import { dirname as dirname2, join as join2, resolve as resolve3 } from "path";
4
+ import { appendFile, mkdir, readFile as readFile4, writeFile } from "fs/promises";
5
+ import { dirname as dirname2, join as join3, resolve as resolve4 } from "path";
6
6
  import { Command } from "commander";
7
7
  import { stringify } from "yaml";
8
8
 
@@ -29,31 +29,52 @@ var dimensionIds = [
29
29
  ];
30
30
  var DimensionIdSchema = z.enum(dimensionIds);
31
31
  var LevelSchema = z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3), z.literal(4)]);
32
+ var EvidenceScopeSchema = z.enum(["repository", "platform", "organization", "outcome"]);
33
+ var ManualEvidenceScopeSchema = z.enum(["platform", "organization", "outcome"]);
34
+ var AssessmentScopeSchema = z.enum(["tracked", "workspace"]);
32
35
  var PathAnySchema = z.object({
33
36
  type: z.literal("path_any"),
34
- patterns: z.array(z.string().min(1)).min(1)
37
+ scope: z.literal("repository").default("repository"),
38
+ patterns: z.array(z.string().min(1)).min(1),
39
+ min_bytes: z.number().int().positive().default(1)
35
40
  });
36
41
  var PathAllSchema = z.object({
37
42
  type: z.literal("path_all"),
38
- patterns: z.array(z.string().min(1)).min(1)
43
+ scope: z.literal("repository").default("repository"),
44
+ patterns: z.array(z.string().min(1)).min(1),
45
+ min_bytes: z.number().int().positive().default(1)
39
46
  });
40
47
  var ContentAnySchema = z.object({
41
48
  type: z.literal("content_any"),
49
+ scope: z.literal("repository").default("repository"),
42
50
  files: z.array(z.string().min(1)).min(1),
43
51
  needles: z.array(z.string().min(1)).min(1)
44
52
  });
45
53
  var ContentAllSchema = z.object({
46
54
  type: z.literal("content_all"),
55
+ scope: z.literal("repository").default("repository"),
47
56
  files: z.array(z.string().min(1)).min(1),
48
57
  needles: z.array(z.string().min(1)).min(1)
49
58
  });
59
+ var ContentTermsSchema = z.object({
60
+ type: z.literal("content_terms"),
61
+ scope: z.literal("repository").default("repository"),
62
+ files: z.array(z.string().min(1)).min(1),
63
+ terms: z.array(z.string().min(1)).min(1),
64
+ min_terms: z.number().int().positive(),
65
+ required_any_terms: z.array(z.string().min(1)).min(1).optional(),
66
+ max_span_lines: z.number().int().positive().max(200).optional(),
67
+ max_files_per_pattern: z.number().int().positive().max(250).optional()
68
+ });
50
69
  var MaxBytesSchema = z.object({
51
70
  type: z.literal("max_bytes"),
71
+ scope: z.literal("repository").default("repository"),
52
72
  patterns: z.array(z.string().min(1)).min(1),
53
73
  max_bytes: z.number().int().positive()
54
74
  });
55
75
  var ManualSchema = z.object({
56
76
  type: z.literal("manual"),
77
+ scope: ManualEvidenceScopeSchema.default("organization"),
57
78
  prompt: z.string().min(1)
58
79
  });
59
80
  var EvidenceCheckSchema = z.discriminatedUnion("type", [
@@ -61,6 +82,7 @@ var EvidenceCheckSchema = z.discriminatedUnion("type", [
61
82
  PathAllSchema,
62
83
  ContentAnySchema,
63
84
  ContentAllSchema,
85
+ ContentTermsSchema,
64
86
  MaxBytesSchema,
65
87
  ManualSchema
66
88
  ]);
@@ -73,13 +95,48 @@ var RawControlSchema = z.object({
73
95
  evidence: z.array(EvidenceCheckSchema).min(1),
74
96
  remediation: z.string().min(1),
75
97
  references: z.array(z.string().min(1)).default([]),
76
- allow_attestation: z.boolean().default(true),
77
- allow_not_applicable: z.boolean().default(false)
98
+ allow_attestation: z.boolean().default(false),
99
+ allow_not_applicable: z.boolean().default(false),
100
+ allow_agent_evidence: z.boolean().default(false),
101
+ agent_evidence_scopes: z.array(EvidenceScopeSchema).default([])
102
+ });
103
+ var LegacyRawControlSchema = RawControlSchema.extend({
104
+ allow_attestation: z.boolean().default(true)
78
105
  });
79
106
  var ControlFileSchema = z.object({
80
107
  dimension: DimensionIdSchema,
81
108
  controls: z.array(RawControlSchema).min(1)
82
109
  });
110
+ var LegacyControlFileSchema = z.object({
111
+ dimension: DimensionIdSchema,
112
+ controls: z.array(LegacyRawControlSchema).min(1)
113
+ });
114
+ var DetectorAdapterExtensionSchema = z.object({
115
+ control_id: z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/),
116
+ evidence_index: z.number().int().nonnegative(),
117
+ patterns: z.array(z.string().min(1)).min(1).optional(),
118
+ files: z.array(z.string().min(1)).min(1).optional(),
119
+ terms: z.array(z.string().min(1)).min(1).optional(),
120
+ required_any_terms: z.array(z.string().min(1)).min(1).optional()
121
+ }).strict().superRefine((extension, context) => {
122
+ const extensionKinds = [
123
+ extension.patterns,
124
+ extension.files,
125
+ extension.terms,
126
+ extension.required_any_terms
127
+ ].filter(Boolean).length;
128
+ if (extensionKinds !== 1) {
129
+ context.addIssue({
130
+ code: z.ZodIssueCode.custom,
131
+ message: "A detector extension must declare exactly one of patterns, files, terms, or required_any_terms"
132
+ });
133
+ }
134
+ });
135
+ var DetectorAdapterSchema = z.object({
136
+ id: z.string().regex(/^[a-z0-9-]+$/),
137
+ benchmark_version: z.string().regex(/^\d+\.\d+\.\d+$/),
138
+ extensions: z.array(DetectorAdapterExtensionSchema).min(1)
139
+ }).strict();
83
140
  var DimensionSchema = z.object({
84
141
  id: DimensionIdSchema,
85
142
  title: z.string().min(1),
@@ -116,16 +173,90 @@ var AttestationSchema = z.object({
116
173
  evidence: z.string().min(1),
117
174
  owner: z.string().min(1),
118
175
  reviewed_at: z.string().date(),
176
+ expires_at: z.string().date()
177
+ });
178
+ var LegacyAttestationSchema = AttestationSchema.extend({
119
179
  expires_at: z.string().date().nullable().default(null)
120
180
  });
121
181
  var AttestationFileSchema = z.object({
122
182
  benchmark_version: z.string().min(1),
123
183
  attestations: z.record(z.string(), AttestationSchema).default({})
124
184
  });
185
+ var LegacyAttestationFileSchema = z.object({
186
+ benchmark_version: z.string().min(1),
187
+ attestations: z.record(z.string(), LegacyAttestationSchema).default({})
188
+ });
189
+ var AgentEvidenceClaimSchema = z.object({
190
+ status: z.enum(["met", "not_met", "unknown"]),
191
+ scope: z.enum(["platform", "organization", "outcome"]),
192
+ summary: z.string().min(1),
193
+ references: z.array(z.string().min(1)).min(1),
194
+ collected_at: z.string().datetime(),
195
+ expires_at: z.string().datetime(),
196
+ error: z.string().min(1).nullable().default(null)
197
+ }).strict().superRefine((claim, context) => {
198
+ if (claim.error && claim.status !== "unknown") {
199
+ context.addIssue({
200
+ code: z.ZodIssueCode.custom,
201
+ message: "A claim with an error must have unknown status",
202
+ path: ["status"]
203
+ });
204
+ }
205
+ });
206
+ var AgentEvidenceFileSchema = z.object({
207
+ schema_version: z.literal("0.2.0"),
208
+ benchmark_version: z.literal("0.2.0"),
209
+ target: z.object({
210
+ repository: z.string().min(1),
211
+ git_head: z.string().min(1).nullable()
212
+ }).strict(),
213
+ collector: z.object({
214
+ name: z.string().min(1),
215
+ version: z.string().min(1)
216
+ }).strict(),
217
+ claims: z.record(z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/), AgentEvidenceClaimSchema)
218
+ }).strict();
219
+ var AgentEvidenceClaimV03Schema = z.object({
220
+ status: z.enum(["met", "not_met", "unknown"]),
221
+ scope: EvidenceScopeSchema,
222
+ summary: z.string().min(1),
223
+ references: z.array(z.string().min(1)).min(1),
224
+ collected_at: z.string().datetime(),
225
+ expires_at: z.string().datetime(),
226
+ error: z.string().min(1).nullable().default(null)
227
+ }).strict().superRefine((claim, context) => {
228
+ if (claim.error && claim.status !== "unknown") {
229
+ context.addIssue({
230
+ code: z.ZodIssueCode.custom,
231
+ message: "A claim with an error must have unknown status",
232
+ path: ["status"]
233
+ });
234
+ }
235
+ if (claim.scope === "repository" && claim.references.some((reference) => !reference.startsWith("repo:"))) {
236
+ context.addIssue({
237
+ code: z.ZodIssueCode.custom,
238
+ message: "Repository claims must use repo:<tracked-path>[#Lx-Ly] references",
239
+ path: ["references"]
240
+ });
241
+ }
242
+ });
243
+ var AgentEvidenceFileV03Schema = z.object({
244
+ schema_version: z.literal("0.3.0"),
245
+ benchmark_version: z.literal("0.3.0"),
246
+ target: z.object({
247
+ repository: z.string().min(1),
248
+ git_head: z.string().min(1)
249
+ }).strict(),
250
+ collector: z.object({
251
+ name: z.string().min(1),
252
+ version: z.string().min(1)
253
+ }).strict(),
254
+ claims: z.record(z.string().regex(/^ADRB-[A-Z]{3}-\d{3}$/), AgentEvidenceClaimV03Schema)
255
+ }).strict();
125
256
 
126
257
  // src/load.ts
127
258
  var packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
128
- var defaultBenchmarkRoot = join(packageRoot, "benchmark", "v0.1");
259
+ var defaultBenchmarkRoot = join(packageRoot, "benchmark", "v0.3");
129
260
  async function readYaml(path) {
130
261
  return parse(await readFile(path, "utf8"));
131
262
  }
@@ -134,15 +265,89 @@ async function loadBenchmark(root = defaultBenchmarkRoot) {
134
265
  const controlPaths = await fg("controls/*.yaml", { cwd: root, absolute: true, onlyFiles: true });
135
266
  const controls = [];
136
267
  for (const path of controlPaths.sort()) {
137
- const file = ControlFileSchema.parse(await readYaml(path));
268
+ const file = benchmark.version === "0.1.0" ? LegacyControlFileSchema.parse(await readYaml(path)) : ControlFileSchema.parse(await readYaml(path));
138
269
  controls.push(...file.controls.map((control) => ({ ...control, dimension: file.dimension })));
139
270
  }
271
+ const adapterPaths = await fg("adapters/*.yaml", {
272
+ cwd: root,
273
+ absolute: true,
274
+ onlyFiles: true
275
+ });
276
+ for (const path of adapterPaths.sort()) {
277
+ const adapter = DetectorAdapterSchema.parse(await readYaml(path));
278
+ applyDetectorAdapter(benchmark, controls, adapter);
279
+ }
140
280
  validateCatalog(benchmark, controls);
141
281
  return { benchmark, controls };
142
282
  }
143
- async function loadAttestations(path, benchmarkVersion) {
283
+ function applyDetectorAdapter(benchmark, controls, adapter) {
284
+ if (adapter.benchmark_version !== benchmark.version) {
285
+ throw new Error(
286
+ `Detector adapter ${adapter.id} targets ${adapter.benchmark_version}, not ${benchmark.version}`
287
+ );
288
+ }
289
+ for (const extension of adapter.extensions) {
290
+ const control = controls.find(({ id }) => id === extension.control_id);
291
+ if (!control) {
292
+ throw new Error(`Detector adapter ${adapter.id} references unknown ${extension.control_id}`);
293
+ }
294
+ const check = control.evidence[extension.evidence_index];
295
+ if (!check) {
296
+ throw new Error(
297
+ `Detector adapter ${adapter.id} references missing evidence index ${extension.evidence_index} on ${control.id}`
298
+ );
299
+ }
300
+ if (extension.patterns) {
301
+ if (!("patterns" in check)) {
302
+ throw new Error(
303
+ `Detector adapter ${adapter.id} cannot add patterns to ${control.id} evidence ${extension.evidence_index}`
304
+ );
305
+ }
306
+ check.patterns = [.../* @__PURE__ */ new Set([...check.patterns, ...extension.patterns])];
307
+ }
308
+ if (extension.files) {
309
+ if (!("files" in check)) {
310
+ throw new Error(
311
+ `Detector adapter ${adapter.id} cannot add files to ${control.id} evidence ${extension.evidence_index}`
312
+ );
313
+ }
314
+ check.files = [.../* @__PURE__ */ new Set([...check.files, ...extension.files])];
315
+ }
316
+ if (extension.terms) {
317
+ if (check.type !== "content_terms") {
318
+ throw new Error(
319
+ `Detector adapter ${adapter.id} cannot add terms to ${control.id} evidence ${extension.evidence_index}`
320
+ );
321
+ }
322
+ check.terms = [.../* @__PURE__ */ new Set([...check.terms, ...extension.terms])];
323
+ }
324
+ if (extension.required_any_terms) {
325
+ if (check.type !== "content_terms") {
326
+ throw new Error(
327
+ `Detector adapter ${adapter.id} cannot add required terms to ${control.id} evidence ${extension.evidence_index}`
328
+ );
329
+ }
330
+ check.required_any_terms = [
331
+ .../* @__PURE__ */ new Set([...check.required_any_terms ?? [], ...extension.required_any_terms])
332
+ ];
333
+ }
334
+ }
335
+ }
336
+ async function loadAttestations(path, benchmarkVersion, options = {}) {
144
337
  try {
145
- const file = AttestationFileSchema.parse(await readYaml(path));
338
+ const rawFile = await readYaml(path);
339
+ const artifactVersion = versionField(rawFile, "benchmark_version");
340
+ if (artifactVersion && handleVersionMismatch(
341
+ "Attestation",
342
+ path,
343
+ artifactVersion,
344
+ benchmarkVersion,
345
+ "init --force",
346
+ options
347
+ )) {
348
+ return null;
349
+ }
350
+ const file = benchmarkVersion === "0.1.0" ? LegacyAttestationFileSchema.parse(rawFile) : AttestationFileSchema.parse(rawFile);
146
351
  if (file.benchmark_version !== benchmarkVersion) {
147
352
  throw new Error(
148
353
  `Attestation benchmark version ${file.benchmark_version} does not match ${benchmarkVersion}`
@@ -154,11 +359,87 @@ async function loadAttestations(path, benchmarkVersion) {
154
359
  throw error;
155
360
  }
156
361
  }
362
+ async function loadAgentEvidence(path, benchmarkVersion, options = {}) {
363
+ try {
364
+ const rawFile = await readYaml(path);
365
+ const artifactVersion = versionField(rawFile, "benchmark_version");
366
+ const schemaVersion = versionField(rawFile, "schema_version");
367
+ const mismatchedVersion = [artifactVersion, schemaVersion].find(
368
+ (version) => version && version !== benchmarkVersion
369
+ );
370
+ if (mismatchedVersion && handleVersionMismatch(
371
+ "Agent evidence",
372
+ path,
373
+ mismatchedVersion,
374
+ benchmarkVersion,
375
+ "init-evidence --force",
376
+ options
377
+ )) {
378
+ return null;
379
+ }
380
+ const file = (() => {
381
+ if (benchmarkVersion === "0.2.0") return AgentEvidenceFileSchema.parse(rawFile);
382
+ if (benchmarkVersion === "0.3.0") return AgentEvidenceFileV03Schema.parse(rawFile);
383
+ throw new Error(`Agent evidence bundles are unsupported for benchmark ${benchmarkVersion}`);
384
+ })();
385
+ if (file.benchmark_version !== benchmarkVersion) {
386
+ throw new Error(
387
+ `Agent evidence benchmark version ${file.benchmark_version} does not match ${benchmarkVersion}`
388
+ );
389
+ }
390
+ return file;
391
+ } catch (error) {
392
+ if (error.code === "ENOENT") return null;
393
+ throw error;
394
+ }
395
+ }
396
+ function versionField(value, field) {
397
+ if (!value || typeof value !== "object" || !(field in value)) return null;
398
+ const version = value[field];
399
+ return typeof version === "string" ? version : null;
400
+ }
401
+ function handleVersionMismatch(label, path, artifactVersion, benchmarkVersion, regenerateCommand, options) {
402
+ if (artifactVersion === benchmarkVersion) return false;
403
+ const mismatch = `${label} file ${path} targets ADRB v${artifactVersion}, not v${benchmarkVersion}`;
404
+ if (!options.ignoreVersionMismatch) {
405
+ throw new Error(
406
+ `${mismatch}. Regenerate it with \`agentic-scorecard ${regenerateCommand}\` or pass a v${benchmarkVersion} file.`
407
+ );
408
+ }
409
+ options.onWarning?.(
410
+ `Ignored auto-loaded ${label.toLowerCase()} file ${path} because it targets ADRB v${artifactVersion}, not v${benchmarkVersion}. Regenerate it with \`agentic-scorecard ${regenerateCommand}\` before relying on its claims.`
411
+ );
412
+ return true;
413
+ }
157
414
  function validateCatalog(benchmark, controls) {
158
415
  const ids = /* @__PURE__ */ new Set();
159
416
  for (const control of controls) {
160
417
  if (ids.has(control.id)) throw new Error(`Duplicate control id: ${control.id}`);
161
418
  ids.add(control.id);
419
+ if (["0.2.0", "0.3.0"].includes(benchmark.version) && control.evidence.some(({ type }) => type === "content_any" || type === "content_all")) {
420
+ throw new Error(
421
+ `${control.id} uses a legacy broad content collector in benchmark ${benchmark.version}`
422
+ );
423
+ }
424
+ const manualScopes = control.evidence.filter(({ type }) => type === "manual").map(({ scope }) => scope);
425
+ const allowedAgentScopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : manualScopes;
426
+ if (control.allow_agent_evidence && allowedAgentScopes.length === 0) {
427
+ throw new Error(`${control.id} allows agent evidence without an eligible evidence scope`);
428
+ }
429
+ if (!control.allow_agent_evidence && control.agent_evidence_scopes.length > 0) {
430
+ throw new Error(`${control.id} declares agent evidence scopes but does not allow it`);
431
+ }
432
+ if (benchmark.version === "0.2.0" && control.agent_evidence_scopes.some((scope) => scope === "repository")) {
433
+ throw new Error(`${control.id} changes immutable v0.2 repository evidence semantics`);
434
+ }
435
+ if (["0.2.0", "0.3.0"].includes(benchmark.version) && control.allow_attestation && !control.evidence.some(({ type }) => type === "manual")) {
436
+ throw new Error(`${control.id} allows attestation for repository-detected evidence`);
437
+ }
438
+ for (const check of control.evidence) {
439
+ if (check.type === "content_terms" && check.min_terms > check.terms.length) {
440
+ throw new Error(`${control.id} requires more content terms than it defines`);
441
+ }
442
+ }
162
443
  }
163
444
  const benchmarkDimensions = benchmark.dimensions.map(({ id }) => id);
164
445
  if (benchmarkDimensions.join(",") !== dimensionIds.join(",")) {
@@ -181,26 +462,114 @@ var statusIcon = {
181
462
  unknown: "UNKNOWN",
182
463
  not_applicable: "N/A"
183
464
  };
465
+ function controlScope(control) {
466
+ return control.evidence.find(({ scope }) => scope !== "repository")?.scope ?? "repository";
467
+ }
468
+ function safeText(value) {
469
+ return value.replace(/\s+/g, " ").replace(/([\\`*_[\]<>|])/g, "\\$1");
470
+ }
471
+ function sentence(value) {
472
+ const safe = safeText(value);
473
+ return /[.!?]$/.test(safe) ? safe : `${safe}.`;
474
+ }
475
+ function references(values) {
476
+ return values.map((value) => `\`${value.replaceAll("`", "'").replace(/\s+/g, " ")}\``).join(", ");
477
+ }
478
+ function evidenceLines(control) {
479
+ const lines = control.evidence.flatMap((evidence) => {
480
+ const evidenceReferences = evidence.references.length > 0 ? ` References: ${references(evidence.references)}.` : "";
481
+ return `- **${evidence.scope}/${evidence.type}:** ${evidence.status} \u2014 ${sentence(evidence.summary)}${evidenceReferences}`;
482
+ });
483
+ if (control.agent_evidence) {
484
+ lines.push(
485
+ `- **Agent-collected:** ${control.agent_evidence.status} \u2014 ${sentence(control.agent_evidence.summary)} References: ${references(control.agent_evidence.references)}.`
486
+ );
487
+ }
488
+ if (control.attestation) {
489
+ lines.push(
490
+ `- **Human-attested:** ${control.attestation.status} \u2014 ${references([control.attestation.evidence])} (owner: ${safeText(control.attestation.owner)}; reviewed: ${control.attestation.reviewed_at}).`
491
+ );
492
+ }
493
+ if (control.agent_evidence && control.attestation && control.agent_evidence.status !== "unknown" && control.attestation.status !== "unknown" && control.agent_evidence.status !== control.attestation.status) {
494
+ lines.push(
495
+ "- **Conflict:** agent-collected and human-attested statuses disagree; fail closed."
496
+ );
497
+ }
498
+ return lines;
499
+ }
500
+ function appendControlDetails(lines, heading, controls) {
501
+ lines.push("", `## ${heading}`, "");
502
+ if (controls.length === 0) {
503
+ lines.push("None.");
504
+ return;
505
+ }
506
+ for (const control of controls) {
507
+ lines.push(
508
+ `### ${statusIcon[control.status]} ${control.id} \u2014 ${control.title}`,
509
+ "",
510
+ `**Risk:** ${control.risk}`,
511
+ "",
512
+ `**Improve:** ${control.remediation}`,
513
+ "",
514
+ `Evidence confidence: ${control.confidence}.`,
515
+ "",
516
+ ...evidenceLines(control),
517
+ ""
518
+ );
519
+ }
520
+ }
184
521
  function toMarkdown(report) {
185
522
  const target = report.profiles.find(({ id }) => id === report.target.profile);
523
+ const targetDependencies = target?.evidence_dependencies;
524
+ const dependencyCount = (targetDependencies?.agent_collected ?? 0) + (targetDependencies?.attested ?? 0);
525
+ const targetProvenance = target?.passed && dependencyCount > 0 ? ` (depends on ${[
526
+ targetDependencies?.agent_collected ? `${targetDependencies.agent_collected} agent-collected` : null,
527
+ targetDependencies?.attested ? `${targetDependencies.attested} human-attested` : null
528
+ ].filter(Boolean).join(" and ")} required ${dependencyCount === 1 ? "control" : "controls"})` : "";
529
+ const established = report.controls.filter(
530
+ ({ status }) => status === "met" || status === "not_applicable"
531
+ );
532
+ const unresolved = report.controls.filter(
533
+ ({ status }) => status === "not_met" || status === "unknown"
534
+ );
535
+ const repositoryGaps = unresolved.filter((control) => controlScope(control) === "repository");
536
+ const externalControls = unresolved.filter(
537
+ (control) => ["platform", "organization"].includes(controlScope(control))
538
+ );
539
+ const outcomeControls = unresolved.filter((control) => controlScope(control) === "outcome");
186
540
  const lines = [
187
541
  "# Agentic Development Readiness Assessment",
188
542
  "",
189
543
  `- Benchmark: ${report.benchmark.id} v${report.benchmark.version}`,
190
544
  `- Repository: \`${report.target.repository}\``,
545
+ `- Assessment scope: **${report.target.scope}**`,
546
+ `- Git commit: ${report.target.git_head ? `\`${report.target.git_head}\`` : "unavailable"}`,
547
+ `- Working tree dirty: ${report.target.working_tree_dirty === null ? "unknown" : String(report.target.working_tree_dirty)}`,
191
548
  `- Assessed: ${report.assessed_at}`,
192
549
  `- Score: **${report.score.total}/${report.score.maximum} (${report.score.percentage}%)**`,
550
+ ...report.score.repository ? [
551
+ `- Repository-detected progress: **${report.score.repository.achieved}/${report.score.repository.ceiling} (${report.score.repository.percentage}%)** of the maturity levels the offline repository collector can establish`
552
+ ] : [],
193
553
  `- Highest readiness profile: **${report.readiness.highest_profile ?? "none"}**`,
194
- `- Target \`${report.target.profile}\`: **${report.readiness.target_passed ? "PASS" : "FAIL"}**`,
195
- `- Evidence: ${report.evidence_summary.verified} verified, ${report.evidence_summary.attested} attested, ${report.evidence_summary.unmet_or_unknown} unmet/unknown`,
554
+ `- Target \`${report.target.profile}\`: **${report.readiness.target_passed ? `PASS${targetProvenance}` : "FAIL"}**`,
555
+ `- Evidence: ${report.evidence_summary.repository_detected} repository-detected, ${report.evidence_summary.agent_collected} agent-collected, ${report.evidence_summary.attested} human-attested, ${report.evidence_summary.unmet} unmet, ${report.evidence_summary.unknown} unknown${report.evidence_summary.resolved !== void 0 && report.evidence_summary.total !== void 0 ? `; ${report.evidence_summary.resolved}/${report.evidence_summary.total} controls resolved` : ""}`,
556
+ ...report.warnings && report.warnings.length > 0 ? [`- Warnings: **${report.warnings.length} \u2014 review before using this assessment**`] : [],
196
557
  "",
558
+ ...report.warnings && report.warnings.length > 0 ? [
559
+ "## Warnings",
560
+ "",
561
+ ...report.warnings.map((warning) => `- WARNING: ${safeText(warning)}`),
562
+ ""
563
+ ] : [],
197
564
  "## Dimensions",
198
565
  "",
199
566
  "| Dimension | Score | Controls met |",
200
567
  "| --- | ---: | ---: |",
201
568
  ...report.dimensions.map(
202
569
  ({ title, score, controls_met: met, controls_total: total }) => `| ${title} | ${score}/4 | ${met}/${total} |`
203
- )
570
+ ),
571
+ "",
572
+ "> A dimension earns only consecutive levels. Controls met above the first gap remain visible but do not increase its score."
204
573
  ];
205
574
  if (target && !target.passed) {
206
575
  lines.push("", "## Target-profile blockers", "");
@@ -210,27 +579,22 @@ function toMarkdown(report) {
210
579
  );
211
580
  }
212
581
  }
213
- lines.push("", "## Controls requiring action", "");
214
- const actionControls = report.controls.filter(
215
- ({ status }) => status === "not_met" || status === "unknown"
216
- );
217
- if (actionControls.length === 0) {
582
+ lines.push("", "## Established controls", "");
583
+ if (established.length === 0) {
218
584
  lines.push("None.");
219
585
  } else {
220
- for (const control of actionControls) {
586
+ lines.push("| Control | Level | Evidence scope | Confidence |", "| --- | ---: | --- | --- |");
587
+ for (const control of established) {
221
588
  lines.push(
222
- `### ${statusIcon[control.status]} ${control.id} \u2014 ${control.title}`,
223
- "",
224
- `**Risk:** ${control.risk}`,
225
- "",
226
- `**Improve:** ${control.remediation}`,
227
- "",
228
- `Evidence confidence: ${control.confidence}.`,
229
- ""
589
+ `| ${control.id} \u2014 ${control.title.replaceAll("|", "\\|")} | ${control.level} | ${controlScope(control)} | ${control.confidence} |`
230
590
  );
231
591
  }
232
592
  }
593
+ appendControlDetails(lines, "Repository evidence gaps", repositoryGaps);
594
+ appendControlDetails(lines, "External controls not established", externalControls);
595
+ appendControlDetails(lines, "Outcome evidence not established", outcomeControls);
233
596
  lines.push(
597
+ "",
234
598
  "## Limitations",
235
599
  "",
236
600
  ...report.limitations.map((limitation) => `- ${limitation}`),
@@ -239,143 +603,401 @@ function toMarkdown(report) {
239
603
  return lines.join("\n");
240
604
  }
241
605
 
606
+ // src/repository.ts
607
+ import { execFile } from "child_process";
608
+ import { realpath } from "fs/promises";
609
+ import { isAbsolute, relative, resolve as resolve2, sep } from "path";
610
+ import { promisify } from "util";
611
+ var execFileAsync = promisify(execFile);
612
+ var generatedEvidenceIgnores = [
613
+ "**/.git/**",
614
+ "**/node_modules/**",
615
+ "**/dist/**",
616
+ "**/coverage/**",
617
+ "**/.agentic/reports/**",
618
+ "**/.agentic/report*.json",
619
+ "**/.agentic/agentic-readiness*.json",
620
+ "**/.agentic/agentic-readiness*.md",
621
+ "**/.agentic/attestations.*",
622
+ "**/.agentic/agent-evidence.*",
623
+ "**/.agentic/evidence-request.*"
624
+ ];
625
+ function relativePathWithin(root, path) {
626
+ const candidate = relative(root, path);
627
+ if (candidate === "") return candidate;
628
+ if (candidate === ".." || candidate.startsWith(`..${sep}`) || isAbsolute(candidate)) {
629
+ return null;
630
+ }
631
+ return candidate;
632
+ }
633
+ function normalizeExcludedPath(path, requestedRoot, canonicalRoot) {
634
+ const absolute = isAbsolute(path) ? resolve2(path) : resolve2(requestedRoot, path);
635
+ return relativePathWithin(requestedRoot, absolute) ?? relativePathWithin(canonicalRoot, absolute);
636
+ }
637
+ function sanitizeRemote(remote) {
638
+ if (!remote) return null;
639
+ try {
640
+ const url = new URL(remote);
641
+ if (url.protocol === "http:" || url.protocol === "https:") {
642
+ url.username = "";
643
+ url.password = "";
644
+ } else if (url.password) {
645
+ url.password = "";
646
+ }
647
+ return url.toString();
648
+ } catch {
649
+ return remote;
650
+ }
651
+ }
652
+ async function git(repo, args) {
653
+ try {
654
+ const { stdout } = await execFileAsync("git", ["-C", repo, ...args], {
655
+ encoding: "utf8",
656
+ maxBuffer: 1e7
657
+ });
658
+ return stdout;
659
+ } catch {
660
+ return null;
661
+ }
662
+ }
663
+ async function createRepositoryContext(repository, scope, excludedPaths = []) {
664
+ const requestedRoot = resolve2(repository);
665
+ const root = await realpath(requestedRoot);
666
+ const [headOutput, remoteOutput, statusOutput, trackedStatusOutput] = await Promise.all([
667
+ git(root, ["rev-parse", "HEAD"]),
668
+ git(root, ["config", "--get", "remote.origin.url"]),
669
+ git(root, ["status", "--porcelain"]),
670
+ git(root, ["status", "--porcelain", "--untracked-files=no"])
671
+ ]);
672
+ let includedPaths = null;
673
+ if (scope === "tracked") {
674
+ const tracked = await git(root, ["ls-files", "-z", "--cached"]);
675
+ if (tracked === null) {
676
+ throw new Error(
677
+ "Tracked assessment requires a Git worktree. Use --scope workspace for a provisional filesystem assessment."
678
+ );
679
+ }
680
+ includedPaths = new Set(tracked.split("\0").filter((path) => path.length > 0));
681
+ }
682
+ return {
683
+ metadata: {
684
+ root,
685
+ scope,
686
+ git_head: headOutput?.trim() || null,
687
+ git_remote: sanitizeRemote(remoteOutput?.trim() || null),
688
+ working_tree_dirty: statusOutput === null ? null : statusOutput.length > 0,
689
+ tracked_tree_dirty: trackedStatusOutput === null ? null : trackedStatusOutput.length > 0
690
+ },
691
+ includedPaths,
692
+ excludedPaths: new Set(
693
+ excludedPaths.flatMap((path) => {
694
+ const normalized = normalizeExcludedPath(path, requestedRoot, root);
695
+ return normalized === null ? [] : [normalized];
696
+ })
697
+ )
698
+ };
699
+ }
700
+ function repositoryEvidenceTarget(metadata) {
701
+ return {
702
+ repository: metadata.git_remote ?? metadata.root,
703
+ git_head: metadata.git_head
704
+ };
705
+ }
706
+
707
+ // src/score.ts
708
+ import { readFile as readFile3 } from "fs/promises";
709
+ import { join as join2 } from "path";
710
+
242
711
  // src/evidence.ts
243
- import { lstat, readFile as readFile2, realpath, stat } from "fs/promises";
244
- import { relative, resolve as resolve2, sep } from "path";
712
+ import { lstat, readFile as readFile2, realpath as realpath2, stat } from "fs/promises";
713
+ import { resolve as resolve3, sep as sep2 } from "path";
245
714
  import fg2 from "fast-glob";
246
- var ignored = ["**/.git/**", "**/node_modules/**", "**/dist/**", "**/coverage/**"];
247
715
  var maxContentFileBytes = 512e3;
248
716
  var maxContentFiles = 250;
249
717
  var maxContentTotalBytes = 5e6;
250
- async function matches(repo, patterns) {
251
- return (await fg2(patterns, {
252
- cwd: repo,
718
+ async function matches(context, patterns) {
719
+ const found = await fg2(patterns, {
720
+ cwd: context.metadata.root,
253
721
  dot: true,
254
- onlyFiles: false,
722
+ onlyFiles: true,
255
723
  unique: true,
256
724
  followSymbolicLinks: false,
257
- ignore: ignored
258
- })).sort();
725
+ ignore: generatedEvidenceIgnores
726
+ });
727
+ return found.filter((path) => context.includedPaths === null || context.includedPaths.has(path)).filter((path) => !context.excludedPaths.has(path)).sort();
728
+ }
729
+ async function safeFileSize(root, path) {
730
+ try {
731
+ const requestedPath = resolve3(root, path);
732
+ if ((await lstat(requestedPath)).isSymbolicLink()) return null;
733
+ const canonicalPath = await realpath2(requestedPath);
734
+ if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep2}`)) return null;
735
+ const metadata = await stat(canonicalPath);
736
+ return metadata.isFile() ? metadata.size : null;
737
+ } catch {
738
+ return null;
739
+ }
259
740
  }
260
- async function evaluatePathAny(repo, check) {
261
- const found = await matches(repo, check.patterns);
741
+ async function nonEmptyMatches(context, patterns, minBytes) {
742
+ const found = await matches(context, patterns);
743
+ const qualifying = await Promise.all(
744
+ found.map(async (path) => ({ path, size: await safeFileSize(context.metadata.root, path) }))
745
+ );
746
+ return qualifying.filter(({ size }) => size !== null && size >= minBytes).map(({ path }) => path);
747
+ }
748
+ async function evaluatePathAny(context, check) {
749
+ const found = await nonEmptyMatches(context, check.patterns, check.min_bytes);
262
750
  return result(
263
751
  check.type,
752
+ check.scope,
264
753
  found.length > 0 ? "met" : "not_met",
265
- `${found.length} matching path(s)`,
754
+ `${found.length} safe, non-empty matching file(s)`,
266
755
  found
267
756
  );
268
757
  }
269
- async function evaluatePathAll(repo, check) {
270
- const groups = await Promise.all(check.patterns.map(async (pattern) => matches(repo, [pattern])));
758
+ async function evaluatePathAll(context, check) {
759
+ const groups = await Promise.all(
760
+ check.patterns.map(async (pattern) => nonEmptyMatches(context, [pattern], check.min_bytes))
761
+ );
271
762
  const missing = check.patterns.filter((_, index) => groups[index]?.length === 0);
272
763
  const found = [...new Set(groups.flat())].sort();
273
764
  return result(
274
765
  check.type,
766
+ check.scope,
275
767
  missing.length === 0 ? "met" : "not_met",
276
- missing.length === 0 ? "Every required path pattern matched" : `Missing patterns: ${missing.join(", ")}`,
768
+ missing.length === 0 ? "Every required pattern matched a safe, non-empty file" : `Missing non-empty patterns: ${missing.join(", ")}`,
277
769
  found
278
770
  );
279
771
  }
280
- async function readSearchableFiles(repo, patterns) {
281
- const root = await realpath(repo);
282
- const paths = (await matches(repo, patterns)).slice(0, maxContentFiles);
772
+ async function readSearchableFiles(context, patterns, maxFilesPerPattern) {
773
+ const root = context.metadata.root;
774
+ const paths = maxFilesPerPattern ? await prioritizedMatches(context, patterns, maxFilesPerPattern) : (await matches(context, patterns)).slice(0, maxContentFiles);
283
775
  const files = [];
284
776
  let totalBytes = 0;
285
777
  for (const path of paths) {
286
778
  try {
287
- const requestedPath = resolve2(root, path);
779
+ const requestedPath = resolve3(root, path);
288
780
  if ((await lstat(requestedPath)).isSymbolicLink()) continue;
289
- const canonicalPath = await realpath(requestedPath);
290
- if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep}`)) continue;
781
+ const canonicalPath = await realpath2(requestedPath);
782
+ if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep2}`)) continue;
291
783
  const metadata = await stat(canonicalPath);
292
- if (!metadata.isFile() || metadata.size > maxContentFileBytes || totalBytes + metadata.size > maxContentTotalBytes) {
784
+ if (!metadata.isFile() || metadata.size === 0 || metadata.size > maxContentFileBytes || totalBytes + metadata.size > maxContentTotalBytes) {
293
785
  continue;
294
786
  }
787
+ const text = (await readFile2(canonicalPath, "utf8")).toLowerCase();
788
+ if (isGeneratedAssessment(text)) continue;
295
789
  totalBytes += metadata.size;
296
- files.push({ path, text: (await readFile2(canonicalPath, "utf8")).toLowerCase() });
790
+ files.push({ path, text });
297
791
  } catch {
298
792
  }
299
793
  }
300
794
  return files;
301
795
  }
302
- async function evaluateContent(repo, check) {
303
- const files = await readSearchableFiles(repo, check.files);
796
+ async function prioritizedMatches(context, patterns, maxFilesPerPattern) {
797
+ const selected = [];
798
+ const seen = /* @__PURE__ */ new Set();
799
+ const groups = await Promise.all(
800
+ patterns.map(
801
+ async (pattern) => (await matches(context, [pattern])).slice(0, maxFilesPerPattern)
802
+ )
803
+ );
804
+ for (let candidateIndex = 0; candidateIndex < maxFilesPerPattern; candidateIndex += 1) {
805
+ for (const group of groups) {
806
+ const path = group[candidateIndex];
807
+ if (!path) continue;
808
+ if (seen.has(path)) continue;
809
+ seen.add(path);
810
+ selected.push(path);
811
+ if (selected.length === maxContentFiles) return selected;
812
+ }
813
+ }
814
+ return selected;
815
+ }
816
+ function isGeneratedAssessment(text) {
817
+ const normalized = text.trimStart();
818
+ if (normalized.startsWith("# agentic development readiness assessment")) return true;
819
+ if (!normalized.startsWith("{")) return false;
820
+ try {
821
+ const candidate = JSON.parse(normalized);
822
+ if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) return false;
823
+ const report = candidate;
824
+ const benchmark = report.benchmark;
825
+ return Boolean(benchmark) && typeof benchmark === "object" && !Array.isArray(benchmark) && benchmark.id === "agentic-development-readiness" && typeof report.assessed_at === "string" && Array.isArray(report.controls);
826
+ } catch {
827
+ return false;
828
+ }
829
+ }
830
+ async function evaluateLegacyContent(context, check) {
831
+ const files = await readSearchableFiles(context, check.files);
832
+ const matchingFiles = files.filter(
833
+ ({ text }) => check.needles.some((needle) => text.includes(needle.toLowerCase()))
834
+ );
304
835
  const matchedNeedles = check.needles.filter(
305
836
  (needle) => files.some(({ text }) => text.includes(needle.toLowerCase()))
306
837
  );
307
838
  const passed = check.type === "content_any" ? matchedNeedles.length > 0 : matchedNeedles.length === check.needles.length;
308
839
  return result(
309
840
  check.type,
841
+ check.scope,
310
842
  passed ? "met" : "not_met",
311
- `Matched ${matchedNeedles.length}/${check.needles.length} required term(s) across ${files.length} file(s)`,
312
- files.map(({ path }) => path)
843
+ `Matched ${matchedNeedles.length}/${check.needles.length} term(s) across ${files.length} candidate file(s)`,
844
+ matchingFiles.map(({ path }) => path)
845
+ );
846
+ }
847
+ async function evaluateContentTerms(context, check) {
848
+ const files = await readSearchableFiles(context, check.files, check.max_files_per_pattern);
849
+ const matchesByFile = files.map(({ path, text }) => ({
850
+ path,
851
+ ...strongestContentMatch(
852
+ text,
853
+ check.terms,
854
+ check.required_any_terms ?? [],
855
+ check.min_terms,
856
+ check.max_span_lines
857
+ )
858
+ }));
859
+ const qualifying = matchesByFile.filter(({ qualifies }) => qualifies);
860
+ const strongest = matchesByFile.reduce((maximum, file) => Math.max(maximum, file.matched), 0);
861
+ const strongestRequired = matchesByFile.reduce(
862
+ (maximum, file) => Math.max(maximum, file.requiredMatched),
863
+ 0
313
864
  );
865
+ const requiredSummary = check.required_any_terms ? `; strongest required match ${strongestRequired}/${check.required_any_terms.length}` : "";
866
+ const proximitySummary = check.max_span_lines ? ` within ${check.max_span_lines}-line window(s)` : "";
867
+ return result(
868
+ check.type,
869
+ check.scope,
870
+ qualifying.length > 0 ? "met" : "not_met",
871
+ `${qualifying.length} qualifying file(s); strongest co-located match ${strongest}/${check.terms.length} term(s)${requiredSummary}${proximitySummary} across ${files.length} candidate file(s); threshold ${check.min_terms}`,
872
+ qualifying.map(({ path }) => path)
873
+ );
874
+ }
875
+ function strongestContentMatch(text, terms, requiredTerms, minTerms, maxSpanLines) {
876
+ if (!maxSpanLines) {
877
+ const matched = terms.filter((term) => containsTerm(text, term)).length;
878
+ const requiredMatched = requiredTerms.filter((term) => containsTerm(text, term)).length;
879
+ return {
880
+ matched,
881
+ requiredMatched,
882
+ qualifies: matched >= minTerms && (requiredTerms.length === 0 || requiredMatched > 0)
883
+ };
884
+ }
885
+ const termCounts = terms.map(() => 0);
886
+ const requiredCounts = requiredTerms.map(() => 0);
887
+ const lines = text.split(/\r?\n/);
888
+ let strongest = 0;
889
+ let strongestRequired = 0;
890
+ let qualifies = false;
891
+ const update = (line, direction) => {
892
+ terms.forEach((term, index) => {
893
+ if (containsTerm(line, term)) termCounts[index] = (termCounts[index] ?? 0) + direction;
894
+ });
895
+ requiredTerms.forEach((term, index) => {
896
+ if (containsTerm(line, term)) {
897
+ requiredCounts[index] = (requiredCounts[index] ?? 0) + direction;
898
+ }
899
+ });
900
+ };
901
+ for (let index = 0; index < lines.length; index += 1) {
902
+ update(lines[index] ?? "", 1);
903
+ if (index >= maxSpanLines) update(lines[index - maxSpanLines] ?? "", -1);
904
+ const matched = termCounts.filter((count) => count > 0).length;
905
+ const requiredMatched = requiredCounts.filter((count) => count > 0).length;
906
+ strongest = Math.max(strongest, matched);
907
+ strongestRequired = Math.max(strongestRequired, requiredMatched);
908
+ if (matched >= minTerms && (requiredTerms.length === 0 || requiredMatched > 0)) {
909
+ qualifies = true;
910
+ }
911
+ }
912
+ return { matched: strongest, requiredMatched: strongestRequired, qualifies };
314
913
  }
315
- async function evaluateMaxBytes(repo, check) {
316
- const paths = await matches(repo, check.patterns);
317
- const root = await realpath(repo);
914
+ function containsTerm(text, term) {
915
+ const pattern = term.trim().split(/\s+/).map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("[\\s_-]+");
916
+ return new RegExp(`(^|[^a-z0-9])${pattern}(?=$|[^a-z0-9])`, "i").test(text);
917
+ }
918
+ async function evaluateMaxBytes(context, check) {
919
+ const paths = await matches(context, check.patterns);
318
920
  let total = 0;
319
- let inspected = 0;
921
+ const inspected = [];
320
922
  for (const path of paths) {
321
- const requestedPath = resolve2(root, path);
322
- if ((await lstat(requestedPath)).isSymbolicLink()) continue;
323
- const canonicalPath = await realpath(requestedPath);
324
- if (canonicalPath !== root && !canonicalPath.startsWith(`${root}${sep}`)) continue;
325
- total += (await stat(canonicalPath)).size;
326
- inspected += 1;
327
- }
328
- const passed = inspected > 0 && total <= check.max_bytes;
923
+ const size = await safeFileSize(context.metadata.root, path);
924
+ if (size === null) continue;
925
+ total += size;
926
+ inspected.push(path);
927
+ }
928
+ const passed = inspected.length > 0 && total <= check.max_bytes;
329
929
  return result(
330
930
  check.type,
931
+ check.scope,
331
932
  passed ? "met" : "not_met",
332
- `${total} byte(s) across ${inspected} safe matching file(s); maximum ${check.max_bytes}`,
333
- paths
933
+ `${total} byte(s) across ${inspected.length} safe matching file(s); maximum ${check.max_bytes}`,
934
+ inspected
334
935
  );
335
936
  }
336
- function result(type, status, summary, references) {
337
- return { type, status, summary, references };
937
+ function result(type, scope, status, summary, references2) {
938
+ return { type, scope, status, summary, references: references2 };
338
939
  }
339
- async function evaluateCheck(repo, check) {
940
+ async function evaluateCheck(context, check) {
340
941
  switch (check.type) {
341
942
  case "path_any":
342
- return evaluatePathAny(repo, check);
943
+ return evaluatePathAny(context, check);
343
944
  case "path_all":
344
- return evaluatePathAll(repo, check);
945
+ return evaluatePathAll(context, check);
345
946
  case "content_any":
346
947
  case "content_all":
347
- return evaluateContent(repo, check);
948
+ return evaluateLegacyContent(context, check);
949
+ case "content_terms":
950
+ return evaluateContentTerms(context, check);
348
951
  case "max_bytes":
349
- return evaluateMaxBytes(repo, check);
952
+ return evaluateMaxBytes(context, check);
350
953
  case "manual":
351
- return result("manual", "unknown", check.prompt, []);
954
+ return result("manual", check.scope, "unknown", check.prompt, []);
352
955
  }
353
956
  }
354
957
  function activeAttestation(control, attestations, now) {
958
+ if (!control.allow_attestation) return null;
355
959
  const attestation = attestations?.attestations[control.id];
356
960
  if (!attestation) return null;
357
961
  if (attestation.expires_at && new Date(attestation.expires_at) < now) return null;
358
962
  if (attestation.status === "not_applicable" && !control.allow_not_applicable) return null;
359
963
  return attestation;
360
964
  }
361
- async function evaluateControl(repo, control, attestations, now = /* @__PURE__ */ new Date()) {
965
+ function activeAgentEvidence(control, claim, now) {
966
+ if (!claim || !control.allow_agent_evidence) return null;
967
+ if (new Date(claim.expires_at) < now) return null;
968
+ return claim;
969
+ }
970
+ async function evaluateControl(context, control, attestations, agentClaim, now = /* @__PURE__ */ new Date()) {
362
971
  const evidence = await Promise.all(
363
- control.evidence.map(async (check) => evaluateCheck(repo, check))
972
+ control.evidence.map(async (check) => evaluateCheck(context, check))
364
973
  );
365
974
  const attestation = activeAttestation(control, attestations, now);
975
+ const agentEvidence = activeAgentEvidence(control, agentClaim, now);
366
976
  const checksPassed = evidence.every(({ status: status2 }) => status2 === "met");
367
977
  const hasManualCheck = control.evidence.some(({ type }) => type === "manual");
368
978
  let status = checksPassed ? "met" : "not_met";
369
- let confidence = checksPassed && !hasManualCheck ? "verified" : "none";
370
- if (attestation?.status === "not_applicable") {
979
+ let confidence = checksPassed && !hasManualCheck ? "repository-detected" : "none";
980
+ const attestationStatus = attestation?.status === "unknown" ? null : attestation?.status ?? null;
981
+ const hasExternalConflict = agentEvidence !== null && agentEvidence.status !== "unknown" && attestationStatus !== null && agentEvidence.status !== attestationStatus;
982
+ if (checksPassed && !hasManualCheck) {
983
+ } else if (hasExternalConflict) {
984
+ status = "unknown";
985
+ confidence = "none";
986
+ } else if (agentEvidence && agentEvidence.status !== "unknown") {
987
+ status = agentEvidence.status;
988
+ confidence = "agent-collected";
989
+ } else if (attestation?.status === "not_applicable") {
371
990
  status = "not_applicable";
372
991
  confidence = "attested";
373
- } else if (attestation?.status === "met" && control.allow_attestation) {
992
+ } else if (attestation?.status === "met") {
374
993
  status = "met";
375
- confidence = checksPassed && !hasManualCheck ? "verified" : "attested";
994
+ confidence = "attested";
376
995
  } else if (attestation?.status === "not_met" || attestation?.status === "unknown") {
377
996
  status = attestation.status;
378
997
  confidence = "attested";
998
+ } else if (agentEvidence?.status === "unknown") {
999
+ status = "unknown";
1000
+ confidence = "agent-collected";
379
1001
  } else if (evidence.some(({ status: checkStatus }) => checkStatus === "unknown")) {
380
1002
  status = "unknown";
381
1003
  }
@@ -389,6 +1011,7 @@ async function evaluateControl(repo, control, attestations, now = /* @__PURE__ *
389
1011
  status,
390
1012
  confidence,
391
1013
  evidence,
1014
+ agent_evidence: agentEvidence,
392
1015
  attestation,
393
1016
  remediation: control.remediation
394
1017
  };
@@ -409,6 +1032,46 @@ function dimensionScore(controls, dimension) {
409
1032
  }
410
1033
  return score;
411
1034
  }
1035
+ function repositoryScore(catalog, results, dimensions) {
1036
+ let achieved = 0;
1037
+ let ceiling = 0;
1038
+ const resultsById = new Map(results.map((result2) => [result2.id, result2]));
1039
+ for (const { id: dimension } of dimensions) {
1040
+ let dimensionAchieved = 0;
1041
+ let dimensionCeiling = 0;
1042
+ let achievedOpen = true;
1043
+ let ceilingOpen = true;
1044
+ for (const level of [1, 2, 3, 4]) {
1045
+ const controlsAtLevel = catalog.filter(
1046
+ (control) => control.dimension === dimension && control.level === level
1047
+ );
1048
+ const repositoryDetectable = controlsAtLevel.every(
1049
+ (control) => control.evidence.every((evidence) => evidence.scope === "repository")
1050
+ );
1051
+ if (ceilingOpen && repositoryDetectable) {
1052
+ dimensionCeiling = level;
1053
+ } else {
1054
+ ceilingOpen = false;
1055
+ }
1056
+ const repositoryEstablished = controlsAtLevel.every((control) => {
1057
+ const result2 = resultsById.get(control.id);
1058
+ return result2?.status === "met" && result2.confidence === "repository-detected";
1059
+ });
1060
+ if (achievedOpen && repositoryDetectable && repositoryEstablished) {
1061
+ dimensionAchieved = level;
1062
+ } else {
1063
+ achievedOpen = false;
1064
+ }
1065
+ }
1066
+ achieved += dimensionAchieved;
1067
+ ceiling += dimensionCeiling;
1068
+ }
1069
+ return {
1070
+ achieved,
1071
+ ceiling,
1072
+ percentage: ceiling === 0 ? 0 : Math.round(achieved / ceiling * 100)
1073
+ };
1074
+ }
412
1075
  function assessProfiles(benchmark, dimensions, controls) {
413
1076
  return benchmark.readiness_profiles.map((profile) => {
414
1077
  const blockers = benchmark.dimensions.flatMap(({ id }) => {
@@ -426,18 +1089,46 @@ function assessProfiles(benchmark, dimensions, controls) {
426
1089
  }
427
1090
  ];
428
1091
  });
429
- return { id: profile.id, title: profile.title, passed: blockers.length === 0, blockers };
1092
+ const requiredControls = controls.filter(
1093
+ (control) => control.level <= profile.floors[control.dimension] && controlPasses(control)
1094
+ );
1095
+ return {
1096
+ id: profile.id,
1097
+ title: profile.title,
1098
+ passed: blockers.length === 0,
1099
+ blockers,
1100
+ ...benchmark.version === "0.3.0" ? {
1101
+ evidence_dependencies: {
1102
+ agent_collected: requiredControls.filter(
1103
+ ({ confidence }) => confidence === "agent-collected"
1104
+ ).length,
1105
+ attested: requiredControls.filter(({ confidence }) => confidence === "attested").length
1106
+ }
1107
+ } : {}
1108
+ };
430
1109
  });
431
1110
  }
432
- async function assess(repo, benchmark, catalog, profileId, attestations, now = /* @__PURE__ */ new Date()) {
1111
+ async function assess(repo, benchmark, catalog, profileId, options = {}) {
433
1112
  const profile = benchmark.readiness_profiles.find(({ id }) => id === profileId);
434
1113
  if (!profile) {
435
1114
  throw new Error(
436
1115
  `Unknown profile ${profileId}. Choose: ${benchmark.readiness_profiles.map(({ id }) => id).join(", ")}`
437
1116
  );
438
1117
  }
1118
+ const scope = options.scope ?? "tracked";
1119
+ const now = options.now ?? /* @__PURE__ */ new Date();
1120
+ const context = await createRepositoryContext(repo, scope, options.excludedPaths);
1121
+ await validateAgentEvidence(benchmark, catalog, context, options.agentEvidence ?? null, now);
439
1122
  const controls = await Promise.all(
440
- catalog.map(async (control) => evaluateControl(repo, control, attestations, now))
1123
+ catalog.map(
1124
+ async (control) => evaluateControl(
1125
+ context,
1126
+ control,
1127
+ options.attestations ?? null,
1128
+ options.agentEvidence?.claims[control.id] ?? null,
1129
+ now
1130
+ )
1131
+ )
441
1132
  );
442
1133
  const dimensions = benchmark.dimensions.map(({ id, title }) => {
443
1134
  const dimensionControls = controls.filter((control) => control.dimension === id);
@@ -451,40 +1142,160 @@ async function assess(repo, benchmark, catalog, profileId, attestations, now = /
451
1142
  });
452
1143
  const profiles = assessProfiles(benchmark, dimensions, controls);
453
1144
  const total = dimensions.reduce((sum, { score }) => sum + score, 0);
1145
+ const repository = benchmark.version === "0.3.0" ? repositoryScore(catalog, controls, benchmark.dimensions) : void 0;
454
1146
  const highestProfile = [...profiles].reverse().find(({ passed }) => passed)?.id ?? null;
455
1147
  const targetPassed = profiles.find(({ id }) => id === profileId)?.passed ?? false;
456
1148
  return {
457
- schema_version: "0.1.0",
1149
+ schema_version: benchmark.version === "0.3.0" ? "0.3.0" : "0.2.0",
458
1150
  benchmark: { id: benchmark.id, version: benchmark.version },
459
- target: { repository: repo, profile: profileId },
1151
+ target: {
1152
+ repository: repo,
1153
+ profile: profileId,
1154
+ scope,
1155
+ git_head: context.metadata.git_head,
1156
+ git_remote: context.metadata.git_remote,
1157
+ working_tree_dirty: context.metadata.working_tree_dirty
1158
+ },
460
1159
  assessed_at: now.toISOString(),
461
- score: { total, maximum: 40, percentage: Math.round(total / 40 * 100) },
1160
+ ...benchmark.version === "0.3.0" ? { warnings: options.warnings ?? [] } : {},
1161
+ score: {
1162
+ total,
1163
+ maximum: 40,
1164
+ percentage: Math.round(total / 40 * 100),
1165
+ ...repository ? { repository } : {}
1166
+ },
462
1167
  evidence_summary: {
463
- verified: controls.filter(
464
- ({ confidence, status }) => confidence === "verified" && status === "met"
1168
+ repository_detected: controls.filter(
1169
+ ({ confidence, status }) => confidence === "repository-detected" && status === "met"
1170
+ ).length,
1171
+ agent_collected: controls.filter(
1172
+ ({ confidence, status }) => confidence === "agent-collected" && status === "met"
465
1173
  ).length,
466
1174
  attested: controls.filter(
467
1175
  ({ confidence, status }) => confidence === "attested" && status === "met"
468
1176
  ).length,
469
- unmet_or_unknown: controls.filter(
470
- ({ status }) => status === "not_met" || status === "unknown"
471
- ).length
1177
+ unmet: controls.filter(({ status }) => status === "not_met").length,
1178
+ unknown: controls.filter(({ status }) => status === "unknown").length,
1179
+ ...benchmark.version === "0.3.0" ? {
1180
+ resolved: controls.filter(({ status }) => status !== "unknown").length,
1181
+ total: controls.length
1182
+ } : {}
472
1183
  },
473
1184
  dimensions,
474
1185
  controls,
475
1186
  profiles,
476
1187
  readiness: { highest_profile: highestProfile, target_passed: targetPassed },
477
1188
  limitations: [
478
- "Repository evidence proves that an artifact exists, not that people consistently follow it.",
479
- "Self-attestations are reported separately and are not tool-verified evidence.",
1189
+ scope === "tracked" ? "Tracked mode considers only Git-tracked paths, using current working-tree contents; uncommitted edits to tracked files can affect the result." : "Workspace mode includes untracked local files and is provisional; do not compare it directly with tracked-mode reports.",
1190
+ "Repository-detected evidence proves a qualifying artifact match, not consistent practice or external enforcement.",
1191
+ ...benchmark.version === "0.3.0" ? [
1192
+ "Repository-detected progress uses only deterministic offline evidence and its attainable ceiling; it is explanatory and does not replace the normative score or readiness floors.",
1193
+ "Agent-collected repository evidence is semantic, target-bound, and source-backed but is not independently verified or relabelled as repository-detected."
1194
+ ] : [],
1195
+ "Agent-collected evidence and human attestations are reported separately and are not independently verified.",
480
1196
  "This assessment does not grant production access, deployment authority, or certification."
481
1197
  ]
482
1198
  };
483
1199
  }
1200
+ async function validateAgentEvidence(benchmark, catalog, context, evidence, now) {
1201
+ if (!evidence) return;
1202
+ if (evidence.benchmark_version !== benchmark.version) {
1203
+ throw new Error(
1204
+ `Agent evidence benchmark ${evidence.benchmark_version} does not match ${benchmark.version}`
1205
+ );
1206
+ }
1207
+ const expectedTarget = repositoryEvidenceTarget(context.metadata);
1208
+ if (evidence.target.repository !== expectedTarget.repository) {
1209
+ throw new Error(
1210
+ `Agent evidence target ${evidence.target.repository} does not match ${expectedTarget.repository}`
1211
+ );
1212
+ }
1213
+ if (evidence.target.git_head !== expectedTarget.git_head) {
1214
+ throw new Error(
1215
+ `Agent evidence commit ${evidence.target.git_head ?? "unavailable"} does not match ${expectedTarget.git_head ?? "an unavailable Git commit"}`
1216
+ );
1217
+ }
1218
+ if (benchmark.version === "0.3.0" && context.metadata.tracked_tree_dirty) {
1219
+ throw new Error("ADRB v0.3 agent evidence requires tracked files to match the bound commit");
1220
+ }
1221
+ const controls = new Map(catalog.map((control) => [control.id, control]));
1222
+ if (Object.keys(evidence.claims).length > 0 && (evidence.collector.name.startsWith("TODO") || evidence.collector.version.startsWith("TODO"))) {
1223
+ throw new Error("Agent evidence with claims must identify the collector name and version");
1224
+ }
1225
+ for (const [controlId, claim] of Object.entries(evidence.claims)) {
1226
+ const control = controls.get(controlId);
1227
+ if (!control) throw new Error(`Agent evidence references unknown control ${controlId}`);
1228
+ if (!control.allow_agent_evidence) {
1229
+ throw new Error(`${controlId} does not permit agent-collected evidence`);
1230
+ }
1231
+ const manualScopes = control.evidence.filter(({ type }) => type === "manual").map(({ scope: evidenceScope }) => evidenceScope);
1232
+ const allowedScopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : manualScopes;
1233
+ if (!allowedScopes.includes(claim.scope)) {
1234
+ throw new Error(`${controlId} does not accept ${claim.scope} evidence`);
1235
+ }
1236
+ if (claim.scope === "repository") {
1237
+ if (context.includedPaths === null) {
1238
+ throw new Error(
1239
+ `${controlId} uses repository-scoped agent evidence, which requires --scope tracked`
1240
+ );
1241
+ }
1242
+ for (const reference of claim.references) {
1243
+ const parsedReference = repositoryReference(reference);
1244
+ if (!parsedReference || !context.includedPaths.has(parsedReference.path) || context.excludedPaths.has(parsedReference.path)) {
1245
+ throw new Error(`${controlId} references an unavailable tracked path: ${reference}`);
1246
+ }
1247
+ if (parsedReference.lines) {
1248
+ const contents = await readFile3(
1249
+ join2(context.metadata.root, parsedReference.path),
1250
+ "utf8"
1251
+ );
1252
+ const lineCount = countLines(contents);
1253
+ if (parsedReference.lines.start < 1 || parsedReference.lines.end < parsedReference.lines.start) {
1254
+ throw new Error(`${controlId} references an invalid line range: ${reference}`);
1255
+ }
1256
+ if (parsedReference.lines.end > lineCount) {
1257
+ throw new Error(
1258
+ `${controlId} references lines beyond ${parsedReference.path}'s ${lineCount} lines: ${reference}`
1259
+ );
1260
+ }
1261
+ }
1262
+ }
1263
+ }
1264
+ if (new Date(claim.collected_at) > new Date(claim.expires_at)) {
1265
+ throw new Error(`${controlId} expires before it was collected`);
1266
+ }
1267
+ if (new Date(claim.collected_at) > now) {
1268
+ throw new Error(`${controlId} has a future collection timestamp`);
1269
+ }
1270
+ if (claim.summary.startsWith("TODO") || claim.references.some((reference) => reference.startsWith("TODO"))) {
1271
+ throw new Error(`${controlId} contains unresolved TODO evidence`);
1272
+ }
1273
+ }
1274
+ }
1275
+ function repositoryReference(reference) {
1276
+ if (!reference.startsWith("repo:")) return null;
1277
+ const withoutPrefix = reference.slice("repo:".length);
1278
+ const lineMatch = withoutPrefix.match(/#L(\d+)(?:-L?(\d+))?$/);
1279
+ const path = lineMatch ? withoutPrefix.slice(0, lineMatch.index) : withoutPrefix;
1280
+ if (path.length === 0 || path.startsWith("/") || path.includes("#") || path.includes("\\") || path.split("/").some((part) => part === ".." || part === ".")) {
1281
+ return null;
1282
+ }
1283
+ const start = lineMatch ? Number(lineMatch[1]) : null;
1284
+ const end = lineMatch ? Number(lineMatch[2] ?? lineMatch[1]) : null;
1285
+ return {
1286
+ path,
1287
+ lines: start === null || end === null ? null : { start, end }
1288
+ };
1289
+ }
1290
+ function countLines(contents) {
1291
+ if (contents.length === 0) return 0;
1292
+ const lines = contents.split("\n").length;
1293
+ return contents.endsWith("\n") ? lines - 1 : lines;
1294
+ }
484
1295
 
485
1296
  // src/cli.ts
486
1297
  var program = new Command();
487
- program.name("agentic-scorecard").description("Evidence-backed readiness assessment for agentic software development harnesses").version("0.1.0");
1298
+ program.name("agentic-scorecard").description("Evidence-backed readiness assessment for agentic software development harnesses").version("0.3.0");
488
1299
  program.command("validate").description("Validate the bundled benchmark catalog").action(async () => {
489
1300
  const { benchmark, controls } = await loadBenchmark();
490
1301
  process.stdout.write(
@@ -510,17 +1321,20 @@ ${control.evidence.map((check) => `- ${stringify(check).trim().replaceAll("\n",
510
1321
  );
511
1322
  });
512
1323
  program.command("init").argument("[repository]", "repository to initialize", ".").option("--force", "replace an existing attestation file", false).description("Create a manual-attestation template").action(async (repository, options) => {
513
- const repo = resolve3(repository);
514
- const path = join2(repo, ".agentic", "attestations.yaml");
1324
+ const repo = resolve4(repository);
1325
+ const path = join3(repo, ".agentic", "attestations.yaml");
515
1326
  if (!options.force) {
516
1327
  try {
517
- await readFile3(path, "utf8");
1328
+ await readFile4(path, "utf8");
518
1329
  throw new Error(`${path} already exists; use --force to replace it`);
519
1330
  } catch (error) {
520
1331
  if (error.code !== "ENOENT") throw error;
521
1332
  }
522
1333
  }
523
1334
  const { benchmark, controls } = await loadBenchmark();
1335
+ const reviewedAt = /* @__PURE__ */ new Date();
1336
+ const expiresAt = new Date(reviewedAt);
1337
+ expiresAt.setDate(expiresAt.getDate() + 90);
524
1338
  const attestations = Object.fromEntries(
525
1339
  controls.filter((control) => control.evidence.some(({ type }) => type === "manual")).map((control) => {
526
1340
  const manualCheck = control.evidence.find(
@@ -532,8 +1346,8 @@ program.command("init").argument("[repository]", "repository to initialize", "."
532
1346
  status: "unknown",
533
1347
  evidence: `TODO: ${manualCheck?.prompt ?? control.outcome}`,
534
1348
  owner: "TODO",
535
- reviewed_at: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
536
- expires_at: null
1349
+ reviewed_at: reviewedAt.toISOString().slice(0, 10),
1350
+ expires_at: expiresAt.toISOString().slice(0, 10)
537
1351
  }
538
1352
  ];
539
1353
  })
@@ -541,30 +1355,145 @@ program.command("init").argument("[repository]", "repository to initialize", "."
541
1355
  await mkdir(dirname2(path), { recursive: true });
542
1356
  await writeFile(
543
1357
  path,
544
- `# Claims are visible as attested, never tool-verified. Link durable evidence; do not paste secrets.
1358
+ `# Claims are visibly human-attested. Link durable evidence; do not paste secrets.
545
1359
  ${stringify({ benchmark_version: benchmark.version, attestations })}`,
546
1360
  "utf8"
547
1361
  );
548
1362
  process.stdout.write(`Created ${path}
549
1363
  `);
550
1364
  });
551
- program.command("assess").argument("[repository]", "repository to assess", ".").option("--profile <profile>", "target autonomy profile", "pr-creation").option("--format <format>", "json or markdown", "markdown").option("--output <path>", "write the report to a file").option("--attestations <path>", "manual attestation file").option("--enforce", "exit non-zero when the target profile fails", false).option("--github-output", "append summary values to $GITHUB_OUTPUT", false).description("Assess a repository using local, read-only evidence collection").action(
1365
+ program.command("init-evidence").argument("[repository]", "repository to prepare external evidence for", ".").option("--output <path>", "agent evidence bundle path").option("--request-output <path>", "human-readable evidence request path").option("--force", "replace an existing agent evidence bundle", false).description("Create a target-bound template for unresolved agent-collected evidence").action(
1366
+ async (repository, options) => {
1367
+ const repo = resolve4(repository);
1368
+ const path = resolve4(options.output ?? join3(repo, ".agentic", "agent-evidence.yaml"));
1369
+ const requestPath = resolve4(
1370
+ options.requestOutput ?? join3(repo, ".agentic", "evidence-request.md")
1371
+ );
1372
+ if (path === requestPath) {
1373
+ throw new Error("Agent evidence bundle and request paths must be different");
1374
+ }
1375
+ if (!options.force) {
1376
+ for (const candidate of [path, requestPath]) {
1377
+ try {
1378
+ await readFile4(candidate, "utf8");
1379
+ throw new Error(`${candidate} already exists; use --force to replace it`);
1380
+ } catch (error) {
1381
+ if (error.code !== "ENOENT") throw error;
1382
+ }
1383
+ }
1384
+ }
1385
+ const { benchmark, controls } = await loadBenchmark();
1386
+ const context = await createRepositoryContext(repo, "tracked").catch((error) => {
1387
+ if (error instanceof Error && error.message.startsWith("Tracked assessment requires a Git worktree")) {
1388
+ throw new Error(
1389
+ "init-evidence requires a Git worktree with a commit so the bundle can be target-bound."
1390
+ );
1391
+ }
1392
+ throw error;
1393
+ });
1394
+ if (!context.metadata.git_head) {
1395
+ throw new Error(
1396
+ "init-evidence requires a Git commit so the bundle can be target-bound. Commit the assessed state and try again."
1397
+ );
1398
+ }
1399
+ if (context.metadata.tracked_tree_dirty) {
1400
+ throw new Error(
1401
+ "init-evidence requires tracked files to match HEAD so every claim binds to the exact assessed commit."
1402
+ );
1403
+ }
1404
+ const baseline = await assess(repo, benchmark, controls, "read-only-analysis", {
1405
+ scope: "tracked"
1406
+ });
1407
+ const unresolved = new Set(
1408
+ baseline.controls.filter(({ status }) => status !== "met" && status !== "not_applicable").map(({ id }) => id)
1409
+ );
1410
+ const eligibleControls = controls.filter(
1411
+ ({ allow_agent_evidence: allowed, id }) => allowed && unresolved.has(id)
1412
+ );
1413
+ const bundle = {
1414
+ schema_version: benchmark.version,
1415
+ benchmark_version: benchmark.version,
1416
+ target: repositoryEvidenceTarget(context.metadata),
1417
+ collector: { name: "TODO: agent or adapter name", version: "TODO" },
1418
+ claims: {}
1419
+ };
1420
+ await mkdir(dirname2(path), { recursive: true });
1421
+ await writeFile(
1422
+ path,
1423
+ `# Use authorized read-only tools. Do not paste secrets or raw sensitive content.
1424
+ ${stringify(bundle)}`,
1425
+ "utf8"
1426
+ );
1427
+ await mkdir(dirname2(requestPath), { recursive: true });
1428
+ await writeFile(
1429
+ requestPath,
1430
+ [
1431
+ "# ADRB v0.3 evidence request",
1432
+ "",
1433
+ `- Repository: ${bundle.target.repository}`,
1434
+ `- Git commit: ${bundle.target.git_head ?? "unavailable"}`,
1435
+ `- Benchmark: ${benchmark.version}`,
1436
+ "",
1437
+ "Repository claims may cite only tracked paths from the bound commit and remain agent-collected, not repository-detected. Obtain authorization before accessing connected systems. Use read-only, least-privileged tools. Add only attempted claims to the bundle; partial or inconclusive evidence remains `unknown`. Never paste source excerpts, secrets, prompts, personal data, or raw sensitive content.",
1438
+ "",
1439
+ ...eligibleControls.flatMap((control) => {
1440
+ const manualCheck = control.evidence.find(
1441
+ (check) => check.type === "manual"
1442
+ );
1443
+ const scopes = control.agent_evidence_scopes.length > 0 ? control.agent_evidence_scopes : [manualCheck?.scope ?? "organization"];
1444
+ return [
1445
+ `## ${control.id} \u2014 ${control.title}`,
1446
+ "",
1447
+ `- Scope: ${scopes.join(", ")}`,
1448
+ `- Request: ${manualCheck?.prompt ?? control.outcome}`,
1449
+ `- Risk: ${control.risk}`,
1450
+ ""
1451
+ ];
1452
+ })
1453
+ ].join("\n"),
1454
+ "utf8"
1455
+ );
1456
+ process.stdout.write(`Created ${path}
1457
+ Created ${requestPath}
1458
+ `);
1459
+ }
1460
+ );
1461
+ program.command("assess").argument("[repository]", "repository to assess", ".").option("--profile <profile>", "target autonomy profile", "pr-creation").option("--format <format>", "json or markdown", "markdown").option("--output <path>", "write the report to a file").option("--attestations <path>", "manual attestation file").option("--agent-evidence <path>", "agent-collected repository or external evidence bundle").option("--scope <scope>", "tracked or workspace", "tracked").option("--enforce", "exit non-zero when the target profile fails", false).option("--github-output", "append summary values to $GITHUB_OUTPUT", false).description("Assess a repository using local, read-only evidence collection").action(
552
1462
  async (repository, options) => {
553
1463
  if (!["json", "markdown"].includes(options.format)) {
554
1464
  throw new Error("--format must be json or markdown");
555
1465
  }
556
- const repo = resolve3(repository);
1466
+ if (!["tracked", "workspace"].includes(options.scope)) {
1467
+ throw new Error("--scope must be tracked or workspace");
1468
+ }
1469
+ const repo = resolve4(repository);
557
1470
  const { benchmark, controls } = await loadBenchmark();
558
- const attestationPath = resolve3(
559
- options.attestations ?? join2(repo, ".agentic", "attestations.yaml")
1471
+ const warnings = [];
1472
+ const attestationPath = resolve4(
1473
+ options.attestations ?? join3(repo, ".agentic", "attestations.yaml")
560
1474
  );
561
- const attestations = await loadAttestations(attestationPath, benchmark.version);
562
- const report = await assess(repo, benchmark, controls, options.profile, attestations);
1475
+ const attestations = await loadAttestations(attestationPath, benchmark.version, {
1476
+ ignoreVersionMismatch: options.attestations === void 0,
1477
+ onWarning: (warning) => warnings.push(warning)
1478
+ });
1479
+ const agentEvidencePath = resolve4(
1480
+ options.agentEvidence ?? join3(repo, ".agentic", "agent-evidence.yaml")
1481
+ );
1482
+ const agentEvidence = await loadAgentEvidence(agentEvidencePath, benchmark.version, {
1483
+ ignoreVersionMismatch: options.agentEvidence === void 0,
1484
+ onWarning: (warning) => warnings.push(warning)
1485
+ });
1486
+ const reportPath = options.output ? resolve4(options.output) : null;
1487
+ const report = await assess(repo, benchmark, controls, options.profile, {
1488
+ scope: options.scope,
1489
+ attestations,
1490
+ agentEvidence,
1491
+ warnings,
1492
+ excludedPaths: [attestationPath, agentEvidencePath, ...reportPath ? [reportPath] : []]
1493
+ });
563
1494
  const output = options.format === "json" ? `${JSON.stringify(report, null, 2)}
564
1495
  ` : toMarkdown(report);
565
- let reportPath = null;
566
- if (options.output) {
567
- reportPath = resolve3(options.output);
1496
+ if (reportPath) {
568
1497
  await mkdir(dirname2(reportPath), { recursive: true });
569
1498
  await writeFile(reportPath, output, "utf8");
570
1499
  process.stdout.write(`Wrote ${reportPath}
@@ -579,6 +1508,9 @@ program.command("assess").argument("[repository]", "repository to assess", ".").
579
1508
  githubOutput,
580
1509
  `score=${report.score.total}
581
1510
  percentage=${report.score.percentage}
1511
+ repository_score=${report.score.repository?.achieved ?? ""}
1512
+ repository_ceiling=${report.score.repository?.ceiling ?? ""}
1513
+ repository_percentage=${report.score.repository?.percentage ?? ""}
582
1514
  highest_profile=${report.readiness.highest_profile ?? "none"}
583
1515
  target_passed=${String(report.readiness.target_passed)}
584
1516
  report_path=${reportPath ?? ""}