@agentspec/sdk 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +46 -0
  2. package/dist/evidence-points-C6j4PSsQ.d.cts +1139 -0
  3. package/dist/evidence-points-C6j4PSsQ.d.ts +1139 -0
  4. package/dist/index.cjs +1727 -345
  5. package/dist/index.cjs.map +1 -1
  6. package/dist/index.d.cts +1026 -2549
  7. package/dist/index.d.ts +1026 -2549
  8. package/dist/index.js +1678 -344
  9. package/dist/index.js.map +1 -1
  10. package/dist/runtime.cjs +930 -0
  11. package/dist/runtime.cjs.map +1 -0
  12. package/dist/runtime.d.cts +63 -0
  13. package/dist/runtime.d.ts +63 -0
  14. package/dist/runtime.js +921 -0
  15. package/dist/runtime.js.map +1 -0
  16. package/package.json +15 -8
  17. package/schemas/evidence/v1/contract.schema.json +2078 -0
  18. package/schemas/evidence/v1/fixtures/drifted/EvalResult.n-as-string.json +1 -0
  19. package/schemas/evidence/v1/fixtures/drifted/EvalResult.pass-rate-above-one.json +1 -0
  20. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.bad-sha.json +1 -0
  21. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.extra-field.json +1 -0
  22. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.fail-without-reason.json +1 -0
  23. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.missing-ttl.json +1 -0
  24. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.not-verifiable-without-reason.json +1 -0
  25. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-epoch.json +1 -0
  26. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-out-of-range.json +1 -0
  27. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-space-separated.json +1 -0
  28. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.sample-size-boolean.json +1 -0
  29. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.ttl-as-string.json +1 -0
  30. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-collector.json +1 -0
  31. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-source.json +1 -0
  32. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-contract-version.json +1 -0
  33. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-record.json +1 -0
  34. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.point-records-over-limit.json +1 -0
  35. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.with-gap.json +1 -0
  36. package/schemas/evidence/v1/fixtures/drifted/ReadinessFinding.bad-severity.json +1 -0
  37. package/schemas/evidence/v1/fixtures/drifted/SidecarPush.missing-sha.json +1 -0
  38. package/schemas/evidence/v1/fixtures/drifted/TrustPolicy.lenient.json +1 -0
  39. package/schemas/evidence/v1/fixtures/valid/EvalResult.live.json +1 -0
  40. package/schemas/evidence/v1/fixtures/valid/EvalResult.null-dataset.json +1 -0
  41. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.fail.json +1 -0
  42. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.null-collector.json +1 -0
  43. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass-null-reason.json +1 -0
  44. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass.json +1 -0
  45. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.timezone-offset.json +1 -0
  46. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.unknown-reason-code.json +1 -0
  47. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.with-collector.json +1 -0
  48. package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-null.json +1 -0
  49. package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-version.json +1 -0
  50. package/schemas/evidence/v1/fixtures/valid/Heartbeat.evidence-points.json +1 -0
  51. package/schemas/evidence/v1/fixtures/valid/Heartbeat.full.json +1 -0
  52. package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk-no-records.json +1 -0
  53. package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk.json +1 -0
  54. package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.open-code.json +1 -0
  55. package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.with-subject.json +1 -0
  56. package/schemas/evidence/v1/fixtures/valid/SidecarPush.declared.json +1 -0
  57. package/schemas/evidence/v1/fixtures/valid/TrustPolicy.standard.json +1 -0
  58. package/schemas/evidence/v1/fixtures/valid/TrustPolicy.strict.json +1 -0
  59. package/schemas/grader/v1/fixtures/accepts-fractional-seconds.json +62 -0
  60. package/schemas/grader/v1/fixtures/accepts-leap-day-in-year-below-100.json +62 -0
  61. package/schemas/grader/v1/fixtures/accepts-more-than-6-fractional-digits.json +62 -0
  62. package/schemas/grader/v1/fixtures/accepts-numeric-offset.json +62 -0
  63. package/schemas/grader/v1/fixtures/all-records-must-pass.json +58 -0
  64. package/schemas/grader/v1/fixtures/below-threshold-not-graded.json +61 -0
  65. package/schemas/grader/v1/fixtures/derived-rule-fails-when-a-parent-fails.json +67 -0
  66. package/schemas/grader/v1/fixtures/derived-rule-follows-memory-write-parent.json +43 -0
  67. package/schemas/grader/v1/fixtures/derived-rule-passes-with-its-parents.json +64 -0
  68. package/schemas/grader/v1/fixtures/derived-rule-unverified-when-a-parent-has-no-evidence.json +51 -0
  69. package/schemas/grader/v1/fixtures/expired-record-is-stale.json +66 -0
  70. package/schemas/grader/v1/fixtures/fail-beats-not-verifiable.json +62 -0
  71. package/schemas/grader/v1/fixtures/grade-band-b.json +62 -0
  72. package/schemas/grader/v1/fixtures/independent-from-listed-collector-is-admitted.json +39 -0
  73. package/schemas/grader/v1/fixtures/independent-from-unlisted-collector-is-rejected.json +47 -0
  74. package/schemas/grader/v1/fixtures/low-severity-coverage-only.json +90 -0
  75. package/schemas/grader/v1/fixtures/malformed-point-is-ignored.json +67 -0
  76. package/schemas/grader/v1/fixtures/never-admitted-sources-standard.json +196 -0
  77. package/schemas/grader/v1/fixtures/never-admitted-sources-strict.json +196 -0
  78. package/schemas/grader/v1/fixtures/not-admitted-outranks-mismatch.json +64 -0
  79. package/schemas/grader/v1/fixtures/not-verifiable-ignored-when-one-passes.json +58 -0
  80. package/schemas/grader/v1/fixtures/not-verifiable-only-keeps-reason.json +71 -0
  81. package/schemas/grader/v1/fixtures/not-wired-point-drops-a-fail-when-no-point-of-the-rule-is-wired.json +59 -0
  82. package/schemas/grader/v1/fixtures/not-wired-point-drops-a-wired-point-pass.json +87 -0
  83. package/schemas/grader/v1/fixtures/not-wired-point-drops-its-own-earlier-fail-when-another-point-is-wired.json +70 -0
  84. package/schemas/grader/v1/fixtures/not-wired-point-ignores-a-wired-point-fail-without-the-rule-id.json +69 -0
  85. package/schemas/grader/v1/fixtures/not-wired-point-keeps-a-wired-point-fail.json +79 -0
  86. package/schemas/grader/v1/fixtures/not-wired-point-keeps-independent-evidence.json +64 -0
  87. package/schemas/grader/v1/fixtures/not-wired-point-never-passes.json +107 -0
  88. package/schemas/grader/v1/fixtures/offline-record-is-not-admitted-yet.json +47 -0
  89. package/schemas/grader/v1/fixtures/record-without-collector-is-admitted-until-producers-set-it.json +38 -0
  90. package/schemas/grader/v1/fixtures/records-without-rule-id-never-graded.json +62 -0
  91. package/schemas/grader/v1/fixtures/rejects-compact-timestamp.json +56 -0
  92. package/schemas/grader/v1/fixtures/rejects-hour-24-lowercase-separator.json +56 -0
  93. package/schemas/grader/v1/fixtures/rejects-hour-24-space-separator.json +56 -0
  94. package/schemas/grader/v1/fixtures/rejects-hour-24.json +56 -0
  95. package/schemas/grader/v1/fixtures/rejects-impossible-date-feb-29.json +56 -0
  96. package/schemas/grader/v1/fixtures/rejects-lowercase-t.json +56 -0
  97. package/schemas/grader/v1/fixtures/rejects-lowercase-z.json +56 -0
  98. package/schemas/grader/v1/fixtures/rejects-offset-hours-above-23.json +56 -0
  99. package/schemas/grader/v1/fixtures/rejects-offset-minutes-above-59.json +56 -0
  100. package/schemas/grader/v1/fixtures/rejects-offsetless-timestamp.json +56 -0
  101. package/schemas/grader/v1/fixtures/rejects-space-separator.json +56 -0
  102. package/schemas/grader/v1/fixtures/rejects-year-zero.json +56 -0
  103. package/schemas/grader/v1/fixtures/rounds-coverage-before-the-threshold.json +850 -0
  104. package/schemas/grader/v1/fixtures/rounds-verified-before-the-band.json +674 -0
  105. package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-admitted-standard.json +39 -0
  106. package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-not-admitted-strict.json +47 -0
  107. package/schemas/grader/v1/fixtures/sdk-observed-standard.json +58 -0
  108. package/schemas/grader/v1/fixtures/sdk-observed-strict.json +66 -0
  109. package/schemas/grader/v1/fixtures/sdk-observed-without-evidence-point-source-is-rejected.json +47 -0
  110. package/schemas/grader/v1/fixtures/sdk-only-agent-all-points-fired.json +116 -0
  111. package/schemas/grader/v1/fixtures/stale-outranks-mismatch.json +64 -0
  112. package/schemas/grader/v1/fixtures/strict-admits-independent.json +39 -0
  113. package/schemas/grader/v1/fixtures/sub-millisecond-expiry-boundary.json +38 -0
  114. package/schemas/v1/agent.schema.json +372 -198
package/dist/index.cjs CHANGED
@@ -1,15 +1,32 @@
1
1
  'use strict';
2
2
 
3
3
  var zod = require('zod');
4
- var zodToJsonSchema = require('zod-to-json-schema');
5
4
  var fs = require('fs');
6
5
  var path = require('path');
7
6
  var yaml = require('js-yaml');
8
7
  var net = require('net');
8
+ var crypto = require('crypto');
9
+ var async_hooks = require('async_hooks');
9
10
 
10
- function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
11
+ function _interopNamespace(e) {
12
+ if (e && e.__esModule) return e;
13
+ var n = Object.create(null);
14
+ if (e) {
15
+ Object.keys(e).forEach(function (k) {
16
+ if (k !== 'default') {
17
+ var d = Object.getOwnPropertyDescriptor(e, k);
18
+ Object.defineProperty(n, k, d.get ? d : {
19
+ enumerable: true,
20
+ get: function () { return e[k]; }
21
+ });
22
+ }
23
+ });
24
+ }
25
+ n.default = e;
26
+ return Object.freeze(n);
27
+ }
11
28
 
12
- var yaml__default = /*#__PURE__*/_interopDefault(yaml);
29
+ var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml);
13
30
 
14
31
  // src/schema/manifest.schema.ts
15
32
  var refOrLiteral = zod.z.string();
@@ -22,17 +39,37 @@ var MetadataSchema = zod.z.object({
22
39
  author: zod.z.string().optional(),
23
40
  license: zod.z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
24
41
  });
42
+ function isHttpUrl(value) {
43
+ if (value.startsWith("$env:")) return true;
44
+ try {
45
+ const url = new URL(value);
46
+ return (url.protocol === "http:" || url.protocol === "https:") && url.hostname !== "";
47
+ } catch {
48
+ return false;
49
+ }
50
+ }
51
+ var baseUrlSchema = zod.z.string().regex(/^(\$env:[A-Za-z_][A-Za-z0-9_]*|https?:\/\/.+)$/, 'must be an http(s) URL or a "$env:VAR" reference').refine(isHttpUrl, 'must be an http(s) URL with a host, or a "$env:VAR" reference').optional().describe('Override the provider endpoint with an OpenAI-compatible base URL (vLLM, Ollama, LiteLLM, an internal proxy, a mock LLM). A literal http(s) URL with a host (checked by the loader, not expressible in JSON Schema) or "$env:VAR". When set, the gateway treats the model as a custom OpenAI-compatible provider for any "provider" value except "anthropic", whose baseUrl is an Anthropic Messages endpoint without "/v1" (the gateway adds it). Omit to use the provider default.');
52
+ var ModelPricingSchema = zod.z.object({
53
+ input: zod.z.number().nonnegative().describe("USD per 1M input (prompt) tokens."),
54
+ output: zod.z.number().nonnegative().describe("USD per 1M output (completion) tokens."),
55
+ cacheRead: zod.z.number().nonnegative().optional().describe("USD per 1M tokens read from the prompt cache."),
56
+ cacheWrite: zod.z.number().nonnegative().optional().describe("USD per 1M tokens written to the prompt cache."),
57
+ reasoning: zod.z.number().nonnegative().optional().describe("USD per 1M reasoning tokens. The gateway uses the output price when omitted.")
58
+ }).describe("Price of this model in USD per 1M tokens. The gateway uses it instead of its built-in price, so a maxMonthlyUSD budget is spent on models it does not price (e.g. o4-mini). Omit to use the gateway's built-in price.");
25
59
  var ModelFallbackSchema = zod.z.object({
26
60
  provider: zod.z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
27
61
  id: zod.z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
28
62
  apiKey: refOrLiteral.describe(refDesc),
63
+ baseUrl: baseUrlSchema,
29
64
  triggerOn: zod.z.array(zod.z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
30
- maxRetries: zod.z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
65
+ maxRetries: zod.z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover)."),
66
+ pricing: ModelPricingSchema.optional()
31
67
  });
32
68
  var ModelSchema = zod.z.object({
33
69
  provider: zod.z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
34
70
  id: zod.z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
35
71
  apiKey: refOrLiteral.describe(refDesc),
72
+ baseUrl: baseUrlSchema,
36
73
  parameters: zod.z.object({
37
74
  temperature: zod.z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
38
75
  maxTokens: zod.z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
@@ -41,6 +78,7 @@ var ModelSchema = zod.z.object({
41
78
  presencePenalty: zod.z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
42
79
  }).optional(),
43
80
  fallback: ModelFallbackSchema.optional(),
81
+ pricing: ModelPricingSchema.optional(),
44
82
  costControls: zod.z.object({
45
83
  maxMonthlyUSD: zod.z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
46
84
  alertAtUSD: zod.z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
@@ -156,6 +194,7 @@ var ChatEndpointSchema = zod.z.object({
156
194
  var ApiSchema = zod.z.object({
157
195
  type: zod.z.enum(["rest", "mcp", "grpc", "websocket"]),
158
196
  port: zod.z.number().int().min(1).max(65535).optional(),
197
+ allowedPorts: zod.z.array(zod.z.number().int().min(1).max(65535)).optional().describe("Extra ports the agent may listen on besides the sidecar-routed API port (e.g. a platform metrics port). The sidecar reports not-ready while the agent listens on any other non-loopback port."),
159
198
  pathPrefix: zod.z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
160
199
  auth: zod.z.object({
161
200
  type: zod.z.enum(["jwt", "apikey", "oauth2", "none"]),
@@ -164,7 +203,8 @@ var ApiSchema = zod.z.object({
164
203
  }).optional(),
165
204
  rateLimit: zod.z.object({
166
205
  requestsPerMinute: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
167
- requestsPerHour: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
206
+ requestsPerHour: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429."),
207
+ tokensPerMinute: zod.z.number().int().min(1).optional().describe("Maximum LLM tokens the gateway allows per minute. Once the bucket is empty, requests receive HTTP 429.")
168
208
  }).optional(),
169
209
  streaming: zod.z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
170
210
  healthEndpoint: zod.z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
@@ -246,25 +286,42 @@ var EvalMetricEnum = zod.z.enum([
246
286
  // RAG: do the retrieved chunks cover the ground-truth answer?
247
287
  "bias",
248
288
  // Does the answer exhibit demographic or political bias?
289
+ "grounded",
290
+ // Are the answer's claims supported by the retrieved context? Replaces faithfulness and hallucination.
291
+ "pii_leakage",
292
+ // Does the answer leak personal data?
293
+ "prompt_injection",
294
+ // Did the answer follow an injected instruction?
249
295
  "custom"
250
- // Custom scorer defined in the evaluation framework config.
296
+ // Rubric-graded scorer, declared in the sibling `custom: { name, rubric }`.
251
297
  ]);
252
- var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for faithfulness, context_precision, hallucination metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
298
+ var LiveMetricEnum = EvalMetricEnum.exclude(["context_recall", "answer_similarity"]);
299
+ var CustomMetricSchema = zod.z.object({
300
+ name: zod.z.string().min(1).describe("Name the custom metric is reported under."),
301
+ rubric: refOrLiteral.describe('Grading rubric. Use "$file:rubrics/<name>.md"; the loader resolves it to the file contents.')
302
+ });
303
+ var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for grounded and context_precision metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
253
304
  var EvaluationSchema = zod.z.object({
254
- framework: zod.z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
305
+ framework: zod.z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]).optional().describe("Deprecated: nothing reads it. validate warns when set; rejected in 0.6.0."),
306
+ live: zod.z.object({
307
+ metrics: zod.z.array(LiveMetricEnum).optional().describe("Metrics scored on live traffic."),
308
+ thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe("Minimum pass rate per live metric (0.0\u20131.0)."),
309
+ custom: CustomMetricSchema.optional()
310
+ }).optional().describe("Live evaluation, separate from dataset evaluation. Only live metrics are allowed."),
311
+ custom: CustomMetricSchema.optional().describe('Rubric for the "custom" metric on datasets.'),
255
312
  datasets: zod.z.array(
256
313
  zod.z.object({
257
314
  name: zod.z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
258
315
  path: refOrLiteral.describe(datasetPathDesc),
259
316
  metrics: zod.z.array(EvalMetricEnum).optional().describe(
260
- 'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["faithfulness","context_recall"] while a safety dataset sets ["toxicity","bias"].'
317
+ 'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["grounded","context_recall"] while a safety dataset sets ["toxicity","bias"].'
261
318
  )
262
319
  })
263
320
  ).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
264
321
  metrics: zod.z.array(EvalMetricEnum).optional().describe(
265
- 'Default metrics to compute across all datasets that do not declare their own metrics. "faithfulness", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires a scorer registered in the evaluation framework config.'
322
+ 'Default metrics to compute across all datasets that do not declare their own metrics. "grounded", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires spec.evaluation.custom: { name, rubric }. "faithfulness" and "hallucination" are deprecated aliases of "grounded".'
266
323
  ),
267
- thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"faithfulness": 0.8, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
324
+ thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"grounded": 0.95, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
268
325
  ciGate: zod.z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
269
326
  });
270
327
  var ObservabilitySchema = zod.z.object({
@@ -363,12 +420,608 @@ var ManifestSchema = zod.z.object({
363
420
  metadata: MetadataSchema,
364
421
  spec: SpecSchema
365
422
  });
423
+ var DRAFT_07 = "http://json-schema.org/draft-07/schema#";
424
+ var options = {
425
+ target: "draft-7",
426
+ io: "input",
427
+ reused: "inline",
428
+ override: ({ zodSchema, jsonSchema }) => {
429
+ const def = zodSchema._zod.def;
430
+ if (def.type === "object" && jsonSchema.additionalProperties === void 0) jsonSchema.additionalProperties = false;
431
+ if (def.type === "record") delete jsonSchema.propertyNames;
432
+ if (def.type === "union" && def.discriminator && jsonSchema.oneOf) {
433
+ jsonSchema.anyOf = jsonSchema.oneOf;
434
+ delete jsonSchema.oneOf;
435
+ }
436
+ if (jsonSchema.type === "integer") {
437
+ if (jsonSchema.maximum === Number.MAX_SAFE_INTEGER) delete jsonSchema.maximum;
438
+ if (jsonSchema.minimum === Number.MIN_SAFE_INTEGER) delete jsonSchema.minimum;
439
+ }
440
+ }
441
+ };
442
+ function toJsonSchema(schema) {
443
+ const { $schema: _, ...body } = zod.z.toJSONSchema(schema, options);
444
+ return { ...body, $schema: DRAFT_07 };
445
+ }
446
+ function toNamedJsonSchema(name, schema) {
447
+ const { $schema, ...body } = toJsonSchema(schema);
448
+ return { $ref: `#/definitions/${name}`, definitions: { [name]: body }, $schema };
449
+ }
366
450
  function exportJsonSchema() {
367
- return zodToJsonSchema.zodToJsonSchema(ManifestSchema, {
368
- name: "AgentSpecManifest",
369
- $refStrategy: "none"
451
+ return toNamedJsonSchema("AgentSpecManifest", ManifestSchema);
452
+ }
453
+ var EvidenceClass = zod.z.enum(["independent", "evidence-point", "offline", "declared"]);
454
+ var Collector = zod.z.enum([
455
+ "gateway-config",
456
+ "gateway-traffic",
457
+ "sidecar-probe",
458
+ "sidecar-forwarder",
459
+ "sidecar-eval",
460
+ "operator-webhook",
461
+ "operator-logs",
462
+ "sdk",
463
+ "lock",
464
+ "manifest"
465
+ ]);
466
+ var EvidencePoint = zod.z.enum(["guardrail", "tool", "mcpClient", "memoryWrite", "vectorStore", "logging"]);
467
+ var Scope = zod.z.enum(["declared", "configured", "enforced", "observed", "ran-where-wired", "every-path", "effective"]);
468
+ var WiredBy = zod.z.enum(["none", "operator-render", "agent-code", "user-schedule", "opt-in"]);
469
+ var Mode = zod.z.enum(["standalone", "operator"]);
470
+ var BuildStatus = zod.z.enum(["built", "partial", "planned", "open"]);
471
+ var RuleEvidenceSource = zod.z.object({
472
+ class: EvidenceClass,
473
+ collector: Collector,
474
+ /** Evidence-point sources only. */
475
+ point: EvidencePoint.nullish(),
476
+ proves: Scope,
477
+ wiredBy: WiredBy,
478
+ modes: zod.z.array(Mode).min(1),
479
+ status: BuildStatus,
480
+ /** A variant such as HTTP MCP, stdio MCP, local tools or opt-in. */
481
+ when: zod.z.string().min(1).nullish(),
482
+ /** A collector that checks the counts, such as gateway calls against guardrail records. */
483
+ crossCheck: Collector.nullish(),
484
+ /** The `agentspec lock` prompt question. */
485
+ question: zod.z.string().min(1).nullish()
486
+ }).strict().refine((s) => s.class === "evidence-point" === (s.point != null), {
487
+ message: "an evidence-point source names its `point`, and no other class does"
488
+ });
489
+ var RuleEvidenceEntry = zod.z.object({
490
+ sources: zod.z.array(RuleEvidenceSource).default([]),
491
+ /** Rules whose verdicts decide this one (all must pass); this rule inherits their sources. */
492
+ derivedFrom: zod.z.array(zod.z.string().min(1)).nullish(),
493
+ /** What a pass does not prove, shown with the result. Free text. */
494
+ notProven: zod.z.string().nullish()
495
+ }).strict().refine((e) => e.sources.length > 0 || (e.derivedFrom?.length ?? 0) > 0, {
496
+ message: "an entry needs at least one source or a `derivedFrom`"
497
+ });
498
+ var BOTH = ["standalone", "operator"];
499
+ var CLASS_DEFAULTS = {
500
+ independent: { wiredBy: "none", status: "built" },
501
+ "evidence-point": { wiredBy: "agent-code", status: "planned" },
502
+ offline: { wiredBy: "user-schedule", status: "planned" },
503
+ declared: { wiredBy: "none", status: "built" }
504
+ };
505
+ var source = (cls, collector, proves, extra = {}) => ({ class: cls, collector, proves, modes: BOTH, ...CLASS_DEFAULTS[cls], ...extra });
506
+ var independent = (collector, proves, extra) => source("independent", collector, proves, extra);
507
+ var point = (p, proves, extra) => source("evidence-point", "sdk", proves, { point: p, ...extra });
508
+ var lock = (question, extra) => source("offline", "lock", "every-path", { question, ...extra });
509
+ var declared = (extra) => source("declared", "manifest", "declared", extra);
510
+ var OPERATOR_ONLY = { modes: ["operator"] };
511
+ var PLANNED = { status: "planned" };
512
+ var WRITES_BYPASS = "Writes that bypass the evidence point at runtime.";
513
+ var entries = {
514
+ "MODEL-01": { sources: [independent("gateway-config", "configured"), independent("gateway-traffic", "observed")] },
515
+ "MODEL-02": {
516
+ sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
517
+ },
518
+ "MODEL-03": { sources: [independent("gateway-config", "enforced")] },
519
+ "MODEL-04": {
520
+ sources: [
521
+ independent("gateway-config", "enforced"),
522
+ lock("Do the agent's own provider clients retry more than spec.model.fallback.maxRetries, or stack retries on the gateway's?", { when: "client retries" })
523
+ ],
524
+ notProven: "That the gateway's attempts match `spec.model.fallback.maxRetries` at runtime beyond the rendered config."
525
+ },
526
+ "MODEL-05": {
527
+ sources: [independent("gateway-traffic", "observed", { when: "agw.ai.usage.cost.total" })],
528
+ notProven: "Requests with no token usage: not counted."
529
+ },
530
+ "SEC-LLM-01": {
531
+ sources: [
532
+ independent("sidecar-probe", "effective", { when: "opt-in", wiredBy: "opt-in", status: "planned" }),
533
+ point("guardrail", "ran-where-wired"),
534
+ lock("Does every path that sends user input to a model run the declared input guardrails first?")
535
+ ]
536
+ },
537
+ "SEC-LLM-02": {
538
+ sources: [
539
+ point("guardrail", "ran-where-wired"),
540
+ lock("Does every path that returns model output run the declared output guardrails first?")
541
+ ],
542
+ notProven: "Effective: a no-op output guardrail still fires."
543
+ },
544
+ "SEC-LLM-03": { sources: [declared()] },
545
+ "SEC-LLM-04": {
546
+ sources: [independent("gateway-config", "enforced"), independent("gateway-traffic", "observed", PLANNED)],
547
+ notProven: "Cost: MODEL-03 covers it."
548
+ },
549
+ "SEC-LLM-05": {
550
+ sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
551
+ },
552
+ "SEC-LLM-06": { derivedFrom: ["MEM-01"], notProven: "Same as MEM-01." },
553
+ "SEC-LLM-07": {
554
+ sources: [
555
+ independent("sidecar-probe", "configured", { when: "HTTP MCP" }),
556
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
557
+ lock("Does the agent call any MCP tool the manifest does not declare, or one that serves no annotations?"),
558
+ declared({ when: "local tools" })
559
+ ]
560
+ },
561
+ "SEC-LLM-08": {
562
+ sources: [
563
+ independent("sidecar-probe", "configured", PLANNED),
564
+ independent("gateway-traffic", "enforced", PLANNED),
565
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" })
566
+ ],
567
+ notProven: "Which tools are destructive: the server declares it; nothing checks the claim."
568
+ },
569
+ "SEC-LLM-09": { derivedFrom: ["EVAL-01", "EVAL-02"] },
570
+ "SEC-LLM-10": {
571
+ sources: [
572
+ independent("operator-webhook", "configured", { ...OPERATOR_ONLY, ...PLANNED }),
573
+ declared({ modes: ["standalone"] })
574
+ ]
575
+ },
576
+ "TOOL-01": {
577
+ sources: [
578
+ independent("gateway-traffic", "observed", { when: "HTTP MCP" }),
579
+ point("tool", "ran-where-wired", { when: "local tools" }),
580
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
581
+ lock("Which tools does the agent code register or call, local, HTTP MCP and stdio MCP?")
582
+ ],
583
+ notProven: "Unwrapped local or stdio calls at runtime: only the lock inventory catches them."
584
+ },
585
+ "MEM-01": {
586
+ sources: [
587
+ point("memoryWrite", "enforced"),
588
+ lock("Does every memory write pass through the PII scrubber for the declared fields?")
589
+ ],
590
+ notProven: "Full coverage: the scrubber is a regex baseline (Presidio optional in Python). " + WRITES_BYPASS
591
+ },
592
+ "MEM-02": {
593
+ sources: [
594
+ point("memoryWrite", "enforced"),
595
+ lock("Does every memory write apply the declared retention TTL?")
596
+ ],
597
+ notProven: WRITES_BYPASS + " Backends that cannot apply a TTL (record `ttl: unsupported`): the evidence point is `not_verifiable` there and only the lock remains."
598
+ },
599
+ "MEM-03": {
600
+ sources: [
601
+ point("memoryWrite", "enforced"),
602
+ lock("Does every memory write honour the declared memory audit setting?")
603
+ ],
604
+ notProven: WRITES_BYPASS
605
+ },
606
+ "MEM-04": {
607
+ sources: [
608
+ point("vectorStore", "ran-where-wired"),
609
+ lock("Does every vector store query and write use the declared namespace or tenant isolation?")
610
+ ],
611
+ notProven: "Isolation: the right namespace does not prove other tenants cannot read it."
612
+ },
613
+ "MEM-05": {
614
+ sources: [independent("gateway-traffic", "observed", { when: "gen_ai.usage.output_tokens" }), declared({ when: "until a call is observed" })],
615
+ notProven: "The tolerance for the system prompt and tool schemas is set in Phase 2, with an ADR-0015 amendment; a call without gen_ai.usage.output_tokens leaves the check not_verifiable."
616
+ },
617
+ "EVAL-01": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
618
+ "EVAL-02": {
619
+ sources: [
620
+ independent("sidecar-eval", "observed", PLANNED),
621
+ lock("Does the agent's CI run the declared evaluation gate before deploy?")
622
+ ],
623
+ notProven: "That a failed gate actually blocks a deploy: CI is outside the system."
624
+ },
625
+ "EVAL-03": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
626
+ "OBS-01": {
627
+ sources: [independent("sidecar-forwarder", "observed"), independent("sidecar-probe", "configured")],
628
+ notProven: "That the agent exports every span: the rule is about the backend, not span coverage. agentops is not a target: it has no OTLP ingest."
629
+ },
630
+ "OBS-02": {
631
+ sources: [
632
+ independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
633
+ point("logging", "ran-where-wired"),
634
+ lock("Does every log statement emit structured JSON?")
635
+ ]
636
+ },
637
+ "OBS-03": {
638
+ sources: [
639
+ independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
640
+ lock("Does any log statement write one of the declared sensitive fields?")
641
+ ],
642
+ notProven: "Full coverage: the scanner is a regex and pattern baseline, so recall is limited."
643
+ },
644
+ "META-01": { sources: [declared()] },
645
+ "META-02": {
646
+ sources: [independent("gateway-traffic", "observed", { when: "gen_ai.request.temperature" })],
647
+ notProven: "Requests that set no temperature: `not_verifiable`, not counted."
648
+ }
649
+ };
650
+ var RULE_EVIDENCE = Object.fromEntries(
651
+ Object.entries(entries).map(([id, entry]) => [id, RuleEvidenceEntry.parse(entry)])
652
+ );
653
+ function ruleSources(ruleId, registry = RULE_EVIDENCE, seen = /* @__PURE__ */ new Set()) {
654
+ if (!Object.hasOwn(registry, ruleId) || seen.has(ruleId)) return [];
655
+ const entry = registry[ruleId];
656
+ const visited = /* @__PURE__ */ new Set([...seen, ruleId]);
657
+ return [...entry.sources, ...(entry.derivedFrom ?? []).flatMap((parent) => ruleSources(parent, registry, visited))];
658
+ }
659
+ function ruleClasses(ruleId, registry = RULE_EVIDENCE) {
660
+ const classes = new Set(ruleSources(ruleId, registry).map((s) => s.class));
661
+ return EvidenceClass.options.filter((c) => classes.has(c));
662
+ }
663
+ var EvidenceStatus = zod.z.enum(["pass", "fail", "not_verifiable"]);
664
+ var EvidenceSource = zod.z.enum(["independent", "sdk-observed", "lower-bound", "self-declared", "declared", "offline"]);
665
+ var EvidenceOrigin = zod.z.enum(["ingress", "gateway", "egress_proxy", "in_process", "probe", "manifest", "judge"]);
666
+ var SeveritySchema = zod.z.enum(["critical", "high", "medium", "low", "info"]);
667
+ var TrustPolicy = zod.z.enum(["standard", "strict"]);
668
+ var EvidenceReason = zod.z.object({ code: zod.z.string().min(1), detail: zod.z.string().nullish() }).strict();
669
+ var TIMESTAMP_PATTERN = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
670
+ var Timestamp = zod.z.string().regex(TIMESTAMP_PATTERN);
671
+ var EvidenceWindow = zod.z.object({ start: Timestamp, end: Timestamp }).strict();
672
+ var Sha256 = zod.z.string().regex(/^[a-f0-9]{64}$/);
673
+ var MAX_TRACE_IDS = 20;
674
+ var recordFields = {
675
+ /** Scoped per subject: `tool:<name>`, `model:<id>`, or a bare check name for agent-wide checks. */
676
+ checkId: zod.z.string().min(1),
677
+ ruleId: zod.z.string().min(1).nullish(),
678
+ source: EvidenceSource,
679
+ origin: EvidenceOrigin,
680
+ /** The registry collector that produced the record. Optional until every producer sets it (#422); checking it against the collectors the registry lists for the rule is the grader's job (#423). */
681
+ collector: Collector.nullish(),
682
+ observedAt: Timestamp,
683
+ window: EvidenceWindow,
684
+ ttlSeconds: zod.z.number().int().nonnegative(),
685
+ sampleSize: zod.z.number().int().nonnegative(),
686
+ traceIds: zod.z.array(zod.z.string().min(1)).max(MAX_TRACE_IDS),
687
+ manifestSha256: Sha256
688
+ };
689
+ var EvidenceRecordPass = zod.z.object({ ...recordFields, status: zod.z.literal("pass"), reason: EvidenceReason.nullish() }).strict();
690
+ var EvidenceRecordNotPass = zod.z.object({
691
+ ...recordFields,
692
+ status: zod.z.enum(["fail", "not_verifiable"]),
693
+ reason: EvidenceReason
694
+ }).strict();
695
+ var EvidenceRecord = zod.z.union([EvidenceRecordPass, EvidenceRecordNotPass]);
696
+ var PodspecFinding = zod.z.object({ code: zod.z.string().min(1), subject: zod.z.string().min(1), message: zod.z.string().min(1) }).strict();
697
+ var ReadinessFinding = zod.z.object({
698
+ code: zod.z.string().min(1),
699
+ severity: SeveritySchema,
700
+ message: zod.z.string().min(1),
701
+ subject: zod.z.string().nullish(),
702
+ ruleIds: zod.z.array(zod.z.string().min(1)).nullish()
703
+ }).strict();
704
+ var EvalResult = zod.z.object({
705
+ metric: zod.z.string().min(1),
706
+ n: zod.z.number().int().nonnegative(),
707
+ passRate: zod.z.number().min(0).max(1),
708
+ ciLow: zod.z.number().min(0).max(1),
709
+ ciHigh: zod.z.number().min(0).max(1),
710
+ threshold: zod.z.number().min(0).max(1),
711
+ window: EvidenceWindow,
712
+ judgeModel: zod.z.string().min(1),
713
+ rubricHash: Sha256,
714
+ source: EvidenceSource,
715
+ dataset: zod.z.string().min(1).nullish()
716
+ }).strict();
717
+ var MAX_POINT_RECORDS = 5;
718
+ var EvidencePointState = zod.z.enum(["fired", "wired", "not_wired"]);
719
+ var EvidencePointAggregate = zod.z.object({
720
+ point: EvidencePoint,
721
+ /** What the point is wired on, a key of `CONTRACT.evidencePoints[point]`. */
722
+ target: zod.z.string().min(1),
723
+ state: EvidencePointState,
724
+ counts: zod.z.object({
725
+ pass: zod.z.number().int().nonnegative(),
726
+ fail: zod.z.number().int().nonnegative(),
727
+ not_verifiable: zod.z.number().int().nonnegative()
728
+ }).strict(),
729
+ records: zod.z.array(EvidenceRecord).max(MAX_POINT_RECORDS)
730
+ }).strict();
731
+ var Passthrough = zod.z.record(zod.z.string(), zod.z.unknown());
732
+ var Heartbeat = zod.z.object({
733
+ /** Contract version of the sending SDK (`CONTRACT.version`); absent or null from older SDKs. */
734
+ contract: zod.z.string().regex(/^\d+\.\d+\.\d+$/).nullish(),
735
+ health: Passthrough,
736
+ records: zod.z.array(EvidenceRecord).default([]),
737
+ /** Every evidence point the registry gives for the manifest, wired or not; absent from older SDKs. */
738
+ evidencePoints: zod.z.array(EvidencePointAggregate).default([]),
739
+ usage: Passthrough.nullish(),
740
+ tools: Passthrough.nullish()
741
+ }).strict();
742
+ var SidecarPush = zod.z.object({
743
+ manifestSha256: Sha256,
744
+ declared: Passthrough.nullish(),
745
+ records: zod.z.array(EvidenceRecord).default([])
746
+ }).strict();
747
+ var EVIDENCE_SHAPES = { EvidenceRecord, ReadinessFinding, EvalResult, TrustPolicy, Heartbeat, SidecarPush };
748
+
749
+ // src/schema/contract.schema.ts
750
+ var fixed = (value) => zod.z.literal(value).default(value);
751
+ var REASON_CODES = [
752
+ "stale",
753
+ "source_not_admitted",
754
+ "source_mismatch",
755
+ "no_evidence",
756
+ "not_wired",
757
+ "no_calls_observed",
758
+ "no_failover_observed",
759
+ "no_budget_evidence",
760
+ "no_approval_token_evidence",
761
+ "no_retrieval_spans",
762
+ "unrecognised_conventions",
763
+ "instrumentation_unavailable",
764
+ "declared_env_missing",
765
+ "provider_key_on_agent",
766
+ "response_cap_exceeded",
767
+ "temperature_above_max",
768
+ "undeclared_tool",
769
+ "undeclared_model",
770
+ "unpinned_model",
771
+ "self_declared",
772
+ "unpriced_model",
773
+ "undeclared_namespace"
774
+ ];
775
+ var ContractSchema = zod.z.object({
776
+ /** Contract version (major.minor.patch) a heartbeat carries: a receiver rejects another major and accepts a newer minor (ADR-0039). */
777
+ version: fixed("1.0.0"),
778
+ env: zod.z.object({
779
+ manifestPath: fixed("MANIFEST_PATH"),
780
+ upstreamUrl: fixed("UPSTREAM_URL"),
781
+ proxyPort: fixed("PROXY_PORT"),
782
+ controlPlanePort: fixed("CONTROL_PLANE_PORT"),
783
+ /** Control plane URL the operator webhook injects into the sidecar and agent containers; SDK push mode falls back to it. */
784
+ controlPlaneUrl: fixed("AGENTSPEC_CONTROL_PLANE_URL"),
785
+ /** Agent JWT from `POST /api/v1/register`; `agentspec register` sends it to the control plane as a Bearer token. The sidecar's own push needs the `sidecarToken` from the same call (see `agentTokenSecretSuffix`). */
786
+ agentKey: fixed("AGENTSPEC_API_KEY"),
787
+ /** Trust policy (`standard | strict`) the grader applies: set on the control plane and the sidecar, never in `agent.yaml` (ADR-0007). */
788
+ trustPolicy: fixed("AGENTSPEC_TRUST_POLICY"),
789
+ /** The webhook's PodSpec findings (the `agentspec.io/podspec-findings` annotation, JSON), injected into the sidecar by the downward API (ADR-0019). */
790
+ podspecFindings: fixed("AGENTSPEC_PODSPEC_FINDINGS"),
791
+ /** Virtual key the agent sends and the gateway accepts (ADR-0001); provider keys stay in the gateway. */
792
+ gatewayKey: fixed("AGENTSPEC_GATEWAY_KEY"),
793
+ /** Bind address and port the agent must honour so the sidecar can own `spec.api.port` (ADR-0028). */
794
+ agentHost: fixed("HOST"),
795
+ agentPort: fixed("PORT"),
796
+ /** Azure resource name the gateway config references (`$AZURE_OPENAI_RESOURCE_NAME`). */
797
+ azureResourceName: fixed("AZURE_OPENAI_RESOURCE_NAME")
798
+ }).strict().prefault({}),
799
+ /** Provider base-URL env vars, keyed by spec.model.provider. */
800
+ providerBaseUrlEnv: zod.z.object({
801
+ openai: fixed("OPENAI_BASE_URL"),
802
+ anthropic: fixed("ANTHROPIC_BASE_URL")
803
+ }).strict().prefault({}),
804
+ paths: zod.z.object({
805
+ manifestDir: fixed("/manifest"),
806
+ manifestFile: fixed("/manifest/agent.yaml"),
807
+ /** SQLite file the gateway keeps USD budget usage in; must be on a writable volume. */
808
+ gatewayDatabase: fixed("/gateway-data/agentgateway.db"),
809
+ /** Directory the init container renders the gateway config into (ADR-0025). */
810
+ gatewayConfigDir: fixed("/gateway"),
811
+ gatewayConfigFile: fixed("/gateway/config.yaml"),
812
+ /** Sidecar bundle inside its image; `node <entrypoint> gateway-config …` is the init container's command (ADR-0025). */
813
+ sidecarEntrypoint: fixed("dist/index.js"),
814
+ gatewayDataDir: fixed("/gateway-data")
815
+ }).strict().prefault({}),
816
+ images: zod.z.object({
817
+ sidecar: fixed("ghcr.io/agents-oss/agentspec-sidecar:latest"),
818
+ /** agentgateway, pinned (ADR-0025). */
819
+ gateway: fixed("cr.agentgateway.dev/agentgateway:v1.6.0")
820
+ }).strict().prefault({}),
821
+ ports: zod.z.object({
822
+ agentDefault: fixed(8e3),
823
+ /** Loopback port the agent binds once the sidecar takes over its declared port (ADR-0028). */
824
+ agentBind: fixed(18e3),
825
+ proxy: fixed(4e3),
826
+ controlPlane: fixed(4001),
827
+ /** agentgateway LLM listener on the pod loopback (ADR-0025). */
828
+ gateway: fixed(4002),
829
+ /** agentgateway's default stats and readiness listeners: wildcard-bound, not the agent's (ADR-0025). */
830
+ gatewayStats: fixed(15020),
831
+ gatewayReadiness: fixed(15021)
832
+ }).strict().prefault({}),
833
+ /** Bare paths served by the sidecar on the control-plane port (ADR-0023). */
834
+ sidecarPaths: zod.z.object({
835
+ healthReady: fixed("/health/ready"),
836
+ gap: fixed("/gap"),
837
+ explore: fixed("/explore"),
838
+ proof: fixed("/proof"),
839
+ traces: fixed("/v1/traces"),
840
+ /** Read-only GET of the buffered, already redacted spans the gateway copied to the sidecar. */
841
+ spans: fixed("/spans")
842
+ }).strict().prefault({}),
843
+ /** Bare paths an agent serves itself (SDK `/agentspec/health`); the sidecar probes them on the upstream. */
844
+ agentPaths: zod.z.object({
845
+ health: fixed("/agentspec/health")
846
+ }).strict().prefault({}),
847
+ /** Control plane API routes shared by the CLI, the sidecar push and the control plane; `{name}` is the registered agent name. */
848
+ controlPlanePaths: zod.z.object({
849
+ /** POST: registers an agent and returns its per-agent token. */
850
+ register: fixed("/api/v1/register"),
851
+ /** POST: the SDK push-mode heartbeat, authenticated with the agent token from `register`. */
852
+ heartbeat: fixed("/api/v1/heartbeat"),
853
+ declaredAudit: fixed("/api/v1/agents/{name}/declared-audit"),
854
+ /** POST: the sidecar's declared audit (at start and on a manifest hash change) and its own evidence records (ADR-0008). */
855
+ sidecarPush: fixed("/api/v1/agents/{name}/sidecar-push"),
856
+ /** GET: the latest stored EvidenceRecord per (ruleId, checkId, source) of an agent, as `{ records }`; read by the mcp-server. */
857
+ evidence: fixed("/api/v1/agents/{name}/evidence")
858
+ }).strict().prefault({}),
859
+ /** Names inside the pod patch that every injection path must use (ADR-0028). */
860
+ podPatch: zod.z.object({
861
+ sidecarContainer: fixed("agentspec-sidecar"),
862
+ /** Pod annotation the webhook writes its PodSpec findings to (JSON list of `{code, subject, message}`). */
863
+ podspecFindingsAnnotation: fixed("agentspec.io/podspec-findings"),
864
+ /** Pod annotation naming the agent; the webhook reads the manifest ConfigMap `<name>-manifest` from it, and `--deploy k8s` writes it. */
865
+ agentNameAnnotation: fixed("agentspec.io/agent-name"),
866
+ manifestVolume: fixed("agentspec-manifest"),
867
+ proxyPortName: fixed("proxy"),
868
+ controlPortName: fixed("control"),
869
+ gatewayContainer: fixed("agentspec-gateway"),
870
+ gatewayConfigInitContainer: fixed("agentspec-gateway-config"),
871
+ /** Sidecar subcommand the init container runs: `node <entrypoint> <this> <manifest> --out <path>`. */
872
+ gatewayConfigCommand: fixed("gateway-config"),
873
+ gatewayConfigVolume: fixed("agentspec-gateway-config"),
874
+ gatewayDataVolume: fixed("agentspec-gateway-data"),
875
+ /** Secret `<metadata.name><suffix>` holding the `sidecarToken` from `POST /api/v1/register`; the webhook gives it to the sidecar only, as `env.agentKey`, when a control plane URL is set. */
876
+ agentTokenSecretSuffix: fixed("-agentspec-token"),
877
+ agentTokenSecretKey: fixed("token"),
878
+ /** Loopback address the agent and sidecar reach each other and the gateway on. */
879
+ loopback: fixed("127.0.0.1"),
880
+ /** Gateway container sizing (ADR-0025): request 25m/32Mi, limit 128Mi. */
881
+ gatewayCpuRequest: fixed("25m"),
882
+ gatewayMemoryRequest: fixed("32Mi"),
883
+ gatewayMemoryLimit: fixed("128Mi")
884
+ }).strict().prefault({}),
885
+ /** Admission webhook the operator serves (ADR-0028). */
886
+ webhook: zod.z.object({
887
+ /** Path kube-apiserver posts to; kopf routes `POST /<handler id>`, so this is the handler id with a leading slash. Hyphenated: kube-apiserver rejects an underscore in a service path segment (RFC 1123). */
888
+ injectPath: fixed("/inject-sidecar"),
889
+ /** Namespace label that selects the namespaces whose pods are injected. */
890
+ namespaceLabel: fixed("agentspec.io/enabled")
891
+ }).strict().prefault({}),
892
+ /** Known `EvidenceRecord.reason.code` values (ADR-0006). */
893
+ reasonCodes: zod.z.array(zod.z.string()).default(REASON_CODES),
894
+ /** Severity weights for coverage and the verified score (ADR-0007). */
895
+ severityWeights: zod.z.object({
896
+ critical: fixed(4),
897
+ high: fixed(3),
898
+ medium: fixed(2),
899
+ low: fixed(1),
900
+ info: fixed(0)
901
+ }).strict().prefault({}),
902
+ /** Minimum verified percentage per grade; below `D` is F (ADR-0007). */
903
+ gradeBands: zod.z.object({ A: fixed(90), B: fixed(75), C: fixed(60), D: fixed(45) }).strict().prefault({}),
904
+ /** Severity-weighted coverage percentage below which no grade is shown (ADR-0007). */
905
+ coverageThreshold: fixed(60),
906
+ /** Rule evidence registry: what can prove each audit rule (ADR-0040). */
907
+ ruleEvidence: zod.z.record(zod.z.string(), RuleEvidenceEntry).default(RULE_EVIDENCE),
908
+ /** Evidence point → target → the rule its records serve; each rule's registry entry lists a source for that point. */
909
+ evidencePoints: zod.z.record(zod.z.string(), zod.z.record(zod.z.string(), zod.z.string())).default({
910
+ guardrail: { input: "SEC-LLM-01", output: "SEC-LLM-02" },
911
+ tool: { local: "TOOL-01" },
912
+ mcpClient: { "tools/list": "SEC-LLM-07", "tools/call": "TOOL-01" },
913
+ memoryWrite: { scrub: "MEM-01", ttl: "MEM-02", audit: "MEM-03" },
914
+ vectorStore: { namespace: "MEM-04" },
915
+ logging: { json: "OBS-02" }
916
+ }),
917
+ /** Records an evidence point aggregate keeps per window. */
918
+ evidencePointRecordLimit: fixed(MAX_POINT_RECORDS),
919
+ /** A heartbeat above this is trimmed to counts before it is sent. */
920
+ heartbeatMaxBytes: fixed(64 * 1024),
921
+ /** OTLP: resource attributes every exported span carries, and the standard endpoint env var. */
922
+ otlp: zod.z.object({
923
+ resourceAgentName: fixed("agentspec.agent.name"),
924
+ resourceManifestSha256: fixed("agentspec.manifest.sha256"),
925
+ endpointEnv: fixed("OTEL_EXPORTER_OTLP_ENDPOINT"),
926
+ /** Resource attribute naming the span's producer, and the value agentgateway exports (its default); only such spans are gateway evidence. */
927
+ resourceServiceName: fixed("service.name"),
928
+ gatewayServiceName: fixed("agentgateway")
929
+ }).strict().prefault({})
930
+ }).strict();
931
+ var CONTRACT = ContractSchema.parse({});
932
+
933
+ // src/schema/evaluation.ts
934
+ var REPLACED_BY = {
935
+ hallucination: "grounded",
936
+ faithfulness: "grounded"
937
+ };
938
+ function canonicalMetric(metric) {
939
+ return REPLACED_BY[metric] ?? metric;
940
+ }
941
+ function evaluationWarnings(manifest) {
942
+ const evaluation = manifest.spec.evaluation;
943
+ if (!evaluation) return [];
944
+ const warnings = [];
945
+ if (evaluation.framework !== void 0) {
946
+ warnings.push(
947
+ "spec.evaluation.framework is deprecated: nothing reads it and AgentSpec computes evaluation evidence itself. Remove it; it is rejected in 0.6.0."
948
+ );
949
+ }
950
+ const declared2 = [
951
+ ...evaluation.metrics ?? [],
952
+ ...evaluation.live?.metrics ?? [],
953
+ ...(evaluation.datasets ?? []).flatMap((d) => d.metrics ?? [])
954
+ ];
955
+ for (const metric of new Set(declared2)) {
956
+ const replacement = canonicalMetric(metric);
957
+ if (replacement !== metric) {
958
+ warnings.push(
959
+ `spec.evaluation metric "${metric}" is deprecated and is read as "${replacement}". Rename it to "${replacement}"; it is rejected in 0.6.0.`
960
+ );
961
+ }
962
+ }
963
+ warnings.push(
964
+ ...thresholdWarnings("spec.evaluation.thresholds", evaluation.thresholds),
965
+ ...thresholdWarnings("spec.evaluation.live.thresholds", evaluation.live?.thresholds)
966
+ );
967
+ return warnings;
968
+ }
969
+ function thresholdWarnings(path, thresholds) {
970
+ return Object.keys(thresholds ?? {}).flatMap((name) => {
971
+ const replacement = canonicalMetric(name);
972
+ if (replacement === name) return [];
973
+ const ignored = replacement in thresholds ? ` It is ignored: ${path}.${replacement} is also declared.` : "";
974
+ return [
975
+ `${path}.${name} is deprecated and is read as ${path}.${replacement}. Rename it to "${replacement}"; it is rejected in 0.6.0.${ignored}`
976
+ ];
370
977
  });
371
978
  }
979
+ function assertRubricRefs(parsed) {
980
+ const evaluation = parsed?.spec?.evaluation;
981
+ const live = evaluation?.live;
982
+ for (const [path, custom] of [
983
+ ["spec.evaluation.custom", evaluation?.custom],
984
+ ["spec.evaluation.live.custom", live?.custom]
985
+ ]) {
986
+ const rubric = custom?.rubric;
987
+ if (typeof rubric === "string" && !rubric.startsWith("$file:")) {
988
+ throw new Error(
989
+ `${path}.rubric must be a "$file:<path>" reference, got "${rubric}". Fix: write it as "$file:${rubric}" (path relative to agent.yaml).`
990
+ );
991
+ }
992
+ }
993
+ }
994
+ function normalizeEvaluation(manifest) {
995
+ const evaluation = manifest.spec.evaluation;
996
+ if (!evaluation) return manifest;
997
+ const metrics = (list) => list?.map(canonicalMetric);
998
+ const thresholds = (map) => {
999
+ if (!map) return map;
1000
+ const out = {};
1001
+ for (const [name, value] of Object.entries(map)) {
1002
+ const key = canonicalMetric(name);
1003
+ if (key === name || !(key in map)) out[key] = value;
1004
+ }
1005
+ return out;
1006
+ };
1007
+ return {
1008
+ ...manifest,
1009
+ spec: {
1010
+ ...manifest.spec,
1011
+ evaluation: {
1012
+ ...evaluation,
1013
+ metrics: metrics(evaluation.metrics),
1014
+ thresholds: thresholds(evaluation.thresholds),
1015
+ datasets: evaluation.datasets?.map((d) => ({ ...d, metrics: metrics(d.metrics) })),
1016
+ live: evaluation.live && {
1017
+ ...evaluation.live,
1018
+ metrics: metrics(evaluation.live.metrics),
1019
+ thresholds: thresholds(evaluation.live.thresholds)
1020
+ }
1021
+ }
1022
+ }
1023
+ };
1024
+ }
372
1025
  function detectRefType(value) {
373
1026
  if (value.startsWith("$env:")) return "env";
374
1027
  if (value.startsWith("$secret:")) return "secret";
@@ -414,8 +1067,9 @@ function resolveRef(value, opts, { optional = false } = {}) {
414
1067
  Use paths relative to agent.yaml location only`
415
1068
  );
416
1069
  }
1070
+ const readPath = checkedReadPath(value, absPath, opts.baseDir);
417
1071
  try {
418
- return fs.readFileSync(absPath, "utf-8");
1072
+ return fs.readFileSync(readPath, "utf-8");
419
1073
  } catch {
420
1074
  if (optional) return "";
421
1075
  throw new Error(
@@ -437,6 +1091,29 @@ function resolveRef(value, opts, { optional = false } = {}) {
437
1091
  }
438
1092
  return value;
439
1093
  }
1094
+ function checkedReadPath(ref, absPath, baseDir) {
1095
+ let realTarget;
1096
+ try {
1097
+ realTarget = fs.realpathSync(absPath);
1098
+ } catch (err) {
1099
+ if (err.code === "ENOENT") return absPath;
1100
+ throw new Error(
1101
+ `Cannot resolve ${ref}
1102
+ ${err.message}
1103
+ Fix the path: every folder in it must exist, be a folder and not loop back through symlinks`
1104
+ );
1105
+ }
1106
+ const realBase = fs.realpathSync(baseDir);
1107
+ if (realTarget !== realBase && !realTarget.startsWith(realBase + path.sep)) {
1108
+ throw new Error(
1109
+ `Path traversal detected in ${ref}
1110
+ Resolved through a symlink to: ${realTarget}
1111
+ Which is outside baseDir: ${baseDir}
1112
+ Replace the symlink with a real folder inside the manifest directory, or move the file there`
1113
+ );
1114
+ }
1115
+ return realTarget;
1116
+ }
440
1117
  function resolveSecret(name, backend) {
441
1118
  switch (backend) {
442
1119
  case "env": {
@@ -463,7 +1140,7 @@ function resolveSecret(name, backend) {
463
1140
  }
464
1141
  function resolveRefs(obj, opts) {
465
1142
  if (typeof obj === "string") {
466
- return resolveRef(obj, opts, { optional: true });
1143
+ return resolveRef(obj, opts);
467
1144
  }
468
1145
  if (Array.isArray(obj)) {
469
1146
  return obj.map((item) => resolveRefs(item, opts));
@@ -515,18 +1192,20 @@ function loadManifest(filePath, opts = {}) {
515
1192
  }
516
1193
  let parsed;
517
1194
  try {
518
- parsed = yaml__default.default.load(raw, { schema: yaml__default.default.JSON_SCHEMA });
1195
+ parsed = yaml__namespace.load(raw, { schema: yaml__namespace.JSON_SCHEMA });
519
1196
  } catch (err) {
520
1197
  throw new Error(`Invalid YAML in ${absPath}
521
1198
  ${String(err)}`);
522
1199
  }
1200
+ assertRubricRefs(parsed);
523
1201
  const toValidate = opts.resolve === true ? resolveRefs(parsed, {
524
1202
  baseDir,
525
1203
  secretBackend: opts.secretBackend,
526
1204
  failOnMissingEnv: opts.failOnMissingEnv ?? true
527
1205
  }) : parsed;
528
- const manifest = ManifestSchema.parse(toValidate);
529
- return { manifest, filePath: absPath, baseDir, raw };
1206
+ const declared2 = ManifestSchema.parse(toValidate);
1207
+ const warnings = evaluationWarnings(declared2);
1208
+ return { manifest: normalizeEvaluation(declared2), warnings, filePath: absPath, baseDir, raw };
530
1209
  }
531
1210
  function tryLoadManifest(filePath, opts = {}) {
532
1211
  try {
@@ -573,8 +1252,8 @@ function isLatestVersion(raw) {
573
1252
  function runEnvChecks(manifest, rawManifest) {
574
1253
  const checks = [];
575
1254
  const envRefs = collectEnvRefs(rawManifest);
576
- const declared = manifest.spec.requires?.envVars ?? [];
577
- for (const v of declared) envRefs.add(v);
1255
+ const declared2 = manifest.spec.requires?.envVars ?? [];
1256
+ for (const v of declared2) envRefs.add(v);
578
1257
  for (const varName of envRefs) {
579
1258
  const present = process.env[varName] !== void 0 && process.env[varName] !== "";
580
1259
  checks.push({
@@ -611,21 +1290,53 @@ var PROVIDER_ENDPOINTS = {
611
1290
  openai: "https://api.openai.com/v1/models",
612
1291
  anthropic: "https://api.anthropic.com/v1/models",
613
1292
  groq: "https://api.groq.com/openai/v1/models",
614
- azure: "https://management.azure.com",
615
- google: "https://generativelanguage.googleapis.com",
1293
+ google: "https://generativelanguage.googleapis.com/v1beta/models",
616
1294
  mistral: "https://api.mistral.ai/v1/models",
617
- cohere: "https://api.cohere.ai/v1",
1295
+ cohere: "https://api.cohere.com/v1/models",
618
1296
  together: "https://api.together.xyz/v1/models",
619
1297
  fireworks: "https://api.fireworks.ai/inference/v1/models"
620
1298
  };
621
- async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
622
- const url = PROVIDER_ENDPOINTS[provider.toLowerCase()];
623
- if (!url) {
1299
+ var AZURE_API_VERSION = "2024-10-21";
1300
+ var AZURE_RESOURCE_HOSTS = [".openai.azure.com", ".cognitiveservices.azure.com", ".openai.azure.us", ".openai.azure.cn"];
1301
+ function isAzureResourceEndpoint(base) {
1302
+ const { hostname, pathname } = new URL(base);
1303
+ return AZURE_RESOURCE_HOSTS.some((suffix) => hostname.endsWith(suffix)) && pathname === "/";
1304
+ }
1305
+ function probeUrl(provider, baseUrl) {
1306
+ const key = provider.toLowerCase();
1307
+ const nativeUrl = PROVIDER_ENDPOINTS[key];
1308
+ if (!nativeUrl && !(key === "azure" && baseUrl)) return void 0;
1309
+ if (!baseUrl) return { url: nativeUrl, native: true };
1310
+ const base = resolveRef(baseUrl, { baseDir: "." }).replace(/\/+$/, "");
1311
+ if (key === "azure" && isAzureResourceEndpoint(base)) {
1312
+ return { url: `${base}/openai/models?api-version=${AZURE_API_VERSION}`, native: false, azureResource: true };
1313
+ }
1314
+ if (key === "anthropic") return { url: `${base}/v1/models`, native: false };
1315
+ return { url: `${base}/models`, native: false };
1316
+ }
1317
+ function unverifiableProbe(provider) {
1318
+ if (provider.toLowerCase() === "azure") {
624
1319
  return {
625
- reachable: true
626
- // Unknown provider — assume reachable, can't check
1320
+ outcome: "not_verifiable",
1321
+ error: "azure keys are checked against your own resource endpoint, and spec.model.baseUrl is not set to name it",
1322
+ remediation: "Set spec.model.baseUrl to your Azure resource endpoint (https://<resource>.openai.azure.com) so the key can be probed, or verify it out of band"
627
1323
  };
628
1324
  }
1325
+ return { outcome: "not_verifiable", error: `unknown provider "${provider}"` };
1326
+ }
1327
+ async function checkModelEndpoint(provider, apiKey, baseUrl, timeoutMs = 5e3) {
1328
+ let target;
1329
+ try {
1330
+ target = probeUrl(provider, baseUrl);
1331
+ } catch (err) {
1332
+ return {
1333
+ outcome: "unreachable",
1334
+ error: err instanceof Error ? err.message.split("\n")[0] : String(err),
1335
+ remediation: `Set the variable behind spec.model baseUrl (${baseUrl}) or remove the override`
1336
+ };
1337
+ }
1338
+ if (!target) return unverifiableProbe(provider);
1339
+ const { url, native, azureResource } = target;
629
1340
  const start = Date.now();
630
1341
  const controller = new AbortController();
631
1342
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -639,19 +1350,30 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
639
1350
  headers["anthropic-version"] = "2023-06-01";
640
1351
  delete headers["Authorization"];
641
1352
  }
1353
+ if (azureResource) {
1354
+ headers["api-key"] = apiKey;
1355
+ delete headers["Authorization"];
1356
+ }
1357
+ const isGemini = provider.toLowerCase() === "google" && native;
1358
+ if (isGemini) {
1359
+ headers["x-goog-api-key"] = apiKey;
1360
+ delete headers["Authorization"];
1361
+ }
642
1362
  const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
643
1363
  const latencyMs = Date.now() - start;
644
- const reachable = res.status < 500;
645
- return {
646
- reachable,
647
- latencyMs,
648
- error: !reachable ? `HTTP ${res.status}` : void 0
649
- };
1364
+ const geminiBadKey = isGemini && res.status === 400;
1365
+ if (res.status === 401 || res.status === 403 || geminiBadKey) return { outcome: "rejected", latencyMs, error: `HTTP ${res.status}` };
1366
+ if (res.status >= 500) return { outcome: "unreachable", latencyMs, error: `HTTP ${res.status}` };
1367
+ if (res.status === 404 || res.status === 405 || res.status === 410) {
1368
+ const { origin, pathname } = new URL(url);
1369
+ return { outcome: "unreachable", latencyMs, error: res.status === 404 ? "HTTP 404 (endpoint not found)" : `HTTP ${res.status} (not a models endpoint)`, remediation: `Check that ${origin}${pathname} is the provider's models endpoint${native ? "" : " (fix spec.model.baseUrl)"}` };
1370
+ }
1371
+ return { outcome: "ok", latencyMs };
650
1372
  } catch (err) {
651
1373
  const latencyMs = Date.now() - start;
652
1374
  const isTimeout = err instanceof Error && err.name === "AbortError";
653
1375
  return {
654
- reachable: false,
1376
+ outcome: "unreachable",
655
1377
  latencyMs,
656
1378
  error: isTimeout ? "Request timed out" : String(err)
657
1379
  };
@@ -659,6 +1381,20 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
659
1381
  clearTimeout(timeout);
660
1382
  }
661
1383
  }
1384
+ function modelProbeCheck(probe, fields, label, provider, keyRef) {
1385
+ const keyName = keyRef.startsWith("$") ? keyRef : "[api-key]";
1386
+ const base = { ...fields, latencyMs: probe.latencyMs };
1387
+ switch (probe.outcome) {
1388
+ case "ok":
1389
+ return { ...base, status: "pass" };
1390
+ case "not_verifiable":
1391
+ return { ...base, status: "not_verifiable", message: `${label} cannot be verified: ${probe.error}`, remediation: probe.remediation ?? `Use a provider the health check knows (${Object.keys(PROVIDER_ENDPOINTS).join(", ")}) or verify ${provider} out of band` };
1392
+ case "rejected":
1393
+ return { ...base, status: "fail", message: `${label} for ${provider} rejected the API key: ${probe.error}`, remediation: `Check that ${keyName} holds a valid ${provider} API key` };
1394
+ default:
1395
+ return { ...base, status: "fail", message: `${label} for ${provider} is unreachable: ${probe.error}`, remediation: probe.remediation ?? `Check that ${keyName} is set and the ${provider} API is reachable` };
1396
+ }
1397
+ }
662
1398
  function resolveApiKey(ref) {
663
1399
  if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
664
1400
  if (ref.startsWith("$")) return "";
@@ -668,16 +1404,14 @@ async function runModelChecks(model) {
668
1404
  const checks = [];
669
1405
  const apiKey = resolveApiKey(model.apiKey);
670
1406
  if (apiKey) {
671
- const { reachable, latencyMs, error } = await checkModelEndpoint(model.provider, apiKey);
672
- checks.push({
673
- id: `model:${model.provider}/${model.id}`,
674
- category: "model",
675
- status: reachable ? "pass" : "fail",
676
- severity: "error",
677
- latencyMs,
678
- message: reachable ? void 0 : `Model endpoint for ${model.provider} is unreachable: ${error}`,
679
- remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
680
- });
1407
+ const probe = await checkModelEndpoint(model.provider, apiKey, model.baseUrl);
1408
+ checks.push(modelProbeCheck(
1409
+ probe,
1410
+ { id: `model:${model.provider}/${model.id}`, category: "model", severity: "error" },
1411
+ "Model endpoint",
1412
+ model.provider,
1413
+ model.apiKey
1414
+ ));
681
1415
  } else {
682
1416
  checks.push({
683
1417
  id: `model:${model.provider}/${model.id}`,
@@ -687,32 +1421,6 @@ async function runModelChecks(model) {
687
1421
  message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
688
1422
  });
689
1423
  }
690
- if (model.fallback) {
691
- const fallbackKey = resolveApiKey(model.fallback.apiKey);
692
- if (fallbackKey) {
693
- const { reachable, latencyMs, error } = await checkModelEndpoint(
694
- model.fallback.provider,
695
- fallbackKey
696
- );
697
- checks.push({
698
- id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
699
- category: "model-fallback",
700
- status: reachable ? "pass" : "fail",
701
- severity: "warning",
702
- latencyMs,
703
- message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
704
- remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
705
- });
706
- } else {
707
- checks.push({
708
- id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
709
- category: "model-fallback",
710
- status: "skip",
711
- severity: "warning",
712
- message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
713
- });
714
- }
715
- }
716
1424
  return checks;
717
1425
  }
718
1426
 
@@ -724,7 +1432,7 @@ async function runMcpChecks(servers) {
724
1432
  const url = server.url;
725
1433
  if (url && !url.startsWith("$")) {
726
1434
  const check = await checkHttpMcpServer(server.name, url);
727
- checks.push(check);
1435
+ checks.push(server.transport === "sse" ? withLegacySseHint(check) : check);
728
1436
  } else {
729
1437
  checks.push({
730
1438
  id: `mcp:${server.name}`,
@@ -741,37 +1449,145 @@ async function runMcpChecks(servers) {
741
1449
  }
742
1450
  return checks;
743
1451
  }
1452
+ var MCP_PROTOCOL_VERSION = "2025-03-26";
1453
+ var HANDSHAKE_TIMEOUT_MS = 5e3;
1454
+ var INITIALIZE_REQUEST_ID = 1;
1455
+ var MAX_SERVER_TEXT = 200;
1456
+ var PROTOCOL_VERSION_FORMAT = /^\d{4}-\d{2}-\d{2}$/;
744
1457
  async function checkHttpMcpServer(name, url) {
745
1458
  const controller = new AbortController();
746
- const timeout = setTimeout(() => controller.abort(), 5e3);
1459
+ const timeout = setTimeout(() => controller.abort(), HANDSHAKE_TIMEOUT_MS);
747
1460
  const start = Date.now();
1461
+ const headers = {
1462
+ "content-type": "application/json",
1463
+ accept: "application/json, text/event-stream"
1464
+ };
748
1465
  try {
749
- const res = await fetch(url, { signal: controller.signal });
1466
+ const res = await fetch(url, {
1467
+ method: "POST",
1468
+ headers,
1469
+ signal: controller.signal,
1470
+ body: JSON.stringify({
1471
+ jsonrpc: "2.0",
1472
+ id: INITIALIZE_REQUEST_ID,
1473
+ method: "initialize",
1474
+ params: {
1475
+ protocolVersion: MCP_PROTOCOL_VERSION,
1476
+ capabilities: {},
1477
+ clientInfo: { name: "agentspec-health", version: "1.0.0" }
1478
+ }
1479
+ })
1480
+ });
1481
+ if (!res.ok) {
1482
+ await res.body?.cancel().catch(() => void 0);
1483
+ return failedHandshake(name, Date.now() - start, `returned HTTP ${res.status} to initialize`);
1484
+ }
1485
+ const parsed = await readInitializeResult(res);
750
1486
  const latencyMs = Date.now() - start;
1487
+ if ("error" in parsed) {
1488
+ return failedHandshake(name, latencyMs, parsed.error);
1489
+ }
1490
+ const result = parsed.result;
1491
+ const sessionId = res.headers.get("mcp-session-id");
1492
+ if (sessionId) headers["mcp-session-id"] = sessionId;
1493
+ headers["mcp-protocol-version"] = result.protocolVersion;
1494
+ await sendBestEffort(url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) }, controller.signal);
1495
+ if (sessionId) await sendBestEffort(url, { method: "DELETE", headers }, controller.signal);
1496
+ const serverName = clip(result.serverInfo?.name);
1497
+ const serverVersion = clip(result.serverInfo?.version);
1498
+ const info = serverName ? ` (${serverName}${serverVersion ? ` ${serverVersion}` : ""})` : "";
751
1499
  return {
752
1500
  id: `mcp:${name}`,
753
1501
  category: "mcp",
754
- status: res.ok || res.status === 404 ? "pass" : "fail",
1502
+ status: "pass",
755
1503
  severity: "warning",
756
1504
  latencyMs,
757
- message: res.status >= 500 ? `MCP server ${name} returned HTTP ${res.status}` : void 0
1505
+ message: `MCP server ${name}${info} initialized, protocolVersion ${clip(result.protocolVersion)}`
758
1506
  };
759
1507
  } catch (err) {
760
1508
  const latencyMs = Date.now() - start;
761
1509
  const isTimeout = err instanceof Error && err.name === "AbortError";
762
- return {
763
- id: `mcp:${name}`,
764
- category: "mcp",
765
- status: "fail",
766
- severity: "warning",
1510
+ return failedHandshake(
1511
+ name,
767
1512
  latencyMs,
768
- message: isTimeout ? `MCP server ${name} health check timed out` : `MCP server ${name} unreachable: ${String(err)}`,
769
- remediation: `Check that the MCP server ${name} is running and accessible`
770
- };
1513
+ isTimeout ? "health check timed out" : `unreachable: ${redactUrlCredentials(String(err))}`
1514
+ );
771
1515
  } finally {
772
1516
  clearTimeout(timeout);
773
1517
  }
774
1518
  }
1519
+ async function sendBestEffort(url, init, signal) {
1520
+ await fetch(url, { ...init, signal }).then((r) => r.body?.cancel()).catch(() => void 0);
1521
+ }
1522
+ function withLegacySseHint(check) {
1523
+ if (check.status !== "fail") return check;
1524
+ return {
1525
+ ...check,
1526
+ remediation: `${check.remediation ?? ""} Note: transport "sse" is probed with the streamable-HTTP handshake; legacy HTTP+SSE servers are not supported by this check.`.trim()
1527
+ };
1528
+ }
1529
+ function redactUrlCredentials(text) {
1530
+ return text.replace(/(\w+:\/\/)[^/\s]*@/g, "$1***@");
1531
+ }
1532
+ function failedHandshake(name, latencyMs, reason) {
1533
+ return {
1534
+ id: `mcp:${name}`,
1535
+ category: "mcp",
1536
+ status: "fail",
1537
+ severity: "warning",
1538
+ latencyMs,
1539
+ message: `MCP server ${name} ${reason}`,
1540
+ remediation: `Check that MCP server ${name} is running and that spec.mcp.servers[].url points at a streamable-HTTP MCP endpoint`
1541
+ };
1542
+ }
1543
+ async function readInitializeResult(res) {
1544
+ const isSse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
1545
+ const text = isSse ? await readSseUntilInitialize(res) : await res.text();
1546
+ const payloads = isSse ? splitSseEvents(text) : [text];
1547
+ for (const payload of payloads) {
1548
+ const outcome = parseInitializeFrame(payload);
1549
+ if (outcome) return outcome;
1550
+ }
1551
+ return { error: "did not return a valid JSON-RPC initialize result" };
1552
+ }
1553
+ function clip(value) {
1554
+ return typeof value === "string" ? value.slice(0, MAX_SERVER_TEXT) : "";
1555
+ }
1556
+ function parseInitializeFrame(payload) {
1557
+ try {
1558
+ const msg = JSON.parse(payload);
1559
+ if (msg.jsonrpc !== "2.0" || msg.id !== INITIALIZE_REQUEST_ID) return null;
1560
+ if (msg.result && typeof msg.result.protocolVersion === "string") {
1561
+ return PROTOCOL_VERSION_FORMAT.test(msg.result.protocolVersion) ? { result: msg.result } : { error: "returned a malformed protocolVersion in its initialize result" };
1562
+ }
1563
+ if (msg.error) return { error: `rejected initialize: ${clip(msg.error.message) || "JSON-RPC error"}` };
1564
+ } catch {
1565
+ }
1566
+ return null;
1567
+ }
1568
+ function splitSseEvents(text) {
1569
+ return text.split(/\r?\n\r?\n/).map(
1570
+ (event) => event.split(/\r?\n/).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).join("\n")
1571
+ );
1572
+ }
1573
+ async function readSseUntilInitialize(res) {
1574
+ if (!res.body) return res.text();
1575
+ const reader = res.body.getReader();
1576
+ const decoder = new TextDecoder();
1577
+ let text = "";
1578
+ try {
1579
+ for (; ; ) {
1580
+ const { done, value } = await reader.read();
1581
+ if (done) break;
1582
+ text += decoder.decode(value, { stream: true });
1583
+ const completeEvents = text.split(/\r?\n\r?\n/).slice(0, -1);
1584
+ if (completeEvents.flatMap((e) => splitSseEvents(e)).some((p) => parseInitializeFrame(p))) break;
1585
+ }
1586
+ } finally {
1587
+ await reader.cancel().catch(() => void 0);
1588
+ }
1589
+ return text;
1590
+ }
775
1591
  async function checkStdioMcpServer(server) {
776
1592
  const command = server.command;
777
1593
  if (!command) {
@@ -1289,7 +2105,7 @@ function computeHealthStatus(checks) {
1289
2105
  const failed = checks.filter((c) => c.status === "fail");
1290
2106
  const warnings = checks.filter((c) => c.status === "warn");
1291
2107
  const passed = checks.filter((c) => c.status === "pass");
1292
- const skipped = checks.filter((c) => c.status === "skip");
2108
+ const skipped = checks.filter((c) => c.status === "skip" || c.status === "not_verifiable");
1293
2109
  const hasErrors = failed.some((c) => c.severity === "error");
1294
2110
  const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
1295
2111
  const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
@@ -1306,8 +2122,10 @@ function computeHealthStatus(checks) {
1306
2122
  async function runHealthCheck(manifest, opts = {}) {
1307
2123
  const checks = [];
1308
2124
  const rawManifest = opts.rawManifest ?? manifest;
1309
- checks.push(...runEnvChecks(manifest, rawManifest));
1310
- checks.push(...await runSecretChecks());
2125
+ if (opts.checkEnv !== false) {
2126
+ checks.push(...runEnvChecks(manifest, rawManifest));
2127
+ checks.push(...await runSecretChecks());
2128
+ }
1311
2129
  if (opts.baseDir) {
1312
2130
  checks.push(...runFileChecks(rawManifest, opts.baseDir));
1313
2131
  }
@@ -1397,7 +2215,381 @@ async function checkA2aEndpoint(name, url) {
1397
2215
  }
1398
2216
  }
1399
2217
 
2218
+ // src/grader/index.ts
2219
+ var ADMITTED = {
2220
+ standard: ["independent", "sdk-observed"],
2221
+ strict: ["independent"]
2222
+ };
2223
+ var SOURCE_CLASS = {
2224
+ independent: "independent",
2225
+ "sdk-observed": "evidence-point",
2226
+ offline: "offline",
2227
+ declared: "declared"
2228
+ };
2229
+ var pct = (part, whole) => whole === 0 ? 0 : part / whole * 100;
2230
+ function isCalendarDate(value) {
2231
+ const [y, m, d] = value.slice(0, 10).split("-").map(Number);
2232
+ const date = /* @__PURE__ */ new Date(0);
2233
+ date.setUTCFullYear(y, m - 1, d);
2234
+ return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
2235
+ }
2236
+ function parseTimestamp(value, field) {
2237
+ const ms = Date.parse(value);
2238
+ const valid = TIMESTAMP_PATTERN.test(value) && !Number.isNaN(ms) && isCalendarDate(value) && !value.startsWith("0000") && !value.includes("T24:");
2239
+ if (!valid) {
2240
+ throw new Error(`grade: ${field} "${value}" is not a valid RFC 3339 timestamp. Fix the producer to emit a real date and time.`);
2241
+ }
2242
+ return ms;
2243
+ }
2244
+ function isExpired(r, nowMs) {
2245
+ return parseTimestamp(r.observedAt, `observedAt of the record for check "${r.checkId}"`) + r.ttlSeconds * 1e3 < nowMs;
2246
+ }
2247
+ function matchesRegistry(r) {
2248
+ const cls = SOURCE_CLASS[r.source];
2249
+ if (!r.ruleId || !cls || !Object.hasOwn(RULE_EVIDENCE, r.ruleId)) return true;
2250
+ const listed = RULE_EVIDENCE[r.ruleId].sources.filter((s) => s.class === cls);
2251
+ return listed.length > 0 && (r.collector == null || listed.some((s) => s.collector === r.collector));
2252
+ }
2253
+ function unverifiedReason(admitted, stale, notAdmitted, mismatched) {
2254
+ const own = admitted.find((r) => r.status === "not_verifiable");
2255
+ if (own?.reason) return own.reason;
2256
+ if (stale.length > 0) return { code: "stale" };
2257
+ if (notAdmitted.length > 0) return { code: "source_not_admitted" };
2258
+ if (mismatched.length > 0) return { code: "source_mismatch" };
2259
+ return { code: "no_evidence" };
2260
+ }
2261
+ function judgeRule(allRecords, policy, nowMs, unwiredFails) {
2262
+ if (unwiredFails) allRecords = [...allRecords.filter((r) => r.source !== "sdk-observed"), ...unwiredFails];
2263
+ const admittedSources = ADMITTED[policy];
2264
+ const registryMatched = [];
2265
+ const mismatched = [];
2266
+ for (const r of allRecords) (matchesRegistry(r) ? registryMatched : mismatched).push(r);
2267
+ const admittedRecords = registryMatched.filter((r) => admittedSources.includes(r.source));
2268
+ const notAdmitted = registryMatched.filter((r) => !admittedSources.includes(r.source));
2269
+ const stale = admittedRecords.filter((r) => isExpired(r, nowMs));
2270
+ const fresh = admittedRecords.filter((r) => !isExpired(r, nowMs));
2271
+ const decisive = fresh.filter((r) => r.status !== "not_verifiable");
2272
+ if (decisive.length === 0) {
2273
+ const unverified = unwiredFails ? { code: "not_wired" } : unverifiedReason(fresh, stale, notAdmitted, mismatched);
2274
+ return { covered: false, pass: false, independent: false, unverified };
2275
+ }
2276
+ return {
2277
+ covered: true,
2278
+ pass: decisive.every((r) => r.status === "pass"),
2279
+ independent: decisive.every((r) => r.source === "independent")
2280
+ };
2281
+ }
2282
+ function unwiredRules(points) {
2283
+ const notWired = /* @__PURE__ */ new Set();
2284
+ const wiredFails = /* @__PURE__ */ new Map();
2285
+ for (const p of points) {
2286
+ const ruleId = CONTRACT.evidencePoints[p.point]?.[p.target];
2287
+ if (!ruleId) continue;
2288
+ if (p.state === "not_wired") notWired.add(ruleId);
2289
+ else {
2290
+ const fails = p.records.filter((r) => r.ruleId === ruleId && r.status === "fail");
2291
+ wiredFails.set(ruleId, [...wiredFails.get(ruleId) ?? [], ...fails]);
2292
+ }
2293
+ }
2294
+ return new Map([...notWired].map((id) => [id, wiredFails.get(id) ?? []]));
2295
+ }
2296
+ function judgeDerived(parents) {
2297
+ const open = parents.find((v) => !v.covered);
2298
+ const failed = parents.some((v) => v.covered && !v.pass);
2299
+ if (open && !failed) return open;
2300
+ const covered = parents.filter((v) => v.covered);
2301
+ return { covered: true, pass: !failed, independent: covered.every((v) => v.independent) };
2302
+ }
2303
+ function judgeRegistered(ruleId, byRule, policy, nowMs, unwired) {
2304
+ const derivedFrom = Object.hasOwn(RULE_EVIDENCE, ruleId) ? RULE_EVIDENCE[ruleId].derivedFrom : void 0;
2305
+ if (!derivedFrom?.length) return judgeRule(byRule.get(ruleId) ?? [], policy, nowMs, unwired.get(ruleId));
2306
+ return judgeDerived(derivedFrom.map((parent) => judgeRegistered(parent, byRule, policy, nowMs, unwired)));
2307
+ }
2308
+ function groupByRule(records) {
2309
+ const byRule = /* @__PURE__ */ new Map();
2310
+ for (const r of records) {
2311
+ if (!r.ruleId) continue;
2312
+ const bucket2 = byRule.get(r.ruleId);
2313
+ if (bucket2) bucket2.push(r);
2314
+ else byRule.set(r.ruleId, [r]);
2315
+ }
2316
+ return byRule;
2317
+ }
2318
+ function scoreToGrade(score) {
2319
+ const { A, B, C, D } = CONTRACT.gradeBands;
2320
+ if (score >= A) return "A";
2321
+ if (score >= B) return "B";
2322
+ if (score >= C) return "C";
2323
+ if (score >= D) return "D";
2324
+ return "F";
2325
+ }
2326
+ function grade(records, applicableRules, policy, now, points = []) {
2327
+ const nowMs = parseTimestamp(now, "now");
2328
+ const byRule = groupByRule(records);
2329
+ const unwired = unwiredRules(points);
2330
+ let totalWeight = 0;
2331
+ let coveredWeight = 0;
2332
+ let passedWeight = 0;
2333
+ let independentWeight = 0;
2334
+ const unverified = [];
2335
+ for (const rule of applicableRules) {
2336
+ const weight = CONTRACT.severityWeights[rule.severity];
2337
+ const verdict = judgeRegistered(rule.id, byRule, policy, nowMs, unwired);
2338
+ totalWeight += weight;
2339
+ if (!verdict.covered) {
2340
+ unverified.push({ ruleId: rule.id, severity: rule.severity, reason: verdict.unverified });
2341
+ continue;
2342
+ }
2343
+ coveredWeight += weight;
2344
+ if (verdict.pass) passedWeight += weight;
2345
+ if (verdict.independent) independentWeight += weight;
2346
+ }
2347
+ const verified = Math.round(pct(passedWeight, coveredWeight));
2348
+ const coverage = Math.round(pct(coveredWeight, totalWeight));
2349
+ return {
2350
+ verified,
2351
+ coverage,
2352
+ independentShare: Math.round(pct(independentWeight, coveredWeight)),
2353
+ grade: coverage >= CONTRACT.coverageThreshold ? scoreToGrade(verified) : "Not graded",
2354
+ unverified
2355
+ };
2356
+ }
2357
+ function formatVerified(result) {
2358
+ if (result.grade === "Not graded") return `Not graded (coverage ${result.coverage}%)`;
2359
+ return `${result.grade} - ${result.verified}% verified (${result.independentShare}% independent) - coverage ${result.coverage}%`;
2360
+ }
2361
+ var NATIVE_PROVIDERS = {
2362
+ openai: "openAI",
2363
+ anthropic: "anthropic",
2364
+ google: "gemini",
2365
+ gemini: "gemini",
2366
+ azure: "azure"
2367
+ };
2368
+ var OPENAI_COMPATIBLE_BASE_URLS = {
2369
+ groq: "https://api.groq.com/openai/v1",
2370
+ mistral: "https://api.mistral.ai/v1"
2371
+ };
2372
+ var CATALOG_PROVIDER_IDS = {
2373
+ openAI: "openai",
2374
+ anthropic: "anthropic",
2375
+ gemini: "gcp.gemini",
2376
+ azure: "azure"
2377
+ };
2378
+ var GATEWAY_NAME = "llm";
2379
+ var ONE_MONTH = "30d";
2380
+ function envRefName(value) {
2381
+ return /^\$env:(.+)$/.exec(value)?.[1];
2382
+ }
2383
+ function secretEnvName(name) {
2384
+ return `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
2385
+ }
2386
+ function keyEnvName(apiKey) {
2387
+ const env = envRefName(apiKey);
2388
+ if (env) return env;
2389
+ const secret = /^\$secret:(.+)$/.exec(apiKey);
2390
+ if (secret) return secretEnvName(secret[1]);
2391
+ throw new Error(
2392
+ 'spec.model.apiKey must be a "$env:VAR" or "$secret:name" reference; a literal key would end up in the gateway config. Replace it with "$env:<VAR>".'
2393
+ );
2394
+ }
2395
+ function baseUrlValue(baseUrl) {
2396
+ const env = envRefName(baseUrl);
2397
+ return env ? `$${env}` : baseUrl;
2398
+ }
2399
+ function baseUrlEnvNames({ baseUrl }) {
2400
+ const name = baseUrl && envRefName(baseUrl);
2401
+ return name ? [name] : [];
2402
+ }
2403
+ function customProvider(provider, id, baseUrl) {
2404
+ const anthropic = provider === "anthropic";
2405
+ return {
2406
+ provider: { custom: { providerOverride: provider, formats: [{ type: anthropic ? "messages" : "completions" }] } },
2407
+ params: { model: id, baseUrl, ...anthropic ? { pathPrefix: "/v1" } : {} }
2408
+ };
2409
+ }
2410
+ function gatewayEnvNames(manifest) {
2411
+ const { model } = manifest.spec;
2412
+ const models = [model, ...model.fallback ? [model.fallback] : []];
2413
+ const names = models.flatMap((m) => [
2414
+ keyEnvName(m.apiKey),
2415
+ ...baseUrlEnvNames(m),
2416
+ ...NATIVE_PROVIDERS[m.provider] === "azure" && !m.baseUrl ? [CONTRACT.env.azureResourceName] : []
2417
+ ]);
2418
+ return [...new Set(names)].sort();
2419
+ }
2420
+ function providerFields({ provider, id, baseUrl }) {
2421
+ if (baseUrl) return customProvider(provider, id, baseUrlValue(baseUrl));
2422
+ const native = NATIVE_PROVIDERS[provider];
2423
+ if (native === "azure") {
2424
+ return {
2425
+ provider: native,
2426
+ params: { model: id, azureResourceName: `$${CONTRACT.env.azureResourceName}`, azureResourceType: "openAI" }
2427
+ };
2428
+ }
2429
+ if (native) return { provider: native, params: { model: id } };
2430
+ const knownBaseUrl = OPENAI_COMPATIBLE_BASE_URLS[provider];
2431
+ if (knownBaseUrl) return customProvider(provider, id, knownBaseUrl);
2432
+ const supported = [...Object.keys(NATIVE_PROVIDERS), ...Object.keys(OPENAI_COMPATIBLE_BASE_URLS)].sort();
2433
+ throw new Error(`spec.model.provider "${provider}" has no gateway mapping. Supported: ${supported.join(", ")}.`);
2434
+ }
2435
+ function renderModel(model) {
2436
+ const { provider, params } = providerFields(model);
2437
+ return {
2438
+ name: `${model.provider}/${model.id}`,
2439
+ visibility: "internal",
2440
+ provider,
2441
+ params: { ...params, apiKey: `$${keyEnvName(model.apiKey)}` }
2442
+ };
2443
+ }
2444
+ function renderVirtualModel(models) {
2445
+ return {
2446
+ name: models[0].id,
2447
+ routing: {
2448
+ failover: {
2449
+ targets: models.map((m, priority) => ({ model: `${m.provider}/${m.id}`, priority }))
2450
+ }
2451
+ }
2452
+ };
2453
+ }
2454
+ function renderRetryPolicy(model) {
2455
+ const attempts = model.fallback?.maxRetries;
2456
+ if (!attempts) return {};
2457
+ return {
2458
+ policies: [{
2459
+ name: { name: "llm-retry", namespace: "default" },
2460
+ target: { gateway: { gatewayName: "default", gatewayNamespace: "default", listenerName: GATEWAY_NAME } },
2461
+ policy: { retry: { attempts, codes: [500, 502, 503, 504] } }
2462
+ }]
2463
+ };
2464
+ }
2465
+ function renderBudgets(spec) {
2466
+ const amount = spec.model.costControls?.maxMonthlyUSD;
2467
+ if (amount === void 0) return {};
2468
+ return {
2469
+ budgets: [{
2470
+ name: "monthly",
2471
+ limit: { unit: "USD", amount },
2472
+ window: { rolling: ONE_MONTH },
2473
+ onBudgetExceeded: "Block"
2474
+ }]
2475
+ };
2476
+ }
2477
+ function renderApiKeyPolicy(manifest) {
2478
+ return {
2479
+ mode: "strict",
2480
+ keys: [{
2481
+ key: `$${CONTRACT.env.gatewayKey}`,
2482
+ metadata: { name: manifest.metadata.name },
2483
+ allowedModels: [manifest.spec.model.id],
2484
+ ...renderBudgets(manifest.spec)
2485
+ }]
2486
+ };
2487
+ }
2488
+ function bucket(type, size, fillInterval) {
2489
+ return { type, maxTokens: size, tokensPerFill: size, fillInterval };
2490
+ }
2491
+ function renderRateLimits(spec) {
2492
+ const { requestsPerMinute, requestsPerHour, tokensPerMinute } = spec.api?.rateLimit ?? {};
2493
+ const buckets = [
2494
+ ...requestsPerMinute === void 0 ? [] : [bucket("requests", requestsPerMinute, "60s")],
2495
+ ...requestsPerHour === void 0 ? [] : [bucket("requests", requestsPerHour, "1h")],
2496
+ ...tokensPerMinute === void 0 ? [] : [bucket("tokens", tokensPerMinute, "60s")]
2497
+ ];
2498
+ return buckets.length === 0 ? {} : { localRateLimit: buckets };
2499
+ }
2500
+ function renderRuntimeConfig(spec) {
2501
+ return {
2502
+ ...spec.model.costControls?.maxMonthlyUSD !== void 0 && {
2503
+ database: { url: CONTRACT.paths.gatewayDatabase }
2504
+ },
2505
+ // agentgateway samples no request by default and keeps prompt and completion out of the
2506
+ // span; the sidecar needs every LLM call with both (ADR-0030), and redacts before buffering.
2507
+ tracing: {
2508
+ otlpEndpoint: `http://${CONTRACT.podPatch.loopback}:${CONTRACT.ports.controlPlane}`,
2509
+ otlpProtocol: "http",
2510
+ path: CONTRACT.sidecarPaths.traces,
2511
+ randomSampling: "true",
2512
+ fields: { add: { "gen_ai.prompt": "llm.prompt", "gen_ai.completion": "llm.completion" } }
2513
+ }
2514
+ };
2515
+ }
2516
+ function catalogProvider(model) {
2517
+ const { provider } = providerFields(model);
2518
+ return typeof provider === "string" ? CATALOG_PROVIDER_IDS[provider] : model.provider;
2519
+ }
2520
+ function catalogRates(pricing) {
2521
+ return Object.fromEntries(Object.entries(pricing).filter(([, v]) => v !== void 0).map(([k, v]) => [k, String(v)]));
2522
+ }
2523
+ function renderModelCatalog(models) {
2524
+ const providers = {};
2525
+ for (const model of models) {
2526
+ if (!model.pricing) continue;
2527
+ const key = catalogProvider(model);
2528
+ providers[key] ??= { models: {} };
2529
+ providers[key].models[model.id] = { rates: catalogRates(model.pricing) };
2530
+ }
2531
+ return Object.keys(providers).length === 0 ? {} : { modelCatalog: [{ inline: { providers } }] };
2532
+ }
2533
+ function renderGatewayConfig(manifest) {
2534
+ const { spec } = manifest;
2535
+ const models = [spec.model, ...spec.model.fallback ? [spec.model.fallback] : []];
2536
+ const config = {
2537
+ config: { ...renderRuntimeConfig(spec), ...renderModelCatalog(models) },
2538
+ gateways: { [GATEWAY_NAME]: { port: CONTRACT.ports.gateway, bindAddress: CONTRACT.podPatch.loopback } },
2539
+ llm: {
2540
+ gateways: GATEWAY_NAME,
2541
+ models: models.map(renderModel),
2542
+ virtualModels: [renderVirtualModel(models)],
2543
+ policies: {
2544
+ apiKey: renderApiKeyPolicy(manifest),
2545
+ ...renderRateLimits(spec)
2546
+ }
2547
+ },
2548
+ ...renderRetryPolicy(spec.model)
2549
+ };
2550
+ return yaml.dump(config, { sortKeys: true, lineWidth: -1, noRefs: true });
2551
+ }
2552
+ function describeManifestError(err) {
2553
+ if (!(err instanceof Error)) return String(err);
2554
+ if (!err.message.startsWith("[")) {
2555
+ return err.message.split("\n")[0];
2556
+ }
2557
+ try {
2558
+ const issues = JSON.parse(err.message);
2559
+ return issues.map((i) => `${i.path.join(".") || "root"}: ${i.message}`).join("\n");
2560
+ } catch {
2561
+ return err.message.split("\n")[0];
2562
+ }
2563
+ }
2564
+
1400
2565
  // src/audit/rules/model.rules.ts
2566
+ var isLatestModelId = (id) => id === "latest" || id.endsWith(":latest");
2567
+ function fallbackRetryResult(model) {
2568
+ if (!model.fallback) return { pass: true };
2569
+ const { maxRetries } = model.fallback;
2570
+ const { policies } = renderRetryPolicy(model);
2571
+ const attempts = policies ? policies[0].policy.retry.attempts : 0;
2572
+ const fail = (message, recommendation) => ({
2573
+ pass: false,
2574
+ message,
2575
+ path: "/spec/model/fallback/maxRetries",
2576
+ recommendation,
2577
+ references: []
2578
+ });
2579
+ if (maxRetries === void 0) {
2580
+ return fail(
2581
+ "Fallback model has no maxRetries configured: the gateway renders no retry, so the request that triggers failover fails",
2582
+ "Add maxRetries: 2 to spec.model.fallback"
2583
+ );
2584
+ }
2585
+ if (attempts !== maxRetries) {
2586
+ return fail(
2587
+ `The rendered gateway config sets ${attempts} retry attempts, but spec.model.fallback declares maxRetries ${maxRetries}`,
2588
+ "Redeploy the agent so its gateway config is rendered again from agent.yaml and retries maxRetries times"
2589
+ );
2590
+ }
2591
+ return { pass: true };
2592
+ }
1401
2593
  var modelRules = [
1402
2594
  {
1403
2595
  id: "MODEL-01",
@@ -1405,9 +2597,7 @@ var modelRules = [
1405
2597
  title: "Model fallback declared",
1406
2598
  description: "A fallback model should be configured to handle rate limits and errors",
1407
2599
  severity: "high",
1408
- evidenceLevel: "external",
1409
- proofTool: "LiteLLM chaos test",
1410
- proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
2600
+ awaitsEvidence: { code: "no_failover_observed", phase: 2 },
1411
2601
  check(manifest) {
1412
2602
  const hasFallback = !!manifest.spec.model.fallback;
1413
2603
  return {
@@ -1425,17 +2615,14 @@ var modelRules = [
1425
2615
  title: "Model version pinned",
1426
2616
  description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
1427
2617
  severity: "medium",
1428
- evidenceLevel: "probed",
1429
- proofTool: "agentspec health",
1430
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1431
2618
  check(manifest) {
1432
2619
  const id = manifest.spec.model.id;
1433
- const isLatest = id === "latest" || id.endsWith(":latest");
2620
+ const isLatest = isLatestModelId(id);
1434
2621
  return {
1435
2622
  pass: !isLatest,
1436
2623
  message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
1437
2624
  path: "/spec/model/id",
1438
- recommendation: 'Pin a specific model version, e.g. "llama-3.3-70b-versatile" or "gpt-4-0125-preview"',
2625
+ recommendation: 'Pin a specific model version, e.g. "openai/gpt-oss-20b" or "gpt-4-0125-preview"',
1439
2626
  references: []
1440
2627
  };
1441
2628
  }
@@ -1446,9 +2633,7 @@ var modelRules = [
1446
2633
  title: "Cost controls declared",
1447
2634
  description: "Monthly cost limits prevent unexpected API bills",
1448
2635
  severity: "medium",
1449
- evidenceLevel: "external",
1450
- proofTool: "LiteLLM Spend Tracking",
1451
- proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
2636
+ awaitsEvidence: { code: "no_budget_evidence", phase: 2 },
1452
2637
  check(manifest) {
1453
2638
  const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
1454
2639
  return {
@@ -1464,19 +2649,61 @@ var modelRules = [
1464
2649
  id: "MODEL-04",
1465
2650
  pack: "model-resilience",
1466
2651
  title: "Fallback retry strategy configured",
1467
- description: "Fallback should have maxRetries to avoid infinite loops",
2652
+ description: "Fallback should have maxRetries, and the rendered gateway config must retry exactly that many times",
1468
2653
  severity: "low",
1469
- evidenceLevel: "external",
1470
- proofTool: "pytest-mockllm",
1471
- proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
1472
2654
  check(manifest) {
1473
- if (!manifest.spec.model.fallback) return { pass: true };
1474
- const hasRetries = manifest.spec.model.fallback.maxRetries !== void 0;
2655
+ return fallbackRetryResult(manifest.spec.model);
2656
+ }
2657
+ },
2658
+ {
2659
+ id: "MODEL-05",
2660
+ pack: "model-resilience",
2661
+ title: "Budgeted model is priced",
2662
+ description: "A monthly USD budget is only enforced for models the gateway can price",
2663
+ severity: "high",
2664
+ // Only the gateway knows its prices: MODEL-05 is graded from the cost on its spans (#494), never guessed from the model id (#495).
2665
+ check() {
2666
+ return { pass: true };
2667
+ }
2668
+ }
2669
+ ];
2670
+
2671
+ // src/audit/rules/metadata.rules.ts
2672
+ var MAX_PRODUCTION_TEMPERATURE = 1.5;
2673
+ var metadataRules = [
2674
+ {
2675
+ id: "META-01",
2676
+ pack: "metadata-quality",
2677
+ title: "Agent description declared",
2678
+ description: "Agents should have a non-empty description for fleet discovery and documentation",
2679
+ severity: "low",
2680
+ check(manifest) {
2681
+ const desc = manifest.metadata.description;
2682
+ const pass = typeof desc === "string" && desc.trim().length > 0;
2683
+ return {
2684
+ pass,
2685
+ message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
2686
+ path: "/metadata/description",
2687
+ recommendation: "Add a concise metadata.description explaining what the agent does",
2688
+ references: []
2689
+ };
2690
+ }
2691
+ },
2692
+ {
2693
+ id: "META-02",
2694
+ pack: "metadata-quality",
2695
+ title: "Model temperature is production-safe",
2696
+ description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
2697
+ severity: "medium",
2698
+ check(manifest) {
2699
+ const temperature = manifest.spec.model.parameters?.temperature;
2700
+ if (temperature === void 0) return { pass: true };
2701
+ const pass = temperature <= MAX_PRODUCTION_TEMPERATURE;
1475
2702
  return {
1476
- pass: hasRetries,
1477
- message: hasRetries ? void 0 : "Fallback model has no maxRetries configured",
1478
- path: "/spec/model/fallback/maxRetries",
1479
- recommendation: "Add maxRetries: 2 to spec.model.fallback",
2703
+ pass,
2704
+ message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above ${MAX_PRODUCTION_TEMPERATURE} produce unreliable outputs in production.`,
2705
+ path: "/spec/model/parameters/temperature",
2706
+ recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
1480
2707
  references: []
1481
2708
  };
1482
2709
  }
@@ -1491,9 +2718,6 @@ var securityRules = [
1491
2718
  title: "Prompt injection guard: input guardrail required",
1492
2719
  description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
1493
2720
  severity: "high",
1494
- evidenceLevel: "behavioral",
1495
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1496
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1497
2721
  check(manifest) {
1498
2722
  const hasGuardrail = manifest.spec.guardrails?.input?.some(
1499
2723
  (g) => g.type === "prompt-injection" || g.type === "topic-filter"
@@ -1515,9 +2739,6 @@ var securityRules = [
1515
2739
  title: "Insecure output handling: output guardrail required",
1516
2740
  description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
1517
2741
  severity: "high",
1518
- evidenceLevel: "behavioral",
1519
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1520
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1521
2742
  check(manifest) {
1522
2743
  const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
1523
2744
  return {
@@ -1537,9 +2758,6 @@ var securityRules = [
1537
2758
  title: "Training/prompt data poisoning: prompt files referenced",
1538
2759
  description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
1539
2760
  severity: "medium",
1540
- evidenceLevel: "probed",
1541
- proofTool: "agentspec health",
1542
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1543
2761
  check(manifest) {
1544
2762
  const system = manifest.spec.prompts.system;
1545
2763
  const usesFile = system.startsWith("$file:");
@@ -1557,21 +2775,16 @@ var securityRules = [
1557
2775
  {
1558
2776
  id: "SEC-LLM-04",
1559
2777
  pack: "owasp-llm-top10",
1560
- title: "Model DoS: rate limiting + cost controls declared",
2778
+ title: "Model DoS: rate limiting configured",
1561
2779
  description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
1562
2780
  severity: "medium",
1563
- evidenceLevel: "external",
1564
- proofTool: "k6 load test",
1565
- proofToolUrl: "https://k6.io",
1566
2781
  check(manifest) {
1567
- const hasRateLimit = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
1568
- const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
1569
- const pass = hasRateLimit || hasCostControls;
2782
+ const pass = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
1570
2783
  return {
1571
2784
  pass,
1572
- message: pass ? void 0 : "No rate limiting or cost controls declared.",
2785
+ message: pass ? void 0 : "No rate limiting declared.",
1573
2786
  path: "/spec/api/rateLimit",
1574
- recommendation: "Add spec.api.rateLimit.requestsPerMinute and spec.model.costControls.maxMonthlyUSD",
2787
+ recommendation: "Add spec.api.rateLimit.requestsPerMinute",
1575
2788
  references: [
1576
2789
  "https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
1577
2790
  ]
@@ -1584,9 +2797,6 @@ var securityRules = [
1584
2797
  title: "Supply chain: model provider and version pinned",
1585
2798
  description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
1586
2799
  severity: "medium",
1587
- evidenceLevel: "probed",
1588
- proofTool: "agentspec health",
1589
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1590
2800
  check(manifest) {
1591
2801
  const hasProvider = !!manifest.spec.model.provider;
1592
2802
  const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
@@ -1608,9 +2818,6 @@ var securityRules = [
1608
2818
  title: "Sensitive data disclosure: PII scrub in memory hygiene",
1609
2819
  description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
1610
2820
  severity: "critical",
1611
- evidenceLevel: "external",
1612
- proofTool: "Microsoft Presidio",
1613
- proofToolUrl: "https://microsoft.github.io/presidio/",
1614
2821
  check(manifest) {
1615
2822
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1616
2823
  const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
@@ -1631,9 +2838,6 @@ var securityRules = [
1631
2838
  title: "Insecure plugin design: tool annotations declared",
1632
2839
  description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
1633
2840
  severity: "medium",
1634
- evidenceLevel: "external",
1635
- proofTool: "Promptfoo",
1636
- proofToolUrl: "https://promptfoo.dev",
1637
2841
  check(manifest) {
1638
2842
  if (!manifest.spec.tools?.length) return { pass: true };
1639
2843
  const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
@@ -1654,9 +2858,7 @@ var securityRules = [
1654
2858
  title: "Excessive agency: destructiveHint declared on tools",
1655
2859
  description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
1656
2860
  severity: "high",
1657
- evidenceLevel: "external",
1658
- proofTool: "Promptfoo",
1659
- proofToolUrl: "https://promptfoo.dev",
2861
+ awaitsEvidence: { code: "no_approval_token_evidence", phase: 5 },
1660
2862
  check(manifest) {
1661
2863
  if (!manifest.spec.tools?.length) return { pass: true };
1662
2864
  const allHaveDestructiveHint = manifest.spec.tools.every(
@@ -1676,21 +2878,18 @@ var securityRules = [
1676
2878
  {
1677
2879
  id: "SEC-LLM-09",
1678
2880
  pack: "owasp-llm-top10",
1679
- title: "Overreliance: evaluation framework + CI gate configured",
2881
+ title: "Overreliance: evaluation + CI gate configured",
1680
2882
  description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
1681
2883
  severity: "medium",
1682
- evidenceLevel: "probed",
1683
- proofTool: "agentspec health",
1684
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1685
2884
  check(manifest) {
1686
- const hasEval = !!manifest.spec.evaluation?.framework;
2885
+ const hasEval = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0 || (manifest.spec.evaluation?.live?.metrics?.length ?? 0) > 0;
1687
2886
  const hasCiGate = manifest.spec.evaluation?.ciGate === true;
1688
2887
  const pass = hasEval && hasCiGate;
1689
2888
  return {
1690
2889
  pass,
1691
- message: pass ? void 0 : !hasEval ? "No evaluation framework configured." : "Evaluation CI gate is not enabled.",
2890
+ message: pass ? void 0 : !hasEval ? "No evaluation datasets or live evaluation configured." : "Evaluation CI gate is not enabled.",
1692
2891
  path: "/spec/evaluation",
1693
- recommendation: "Configure spec.evaluation.framework and set spec.evaluation.ciGate: true",
2892
+ recommendation: "Configure spec.evaluation.datasets (or spec.evaluation.live) and set spec.evaluation.ciGate: true",
1694
2893
  references: [
1695
2894
  "https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
1696
2895
  ]
@@ -1703,9 +2902,6 @@ var securityRules = [
1703
2902
  title: "Model theft: API keys use $secret, not $env",
1704
2903
  description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
1705
2904
  severity: "high",
1706
- evidenceLevel: "probed",
1707
- proofTool: "agentspec health",
1708
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1709
2905
  check(manifest) {
1710
2906
  const apiKey = manifest.spec.model.apiKey;
1711
2907
  const usesSecret = apiKey.startsWith("$secret:");
@@ -1721,6 +2917,18 @@ var securityRules = [
1721
2917
  ]
1722
2918
  };
1723
2919
  }
2920
+ },
2921
+ {
2922
+ id: "TOOL-01",
2923
+ pack: "owasp-llm-top10",
2924
+ title: "Only declared tools are called",
2925
+ description: "The agent must call only tools declared in spec.tools; undeclared calls are excessive agency",
2926
+ severity: "high",
2927
+ awaitsEvidence: { code: "no_calls_observed", phase: 2 },
2928
+ // The manifest alone cannot show an undeclared call: passes declaratively, verified only from observed spans (#152, D17).
2929
+ check() {
2930
+ return { pass: true };
2931
+ }
1724
2932
  }
1725
2933
  ];
1726
2934
 
@@ -1732,9 +2940,6 @@ var memoryRules = [
1732
2940
  title: "PII scrub fields declared for long-term memory",
1733
2941
  description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
1734
2942
  severity: "critical",
1735
- evidenceLevel: "external",
1736
- proofTool: "Microsoft Presidio",
1737
- proofToolUrl: "https://microsoft.github.io/presidio/",
1738
2943
  check(manifest) {
1739
2944
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1740
2945
  const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
@@ -1753,9 +2958,6 @@ var memoryRules = [
1753
2958
  title: "Retention policy / TTL set for all memory backends",
1754
2959
  description: "Memory without TTL grows unbounded and increases PII exposure risk",
1755
2960
  severity: "high",
1756
- evidenceLevel: "probed",
1757
- proofTool: "agentspec health",
1758
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1759
2961
  check(manifest) {
1760
2962
  const mem = manifest.spec.memory;
1761
2963
  if (!mem) return { pass: true };
@@ -1780,9 +2982,6 @@ var memoryRules = [
1780
2982
  title: "Audit log enabled for long-term memory",
1781
2983
  description: "Audit logging enables compliance investigations and debugging",
1782
2984
  severity: "medium",
1783
- evidenceLevel: "probed",
1784
- proofTool: "agentspec health",
1785
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1786
2985
  check(manifest) {
1787
2986
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1788
2987
  const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
@@ -1801,9 +3000,6 @@ var memoryRules = [
1801
3000
  title: "Vector store namespace isolated",
1802
3001
  description: "Shared vector store namespaces risk data leakage between agents",
1803
3002
  severity: "medium",
1804
- evidenceLevel: "probed",
1805
- proofTool: "agentspec health",
1806
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1807
3003
  check(manifest) {
1808
3004
  if (!manifest.spec.memory?.vector) return { pass: true };
1809
3005
  const hasNamespace = !!manifest.spec.memory.vector.namespace;
@@ -1822,9 +3018,6 @@ var memoryRules = [
1822
3018
  title: "Short-term memory max tokens bounded",
1823
3019
  description: "Unbounded short-term memory can cause token budget overruns",
1824
3020
  severity: "low",
1825
- evidenceLevel: "probed",
1826
- proofTool: "agentspec health",
1827
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1828
3021
  check(manifest) {
1829
3022
  if (!manifest.spec.memory?.shortTerm) return { pass: true };
1830
3023
  const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
@@ -1847,9 +3040,6 @@ var evaluationRules = [
1847
3040
  title: "Evaluation dataset declared",
1848
3041
  description: "At least one evaluation dataset is required for quality assurance",
1849
3042
  severity: "medium",
1850
- evidenceLevel: "probed",
1851
- proofTool: "agentspec health",
1852
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1853
3043
  check(manifest) {
1854
3044
  const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
1855
3045
  return {
@@ -1867,9 +3057,6 @@ var evaluationRules = [
1867
3057
  title: "CI gate enabled",
1868
3058
  description: "Evaluation CI gate prevents regressions from reaching production",
1869
3059
  severity: "medium",
1870
- evidenceLevel: "probed",
1871
- proofTool: "agentspec health",
1872
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1873
3060
  check(manifest) {
1874
3061
  const hasCiGate = manifest.spec.evaluation?.ciGate === true;
1875
3062
  return {
@@ -1884,22 +3071,22 @@ var evaluationRules = [
1884
3071
  {
1885
3072
  id: "EVAL-03",
1886
3073
  pack: "evaluation-coverage",
1887
- title: "Hallucination metric threshold configured",
1888
- description: "A hallucination threshold below 0.1 is required for production agents",
3074
+ title: "Grounded metric threshold configured",
3075
+ description: "A grounded threshold (minimum pass rate, 0.95 recommended) is required for production agents",
1889
3076
  severity: "medium",
1890
- evidenceLevel: "probed",
1891
- proofTool: "agentspec health",
1892
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1893
3077
  check(manifest) {
1894
- const metrics = manifest.spec.evaluation?.metrics ?? [];
1895
- const hasHallucinationMetric = metrics.includes("hallucination");
1896
- const threshold = manifest.spec.evaluation?.thresholds?.["hallucination"];
1897
- const pass = hasHallucinationMetric && threshold !== void 0;
3078
+ const evaluation = manifest.spec.evaluation;
3079
+ const scopes = [evaluation, evaluation?.live];
3080
+ const hasGrounded = (scope) => !!scope?.metrics?.map(canonicalMetric).includes("grounded");
3081
+ const hasGroundedMetric = scopes.some(hasGrounded);
3082
+ const pass = scopes.some(
3083
+ (scope) => hasGrounded(scope) && Object.keys(scope?.thresholds ?? {}).some((n) => canonicalMetric(n) === "grounded")
3084
+ );
1898
3085
  return {
1899
3086
  pass,
1900
- message: pass ? void 0 : !hasHallucinationMetric ? "hallucination metric not in spec.evaluation.metrics" : "No hallucination threshold configured",
1901
- path: "/spec/evaluation/thresholds/hallucination",
1902
- recommendation: "Add hallucination to spec.evaluation.metrics and set spec.evaluation.thresholds.hallucination: 0.05",
3087
+ message: pass ? void 0 : !hasGroundedMetric ? "grounded metric not in spec.evaluation.metrics or spec.evaluation.live.metrics" : "No grounded threshold next to the grounded metric",
3088
+ path: "/spec/evaluation/thresholds/grounded",
3089
+ recommendation: "Add grounded to spec.evaluation.metrics (or spec.evaluation.live.metrics) and set the matching thresholds.grounded: 0.95",
1903
3090
  references: []
1904
3091
  };
1905
3092
  }
@@ -1914,9 +3101,6 @@ var observabilityRules = [
1914
3101
  title: "Tracing backend declared",
1915
3102
  description: "Agents should declare a tracing backend so runs are observable and debuggable",
1916
3103
  severity: "medium",
1917
- evidenceLevel: "probed",
1918
- proofTool: "agentspec health",
1919
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1920
3104
  check(manifest) {
1921
3105
  const pass = !!manifest.spec.observability?.tracing?.backend;
1922
3106
  return {
@@ -1933,9 +3117,6 @@ var observabilityRules = [
1933
3117
  title: "Structured logging enabled",
1934
3118
  description: "Structured logging makes log aggregation and searching reliable",
1935
3119
  severity: "low",
1936
- evidenceLevel: "behavioral",
1937
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1938
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1939
3120
  check(manifest) {
1940
3121
  const pass = manifest.spec.observability?.logging?.structured !== false;
1941
3122
  return {
@@ -1952,9 +3133,6 @@ var observabilityRules = [
1952
3133
  title: "Sensitive fields redacted from logs",
1953
3134
  description: "Log redaction prevents secrets and PII from appearing in log aggregators",
1954
3135
  severity: "medium",
1955
- evidenceLevel: "external",
1956
- proofTool: "Microsoft Presidio",
1957
- proofToolUrl: "https://microsoft.github.io/presidio/",
1958
3136
  check(manifest) {
1959
3137
  const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
1960
3138
  return {
@@ -1967,57 +3145,8 @@ var observabilityRules = [
1967
3145
  }
1968
3146
  ];
1969
3147
 
1970
- // src/audit/rules/metadata.rules.ts
1971
- var metadataRules = [
1972
- {
1973
- id: "META-01",
1974
- pack: "metadata-quality",
1975
- title: "Agent description declared",
1976
- description: "Agents should have a non-empty description for fleet discovery and documentation",
1977
- severity: "low",
1978
- evidenceLevel: "declarative",
1979
- check(manifest) {
1980
- const desc = manifest.metadata.description;
1981
- const pass = typeof desc === "string" && desc.trim().length > 0;
1982
- return {
1983
- pass,
1984
- message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
1985
- path: "/metadata/description",
1986
- recommendation: "Add a concise metadata.description explaining what the agent does",
1987
- references: []
1988
- };
1989
- }
1990
- },
1991
- {
1992
- id: "META-02",
1993
- pack: "metadata-quality",
1994
- title: "Model temperature is production-safe",
1995
- description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
1996
- severity: "medium",
1997
- evidenceLevel: "declarative",
1998
- check(manifest) {
1999
- const temperature = manifest.spec.model.parameters?.temperature;
2000
- if (temperature === void 0) return { pass: true };
2001
- const pass = temperature <= 1.5;
2002
- return {
2003
- pass,
2004
- message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above 1.5 produce unreliable outputs in production.`,
2005
- path: "/spec/model/parameters/temperature",
2006
- recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
2007
- references: []
2008
- };
2009
- }
2010
- }
2011
- ];
2012
-
2013
3148
  // src/audit/index.ts
2014
- var SEVERITY_WEIGHTS = {
2015
- critical: 4,
2016
- high: 3,
2017
- medium: 2,
2018
- low: 1,
2019
- info: 0
2020
- };
3149
+ var SEVERITY_WEIGHTS = CONTRACT.severityWeights;
2021
3150
  var ALL_RULES = [
2022
3151
  ...modelRules,
2023
3152
  ...securityRules,
@@ -2027,19 +3156,10 @@ var ALL_RULES = [
2027
3156
  ...metadataRules
2028
3157
  ];
2029
3158
  var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
2030
- function isProofRecord(r) {
2031
- return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
2032
- }
2033
- function scoreToGrade(score) {
2034
- if (score >= 90) return "A";
2035
- if (score >= 75) return "B";
2036
- if (score >= 60) return "C";
2037
- if (score >= 45) return "D";
2038
- return "F";
2039
- }
3159
+ var COMPLIANCE_PACKS = [...new Set(ALL_RULES.map((r) => r.pack))];
2040
3160
  function resolveActiveRules(manifest, opts) {
2041
3161
  const manifestPacks = manifest.spec.compliance?.packs ?? [];
2042
- const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : [...new Set(ALL_RULES.map((r) => r.pack))]);
3162
+ const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : COMPLIANCE_PACKS);
2043
3163
  return ALL_RULES.filter((r) => activePacks.includes(r.pack));
2044
3164
  }
2045
3165
  function collectSuppressions(manifest) {
@@ -2068,10 +3188,10 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2068
3188
  const violations = [];
2069
3189
  const packBreakdown = {};
2070
3190
  const evidenceBreakdown = {
2071
- declarative: { passed: 0, total: 0 },
2072
- probed: { passed: 0, total: 0 },
2073
- behavioral: { passed: 0, total: 0 },
2074
- external: { passed: 0, total: 0 }
3191
+ independent: { passed: 0, total: 0 },
3192
+ "evidence-point": { passed: 0, total: 0 },
3193
+ offline: { passed: 0, total: 0 },
3194
+ declared: { passed: 0, total: 0 }
2075
3195
  };
2076
3196
  let totalWeight = 0;
2077
3197
  let passedWeight = 0;
@@ -2082,19 +3202,20 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2082
3202
  const suppressed = suppressedIds.has(rule.id);
2083
3203
  const weight = SEVERITY_WEIGHTS[rule.severity];
2084
3204
  const result = ruleResults.get(rule.id);
2085
- const tier = evidenceBreakdown[rule.evidenceLevel];
3205
+ const classes = ruleClasses(rule.id);
2086
3206
  packBreakdown[rule.pack].total += 1;
2087
- tier.total += 1;
3207
+ tally(evidenceBreakdown, classes, "total");
2088
3208
  if (suppressed) {
2089
3209
  packBreakdown[rule.pack].passed += 1;
2090
- tier.passed += 1;
3210
+ tally(evidenceBreakdown, classes, "passed");
2091
3211
  continue;
2092
3212
  }
2093
3213
  if (result.pass) {
2094
3214
  packBreakdown[rule.pack].passed += 1;
2095
- tier.passed += 1;
3215
+ tally(evidenceBreakdown, classes, "passed");
2096
3216
  passedWeight += weight;
2097
3217
  } else {
3218
+ const hint = proofHint(rule.id);
2098
3219
  violations.push({
2099
3220
  ruleId: rule.id,
2100
3221
  severity: rule.severity,
@@ -2103,9 +3224,9 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2103
3224
  path: result.path,
2104
3225
  recommendation: result.recommendation,
2105
3226
  references: result.references,
2106
- evidenceLevel: rule.evidenceLevel,
2107
- proofTool: rule.proofTool,
2108
- proofToolUrl: rule.proofToolUrl
3227
+ evidenceClasses: classes,
3228
+ proofTool: hint?.tool,
3229
+ proofToolUrl: hint?.url
2109
3230
  });
2110
3231
  }
2111
3232
  totalWeight += weight;
@@ -2117,24 +3238,36 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2117
3238
  }
2118
3239
  return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
2119
3240
  }
2120
- function computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, proofRecords) {
2121
- const now = /* @__PURE__ */ new Date();
2122
- const proofSet = new Set(
2123
- proofRecords.filter((r) => !r.expiresAt || new Date(r.expiresAt) > now).map((r) => r.ruleId)
2124
- );
2125
- let provedPassedWeight = 0;
2126
- for (const rule of rules) {
2127
- if (suppressedIds.has(rule.id)) continue;
2128
- const result = ruleResults.get(rule.id);
2129
- const weight = SEVERITY_WEIGHTS[rule.severity];
2130
- const isProved = rule.evidenceLevel === "probed" && result.pass || rule.evidenceLevel === "behavioral" && result.pass || rule.evidenceLevel === "external" && proofSet.has(rule.id);
2131
- if (isProved) provedPassedWeight += weight;
3241
+ function tally(breakdown, classes, field) {
3242
+ for (const c of classes) breakdown[c][field] += 1;
3243
+ }
3244
+ var SIDECAR_GAP = { tool: "AgentSpec sidecar gap analysis", url: "https://agentspec.io/docs/concepts/probe-coverage" };
3245
+ var COLLECTOR_HINTS = {
3246
+ "sidecar-probe": SIDECAR_GAP,
3247
+ "sidecar-forwarder": SIDECAR_GAP
3248
+ };
3249
+ function proofHint(ruleId) {
3250
+ for (const { collector, status } of ruleSources(ruleId)) {
3251
+ const hint = COLLECTOR_HINTS[collector];
3252
+ if (status === "built" && hint) return { collector, ...hint };
2132
3253
  }
2133
- const provedScore = totalWeight === 0 ? 100 : Math.round(provedPassedWeight / totalWeight * 100);
2134
- const pendingProofCount = rules.filter(
2135
- (r) => r.evidenceLevel === "external" && !suppressedIds.has(r.id) && ruleResults.get(r.id).pass && !proofSet.has(r.id)
2136
- ).length;
2137
- return { provedScore, provedGrade: scoreToGrade(provedScore), pendingProofCount };
3254
+ return void 0;
3255
+ }
3256
+ function explainPendingEvidence(verified, rules) {
3257
+ const pending = new Map(rules.filter((r) => r.awaitsEvidence).map((r) => [r.id, r.awaitsEvidence]));
3258
+ return {
3259
+ ...verified,
3260
+ unverified: verified.unverified.map((u) => {
3261
+ const awaits = pending.get(u.ruleId);
3262
+ if (!awaits || u.reason.code !== "no_evidence") return u;
3263
+ return { ...u, reason: { code: awaits.code, detail: `evidence source arrives in phase ${awaits.phase}` } };
3264
+ })
3265
+ };
3266
+ }
3267
+ function summariseRuleEvidence(rules) {
3268
+ return Object.fromEntries(
3269
+ rules.map((r) => [r.id, { classes: ruleClasses(r.id), notProven: RULE_EVIDENCE[r.id]?.notProven ?? null }])
3270
+ );
2138
3271
  }
2139
3272
  function runAudit(manifest, opts = {}) {
2140
3273
  const rules = resolveActiveRules(manifest, opts);
@@ -2144,11 +3277,21 @@ function runAudit(manifest, opts = {}) {
2144
3277
  violations,
2145
3278
  packBreakdown,
2146
3279
  evidenceBreakdown,
2147
- totalWeight,
2148
3280
  overallScore,
2149
3281
  categoryScores
2150
3282
  } = computeScoring(rules, ruleResults, suppressedIds);
2151
- const proved = opts.proofRecords !== void 0 ? computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, opts.proofRecords) : {};
3283
+ const applicable = rules.filter((r) => !suppressedIds.has(r.id));
3284
+ const applicableRules = applicable.map((r) => ({ id: r.id, severity: r.severity }));
3285
+ const verified = explainPendingEvidence(
3286
+ grade(
3287
+ opts.records ?? [],
3288
+ applicableRules,
3289
+ opts.trust ?? "standard",
3290
+ (/* @__PURE__ */ new Date()).toISOString(),
3291
+ opts.evidencePoints
3292
+ ),
3293
+ applicable
3294
+ );
2152
3295
  return {
2153
3296
  agentName: manifest.metadata.name,
2154
3297
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2161,9 +3304,26 @@ function runAudit(manifest, opts = {}) {
2161
3304
  totalRules: rules.length,
2162
3305
  packBreakdown,
2163
3306
  evidenceBreakdown,
2164
- ...proved
3307
+ verified,
3308
+ applicableRules,
3309
+ ruleEvidence: summariseRuleEvidence(rules)
2165
3310
  };
2166
3311
  }
3312
+ function canonicalJson(value) {
3313
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
3314
+ if (value !== null && typeof value === "object") {
3315
+ const obj = value;
3316
+ const members = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
3317
+ return `{${members.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
3318
+ }
3319
+ return JSON.stringify(value);
3320
+ }
3321
+ function canonicalManifestSha256(manifest) {
3322
+ return crypto.createHash("sha256").update(canonicalJson(manifest), "utf8").digest("hex");
3323
+ }
3324
+ function declaredManifestSha256(raw) {
3325
+ return canonicalManifestSha256(yaml__namespace.load(raw, { schema: yaml__namespace.JSON_SCHEMA }));
3326
+ }
2167
3327
 
2168
3328
  // src/generate/index.ts
2169
3329
  var _registry = /* @__PURE__ */ new Map();
@@ -2176,7 +3336,7 @@ function getAdapter(framework) {
2176
3336
  function listAdapters() {
2177
3337
  return [..._registry.keys()];
2178
3338
  }
2179
- function generateAdapter(manifest, framework, options) {
3339
+ function generateAdapter(manifest, framework, options2) {
2180
3340
  const adapter = _registry.get(framework.toLowerCase());
2181
3341
  if (!adapter) {
2182
3342
  throw new Error(
@@ -2185,7 +3345,7 @@ function generateAdapter(manifest, framework, options) {
2185
3345
  Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
2186
3346
  );
2187
3347
  }
2188
- return adapter.generate(manifest, options);
3348
+ return adapter.generate(manifest, options2);
2189
3349
  }
2190
3350
 
2191
3351
  // src/agent/usage-ledger.ts
@@ -2231,8 +3391,8 @@ var UsageLedger = class {
2231
3391
  * Return a frozen snapshot of the current usage window.
2232
3392
  *
2233
3393
  * @param reset - When true (default), clears all counters and starts a new
2234
- * window. Use reset=true in the heartbeat push so each window is shipped
2235
- * exactly once. Use reset=false for inspection without draining.
3394
+ * window. Use reset=false for inspection without draining; the heartbeat
3395
+ * push takes a reset=false snapshot and calls subtract() with it.
2236
3396
  */
2237
3397
  snapshot(reset = true) {
2238
3398
  const snap = buildSnapshot(this.counters, this.windowStart);
@@ -2242,39 +3402,220 @@ var UsageLedger = class {
2242
3402
  }
2243
3403
  return snap;
2244
3404
  }
3405
+ /**
3406
+ * Remove a shipped snapshot from the ledger; a new window starts, because the control plane dedupes heartbeats on windowStartedAt.
3407
+ * Usage recorded after the snapshot was taken stays, so every token is
3408
+ * shipped exactly once even when a call finishes while a heartbeat is in flight.
3409
+ */
3410
+ subtract(shipped) {
3411
+ for (const s of shipped.models) {
3412
+ const entry = this.counters.get(s.modelId);
3413
+ if (!entry) continue;
3414
+ entry.promptTokens -= s.promptTokens;
3415
+ entry.completionTokens -= s.completionTokens;
3416
+ entry.totalTokens -= s.totalTokens;
3417
+ entry.callCount -= s.callCount;
3418
+ if (entry.callCount <= 0) this.counters.delete(s.modelId);
3419
+ }
3420
+ this.windowStart = /* @__PURE__ */ new Date();
3421
+ }
2245
3422
  };
2246
3423
 
3424
+ // src/runtime/evidence-points.ts
3425
+ var GUARDRAIL = EvidencePoint.enum.guardrail;
3426
+ var [MCP_LIST, MCP_CALL] = Object.keys(CONTRACT.evidencePoints.mcpClient);
3427
+ var [SCRUB, TTL, AUDIT] = Object.keys(CONTRACT.evidencePoints.memoryWrite);
3428
+ var [NAMESPACE] = Object.keys(CONTRACT.evidencePoints.vectorStore);
3429
+ var [TOOL_TARGET] = Object.keys(CONTRACT.evidencePoints.tool);
3430
+ var [LOGGING_TARGET] = Object.keys(CONTRACT.evidencePoints.logging);
3431
+ var keyOf = (point2, target) => `${point2}\0${target}`;
3432
+ var DEFAULT_KEY = /* @__PURE__ */ Symbol.for("agentspec.runtime.defaultEvidencePoints");
3433
+ var defaults = globalThis;
3434
+ var rec = (value) => value && typeof value === "object" ? value : {};
3435
+ var stdioServers = (spec) => (Array.isArray(rec(spec.mcp).servers) ? rec(spec.mcp).servers : []).filter((s) => s?.transport === "stdio");
3436
+ var isDeclared = (value) => Array.isArray(value) ? value.length > 0 : Boolean(value);
3437
+ function memoryWriteTargets(memory) {
3438
+ const hygiene = rec(memory.hygiene);
3439
+ const targets = /* @__PURE__ */ new Set();
3440
+ if (isDeclared(hygiene.piiScrubFields)) targets.add(SCRUB);
3441
+ if ([rec(memory.shortTerm).ttlSeconds, rec(memory.longTerm).ttlDays].some((ttl) => ttl !== void 0 && ttl !== null)) targets.add(TTL);
3442
+ if (memory.longTerm && hygiene.auditLog === true) targets.add(AUDIT);
3443
+ return targets;
3444
+ }
3445
+ function declaredTargets(spec) {
3446
+ const all = (point2, declared2) => new Set(declared2 ? Object.keys(CONTRACT.evidencePoints[point2] ?? {}) : []);
3447
+ const memory = rec(spec.memory);
3448
+ const logging = rec(rec(spec.observability).logging);
3449
+ return {
3450
+ [EvidencePoint.enum.memoryWrite]: memoryWriteTargets(memory),
3451
+ [EvidencePoint.enum.vectorStore]: new Set(rec(memory.vector).namespace ? [NAMESPACE] : []),
3452
+ [GUARDRAIL]: new Set(Object.keys(CONTRACT.evidencePoints[GUARDRAIL] ?? {}).filter((t) => isDeclared(rec(spec.guardrails)[t]))),
3453
+ [EvidencePoint.enum.tool]: all(EvidencePoint.enum.tool, (Array.isArray(spec.tools) ? spec.tools : []).some((t) => t?.type === "function")),
3454
+ [EvidencePoint.enum.mcpClient]: all(EvidencePoint.enum.mcpClient, stdioServers(spec).length > 0),
3455
+ [EvidencePoint.enum.logging]: all(EvidencePoint.enum.logging, rec(spec.observability).logging !== void 0 && logging.structured !== false)
3456
+ };
3457
+ }
3458
+ function applicablePoints(spec) {
3459
+ const declared2 = declaredTargets(spec);
3460
+ const applicable = /* @__PURE__ */ new Map();
3461
+ for (const [point2, targets] of Object.entries(CONTRACT.evidencePoints)) {
3462
+ for (const [target, ruleId] of Object.entries(targets)) {
3463
+ const listed = CONTRACT.ruleEvidence[ruleId]?.sources.some((s) => s.point === point2);
3464
+ if (listed && declared2[point2]?.has(target)) applicable.set(keyOf(point2, target), { point: EvidencePoint.parse(point2), target, ruleId });
3465
+ }
3466
+ }
3467
+ return applicable;
3468
+ }
3469
+ function createEvidencePoints(manifest) {
3470
+ const manifestSha256 = canonicalManifestSha256(manifest);
3471
+ const spec = rec(rec(manifest).spec);
3472
+ const applicable = applicablePoints(spec);
3473
+ const mcpServers = stdioServers(spec).map((s) => String(s.name));
3474
+ const wired = /* @__PURE__ */ new Set();
3475
+ const emptyWindow = () => new Map([...applicable.keys()].map((k) => [k, { total: 0, counts: { pass: 0, fail: 0, not_verifiable: 0 }, last: [] }]));
3476
+ let calls = emptyWindow();
3477
+ let windowStart = /* @__PURE__ */ new Date();
3478
+ let seq = 0;
3479
+ const shippedUpTo = /* @__PURE__ */ new WeakMap();
3480
+ const points = {
3481
+ register: (point2, target) => {
3482
+ wired.add(keyOf(point2, target));
3483
+ },
3484
+ applies: (point2, target) => applicable.has(keyOf(point2, target)),
3485
+ memorySpec: () => rec(spec.memory),
3486
+ declaredMcpServers: () => [...mcpServers],
3487
+ recordCall: (point2, target, { subject, status = "pass", reason } = {}) => {
3488
+ if (status !== "pass" && !reason) throw new Error(`recordCall: a ${status} call of ${point2}:${target} needs a reason`);
3489
+ const c = calls.get(keyOf(point2, target));
3490
+ if (!c) return;
3491
+ c.total += 1;
3492
+ c.counts[status] += 1;
3493
+ c.last.push({ at: /* @__PURE__ */ new Date(), seq: ++seq, subject, status, reason });
3494
+ if (c.last.length > CONTRACT.evidencePointRecordLimit) c.last.shift();
3495
+ },
3496
+ snapshot: (ttlSeconds, reset = true) => {
3497
+ const windowEnd = /* @__PURE__ */ new Date();
3498
+ const lastSeq = /* @__PURE__ */ new Map();
3499
+ const window = { start: windowStart.toISOString(), end: windowEnd.toISOString() };
3500
+ const aggregates = [...applicable].map(([key, { point: point2, target, ruleId }]) => {
3501
+ const c = calls.get(key);
3502
+ const state = c.total ? "fired" : wired.has(key) ? "wired" : "not_wired";
3503
+ lastSeq.set(key, seq);
3504
+ const records = c.last.map(({ at, subject, status, reason }) => ({
3505
+ checkId: subject ?? `${point2}:${target}`,
3506
+ ruleId,
3507
+ status: EvidenceStatus.enum[status],
3508
+ ...reason ? { reason } : {},
3509
+ source: EvidenceSource.enum["sdk-observed"],
3510
+ origin: EvidenceOrigin.enum.in_process,
3511
+ collector: Collector.enum.sdk,
3512
+ observedAt: at.toISOString(),
3513
+ window,
3514
+ ttlSeconds,
3515
+ sampleSize: c.total,
3516
+ traceIds: [],
3517
+ manifestSha256
3518
+ }));
3519
+ return { point: point2, target, state, counts: { ...c.counts }, records };
3520
+ });
3521
+ if (reset) {
3522
+ calls = emptyWindow();
3523
+ windowStart = windowEnd;
3524
+ } else {
3525
+ shippedUpTo.set(aggregates, { lastSeq, windowEnd });
3526
+ }
3527
+ return aggregates;
3528
+ },
3529
+ subtract: (shipped) => {
3530
+ const taken = shippedUpTo.get(shipped);
3531
+ if (!taken) throw new Error("EvidencePoints.subtract: pass the array snapshot() returned");
3532
+ for (const a of shipped) {
3533
+ const key = keyOf(a.point, a.target);
3534
+ const c = calls.get(key);
3535
+ c.total -= a.counts.pass + a.counts.fail + a.counts.not_verifiable;
3536
+ for (const status of ["pass", "fail", "not_verifiable"]) c.counts[status] -= a.counts[status];
3537
+ c.last = c.last.filter((l) => l.seq > taken.lastSeq.get(key));
3538
+ }
3539
+ windowStart = taken.windowEnd;
3540
+ }
3541
+ };
3542
+ defaults[DEFAULT_KEY] = points;
3543
+ return points;
3544
+ }
3545
+
3546
+ // src/health/tool-check-id.ts
3547
+ function slugify(s) {
3548
+ return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
3549
+ }
3550
+ function toolCheckId(toolName) {
3551
+ return `tool:${slugify(toolName)}`;
3552
+ }
3553
+
3554
+ // src/runtime/call-points.ts
3555
+ var { tool: TOOL, mcpClient: MCP_CLIENT, logging: LOGGING } = EvidencePoint.enum;
3556
+ var { memoryWrite: MEMORY_WRITE, vectorStore: VECTOR_STORE } = EvidencePoint.enum;
3557
+ var NOT_VERIFIABLE_CODE = "instrumentation_unavailable";
3558
+ var WRONG_NAMESPACE_CODE = "undeclared_namespace";
3559
+ var NO_NAMESPACE_CODE = "no_evidence";
3560
+ for (const code of [NOT_VERIFIABLE_CODE, WRONG_NAMESPACE_CODE, NO_NAMESPACE_CODE]) {
3561
+ if (!CONTRACT.reasonCodes.includes(code)) throw new Error(`contract.reasonCodes lacks '${code}'`);
3562
+ }
3563
+ new async_hooks.AsyncLocalStorage();
3564
+
2247
3565
  // src/agent/reporter.ts
3566
+ var HEARTBEAT_POST_TIMEOUT_MS = 1e4;
3567
+ var DEFAULT_PUSH_INTERVAL_SECONDS = 30;
3568
+ var RECORD_TTL_INTERVALS = 2;
3569
+ function buildCappedBody(health, aggregates, usage) {
3570
+ const trimmedChecks = [...health.checks];
3571
+ const serialise = (checks, evidencePoints) => JSON.stringify({
3572
+ contract: CONTRACT.version,
3573
+ health: { ...health, checks },
3574
+ records: [],
3575
+ // the control plane reads the records inside each aggregate (#599)
3576
+ evidencePoints,
3577
+ usage
3578
+ });
3579
+ const size = (body2) => Buffer.byteLength(body2, "utf8");
3580
+ let body = serialise(trimmedChecks, aggregates);
3581
+ if (size(body) > CONTRACT.heartbeatMaxBytes) {
3582
+ aggregates = aggregates.map((a) => ({ ...a, records: [] }));
3583
+ body = serialise(trimmedChecks, aggregates);
3584
+ }
3585
+ while (size(body) > CONTRACT.heartbeatMaxBytes && trimmedChecks.length > 0) {
3586
+ trimmedChecks.pop();
3587
+ body = serialise(trimmedChecks, aggregates);
3588
+ }
3589
+ return { body, bytes: size(body) };
3590
+ }
2248
3591
  var AgentSpecReporter = class {
2249
3592
  constructor(manifest, opts = {}) {
2250
3593
  this.manifest = manifest;
2251
3594
  this.opts = opts;
2252
3595
  this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
2253
3596
  this.staleAfterMs = opts.staleAfterMs ?? 6e4;
3597
+ this.evidence = createEvidencePoints(opts.rawManifest ?? manifest);
2254
3598
  }
3599
+ manifest;
3600
+ opts;
2255
3601
  cached = null;
2256
3602
  timer = null;
2257
3603
  _pushTimer = null;
2258
3604
  refreshIntervalMs;
2259
3605
  staleAfterMs;
2260
- refreshing = false;
3606
+ inflight = null;
3607
+ /** True while a heartbeat POST is in flight; a tick that finds it set sends nothing. */
3608
+ heartbeatInFlight = false;
2261
3609
  stopped = false;
2262
- registeredTools = /* @__PURE__ */ new Set();
2263
3610
  /** In-process token usage accumulator. */
2264
3611
  usage = new UsageLedger();
3612
+ /** Evidence points the agent's code wires (`guardrail(...)` reports here); each heartbeat sends the window's aggregates. */
3613
+ evidence;
2265
3614
  /**
2266
- * Register a tool handler by name.
2267
- * Only registered tools report status: 'pass' in the live health check.
2268
- * Declared tools that are not registered report status: 'fail'.
2269
- *
2270
- * Call this once per tool after wiring up your tool handlers:
2271
- * reporter.registerTool('search-arxiv', handler)
2272
- *
2273
- * The handler argument is accepted but not called — it is retained solely so
2274
- * callers can pass the real function object for documentation purposes.
3615
+ * @deprecated No-op, kept so existing callers compile. Registering a handler proves nothing about
3616
+ * which tools the agent calls, so the health report no longer carries `tool:<name>` checks (ADR-0017).
2275
3617
  */
2276
- registerTool(name, _handler) {
2277
- this.registeredTools.add(name);
3618
+ registerTool(_name, _handler) {
2278
3619
  }
2279
3620
  /**
2280
3621
  * Start background refresh. Call once during app startup.
@@ -2338,52 +3679,74 @@ var AgentSpecReporter = class {
2338
3679
  */
2339
3680
  startPushMode(opts) {
2340
3681
  if (this._pushTimer !== null) return;
2341
- const intervalMs = (opts.intervalSeconds ?? 30) * 1e3;
2342
- void this._pushHeartbeat(opts);
3682
+ const controlPlaneUrl = opts.controlPlaneUrl || process.env[CONTRACT.env.controlPlaneUrl];
3683
+ if (!controlPlaneUrl) {
3684
+ throw new Error(
3685
+ `startPushMode: no control plane URL \u2014 pass controlPlaneUrl or set ${CONTRACT.env.controlPlaneUrl}`
3686
+ );
3687
+ }
3688
+ if (!opts.apiKey) {
3689
+ throw new Error("startPushMode: apiKey is required (the per-agent token from POST /api/v1/register)");
3690
+ }
3691
+ const resolved = { ...opts, controlPlaneUrl };
3692
+ const intervalMs = (opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * 1e3;
3693
+ void this._pushHeartbeat(resolved);
2343
3694
  this._pushTimer = setInterval(() => {
2344
- void this._pushHeartbeat(opts);
3695
+ void this._pushHeartbeat(resolved);
2345
3696
  }, intervalMs);
2346
3697
  if (this._pushTimer.unref) {
2347
3698
  this._pushTimer.unref();
2348
3699
  }
2349
3700
  }
2350
3701
  async _pushHeartbeat(opts) {
3702
+ if (this.heartbeatInFlight) return;
3703
+ this.heartbeatInFlight = true;
2351
3704
  try {
2352
3705
  const health = await this.getReport();
2353
- const gap = runAudit(this.manifest);
2354
3706
  const usage = this.usage.snapshot(false);
2355
- const trimmedChecks = [...health.checks];
2356
- let body = JSON.stringify({ health, gap, usage });
2357
- if (body.length > 65536) {
2358
- while (trimmedChecks.length > 0) {
2359
- trimmedChecks.pop();
2360
- body = JSON.stringify({ health: { ...health, checks: trimmedChecks }, gap, usage });
2361
- if (body.length <= 65536) break;
2362
- }
3707
+ const aggregates = this.evidence.snapshot(Math.ceil((opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * RECORD_TTL_INTERVALS), false);
3708
+ const { body, bytes } = buildCappedBody(health, aggregates, usage);
3709
+ if (bytes > CONTRACT.heartbeatMaxBytes) {
3710
+ opts.onError?.(
3711
+ new Error(`Heartbeat not sent: ${bytes} bytes is over the ${CONTRACT.heartbeatMaxBytes}-byte cap with every check trimmed`)
3712
+ );
3713
+ return;
2363
3714
  }
2364
3715
  let res;
3716
+ const abort = new AbortController();
3717
+ const abortTimer = setTimeout(
3718
+ () => abort.abort(new Error(`heartbeat POST timed out after ${HEARTBEAT_POST_TIMEOUT_MS}ms`)),
3719
+ HEARTBEAT_POST_TIMEOUT_MS
3720
+ );
3721
+ abortTimer.unref?.();
2365
3722
  try {
2366
- res = await fetch(`${opts.controlPlaneUrl}/api/v1/heartbeat`, {
3723
+ res = await fetch(`${opts.controlPlaneUrl}${CONTRACT.controlPlanePaths.heartbeat}`, {
2367
3724
  method: "POST",
2368
3725
  headers: {
2369
3726
  "Authorization": `Bearer ${opts.apiKey}`,
2370
3727
  "Content-Type": "application/json"
2371
3728
  },
2372
- body
3729
+ body,
3730
+ signal: abort.signal
2373
3731
  });
2374
3732
  } catch (err) {
2375
3733
  const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
2376
3734
  opts.onError?.(new Error(sanitized));
2377
3735
  return;
3736
+ } finally {
3737
+ clearTimeout(abortTimer);
2378
3738
  }
2379
3739
  if (res.ok) {
2380
- this.usage.snapshot(true);
3740
+ this.usage.subtract(usage);
3741
+ this.evidence.subtract(aggregates);
2381
3742
  } else {
2382
3743
  opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
2383
3744
  }
2384
3745
  } catch (err) {
2385
3746
  const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
2386
3747
  opts.onError?.(new Error(sanitized));
3748
+ } finally {
3749
+ this.heartbeatInFlight = false;
2387
3750
  }
2388
3751
  }
2389
3752
  /** Stop sending heartbeats to the control plane. */
@@ -2397,38 +3760,26 @@ var AgentSpecReporter = class {
2397
3760
  isPushModeActive() {
2398
3761
  return this._pushTimer !== null;
2399
3762
  }
2400
- async refresh() {
2401
- if (this.stopped || this.refreshing) return;
2402
- this.refreshing = true;
3763
+ /** Runs one refresh; concurrent callers share the in-flight promise so they all see a populated cache. */
3764
+ refresh() {
3765
+ if (this.inflight) return this.inflight;
3766
+ if (this.stopped) return Promise.resolve();
3767
+ this.inflight = this.runRefresh().finally(() => {
3768
+ this.inflight = null;
3769
+ });
3770
+ return this.inflight;
3771
+ }
3772
+ async runRefresh() {
2403
3773
  try {
2404
3774
  const report = await runHealthCheck(this.manifest, {
2405
3775
  checkModel: true,
2406
3776
  checkMcp: true,
2407
3777
  checkMemory: true,
2408
- checkServices: true
2409
- });
2410
- const toolChecks = (this.manifest.spec.tools ?? []).map((tool) => {
2411
- const registered = this.registeredTools.has(tool.name);
2412
- return {
2413
- id: `tool:${tool.name}`,
2414
- category: "tool",
2415
- status: registered ? "pass" : "fail",
2416
- severity: "info",
2417
- ...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
2418
- };
3778
+ checkServices: true,
3779
+ baseDir: this.opts.baseDir,
3780
+ rawManifest: this.opts.rawManifest
2419
3781
  });
2420
- const toolPassed = toolChecks.filter((c) => c.status === "pass").length;
2421
- const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
2422
- const enrichedReport = {
2423
- ...report,
2424
- checks: [...report.checks, ...toolChecks],
2425
- summary: {
2426
- ...report.summary,
2427
- passed: report.summary.passed + toolPassed,
2428
- failed: report.summary.failed + toolFailed
2429
- }
2430
- };
2431
- this.cached = { report: enrichedReport, ts: Date.now() };
3782
+ this.cached = { report, ts: Date.now() };
2432
3783
  } catch {
2433
3784
  if (!this.cached) {
2434
3785
  this.cached = {
@@ -2452,8 +3803,6 @@ var AgentSpecReporter = class {
2452
3803
  } else {
2453
3804
  this.cached = { ...this.cached, ts: Date.now() };
2454
3805
  }
2455
- } finally {
2456
- this.refreshing = false;
2457
3806
  }
2458
3807
  }
2459
3808
  };
@@ -2461,7 +3810,7 @@ var AgentSpecReporter = class {
2461
3810
  // src/agent/adapters/fastify.ts
2462
3811
  function agentSpecFastifyPlugin(reporter) {
2463
3812
  return async (app) => {
2464
- app.get("/agentspec/health", async (_req, reply) => {
3813
+ app.get(CONTRACT.agentPaths.health, async (_req, reply) => {
2465
3814
  try {
2466
3815
  const report = await reporter.getReport();
2467
3816
  reply.status(200).send(report);
@@ -2526,30 +3875,63 @@ function agentSpecExpressRouter(reporter) {
2526
3875
 
2527
3876
  exports.AUDIT_RULE_IDS = AUDIT_RULE_IDS;
2528
3877
  exports.AgentSpecReporter = AgentSpecReporter;
3878
+ exports.COMPLIANCE_PACKS = COMPLIANCE_PACKS;
3879
+ exports.CONTRACT = CONTRACT;
3880
+ exports.ContractSchema = ContractSchema;
3881
+ exports.EVIDENCE_SHAPES = EVIDENCE_SHAPES;
3882
+ exports.EvalResult = EvalResult;
3883
+ exports.EvidenceOrigin = EvidenceOrigin;
3884
+ exports.EvidenceReason = EvidenceReason;
3885
+ exports.EvidenceRecord = EvidenceRecord;
3886
+ exports.EvidenceSource = EvidenceSource;
3887
+ exports.EvidenceStatus = EvidenceStatus;
3888
+ exports.EvidenceWindow = EvidenceWindow;
3889
+ exports.Heartbeat = Heartbeat;
2529
3890
  exports.LATEST_API_VERSION = LATEST_API_VERSION;
3891
+ exports.MAX_PRODUCTION_TEMPERATURE = MAX_PRODUCTION_TEMPERATURE;
3892
+ exports.MAX_TRACE_IDS = MAX_TRACE_IDS;
2530
3893
  exports.ManifestSchema = ManifestSchema;
3894
+ exports.PodspecFinding = PodspecFinding;
3895
+ exports.RULE_EVIDENCE = RULE_EVIDENCE;
3896
+ exports.ReadinessFinding = ReadinessFinding;
3897
+ exports.RuleEvidenceEntry = RuleEvidenceEntry;
3898
+ exports.SeveritySchema = SeveritySchema;
3899
+ exports.SidecarPush = SidecarPush;
3900
+ exports.TrustPolicy = TrustPolicy;
2531
3901
  exports.UsageLedger = UsageLedger;
2532
3902
  exports.agentSpecExpressRouter = agentSpecExpressRouter;
2533
3903
  exports.agentSpecFastifyPlugin = agentSpecFastifyPlugin;
3904
+ exports.canonicalManifestSha256 = canonicalManifestSha256;
2534
3905
  exports.collectEnvRefs = collectEnvRefs;
2535
3906
  exports.collectFileRefs = collectFileRefs;
3907
+ exports.computeHealthStatus = computeHealthStatus;
3908
+ exports.declaredManifestSha256 = declaredManifestSha256;
3909
+ exports.describeManifestError = describeManifestError;
2536
3910
  exports.detectRefType = detectRefType;
2537
3911
  exports.detectVersion = detectVersion;
2538
3912
  exports.exportJsonSchema = exportJsonSchema;
3913
+ exports.formatVerified = formatVerified;
3914
+ exports.gatewayEnvNames = gatewayEnvNames;
2539
3915
  exports.generateAdapter = generateAdapter;
2540
3916
  exports.getAdapter = getAdapter;
3917
+ exports.grade = grade;
3918
+ exports.isLatestModelId = isLatestModelId;
2541
3919
  exports.isLatestVersion = isLatestVersion;
2542
- exports.isProofRecord = isProofRecord;
2543
3920
  exports.listAdapters = listAdapters;
2544
3921
  exports.loadManifest = loadManifest;
2545
3922
  exports.migrateManifest = migrateManifest;
2546
3923
  exports.registerAdapter = registerAdapter;
3924
+ exports.renderGatewayConfig = renderGatewayConfig;
2547
3925
  exports.resolveRef = resolveRef;
2548
3926
  exports.resolveRefs = resolveRefs;
3927
+ exports.ruleClasses = ruleClasses;
2549
3928
  exports.runAudit = runAudit;
2550
3929
  exports.runHealthCheck = runHealthCheck;
2551
3930
  exports.runServiceChecks = runServiceChecks;
2552
3931
  exports.scoreToGrade = scoreToGrade;
3932
+ exports.secretEnvName = secretEnvName;
3933
+ exports.slugify = slugify;
3934
+ exports.toolCheckId = toolCheckId;
2553
3935
  exports.tryLoadManifest = tryLoadManifest;
2554
3936
  //# sourceMappingURL=index.cjs.map
2555
3937
  //# sourceMappingURL=index.cjs.map