@agentspec/sdk 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +46 -0
  2. package/dist/evidence-points-C6j4PSsQ.d.cts +1139 -0
  3. package/dist/evidence-points-C6j4PSsQ.d.ts +1139 -0
  4. package/dist/index.cjs +1733 -346
  5. package/dist/index.cjs.map +1 -1
  6. package/dist/index.d.cts +1026 -2549
  7. package/dist/index.d.ts +1026 -2549
  8. package/dist/index.js +1684 -345
  9. package/dist/index.js.map +1 -1
  10. package/dist/runtime.cjs +930 -0
  11. package/dist/runtime.cjs.map +1 -0
  12. package/dist/runtime.d.cts +63 -0
  13. package/dist/runtime.d.ts +63 -0
  14. package/dist/runtime.js +921 -0
  15. package/dist/runtime.js.map +1 -0
  16. package/package.json +15 -8
  17. package/schemas/evidence/v1/contract.schema.json +2078 -0
  18. package/schemas/evidence/v1/fixtures/drifted/EvalResult.n-as-string.json +1 -0
  19. package/schemas/evidence/v1/fixtures/drifted/EvalResult.pass-rate-above-one.json +1 -0
  20. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.bad-sha.json +1 -0
  21. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.extra-field.json +1 -0
  22. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.fail-without-reason.json +1 -0
  23. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.missing-ttl.json +1 -0
  24. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.not-verifiable-without-reason.json +1 -0
  25. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-epoch.json +1 -0
  26. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-out-of-range.json +1 -0
  27. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-space-separated.json +1 -0
  28. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.sample-size-boolean.json +1 -0
  29. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.ttl-as-string.json +1 -0
  30. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-collector.json +1 -0
  31. package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-source.json +1 -0
  32. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-contract-version.json +1 -0
  33. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-record.json +1 -0
  34. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.point-records-over-limit.json +1 -0
  35. package/schemas/evidence/v1/fixtures/drifted/Heartbeat.with-gap.json +1 -0
  36. package/schemas/evidence/v1/fixtures/drifted/ReadinessFinding.bad-severity.json +1 -0
  37. package/schemas/evidence/v1/fixtures/drifted/SidecarPush.missing-sha.json +1 -0
  38. package/schemas/evidence/v1/fixtures/drifted/TrustPolicy.lenient.json +1 -0
  39. package/schemas/evidence/v1/fixtures/valid/EvalResult.live.json +1 -0
  40. package/schemas/evidence/v1/fixtures/valid/EvalResult.null-dataset.json +1 -0
  41. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.fail.json +1 -0
  42. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.null-collector.json +1 -0
  43. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass-null-reason.json +1 -0
  44. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass.json +1 -0
  45. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.timezone-offset.json +1 -0
  46. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.unknown-reason-code.json +1 -0
  47. package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.with-collector.json +1 -0
  48. package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-null.json +1 -0
  49. package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-version.json +1 -0
  50. package/schemas/evidence/v1/fixtures/valid/Heartbeat.evidence-points.json +1 -0
  51. package/schemas/evidence/v1/fixtures/valid/Heartbeat.full.json +1 -0
  52. package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk-no-records.json +1 -0
  53. package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk.json +1 -0
  54. package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.open-code.json +1 -0
  55. package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.with-subject.json +1 -0
  56. package/schemas/evidence/v1/fixtures/valid/SidecarPush.declared.json +1 -0
  57. package/schemas/evidence/v1/fixtures/valid/TrustPolicy.standard.json +1 -0
  58. package/schemas/evidence/v1/fixtures/valid/TrustPolicy.strict.json +1 -0
  59. package/schemas/grader/v1/fixtures/accepts-fractional-seconds.json +62 -0
  60. package/schemas/grader/v1/fixtures/accepts-leap-day-in-year-below-100.json +62 -0
  61. package/schemas/grader/v1/fixtures/accepts-more-than-6-fractional-digits.json +62 -0
  62. package/schemas/grader/v1/fixtures/accepts-numeric-offset.json +62 -0
  63. package/schemas/grader/v1/fixtures/all-records-must-pass.json +58 -0
  64. package/schemas/grader/v1/fixtures/below-threshold-not-graded.json +61 -0
  65. package/schemas/grader/v1/fixtures/derived-rule-fails-when-a-parent-fails.json +67 -0
  66. package/schemas/grader/v1/fixtures/derived-rule-follows-memory-write-parent.json +43 -0
  67. package/schemas/grader/v1/fixtures/derived-rule-passes-with-its-parents.json +64 -0
  68. package/schemas/grader/v1/fixtures/derived-rule-unverified-when-a-parent-has-no-evidence.json +51 -0
  69. package/schemas/grader/v1/fixtures/expired-record-is-stale.json +66 -0
  70. package/schemas/grader/v1/fixtures/fail-beats-not-verifiable.json +62 -0
  71. package/schemas/grader/v1/fixtures/grade-band-b.json +62 -0
  72. package/schemas/grader/v1/fixtures/independent-from-listed-collector-is-admitted.json +39 -0
  73. package/schemas/grader/v1/fixtures/independent-from-unlisted-collector-is-rejected.json +47 -0
  74. package/schemas/grader/v1/fixtures/low-severity-coverage-only.json +90 -0
  75. package/schemas/grader/v1/fixtures/malformed-point-is-ignored.json +67 -0
  76. package/schemas/grader/v1/fixtures/never-admitted-sources-standard.json +196 -0
  77. package/schemas/grader/v1/fixtures/never-admitted-sources-strict.json +196 -0
  78. package/schemas/grader/v1/fixtures/not-admitted-outranks-mismatch.json +64 -0
  79. package/schemas/grader/v1/fixtures/not-verifiable-ignored-when-one-passes.json +58 -0
  80. package/schemas/grader/v1/fixtures/not-verifiable-only-keeps-reason.json +71 -0
  81. package/schemas/grader/v1/fixtures/not-wired-point-drops-a-fail-when-no-point-of-the-rule-is-wired.json +59 -0
  82. package/schemas/grader/v1/fixtures/not-wired-point-drops-a-wired-point-pass.json +87 -0
  83. package/schemas/grader/v1/fixtures/not-wired-point-drops-its-own-earlier-fail-when-another-point-is-wired.json +70 -0
  84. package/schemas/grader/v1/fixtures/not-wired-point-ignores-a-wired-point-fail-without-the-rule-id.json +69 -0
  85. package/schemas/grader/v1/fixtures/not-wired-point-keeps-a-wired-point-fail.json +79 -0
  86. package/schemas/grader/v1/fixtures/not-wired-point-keeps-independent-evidence.json +64 -0
  87. package/schemas/grader/v1/fixtures/not-wired-point-never-passes.json +107 -0
  88. package/schemas/grader/v1/fixtures/offline-record-is-not-admitted-yet.json +47 -0
  89. package/schemas/grader/v1/fixtures/record-without-collector-is-admitted-until-producers-set-it.json +38 -0
  90. package/schemas/grader/v1/fixtures/records-without-rule-id-never-graded.json +62 -0
  91. package/schemas/grader/v1/fixtures/rejects-compact-timestamp.json +56 -0
  92. package/schemas/grader/v1/fixtures/rejects-hour-24-lowercase-separator.json +56 -0
  93. package/schemas/grader/v1/fixtures/rejects-hour-24-space-separator.json +56 -0
  94. package/schemas/grader/v1/fixtures/rejects-hour-24.json +56 -0
  95. package/schemas/grader/v1/fixtures/rejects-impossible-date-feb-29.json +56 -0
  96. package/schemas/grader/v1/fixtures/rejects-lowercase-t.json +56 -0
  97. package/schemas/grader/v1/fixtures/rejects-lowercase-z.json +56 -0
  98. package/schemas/grader/v1/fixtures/rejects-offset-hours-above-23.json +56 -0
  99. package/schemas/grader/v1/fixtures/rejects-offset-minutes-above-59.json +56 -0
  100. package/schemas/grader/v1/fixtures/rejects-offsetless-timestamp.json +56 -0
  101. package/schemas/grader/v1/fixtures/rejects-space-separator.json +56 -0
  102. package/schemas/grader/v1/fixtures/rejects-year-zero.json +56 -0
  103. package/schemas/grader/v1/fixtures/rounds-coverage-before-the-threshold.json +850 -0
  104. package/schemas/grader/v1/fixtures/rounds-verified-before-the-band.json +674 -0
  105. package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-admitted-standard.json +39 -0
  106. package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-not-admitted-strict.json +47 -0
  107. package/schemas/grader/v1/fixtures/sdk-observed-standard.json +58 -0
  108. package/schemas/grader/v1/fixtures/sdk-observed-strict.json +66 -0
  109. package/schemas/grader/v1/fixtures/sdk-observed-without-evidence-point-source-is-rejected.json +47 -0
  110. package/schemas/grader/v1/fixtures/sdk-only-agent-all-points-fired.json +116 -0
  111. package/schemas/grader/v1/fixtures/stale-outranks-mismatch.json +64 -0
  112. package/schemas/grader/v1/fixtures/strict-admits-independent.json +39 -0
  113. package/schemas/grader/v1/fixtures/sub-millisecond-expiry-boundary.json +38 -0
  114. package/schemas/v1/agent.schema.json +372 -198
package/dist/index.js CHANGED
@@ -1,9 +1,11 @@
1
1
  import { z } from 'zod';
2
- import { zodToJsonSchema } from 'zod-to-json-schema';
3
- import { readFileSync, existsSync } from 'fs';
2
+ import { readFileSync, realpathSync, existsSync } from 'fs';
4
3
  import { resolve, sep, dirname } from 'path';
5
- import yaml from 'js-yaml';
4
+ import * as yaml from 'js-yaml';
5
+ import { dump } from 'js-yaml';
6
6
  import { isIP } from 'net';
7
+ import { createHash } from 'crypto';
8
+ import { AsyncLocalStorage } from 'async_hooks';
7
9
 
8
10
  // src/schema/manifest.schema.ts
9
11
  var refOrLiteral = z.string();
@@ -16,17 +18,37 @@ var MetadataSchema = z.object({
16
18
  author: z.string().optional(),
17
19
  license: z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
18
20
  });
21
+ function isHttpUrl(value) {
22
+ if (value.startsWith("$env:")) return true;
23
+ try {
24
+ const url = new URL(value);
25
+ return (url.protocol === "http:" || url.protocol === "https:") && url.hostname !== "";
26
+ } catch {
27
+ return false;
28
+ }
29
+ }
30
+ var baseUrlSchema = z.string().regex(/^(\$env:[A-Za-z_][A-Za-z0-9_]*|https?:\/\/.+)$/, 'must be an http(s) URL or a "$env:VAR" reference').refine(isHttpUrl, 'must be an http(s) URL with a host, or a "$env:VAR" reference').optional().describe('Override the provider endpoint with an OpenAI-compatible base URL (vLLM, Ollama, LiteLLM, an internal proxy, a mock LLM). A literal http(s) URL with a host (checked by the loader, not expressible in JSON Schema) or "$env:VAR". When set, the gateway treats the model as a custom OpenAI-compatible provider for any "provider" value except "anthropic", whose baseUrl is an Anthropic Messages endpoint without "/v1" (the gateway adds it). Omit to use the provider default.');
31
+ var ModelPricingSchema = z.object({
32
+ input: z.number().nonnegative().describe("USD per 1M input (prompt) tokens."),
33
+ output: z.number().nonnegative().describe("USD per 1M output (completion) tokens."),
34
+ cacheRead: z.number().nonnegative().optional().describe("USD per 1M tokens read from the prompt cache."),
35
+ cacheWrite: z.number().nonnegative().optional().describe("USD per 1M tokens written to the prompt cache."),
36
+ reasoning: z.number().nonnegative().optional().describe("USD per 1M reasoning tokens. The gateway uses the output price when omitted.")
37
+ }).describe("Price of this model in USD per 1M tokens. The gateway uses it instead of its built-in price, so a maxMonthlyUSD budget is spent on models it does not price (e.g. o4-mini). Omit to use the gateway's built-in price.");
19
38
  var ModelFallbackSchema = z.object({
20
39
  provider: z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
21
40
  id: z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
22
41
  apiKey: refOrLiteral.describe(refDesc),
42
+ baseUrl: baseUrlSchema,
23
43
  triggerOn: z.array(z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
24
- maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
44
+ maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover)."),
45
+ pricing: ModelPricingSchema.optional()
25
46
  });
26
47
  var ModelSchema = z.object({
27
48
  provider: z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
28
49
  id: z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
29
50
  apiKey: refOrLiteral.describe(refDesc),
51
+ baseUrl: baseUrlSchema,
30
52
  parameters: z.object({
31
53
  temperature: z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
32
54
  maxTokens: z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
@@ -35,6 +57,7 @@ var ModelSchema = z.object({
35
57
  presencePenalty: z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
36
58
  }).optional(),
37
59
  fallback: ModelFallbackSchema.optional(),
60
+ pricing: ModelPricingSchema.optional(),
38
61
  costControls: z.object({
39
62
  maxMonthlyUSD: z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
40
63
  alertAtUSD: z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
@@ -69,7 +92,12 @@ var McpServerSchema = z.object({
69
92
  transport: z.enum(["stdio", "sse", "http"]).describe('"stdio" = spawn a local process (requires "command"), "sse" = Server-Sent Events over HTTP, "http" = streamable HTTP (requires "url").'),
70
93
  command: z.string().optional().describe('Shell command to spawn the MCP server process. Required for transport "stdio" (e.g. "npx", "python").'),
71
94
  args: z.array(z.string()).optional().describe('Arguments passed to the command (e.g. ["-m", "my_mcp_server"]). Only used for transport "stdio".'),
72
- url: z.string().url().optional().describe('Base URL of the remote MCP server. Required for transport "sse" or "http". ' + refDesc),
95
+ // refOrLiteral, not z.string().url(). This field's own description promises reference syntax,
96
+ // and `.url()` rejected every form of it — so an MCP server whose address comes from the
97
+ // environment (`$env:GITHUB_MCP_URL`, which is the normal case in a cluster, where the URL is
98
+ // a Service name that differs per namespace) could not be declared at all. Every sibling
99
+ // address field in this schema already uses refOrLiteral; this one was the outlier.
100
+ url: refOrLiteral.optional().describe('Base URL of the remote MCP server. Required for transport "sse" or "http". ' + refDesc),
73
101
  env: z.record(z.string(), refOrLiteral).optional().describe('Environment variables injected into the MCP server process. Values support reference syntax. Use "$env:API_KEY" to forward secrets without hardcoding.'),
74
102
  healthCheck: z.object({
75
103
  timeoutSeconds: z.number().int().min(1).max(60).default(5).describe("Seconds to wait for the MCP server to respond during health checks. Increase for slow-starting processes.")
@@ -145,6 +173,7 @@ var ChatEndpointSchema = z.object({
145
173
  var ApiSchema = z.object({
146
174
  type: z.enum(["rest", "mcp", "grpc", "websocket"]),
147
175
  port: z.number().int().min(1).max(65535).optional(),
176
+ allowedPorts: z.array(z.number().int().min(1).max(65535)).optional().describe("Extra ports the agent may listen on besides the sidecar-routed API port (e.g. a platform metrics port). The sidecar reports not-ready while the agent listens on any other non-loopback port."),
148
177
  pathPrefix: z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
149
178
  auth: z.object({
150
179
  type: z.enum(["jwt", "apikey", "oauth2", "none"]),
@@ -153,7 +182,8 @@ var ApiSchema = z.object({
153
182
  }).optional(),
154
183
  rateLimit: z.object({
155
184
  requestsPerMinute: z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
156
- requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
185
+ requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429."),
186
+ tokensPerMinute: z.number().int().min(1).optional().describe("Maximum LLM tokens the gateway allows per minute. Once the bucket is empty, requests receive HTTP 429.")
157
187
  }).optional(),
158
188
  streaming: z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
159
189
  healthEndpoint: z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
@@ -235,25 +265,42 @@ var EvalMetricEnum = z.enum([
235
265
  // RAG: do the retrieved chunks cover the ground-truth answer?
236
266
  "bias",
237
267
  // Does the answer exhibit demographic or political bias?
268
+ "grounded",
269
+ // Are the answer's claims supported by the retrieved context? Replaces faithfulness and hallucination.
270
+ "pii_leakage",
271
+ // Does the answer leak personal data?
272
+ "prompt_injection",
273
+ // Did the answer follow an injected instruction?
238
274
  "custom"
239
- // Custom scorer defined in the evaluation framework config.
275
+ // Rubric-graded scorer, declared in the sibling `custom: { name, rubric }`.
240
276
  ]);
241
- var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for faithfulness, context_precision, hallucination metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
277
+ var LiveMetricEnum = EvalMetricEnum.exclude(["context_recall", "answer_similarity"]);
278
+ var CustomMetricSchema = z.object({
279
+ name: z.string().min(1).describe("Name the custom metric is reported under."),
280
+ rubric: refOrLiteral.describe('Grading rubric. Use "$file:rubrics/<name>.md"; the loader resolves it to the file contents.')
281
+ });
282
+ var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for grounded and context_precision metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
242
283
  var EvaluationSchema = z.object({
243
- framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
284
+ framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]).optional().describe("Deprecated: nothing reads it. validate warns when set; rejected in 0.6.0."),
285
+ live: z.object({
286
+ metrics: z.array(LiveMetricEnum).optional().describe("Metrics scored on live traffic."),
287
+ thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe("Minimum pass rate per live metric (0.0\u20131.0)."),
288
+ custom: CustomMetricSchema.optional()
289
+ }).optional().describe("Live evaluation, separate from dataset evaluation. Only live metrics are allowed."),
290
+ custom: CustomMetricSchema.optional().describe('Rubric for the "custom" metric on datasets.'),
244
291
  datasets: z.array(
245
292
  z.object({
246
293
  name: z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
247
294
  path: refOrLiteral.describe(datasetPathDesc),
248
295
  metrics: z.array(EvalMetricEnum).optional().describe(
249
- 'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["faithfulness","context_recall"] while a safety dataset sets ["toxicity","bias"].'
296
+ 'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["grounded","context_recall"] while a safety dataset sets ["toxicity","bias"].'
250
297
  )
251
298
  })
252
299
  ).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
253
300
  metrics: z.array(EvalMetricEnum).optional().describe(
254
- 'Default metrics to compute across all datasets that do not declare their own metrics. "faithfulness", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires a scorer registered in the evaluation framework config.'
301
+ 'Default metrics to compute across all datasets that do not declare their own metrics. "grounded", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires spec.evaluation.custom: { name, rubric }. "faithfulness" and "hallucination" are deprecated aliases of "grounded".'
255
302
  ),
256
- thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"faithfulness": 0.8, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
303
+ thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"grounded": 0.95, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
257
304
  ciGate: z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
258
305
  });
259
306
  var ObservabilitySchema = z.object({
@@ -352,12 +399,608 @@ var ManifestSchema = z.object({
352
399
  metadata: MetadataSchema,
353
400
  spec: SpecSchema
354
401
  });
402
+ var DRAFT_07 = "http://json-schema.org/draft-07/schema#";
403
+ var options = {
404
+ target: "draft-7",
405
+ io: "input",
406
+ reused: "inline",
407
+ override: ({ zodSchema, jsonSchema }) => {
408
+ const def = zodSchema._zod.def;
409
+ if (def.type === "object" && jsonSchema.additionalProperties === void 0) jsonSchema.additionalProperties = false;
410
+ if (def.type === "record") delete jsonSchema.propertyNames;
411
+ if (def.type === "union" && def.discriminator && jsonSchema.oneOf) {
412
+ jsonSchema.anyOf = jsonSchema.oneOf;
413
+ delete jsonSchema.oneOf;
414
+ }
415
+ if (jsonSchema.type === "integer") {
416
+ if (jsonSchema.maximum === Number.MAX_SAFE_INTEGER) delete jsonSchema.maximum;
417
+ if (jsonSchema.minimum === Number.MIN_SAFE_INTEGER) delete jsonSchema.minimum;
418
+ }
419
+ }
420
+ };
421
+ function toJsonSchema(schema) {
422
+ const { $schema: _, ...body } = z.toJSONSchema(schema, options);
423
+ return { ...body, $schema: DRAFT_07 };
424
+ }
425
+ function toNamedJsonSchema(name, schema) {
426
+ const { $schema, ...body } = toJsonSchema(schema);
427
+ return { $ref: `#/definitions/${name}`, definitions: { [name]: body }, $schema };
428
+ }
355
429
  function exportJsonSchema() {
356
- return zodToJsonSchema(ManifestSchema, {
357
- name: "AgentSpecManifest",
358
- $refStrategy: "none"
430
+ return toNamedJsonSchema("AgentSpecManifest", ManifestSchema);
431
+ }
432
+ var EvidenceClass = z.enum(["independent", "evidence-point", "offline", "declared"]);
433
+ var Collector = z.enum([
434
+ "gateway-config",
435
+ "gateway-traffic",
436
+ "sidecar-probe",
437
+ "sidecar-forwarder",
438
+ "sidecar-eval",
439
+ "operator-webhook",
440
+ "operator-logs",
441
+ "sdk",
442
+ "lock",
443
+ "manifest"
444
+ ]);
445
+ var EvidencePoint = z.enum(["guardrail", "tool", "mcpClient", "memoryWrite", "vectorStore", "logging"]);
446
+ var Scope = z.enum(["declared", "configured", "enforced", "observed", "ran-where-wired", "every-path", "effective"]);
447
+ var WiredBy = z.enum(["none", "operator-render", "agent-code", "user-schedule", "opt-in"]);
448
+ var Mode = z.enum(["standalone", "operator"]);
449
+ var BuildStatus = z.enum(["built", "partial", "planned", "open"]);
450
+ var RuleEvidenceSource = z.object({
451
+ class: EvidenceClass,
452
+ collector: Collector,
453
+ /** Evidence-point sources only. */
454
+ point: EvidencePoint.nullish(),
455
+ proves: Scope,
456
+ wiredBy: WiredBy,
457
+ modes: z.array(Mode).min(1),
458
+ status: BuildStatus,
459
+ /** A variant such as HTTP MCP, stdio MCP, local tools or opt-in. */
460
+ when: z.string().min(1).nullish(),
461
+ /** A collector that checks the counts, such as gateway calls against guardrail records. */
462
+ crossCheck: Collector.nullish(),
463
+ /** The `agentspec lock` prompt question. */
464
+ question: z.string().min(1).nullish()
465
+ }).strict().refine((s) => s.class === "evidence-point" === (s.point != null), {
466
+ message: "an evidence-point source names its `point`, and no other class does"
467
+ });
468
+ var RuleEvidenceEntry = z.object({
469
+ sources: z.array(RuleEvidenceSource).default([]),
470
+ /** Rules whose verdicts decide this one (all must pass); this rule inherits their sources. */
471
+ derivedFrom: z.array(z.string().min(1)).nullish(),
472
+ /** What a pass does not prove, shown with the result. Free text. */
473
+ notProven: z.string().nullish()
474
+ }).strict().refine((e) => e.sources.length > 0 || (e.derivedFrom?.length ?? 0) > 0, {
475
+ message: "an entry needs at least one source or a `derivedFrom`"
476
+ });
477
+ var BOTH = ["standalone", "operator"];
478
+ var CLASS_DEFAULTS = {
479
+ independent: { wiredBy: "none", status: "built" },
480
+ "evidence-point": { wiredBy: "agent-code", status: "planned" },
481
+ offline: { wiredBy: "user-schedule", status: "planned" },
482
+ declared: { wiredBy: "none", status: "built" }
483
+ };
484
+ var source = (cls, collector, proves, extra = {}) => ({ class: cls, collector, proves, modes: BOTH, ...CLASS_DEFAULTS[cls], ...extra });
485
+ var independent = (collector, proves, extra) => source("independent", collector, proves, extra);
486
+ var point = (p, proves, extra) => source("evidence-point", "sdk", proves, { point: p, ...extra });
487
+ var lock = (question, extra) => source("offline", "lock", "every-path", { question, ...extra });
488
+ var declared = (extra) => source("declared", "manifest", "declared", extra);
489
+ var OPERATOR_ONLY = { modes: ["operator"] };
490
+ var PLANNED = { status: "planned" };
491
+ var WRITES_BYPASS = "Writes that bypass the evidence point at runtime.";
492
+ var entries = {
493
+ "MODEL-01": { sources: [independent("gateway-config", "configured"), independent("gateway-traffic", "observed")] },
494
+ "MODEL-02": {
495
+ sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
496
+ },
497
+ "MODEL-03": { sources: [independent("gateway-config", "enforced")] },
498
+ "MODEL-04": {
499
+ sources: [
500
+ independent("gateway-config", "enforced"),
501
+ lock("Do the agent's own provider clients retry more than spec.model.fallback.maxRetries, or stack retries on the gateway's?", { when: "client retries" })
502
+ ],
503
+ notProven: "That the gateway's attempts match `spec.model.fallback.maxRetries` at runtime beyond the rendered config."
504
+ },
505
+ "MODEL-05": {
506
+ sources: [independent("gateway-traffic", "observed", { when: "agw.ai.usage.cost.total" })],
507
+ notProven: "Requests with no token usage: not counted."
508
+ },
509
+ "SEC-LLM-01": {
510
+ sources: [
511
+ independent("sidecar-probe", "effective", { when: "opt-in", wiredBy: "opt-in", status: "planned" }),
512
+ point("guardrail", "ran-where-wired"),
513
+ lock("Does every path that sends user input to a model run the declared input guardrails first?")
514
+ ]
515
+ },
516
+ "SEC-LLM-02": {
517
+ sources: [
518
+ point("guardrail", "ran-where-wired"),
519
+ lock("Does every path that returns model output run the declared output guardrails first?")
520
+ ],
521
+ notProven: "Effective: a no-op output guardrail still fires."
522
+ },
523
+ "SEC-LLM-03": { sources: [declared()] },
524
+ "SEC-LLM-04": {
525
+ sources: [independent("gateway-config", "enforced"), independent("gateway-traffic", "observed", PLANNED)],
526
+ notProven: "Cost: MODEL-03 covers it."
527
+ },
528
+ "SEC-LLM-05": {
529
+ sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
530
+ },
531
+ "SEC-LLM-06": { derivedFrom: ["MEM-01"], notProven: "Same as MEM-01." },
532
+ "SEC-LLM-07": {
533
+ sources: [
534
+ independent("sidecar-probe", "configured", { when: "HTTP MCP" }),
535
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
536
+ lock("Does the agent call any MCP tool the manifest does not declare, or one that serves no annotations?"),
537
+ declared({ when: "local tools" })
538
+ ]
539
+ },
540
+ "SEC-LLM-08": {
541
+ sources: [
542
+ independent("sidecar-probe", "configured", PLANNED),
543
+ independent("gateway-traffic", "enforced", PLANNED),
544
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" })
545
+ ],
546
+ notProven: "Which tools are destructive: the server declares it; nothing checks the claim."
547
+ },
548
+ "SEC-LLM-09": { derivedFrom: ["EVAL-01", "EVAL-02"] },
549
+ "SEC-LLM-10": {
550
+ sources: [
551
+ independent("operator-webhook", "configured", { ...OPERATOR_ONLY, ...PLANNED }),
552
+ declared({ modes: ["standalone"] })
553
+ ]
554
+ },
555
+ "TOOL-01": {
556
+ sources: [
557
+ independent("gateway-traffic", "observed", { when: "HTTP MCP" }),
558
+ point("tool", "ran-where-wired", { when: "local tools" }),
559
+ point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
560
+ lock("Which tools does the agent code register or call, local, HTTP MCP and stdio MCP?")
561
+ ],
562
+ notProven: "Unwrapped local or stdio calls at runtime: only the lock inventory catches them."
563
+ },
564
+ "MEM-01": {
565
+ sources: [
566
+ point("memoryWrite", "enforced"),
567
+ lock("Does every memory write pass through the PII scrubber for the declared fields?")
568
+ ],
569
+ notProven: "Full coverage: the scrubber is a regex baseline (Presidio optional in Python). " + WRITES_BYPASS
570
+ },
571
+ "MEM-02": {
572
+ sources: [
573
+ point("memoryWrite", "enforced"),
574
+ lock("Does every memory write apply the declared retention TTL?")
575
+ ],
576
+ notProven: WRITES_BYPASS + " Backends that cannot apply a TTL (record `ttl: unsupported`): the evidence point is `not_verifiable` there and only the lock remains."
577
+ },
578
+ "MEM-03": {
579
+ sources: [
580
+ point("memoryWrite", "enforced"),
581
+ lock("Does every memory write honour the declared memory audit setting?")
582
+ ],
583
+ notProven: WRITES_BYPASS
584
+ },
585
+ "MEM-04": {
586
+ sources: [
587
+ point("vectorStore", "ran-where-wired"),
588
+ lock("Does every vector store query and write use the declared namespace or tenant isolation?")
589
+ ],
590
+ notProven: "Isolation: the right namespace does not prove other tenants cannot read it."
591
+ },
592
+ "MEM-05": {
593
+ sources: [independent("gateway-traffic", "observed", { when: "gen_ai.usage.output_tokens" }), declared({ when: "until a call is observed" })],
594
+ notProven: "The tolerance for the system prompt and tool schemas is set in Phase 2, with an ADR-0015 amendment; a call without gen_ai.usage.output_tokens leaves the check not_verifiable."
595
+ },
596
+ "EVAL-01": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
597
+ "EVAL-02": {
598
+ sources: [
599
+ independent("sidecar-eval", "observed", PLANNED),
600
+ lock("Does the agent's CI run the declared evaluation gate before deploy?")
601
+ ],
602
+ notProven: "That a failed gate actually blocks a deploy: CI is outside the system."
603
+ },
604
+ "EVAL-03": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
605
+ "OBS-01": {
606
+ sources: [independent("sidecar-forwarder", "observed"), independent("sidecar-probe", "configured")],
607
+ notProven: "That the agent exports every span: the rule is about the backend, not span coverage. agentops is not a target: it has no OTLP ingest."
608
+ },
609
+ "OBS-02": {
610
+ sources: [
611
+ independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
612
+ point("logging", "ran-where-wired"),
613
+ lock("Does every log statement emit structured JSON?")
614
+ ]
615
+ },
616
+ "OBS-03": {
617
+ sources: [
618
+ independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
619
+ lock("Does any log statement write one of the declared sensitive fields?")
620
+ ],
621
+ notProven: "Full coverage: the scanner is a regex and pattern baseline, so recall is limited."
622
+ },
623
+ "META-01": { sources: [declared()] },
624
+ "META-02": {
625
+ sources: [independent("gateway-traffic", "observed", { when: "gen_ai.request.temperature" })],
626
+ notProven: "Requests that set no temperature: `not_verifiable`, not counted."
627
+ }
628
+ };
629
+ var RULE_EVIDENCE = Object.fromEntries(
630
+ Object.entries(entries).map(([id, entry]) => [id, RuleEvidenceEntry.parse(entry)])
631
+ );
632
+ function ruleSources(ruleId, registry = RULE_EVIDENCE, seen = /* @__PURE__ */ new Set()) {
633
+ if (!Object.hasOwn(registry, ruleId) || seen.has(ruleId)) return [];
634
+ const entry = registry[ruleId];
635
+ const visited = /* @__PURE__ */ new Set([...seen, ruleId]);
636
+ return [...entry.sources, ...(entry.derivedFrom ?? []).flatMap((parent) => ruleSources(parent, registry, visited))];
637
+ }
638
+ function ruleClasses(ruleId, registry = RULE_EVIDENCE) {
639
+ const classes = new Set(ruleSources(ruleId, registry).map((s) => s.class));
640
+ return EvidenceClass.options.filter((c) => classes.has(c));
641
+ }
642
+ var EvidenceStatus = z.enum(["pass", "fail", "not_verifiable"]);
643
+ var EvidenceSource = z.enum(["independent", "sdk-observed", "lower-bound", "self-declared", "declared", "offline"]);
644
+ var EvidenceOrigin = z.enum(["ingress", "gateway", "egress_proxy", "in_process", "probe", "manifest", "judge"]);
645
+ var SeveritySchema = z.enum(["critical", "high", "medium", "low", "info"]);
646
+ var TrustPolicy = z.enum(["standard", "strict"]);
647
+ var EvidenceReason = z.object({ code: z.string().min(1), detail: z.string().nullish() }).strict();
648
+ var TIMESTAMP_PATTERN = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
649
+ var Timestamp = z.string().regex(TIMESTAMP_PATTERN);
650
+ var EvidenceWindow = z.object({ start: Timestamp, end: Timestamp }).strict();
651
+ var Sha256 = z.string().regex(/^[a-f0-9]{64}$/);
652
+ var MAX_TRACE_IDS = 20;
653
+ var recordFields = {
654
+ /** Scoped per subject: `tool:<name>`, `model:<id>`, or a bare check name for agent-wide checks. */
655
+ checkId: z.string().min(1),
656
+ ruleId: z.string().min(1).nullish(),
657
+ source: EvidenceSource,
658
+ origin: EvidenceOrigin,
659
+ /** The registry collector that produced the record. Optional until every producer sets it (#422); checking it against the collectors the registry lists for the rule is the grader's job (#423). */
660
+ collector: Collector.nullish(),
661
+ observedAt: Timestamp,
662
+ window: EvidenceWindow,
663
+ ttlSeconds: z.number().int().nonnegative(),
664
+ sampleSize: z.number().int().nonnegative(),
665
+ traceIds: z.array(z.string().min(1)).max(MAX_TRACE_IDS),
666
+ manifestSha256: Sha256
667
+ };
668
+ var EvidenceRecordPass = z.object({ ...recordFields, status: z.literal("pass"), reason: EvidenceReason.nullish() }).strict();
669
+ var EvidenceRecordNotPass = z.object({
670
+ ...recordFields,
671
+ status: z.enum(["fail", "not_verifiable"]),
672
+ reason: EvidenceReason
673
+ }).strict();
674
+ var EvidenceRecord = z.union([EvidenceRecordPass, EvidenceRecordNotPass]);
675
+ var PodspecFinding = z.object({ code: z.string().min(1), subject: z.string().min(1), message: z.string().min(1) }).strict();
676
+ var ReadinessFinding = z.object({
677
+ code: z.string().min(1),
678
+ severity: SeveritySchema,
679
+ message: z.string().min(1),
680
+ subject: z.string().nullish(),
681
+ ruleIds: z.array(z.string().min(1)).nullish()
682
+ }).strict();
683
+ var EvalResult = z.object({
684
+ metric: z.string().min(1),
685
+ n: z.number().int().nonnegative(),
686
+ passRate: z.number().min(0).max(1),
687
+ ciLow: z.number().min(0).max(1),
688
+ ciHigh: z.number().min(0).max(1),
689
+ threshold: z.number().min(0).max(1),
690
+ window: EvidenceWindow,
691
+ judgeModel: z.string().min(1),
692
+ rubricHash: Sha256,
693
+ source: EvidenceSource,
694
+ dataset: z.string().min(1).nullish()
695
+ }).strict();
696
+ var MAX_POINT_RECORDS = 5;
697
+ var EvidencePointState = z.enum(["fired", "wired", "not_wired"]);
698
+ var EvidencePointAggregate = z.object({
699
+ point: EvidencePoint,
700
+ /** What the point is wired on, a key of `CONTRACT.evidencePoints[point]`. */
701
+ target: z.string().min(1),
702
+ state: EvidencePointState,
703
+ counts: z.object({
704
+ pass: z.number().int().nonnegative(),
705
+ fail: z.number().int().nonnegative(),
706
+ not_verifiable: z.number().int().nonnegative()
707
+ }).strict(),
708
+ records: z.array(EvidenceRecord).max(MAX_POINT_RECORDS)
709
+ }).strict();
710
+ var Passthrough = z.record(z.string(), z.unknown());
711
+ var Heartbeat = z.object({
712
+ /** Contract version of the sending SDK (`CONTRACT.version`); absent or null from older SDKs. */
713
+ contract: z.string().regex(/^\d+\.\d+\.\d+$/).nullish(),
714
+ health: Passthrough,
715
+ records: z.array(EvidenceRecord).default([]),
716
+ /** Every evidence point the registry gives for the manifest, wired or not; absent from older SDKs. */
717
+ evidencePoints: z.array(EvidencePointAggregate).default([]),
718
+ usage: Passthrough.nullish(),
719
+ tools: Passthrough.nullish()
720
+ }).strict();
721
+ var SidecarPush = z.object({
722
+ manifestSha256: Sha256,
723
+ declared: Passthrough.nullish(),
724
+ records: z.array(EvidenceRecord).default([])
725
+ }).strict();
726
+ var EVIDENCE_SHAPES = { EvidenceRecord, ReadinessFinding, EvalResult, TrustPolicy, Heartbeat, SidecarPush };
727
+
728
+ // src/schema/contract.schema.ts
729
+ var fixed = (value) => z.literal(value).default(value);
730
+ var REASON_CODES = [
731
+ "stale",
732
+ "source_not_admitted",
733
+ "source_mismatch",
734
+ "no_evidence",
735
+ "not_wired",
736
+ "no_calls_observed",
737
+ "no_failover_observed",
738
+ "no_budget_evidence",
739
+ "no_approval_token_evidence",
740
+ "no_retrieval_spans",
741
+ "unrecognised_conventions",
742
+ "instrumentation_unavailable",
743
+ "declared_env_missing",
744
+ "provider_key_on_agent",
745
+ "response_cap_exceeded",
746
+ "temperature_above_max",
747
+ "undeclared_tool",
748
+ "undeclared_model",
749
+ "unpinned_model",
750
+ "self_declared",
751
+ "unpriced_model",
752
+ "undeclared_namespace"
753
+ ];
754
+ var ContractSchema = z.object({
755
+ /** Contract version (major.minor.patch) a heartbeat carries: a receiver rejects another major and accepts a newer minor (ADR-0039). */
756
+ version: fixed("1.0.0"),
757
+ env: z.object({
758
+ manifestPath: fixed("MANIFEST_PATH"),
759
+ upstreamUrl: fixed("UPSTREAM_URL"),
760
+ proxyPort: fixed("PROXY_PORT"),
761
+ controlPlanePort: fixed("CONTROL_PLANE_PORT"),
762
+ /** Control plane URL the operator webhook injects into the sidecar and agent containers; SDK push mode falls back to it. */
763
+ controlPlaneUrl: fixed("AGENTSPEC_CONTROL_PLANE_URL"),
764
+ /** Agent JWT from `POST /api/v1/register`; `agentspec register` sends it to the control plane as a Bearer token. The sidecar's own push needs the `sidecarToken` from the same call (see `agentTokenSecretSuffix`). */
765
+ agentKey: fixed("AGENTSPEC_API_KEY"),
766
+ /** Trust policy (`standard | strict`) the grader applies: set on the control plane and the sidecar, never in `agent.yaml` (ADR-0007). */
767
+ trustPolicy: fixed("AGENTSPEC_TRUST_POLICY"),
768
+ /** The webhook's PodSpec findings (the `agentspec.io/podspec-findings` annotation, JSON), injected into the sidecar by the downward API (ADR-0019). */
769
+ podspecFindings: fixed("AGENTSPEC_PODSPEC_FINDINGS"),
770
+ /** Virtual key the agent sends and the gateway accepts (ADR-0001); provider keys stay in the gateway. */
771
+ gatewayKey: fixed("AGENTSPEC_GATEWAY_KEY"),
772
+ /** Bind address and port the agent must honour so the sidecar can own `spec.api.port` (ADR-0028). */
773
+ agentHost: fixed("HOST"),
774
+ agentPort: fixed("PORT"),
775
+ /** Azure resource name the gateway config references (`$AZURE_OPENAI_RESOURCE_NAME`). */
776
+ azureResourceName: fixed("AZURE_OPENAI_RESOURCE_NAME")
777
+ }).strict().prefault({}),
778
+ /** Provider base-URL env vars, keyed by spec.model.provider. */
779
+ providerBaseUrlEnv: z.object({
780
+ openai: fixed("OPENAI_BASE_URL"),
781
+ anthropic: fixed("ANTHROPIC_BASE_URL")
782
+ }).strict().prefault({}),
783
+ paths: z.object({
784
+ manifestDir: fixed("/manifest"),
785
+ manifestFile: fixed("/manifest/agent.yaml"),
786
+ /** SQLite file the gateway keeps USD budget usage in; must be on a writable volume. */
787
+ gatewayDatabase: fixed("/gateway-data/agentgateway.db"),
788
+ /** Directory the init container renders the gateway config into (ADR-0025). */
789
+ gatewayConfigDir: fixed("/gateway"),
790
+ gatewayConfigFile: fixed("/gateway/config.yaml"),
791
+ /** Sidecar bundle inside its image; `node <entrypoint> gateway-config …` is the init container's command (ADR-0025). */
792
+ sidecarEntrypoint: fixed("dist/index.js"),
793
+ gatewayDataDir: fixed("/gateway-data")
794
+ }).strict().prefault({}),
795
+ images: z.object({
796
+ sidecar: fixed("ghcr.io/agents-oss/agentspec-sidecar:latest"),
797
+ /** agentgateway, pinned (ADR-0025). */
798
+ gateway: fixed("cr.agentgateway.dev/agentgateway:v1.6.0")
799
+ }).strict().prefault({}),
800
+ ports: z.object({
801
+ agentDefault: fixed(8e3),
802
+ /** Loopback port the agent binds once the sidecar takes over its declared port (ADR-0028). */
803
+ agentBind: fixed(18e3),
804
+ proxy: fixed(4e3),
805
+ controlPlane: fixed(4001),
806
+ /** agentgateway LLM listener on the pod loopback (ADR-0025). */
807
+ gateway: fixed(4002),
808
+ /** agentgateway's default stats and readiness listeners: wildcard-bound, not the agent's (ADR-0025). */
809
+ gatewayStats: fixed(15020),
810
+ gatewayReadiness: fixed(15021)
811
+ }).strict().prefault({}),
812
+ /** Bare paths served by the sidecar on the control-plane port (ADR-0023). */
813
+ sidecarPaths: z.object({
814
+ healthReady: fixed("/health/ready"),
815
+ gap: fixed("/gap"),
816
+ explore: fixed("/explore"),
817
+ proof: fixed("/proof"),
818
+ traces: fixed("/v1/traces"),
819
+ /** Read-only GET of the buffered, already redacted spans the gateway copied to the sidecar. */
820
+ spans: fixed("/spans")
821
+ }).strict().prefault({}),
822
+ /** Bare paths an agent serves itself (SDK `/agentspec/health`); the sidecar probes them on the upstream. */
823
+ agentPaths: z.object({
824
+ health: fixed("/agentspec/health")
825
+ }).strict().prefault({}),
826
+ /** Control plane API routes shared by the CLI, the sidecar push and the control plane; `{name}` is the registered agent name. */
827
+ controlPlanePaths: z.object({
828
+ /** POST: registers an agent and returns its per-agent token. */
829
+ register: fixed("/api/v1/register"),
830
+ /** POST: the SDK push-mode heartbeat, authenticated with the agent token from `register`. */
831
+ heartbeat: fixed("/api/v1/heartbeat"),
832
+ declaredAudit: fixed("/api/v1/agents/{name}/declared-audit"),
833
+ /** POST: the sidecar's declared audit (at start and on a manifest hash change) and its own evidence records (ADR-0008). */
834
+ sidecarPush: fixed("/api/v1/agents/{name}/sidecar-push"),
835
+ /** GET: the latest stored EvidenceRecord per (ruleId, checkId, source) of an agent, as `{ records }`; read by the mcp-server. */
836
+ evidence: fixed("/api/v1/agents/{name}/evidence")
837
+ }).strict().prefault({}),
838
+ /** Names inside the pod patch that every injection path must use (ADR-0028). */
839
+ podPatch: z.object({
840
+ sidecarContainer: fixed("agentspec-sidecar"),
841
+ /** Pod annotation the webhook writes its PodSpec findings to (JSON list of `{code, subject, message}`). */
842
+ podspecFindingsAnnotation: fixed("agentspec.io/podspec-findings"),
843
+ /** Pod annotation naming the agent; the webhook reads the manifest ConfigMap `<name>-manifest` from it, and `--deploy k8s` writes it. */
844
+ agentNameAnnotation: fixed("agentspec.io/agent-name"),
845
+ manifestVolume: fixed("agentspec-manifest"),
846
+ proxyPortName: fixed("proxy"),
847
+ controlPortName: fixed("control"),
848
+ gatewayContainer: fixed("agentspec-gateway"),
849
+ gatewayConfigInitContainer: fixed("agentspec-gateway-config"),
850
+ /** Sidecar subcommand the init container runs: `node <entrypoint> <this> <manifest> --out <path>`. */
851
+ gatewayConfigCommand: fixed("gateway-config"),
852
+ gatewayConfigVolume: fixed("agentspec-gateway-config"),
853
+ gatewayDataVolume: fixed("agentspec-gateway-data"),
854
+ /** Secret `<metadata.name><suffix>` holding the `sidecarToken` from `POST /api/v1/register`; the webhook gives it to the sidecar only, as `env.agentKey`, when a control plane URL is set. */
855
+ agentTokenSecretSuffix: fixed("-agentspec-token"),
856
+ agentTokenSecretKey: fixed("token"),
857
+ /** Loopback address the agent and sidecar reach each other and the gateway on. */
858
+ loopback: fixed("127.0.0.1"),
859
+ /** Gateway container sizing (ADR-0025): request 25m/32Mi, limit 128Mi. */
860
+ gatewayCpuRequest: fixed("25m"),
861
+ gatewayMemoryRequest: fixed("32Mi"),
862
+ gatewayMemoryLimit: fixed("128Mi")
863
+ }).strict().prefault({}),
864
+ /** Admission webhook the operator serves (ADR-0028). */
865
+ webhook: z.object({
866
+ /** Path kube-apiserver posts to; kopf routes `POST /<handler id>`, so this is the handler id with a leading slash. Hyphenated: kube-apiserver rejects an underscore in a service path segment (RFC 1123). */
867
+ injectPath: fixed("/inject-sidecar"),
868
+ /** Namespace label that selects the namespaces whose pods are injected. */
869
+ namespaceLabel: fixed("agentspec.io/enabled")
870
+ }).strict().prefault({}),
871
+ /** Known `EvidenceRecord.reason.code` values (ADR-0006). */
872
+ reasonCodes: z.array(z.string()).default(REASON_CODES),
873
+ /** Severity weights for coverage and the verified score (ADR-0007). */
874
+ severityWeights: z.object({
875
+ critical: fixed(4),
876
+ high: fixed(3),
877
+ medium: fixed(2),
878
+ low: fixed(1),
879
+ info: fixed(0)
880
+ }).strict().prefault({}),
881
+ /** Minimum verified percentage per grade; below `D` is F (ADR-0007). */
882
+ gradeBands: z.object({ A: fixed(90), B: fixed(75), C: fixed(60), D: fixed(45) }).strict().prefault({}),
883
+ /** Severity-weighted coverage percentage below which no grade is shown (ADR-0007). */
884
+ coverageThreshold: fixed(60),
885
+ /** Rule evidence registry: what can prove each audit rule (ADR-0040). */
886
+ ruleEvidence: z.record(z.string(), RuleEvidenceEntry).default(RULE_EVIDENCE),
887
+ /** Evidence point → target → the rule its records serve; each rule's registry entry lists a source for that point. */
888
+ evidencePoints: z.record(z.string(), z.record(z.string(), z.string())).default({
889
+ guardrail: { input: "SEC-LLM-01", output: "SEC-LLM-02" },
890
+ tool: { local: "TOOL-01" },
891
+ mcpClient: { "tools/list": "SEC-LLM-07", "tools/call": "TOOL-01" },
892
+ memoryWrite: { scrub: "MEM-01", ttl: "MEM-02", audit: "MEM-03" },
893
+ vectorStore: { namespace: "MEM-04" },
894
+ logging: { json: "OBS-02" }
895
+ }),
896
+ /** Records an evidence point aggregate keeps per window. */
897
+ evidencePointRecordLimit: fixed(MAX_POINT_RECORDS),
898
+ /** A heartbeat above this is trimmed to counts before it is sent. */
899
+ heartbeatMaxBytes: fixed(64 * 1024),
900
+ /** OTLP: resource attributes every exported span carries, and the standard endpoint env var. */
901
+ otlp: z.object({
902
+ resourceAgentName: fixed("agentspec.agent.name"),
903
+ resourceManifestSha256: fixed("agentspec.manifest.sha256"),
904
+ endpointEnv: fixed("OTEL_EXPORTER_OTLP_ENDPOINT"),
905
+ /** Resource attribute naming the span's producer, and the value agentgateway exports (its default); only such spans are gateway evidence. */
906
+ resourceServiceName: fixed("service.name"),
907
+ gatewayServiceName: fixed("agentgateway")
908
+ }).strict().prefault({})
909
+ }).strict();
910
+ var CONTRACT = ContractSchema.parse({});
911
+
912
+ // src/schema/evaluation.ts
913
+ var REPLACED_BY = {
914
+ hallucination: "grounded",
915
+ faithfulness: "grounded"
916
+ };
917
+ function canonicalMetric(metric) {
918
+ return REPLACED_BY[metric] ?? metric;
919
+ }
920
+ function evaluationWarnings(manifest) {
921
+ const evaluation = manifest.spec.evaluation;
922
+ if (!evaluation) return [];
923
+ const warnings = [];
924
+ if (evaluation.framework !== void 0) {
925
+ warnings.push(
926
+ "spec.evaluation.framework is deprecated: nothing reads it and AgentSpec computes evaluation evidence itself. Remove it; it is rejected in 0.6.0."
927
+ );
928
+ }
929
+ const declared2 = [
930
+ ...evaluation.metrics ?? [],
931
+ ...evaluation.live?.metrics ?? [],
932
+ ...(evaluation.datasets ?? []).flatMap((d) => d.metrics ?? [])
933
+ ];
934
+ for (const metric of new Set(declared2)) {
935
+ const replacement = canonicalMetric(metric);
936
+ if (replacement !== metric) {
937
+ warnings.push(
938
+ `spec.evaluation metric "${metric}" is deprecated and is read as "${replacement}". Rename it to "${replacement}"; it is rejected in 0.6.0.`
939
+ );
940
+ }
941
+ }
942
+ warnings.push(
943
+ ...thresholdWarnings("spec.evaluation.thresholds", evaluation.thresholds),
944
+ ...thresholdWarnings("spec.evaluation.live.thresholds", evaluation.live?.thresholds)
945
+ );
946
+ return warnings;
947
+ }
948
+ function thresholdWarnings(path, thresholds) {
949
+ return Object.keys(thresholds ?? {}).flatMap((name) => {
950
+ const replacement = canonicalMetric(name);
951
+ if (replacement === name) return [];
952
+ const ignored = replacement in thresholds ? ` It is ignored: ${path}.${replacement} is also declared.` : "";
953
+ return [
954
+ `${path}.${name} is deprecated and is read as ${path}.${replacement}. Rename it to "${replacement}"; it is rejected in 0.6.0.${ignored}`
955
+ ];
359
956
  });
360
957
  }
958
+ function assertRubricRefs(parsed) {
959
+ const evaluation = parsed?.spec?.evaluation;
960
+ const live = evaluation?.live;
961
+ for (const [path, custom] of [
962
+ ["spec.evaluation.custom", evaluation?.custom],
963
+ ["spec.evaluation.live.custom", live?.custom]
964
+ ]) {
965
+ const rubric = custom?.rubric;
966
+ if (typeof rubric === "string" && !rubric.startsWith("$file:")) {
967
+ throw new Error(
968
+ `${path}.rubric must be a "$file:<path>" reference, got "${rubric}". Fix: write it as "$file:${rubric}" (path relative to agent.yaml).`
969
+ );
970
+ }
971
+ }
972
+ }
973
+ function normalizeEvaluation(manifest) {
974
+ const evaluation = manifest.spec.evaluation;
975
+ if (!evaluation) return manifest;
976
+ const metrics = (list) => list?.map(canonicalMetric);
977
+ const thresholds = (map) => {
978
+ if (!map) return map;
979
+ const out = {};
980
+ for (const [name, value] of Object.entries(map)) {
981
+ const key = canonicalMetric(name);
982
+ if (key === name || !(key in map)) out[key] = value;
983
+ }
984
+ return out;
985
+ };
986
+ return {
987
+ ...manifest,
988
+ spec: {
989
+ ...manifest.spec,
990
+ evaluation: {
991
+ ...evaluation,
992
+ metrics: metrics(evaluation.metrics),
993
+ thresholds: thresholds(evaluation.thresholds),
994
+ datasets: evaluation.datasets?.map((d) => ({ ...d, metrics: metrics(d.metrics) })),
995
+ live: evaluation.live && {
996
+ ...evaluation.live,
997
+ metrics: metrics(evaluation.live.metrics),
998
+ thresholds: thresholds(evaluation.live.thresholds)
999
+ }
1000
+ }
1001
+ }
1002
+ };
1003
+ }
361
1004
  function detectRefType(value) {
362
1005
  if (value.startsWith("$env:")) return "env";
363
1006
  if (value.startsWith("$secret:")) return "secret";
@@ -403,8 +1046,9 @@ function resolveRef(value, opts, { optional = false } = {}) {
403
1046
  Use paths relative to agent.yaml location only`
404
1047
  );
405
1048
  }
1049
+ const readPath = checkedReadPath(value, absPath, opts.baseDir);
406
1050
  try {
407
- return readFileSync(absPath, "utf-8");
1051
+ return readFileSync(readPath, "utf-8");
408
1052
  } catch {
409
1053
  if (optional) return "";
410
1054
  throw new Error(
@@ -426,6 +1070,29 @@ function resolveRef(value, opts, { optional = false } = {}) {
426
1070
  }
427
1071
  return value;
428
1072
  }
1073
+ function checkedReadPath(ref, absPath, baseDir) {
1074
+ let realTarget;
1075
+ try {
1076
+ realTarget = realpathSync(absPath);
1077
+ } catch (err) {
1078
+ if (err.code === "ENOENT") return absPath;
1079
+ throw new Error(
1080
+ `Cannot resolve ${ref}
1081
+ ${err.message}
1082
+ Fix the path: every folder in it must exist, be a folder and not loop back through symlinks`
1083
+ );
1084
+ }
1085
+ const realBase = realpathSync(baseDir);
1086
+ if (realTarget !== realBase && !realTarget.startsWith(realBase + sep)) {
1087
+ throw new Error(
1088
+ `Path traversal detected in ${ref}
1089
+ Resolved through a symlink to: ${realTarget}
1090
+ Which is outside baseDir: ${baseDir}
1091
+ Replace the symlink with a real folder inside the manifest directory, or move the file there`
1092
+ );
1093
+ }
1094
+ return realTarget;
1095
+ }
429
1096
  function resolveSecret(name, backend) {
430
1097
  switch (backend) {
431
1098
  case "env": {
@@ -452,7 +1119,7 @@ function resolveSecret(name, backend) {
452
1119
  }
453
1120
  function resolveRefs(obj, opts) {
454
1121
  if (typeof obj === "string") {
455
- return resolveRef(obj, opts, { optional: true });
1122
+ return resolveRef(obj, opts);
456
1123
  }
457
1124
  if (Array.isArray(obj)) {
458
1125
  return obj.map((item) => resolveRefs(item, opts));
@@ -509,13 +1176,15 @@ function loadManifest(filePath, opts = {}) {
509
1176
  throw new Error(`Invalid YAML in ${absPath}
510
1177
  ${String(err)}`);
511
1178
  }
1179
+ assertRubricRefs(parsed);
512
1180
  const toValidate = opts.resolve === true ? resolveRefs(parsed, {
513
1181
  baseDir,
514
1182
  secretBackend: opts.secretBackend,
515
1183
  failOnMissingEnv: opts.failOnMissingEnv ?? true
516
1184
  }) : parsed;
517
- const manifest = ManifestSchema.parse(toValidate);
518
- return { manifest, filePath: absPath, baseDir, raw };
1185
+ const declared2 = ManifestSchema.parse(toValidate);
1186
+ const warnings = evaluationWarnings(declared2);
1187
+ return { manifest: normalizeEvaluation(declared2), warnings, filePath: absPath, baseDir, raw };
519
1188
  }
520
1189
  function tryLoadManifest(filePath, opts = {}) {
521
1190
  try {
@@ -562,8 +1231,8 @@ function isLatestVersion(raw) {
562
1231
  function runEnvChecks(manifest, rawManifest) {
563
1232
  const checks = [];
564
1233
  const envRefs = collectEnvRefs(rawManifest);
565
- const declared = manifest.spec.requires?.envVars ?? [];
566
- for (const v of declared) envRefs.add(v);
1234
+ const declared2 = manifest.spec.requires?.envVars ?? [];
1235
+ for (const v of declared2) envRefs.add(v);
567
1236
  for (const varName of envRefs) {
568
1237
  const present = process.env[varName] !== void 0 && process.env[varName] !== "";
569
1238
  checks.push({
@@ -600,21 +1269,53 @@ var PROVIDER_ENDPOINTS = {
600
1269
  openai: "https://api.openai.com/v1/models",
601
1270
  anthropic: "https://api.anthropic.com/v1/models",
602
1271
  groq: "https://api.groq.com/openai/v1/models",
603
- azure: "https://management.azure.com",
604
- google: "https://generativelanguage.googleapis.com",
1272
+ google: "https://generativelanguage.googleapis.com/v1beta/models",
605
1273
  mistral: "https://api.mistral.ai/v1/models",
606
- cohere: "https://api.cohere.ai/v1",
1274
+ cohere: "https://api.cohere.com/v1/models",
607
1275
  together: "https://api.together.xyz/v1/models",
608
1276
  fireworks: "https://api.fireworks.ai/inference/v1/models"
609
1277
  };
610
- async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
611
- const url = PROVIDER_ENDPOINTS[provider.toLowerCase()];
612
- if (!url) {
1278
+ var AZURE_API_VERSION = "2024-10-21";
1279
+ var AZURE_RESOURCE_HOSTS = [".openai.azure.com", ".cognitiveservices.azure.com", ".openai.azure.us", ".openai.azure.cn"];
1280
+ function isAzureResourceEndpoint(base) {
1281
+ const { hostname, pathname } = new URL(base);
1282
+ return AZURE_RESOURCE_HOSTS.some((suffix) => hostname.endsWith(suffix)) && pathname === "/";
1283
+ }
1284
+ function probeUrl(provider, baseUrl) {
1285
+ const key = provider.toLowerCase();
1286
+ const nativeUrl = PROVIDER_ENDPOINTS[key];
1287
+ if (!nativeUrl && !(key === "azure" && baseUrl)) return void 0;
1288
+ if (!baseUrl) return { url: nativeUrl, native: true };
1289
+ const base = resolveRef(baseUrl, { baseDir: "." }).replace(/\/+$/, "");
1290
+ if (key === "azure" && isAzureResourceEndpoint(base)) {
1291
+ return { url: `${base}/openai/models?api-version=${AZURE_API_VERSION}`, native: false, azureResource: true };
1292
+ }
1293
+ if (key === "anthropic") return { url: `${base}/v1/models`, native: false };
1294
+ return { url: `${base}/models`, native: false };
1295
+ }
1296
+ function unverifiableProbe(provider) {
1297
+ if (provider.toLowerCase() === "azure") {
613
1298
  return {
614
- reachable: true
615
- // Unknown provider — assume reachable, can't check
1299
+ outcome: "not_verifiable",
1300
+ error: "azure keys are checked against your own resource endpoint, and spec.model.baseUrl is not set to name it",
1301
+ remediation: "Set spec.model.baseUrl to your Azure resource endpoint (https://<resource>.openai.azure.com) so the key can be probed, or verify it out of band"
616
1302
  };
617
1303
  }
1304
+ return { outcome: "not_verifiable", error: `unknown provider "${provider}"` };
1305
+ }
1306
+ async function checkModelEndpoint(provider, apiKey, baseUrl, timeoutMs = 5e3) {
1307
+ let target;
1308
+ try {
1309
+ target = probeUrl(provider, baseUrl);
1310
+ } catch (err) {
1311
+ return {
1312
+ outcome: "unreachable",
1313
+ error: err instanceof Error ? err.message.split("\n")[0] : String(err),
1314
+ remediation: `Set the variable behind spec.model baseUrl (${baseUrl}) or remove the override`
1315
+ };
1316
+ }
1317
+ if (!target) return unverifiableProbe(provider);
1318
+ const { url, native, azureResource } = target;
618
1319
  const start = Date.now();
619
1320
  const controller = new AbortController();
620
1321
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -628,19 +1329,30 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
628
1329
  headers["anthropic-version"] = "2023-06-01";
629
1330
  delete headers["Authorization"];
630
1331
  }
1332
+ if (azureResource) {
1333
+ headers["api-key"] = apiKey;
1334
+ delete headers["Authorization"];
1335
+ }
1336
+ const isGemini = provider.toLowerCase() === "google" && native;
1337
+ if (isGemini) {
1338
+ headers["x-goog-api-key"] = apiKey;
1339
+ delete headers["Authorization"];
1340
+ }
631
1341
  const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
632
1342
  const latencyMs = Date.now() - start;
633
- const reachable = res.status < 500;
634
- return {
635
- reachable,
636
- latencyMs,
637
- error: !reachable ? `HTTP ${res.status}` : void 0
638
- };
1343
+ const geminiBadKey = isGemini && res.status === 400;
1344
+ if (res.status === 401 || res.status === 403 || geminiBadKey) return { outcome: "rejected", latencyMs, error: `HTTP ${res.status}` };
1345
+ if (res.status >= 500) return { outcome: "unreachable", latencyMs, error: `HTTP ${res.status}` };
1346
+ if (res.status === 404 || res.status === 405 || res.status === 410) {
1347
+ const { origin, pathname } = new URL(url);
1348
+ return { outcome: "unreachable", latencyMs, error: res.status === 404 ? "HTTP 404 (endpoint not found)" : `HTTP ${res.status} (not a models endpoint)`, remediation: `Check that ${origin}${pathname} is the provider's models endpoint${native ? "" : " (fix spec.model.baseUrl)"}` };
1349
+ }
1350
+ return { outcome: "ok", latencyMs };
639
1351
  } catch (err) {
640
1352
  const latencyMs = Date.now() - start;
641
1353
  const isTimeout = err instanceof Error && err.name === "AbortError";
642
1354
  return {
643
- reachable: false,
1355
+ outcome: "unreachable",
644
1356
  latencyMs,
645
1357
  error: isTimeout ? "Request timed out" : String(err)
646
1358
  };
@@ -648,6 +1360,20 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
648
1360
  clearTimeout(timeout);
649
1361
  }
650
1362
  }
1363
+ function modelProbeCheck(probe, fields, label, provider, keyRef) {
1364
+ const keyName = keyRef.startsWith("$") ? keyRef : "[api-key]";
1365
+ const base = { ...fields, latencyMs: probe.latencyMs };
1366
+ switch (probe.outcome) {
1367
+ case "ok":
1368
+ return { ...base, status: "pass" };
1369
+ case "not_verifiable":
1370
+ return { ...base, status: "not_verifiable", message: `${label} cannot be verified: ${probe.error}`, remediation: probe.remediation ?? `Use a provider the health check knows (${Object.keys(PROVIDER_ENDPOINTS).join(", ")}) or verify ${provider} out of band` };
1371
+ case "rejected":
1372
+ return { ...base, status: "fail", message: `${label} for ${provider} rejected the API key: ${probe.error}`, remediation: `Check that ${keyName} holds a valid ${provider} API key` };
1373
+ default:
1374
+ return { ...base, status: "fail", message: `${label} for ${provider} is unreachable: ${probe.error}`, remediation: probe.remediation ?? `Check that ${keyName} is set and the ${provider} API is reachable` };
1375
+ }
1376
+ }
651
1377
  function resolveApiKey(ref) {
652
1378
  if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
653
1379
  if (ref.startsWith("$")) return "";
@@ -657,16 +1383,14 @@ async function runModelChecks(model) {
657
1383
  const checks = [];
658
1384
  const apiKey = resolveApiKey(model.apiKey);
659
1385
  if (apiKey) {
660
- const { reachable, latencyMs, error } = await checkModelEndpoint(model.provider, apiKey);
661
- checks.push({
662
- id: `model:${model.provider}/${model.id}`,
663
- category: "model",
664
- status: reachable ? "pass" : "fail",
665
- severity: "error",
666
- latencyMs,
667
- message: reachable ? void 0 : `Model endpoint for ${model.provider} is unreachable: ${error}`,
668
- remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
669
- });
1386
+ const probe = await checkModelEndpoint(model.provider, apiKey, model.baseUrl);
1387
+ checks.push(modelProbeCheck(
1388
+ probe,
1389
+ { id: `model:${model.provider}/${model.id}`, category: "model", severity: "error" },
1390
+ "Model endpoint",
1391
+ model.provider,
1392
+ model.apiKey
1393
+ ));
670
1394
  } else {
671
1395
  checks.push({
672
1396
  id: `model:${model.provider}/${model.id}`,
@@ -676,32 +1400,6 @@ async function runModelChecks(model) {
676
1400
  message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
677
1401
  });
678
1402
  }
679
- if (model.fallback) {
680
- const fallbackKey = resolveApiKey(model.fallback.apiKey);
681
- if (fallbackKey) {
682
- const { reachable, latencyMs, error } = await checkModelEndpoint(
683
- model.fallback.provider,
684
- fallbackKey
685
- );
686
- checks.push({
687
- id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
688
- category: "model-fallback",
689
- status: reachable ? "pass" : "fail",
690
- severity: "warning",
691
- latencyMs,
692
- message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
693
- remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
694
- });
695
- } else {
696
- checks.push({
697
- id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
698
- category: "model-fallback",
699
- status: "skip",
700
- severity: "warning",
701
- message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
702
- });
703
- }
704
- }
705
1403
  return checks;
706
1404
  }
707
1405
 
@@ -713,7 +1411,7 @@ async function runMcpChecks(servers) {
713
1411
  const url = server.url;
714
1412
  if (url && !url.startsWith("$")) {
715
1413
  const check = await checkHttpMcpServer(server.name, url);
716
- checks.push(check);
1414
+ checks.push(server.transport === "sse" ? withLegacySseHint(check) : check);
717
1415
  } else {
718
1416
  checks.push({
719
1417
  id: `mcp:${server.name}`,
@@ -730,37 +1428,145 @@ async function runMcpChecks(servers) {
730
1428
  }
731
1429
  return checks;
732
1430
  }
1431
+ var MCP_PROTOCOL_VERSION = "2025-03-26";
1432
+ var HANDSHAKE_TIMEOUT_MS = 5e3;
1433
+ var INITIALIZE_REQUEST_ID = 1;
1434
+ var MAX_SERVER_TEXT = 200;
1435
+ var PROTOCOL_VERSION_FORMAT = /^\d{4}-\d{2}-\d{2}$/;
733
1436
  async function checkHttpMcpServer(name, url) {
734
1437
  const controller = new AbortController();
735
- const timeout = setTimeout(() => controller.abort(), 5e3);
1438
+ const timeout = setTimeout(() => controller.abort(), HANDSHAKE_TIMEOUT_MS);
736
1439
  const start = Date.now();
1440
+ const headers = {
1441
+ "content-type": "application/json",
1442
+ accept: "application/json, text/event-stream"
1443
+ };
737
1444
  try {
738
- const res = await fetch(url, { signal: controller.signal });
1445
+ const res = await fetch(url, {
1446
+ method: "POST",
1447
+ headers,
1448
+ signal: controller.signal,
1449
+ body: JSON.stringify({
1450
+ jsonrpc: "2.0",
1451
+ id: INITIALIZE_REQUEST_ID,
1452
+ method: "initialize",
1453
+ params: {
1454
+ protocolVersion: MCP_PROTOCOL_VERSION,
1455
+ capabilities: {},
1456
+ clientInfo: { name: "agentspec-health", version: "1.0.0" }
1457
+ }
1458
+ })
1459
+ });
1460
+ if (!res.ok) {
1461
+ await res.body?.cancel().catch(() => void 0);
1462
+ return failedHandshake(name, Date.now() - start, `returned HTTP ${res.status} to initialize`);
1463
+ }
1464
+ const parsed = await readInitializeResult(res);
739
1465
  const latencyMs = Date.now() - start;
1466
+ if ("error" in parsed) {
1467
+ return failedHandshake(name, latencyMs, parsed.error);
1468
+ }
1469
+ const result = parsed.result;
1470
+ const sessionId = res.headers.get("mcp-session-id");
1471
+ if (sessionId) headers["mcp-session-id"] = sessionId;
1472
+ headers["mcp-protocol-version"] = result.protocolVersion;
1473
+ await sendBestEffort(url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) }, controller.signal);
1474
+ if (sessionId) await sendBestEffort(url, { method: "DELETE", headers }, controller.signal);
1475
+ const serverName = clip(result.serverInfo?.name);
1476
+ const serverVersion = clip(result.serverInfo?.version);
1477
+ const info = serverName ? ` (${serverName}${serverVersion ? ` ${serverVersion}` : ""})` : "";
740
1478
  return {
741
1479
  id: `mcp:${name}`,
742
1480
  category: "mcp",
743
- status: res.ok || res.status === 404 ? "pass" : "fail",
1481
+ status: "pass",
744
1482
  severity: "warning",
745
1483
  latencyMs,
746
- message: res.status >= 500 ? `MCP server ${name} returned HTTP ${res.status}` : void 0
1484
+ message: `MCP server ${name}${info} initialized, protocolVersion ${clip(result.protocolVersion)}`
747
1485
  };
748
1486
  } catch (err) {
749
1487
  const latencyMs = Date.now() - start;
750
1488
  const isTimeout = err instanceof Error && err.name === "AbortError";
751
- return {
752
- id: `mcp:${name}`,
753
- category: "mcp",
754
- status: "fail",
755
- severity: "warning",
1489
+ return failedHandshake(
1490
+ name,
756
1491
  latencyMs,
757
- message: isTimeout ? `MCP server ${name} health check timed out` : `MCP server ${name} unreachable: ${String(err)}`,
758
- remediation: `Check that the MCP server ${name} is running and accessible`
759
- };
1492
+ isTimeout ? "health check timed out" : `unreachable: ${redactUrlCredentials(String(err))}`
1493
+ );
760
1494
  } finally {
761
1495
  clearTimeout(timeout);
762
1496
  }
763
1497
  }
1498
+ async function sendBestEffort(url, init, signal) {
1499
+ await fetch(url, { ...init, signal }).then((r) => r.body?.cancel()).catch(() => void 0);
1500
+ }
1501
+ function withLegacySseHint(check) {
1502
+ if (check.status !== "fail") return check;
1503
+ return {
1504
+ ...check,
1505
+ remediation: `${check.remediation ?? ""} Note: transport "sse" is probed with the streamable-HTTP handshake; legacy HTTP+SSE servers are not supported by this check.`.trim()
1506
+ };
1507
+ }
1508
+ function redactUrlCredentials(text) {
1509
+ return text.replace(/(\w+:\/\/)[^/\s]*@/g, "$1***@");
1510
+ }
1511
+ function failedHandshake(name, latencyMs, reason) {
1512
+ return {
1513
+ id: `mcp:${name}`,
1514
+ category: "mcp",
1515
+ status: "fail",
1516
+ severity: "warning",
1517
+ latencyMs,
1518
+ message: `MCP server ${name} ${reason}`,
1519
+ remediation: `Check that MCP server ${name} is running and that spec.mcp.servers[].url points at a streamable-HTTP MCP endpoint`
1520
+ };
1521
+ }
1522
+ async function readInitializeResult(res) {
1523
+ const isSse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
1524
+ const text = isSse ? await readSseUntilInitialize(res) : await res.text();
1525
+ const payloads = isSse ? splitSseEvents(text) : [text];
1526
+ for (const payload of payloads) {
1527
+ const outcome = parseInitializeFrame(payload);
1528
+ if (outcome) return outcome;
1529
+ }
1530
+ return { error: "did not return a valid JSON-RPC initialize result" };
1531
+ }
1532
+ function clip(value) {
1533
+ return typeof value === "string" ? value.slice(0, MAX_SERVER_TEXT) : "";
1534
+ }
1535
+ function parseInitializeFrame(payload) {
1536
+ try {
1537
+ const msg = JSON.parse(payload);
1538
+ if (msg.jsonrpc !== "2.0" || msg.id !== INITIALIZE_REQUEST_ID) return null;
1539
+ if (msg.result && typeof msg.result.protocolVersion === "string") {
1540
+ return PROTOCOL_VERSION_FORMAT.test(msg.result.protocolVersion) ? { result: msg.result } : { error: "returned a malformed protocolVersion in its initialize result" };
1541
+ }
1542
+ if (msg.error) return { error: `rejected initialize: ${clip(msg.error.message) || "JSON-RPC error"}` };
1543
+ } catch {
1544
+ }
1545
+ return null;
1546
+ }
1547
+ function splitSseEvents(text) {
1548
+ return text.split(/\r?\n\r?\n/).map(
1549
+ (event) => event.split(/\r?\n/).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).join("\n")
1550
+ );
1551
+ }
1552
+ async function readSseUntilInitialize(res) {
1553
+ if (!res.body) return res.text();
1554
+ const reader = res.body.getReader();
1555
+ const decoder = new TextDecoder();
1556
+ let text = "";
1557
+ try {
1558
+ for (; ; ) {
1559
+ const { done, value } = await reader.read();
1560
+ if (done) break;
1561
+ text += decoder.decode(value, { stream: true });
1562
+ const completeEvents = text.split(/\r?\n\r?\n/).slice(0, -1);
1563
+ if (completeEvents.flatMap((e) => splitSseEvents(e)).some((p) => parseInitializeFrame(p))) break;
1564
+ }
1565
+ } finally {
1566
+ await reader.cancel().catch(() => void 0);
1567
+ }
1568
+ return text;
1569
+ }
764
1570
  async function checkStdioMcpServer(server) {
765
1571
  const command = server.command;
766
1572
  if (!command) {
@@ -1278,7 +2084,7 @@ function computeHealthStatus(checks) {
1278
2084
  const failed = checks.filter((c) => c.status === "fail");
1279
2085
  const warnings = checks.filter((c) => c.status === "warn");
1280
2086
  const passed = checks.filter((c) => c.status === "pass");
1281
- const skipped = checks.filter((c) => c.status === "skip");
2087
+ const skipped = checks.filter((c) => c.status === "skip" || c.status === "not_verifiable");
1282
2088
  const hasErrors = failed.some((c) => c.severity === "error");
1283
2089
  const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
1284
2090
  const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
@@ -1295,8 +2101,10 @@ function computeHealthStatus(checks) {
1295
2101
  async function runHealthCheck(manifest, opts = {}) {
1296
2102
  const checks = [];
1297
2103
  const rawManifest = opts.rawManifest ?? manifest;
1298
- checks.push(...runEnvChecks(manifest, rawManifest));
1299
- checks.push(...await runSecretChecks());
2104
+ if (opts.checkEnv !== false) {
2105
+ checks.push(...runEnvChecks(manifest, rawManifest));
2106
+ checks.push(...await runSecretChecks());
2107
+ }
1300
2108
  if (opts.baseDir) {
1301
2109
  checks.push(...runFileChecks(rawManifest, opts.baseDir));
1302
2110
  }
@@ -1386,7 +2194,381 @@ async function checkA2aEndpoint(name, url) {
1386
2194
  }
1387
2195
  }
1388
2196
 
2197
+ // src/grader/index.ts
2198
+ var ADMITTED = {
2199
+ standard: ["independent", "sdk-observed"],
2200
+ strict: ["independent"]
2201
+ };
2202
+ var SOURCE_CLASS = {
2203
+ independent: "independent",
2204
+ "sdk-observed": "evidence-point",
2205
+ offline: "offline",
2206
+ declared: "declared"
2207
+ };
2208
+ var pct = (part, whole) => whole === 0 ? 0 : part / whole * 100;
2209
+ function isCalendarDate(value) {
2210
+ const [y, m, d] = value.slice(0, 10).split("-").map(Number);
2211
+ const date = /* @__PURE__ */ new Date(0);
2212
+ date.setUTCFullYear(y, m - 1, d);
2213
+ return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
2214
+ }
2215
+ function parseTimestamp(value, field) {
2216
+ const ms = Date.parse(value);
2217
+ const valid = TIMESTAMP_PATTERN.test(value) && !Number.isNaN(ms) && isCalendarDate(value) && !value.startsWith("0000") && !value.includes("T24:");
2218
+ if (!valid) {
2219
+ throw new Error(`grade: ${field} "${value}" is not a valid RFC 3339 timestamp. Fix the producer to emit a real date and time.`);
2220
+ }
2221
+ return ms;
2222
+ }
2223
+ function isExpired(r, nowMs) {
2224
+ return parseTimestamp(r.observedAt, `observedAt of the record for check "${r.checkId}"`) + r.ttlSeconds * 1e3 < nowMs;
2225
+ }
2226
+ function matchesRegistry(r) {
2227
+ const cls = SOURCE_CLASS[r.source];
2228
+ if (!r.ruleId || !cls || !Object.hasOwn(RULE_EVIDENCE, r.ruleId)) return true;
2229
+ const listed = RULE_EVIDENCE[r.ruleId].sources.filter((s) => s.class === cls);
2230
+ return listed.length > 0 && (r.collector == null || listed.some((s) => s.collector === r.collector));
2231
+ }
2232
+ function unverifiedReason(admitted, stale, notAdmitted, mismatched) {
2233
+ const own = admitted.find((r) => r.status === "not_verifiable");
2234
+ if (own?.reason) return own.reason;
2235
+ if (stale.length > 0) return { code: "stale" };
2236
+ if (notAdmitted.length > 0) return { code: "source_not_admitted" };
2237
+ if (mismatched.length > 0) return { code: "source_mismatch" };
2238
+ return { code: "no_evidence" };
2239
+ }
2240
+ function judgeRule(allRecords, policy, nowMs, unwiredFails) {
2241
+ if (unwiredFails) allRecords = [...allRecords.filter((r) => r.source !== "sdk-observed"), ...unwiredFails];
2242
+ const admittedSources = ADMITTED[policy];
2243
+ const registryMatched = [];
2244
+ const mismatched = [];
2245
+ for (const r of allRecords) (matchesRegistry(r) ? registryMatched : mismatched).push(r);
2246
+ const admittedRecords = registryMatched.filter((r) => admittedSources.includes(r.source));
2247
+ const notAdmitted = registryMatched.filter((r) => !admittedSources.includes(r.source));
2248
+ const stale = admittedRecords.filter((r) => isExpired(r, nowMs));
2249
+ const fresh = admittedRecords.filter((r) => !isExpired(r, nowMs));
2250
+ const decisive = fresh.filter((r) => r.status !== "not_verifiable");
2251
+ if (decisive.length === 0) {
2252
+ const unverified = unwiredFails ? { code: "not_wired" } : unverifiedReason(fresh, stale, notAdmitted, mismatched);
2253
+ return { covered: false, pass: false, independent: false, unverified };
2254
+ }
2255
+ return {
2256
+ covered: true,
2257
+ pass: decisive.every((r) => r.status === "pass"),
2258
+ independent: decisive.every((r) => r.source === "independent")
2259
+ };
2260
+ }
2261
+ function unwiredRules(points) {
2262
+ const notWired = /* @__PURE__ */ new Set();
2263
+ const wiredFails = /* @__PURE__ */ new Map();
2264
+ for (const p of points) {
2265
+ const ruleId = CONTRACT.evidencePoints[p.point]?.[p.target];
2266
+ if (!ruleId) continue;
2267
+ if (p.state === "not_wired") notWired.add(ruleId);
2268
+ else {
2269
+ const fails = p.records.filter((r) => r.ruleId === ruleId && r.status === "fail");
2270
+ wiredFails.set(ruleId, [...wiredFails.get(ruleId) ?? [], ...fails]);
2271
+ }
2272
+ }
2273
+ return new Map([...notWired].map((id) => [id, wiredFails.get(id) ?? []]));
2274
+ }
2275
+ function judgeDerived(parents) {
2276
+ const open = parents.find((v) => !v.covered);
2277
+ const failed = parents.some((v) => v.covered && !v.pass);
2278
+ if (open && !failed) return open;
2279
+ const covered = parents.filter((v) => v.covered);
2280
+ return { covered: true, pass: !failed, independent: covered.every((v) => v.independent) };
2281
+ }
2282
+ function judgeRegistered(ruleId, byRule, policy, nowMs, unwired) {
2283
+ const derivedFrom = Object.hasOwn(RULE_EVIDENCE, ruleId) ? RULE_EVIDENCE[ruleId].derivedFrom : void 0;
2284
+ if (!derivedFrom?.length) return judgeRule(byRule.get(ruleId) ?? [], policy, nowMs, unwired.get(ruleId));
2285
+ return judgeDerived(derivedFrom.map((parent) => judgeRegistered(parent, byRule, policy, nowMs, unwired)));
2286
+ }
2287
+ function groupByRule(records) {
2288
+ const byRule = /* @__PURE__ */ new Map();
2289
+ for (const r of records) {
2290
+ if (!r.ruleId) continue;
2291
+ const bucket2 = byRule.get(r.ruleId);
2292
+ if (bucket2) bucket2.push(r);
2293
+ else byRule.set(r.ruleId, [r]);
2294
+ }
2295
+ return byRule;
2296
+ }
2297
+ function scoreToGrade(score) {
2298
+ const { A, B, C, D } = CONTRACT.gradeBands;
2299
+ if (score >= A) return "A";
2300
+ if (score >= B) return "B";
2301
+ if (score >= C) return "C";
2302
+ if (score >= D) return "D";
2303
+ return "F";
2304
+ }
2305
+ function grade(records, applicableRules, policy, now, points = []) {
2306
+ const nowMs = parseTimestamp(now, "now");
2307
+ const byRule = groupByRule(records);
2308
+ const unwired = unwiredRules(points);
2309
+ let totalWeight = 0;
2310
+ let coveredWeight = 0;
2311
+ let passedWeight = 0;
2312
+ let independentWeight = 0;
2313
+ const unverified = [];
2314
+ for (const rule of applicableRules) {
2315
+ const weight = CONTRACT.severityWeights[rule.severity];
2316
+ const verdict = judgeRegistered(rule.id, byRule, policy, nowMs, unwired);
2317
+ totalWeight += weight;
2318
+ if (!verdict.covered) {
2319
+ unverified.push({ ruleId: rule.id, severity: rule.severity, reason: verdict.unverified });
2320
+ continue;
2321
+ }
2322
+ coveredWeight += weight;
2323
+ if (verdict.pass) passedWeight += weight;
2324
+ if (verdict.independent) independentWeight += weight;
2325
+ }
2326
+ const verified = Math.round(pct(passedWeight, coveredWeight));
2327
+ const coverage = Math.round(pct(coveredWeight, totalWeight));
2328
+ return {
2329
+ verified,
2330
+ coverage,
2331
+ independentShare: Math.round(pct(independentWeight, coveredWeight)),
2332
+ grade: coverage >= CONTRACT.coverageThreshold ? scoreToGrade(verified) : "Not graded",
2333
+ unverified
2334
+ };
2335
+ }
2336
+ function formatVerified(result) {
2337
+ if (result.grade === "Not graded") return `Not graded (coverage ${result.coverage}%)`;
2338
+ return `${result.grade} - ${result.verified}% verified (${result.independentShare}% independent) - coverage ${result.coverage}%`;
2339
+ }
2340
+ var NATIVE_PROVIDERS = {
2341
+ openai: "openAI",
2342
+ anthropic: "anthropic",
2343
+ google: "gemini",
2344
+ gemini: "gemini",
2345
+ azure: "azure"
2346
+ };
2347
+ var OPENAI_COMPATIBLE_BASE_URLS = {
2348
+ groq: "https://api.groq.com/openai/v1",
2349
+ mistral: "https://api.mistral.ai/v1"
2350
+ };
2351
+ var CATALOG_PROVIDER_IDS = {
2352
+ openAI: "openai",
2353
+ anthropic: "anthropic",
2354
+ gemini: "gcp.gemini",
2355
+ azure: "azure"
2356
+ };
2357
+ var GATEWAY_NAME = "llm";
2358
+ var ONE_MONTH = "30d";
2359
+ function envRefName(value) {
2360
+ return /^\$env:(.+)$/.exec(value)?.[1];
2361
+ }
2362
+ function secretEnvName(name) {
2363
+ return `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
2364
+ }
2365
+ function keyEnvName(apiKey) {
2366
+ const env = envRefName(apiKey);
2367
+ if (env) return env;
2368
+ const secret = /^\$secret:(.+)$/.exec(apiKey);
2369
+ if (secret) return secretEnvName(secret[1]);
2370
+ throw new Error(
2371
+ 'spec.model.apiKey must be a "$env:VAR" or "$secret:name" reference; a literal key would end up in the gateway config. Replace it with "$env:<VAR>".'
2372
+ );
2373
+ }
2374
+ function baseUrlValue(baseUrl) {
2375
+ const env = envRefName(baseUrl);
2376
+ return env ? `$${env}` : baseUrl;
2377
+ }
2378
+ function baseUrlEnvNames({ baseUrl }) {
2379
+ const name = baseUrl && envRefName(baseUrl);
2380
+ return name ? [name] : [];
2381
+ }
2382
+ function customProvider(provider, id, baseUrl) {
2383
+ const anthropic = provider === "anthropic";
2384
+ return {
2385
+ provider: { custom: { providerOverride: provider, formats: [{ type: anthropic ? "messages" : "completions" }] } },
2386
+ params: { model: id, baseUrl, ...anthropic ? { pathPrefix: "/v1" } : {} }
2387
+ };
2388
+ }
2389
+ function gatewayEnvNames(manifest) {
2390
+ const { model } = manifest.spec;
2391
+ const models = [model, ...model.fallback ? [model.fallback] : []];
2392
+ const names = models.flatMap((m) => [
2393
+ keyEnvName(m.apiKey),
2394
+ ...baseUrlEnvNames(m),
2395
+ ...NATIVE_PROVIDERS[m.provider] === "azure" && !m.baseUrl ? [CONTRACT.env.azureResourceName] : []
2396
+ ]);
2397
+ return [...new Set(names)].sort();
2398
+ }
2399
+ function providerFields({ provider, id, baseUrl }) {
2400
+ if (baseUrl) return customProvider(provider, id, baseUrlValue(baseUrl));
2401
+ const native = NATIVE_PROVIDERS[provider];
2402
+ if (native === "azure") {
2403
+ return {
2404
+ provider: native,
2405
+ params: { model: id, azureResourceName: `$${CONTRACT.env.azureResourceName}`, azureResourceType: "openAI" }
2406
+ };
2407
+ }
2408
+ if (native) return { provider: native, params: { model: id } };
2409
+ const knownBaseUrl = OPENAI_COMPATIBLE_BASE_URLS[provider];
2410
+ if (knownBaseUrl) return customProvider(provider, id, knownBaseUrl);
2411
+ const supported = [...Object.keys(NATIVE_PROVIDERS), ...Object.keys(OPENAI_COMPATIBLE_BASE_URLS)].sort();
2412
+ throw new Error(`spec.model.provider "${provider}" has no gateway mapping. Supported: ${supported.join(", ")}.`);
2413
+ }
2414
+ function renderModel(model) {
2415
+ const { provider, params } = providerFields(model);
2416
+ return {
2417
+ name: `${model.provider}/${model.id}`,
2418
+ visibility: "internal",
2419
+ provider,
2420
+ params: { ...params, apiKey: `$${keyEnvName(model.apiKey)}` }
2421
+ };
2422
+ }
2423
+ function renderVirtualModel(models) {
2424
+ return {
2425
+ name: models[0].id,
2426
+ routing: {
2427
+ failover: {
2428
+ targets: models.map((m, priority) => ({ model: `${m.provider}/${m.id}`, priority }))
2429
+ }
2430
+ }
2431
+ };
2432
+ }
2433
+ function renderRetryPolicy(model) {
2434
+ const attempts = model.fallback?.maxRetries;
2435
+ if (!attempts) return {};
2436
+ return {
2437
+ policies: [{
2438
+ name: { name: "llm-retry", namespace: "default" },
2439
+ target: { gateway: { gatewayName: "default", gatewayNamespace: "default", listenerName: GATEWAY_NAME } },
2440
+ policy: { retry: { attempts, codes: [500, 502, 503, 504] } }
2441
+ }]
2442
+ };
2443
+ }
2444
+ function renderBudgets(spec) {
2445
+ const amount = spec.model.costControls?.maxMonthlyUSD;
2446
+ if (amount === void 0) return {};
2447
+ return {
2448
+ budgets: [{
2449
+ name: "monthly",
2450
+ limit: { unit: "USD", amount },
2451
+ window: { rolling: ONE_MONTH },
2452
+ onBudgetExceeded: "Block"
2453
+ }]
2454
+ };
2455
+ }
2456
+ function renderApiKeyPolicy(manifest) {
2457
+ return {
2458
+ mode: "strict",
2459
+ keys: [{
2460
+ key: `$${CONTRACT.env.gatewayKey}`,
2461
+ metadata: { name: manifest.metadata.name },
2462
+ allowedModels: [manifest.spec.model.id],
2463
+ ...renderBudgets(manifest.spec)
2464
+ }]
2465
+ };
2466
+ }
2467
+ function bucket(type, size, fillInterval) {
2468
+ return { type, maxTokens: size, tokensPerFill: size, fillInterval };
2469
+ }
2470
+ function renderRateLimits(spec) {
2471
+ const { requestsPerMinute, requestsPerHour, tokensPerMinute } = spec.api?.rateLimit ?? {};
2472
+ const buckets = [
2473
+ ...requestsPerMinute === void 0 ? [] : [bucket("requests", requestsPerMinute, "60s")],
2474
+ ...requestsPerHour === void 0 ? [] : [bucket("requests", requestsPerHour, "1h")],
2475
+ ...tokensPerMinute === void 0 ? [] : [bucket("tokens", tokensPerMinute, "60s")]
2476
+ ];
2477
+ return buckets.length === 0 ? {} : { localRateLimit: buckets };
2478
+ }
2479
+ function renderRuntimeConfig(spec) {
2480
+ return {
2481
+ ...spec.model.costControls?.maxMonthlyUSD !== void 0 && {
2482
+ database: { url: CONTRACT.paths.gatewayDatabase }
2483
+ },
2484
+ // agentgateway samples no request by default and keeps prompt and completion out of the
2485
+ // span; the sidecar needs every LLM call with both (ADR-0030), and redacts before buffering.
2486
+ tracing: {
2487
+ otlpEndpoint: `http://${CONTRACT.podPatch.loopback}:${CONTRACT.ports.controlPlane}`,
2488
+ otlpProtocol: "http",
2489
+ path: CONTRACT.sidecarPaths.traces,
2490
+ randomSampling: "true",
2491
+ fields: { add: { "gen_ai.prompt": "llm.prompt", "gen_ai.completion": "llm.completion" } }
2492
+ }
2493
+ };
2494
+ }
2495
+ function catalogProvider(model) {
2496
+ const { provider } = providerFields(model);
2497
+ return typeof provider === "string" ? CATALOG_PROVIDER_IDS[provider] : model.provider;
2498
+ }
2499
+ function catalogRates(pricing) {
2500
+ return Object.fromEntries(Object.entries(pricing).filter(([, v]) => v !== void 0).map(([k, v]) => [k, String(v)]));
2501
+ }
2502
+ function renderModelCatalog(models) {
2503
+ const providers = {};
2504
+ for (const model of models) {
2505
+ if (!model.pricing) continue;
2506
+ const key = catalogProvider(model);
2507
+ providers[key] ??= { models: {} };
2508
+ providers[key].models[model.id] = { rates: catalogRates(model.pricing) };
2509
+ }
2510
+ return Object.keys(providers).length === 0 ? {} : { modelCatalog: [{ inline: { providers } }] };
2511
+ }
2512
+ function renderGatewayConfig(manifest) {
2513
+ const { spec } = manifest;
2514
+ const models = [spec.model, ...spec.model.fallback ? [spec.model.fallback] : []];
2515
+ const config = {
2516
+ config: { ...renderRuntimeConfig(spec), ...renderModelCatalog(models) },
2517
+ gateways: { [GATEWAY_NAME]: { port: CONTRACT.ports.gateway, bindAddress: CONTRACT.podPatch.loopback } },
2518
+ llm: {
2519
+ gateways: GATEWAY_NAME,
2520
+ models: models.map(renderModel),
2521
+ virtualModels: [renderVirtualModel(models)],
2522
+ policies: {
2523
+ apiKey: renderApiKeyPolicy(manifest),
2524
+ ...renderRateLimits(spec)
2525
+ }
2526
+ },
2527
+ ...renderRetryPolicy(spec.model)
2528
+ };
2529
+ return dump(config, { sortKeys: true, lineWidth: -1, noRefs: true });
2530
+ }
2531
+ function describeManifestError(err) {
2532
+ if (!(err instanceof Error)) return String(err);
2533
+ if (!err.message.startsWith("[")) {
2534
+ return err.message.split("\n")[0];
2535
+ }
2536
+ try {
2537
+ const issues = JSON.parse(err.message);
2538
+ return issues.map((i) => `${i.path.join(".") || "root"}: ${i.message}`).join("\n");
2539
+ } catch {
2540
+ return err.message.split("\n")[0];
2541
+ }
2542
+ }
2543
+
1389
2544
  // src/audit/rules/model.rules.ts
2545
+ var isLatestModelId = (id) => id === "latest" || id.endsWith(":latest");
2546
+ function fallbackRetryResult(model) {
2547
+ if (!model.fallback) return { pass: true };
2548
+ const { maxRetries } = model.fallback;
2549
+ const { policies } = renderRetryPolicy(model);
2550
+ const attempts = policies ? policies[0].policy.retry.attempts : 0;
2551
+ const fail = (message, recommendation) => ({
2552
+ pass: false,
2553
+ message,
2554
+ path: "/spec/model/fallback/maxRetries",
2555
+ recommendation,
2556
+ references: []
2557
+ });
2558
+ if (maxRetries === void 0) {
2559
+ return fail(
2560
+ "Fallback model has no maxRetries configured: the gateway renders no retry, so the request that triggers failover fails",
2561
+ "Add maxRetries: 2 to spec.model.fallback"
2562
+ );
2563
+ }
2564
+ if (attempts !== maxRetries) {
2565
+ return fail(
2566
+ `The rendered gateway config sets ${attempts} retry attempts, but spec.model.fallback declares maxRetries ${maxRetries}`,
2567
+ "Redeploy the agent so its gateway config is rendered again from agent.yaml and retries maxRetries times"
2568
+ );
2569
+ }
2570
+ return { pass: true };
2571
+ }
1390
2572
  var modelRules = [
1391
2573
  {
1392
2574
  id: "MODEL-01",
@@ -1394,9 +2576,7 @@ var modelRules = [
1394
2576
  title: "Model fallback declared",
1395
2577
  description: "A fallback model should be configured to handle rate limits and errors",
1396
2578
  severity: "high",
1397
- evidenceLevel: "external",
1398
- proofTool: "LiteLLM chaos test",
1399
- proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
2579
+ awaitsEvidence: { code: "no_failover_observed", phase: 2 },
1400
2580
  check(manifest) {
1401
2581
  const hasFallback = !!manifest.spec.model.fallback;
1402
2582
  return {
@@ -1414,17 +2594,14 @@ var modelRules = [
1414
2594
  title: "Model version pinned",
1415
2595
  description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
1416
2596
  severity: "medium",
1417
- evidenceLevel: "probed",
1418
- proofTool: "agentspec health",
1419
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1420
2597
  check(manifest) {
1421
2598
  const id = manifest.spec.model.id;
1422
- const isLatest = id === "latest" || id.endsWith(":latest");
2599
+ const isLatest = isLatestModelId(id);
1423
2600
  return {
1424
2601
  pass: !isLatest,
1425
2602
  message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
1426
2603
  path: "/spec/model/id",
1427
- recommendation: 'Pin a specific model version, e.g. "llama-3.3-70b-versatile" or "gpt-4-0125-preview"',
2604
+ recommendation: 'Pin a specific model version, e.g. "openai/gpt-oss-20b" or "gpt-4-0125-preview"',
1428
2605
  references: []
1429
2606
  };
1430
2607
  }
@@ -1435,9 +2612,7 @@ var modelRules = [
1435
2612
  title: "Cost controls declared",
1436
2613
  description: "Monthly cost limits prevent unexpected API bills",
1437
2614
  severity: "medium",
1438
- evidenceLevel: "external",
1439
- proofTool: "LiteLLM Spend Tracking",
1440
- proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
2615
+ awaitsEvidence: { code: "no_budget_evidence", phase: 2 },
1441
2616
  check(manifest) {
1442
2617
  const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
1443
2618
  return {
@@ -1453,19 +2628,61 @@ var modelRules = [
1453
2628
  id: "MODEL-04",
1454
2629
  pack: "model-resilience",
1455
2630
  title: "Fallback retry strategy configured",
1456
- description: "Fallback should have maxRetries to avoid infinite loops",
2631
+ description: "Fallback should have maxRetries, and the rendered gateway config must retry exactly that many times",
2632
+ severity: "low",
2633
+ check(manifest) {
2634
+ return fallbackRetryResult(manifest.spec.model);
2635
+ }
2636
+ },
2637
+ {
2638
+ id: "MODEL-05",
2639
+ pack: "model-resilience",
2640
+ title: "Budgeted model is priced",
2641
+ description: "A monthly USD budget is only enforced for models the gateway can price",
2642
+ severity: "high",
2643
+ // Only the gateway knows its prices: MODEL-05 is graded from the cost on its spans (#494), never guessed from the model id (#495).
2644
+ check() {
2645
+ return { pass: true };
2646
+ }
2647
+ }
2648
+ ];
2649
+
2650
+ // src/audit/rules/metadata.rules.ts
2651
+ var MAX_PRODUCTION_TEMPERATURE = 1.5;
2652
+ var metadataRules = [
2653
+ {
2654
+ id: "META-01",
2655
+ pack: "metadata-quality",
2656
+ title: "Agent description declared",
2657
+ description: "Agents should have a non-empty description for fleet discovery and documentation",
1457
2658
  severity: "low",
1458
- evidenceLevel: "external",
1459
- proofTool: "pytest-mockllm",
1460
- proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
1461
2659
  check(manifest) {
1462
- if (!manifest.spec.model.fallback) return { pass: true };
1463
- const hasRetries = manifest.spec.model.fallback.maxRetries !== void 0;
2660
+ const desc = manifest.metadata.description;
2661
+ const pass = typeof desc === "string" && desc.trim().length > 0;
2662
+ return {
2663
+ pass,
2664
+ message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
2665
+ path: "/metadata/description",
2666
+ recommendation: "Add a concise metadata.description explaining what the agent does",
2667
+ references: []
2668
+ };
2669
+ }
2670
+ },
2671
+ {
2672
+ id: "META-02",
2673
+ pack: "metadata-quality",
2674
+ title: "Model temperature is production-safe",
2675
+ description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
2676
+ severity: "medium",
2677
+ check(manifest) {
2678
+ const temperature = manifest.spec.model.parameters?.temperature;
2679
+ if (temperature === void 0) return { pass: true };
2680
+ const pass = temperature <= MAX_PRODUCTION_TEMPERATURE;
1464
2681
  return {
1465
- pass: hasRetries,
1466
- message: hasRetries ? void 0 : "Fallback model has no maxRetries configured",
1467
- path: "/spec/model/fallback/maxRetries",
1468
- recommendation: "Add maxRetries: 2 to spec.model.fallback",
2682
+ pass,
2683
+ message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above ${MAX_PRODUCTION_TEMPERATURE} produce unreliable outputs in production.`,
2684
+ path: "/spec/model/parameters/temperature",
2685
+ recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
1469
2686
  references: []
1470
2687
  };
1471
2688
  }
@@ -1480,9 +2697,6 @@ var securityRules = [
1480
2697
  title: "Prompt injection guard: input guardrail required",
1481
2698
  description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
1482
2699
  severity: "high",
1483
- evidenceLevel: "behavioral",
1484
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1485
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1486
2700
  check(manifest) {
1487
2701
  const hasGuardrail = manifest.spec.guardrails?.input?.some(
1488
2702
  (g) => g.type === "prompt-injection" || g.type === "topic-filter"
@@ -1504,9 +2718,6 @@ var securityRules = [
1504
2718
  title: "Insecure output handling: output guardrail required",
1505
2719
  description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
1506
2720
  severity: "high",
1507
- evidenceLevel: "behavioral",
1508
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1509
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1510
2721
  check(manifest) {
1511
2722
  const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
1512
2723
  return {
@@ -1526,9 +2737,6 @@ var securityRules = [
1526
2737
  title: "Training/prompt data poisoning: prompt files referenced",
1527
2738
  description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
1528
2739
  severity: "medium",
1529
- evidenceLevel: "probed",
1530
- proofTool: "agentspec health",
1531
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1532
2740
  check(manifest) {
1533
2741
  const system = manifest.spec.prompts.system;
1534
2742
  const usesFile = system.startsWith("$file:");
@@ -1546,21 +2754,16 @@ var securityRules = [
1546
2754
  {
1547
2755
  id: "SEC-LLM-04",
1548
2756
  pack: "owasp-llm-top10",
1549
- title: "Model DoS: rate limiting + cost controls declared",
2757
+ title: "Model DoS: rate limiting configured",
1550
2758
  description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
1551
2759
  severity: "medium",
1552
- evidenceLevel: "external",
1553
- proofTool: "k6 load test",
1554
- proofToolUrl: "https://k6.io",
1555
2760
  check(manifest) {
1556
- const hasRateLimit = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
1557
- const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
1558
- const pass = hasRateLimit || hasCostControls;
2761
+ const pass = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
1559
2762
  return {
1560
2763
  pass,
1561
- message: pass ? void 0 : "No rate limiting or cost controls declared.",
2764
+ message: pass ? void 0 : "No rate limiting declared.",
1562
2765
  path: "/spec/api/rateLimit",
1563
- recommendation: "Add spec.api.rateLimit.requestsPerMinute and spec.model.costControls.maxMonthlyUSD",
2766
+ recommendation: "Add spec.api.rateLimit.requestsPerMinute",
1564
2767
  references: [
1565
2768
  "https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
1566
2769
  ]
@@ -1573,9 +2776,6 @@ var securityRules = [
1573
2776
  title: "Supply chain: model provider and version pinned",
1574
2777
  description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
1575
2778
  severity: "medium",
1576
- evidenceLevel: "probed",
1577
- proofTool: "agentspec health",
1578
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1579
2779
  check(manifest) {
1580
2780
  const hasProvider = !!manifest.spec.model.provider;
1581
2781
  const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
@@ -1597,9 +2797,6 @@ var securityRules = [
1597
2797
  title: "Sensitive data disclosure: PII scrub in memory hygiene",
1598
2798
  description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
1599
2799
  severity: "critical",
1600
- evidenceLevel: "external",
1601
- proofTool: "Microsoft Presidio",
1602
- proofToolUrl: "https://microsoft.github.io/presidio/",
1603
2800
  check(manifest) {
1604
2801
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1605
2802
  const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
@@ -1620,9 +2817,6 @@ var securityRules = [
1620
2817
  title: "Insecure plugin design: tool annotations declared",
1621
2818
  description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
1622
2819
  severity: "medium",
1623
- evidenceLevel: "external",
1624
- proofTool: "Promptfoo",
1625
- proofToolUrl: "https://promptfoo.dev",
1626
2820
  check(manifest) {
1627
2821
  if (!manifest.spec.tools?.length) return { pass: true };
1628
2822
  const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
@@ -1643,9 +2837,7 @@ var securityRules = [
1643
2837
  title: "Excessive agency: destructiveHint declared on tools",
1644
2838
  description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
1645
2839
  severity: "high",
1646
- evidenceLevel: "external",
1647
- proofTool: "Promptfoo",
1648
- proofToolUrl: "https://promptfoo.dev",
2840
+ awaitsEvidence: { code: "no_approval_token_evidence", phase: 5 },
1649
2841
  check(manifest) {
1650
2842
  if (!manifest.spec.tools?.length) return { pass: true };
1651
2843
  const allHaveDestructiveHint = manifest.spec.tools.every(
@@ -1665,21 +2857,18 @@ var securityRules = [
1665
2857
  {
1666
2858
  id: "SEC-LLM-09",
1667
2859
  pack: "owasp-llm-top10",
1668
- title: "Overreliance: evaluation framework + CI gate configured",
2860
+ title: "Overreliance: evaluation + CI gate configured",
1669
2861
  description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
1670
2862
  severity: "medium",
1671
- evidenceLevel: "probed",
1672
- proofTool: "agentspec health",
1673
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1674
2863
  check(manifest) {
1675
- const hasEval = !!manifest.spec.evaluation?.framework;
2864
+ const hasEval = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0 || (manifest.spec.evaluation?.live?.metrics?.length ?? 0) > 0;
1676
2865
  const hasCiGate = manifest.spec.evaluation?.ciGate === true;
1677
2866
  const pass = hasEval && hasCiGate;
1678
2867
  return {
1679
2868
  pass,
1680
- message: pass ? void 0 : !hasEval ? "No evaluation framework configured." : "Evaluation CI gate is not enabled.",
2869
+ message: pass ? void 0 : !hasEval ? "No evaluation datasets or live evaluation configured." : "Evaluation CI gate is not enabled.",
1681
2870
  path: "/spec/evaluation",
1682
- recommendation: "Configure spec.evaluation.framework and set spec.evaluation.ciGate: true",
2871
+ recommendation: "Configure spec.evaluation.datasets (or spec.evaluation.live) and set spec.evaluation.ciGate: true",
1683
2872
  references: [
1684
2873
  "https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
1685
2874
  ]
@@ -1692,9 +2881,6 @@ var securityRules = [
1692
2881
  title: "Model theft: API keys use $secret, not $env",
1693
2882
  description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
1694
2883
  severity: "high",
1695
- evidenceLevel: "probed",
1696
- proofTool: "agentspec health",
1697
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1698
2884
  check(manifest) {
1699
2885
  const apiKey = manifest.spec.model.apiKey;
1700
2886
  const usesSecret = apiKey.startsWith("$secret:");
@@ -1710,6 +2896,18 @@ var securityRules = [
1710
2896
  ]
1711
2897
  };
1712
2898
  }
2899
+ },
2900
+ {
2901
+ id: "TOOL-01",
2902
+ pack: "owasp-llm-top10",
2903
+ title: "Only declared tools are called",
2904
+ description: "The agent must call only tools declared in spec.tools; undeclared calls are excessive agency",
2905
+ severity: "high",
2906
+ awaitsEvidence: { code: "no_calls_observed", phase: 2 },
2907
+ // The manifest alone cannot show an undeclared call: passes declaratively, verified only from observed spans (#152, D17).
2908
+ check() {
2909
+ return { pass: true };
2910
+ }
1713
2911
  }
1714
2912
  ];
1715
2913
 
@@ -1721,9 +2919,6 @@ var memoryRules = [
1721
2919
  title: "PII scrub fields declared for long-term memory",
1722
2920
  description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
1723
2921
  severity: "critical",
1724
- evidenceLevel: "external",
1725
- proofTool: "Microsoft Presidio",
1726
- proofToolUrl: "https://microsoft.github.io/presidio/",
1727
2922
  check(manifest) {
1728
2923
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1729
2924
  const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
@@ -1742,9 +2937,6 @@ var memoryRules = [
1742
2937
  title: "Retention policy / TTL set for all memory backends",
1743
2938
  description: "Memory without TTL grows unbounded and increases PII exposure risk",
1744
2939
  severity: "high",
1745
- evidenceLevel: "probed",
1746
- proofTool: "agentspec health",
1747
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1748
2940
  check(manifest) {
1749
2941
  const mem = manifest.spec.memory;
1750
2942
  if (!mem) return { pass: true };
@@ -1769,9 +2961,6 @@ var memoryRules = [
1769
2961
  title: "Audit log enabled for long-term memory",
1770
2962
  description: "Audit logging enables compliance investigations and debugging",
1771
2963
  severity: "medium",
1772
- evidenceLevel: "probed",
1773
- proofTool: "agentspec health",
1774
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1775
2964
  check(manifest) {
1776
2965
  if (!manifest.spec.memory?.longTerm) return { pass: true };
1777
2966
  const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
@@ -1790,9 +2979,6 @@ var memoryRules = [
1790
2979
  title: "Vector store namespace isolated",
1791
2980
  description: "Shared vector store namespaces risk data leakage between agents",
1792
2981
  severity: "medium",
1793
- evidenceLevel: "probed",
1794
- proofTool: "agentspec health",
1795
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1796
2982
  check(manifest) {
1797
2983
  if (!manifest.spec.memory?.vector) return { pass: true };
1798
2984
  const hasNamespace = !!manifest.spec.memory.vector.namespace;
@@ -1811,9 +2997,6 @@ var memoryRules = [
1811
2997
  title: "Short-term memory max tokens bounded",
1812
2998
  description: "Unbounded short-term memory can cause token budget overruns",
1813
2999
  severity: "low",
1814
- evidenceLevel: "probed",
1815
- proofTool: "agentspec health",
1816
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1817
3000
  check(manifest) {
1818
3001
  if (!manifest.spec.memory?.shortTerm) return { pass: true };
1819
3002
  const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
@@ -1836,9 +3019,6 @@ var evaluationRules = [
1836
3019
  title: "Evaluation dataset declared",
1837
3020
  description: "At least one evaluation dataset is required for quality assurance",
1838
3021
  severity: "medium",
1839
- evidenceLevel: "probed",
1840
- proofTool: "agentspec health",
1841
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1842
3022
  check(manifest) {
1843
3023
  const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
1844
3024
  return {
@@ -1856,9 +3036,6 @@ var evaluationRules = [
1856
3036
  title: "CI gate enabled",
1857
3037
  description: "Evaluation CI gate prevents regressions from reaching production",
1858
3038
  severity: "medium",
1859
- evidenceLevel: "probed",
1860
- proofTool: "agentspec health",
1861
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1862
3039
  check(manifest) {
1863
3040
  const hasCiGate = manifest.spec.evaluation?.ciGate === true;
1864
3041
  return {
@@ -1873,22 +3050,22 @@ var evaluationRules = [
1873
3050
  {
1874
3051
  id: "EVAL-03",
1875
3052
  pack: "evaluation-coverage",
1876
- title: "Hallucination metric threshold configured",
1877
- description: "A hallucination threshold below 0.1 is required for production agents",
3053
+ title: "Grounded metric threshold configured",
3054
+ description: "A grounded threshold (minimum pass rate, 0.95 recommended) is required for production agents",
1878
3055
  severity: "medium",
1879
- evidenceLevel: "probed",
1880
- proofTool: "agentspec health",
1881
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1882
3056
  check(manifest) {
1883
- const metrics = manifest.spec.evaluation?.metrics ?? [];
1884
- const hasHallucinationMetric = metrics.includes("hallucination");
1885
- const threshold = manifest.spec.evaluation?.thresholds?.["hallucination"];
1886
- const pass = hasHallucinationMetric && threshold !== void 0;
3057
+ const evaluation = manifest.spec.evaluation;
3058
+ const scopes = [evaluation, evaluation?.live];
3059
+ const hasGrounded = (scope) => !!scope?.metrics?.map(canonicalMetric).includes("grounded");
3060
+ const hasGroundedMetric = scopes.some(hasGrounded);
3061
+ const pass = scopes.some(
3062
+ (scope) => hasGrounded(scope) && Object.keys(scope?.thresholds ?? {}).some((n) => canonicalMetric(n) === "grounded")
3063
+ );
1887
3064
  return {
1888
3065
  pass,
1889
- message: pass ? void 0 : !hasHallucinationMetric ? "hallucination metric not in spec.evaluation.metrics" : "No hallucination threshold configured",
1890
- path: "/spec/evaluation/thresholds/hallucination",
1891
- recommendation: "Add hallucination to spec.evaluation.metrics and set spec.evaluation.thresholds.hallucination: 0.05",
3066
+ message: pass ? void 0 : !hasGroundedMetric ? "grounded metric not in spec.evaluation.metrics or spec.evaluation.live.metrics" : "No grounded threshold next to the grounded metric",
3067
+ path: "/spec/evaluation/thresholds/grounded",
3068
+ recommendation: "Add grounded to spec.evaluation.metrics (or spec.evaluation.live.metrics) and set the matching thresholds.grounded: 0.95",
1892
3069
  references: []
1893
3070
  };
1894
3071
  }
@@ -1903,9 +3080,6 @@ var observabilityRules = [
1903
3080
  title: "Tracing backend declared",
1904
3081
  description: "Agents should declare a tracing backend so runs are observable and debuggable",
1905
3082
  severity: "medium",
1906
- evidenceLevel: "probed",
1907
- proofTool: "agentspec health",
1908
- proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
1909
3083
  check(manifest) {
1910
3084
  const pass = !!manifest.spec.observability?.tracing?.backend;
1911
3085
  return {
@@ -1922,9 +3096,6 @@ var observabilityRules = [
1922
3096
  title: "Structured logging enabled",
1923
3097
  description: "Structured logging makes log aggregation and searching reliable",
1924
3098
  severity: "low",
1925
- evidenceLevel: "behavioral",
1926
- proofTool: "AgentSpec EventPush + sidecar gap analysis",
1927
- proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
1928
3099
  check(manifest) {
1929
3100
  const pass = manifest.spec.observability?.logging?.structured !== false;
1930
3101
  return {
@@ -1941,9 +3112,6 @@ var observabilityRules = [
1941
3112
  title: "Sensitive fields redacted from logs",
1942
3113
  description: "Log redaction prevents secrets and PII from appearing in log aggregators",
1943
3114
  severity: "medium",
1944
- evidenceLevel: "external",
1945
- proofTool: "Microsoft Presidio",
1946
- proofToolUrl: "https://microsoft.github.io/presidio/",
1947
3115
  check(manifest) {
1948
3116
  const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
1949
3117
  return {
@@ -1956,57 +3124,8 @@ var observabilityRules = [
1956
3124
  }
1957
3125
  ];
1958
3126
 
1959
- // src/audit/rules/metadata.rules.ts
1960
- var metadataRules = [
1961
- {
1962
- id: "META-01",
1963
- pack: "metadata-quality",
1964
- title: "Agent description declared",
1965
- description: "Agents should have a non-empty description for fleet discovery and documentation",
1966
- severity: "low",
1967
- evidenceLevel: "declarative",
1968
- check(manifest) {
1969
- const desc = manifest.metadata.description;
1970
- const pass = typeof desc === "string" && desc.trim().length > 0;
1971
- return {
1972
- pass,
1973
- message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
1974
- path: "/metadata/description",
1975
- recommendation: "Add a concise metadata.description explaining what the agent does",
1976
- references: []
1977
- };
1978
- }
1979
- },
1980
- {
1981
- id: "META-02",
1982
- pack: "metadata-quality",
1983
- title: "Model temperature is production-safe",
1984
- description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
1985
- severity: "medium",
1986
- evidenceLevel: "declarative",
1987
- check(manifest) {
1988
- const temperature = manifest.spec.model.parameters?.temperature;
1989
- if (temperature === void 0) return { pass: true };
1990
- const pass = temperature <= 1.5;
1991
- return {
1992
- pass,
1993
- message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above 1.5 produce unreliable outputs in production.`,
1994
- path: "/spec/model/parameters/temperature",
1995
- recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
1996
- references: []
1997
- };
1998
- }
1999
- }
2000
- ];
2001
-
2002
3127
  // src/audit/index.ts
2003
- var SEVERITY_WEIGHTS = {
2004
- critical: 4,
2005
- high: 3,
2006
- medium: 2,
2007
- low: 1,
2008
- info: 0
2009
- };
3128
+ var SEVERITY_WEIGHTS = CONTRACT.severityWeights;
2010
3129
  var ALL_RULES = [
2011
3130
  ...modelRules,
2012
3131
  ...securityRules,
@@ -2016,19 +3135,10 @@ var ALL_RULES = [
2016
3135
  ...metadataRules
2017
3136
  ];
2018
3137
  var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
2019
- function isProofRecord(r) {
2020
- return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
2021
- }
2022
- function scoreToGrade(score) {
2023
- if (score >= 90) return "A";
2024
- if (score >= 75) return "B";
2025
- if (score >= 60) return "C";
2026
- if (score >= 45) return "D";
2027
- return "F";
2028
- }
3138
+ var COMPLIANCE_PACKS = [...new Set(ALL_RULES.map((r) => r.pack))];
2029
3139
  function resolveActiveRules(manifest, opts) {
2030
3140
  const manifestPacks = manifest.spec.compliance?.packs ?? [];
2031
- const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : [...new Set(ALL_RULES.map((r) => r.pack))]);
3141
+ const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : COMPLIANCE_PACKS);
2032
3142
  return ALL_RULES.filter((r) => activePacks.includes(r.pack));
2033
3143
  }
2034
3144
  function collectSuppressions(manifest) {
@@ -2057,10 +3167,10 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2057
3167
  const violations = [];
2058
3168
  const packBreakdown = {};
2059
3169
  const evidenceBreakdown = {
2060
- declarative: { passed: 0, total: 0 },
2061
- probed: { passed: 0, total: 0 },
2062
- behavioral: { passed: 0, total: 0 },
2063
- external: { passed: 0, total: 0 }
3170
+ independent: { passed: 0, total: 0 },
3171
+ "evidence-point": { passed: 0, total: 0 },
3172
+ offline: { passed: 0, total: 0 },
3173
+ declared: { passed: 0, total: 0 }
2064
3174
  };
2065
3175
  let totalWeight = 0;
2066
3176
  let passedWeight = 0;
@@ -2071,19 +3181,20 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2071
3181
  const suppressed = suppressedIds.has(rule.id);
2072
3182
  const weight = SEVERITY_WEIGHTS[rule.severity];
2073
3183
  const result = ruleResults.get(rule.id);
2074
- const tier = evidenceBreakdown[rule.evidenceLevel];
3184
+ const classes = ruleClasses(rule.id);
2075
3185
  packBreakdown[rule.pack].total += 1;
2076
- tier.total += 1;
3186
+ tally(evidenceBreakdown, classes, "total");
2077
3187
  if (suppressed) {
2078
3188
  packBreakdown[rule.pack].passed += 1;
2079
- tier.passed += 1;
3189
+ tally(evidenceBreakdown, classes, "passed");
2080
3190
  continue;
2081
3191
  }
2082
3192
  if (result.pass) {
2083
3193
  packBreakdown[rule.pack].passed += 1;
2084
- tier.passed += 1;
3194
+ tally(evidenceBreakdown, classes, "passed");
2085
3195
  passedWeight += weight;
2086
3196
  } else {
3197
+ const hint = proofHint(rule.id);
2087
3198
  violations.push({
2088
3199
  ruleId: rule.id,
2089
3200
  severity: rule.severity,
@@ -2092,9 +3203,9 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2092
3203
  path: result.path,
2093
3204
  recommendation: result.recommendation,
2094
3205
  references: result.references,
2095
- evidenceLevel: rule.evidenceLevel,
2096
- proofTool: rule.proofTool,
2097
- proofToolUrl: rule.proofToolUrl
3206
+ evidenceClasses: classes,
3207
+ proofTool: hint?.tool,
3208
+ proofToolUrl: hint?.url
2098
3209
  });
2099
3210
  }
2100
3211
  totalWeight += weight;
@@ -2106,24 +3217,36 @@ function computeScoring(rules, ruleResults, suppressedIds) {
2106
3217
  }
2107
3218
  return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
2108
3219
  }
2109
- function computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, proofRecords) {
2110
- const now = /* @__PURE__ */ new Date();
2111
- const proofSet = new Set(
2112
- proofRecords.filter((r) => !r.expiresAt || new Date(r.expiresAt) > now).map((r) => r.ruleId)
2113
- );
2114
- let provedPassedWeight = 0;
2115
- for (const rule of rules) {
2116
- if (suppressedIds.has(rule.id)) continue;
2117
- const result = ruleResults.get(rule.id);
2118
- const weight = SEVERITY_WEIGHTS[rule.severity];
2119
- const isProved = rule.evidenceLevel === "probed" && result.pass || rule.evidenceLevel === "behavioral" && result.pass || rule.evidenceLevel === "external" && proofSet.has(rule.id);
2120
- if (isProved) provedPassedWeight += weight;
3220
+ function tally(breakdown, classes, field) {
3221
+ for (const c of classes) breakdown[c][field] += 1;
3222
+ }
3223
+ var SIDECAR_GAP = { tool: "AgentSpec sidecar gap analysis", url: "https://agentspec.io/docs/concepts/probe-coverage" };
3224
+ var COLLECTOR_HINTS = {
3225
+ "sidecar-probe": SIDECAR_GAP,
3226
+ "sidecar-forwarder": SIDECAR_GAP
3227
+ };
3228
+ function proofHint(ruleId) {
3229
+ for (const { collector, status } of ruleSources(ruleId)) {
3230
+ const hint = COLLECTOR_HINTS[collector];
3231
+ if (status === "built" && hint) return { collector, ...hint };
2121
3232
  }
2122
- const provedScore = totalWeight === 0 ? 100 : Math.round(provedPassedWeight / totalWeight * 100);
2123
- const pendingProofCount = rules.filter(
2124
- (r) => r.evidenceLevel === "external" && !suppressedIds.has(r.id) && ruleResults.get(r.id).pass && !proofSet.has(r.id)
2125
- ).length;
2126
- return { provedScore, provedGrade: scoreToGrade(provedScore), pendingProofCount };
3233
+ return void 0;
3234
+ }
3235
+ function explainPendingEvidence(verified, rules) {
3236
+ const pending = new Map(rules.filter((r) => r.awaitsEvidence).map((r) => [r.id, r.awaitsEvidence]));
3237
+ return {
3238
+ ...verified,
3239
+ unverified: verified.unverified.map((u) => {
3240
+ const awaits = pending.get(u.ruleId);
3241
+ if (!awaits || u.reason.code !== "no_evidence") return u;
3242
+ return { ...u, reason: { code: awaits.code, detail: `evidence source arrives in phase ${awaits.phase}` } };
3243
+ })
3244
+ };
3245
+ }
3246
+ function summariseRuleEvidence(rules) {
3247
+ return Object.fromEntries(
3248
+ rules.map((r) => [r.id, { classes: ruleClasses(r.id), notProven: RULE_EVIDENCE[r.id]?.notProven ?? null }])
3249
+ );
2127
3250
  }
2128
3251
  function runAudit(manifest, opts = {}) {
2129
3252
  const rules = resolveActiveRules(manifest, opts);
@@ -2133,11 +3256,21 @@ function runAudit(manifest, opts = {}) {
2133
3256
  violations,
2134
3257
  packBreakdown,
2135
3258
  evidenceBreakdown,
2136
- totalWeight,
2137
3259
  overallScore,
2138
3260
  categoryScores
2139
3261
  } = computeScoring(rules, ruleResults, suppressedIds);
2140
- const proved = opts.proofRecords !== void 0 ? computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, opts.proofRecords) : {};
3262
+ const applicable = rules.filter((r) => !suppressedIds.has(r.id));
3263
+ const applicableRules = applicable.map((r) => ({ id: r.id, severity: r.severity }));
3264
+ const verified = explainPendingEvidence(
3265
+ grade(
3266
+ opts.records ?? [],
3267
+ applicableRules,
3268
+ opts.trust ?? "standard",
3269
+ (/* @__PURE__ */ new Date()).toISOString(),
3270
+ opts.evidencePoints
3271
+ ),
3272
+ applicable
3273
+ );
2141
3274
  return {
2142
3275
  agentName: manifest.metadata.name,
2143
3276
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2150,9 +3283,26 @@ function runAudit(manifest, opts = {}) {
2150
3283
  totalRules: rules.length,
2151
3284
  packBreakdown,
2152
3285
  evidenceBreakdown,
2153
- ...proved
3286
+ verified,
3287
+ applicableRules,
3288
+ ruleEvidence: summariseRuleEvidence(rules)
2154
3289
  };
2155
3290
  }
3291
+ function canonicalJson(value) {
3292
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
3293
+ if (value !== null && typeof value === "object") {
3294
+ const obj = value;
3295
+ const members = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
3296
+ return `{${members.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
3297
+ }
3298
+ return JSON.stringify(value);
3299
+ }
3300
+ function canonicalManifestSha256(manifest) {
3301
+ return createHash("sha256").update(canonicalJson(manifest), "utf8").digest("hex");
3302
+ }
3303
+ function declaredManifestSha256(raw) {
3304
+ return canonicalManifestSha256(yaml.load(raw, { schema: yaml.JSON_SCHEMA }));
3305
+ }
2156
3306
 
2157
3307
  // src/generate/index.ts
2158
3308
  var _registry = /* @__PURE__ */ new Map();
@@ -2165,7 +3315,7 @@ function getAdapter(framework) {
2165
3315
  function listAdapters() {
2166
3316
  return [..._registry.keys()];
2167
3317
  }
2168
- function generateAdapter(manifest, framework, options) {
3318
+ function generateAdapter(manifest, framework, options2) {
2169
3319
  const adapter = _registry.get(framework.toLowerCase());
2170
3320
  if (!adapter) {
2171
3321
  throw new Error(
@@ -2174,7 +3324,7 @@ function generateAdapter(manifest, framework, options) {
2174
3324
  Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
2175
3325
  );
2176
3326
  }
2177
- return adapter.generate(manifest, options);
3327
+ return adapter.generate(manifest, options2);
2178
3328
  }
2179
3329
 
2180
3330
  // src/agent/usage-ledger.ts
@@ -2220,8 +3370,8 @@ var UsageLedger = class {
2220
3370
  * Return a frozen snapshot of the current usage window.
2221
3371
  *
2222
3372
  * @param reset - When true (default), clears all counters and starts a new
2223
- * window. Use reset=true in the heartbeat push so each window is shipped
2224
- * exactly once. Use reset=false for inspection without draining.
3373
+ * window. Use reset=false for inspection without draining; the heartbeat
3374
+ * push takes a reset=false snapshot and calls subtract() with it.
2225
3375
  */
2226
3376
  snapshot(reset = true) {
2227
3377
  const snap = buildSnapshot(this.counters, this.windowStart);
@@ -2231,39 +3381,220 @@ var UsageLedger = class {
2231
3381
  }
2232
3382
  return snap;
2233
3383
  }
3384
+ /**
3385
+ * Remove a shipped snapshot from the ledger; a new window starts, because the control plane dedupes heartbeats on windowStartedAt.
3386
+ * Usage recorded after the snapshot was taken stays, so every token is
3387
+ * shipped exactly once even when a call finishes while a heartbeat is in flight.
3388
+ */
3389
+ subtract(shipped) {
3390
+ for (const s of shipped.models) {
3391
+ const entry = this.counters.get(s.modelId);
3392
+ if (!entry) continue;
3393
+ entry.promptTokens -= s.promptTokens;
3394
+ entry.completionTokens -= s.completionTokens;
3395
+ entry.totalTokens -= s.totalTokens;
3396
+ entry.callCount -= s.callCount;
3397
+ if (entry.callCount <= 0) this.counters.delete(s.modelId);
3398
+ }
3399
+ this.windowStart = /* @__PURE__ */ new Date();
3400
+ }
2234
3401
  };
2235
3402
 
3403
+ // src/runtime/evidence-points.ts
3404
+ var GUARDRAIL = EvidencePoint.enum.guardrail;
3405
+ var [MCP_LIST, MCP_CALL] = Object.keys(CONTRACT.evidencePoints.mcpClient);
3406
+ var [SCRUB, TTL, AUDIT] = Object.keys(CONTRACT.evidencePoints.memoryWrite);
3407
+ var [NAMESPACE] = Object.keys(CONTRACT.evidencePoints.vectorStore);
3408
+ var [TOOL_TARGET] = Object.keys(CONTRACT.evidencePoints.tool);
3409
+ var [LOGGING_TARGET] = Object.keys(CONTRACT.evidencePoints.logging);
3410
+ var keyOf = (point2, target) => `${point2}\0${target}`;
3411
+ var DEFAULT_KEY = /* @__PURE__ */ Symbol.for("agentspec.runtime.defaultEvidencePoints");
3412
+ var defaults = globalThis;
3413
+ var rec = (value) => value && typeof value === "object" ? value : {};
3414
+ var stdioServers = (spec) => (Array.isArray(rec(spec.mcp).servers) ? rec(spec.mcp).servers : []).filter((s) => s?.transport === "stdio");
3415
+ var isDeclared = (value) => Array.isArray(value) ? value.length > 0 : Boolean(value);
3416
+ function memoryWriteTargets(memory) {
3417
+ const hygiene = rec(memory.hygiene);
3418
+ const targets = /* @__PURE__ */ new Set();
3419
+ if (isDeclared(hygiene.piiScrubFields)) targets.add(SCRUB);
3420
+ if ([rec(memory.shortTerm).ttlSeconds, rec(memory.longTerm).ttlDays].some((ttl) => ttl !== void 0 && ttl !== null)) targets.add(TTL);
3421
+ if (memory.longTerm && hygiene.auditLog === true) targets.add(AUDIT);
3422
+ return targets;
3423
+ }
3424
+ function declaredTargets(spec) {
3425
+ const all = (point2, declared2) => new Set(declared2 ? Object.keys(CONTRACT.evidencePoints[point2] ?? {}) : []);
3426
+ const memory = rec(spec.memory);
3427
+ const logging = rec(rec(spec.observability).logging);
3428
+ return {
3429
+ [EvidencePoint.enum.memoryWrite]: memoryWriteTargets(memory),
3430
+ [EvidencePoint.enum.vectorStore]: new Set(rec(memory.vector).namespace ? [NAMESPACE] : []),
3431
+ [GUARDRAIL]: new Set(Object.keys(CONTRACT.evidencePoints[GUARDRAIL] ?? {}).filter((t) => isDeclared(rec(spec.guardrails)[t]))),
3432
+ [EvidencePoint.enum.tool]: all(EvidencePoint.enum.tool, (Array.isArray(spec.tools) ? spec.tools : []).some((t) => t?.type === "function")),
3433
+ [EvidencePoint.enum.mcpClient]: all(EvidencePoint.enum.mcpClient, stdioServers(spec).length > 0),
3434
+ [EvidencePoint.enum.logging]: all(EvidencePoint.enum.logging, rec(spec.observability).logging !== void 0 && logging.structured !== false)
3435
+ };
3436
+ }
3437
+ function applicablePoints(spec) {
3438
+ const declared2 = declaredTargets(spec);
3439
+ const applicable = /* @__PURE__ */ new Map();
3440
+ for (const [point2, targets] of Object.entries(CONTRACT.evidencePoints)) {
3441
+ for (const [target, ruleId] of Object.entries(targets)) {
3442
+ const listed = CONTRACT.ruleEvidence[ruleId]?.sources.some((s) => s.point === point2);
3443
+ if (listed && declared2[point2]?.has(target)) applicable.set(keyOf(point2, target), { point: EvidencePoint.parse(point2), target, ruleId });
3444
+ }
3445
+ }
3446
+ return applicable;
3447
+ }
3448
+ function createEvidencePoints(manifest) {
3449
+ const manifestSha256 = canonicalManifestSha256(manifest);
3450
+ const spec = rec(rec(manifest).spec);
3451
+ const applicable = applicablePoints(spec);
3452
+ const mcpServers = stdioServers(spec).map((s) => String(s.name));
3453
+ const wired = /* @__PURE__ */ new Set();
3454
+ const emptyWindow = () => new Map([...applicable.keys()].map((k) => [k, { total: 0, counts: { pass: 0, fail: 0, not_verifiable: 0 }, last: [] }]));
3455
+ let calls = emptyWindow();
3456
+ let windowStart = /* @__PURE__ */ new Date();
3457
+ let seq = 0;
3458
+ const shippedUpTo = /* @__PURE__ */ new WeakMap();
3459
+ const points = {
3460
+ register: (point2, target) => {
3461
+ wired.add(keyOf(point2, target));
3462
+ },
3463
+ applies: (point2, target) => applicable.has(keyOf(point2, target)),
3464
+ memorySpec: () => rec(spec.memory),
3465
+ declaredMcpServers: () => [...mcpServers],
3466
+ recordCall: (point2, target, { subject, status = "pass", reason } = {}) => {
3467
+ if (status !== "pass" && !reason) throw new Error(`recordCall: a ${status} call of ${point2}:${target} needs a reason`);
3468
+ const c = calls.get(keyOf(point2, target));
3469
+ if (!c) return;
3470
+ c.total += 1;
3471
+ c.counts[status] += 1;
3472
+ c.last.push({ at: /* @__PURE__ */ new Date(), seq: ++seq, subject, status, reason });
3473
+ if (c.last.length > CONTRACT.evidencePointRecordLimit) c.last.shift();
3474
+ },
3475
+ snapshot: (ttlSeconds, reset = true) => {
3476
+ const windowEnd = /* @__PURE__ */ new Date();
3477
+ const lastSeq = /* @__PURE__ */ new Map();
3478
+ const window = { start: windowStart.toISOString(), end: windowEnd.toISOString() };
3479
+ const aggregates = [...applicable].map(([key, { point: point2, target, ruleId }]) => {
3480
+ const c = calls.get(key);
3481
+ const state = c.total ? "fired" : wired.has(key) ? "wired" : "not_wired";
3482
+ lastSeq.set(key, seq);
3483
+ const records = c.last.map(({ at, subject, status, reason }) => ({
3484
+ checkId: subject ?? `${point2}:${target}`,
3485
+ ruleId,
3486
+ status: EvidenceStatus.enum[status],
3487
+ ...reason ? { reason } : {},
3488
+ source: EvidenceSource.enum["sdk-observed"],
3489
+ origin: EvidenceOrigin.enum.in_process,
3490
+ collector: Collector.enum.sdk,
3491
+ observedAt: at.toISOString(),
3492
+ window,
3493
+ ttlSeconds,
3494
+ sampleSize: c.total,
3495
+ traceIds: [],
3496
+ manifestSha256
3497
+ }));
3498
+ return { point: point2, target, state, counts: { ...c.counts }, records };
3499
+ });
3500
+ if (reset) {
3501
+ calls = emptyWindow();
3502
+ windowStart = windowEnd;
3503
+ } else {
3504
+ shippedUpTo.set(aggregates, { lastSeq, windowEnd });
3505
+ }
3506
+ return aggregates;
3507
+ },
3508
+ subtract: (shipped) => {
3509
+ const taken = shippedUpTo.get(shipped);
3510
+ if (!taken) throw new Error("EvidencePoints.subtract: pass the array snapshot() returned");
3511
+ for (const a of shipped) {
3512
+ const key = keyOf(a.point, a.target);
3513
+ const c = calls.get(key);
3514
+ c.total -= a.counts.pass + a.counts.fail + a.counts.not_verifiable;
3515
+ for (const status of ["pass", "fail", "not_verifiable"]) c.counts[status] -= a.counts[status];
3516
+ c.last = c.last.filter((l) => l.seq > taken.lastSeq.get(key));
3517
+ }
3518
+ windowStart = taken.windowEnd;
3519
+ }
3520
+ };
3521
+ defaults[DEFAULT_KEY] = points;
3522
+ return points;
3523
+ }
3524
+
3525
+ // src/health/tool-check-id.ts
3526
+ function slugify(s) {
3527
+ return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
3528
+ }
3529
+ function toolCheckId(toolName) {
3530
+ return `tool:${slugify(toolName)}`;
3531
+ }
3532
+
3533
+ // src/runtime/call-points.ts
3534
+ var { tool: TOOL, mcpClient: MCP_CLIENT, logging: LOGGING } = EvidencePoint.enum;
3535
+ var { memoryWrite: MEMORY_WRITE, vectorStore: VECTOR_STORE } = EvidencePoint.enum;
3536
+ var NOT_VERIFIABLE_CODE = "instrumentation_unavailable";
3537
+ var WRONG_NAMESPACE_CODE = "undeclared_namespace";
3538
+ var NO_NAMESPACE_CODE = "no_evidence";
3539
+ for (const code of [NOT_VERIFIABLE_CODE, WRONG_NAMESPACE_CODE, NO_NAMESPACE_CODE]) {
3540
+ if (!CONTRACT.reasonCodes.includes(code)) throw new Error(`contract.reasonCodes lacks '${code}'`);
3541
+ }
3542
+ new AsyncLocalStorage();
3543
+
2236
3544
  // src/agent/reporter.ts
3545
+ var HEARTBEAT_POST_TIMEOUT_MS = 1e4;
3546
+ var DEFAULT_PUSH_INTERVAL_SECONDS = 30;
3547
+ var RECORD_TTL_INTERVALS = 2;
3548
+ function buildCappedBody(health, aggregates, usage) {
3549
+ const trimmedChecks = [...health.checks];
3550
+ const serialise = (checks, evidencePoints) => JSON.stringify({
3551
+ contract: CONTRACT.version,
3552
+ health: { ...health, checks },
3553
+ records: [],
3554
+ // the control plane reads the records inside each aggregate (#599)
3555
+ evidencePoints,
3556
+ usage
3557
+ });
3558
+ const size = (body2) => Buffer.byteLength(body2, "utf8");
3559
+ let body = serialise(trimmedChecks, aggregates);
3560
+ if (size(body) > CONTRACT.heartbeatMaxBytes) {
3561
+ aggregates = aggregates.map((a) => ({ ...a, records: [] }));
3562
+ body = serialise(trimmedChecks, aggregates);
3563
+ }
3564
+ while (size(body) > CONTRACT.heartbeatMaxBytes && trimmedChecks.length > 0) {
3565
+ trimmedChecks.pop();
3566
+ body = serialise(trimmedChecks, aggregates);
3567
+ }
3568
+ return { body, bytes: size(body) };
3569
+ }
2237
3570
  var AgentSpecReporter = class {
2238
3571
  constructor(manifest, opts = {}) {
2239
3572
  this.manifest = manifest;
2240
3573
  this.opts = opts;
2241
3574
  this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
2242
3575
  this.staleAfterMs = opts.staleAfterMs ?? 6e4;
3576
+ this.evidence = createEvidencePoints(opts.rawManifest ?? manifest);
2243
3577
  }
3578
+ manifest;
3579
+ opts;
2244
3580
  cached = null;
2245
3581
  timer = null;
2246
3582
  _pushTimer = null;
2247
3583
  refreshIntervalMs;
2248
3584
  staleAfterMs;
2249
- refreshing = false;
3585
+ inflight = null;
3586
+ /** True while a heartbeat POST is in flight; a tick that finds it set sends nothing. */
3587
+ heartbeatInFlight = false;
2250
3588
  stopped = false;
2251
- registeredTools = /* @__PURE__ */ new Set();
2252
3589
  /** In-process token usage accumulator. */
2253
3590
  usage = new UsageLedger();
3591
+ /** Evidence points the agent's code wires (`guardrail(...)` reports here); each heartbeat sends the window's aggregates. */
3592
+ evidence;
2254
3593
  /**
2255
- * Register a tool handler by name.
2256
- * Only registered tools report status: 'pass' in the live health check.
2257
- * Declared tools that are not registered report status: 'fail'.
2258
- *
2259
- * Call this once per tool after wiring up your tool handlers:
2260
- * reporter.registerTool('search-arxiv', handler)
2261
- *
2262
- * The handler argument is accepted but not called — it is retained solely so
2263
- * callers can pass the real function object for documentation purposes.
3594
+ * @deprecated No-op, kept so existing callers compile. Registering a handler proves nothing about
3595
+ * which tools the agent calls, so the health report no longer carries `tool:<name>` checks (ADR-0017).
2264
3596
  */
2265
- registerTool(name, _handler) {
2266
- this.registeredTools.add(name);
3597
+ registerTool(_name, _handler) {
2267
3598
  }
2268
3599
  /**
2269
3600
  * Start background refresh. Call once during app startup.
@@ -2327,52 +3658,74 @@ var AgentSpecReporter = class {
2327
3658
  */
2328
3659
  startPushMode(opts) {
2329
3660
  if (this._pushTimer !== null) return;
2330
- const intervalMs = (opts.intervalSeconds ?? 30) * 1e3;
2331
- void this._pushHeartbeat(opts);
3661
+ const controlPlaneUrl = opts.controlPlaneUrl || process.env[CONTRACT.env.controlPlaneUrl];
3662
+ if (!controlPlaneUrl) {
3663
+ throw new Error(
3664
+ `startPushMode: no control plane URL \u2014 pass controlPlaneUrl or set ${CONTRACT.env.controlPlaneUrl}`
3665
+ );
3666
+ }
3667
+ if (!opts.apiKey) {
3668
+ throw new Error("startPushMode: apiKey is required (the per-agent token from POST /api/v1/register)");
3669
+ }
3670
+ const resolved = { ...opts, controlPlaneUrl };
3671
+ const intervalMs = (opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * 1e3;
3672
+ void this._pushHeartbeat(resolved);
2332
3673
  this._pushTimer = setInterval(() => {
2333
- void this._pushHeartbeat(opts);
3674
+ void this._pushHeartbeat(resolved);
2334
3675
  }, intervalMs);
2335
3676
  if (this._pushTimer.unref) {
2336
3677
  this._pushTimer.unref();
2337
3678
  }
2338
3679
  }
2339
3680
  async _pushHeartbeat(opts) {
3681
+ if (this.heartbeatInFlight) return;
3682
+ this.heartbeatInFlight = true;
2340
3683
  try {
2341
3684
  const health = await this.getReport();
2342
- const gap = runAudit(this.manifest);
2343
3685
  const usage = this.usage.snapshot(false);
2344
- const trimmedChecks = [...health.checks];
2345
- let body = JSON.stringify({ health, gap, usage });
2346
- if (body.length > 65536) {
2347
- while (trimmedChecks.length > 0) {
2348
- trimmedChecks.pop();
2349
- body = JSON.stringify({ health: { ...health, checks: trimmedChecks }, gap, usage });
2350
- if (body.length <= 65536) break;
2351
- }
3686
+ const aggregates = this.evidence.snapshot(Math.ceil((opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * RECORD_TTL_INTERVALS), false);
3687
+ const { body, bytes } = buildCappedBody(health, aggregates, usage);
3688
+ if (bytes > CONTRACT.heartbeatMaxBytes) {
3689
+ opts.onError?.(
3690
+ new Error(`Heartbeat not sent: ${bytes} bytes is over the ${CONTRACT.heartbeatMaxBytes}-byte cap with every check trimmed`)
3691
+ );
3692
+ return;
2352
3693
  }
2353
3694
  let res;
3695
+ const abort = new AbortController();
3696
+ const abortTimer = setTimeout(
3697
+ () => abort.abort(new Error(`heartbeat POST timed out after ${HEARTBEAT_POST_TIMEOUT_MS}ms`)),
3698
+ HEARTBEAT_POST_TIMEOUT_MS
3699
+ );
3700
+ abortTimer.unref?.();
2354
3701
  try {
2355
- res = await fetch(`${opts.controlPlaneUrl}/api/v1/heartbeat`, {
3702
+ res = await fetch(`${opts.controlPlaneUrl}${CONTRACT.controlPlanePaths.heartbeat}`, {
2356
3703
  method: "POST",
2357
3704
  headers: {
2358
3705
  "Authorization": `Bearer ${opts.apiKey}`,
2359
3706
  "Content-Type": "application/json"
2360
3707
  },
2361
- body
3708
+ body,
3709
+ signal: abort.signal
2362
3710
  });
2363
3711
  } catch (err) {
2364
3712
  const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
2365
3713
  opts.onError?.(new Error(sanitized));
2366
3714
  return;
3715
+ } finally {
3716
+ clearTimeout(abortTimer);
2367
3717
  }
2368
3718
  if (res.ok) {
2369
- this.usage.snapshot(true);
3719
+ this.usage.subtract(usage);
3720
+ this.evidence.subtract(aggregates);
2370
3721
  } else {
2371
3722
  opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
2372
3723
  }
2373
3724
  } catch (err) {
2374
3725
  const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
2375
3726
  opts.onError?.(new Error(sanitized));
3727
+ } finally {
3728
+ this.heartbeatInFlight = false;
2376
3729
  }
2377
3730
  }
2378
3731
  /** Stop sending heartbeats to the control plane. */
@@ -2386,38 +3739,26 @@ var AgentSpecReporter = class {
2386
3739
  isPushModeActive() {
2387
3740
  return this._pushTimer !== null;
2388
3741
  }
2389
- async refresh() {
2390
- if (this.stopped || this.refreshing) return;
2391
- this.refreshing = true;
3742
+ /** Runs one refresh; concurrent callers share the in-flight promise so they all see a populated cache. */
3743
+ refresh() {
3744
+ if (this.inflight) return this.inflight;
3745
+ if (this.stopped) return Promise.resolve();
3746
+ this.inflight = this.runRefresh().finally(() => {
3747
+ this.inflight = null;
3748
+ });
3749
+ return this.inflight;
3750
+ }
3751
+ async runRefresh() {
2392
3752
  try {
2393
3753
  const report = await runHealthCheck(this.manifest, {
2394
3754
  checkModel: true,
2395
3755
  checkMcp: true,
2396
3756
  checkMemory: true,
2397
- checkServices: true
2398
- });
2399
- const toolChecks = (this.manifest.spec.tools ?? []).map((tool) => {
2400
- const registered = this.registeredTools.has(tool.name);
2401
- return {
2402
- id: `tool:${tool.name}`,
2403
- category: "tool",
2404
- status: registered ? "pass" : "fail",
2405
- severity: "info",
2406
- ...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
2407
- };
3757
+ checkServices: true,
3758
+ baseDir: this.opts.baseDir,
3759
+ rawManifest: this.opts.rawManifest
2408
3760
  });
2409
- const toolPassed = toolChecks.filter((c) => c.status === "pass").length;
2410
- const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
2411
- const enrichedReport = {
2412
- ...report,
2413
- checks: [...report.checks, ...toolChecks],
2414
- summary: {
2415
- ...report.summary,
2416
- passed: report.summary.passed + toolPassed,
2417
- failed: report.summary.failed + toolFailed
2418
- }
2419
- };
2420
- this.cached = { report: enrichedReport, ts: Date.now() };
3761
+ this.cached = { report, ts: Date.now() };
2421
3762
  } catch {
2422
3763
  if (!this.cached) {
2423
3764
  this.cached = {
@@ -2441,8 +3782,6 @@ var AgentSpecReporter = class {
2441
3782
  } else {
2442
3783
  this.cached = { ...this.cached, ts: Date.now() };
2443
3784
  }
2444
- } finally {
2445
- this.refreshing = false;
2446
3785
  }
2447
3786
  }
2448
3787
  };
@@ -2450,7 +3789,7 @@ var AgentSpecReporter = class {
2450
3789
  // src/agent/adapters/fastify.ts
2451
3790
  function agentSpecFastifyPlugin(reporter) {
2452
3791
  return async (app) => {
2453
- app.get("/agentspec/health", async (_req, reply) => {
3792
+ app.get(CONTRACT.agentPaths.health, async (_req, reply) => {
2454
3793
  try {
2455
3794
  const report = await reporter.getReport();
2456
3795
  reply.status(200).send(report);
@@ -2513,6 +3852,6 @@ function agentSpecExpressRouter(reporter) {
2513
3852
  return handler;
2514
3853
  }
2515
3854
 
2516
- export { AUDIT_RULE_IDS, AgentSpecReporter, LATEST_API_VERSION, ManifestSchema, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, collectEnvRefs, collectFileRefs, detectRefType, detectVersion, exportJsonSchema, generateAdapter, getAdapter, isLatestVersion, isProofRecord, listAdapters, loadManifest, migrateManifest, registerAdapter, resolveRef, resolveRefs, runAudit, runHealthCheck, runServiceChecks, scoreToGrade, tryLoadManifest };
3855
+ export { AUDIT_RULE_IDS, AgentSpecReporter, COMPLIANCE_PACKS, CONTRACT, ContractSchema, EVIDENCE_SHAPES, EvalResult, EvidenceOrigin, EvidenceReason, EvidenceRecord, EvidenceSource, EvidenceStatus, EvidenceWindow, Heartbeat, LATEST_API_VERSION, MAX_PRODUCTION_TEMPERATURE, MAX_TRACE_IDS, ManifestSchema, PodspecFinding, RULE_EVIDENCE, ReadinessFinding, RuleEvidenceEntry, SeveritySchema, SidecarPush, TrustPolicy, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, canonicalManifestSha256, collectEnvRefs, collectFileRefs, computeHealthStatus, declaredManifestSha256, describeManifestError, detectRefType, detectVersion, exportJsonSchema, formatVerified, gatewayEnvNames, generateAdapter, getAdapter, grade, isLatestModelId, isLatestVersion, listAdapters, loadManifest, migrateManifest, registerAdapter, renderGatewayConfig, resolveRef, resolveRefs, ruleClasses, runAudit, runHealthCheck, runServiceChecks, scoreToGrade, secretEnvName, slugify, toolCheckId, tryLoadManifest };
2517
3856
  //# sourceMappingURL=index.js.map
2518
3857
  //# sourceMappingURL=index.js.map