@agentspec/sdk 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/evidence-points-C6j4PSsQ.d.cts +1139 -0
- package/dist/evidence-points-C6j4PSsQ.d.ts +1139 -0
- package/dist/index.cjs +1733 -346
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1026 -2549
- package/dist/index.d.ts +1026 -2549
- package/dist/index.js +1684 -345
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +930 -0
- package/dist/runtime.cjs.map +1 -0
- package/dist/runtime.d.cts +63 -0
- package/dist/runtime.d.ts +63 -0
- package/dist/runtime.js +921 -0
- package/dist/runtime.js.map +1 -0
- package/package.json +15 -8
- package/schemas/evidence/v1/contract.schema.json +2078 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.n-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.pass-rate-above-one.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.bad-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.extra-field.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.fail-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.missing-ttl.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.not-verifiable-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-epoch.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-out-of-range.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-space-separated.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.sample-size-boolean.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.ttl-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-source.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-record.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.point-records-over-limit.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.with-gap.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/ReadinessFinding.bad-severity.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/SidecarPush.missing-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/TrustPolicy.lenient.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.live.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.null-dataset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.fail.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.null-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass-null-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.timezone-offset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.unknown-reason-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.with-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-null.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.evidence-points.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.full.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk-no-records.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.open-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.with-subject.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/SidecarPush.declared.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.standard.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.strict.json +1 -0
- package/schemas/grader/v1/fixtures/accepts-fractional-seconds.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-leap-day-in-year-below-100.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-more-than-6-fractional-digits.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-numeric-offset.json +62 -0
- package/schemas/grader/v1/fixtures/all-records-must-pass.json +58 -0
- package/schemas/grader/v1/fixtures/below-threshold-not-graded.json +61 -0
- package/schemas/grader/v1/fixtures/derived-rule-fails-when-a-parent-fails.json +67 -0
- package/schemas/grader/v1/fixtures/derived-rule-follows-memory-write-parent.json +43 -0
- package/schemas/grader/v1/fixtures/derived-rule-passes-with-its-parents.json +64 -0
- package/schemas/grader/v1/fixtures/derived-rule-unverified-when-a-parent-has-no-evidence.json +51 -0
- package/schemas/grader/v1/fixtures/expired-record-is-stale.json +66 -0
- package/schemas/grader/v1/fixtures/fail-beats-not-verifiable.json +62 -0
- package/schemas/grader/v1/fixtures/grade-band-b.json +62 -0
- package/schemas/grader/v1/fixtures/independent-from-listed-collector-is-admitted.json +39 -0
- package/schemas/grader/v1/fixtures/independent-from-unlisted-collector-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/low-severity-coverage-only.json +90 -0
- package/schemas/grader/v1/fixtures/malformed-point-is-ignored.json +67 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-standard.json +196 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-strict.json +196 -0
- package/schemas/grader/v1/fixtures/not-admitted-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/not-verifiable-ignored-when-one-passes.json +58 -0
- package/schemas/grader/v1/fixtures/not-verifiable-only-keeps-reason.json +71 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-fail-when-no-point-of-the-rule-is-wired.json +59 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-wired-point-pass.json +87 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-its-own-earlier-fail-when-another-point-is-wired.json +70 -0
- package/schemas/grader/v1/fixtures/not-wired-point-ignores-a-wired-point-fail-without-the-rule-id.json +69 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-a-wired-point-fail.json +79 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-independent-evidence.json +64 -0
- package/schemas/grader/v1/fixtures/not-wired-point-never-passes.json +107 -0
- package/schemas/grader/v1/fixtures/offline-record-is-not-admitted-yet.json +47 -0
- package/schemas/grader/v1/fixtures/record-without-collector-is-admitted-until-producers-set-it.json +38 -0
- package/schemas/grader/v1/fixtures/records-without-rule-id-never-graded.json +62 -0
- package/schemas/grader/v1/fixtures/rejects-compact-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-lowercase-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-impossible-date-feb-29.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-t.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-z.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-hours-above-23.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-minutes-above-59.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offsetless-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-year-zero.json +56 -0
- package/schemas/grader/v1/fixtures/rounds-coverage-before-the-threshold.json +850 -0
- package/schemas/grader/v1/fixtures/rounds-verified-before-the-band.json +674 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-admitted-standard.json +39 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-not-admitted-strict.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-observed-standard.json +58 -0
- package/schemas/grader/v1/fixtures/sdk-observed-strict.json +66 -0
- package/schemas/grader/v1/fixtures/sdk-observed-without-evidence-point-source-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-only-agent-all-points-fired.json +116 -0
- package/schemas/grader/v1/fixtures/stale-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/strict-admits-independent.json +39 -0
- package/schemas/grader/v1/fixtures/sub-millisecond-expiry-boundary.json +38 -0
- package/schemas/v1/agent.schema.json +372 -198
package/dist/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import {
|
|
3
|
-
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { readFileSync, realpathSync, existsSync } from 'fs';
|
|
4
3
|
import { resolve, sep, dirname } from 'path';
|
|
5
|
-
import yaml from 'js-yaml';
|
|
4
|
+
import * as yaml from 'js-yaml';
|
|
5
|
+
import { dump } from 'js-yaml';
|
|
6
6
|
import { isIP } from 'net';
|
|
7
|
+
import { createHash } from 'crypto';
|
|
8
|
+
import { AsyncLocalStorage } from 'async_hooks';
|
|
7
9
|
|
|
8
10
|
// src/schema/manifest.schema.ts
|
|
9
11
|
var refOrLiteral = z.string();
|
|
@@ -16,17 +18,37 @@ var MetadataSchema = z.object({
|
|
|
16
18
|
author: z.string().optional(),
|
|
17
19
|
license: z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
|
|
18
20
|
});
|
|
21
|
+
function isHttpUrl(value) {
|
|
22
|
+
if (value.startsWith("$env:")) return true;
|
|
23
|
+
try {
|
|
24
|
+
const url = new URL(value);
|
|
25
|
+
return (url.protocol === "http:" || url.protocol === "https:") && url.hostname !== "";
|
|
26
|
+
} catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
var baseUrlSchema = z.string().regex(/^(\$env:[A-Za-z_][A-Za-z0-9_]*|https?:\/\/.+)$/, 'must be an http(s) URL or a "$env:VAR" reference').refine(isHttpUrl, 'must be an http(s) URL with a host, or a "$env:VAR" reference').optional().describe('Override the provider endpoint with an OpenAI-compatible base URL (vLLM, Ollama, LiteLLM, an internal proxy, a mock LLM). A literal http(s) URL with a host (checked by the loader, not expressible in JSON Schema) or "$env:VAR". When set, the gateway treats the model as a custom OpenAI-compatible provider for any "provider" value except "anthropic", whose baseUrl is an Anthropic Messages endpoint without "/v1" (the gateway adds it). Omit to use the provider default.');
|
|
31
|
+
var ModelPricingSchema = z.object({
|
|
32
|
+
input: z.number().nonnegative().describe("USD per 1M input (prompt) tokens."),
|
|
33
|
+
output: z.number().nonnegative().describe("USD per 1M output (completion) tokens."),
|
|
34
|
+
cacheRead: z.number().nonnegative().optional().describe("USD per 1M tokens read from the prompt cache."),
|
|
35
|
+
cacheWrite: z.number().nonnegative().optional().describe("USD per 1M tokens written to the prompt cache."),
|
|
36
|
+
reasoning: z.number().nonnegative().optional().describe("USD per 1M reasoning tokens. The gateway uses the output price when omitted.")
|
|
37
|
+
}).describe("Price of this model in USD per 1M tokens. The gateway uses it instead of its built-in price, so a maxMonthlyUSD budget is spent on models it does not price (e.g. o4-mini). Omit to use the gateway's built-in price.");
|
|
19
38
|
var ModelFallbackSchema = z.object({
|
|
20
39
|
provider: z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
|
|
21
40
|
id: z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
|
|
22
41
|
apiKey: refOrLiteral.describe(refDesc),
|
|
42
|
+
baseUrl: baseUrlSchema,
|
|
23
43
|
triggerOn: z.array(z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
|
|
24
|
-
maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
|
|
44
|
+
maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover)."),
|
|
45
|
+
pricing: ModelPricingSchema.optional()
|
|
25
46
|
});
|
|
26
47
|
var ModelSchema = z.object({
|
|
27
48
|
provider: z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
|
|
28
49
|
id: z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
|
|
29
50
|
apiKey: refOrLiteral.describe(refDesc),
|
|
51
|
+
baseUrl: baseUrlSchema,
|
|
30
52
|
parameters: z.object({
|
|
31
53
|
temperature: z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
|
|
32
54
|
maxTokens: z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
|
|
@@ -35,6 +57,7 @@ var ModelSchema = z.object({
|
|
|
35
57
|
presencePenalty: z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
|
|
36
58
|
}).optional(),
|
|
37
59
|
fallback: ModelFallbackSchema.optional(),
|
|
60
|
+
pricing: ModelPricingSchema.optional(),
|
|
38
61
|
costControls: z.object({
|
|
39
62
|
maxMonthlyUSD: z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
|
|
40
63
|
alertAtUSD: z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
|
|
@@ -69,7 +92,12 @@ var McpServerSchema = z.object({
|
|
|
69
92
|
transport: z.enum(["stdio", "sse", "http"]).describe('"stdio" = spawn a local process (requires "command"), "sse" = Server-Sent Events over HTTP, "http" = streamable HTTP (requires "url").'),
|
|
70
93
|
command: z.string().optional().describe('Shell command to spawn the MCP server process. Required for transport "stdio" (e.g. "npx", "python").'),
|
|
71
94
|
args: z.array(z.string()).optional().describe('Arguments passed to the command (e.g. ["-m", "my_mcp_server"]). Only used for transport "stdio".'),
|
|
72
|
-
|
|
95
|
+
// refOrLiteral, not z.string().url(). This field's own description promises reference syntax,
|
|
96
|
+
// and `.url()` rejected every form of it — so an MCP server whose address comes from the
|
|
97
|
+
// environment (`$env:GITHUB_MCP_URL`, which is the normal case in a cluster, where the URL is
|
|
98
|
+
// a Service name that differs per namespace) could not be declared at all. Every sibling
|
|
99
|
+
// address field in this schema already uses refOrLiteral; this one was the outlier.
|
|
100
|
+
url: refOrLiteral.optional().describe('Base URL of the remote MCP server. Required for transport "sse" or "http". ' + refDesc),
|
|
73
101
|
env: z.record(z.string(), refOrLiteral).optional().describe('Environment variables injected into the MCP server process. Values support reference syntax. Use "$env:API_KEY" to forward secrets without hardcoding.'),
|
|
74
102
|
healthCheck: z.object({
|
|
75
103
|
timeoutSeconds: z.number().int().min(1).max(60).default(5).describe("Seconds to wait for the MCP server to respond during health checks. Increase for slow-starting processes.")
|
|
@@ -145,6 +173,7 @@ var ChatEndpointSchema = z.object({
|
|
|
145
173
|
var ApiSchema = z.object({
|
|
146
174
|
type: z.enum(["rest", "mcp", "grpc", "websocket"]),
|
|
147
175
|
port: z.number().int().min(1).max(65535).optional(),
|
|
176
|
+
allowedPorts: z.array(z.number().int().min(1).max(65535)).optional().describe("Extra ports the agent may listen on besides the sidecar-routed API port (e.g. a platform metrics port). The sidecar reports not-ready while the agent listens on any other non-loopback port."),
|
|
148
177
|
pathPrefix: z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
|
|
149
178
|
auth: z.object({
|
|
150
179
|
type: z.enum(["jwt", "apikey", "oauth2", "none"]),
|
|
@@ -153,7 +182,8 @@ var ApiSchema = z.object({
|
|
|
153
182
|
}).optional(),
|
|
154
183
|
rateLimit: z.object({
|
|
155
184
|
requestsPerMinute: z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
|
|
156
|
-
requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
|
|
185
|
+
requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429."),
|
|
186
|
+
tokensPerMinute: z.number().int().min(1).optional().describe("Maximum LLM tokens the gateway allows per minute. Once the bucket is empty, requests receive HTTP 429.")
|
|
157
187
|
}).optional(),
|
|
158
188
|
streaming: z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
|
|
159
189
|
healthEndpoint: z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
|
|
@@ -235,25 +265,42 @@ var EvalMetricEnum = z.enum([
|
|
|
235
265
|
// RAG: do the retrieved chunks cover the ground-truth answer?
|
|
236
266
|
"bias",
|
|
237
267
|
// Does the answer exhibit demographic or political bias?
|
|
268
|
+
"grounded",
|
|
269
|
+
// Are the answer's claims supported by the retrieved context? Replaces faithfulness and hallucination.
|
|
270
|
+
"pii_leakage",
|
|
271
|
+
// Does the answer leak personal data?
|
|
272
|
+
"prompt_injection",
|
|
273
|
+
// Did the answer follow an injected instruction?
|
|
238
274
|
"custom"
|
|
239
|
-
//
|
|
275
|
+
// Rubric-graded scorer, declared in the sibling `custom: { name, rubric }`.
|
|
240
276
|
]);
|
|
241
|
-
var
|
|
277
|
+
var LiveMetricEnum = EvalMetricEnum.exclude(["context_recall", "answer_similarity"]);
|
|
278
|
+
var CustomMetricSchema = z.object({
|
|
279
|
+
name: z.string().min(1).describe("Name the custom metric is reported under."),
|
|
280
|
+
rubric: refOrLiteral.describe('Grading rubric. Use "$file:rubrics/<name>.md"; the loader resolves it to the file contents.')
|
|
281
|
+
});
|
|
282
|
+
var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for grounded and context_precision metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
|
|
242
283
|
var EvaluationSchema = z.object({
|
|
243
|
-
framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
|
|
284
|
+
framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]).optional().describe("Deprecated: nothing reads it. validate warns when set; rejected in 0.6.0."),
|
|
285
|
+
live: z.object({
|
|
286
|
+
metrics: z.array(LiveMetricEnum).optional().describe("Metrics scored on live traffic."),
|
|
287
|
+
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe("Minimum pass rate per live metric (0.0\u20131.0)."),
|
|
288
|
+
custom: CustomMetricSchema.optional()
|
|
289
|
+
}).optional().describe("Live evaluation, separate from dataset evaluation. Only live metrics are allowed."),
|
|
290
|
+
custom: CustomMetricSchema.optional().describe('Rubric for the "custom" metric on datasets.'),
|
|
244
291
|
datasets: z.array(
|
|
245
292
|
z.object({
|
|
246
293
|
name: z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
|
|
247
294
|
path: refOrLiteral.describe(datasetPathDesc),
|
|
248
295
|
metrics: z.array(EvalMetricEnum).optional().describe(
|
|
249
|
-
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["
|
|
296
|
+
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["grounded","context_recall"] while a safety dataset sets ["toxicity","bias"].'
|
|
250
297
|
)
|
|
251
298
|
})
|
|
252
299
|
).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
|
|
253
300
|
metrics: z.array(EvalMetricEnum).optional().describe(
|
|
254
|
-
'Default metrics to compute across all datasets that do not declare their own metrics. "
|
|
301
|
+
'Default metrics to compute across all datasets that do not declare their own metrics. "grounded", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires spec.evaluation.custom: { name, rubric }. "faithfulness" and "hallucination" are deprecated aliases of "grounded".'
|
|
255
302
|
),
|
|
256
|
-
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"
|
|
303
|
+
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"grounded": 0.95, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
|
|
257
304
|
ciGate: z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
|
|
258
305
|
});
|
|
259
306
|
var ObservabilitySchema = z.object({
|
|
@@ -352,12 +399,608 @@ var ManifestSchema = z.object({
|
|
|
352
399
|
metadata: MetadataSchema,
|
|
353
400
|
spec: SpecSchema
|
|
354
401
|
});
|
|
402
|
+
var DRAFT_07 = "http://json-schema.org/draft-07/schema#";
|
|
403
|
+
var options = {
|
|
404
|
+
target: "draft-7",
|
|
405
|
+
io: "input",
|
|
406
|
+
reused: "inline",
|
|
407
|
+
override: ({ zodSchema, jsonSchema }) => {
|
|
408
|
+
const def = zodSchema._zod.def;
|
|
409
|
+
if (def.type === "object" && jsonSchema.additionalProperties === void 0) jsonSchema.additionalProperties = false;
|
|
410
|
+
if (def.type === "record") delete jsonSchema.propertyNames;
|
|
411
|
+
if (def.type === "union" && def.discriminator && jsonSchema.oneOf) {
|
|
412
|
+
jsonSchema.anyOf = jsonSchema.oneOf;
|
|
413
|
+
delete jsonSchema.oneOf;
|
|
414
|
+
}
|
|
415
|
+
if (jsonSchema.type === "integer") {
|
|
416
|
+
if (jsonSchema.maximum === Number.MAX_SAFE_INTEGER) delete jsonSchema.maximum;
|
|
417
|
+
if (jsonSchema.minimum === Number.MIN_SAFE_INTEGER) delete jsonSchema.minimum;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
function toJsonSchema(schema) {
|
|
422
|
+
const { $schema: _, ...body } = z.toJSONSchema(schema, options);
|
|
423
|
+
return { ...body, $schema: DRAFT_07 };
|
|
424
|
+
}
|
|
425
|
+
function toNamedJsonSchema(name, schema) {
|
|
426
|
+
const { $schema, ...body } = toJsonSchema(schema);
|
|
427
|
+
return { $ref: `#/definitions/${name}`, definitions: { [name]: body }, $schema };
|
|
428
|
+
}
|
|
355
429
|
function exportJsonSchema() {
|
|
356
|
-
return
|
|
357
|
-
|
|
358
|
-
|
|
430
|
+
return toNamedJsonSchema("AgentSpecManifest", ManifestSchema);
|
|
431
|
+
}
|
|
432
|
+
var EvidenceClass = z.enum(["independent", "evidence-point", "offline", "declared"]);
|
|
433
|
+
var Collector = z.enum([
|
|
434
|
+
"gateway-config",
|
|
435
|
+
"gateway-traffic",
|
|
436
|
+
"sidecar-probe",
|
|
437
|
+
"sidecar-forwarder",
|
|
438
|
+
"sidecar-eval",
|
|
439
|
+
"operator-webhook",
|
|
440
|
+
"operator-logs",
|
|
441
|
+
"sdk",
|
|
442
|
+
"lock",
|
|
443
|
+
"manifest"
|
|
444
|
+
]);
|
|
445
|
+
var EvidencePoint = z.enum(["guardrail", "tool", "mcpClient", "memoryWrite", "vectorStore", "logging"]);
|
|
446
|
+
var Scope = z.enum(["declared", "configured", "enforced", "observed", "ran-where-wired", "every-path", "effective"]);
|
|
447
|
+
var WiredBy = z.enum(["none", "operator-render", "agent-code", "user-schedule", "opt-in"]);
|
|
448
|
+
var Mode = z.enum(["standalone", "operator"]);
|
|
449
|
+
var BuildStatus = z.enum(["built", "partial", "planned", "open"]);
|
|
450
|
+
var RuleEvidenceSource = z.object({
|
|
451
|
+
class: EvidenceClass,
|
|
452
|
+
collector: Collector,
|
|
453
|
+
/** Evidence-point sources only. */
|
|
454
|
+
point: EvidencePoint.nullish(),
|
|
455
|
+
proves: Scope,
|
|
456
|
+
wiredBy: WiredBy,
|
|
457
|
+
modes: z.array(Mode).min(1),
|
|
458
|
+
status: BuildStatus,
|
|
459
|
+
/** A variant such as HTTP MCP, stdio MCP, local tools or opt-in. */
|
|
460
|
+
when: z.string().min(1).nullish(),
|
|
461
|
+
/** A collector that checks the counts, such as gateway calls against guardrail records. */
|
|
462
|
+
crossCheck: Collector.nullish(),
|
|
463
|
+
/** The `agentspec lock` prompt question. */
|
|
464
|
+
question: z.string().min(1).nullish()
|
|
465
|
+
}).strict().refine((s) => s.class === "evidence-point" === (s.point != null), {
|
|
466
|
+
message: "an evidence-point source names its `point`, and no other class does"
|
|
467
|
+
});
|
|
468
|
+
var RuleEvidenceEntry = z.object({
|
|
469
|
+
sources: z.array(RuleEvidenceSource).default([]),
|
|
470
|
+
/** Rules whose verdicts decide this one (all must pass); this rule inherits their sources. */
|
|
471
|
+
derivedFrom: z.array(z.string().min(1)).nullish(),
|
|
472
|
+
/** What a pass does not prove, shown with the result. Free text. */
|
|
473
|
+
notProven: z.string().nullish()
|
|
474
|
+
}).strict().refine((e) => e.sources.length > 0 || (e.derivedFrom?.length ?? 0) > 0, {
|
|
475
|
+
message: "an entry needs at least one source or a `derivedFrom`"
|
|
476
|
+
});
|
|
477
|
+
var BOTH = ["standalone", "operator"];
|
|
478
|
+
var CLASS_DEFAULTS = {
|
|
479
|
+
independent: { wiredBy: "none", status: "built" },
|
|
480
|
+
"evidence-point": { wiredBy: "agent-code", status: "planned" },
|
|
481
|
+
offline: { wiredBy: "user-schedule", status: "planned" },
|
|
482
|
+
declared: { wiredBy: "none", status: "built" }
|
|
483
|
+
};
|
|
484
|
+
var source = (cls, collector, proves, extra = {}) => ({ class: cls, collector, proves, modes: BOTH, ...CLASS_DEFAULTS[cls], ...extra });
|
|
485
|
+
var independent = (collector, proves, extra) => source("independent", collector, proves, extra);
|
|
486
|
+
var point = (p, proves, extra) => source("evidence-point", "sdk", proves, { point: p, ...extra });
|
|
487
|
+
var lock = (question, extra) => source("offline", "lock", "every-path", { question, ...extra });
|
|
488
|
+
var declared = (extra) => source("declared", "manifest", "declared", extra);
|
|
489
|
+
var OPERATOR_ONLY = { modes: ["operator"] };
|
|
490
|
+
var PLANNED = { status: "planned" };
|
|
491
|
+
var WRITES_BYPASS = "Writes that bypass the evidence point at runtime.";
|
|
492
|
+
var entries = {
|
|
493
|
+
"MODEL-01": { sources: [independent("gateway-config", "configured"), independent("gateway-traffic", "observed")] },
|
|
494
|
+
"MODEL-02": {
|
|
495
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
496
|
+
},
|
|
497
|
+
"MODEL-03": { sources: [independent("gateway-config", "enforced")] },
|
|
498
|
+
"MODEL-04": {
|
|
499
|
+
sources: [
|
|
500
|
+
independent("gateway-config", "enforced"),
|
|
501
|
+
lock("Do the agent's own provider clients retry more than spec.model.fallback.maxRetries, or stack retries on the gateway's?", { when: "client retries" })
|
|
502
|
+
],
|
|
503
|
+
notProven: "That the gateway's attempts match `spec.model.fallback.maxRetries` at runtime beyond the rendered config."
|
|
504
|
+
},
|
|
505
|
+
"MODEL-05": {
|
|
506
|
+
sources: [independent("gateway-traffic", "observed", { when: "agw.ai.usage.cost.total" })],
|
|
507
|
+
notProven: "Requests with no token usage: not counted."
|
|
508
|
+
},
|
|
509
|
+
"SEC-LLM-01": {
|
|
510
|
+
sources: [
|
|
511
|
+
independent("sidecar-probe", "effective", { when: "opt-in", wiredBy: "opt-in", status: "planned" }),
|
|
512
|
+
point("guardrail", "ran-where-wired"),
|
|
513
|
+
lock("Does every path that sends user input to a model run the declared input guardrails first?")
|
|
514
|
+
]
|
|
515
|
+
},
|
|
516
|
+
"SEC-LLM-02": {
|
|
517
|
+
sources: [
|
|
518
|
+
point("guardrail", "ran-where-wired"),
|
|
519
|
+
lock("Does every path that returns model output run the declared output guardrails first?")
|
|
520
|
+
],
|
|
521
|
+
notProven: "Effective: a no-op output guardrail still fires."
|
|
522
|
+
},
|
|
523
|
+
"SEC-LLM-03": { sources: [declared()] },
|
|
524
|
+
"SEC-LLM-04": {
|
|
525
|
+
sources: [independent("gateway-config", "enforced"), independent("gateway-traffic", "observed", PLANNED)],
|
|
526
|
+
notProven: "Cost: MODEL-03 covers it."
|
|
527
|
+
},
|
|
528
|
+
"SEC-LLM-05": {
|
|
529
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
530
|
+
},
|
|
531
|
+
"SEC-LLM-06": { derivedFrom: ["MEM-01"], notProven: "Same as MEM-01." },
|
|
532
|
+
"SEC-LLM-07": {
|
|
533
|
+
sources: [
|
|
534
|
+
independent("sidecar-probe", "configured", { when: "HTTP MCP" }),
|
|
535
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
536
|
+
lock("Does the agent call any MCP tool the manifest does not declare, or one that serves no annotations?"),
|
|
537
|
+
declared({ when: "local tools" })
|
|
538
|
+
]
|
|
539
|
+
},
|
|
540
|
+
"SEC-LLM-08": {
|
|
541
|
+
sources: [
|
|
542
|
+
independent("sidecar-probe", "configured", PLANNED),
|
|
543
|
+
independent("gateway-traffic", "enforced", PLANNED),
|
|
544
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" })
|
|
545
|
+
],
|
|
546
|
+
notProven: "Which tools are destructive: the server declares it; nothing checks the claim."
|
|
547
|
+
},
|
|
548
|
+
"SEC-LLM-09": { derivedFrom: ["EVAL-01", "EVAL-02"] },
|
|
549
|
+
"SEC-LLM-10": {
|
|
550
|
+
sources: [
|
|
551
|
+
independent("operator-webhook", "configured", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
552
|
+
declared({ modes: ["standalone"] })
|
|
553
|
+
]
|
|
554
|
+
},
|
|
555
|
+
"TOOL-01": {
|
|
556
|
+
sources: [
|
|
557
|
+
independent("gateway-traffic", "observed", { when: "HTTP MCP" }),
|
|
558
|
+
point("tool", "ran-where-wired", { when: "local tools" }),
|
|
559
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
560
|
+
lock("Which tools does the agent code register or call, local, HTTP MCP and stdio MCP?")
|
|
561
|
+
],
|
|
562
|
+
notProven: "Unwrapped local or stdio calls at runtime: only the lock inventory catches them."
|
|
563
|
+
},
|
|
564
|
+
"MEM-01": {
|
|
565
|
+
sources: [
|
|
566
|
+
point("memoryWrite", "enforced"),
|
|
567
|
+
lock("Does every memory write pass through the PII scrubber for the declared fields?")
|
|
568
|
+
],
|
|
569
|
+
notProven: "Full coverage: the scrubber is a regex baseline (Presidio optional in Python). " + WRITES_BYPASS
|
|
570
|
+
},
|
|
571
|
+
"MEM-02": {
|
|
572
|
+
sources: [
|
|
573
|
+
point("memoryWrite", "enforced"),
|
|
574
|
+
lock("Does every memory write apply the declared retention TTL?")
|
|
575
|
+
],
|
|
576
|
+
notProven: WRITES_BYPASS + " Backends that cannot apply a TTL (record `ttl: unsupported`): the evidence point is `not_verifiable` there and only the lock remains."
|
|
577
|
+
},
|
|
578
|
+
"MEM-03": {
|
|
579
|
+
sources: [
|
|
580
|
+
point("memoryWrite", "enforced"),
|
|
581
|
+
lock("Does every memory write honour the declared memory audit setting?")
|
|
582
|
+
],
|
|
583
|
+
notProven: WRITES_BYPASS
|
|
584
|
+
},
|
|
585
|
+
"MEM-04": {
|
|
586
|
+
sources: [
|
|
587
|
+
point("vectorStore", "ran-where-wired"),
|
|
588
|
+
lock("Does every vector store query and write use the declared namespace or tenant isolation?")
|
|
589
|
+
],
|
|
590
|
+
notProven: "Isolation: the right namespace does not prove other tenants cannot read it."
|
|
591
|
+
},
|
|
592
|
+
"MEM-05": {
|
|
593
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.usage.output_tokens" }), declared({ when: "until a call is observed" })],
|
|
594
|
+
notProven: "The tolerance for the system prompt and tool schemas is set in Phase 2, with an ADR-0015 amendment; a call without gen_ai.usage.output_tokens leaves the check not_verifiable."
|
|
595
|
+
},
|
|
596
|
+
"EVAL-01": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
597
|
+
"EVAL-02": {
|
|
598
|
+
sources: [
|
|
599
|
+
independent("sidecar-eval", "observed", PLANNED),
|
|
600
|
+
lock("Does the agent's CI run the declared evaluation gate before deploy?")
|
|
601
|
+
],
|
|
602
|
+
notProven: "That a failed gate actually blocks a deploy: CI is outside the system."
|
|
603
|
+
},
|
|
604
|
+
"EVAL-03": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
605
|
+
"OBS-01": {
|
|
606
|
+
sources: [independent("sidecar-forwarder", "observed"), independent("sidecar-probe", "configured")],
|
|
607
|
+
notProven: "That the agent exports every span: the rule is about the backend, not span coverage. agentops is not a target: it has no OTLP ingest."
|
|
608
|
+
},
|
|
609
|
+
"OBS-02": {
|
|
610
|
+
sources: [
|
|
611
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
612
|
+
point("logging", "ran-where-wired"),
|
|
613
|
+
lock("Does every log statement emit structured JSON?")
|
|
614
|
+
]
|
|
615
|
+
},
|
|
616
|
+
"OBS-03": {
|
|
617
|
+
sources: [
|
|
618
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
619
|
+
lock("Does any log statement write one of the declared sensitive fields?")
|
|
620
|
+
],
|
|
621
|
+
notProven: "Full coverage: the scanner is a regex and pattern baseline, so recall is limited."
|
|
622
|
+
},
|
|
623
|
+
"META-01": { sources: [declared()] },
|
|
624
|
+
"META-02": {
|
|
625
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.request.temperature" })],
|
|
626
|
+
notProven: "Requests that set no temperature: `not_verifiable`, not counted."
|
|
627
|
+
}
|
|
628
|
+
};
|
|
629
|
+
var RULE_EVIDENCE = Object.fromEntries(
|
|
630
|
+
Object.entries(entries).map(([id, entry]) => [id, RuleEvidenceEntry.parse(entry)])
|
|
631
|
+
);
|
|
632
|
+
function ruleSources(ruleId, registry = RULE_EVIDENCE, seen = /* @__PURE__ */ new Set()) {
|
|
633
|
+
if (!Object.hasOwn(registry, ruleId) || seen.has(ruleId)) return [];
|
|
634
|
+
const entry = registry[ruleId];
|
|
635
|
+
const visited = /* @__PURE__ */ new Set([...seen, ruleId]);
|
|
636
|
+
return [...entry.sources, ...(entry.derivedFrom ?? []).flatMap((parent) => ruleSources(parent, registry, visited))];
|
|
637
|
+
}
|
|
638
|
+
function ruleClasses(ruleId, registry = RULE_EVIDENCE) {
|
|
639
|
+
const classes = new Set(ruleSources(ruleId, registry).map((s) => s.class));
|
|
640
|
+
return EvidenceClass.options.filter((c) => classes.has(c));
|
|
641
|
+
}
|
|
642
|
+
var EvidenceStatus = z.enum(["pass", "fail", "not_verifiable"]);
|
|
643
|
+
var EvidenceSource = z.enum(["independent", "sdk-observed", "lower-bound", "self-declared", "declared", "offline"]);
|
|
644
|
+
var EvidenceOrigin = z.enum(["ingress", "gateway", "egress_proxy", "in_process", "probe", "manifest", "judge"]);
|
|
645
|
+
var SeveritySchema = z.enum(["critical", "high", "medium", "low", "info"]);
|
|
646
|
+
var TrustPolicy = z.enum(["standard", "strict"]);
|
|
647
|
+
var EvidenceReason = z.object({ code: z.string().min(1), detail: z.string().nullish() }).strict();
|
|
648
|
+
var TIMESTAMP_PATTERN = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
|
|
649
|
+
var Timestamp = z.string().regex(TIMESTAMP_PATTERN);
|
|
650
|
+
var EvidenceWindow = z.object({ start: Timestamp, end: Timestamp }).strict();
|
|
651
|
+
var Sha256 = z.string().regex(/^[a-f0-9]{64}$/);
|
|
652
|
+
var MAX_TRACE_IDS = 20;
|
|
653
|
+
var recordFields = {
|
|
654
|
+
/** Scoped per subject: `tool:<name>`, `model:<id>`, or a bare check name for agent-wide checks. */
|
|
655
|
+
checkId: z.string().min(1),
|
|
656
|
+
ruleId: z.string().min(1).nullish(),
|
|
657
|
+
source: EvidenceSource,
|
|
658
|
+
origin: EvidenceOrigin,
|
|
659
|
+
/** The registry collector that produced the record. Optional until every producer sets it (#422); checking it against the collectors the registry lists for the rule is the grader's job (#423). */
|
|
660
|
+
collector: Collector.nullish(),
|
|
661
|
+
observedAt: Timestamp,
|
|
662
|
+
window: EvidenceWindow,
|
|
663
|
+
ttlSeconds: z.number().int().nonnegative(),
|
|
664
|
+
sampleSize: z.number().int().nonnegative(),
|
|
665
|
+
traceIds: z.array(z.string().min(1)).max(MAX_TRACE_IDS),
|
|
666
|
+
manifestSha256: Sha256
|
|
667
|
+
};
|
|
668
|
+
var EvidenceRecordPass = z.object({ ...recordFields, status: z.literal("pass"), reason: EvidenceReason.nullish() }).strict();
|
|
669
|
+
var EvidenceRecordNotPass = z.object({
|
|
670
|
+
...recordFields,
|
|
671
|
+
status: z.enum(["fail", "not_verifiable"]),
|
|
672
|
+
reason: EvidenceReason
|
|
673
|
+
}).strict();
|
|
674
|
+
var EvidenceRecord = z.union([EvidenceRecordPass, EvidenceRecordNotPass]);
|
|
675
|
+
var PodspecFinding = z.object({ code: z.string().min(1), subject: z.string().min(1), message: z.string().min(1) }).strict();
|
|
676
|
+
var ReadinessFinding = z.object({
|
|
677
|
+
code: z.string().min(1),
|
|
678
|
+
severity: SeveritySchema,
|
|
679
|
+
message: z.string().min(1),
|
|
680
|
+
subject: z.string().nullish(),
|
|
681
|
+
ruleIds: z.array(z.string().min(1)).nullish()
|
|
682
|
+
}).strict();
|
|
683
|
+
var EvalResult = z.object({
|
|
684
|
+
metric: z.string().min(1),
|
|
685
|
+
n: z.number().int().nonnegative(),
|
|
686
|
+
passRate: z.number().min(0).max(1),
|
|
687
|
+
ciLow: z.number().min(0).max(1),
|
|
688
|
+
ciHigh: z.number().min(0).max(1),
|
|
689
|
+
threshold: z.number().min(0).max(1),
|
|
690
|
+
window: EvidenceWindow,
|
|
691
|
+
judgeModel: z.string().min(1),
|
|
692
|
+
rubricHash: Sha256,
|
|
693
|
+
source: EvidenceSource,
|
|
694
|
+
dataset: z.string().min(1).nullish()
|
|
695
|
+
}).strict();
|
|
696
|
+
var MAX_POINT_RECORDS = 5;
|
|
697
|
+
var EvidencePointState = z.enum(["fired", "wired", "not_wired"]);
|
|
698
|
+
var EvidencePointAggregate = z.object({
|
|
699
|
+
point: EvidencePoint,
|
|
700
|
+
/** What the point is wired on, a key of `CONTRACT.evidencePoints[point]`. */
|
|
701
|
+
target: z.string().min(1),
|
|
702
|
+
state: EvidencePointState,
|
|
703
|
+
counts: z.object({
|
|
704
|
+
pass: z.number().int().nonnegative(),
|
|
705
|
+
fail: z.number().int().nonnegative(),
|
|
706
|
+
not_verifiable: z.number().int().nonnegative()
|
|
707
|
+
}).strict(),
|
|
708
|
+
records: z.array(EvidenceRecord).max(MAX_POINT_RECORDS)
|
|
709
|
+
}).strict();
|
|
710
|
+
var Passthrough = z.record(z.string(), z.unknown());
|
|
711
|
+
var Heartbeat = z.object({
|
|
712
|
+
/** Contract version of the sending SDK (`CONTRACT.version`); absent or null from older SDKs. */
|
|
713
|
+
contract: z.string().regex(/^\d+\.\d+\.\d+$/).nullish(),
|
|
714
|
+
health: Passthrough,
|
|
715
|
+
records: z.array(EvidenceRecord).default([]),
|
|
716
|
+
/** Every evidence point the registry gives for the manifest, wired or not; absent from older SDKs. */
|
|
717
|
+
evidencePoints: z.array(EvidencePointAggregate).default([]),
|
|
718
|
+
usage: Passthrough.nullish(),
|
|
719
|
+
tools: Passthrough.nullish()
|
|
720
|
+
}).strict();
|
|
721
|
+
var SidecarPush = z.object({
|
|
722
|
+
manifestSha256: Sha256,
|
|
723
|
+
declared: Passthrough.nullish(),
|
|
724
|
+
records: z.array(EvidenceRecord).default([])
|
|
725
|
+
}).strict();
|
|
726
|
+
var EVIDENCE_SHAPES = { EvidenceRecord, ReadinessFinding, EvalResult, TrustPolicy, Heartbeat, SidecarPush };
|
|
727
|
+
|
|
728
|
+
// src/schema/contract.schema.ts
|
|
729
|
+
var fixed = (value) => z.literal(value).default(value);
|
|
730
|
+
var REASON_CODES = [
|
|
731
|
+
"stale",
|
|
732
|
+
"source_not_admitted",
|
|
733
|
+
"source_mismatch",
|
|
734
|
+
"no_evidence",
|
|
735
|
+
"not_wired",
|
|
736
|
+
"no_calls_observed",
|
|
737
|
+
"no_failover_observed",
|
|
738
|
+
"no_budget_evidence",
|
|
739
|
+
"no_approval_token_evidence",
|
|
740
|
+
"no_retrieval_spans",
|
|
741
|
+
"unrecognised_conventions",
|
|
742
|
+
"instrumentation_unavailable",
|
|
743
|
+
"declared_env_missing",
|
|
744
|
+
"provider_key_on_agent",
|
|
745
|
+
"response_cap_exceeded",
|
|
746
|
+
"temperature_above_max",
|
|
747
|
+
"undeclared_tool",
|
|
748
|
+
"undeclared_model",
|
|
749
|
+
"unpinned_model",
|
|
750
|
+
"self_declared",
|
|
751
|
+
"unpriced_model",
|
|
752
|
+
"undeclared_namespace"
|
|
753
|
+
];
|
|
754
|
+
var ContractSchema = z.object({
|
|
755
|
+
/** Contract version (major.minor.patch) a heartbeat carries: a receiver rejects another major and accepts a newer minor (ADR-0039). */
|
|
756
|
+
version: fixed("1.0.0"),
|
|
757
|
+
env: z.object({
|
|
758
|
+
manifestPath: fixed("MANIFEST_PATH"),
|
|
759
|
+
upstreamUrl: fixed("UPSTREAM_URL"),
|
|
760
|
+
proxyPort: fixed("PROXY_PORT"),
|
|
761
|
+
controlPlanePort: fixed("CONTROL_PLANE_PORT"),
|
|
762
|
+
/** Control plane URL the operator webhook injects into the sidecar and agent containers; SDK push mode falls back to it. */
|
|
763
|
+
controlPlaneUrl: fixed("AGENTSPEC_CONTROL_PLANE_URL"),
|
|
764
|
+
/** Agent JWT from `POST /api/v1/register`; `agentspec register` sends it to the control plane as a Bearer token. The sidecar's own push needs the `sidecarToken` from the same call (see `agentTokenSecretSuffix`). */
|
|
765
|
+
agentKey: fixed("AGENTSPEC_API_KEY"),
|
|
766
|
+
/** Trust policy (`standard | strict`) the grader applies: set on the control plane and the sidecar, never in `agent.yaml` (ADR-0007). */
|
|
767
|
+
trustPolicy: fixed("AGENTSPEC_TRUST_POLICY"),
|
|
768
|
+
/** The webhook's PodSpec findings (the `agentspec.io/podspec-findings` annotation, JSON), injected into the sidecar by the downward API (ADR-0019). */
|
|
769
|
+
podspecFindings: fixed("AGENTSPEC_PODSPEC_FINDINGS"),
|
|
770
|
+
/** Virtual key the agent sends and the gateway accepts (ADR-0001); provider keys stay in the gateway. */
|
|
771
|
+
gatewayKey: fixed("AGENTSPEC_GATEWAY_KEY"),
|
|
772
|
+
/** Bind address and port the agent must honour so the sidecar can own `spec.api.port` (ADR-0028). */
|
|
773
|
+
agentHost: fixed("HOST"),
|
|
774
|
+
agentPort: fixed("PORT"),
|
|
775
|
+
/** Azure resource name the gateway config references (`$AZURE_OPENAI_RESOURCE_NAME`). */
|
|
776
|
+
azureResourceName: fixed("AZURE_OPENAI_RESOURCE_NAME")
|
|
777
|
+
}).strict().prefault({}),
|
|
778
|
+
/** Provider base-URL env vars, keyed by spec.model.provider. */
|
|
779
|
+
providerBaseUrlEnv: z.object({
|
|
780
|
+
openai: fixed("OPENAI_BASE_URL"),
|
|
781
|
+
anthropic: fixed("ANTHROPIC_BASE_URL")
|
|
782
|
+
}).strict().prefault({}),
|
|
783
|
+
paths: z.object({
|
|
784
|
+
manifestDir: fixed("/manifest"),
|
|
785
|
+
manifestFile: fixed("/manifest/agent.yaml"),
|
|
786
|
+
/** SQLite file the gateway keeps USD budget usage in; must be on a writable volume. */
|
|
787
|
+
gatewayDatabase: fixed("/gateway-data/agentgateway.db"),
|
|
788
|
+
/** Directory the init container renders the gateway config into (ADR-0025). */
|
|
789
|
+
gatewayConfigDir: fixed("/gateway"),
|
|
790
|
+
gatewayConfigFile: fixed("/gateway/config.yaml"),
|
|
791
|
+
/** Sidecar bundle inside its image; `node <entrypoint> gateway-config …` is the init container's command (ADR-0025). */
|
|
792
|
+
sidecarEntrypoint: fixed("dist/index.js"),
|
|
793
|
+
gatewayDataDir: fixed("/gateway-data")
|
|
794
|
+
}).strict().prefault({}),
|
|
795
|
+
images: z.object({
|
|
796
|
+
sidecar: fixed("ghcr.io/agents-oss/agentspec-sidecar:latest"),
|
|
797
|
+
/** agentgateway, pinned (ADR-0025). */
|
|
798
|
+
gateway: fixed("cr.agentgateway.dev/agentgateway:v1.6.0")
|
|
799
|
+
}).strict().prefault({}),
|
|
800
|
+
ports: z.object({
|
|
801
|
+
agentDefault: fixed(8e3),
|
|
802
|
+
/** Loopback port the agent binds once the sidecar takes over its declared port (ADR-0028). */
|
|
803
|
+
agentBind: fixed(18e3),
|
|
804
|
+
proxy: fixed(4e3),
|
|
805
|
+
controlPlane: fixed(4001),
|
|
806
|
+
/** agentgateway LLM listener on the pod loopback (ADR-0025). */
|
|
807
|
+
gateway: fixed(4002),
|
|
808
|
+
/** agentgateway's default stats and readiness listeners: wildcard-bound, not the agent's (ADR-0025). */
|
|
809
|
+
gatewayStats: fixed(15020),
|
|
810
|
+
gatewayReadiness: fixed(15021)
|
|
811
|
+
}).strict().prefault({}),
|
|
812
|
+
/** Bare paths served by the sidecar on the control-plane port (ADR-0023). */
|
|
813
|
+
sidecarPaths: z.object({
|
|
814
|
+
healthReady: fixed("/health/ready"),
|
|
815
|
+
gap: fixed("/gap"),
|
|
816
|
+
explore: fixed("/explore"),
|
|
817
|
+
proof: fixed("/proof"),
|
|
818
|
+
traces: fixed("/v1/traces"),
|
|
819
|
+
/** Read-only GET of the buffered, already redacted spans the gateway copied to the sidecar. */
|
|
820
|
+
spans: fixed("/spans")
|
|
821
|
+
}).strict().prefault({}),
|
|
822
|
+
/** Bare paths an agent serves itself (SDK `/agentspec/health`); the sidecar probes them on the upstream. */
|
|
823
|
+
agentPaths: z.object({
|
|
824
|
+
health: fixed("/agentspec/health")
|
|
825
|
+
}).strict().prefault({}),
|
|
826
|
+
/** Control plane API routes shared by the CLI, the sidecar push and the control plane; `{name}` is the registered agent name. */
|
|
827
|
+
controlPlanePaths: z.object({
|
|
828
|
+
/** POST: registers an agent and returns its per-agent token. */
|
|
829
|
+
register: fixed("/api/v1/register"),
|
|
830
|
+
/** POST: the SDK push-mode heartbeat, authenticated with the agent token from `register`. */
|
|
831
|
+
heartbeat: fixed("/api/v1/heartbeat"),
|
|
832
|
+
declaredAudit: fixed("/api/v1/agents/{name}/declared-audit"),
|
|
833
|
+
/** POST: the sidecar's declared audit (at start and on a manifest hash change) and its own evidence records (ADR-0008). */
|
|
834
|
+
sidecarPush: fixed("/api/v1/agents/{name}/sidecar-push"),
|
|
835
|
+
/** GET: the latest stored EvidenceRecord per (ruleId, checkId, source) of an agent, as `{ records }`; read by the mcp-server. */
|
|
836
|
+
evidence: fixed("/api/v1/agents/{name}/evidence")
|
|
837
|
+
}).strict().prefault({}),
|
|
838
|
+
/** Names inside the pod patch that every injection path must use (ADR-0028). */
|
|
839
|
+
podPatch: z.object({
|
|
840
|
+
sidecarContainer: fixed("agentspec-sidecar"),
|
|
841
|
+
/** Pod annotation the webhook writes its PodSpec findings to (JSON list of `{code, subject, message}`). */
|
|
842
|
+
podspecFindingsAnnotation: fixed("agentspec.io/podspec-findings"),
|
|
843
|
+
/** Pod annotation naming the agent; the webhook reads the manifest ConfigMap `<name>-manifest` from it, and `--deploy k8s` writes it. */
|
|
844
|
+
agentNameAnnotation: fixed("agentspec.io/agent-name"),
|
|
845
|
+
manifestVolume: fixed("agentspec-manifest"),
|
|
846
|
+
proxyPortName: fixed("proxy"),
|
|
847
|
+
controlPortName: fixed("control"),
|
|
848
|
+
gatewayContainer: fixed("agentspec-gateway"),
|
|
849
|
+
gatewayConfigInitContainer: fixed("agentspec-gateway-config"),
|
|
850
|
+
/** Sidecar subcommand the init container runs: `node <entrypoint> <this> <manifest> --out <path>`. */
|
|
851
|
+
gatewayConfigCommand: fixed("gateway-config"),
|
|
852
|
+
gatewayConfigVolume: fixed("agentspec-gateway-config"),
|
|
853
|
+
gatewayDataVolume: fixed("agentspec-gateway-data"),
|
|
854
|
+
/** Secret `<metadata.name><suffix>` holding the `sidecarToken` from `POST /api/v1/register`; the webhook gives it to the sidecar only, as `env.agentKey`, when a control plane URL is set. */
|
|
855
|
+
agentTokenSecretSuffix: fixed("-agentspec-token"),
|
|
856
|
+
agentTokenSecretKey: fixed("token"),
|
|
857
|
+
/** Loopback address the agent and sidecar reach each other and the gateway on. */
|
|
858
|
+
loopback: fixed("127.0.0.1"),
|
|
859
|
+
/** Gateway container sizing (ADR-0025): request 25m/32Mi, limit 128Mi. */
|
|
860
|
+
gatewayCpuRequest: fixed("25m"),
|
|
861
|
+
gatewayMemoryRequest: fixed("32Mi"),
|
|
862
|
+
gatewayMemoryLimit: fixed("128Mi")
|
|
863
|
+
}).strict().prefault({}),
|
|
864
|
+
/** Admission webhook the operator serves (ADR-0028). */
|
|
865
|
+
webhook: z.object({
|
|
866
|
+
/** Path kube-apiserver posts to; kopf routes `POST /<handler id>`, so this is the handler id with a leading slash. Hyphenated: kube-apiserver rejects an underscore in a service path segment (RFC 1123). */
|
|
867
|
+
injectPath: fixed("/inject-sidecar"),
|
|
868
|
+
/** Namespace label that selects the namespaces whose pods are injected. */
|
|
869
|
+
namespaceLabel: fixed("agentspec.io/enabled")
|
|
870
|
+
}).strict().prefault({}),
|
|
871
|
+
/** Known `EvidenceRecord.reason.code` values (ADR-0006). */
|
|
872
|
+
reasonCodes: z.array(z.string()).default(REASON_CODES),
|
|
873
|
+
/** Severity weights for coverage and the verified score (ADR-0007). */
|
|
874
|
+
severityWeights: z.object({
|
|
875
|
+
critical: fixed(4),
|
|
876
|
+
high: fixed(3),
|
|
877
|
+
medium: fixed(2),
|
|
878
|
+
low: fixed(1),
|
|
879
|
+
info: fixed(0)
|
|
880
|
+
}).strict().prefault({}),
|
|
881
|
+
/** Minimum verified percentage per grade; below `D` is F (ADR-0007). */
|
|
882
|
+
gradeBands: z.object({ A: fixed(90), B: fixed(75), C: fixed(60), D: fixed(45) }).strict().prefault({}),
|
|
883
|
+
/** Severity-weighted coverage percentage below which no grade is shown (ADR-0007). */
|
|
884
|
+
coverageThreshold: fixed(60),
|
|
885
|
+
/** Rule evidence registry: what can prove each audit rule (ADR-0040). */
|
|
886
|
+
ruleEvidence: z.record(z.string(), RuleEvidenceEntry).default(RULE_EVIDENCE),
|
|
887
|
+
/** Evidence point → target → the rule its records serve; each rule's registry entry lists a source for that point. */
|
|
888
|
+
evidencePoints: z.record(z.string(), z.record(z.string(), z.string())).default({
|
|
889
|
+
guardrail: { input: "SEC-LLM-01", output: "SEC-LLM-02" },
|
|
890
|
+
tool: { local: "TOOL-01" },
|
|
891
|
+
mcpClient: { "tools/list": "SEC-LLM-07", "tools/call": "TOOL-01" },
|
|
892
|
+
memoryWrite: { scrub: "MEM-01", ttl: "MEM-02", audit: "MEM-03" },
|
|
893
|
+
vectorStore: { namespace: "MEM-04" },
|
|
894
|
+
logging: { json: "OBS-02" }
|
|
895
|
+
}),
|
|
896
|
+
/** Records an evidence point aggregate keeps per window. */
|
|
897
|
+
evidencePointRecordLimit: fixed(MAX_POINT_RECORDS),
|
|
898
|
+
/** A heartbeat above this is trimmed to counts before it is sent. */
|
|
899
|
+
heartbeatMaxBytes: fixed(64 * 1024),
|
|
900
|
+
/** OTLP: resource attributes every exported span carries, and the standard endpoint env var. */
|
|
901
|
+
otlp: z.object({
|
|
902
|
+
resourceAgentName: fixed("agentspec.agent.name"),
|
|
903
|
+
resourceManifestSha256: fixed("agentspec.manifest.sha256"),
|
|
904
|
+
endpointEnv: fixed("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
|
905
|
+
/** Resource attribute naming the span's producer, and the value agentgateway exports (its default); only such spans are gateway evidence. */
|
|
906
|
+
resourceServiceName: fixed("service.name"),
|
|
907
|
+
gatewayServiceName: fixed("agentgateway")
|
|
908
|
+
}).strict().prefault({})
|
|
909
|
+
}).strict();
|
|
910
|
+
var CONTRACT = ContractSchema.parse({});
|
|
911
|
+
|
|
912
|
+
// src/schema/evaluation.ts
|
|
913
|
+
var REPLACED_BY = {
|
|
914
|
+
hallucination: "grounded",
|
|
915
|
+
faithfulness: "grounded"
|
|
916
|
+
};
|
|
917
|
+
function canonicalMetric(metric) {
|
|
918
|
+
return REPLACED_BY[metric] ?? metric;
|
|
919
|
+
}
|
|
920
|
+
function evaluationWarnings(manifest) {
|
|
921
|
+
const evaluation = manifest.spec.evaluation;
|
|
922
|
+
if (!evaluation) return [];
|
|
923
|
+
const warnings = [];
|
|
924
|
+
if (evaluation.framework !== void 0) {
|
|
925
|
+
warnings.push(
|
|
926
|
+
"spec.evaluation.framework is deprecated: nothing reads it and AgentSpec computes evaluation evidence itself. Remove it; it is rejected in 0.6.0."
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
const declared2 = [
|
|
930
|
+
...evaluation.metrics ?? [],
|
|
931
|
+
...evaluation.live?.metrics ?? [],
|
|
932
|
+
...(evaluation.datasets ?? []).flatMap((d) => d.metrics ?? [])
|
|
933
|
+
];
|
|
934
|
+
for (const metric of new Set(declared2)) {
|
|
935
|
+
const replacement = canonicalMetric(metric);
|
|
936
|
+
if (replacement !== metric) {
|
|
937
|
+
warnings.push(
|
|
938
|
+
`spec.evaluation metric "${metric}" is deprecated and is read as "${replacement}". Rename it to "${replacement}"; it is rejected in 0.6.0.`
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
warnings.push(
|
|
943
|
+
...thresholdWarnings("spec.evaluation.thresholds", evaluation.thresholds),
|
|
944
|
+
...thresholdWarnings("spec.evaluation.live.thresholds", evaluation.live?.thresholds)
|
|
945
|
+
);
|
|
946
|
+
return warnings;
|
|
947
|
+
}
|
|
948
|
+
function thresholdWarnings(path, thresholds) {
|
|
949
|
+
return Object.keys(thresholds ?? {}).flatMap((name) => {
|
|
950
|
+
const replacement = canonicalMetric(name);
|
|
951
|
+
if (replacement === name) return [];
|
|
952
|
+
const ignored = replacement in thresholds ? ` It is ignored: ${path}.${replacement} is also declared.` : "";
|
|
953
|
+
return [
|
|
954
|
+
`${path}.${name} is deprecated and is read as ${path}.${replacement}. Rename it to "${replacement}"; it is rejected in 0.6.0.${ignored}`
|
|
955
|
+
];
|
|
359
956
|
});
|
|
360
957
|
}
|
|
958
|
+
function assertRubricRefs(parsed) {
|
|
959
|
+
const evaluation = parsed?.spec?.evaluation;
|
|
960
|
+
const live = evaluation?.live;
|
|
961
|
+
for (const [path, custom] of [
|
|
962
|
+
["spec.evaluation.custom", evaluation?.custom],
|
|
963
|
+
["spec.evaluation.live.custom", live?.custom]
|
|
964
|
+
]) {
|
|
965
|
+
const rubric = custom?.rubric;
|
|
966
|
+
if (typeof rubric === "string" && !rubric.startsWith("$file:")) {
|
|
967
|
+
throw new Error(
|
|
968
|
+
`${path}.rubric must be a "$file:<path>" reference, got "${rubric}". Fix: write it as "$file:${rubric}" (path relative to agent.yaml).`
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
function normalizeEvaluation(manifest) {
|
|
974
|
+
const evaluation = manifest.spec.evaluation;
|
|
975
|
+
if (!evaluation) return manifest;
|
|
976
|
+
const metrics = (list) => list?.map(canonicalMetric);
|
|
977
|
+
const thresholds = (map) => {
|
|
978
|
+
if (!map) return map;
|
|
979
|
+
const out = {};
|
|
980
|
+
for (const [name, value] of Object.entries(map)) {
|
|
981
|
+
const key = canonicalMetric(name);
|
|
982
|
+
if (key === name || !(key in map)) out[key] = value;
|
|
983
|
+
}
|
|
984
|
+
return out;
|
|
985
|
+
};
|
|
986
|
+
return {
|
|
987
|
+
...manifest,
|
|
988
|
+
spec: {
|
|
989
|
+
...manifest.spec,
|
|
990
|
+
evaluation: {
|
|
991
|
+
...evaluation,
|
|
992
|
+
metrics: metrics(evaluation.metrics),
|
|
993
|
+
thresholds: thresholds(evaluation.thresholds),
|
|
994
|
+
datasets: evaluation.datasets?.map((d) => ({ ...d, metrics: metrics(d.metrics) })),
|
|
995
|
+
live: evaluation.live && {
|
|
996
|
+
...evaluation.live,
|
|
997
|
+
metrics: metrics(evaluation.live.metrics),
|
|
998
|
+
thresholds: thresholds(evaluation.live.thresholds)
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
361
1004
|
function detectRefType(value) {
|
|
362
1005
|
if (value.startsWith("$env:")) return "env";
|
|
363
1006
|
if (value.startsWith("$secret:")) return "secret";
|
|
@@ -403,8 +1046,9 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
403
1046
|
Use paths relative to agent.yaml location only`
|
|
404
1047
|
);
|
|
405
1048
|
}
|
|
1049
|
+
const readPath = checkedReadPath(value, absPath, opts.baseDir);
|
|
406
1050
|
try {
|
|
407
|
-
return readFileSync(
|
|
1051
|
+
return readFileSync(readPath, "utf-8");
|
|
408
1052
|
} catch {
|
|
409
1053
|
if (optional) return "";
|
|
410
1054
|
throw new Error(
|
|
@@ -426,6 +1070,29 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
426
1070
|
}
|
|
427
1071
|
return value;
|
|
428
1072
|
}
|
|
1073
|
+
function checkedReadPath(ref, absPath, baseDir) {
|
|
1074
|
+
let realTarget;
|
|
1075
|
+
try {
|
|
1076
|
+
realTarget = realpathSync(absPath);
|
|
1077
|
+
} catch (err) {
|
|
1078
|
+
if (err.code === "ENOENT") return absPath;
|
|
1079
|
+
throw new Error(
|
|
1080
|
+
`Cannot resolve ${ref}
|
|
1081
|
+
${err.message}
|
|
1082
|
+
Fix the path: every folder in it must exist, be a folder and not loop back through symlinks`
|
|
1083
|
+
);
|
|
1084
|
+
}
|
|
1085
|
+
const realBase = realpathSync(baseDir);
|
|
1086
|
+
if (realTarget !== realBase && !realTarget.startsWith(realBase + sep)) {
|
|
1087
|
+
throw new Error(
|
|
1088
|
+
`Path traversal detected in ${ref}
|
|
1089
|
+
Resolved through a symlink to: ${realTarget}
|
|
1090
|
+
Which is outside baseDir: ${baseDir}
|
|
1091
|
+
Replace the symlink with a real folder inside the manifest directory, or move the file there`
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
return realTarget;
|
|
1095
|
+
}
|
|
429
1096
|
function resolveSecret(name, backend) {
|
|
430
1097
|
switch (backend) {
|
|
431
1098
|
case "env": {
|
|
@@ -452,7 +1119,7 @@ function resolveSecret(name, backend) {
|
|
|
452
1119
|
}
|
|
453
1120
|
function resolveRefs(obj, opts) {
|
|
454
1121
|
if (typeof obj === "string") {
|
|
455
|
-
return resolveRef(obj, opts
|
|
1122
|
+
return resolveRef(obj, opts);
|
|
456
1123
|
}
|
|
457
1124
|
if (Array.isArray(obj)) {
|
|
458
1125
|
return obj.map((item) => resolveRefs(item, opts));
|
|
@@ -509,13 +1176,15 @@ function loadManifest(filePath, opts = {}) {
|
|
|
509
1176
|
throw new Error(`Invalid YAML in ${absPath}
|
|
510
1177
|
${String(err)}`);
|
|
511
1178
|
}
|
|
1179
|
+
assertRubricRefs(parsed);
|
|
512
1180
|
const toValidate = opts.resolve === true ? resolveRefs(parsed, {
|
|
513
1181
|
baseDir,
|
|
514
1182
|
secretBackend: opts.secretBackend,
|
|
515
1183
|
failOnMissingEnv: opts.failOnMissingEnv ?? true
|
|
516
1184
|
}) : parsed;
|
|
517
|
-
const
|
|
518
|
-
|
|
1185
|
+
const declared2 = ManifestSchema.parse(toValidate);
|
|
1186
|
+
const warnings = evaluationWarnings(declared2);
|
|
1187
|
+
return { manifest: normalizeEvaluation(declared2), warnings, filePath: absPath, baseDir, raw };
|
|
519
1188
|
}
|
|
520
1189
|
function tryLoadManifest(filePath, opts = {}) {
|
|
521
1190
|
try {
|
|
@@ -562,8 +1231,8 @@ function isLatestVersion(raw) {
|
|
|
562
1231
|
function runEnvChecks(manifest, rawManifest) {
|
|
563
1232
|
const checks = [];
|
|
564
1233
|
const envRefs = collectEnvRefs(rawManifest);
|
|
565
|
-
const
|
|
566
|
-
for (const v of
|
|
1234
|
+
const declared2 = manifest.spec.requires?.envVars ?? [];
|
|
1235
|
+
for (const v of declared2) envRefs.add(v);
|
|
567
1236
|
for (const varName of envRefs) {
|
|
568
1237
|
const present = process.env[varName] !== void 0 && process.env[varName] !== "";
|
|
569
1238
|
checks.push({
|
|
@@ -600,21 +1269,53 @@ var PROVIDER_ENDPOINTS = {
|
|
|
600
1269
|
openai: "https://api.openai.com/v1/models",
|
|
601
1270
|
anthropic: "https://api.anthropic.com/v1/models",
|
|
602
1271
|
groq: "https://api.groq.com/openai/v1/models",
|
|
603
|
-
|
|
604
|
-
google: "https://generativelanguage.googleapis.com",
|
|
1272
|
+
google: "https://generativelanguage.googleapis.com/v1beta/models",
|
|
605
1273
|
mistral: "https://api.mistral.ai/v1/models",
|
|
606
|
-
cohere: "https://api.cohere.
|
|
1274
|
+
cohere: "https://api.cohere.com/v1/models",
|
|
607
1275
|
together: "https://api.together.xyz/v1/models",
|
|
608
1276
|
fireworks: "https://api.fireworks.ai/inference/v1/models"
|
|
609
1277
|
};
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
1278
|
+
var AZURE_API_VERSION = "2024-10-21";
|
|
1279
|
+
var AZURE_RESOURCE_HOSTS = [".openai.azure.com", ".cognitiveservices.azure.com", ".openai.azure.us", ".openai.azure.cn"];
|
|
1280
|
+
function isAzureResourceEndpoint(base) {
|
|
1281
|
+
const { hostname, pathname } = new URL(base);
|
|
1282
|
+
return AZURE_RESOURCE_HOSTS.some((suffix) => hostname.endsWith(suffix)) && pathname === "/";
|
|
1283
|
+
}
|
|
1284
|
+
function probeUrl(provider, baseUrl) {
|
|
1285
|
+
const key = provider.toLowerCase();
|
|
1286
|
+
const nativeUrl = PROVIDER_ENDPOINTS[key];
|
|
1287
|
+
if (!nativeUrl && !(key === "azure" && baseUrl)) return void 0;
|
|
1288
|
+
if (!baseUrl) return { url: nativeUrl, native: true };
|
|
1289
|
+
const base = resolveRef(baseUrl, { baseDir: "." }).replace(/\/+$/, "");
|
|
1290
|
+
if (key === "azure" && isAzureResourceEndpoint(base)) {
|
|
1291
|
+
return { url: `${base}/openai/models?api-version=${AZURE_API_VERSION}`, native: false, azureResource: true };
|
|
1292
|
+
}
|
|
1293
|
+
if (key === "anthropic") return { url: `${base}/v1/models`, native: false };
|
|
1294
|
+
return { url: `${base}/models`, native: false };
|
|
1295
|
+
}
|
|
1296
|
+
function unverifiableProbe(provider) {
|
|
1297
|
+
if (provider.toLowerCase() === "azure") {
|
|
613
1298
|
return {
|
|
614
|
-
|
|
615
|
-
|
|
1299
|
+
outcome: "not_verifiable",
|
|
1300
|
+
error: "azure keys are checked against your own resource endpoint, and spec.model.baseUrl is not set to name it",
|
|
1301
|
+
remediation: "Set spec.model.baseUrl to your Azure resource endpoint (https://<resource>.openai.azure.com) so the key can be probed, or verify it out of band"
|
|
616
1302
|
};
|
|
617
1303
|
}
|
|
1304
|
+
return { outcome: "not_verifiable", error: `unknown provider "${provider}"` };
|
|
1305
|
+
}
|
|
1306
|
+
async function checkModelEndpoint(provider, apiKey, baseUrl, timeoutMs = 5e3) {
|
|
1307
|
+
let target;
|
|
1308
|
+
try {
|
|
1309
|
+
target = probeUrl(provider, baseUrl);
|
|
1310
|
+
} catch (err) {
|
|
1311
|
+
return {
|
|
1312
|
+
outcome: "unreachable",
|
|
1313
|
+
error: err instanceof Error ? err.message.split("\n")[0] : String(err),
|
|
1314
|
+
remediation: `Set the variable behind spec.model baseUrl (${baseUrl}) or remove the override`
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
if (!target) return unverifiableProbe(provider);
|
|
1318
|
+
const { url, native, azureResource } = target;
|
|
618
1319
|
const start = Date.now();
|
|
619
1320
|
const controller = new AbortController();
|
|
620
1321
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -628,19 +1329,30 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
628
1329
|
headers["anthropic-version"] = "2023-06-01";
|
|
629
1330
|
delete headers["Authorization"];
|
|
630
1331
|
}
|
|
1332
|
+
if (azureResource) {
|
|
1333
|
+
headers["api-key"] = apiKey;
|
|
1334
|
+
delete headers["Authorization"];
|
|
1335
|
+
}
|
|
1336
|
+
const isGemini = provider.toLowerCase() === "google" && native;
|
|
1337
|
+
if (isGemini) {
|
|
1338
|
+
headers["x-goog-api-key"] = apiKey;
|
|
1339
|
+
delete headers["Authorization"];
|
|
1340
|
+
}
|
|
631
1341
|
const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
|
|
632
1342
|
const latencyMs = Date.now() - start;
|
|
633
|
-
const
|
|
634
|
-
return {
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
1343
|
+
const geminiBadKey = isGemini && res.status === 400;
|
|
1344
|
+
if (res.status === 401 || res.status === 403 || geminiBadKey) return { outcome: "rejected", latencyMs, error: `HTTP ${res.status}` };
|
|
1345
|
+
if (res.status >= 500) return { outcome: "unreachable", latencyMs, error: `HTTP ${res.status}` };
|
|
1346
|
+
if (res.status === 404 || res.status === 405 || res.status === 410) {
|
|
1347
|
+
const { origin, pathname } = new URL(url);
|
|
1348
|
+
return { outcome: "unreachable", latencyMs, error: res.status === 404 ? "HTTP 404 (endpoint not found)" : `HTTP ${res.status} (not a models endpoint)`, remediation: `Check that ${origin}${pathname} is the provider's models endpoint${native ? "" : " (fix spec.model.baseUrl)"}` };
|
|
1349
|
+
}
|
|
1350
|
+
return { outcome: "ok", latencyMs };
|
|
639
1351
|
} catch (err) {
|
|
640
1352
|
const latencyMs = Date.now() - start;
|
|
641
1353
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
642
1354
|
return {
|
|
643
|
-
|
|
1355
|
+
outcome: "unreachable",
|
|
644
1356
|
latencyMs,
|
|
645
1357
|
error: isTimeout ? "Request timed out" : String(err)
|
|
646
1358
|
};
|
|
@@ -648,6 +1360,20 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
648
1360
|
clearTimeout(timeout);
|
|
649
1361
|
}
|
|
650
1362
|
}
|
|
1363
|
+
function modelProbeCheck(probe, fields, label, provider, keyRef) {
|
|
1364
|
+
const keyName = keyRef.startsWith("$") ? keyRef : "[api-key]";
|
|
1365
|
+
const base = { ...fields, latencyMs: probe.latencyMs };
|
|
1366
|
+
switch (probe.outcome) {
|
|
1367
|
+
case "ok":
|
|
1368
|
+
return { ...base, status: "pass" };
|
|
1369
|
+
case "not_verifiable":
|
|
1370
|
+
return { ...base, status: "not_verifiable", message: `${label} cannot be verified: ${probe.error}`, remediation: probe.remediation ?? `Use a provider the health check knows (${Object.keys(PROVIDER_ENDPOINTS).join(", ")}) or verify ${provider} out of band` };
|
|
1371
|
+
case "rejected":
|
|
1372
|
+
return { ...base, status: "fail", message: `${label} for ${provider} rejected the API key: ${probe.error}`, remediation: `Check that ${keyName} holds a valid ${provider} API key` };
|
|
1373
|
+
default:
|
|
1374
|
+
return { ...base, status: "fail", message: `${label} for ${provider} is unreachable: ${probe.error}`, remediation: probe.remediation ?? `Check that ${keyName} is set and the ${provider} API is reachable` };
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
651
1377
|
function resolveApiKey(ref) {
|
|
652
1378
|
if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
|
|
653
1379
|
if (ref.startsWith("$")) return "";
|
|
@@ -657,16 +1383,14 @@ async function runModelChecks(model) {
|
|
|
657
1383
|
const checks = [];
|
|
658
1384
|
const apiKey = resolveApiKey(model.apiKey);
|
|
659
1385
|
if (apiKey) {
|
|
660
|
-
const
|
|
661
|
-
checks.push(
|
|
662
|
-
|
|
663
|
-
category: "model",
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
|
|
669
|
-
});
|
|
1386
|
+
const probe = await checkModelEndpoint(model.provider, apiKey, model.baseUrl);
|
|
1387
|
+
checks.push(modelProbeCheck(
|
|
1388
|
+
probe,
|
|
1389
|
+
{ id: `model:${model.provider}/${model.id}`, category: "model", severity: "error" },
|
|
1390
|
+
"Model endpoint",
|
|
1391
|
+
model.provider,
|
|
1392
|
+
model.apiKey
|
|
1393
|
+
));
|
|
670
1394
|
} else {
|
|
671
1395
|
checks.push({
|
|
672
1396
|
id: `model:${model.provider}/${model.id}`,
|
|
@@ -676,32 +1400,6 @@ async function runModelChecks(model) {
|
|
|
676
1400
|
message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
|
|
677
1401
|
});
|
|
678
1402
|
}
|
|
679
|
-
if (model.fallback) {
|
|
680
|
-
const fallbackKey = resolveApiKey(model.fallback.apiKey);
|
|
681
|
-
if (fallbackKey) {
|
|
682
|
-
const { reachable, latencyMs, error } = await checkModelEndpoint(
|
|
683
|
-
model.fallback.provider,
|
|
684
|
-
fallbackKey
|
|
685
|
-
);
|
|
686
|
-
checks.push({
|
|
687
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
688
|
-
category: "model-fallback",
|
|
689
|
-
status: reachable ? "pass" : "fail",
|
|
690
|
-
severity: "warning",
|
|
691
|
-
latencyMs,
|
|
692
|
-
message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
|
|
693
|
-
remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
|
|
694
|
-
});
|
|
695
|
-
} else {
|
|
696
|
-
checks.push({
|
|
697
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
698
|
-
category: "model-fallback",
|
|
699
|
-
status: "skip",
|
|
700
|
-
severity: "warning",
|
|
701
|
-
message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
|
|
702
|
-
});
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
1403
|
return checks;
|
|
706
1404
|
}
|
|
707
1405
|
|
|
@@ -713,7 +1411,7 @@ async function runMcpChecks(servers) {
|
|
|
713
1411
|
const url = server.url;
|
|
714
1412
|
if (url && !url.startsWith("$")) {
|
|
715
1413
|
const check = await checkHttpMcpServer(server.name, url);
|
|
716
|
-
checks.push(check);
|
|
1414
|
+
checks.push(server.transport === "sse" ? withLegacySseHint(check) : check);
|
|
717
1415
|
} else {
|
|
718
1416
|
checks.push({
|
|
719
1417
|
id: `mcp:${server.name}`,
|
|
@@ -730,37 +1428,145 @@ async function runMcpChecks(servers) {
|
|
|
730
1428
|
}
|
|
731
1429
|
return checks;
|
|
732
1430
|
}
|
|
1431
|
+
var MCP_PROTOCOL_VERSION = "2025-03-26";
|
|
1432
|
+
var HANDSHAKE_TIMEOUT_MS = 5e3;
|
|
1433
|
+
var INITIALIZE_REQUEST_ID = 1;
|
|
1434
|
+
var MAX_SERVER_TEXT = 200;
|
|
1435
|
+
var PROTOCOL_VERSION_FORMAT = /^\d{4}-\d{2}-\d{2}$/;
|
|
733
1436
|
async function checkHttpMcpServer(name, url) {
|
|
734
1437
|
const controller = new AbortController();
|
|
735
|
-
const timeout = setTimeout(() => controller.abort(),
|
|
1438
|
+
const timeout = setTimeout(() => controller.abort(), HANDSHAKE_TIMEOUT_MS);
|
|
736
1439
|
const start = Date.now();
|
|
1440
|
+
const headers = {
|
|
1441
|
+
"content-type": "application/json",
|
|
1442
|
+
accept: "application/json, text/event-stream"
|
|
1443
|
+
};
|
|
737
1444
|
try {
|
|
738
|
-
const res = await fetch(url, {
|
|
1445
|
+
const res = await fetch(url, {
|
|
1446
|
+
method: "POST",
|
|
1447
|
+
headers,
|
|
1448
|
+
signal: controller.signal,
|
|
1449
|
+
body: JSON.stringify({
|
|
1450
|
+
jsonrpc: "2.0",
|
|
1451
|
+
id: INITIALIZE_REQUEST_ID,
|
|
1452
|
+
method: "initialize",
|
|
1453
|
+
params: {
|
|
1454
|
+
protocolVersion: MCP_PROTOCOL_VERSION,
|
|
1455
|
+
capabilities: {},
|
|
1456
|
+
clientInfo: { name: "agentspec-health", version: "1.0.0" }
|
|
1457
|
+
}
|
|
1458
|
+
})
|
|
1459
|
+
});
|
|
1460
|
+
if (!res.ok) {
|
|
1461
|
+
await res.body?.cancel().catch(() => void 0);
|
|
1462
|
+
return failedHandshake(name, Date.now() - start, `returned HTTP ${res.status} to initialize`);
|
|
1463
|
+
}
|
|
1464
|
+
const parsed = await readInitializeResult(res);
|
|
739
1465
|
const latencyMs = Date.now() - start;
|
|
1466
|
+
if ("error" in parsed) {
|
|
1467
|
+
return failedHandshake(name, latencyMs, parsed.error);
|
|
1468
|
+
}
|
|
1469
|
+
const result = parsed.result;
|
|
1470
|
+
const sessionId = res.headers.get("mcp-session-id");
|
|
1471
|
+
if (sessionId) headers["mcp-session-id"] = sessionId;
|
|
1472
|
+
headers["mcp-protocol-version"] = result.protocolVersion;
|
|
1473
|
+
await sendBestEffort(url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) }, controller.signal);
|
|
1474
|
+
if (sessionId) await sendBestEffort(url, { method: "DELETE", headers }, controller.signal);
|
|
1475
|
+
const serverName = clip(result.serverInfo?.name);
|
|
1476
|
+
const serverVersion = clip(result.serverInfo?.version);
|
|
1477
|
+
const info = serverName ? ` (${serverName}${serverVersion ? ` ${serverVersion}` : ""})` : "";
|
|
740
1478
|
return {
|
|
741
1479
|
id: `mcp:${name}`,
|
|
742
1480
|
category: "mcp",
|
|
743
|
-
status:
|
|
1481
|
+
status: "pass",
|
|
744
1482
|
severity: "warning",
|
|
745
1483
|
latencyMs,
|
|
746
|
-
message:
|
|
1484
|
+
message: `MCP server ${name}${info} initialized, protocolVersion ${clip(result.protocolVersion)}`
|
|
747
1485
|
};
|
|
748
1486
|
} catch (err) {
|
|
749
1487
|
const latencyMs = Date.now() - start;
|
|
750
1488
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
751
|
-
return
|
|
752
|
-
|
|
753
|
-
category: "mcp",
|
|
754
|
-
status: "fail",
|
|
755
|
-
severity: "warning",
|
|
1489
|
+
return failedHandshake(
|
|
1490
|
+
name,
|
|
756
1491
|
latencyMs,
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
};
|
|
1492
|
+
isTimeout ? "health check timed out" : `unreachable: ${redactUrlCredentials(String(err))}`
|
|
1493
|
+
);
|
|
760
1494
|
} finally {
|
|
761
1495
|
clearTimeout(timeout);
|
|
762
1496
|
}
|
|
763
1497
|
}
|
|
1498
|
+
async function sendBestEffort(url, init, signal) {
|
|
1499
|
+
await fetch(url, { ...init, signal }).then((r) => r.body?.cancel()).catch(() => void 0);
|
|
1500
|
+
}
|
|
1501
|
+
function withLegacySseHint(check) {
|
|
1502
|
+
if (check.status !== "fail") return check;
|
|
1503
|
+
return {
|
|
1504
|
+
...check,
|
|
1505
|
+
remediation: `${check.remediation ?? ""} Note: transport "sse" is probed with the streamable-HTTP handshake; legacy HTTP+SSE servers are not supported by this check.`.trim()
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
function redactUrlCredentials(text) {
|
|
1509
|
+
return text.replace(/(\w+:\/\/)[^/\s]*@/g, "$1***@");
|
|
1510
|
+
}
|
|
1511
|
+
function failedHandshake(name, latencyMs, reason) {
|
|
1512
|
+
return {
|
|
1513
|
+
id: `mcp:${name}`,
|
|
1514
|
+
category: "mcp",
|
|
1515
|
+
status: "fail",
|
|
1516
|
+
severity: "warning",
|
|
1517
|
+
latencyMs,
|
|
1518
|
+
message: `MCP server ${name} ${reason}`,
|
|
1519
|
+
remediation: `Check that MCP server ${name} is running and that spec.mcp.servers[].url points at a streamable-HTTP MCP endpoint`
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
async function readInitializeResult(res) {
|
|
1523
|
+
const isSse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
|
|
1524
|
+
const text = isSse ? await readSseUntilInitialize(res) : await res.text();
|
|
1525
|
+
const payloads = isSse ? splitSseEvents(text) : [text];
|
|
1526
|
+
for (const payload of payloads) {
|
|
1527
|
+
const outcome = parseInitializeFrame(payload);
|
|
1528
|
+
if (outcome) return outcome;
|
|
1529
|
+
}
|
|
1530
|
+
return { error: "did not return a valid JSON-RPC initialize result" };
|
|
1531
|
+
}
|
|
1532
|
+
function clip(value) {
|
|
1533
|
+
return typeof value === "string" ? value.slice(0, MAX_SERVER_TEXT) : "";
|
|
1534
|
+
}
|
|
1535
|
+
function parseInitializeFrame(payload) {
|
|
1536
|
+
try {
|
|
1537
|
+
const msg = JSON.parse(payload);
|
|
1538
|
+
if (msg.jsonrpc !== "2.0" || msg.id !== INITIALIZE_REQUEST_ID) return null;
|
|
1539
|
+
if (msg.result && typeof msg.result.protocolVersion === "string") {
|
|
1540
|
+
return PROTOCOL_VERSION_FORMAT.test(msg.result.protocolVersion) ? { result: msg.result } : { error: "returned a malformed protocolVersion in its initialize result" };
|
|
1541
|
+
}
|
|
1542
|
+
if (msg.error) return { error: `rejected initialize: ${clip(msg.error.message) || "JSON-RPC error"}` };
|
|
1543
|
+
} catch {
|
|
1544
|
+
}
|
|
1545
|
+
return null;
|
|
1546
|
+
}
|
|
1547
|
+
function splitSseEvents(text) {
|
|
1548
|
+
return text.split(/\r?\n\r?\n/).map(
|
|
1549
|
+
(event) => event.split(/\r?\n/).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).join("\n")
|
|
1550
|
+
);
|
|
1551
|
+
}
|
|
1552
|
+
async function readSseUntilInitialize(res) {
|
|
1553
|
+
if (!res.body) return res.text();
|
|
1554
|
+
const reader = res.body.getReader();
|
|
1555
|
+
const decoder = new TextDecoder();
|
|
1556
|
+
let text = "";
|
|
1557
|
+
try {
|
|
1558
|
+
for (; ; ) {
|
|
1559
|
+
const { done, value } = await reader.read();
|
|
1560
|
+
if (done) break;
|
|
1561
|
+
text += decoder.decode(value, { stream: true });
|
|
1562
|
+
const completeEvents = text.split(/\r?\n\r?\n/).slice(0, -1);
|
|
1563
|
+
if (completeEvents.flatMap((e) => splitSseEvents(e)).some((p) => parseInitializeFrame(p))) break;
|
|
1564
|
+
}
|
|
1565
|
+
} finally {
|
|
1566
|
+
await reader.cancel().catch(() => void 0);
|
|
1567
|
+
}
|
|
1568
|
+
return text;
|
|
1569
|
+
}
|
|
764
1570
|
async function checkStdioMcpServer(server) {
|
|
765
1571
|
const command = server.command;
|
|
766
1572
|
if (!command) {
|
|
@@ -1278,7 +2084,7 @@ function computeHealthStatus(checks) {
|
|
|
1278
2084
|
const failed = checks.filter((c) => c.status === "fail");
|
|
1279
2085
|
const warnings = checks.filter((c) => c.status === "warn");
|
|
1280
2086
|
const passed = checks.filter((c) => c.status === "pass");
|
|
1281
|
-
const skipped = checks.filter((c) => c.status === "skip");
|
|
2087
|
+
const skipped = checks.filter((c) => c.status === "skip" || c.status === "not_verifiable");
|
|
1282
2088
|
const hasErrors = failed.some((c) => c.severity === "error");
|
|
1283
2089
|
const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
|
|
1284
2090
|
const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
|
|
@@ -1295,8 +2101,10 @@ function computeHealthStatus(checks) {
|
|
|
1295
2101
|
async function runHealthCheck(manifest, opts = {}) {
|
|
1296
2102
|
const checks = [];
|
|
1297
2103
|
const rawManifest = opts.rawManifest ?? manifest;
|
|
1298
|
-
|
|
1299
|
-
|
|
2104
|
+
if (opts.checkEnv !== false) {
|
|
2105
|
+
checks.push(...runEnvChecks(manifest, rawManifest));
|
|
2106
|
+
checks.push(...await runSecretChecks());
|
|
2107
|
+
}
|
|
1300
2108
|
if (opts.baseDir) {
|
|
1301
2109
|
checks.push(...runFileChecks(rawManifest, opts.baseDir));
|
|
1302
2110
|
}
|
|
@@ -1386,7 +2194,381 @@ async function checkA2aEndpoint(name, url) {
|
|
|
1386
2194
|
}
|
|
1387
2195
|
}
|
|
1388
2196
|
|
|
2197
|
+
// src/grader/index.ts
|
|
2198
|
+
var ADMITTED = {
|
|
2199
|
+
standard: ["independent", "sdk-observed"],
|
|
2200
|
+
strict: ["independent"]
|
|
2201
|
+
};
|
|
2202
|
+
var SOURCE_CLASS = {
|
|
2203
|
+
independent: "independent",
|
|
2204
|
+
"sdk-observed": "evidence-point",
|
|
2205
|
+
offline: "offline",
|
|
2206
|
+
declared: "declared"
|
|
2207
|
+
};
|
|
2208
|
+
var pct = (part, whole) => whole === 0 ? 0 : part / whole * 100;
|
|
2209
|
+
function isCalendarDate(value) {
|
|
2210
|
+
const [y, m, d] = value.slice(0, 10).split("-").map(Number);
|
|
2211
|
+
const date = /* @__PURE__ */ new Date(0);
|
|
2212
|
+
date.setUTCFullYear(y, m - 1, d);
|
|
2213
|
+
return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
|
|
2214
|
+
}
|
|
2215
|
+
function parseTimestamp(value, field) {
|
|
2216
|
+
const ms = Date.parse(value);
|
|
2217
|
+
const valid = TIMESTAMP_PATTERN.test(value) && !Number.isNaN(ms) && isCalendarDate(value) && !value.startsWith("0000") && !value.includes("T24:");
|
|
2218
|
+
if (!valid) {
|
|
2219
|
+
throw new Error(`grade: ${field} "${value}" is not a valid RFC 3339 timestamp. Fix the producer to emit a real date and time.`);
|
|
2220
|
+
}
|
|
2221
|
+
return ms;
|
|
2222
|
+
}
|
|
2223
|
+
function isExpired(r, nowMs) {
|
|
2224
|
+
return parseTimestamp(r.observedAt, `observedAt of the record for check "${r.checkId}"`) + r.ttlSeconds * 1e3 < nowMs;
|
|
2225
|
+
}
|
|
2226
|
+
function matchesRegistry(r) {
|
|
2227
|
+
const cls = SOURCE_CLASS[r.source];
|
|
2228
|
+
if (!r.ruleId || !cls || !Object.hasOwn(RULE_EVIDENCE, r.ruleId)) return true;
|
|
2229
|
+
const listed = RULE_EVIDENCE[r.ruleId].sources.filter((s) => s.class === cls);
|
|
2230
|
+
return listed.length > 0 && (r.collector == null || listed.some((s) => s.collector === r.collector));
|
|
2231
|
+
}
|
|
2232
|
+
function unverifiedReason(admitted, stale, notAdmitted, mismatched) {
|
|
2233
|
+
const own = admitted.find((r) => r.status === "not_verifiable");
|
|
2234
|
+
if (own?.reason) return own.reason;
|
|
2235
|
+
if (stale.length > 0) return { code: "stale" };
|
|
2236
|
+
if (notAdmitted.length > 0) return { code: "source_not_admitted" };
|
|
2237
|
+
if (mismatched.length > 0) return { code: "source_mismatch" };
|
|
2238
|
+
return { code: "no_evidence" };
|
|
2239
|
+
}
|
|
2240
|
+
function judgeRule(allRecords, policy, nowMs, unwiredFails) {
|
|
2241
|
+
if (unwiredFails) allRecords = [...allRecords.filter((r) => r.source !== "sdk-observed"), ...unwiredFails];
|
|
2242
|
+
const admittedSources = ADMITTED[policy];
|
|
2243
|
+
const registryMatched = [];
|
|
2244
|
+
const mismatched = [];
|
|
2245
|
+
for (const r of allRecords) (matchesRegistry(r) ? registryMatched : mismatched).push(r);
|
|
2246
|
+
const admittedRecords = registryMatched.filter((r) => admittedSources.includes(r.source));
|
|
2247
|
+
const notAdmitted = registryMatched.filter((r) => !admittedSources.includes(r.source));
|
|
2248
|
+
const stale = admittedRecords.filter((r) => isExpired(r, nowMs));
|
|
2249
|
+
const fresh = admittedRecords.filter((r) => !isExpired(r, nowMs));
|
|
2250
|
+
const decisive = fresh.filter((r) => r.status !== "not_verifiable");
|
|
2251
|
+
if (decisive.length === 0) {
|
|
2252
|
+
const unverified = unwiredFails ? { code: "not_wired" } : unverifiedReason(fresh, stale, notAdmitted, mismatched);
|
|
2253
|
+
return { covered: false, pass: false, independent: false, unverified };
|
|
2254
|
+
}
|
|
2255
|
+
return {
|
|
2256
|
+
covered: true,
|
|
2257
|
+
pass: decisive.every((r) => r.status === "pass"),
|
|
2258
|
+
independent: decisive.every((r) => r.source === "independent")
|
|
2259
|
+
};
|
|
2260
|
+
}
|
|
2261
|
+
function unwiredRules(points) {
|
|
2262
|
+
const notWired = /* @__PURE__ */ new Set();
|
|
2263
|
+
const wiredFails = /* @__PURE__ */ new Map();
|
|
2264
|
+
for (const p of points) {
|
|
2265
|
+
const ruleId = CONTRACT.evidencePoints[p.point]?.[p.target];
|
|
2266
|
+
if (!ruleId) continue;
|
|
2267
|
+
if (p.state === "not_wired") notWired.add(ruleId);
|
|
2268
|
+
else {
|
|
2269
|
+
const fails = p.records.filter((r) => r.ruleId === ruleId && r.status === "fail");
|
|
2270
|
+
wiredFails.set(ruleId, [...wiredFails.get(ruleId) ?? [], ...fails]);
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
return new Map([...notWired].map((id) => [id, wiredFails.get(id) ?? []]));
|
|
2274
|
+
}
|
|
2275
|
+
function judgeDerived(parents) {
|
|
2276
|
+
const open = parents.find((v) => !v.covered);
|
|
2277
|
+
const failed = parents.some((v) => v.covered && !v.pass);
|
|
2278
|
+
if (open && !failed) return open;
|
|
2279
|
+
const covered = parents.filter((v) => v.covered);
|
|
2280
|
+
return { covered: true, pass: !failed, independent: covered.every((v) => v.independent) };
|
|
2281
|
+
}
|
|
2282
|
+
function judgeRegistered(ruleId, byRule, policy, nowMs, unwired) {
|
|
2283
|
+
const derivedFrom = Object.hasOwn(RULE_EVIDENCE, ruleId) ? RULE_EVIDENCE[ruleId].derivedFrom : void 0;
|
|
2284
|
+
if (!derivedFrom?.length) return judgeRule(byRule.get(ruleId) ?? [], policy, nowMs, unwired.get(ruleId));
|
|
2285
|
+
return judgeDerived(derivedFrom.map((parent) => judgeRegistered(parent, byRule, policy, nowMs, unwired)));
|
|
2286
|
+
}
|
|
2287
|
+
function groupByRule(records) {
|
|
2288
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
2289
|
+
for (const r of records) {
|
|
2290
|
+
if (!r.ruleId) continue;
|
|
2291
|
+
const bucket2 = byRule.get(r.ruleId);
|
|
2292
|
+
if (bucket2) bucket2.push(r);
|
|
2293
|
+
else byRule.set(r.ruleId, [r]);
|
|
2294
|
+
}
|
|
2295
|
+
return byRule;
|
|
2296
|
+
}
|
|
2297
|
+
function scoreToGrade(score) {
|
|
2298
|
+
const { A, B, C, D } = CONTRACT.gradeBands;
|
|
2299
|
+
if (score >= A) return "A";
|
|
2300
|
+
if (score >= B) return "B";
|
|
2301
|
+
if (score >= C) return "C";
|
|
2302
|
+
if (score >= D) return "D";
|
|
2303
|
+
return "F";
|
|
2304
|
+
}
|
|
2305
|
+
function grade(records, applicableRules, policy, now, points = []) {
|
|
2306
|
+
const nowMs = parseTimestamp(now, "now");
|
|
2307
|
+
const byRule = groupByRule(records);
|
|
2308
|
+
const unwired = unwiredRules(points);
|
|
2309
|
+
let totalWeight = 0;
|
|
2310
|
+
let coveredWeight = 0;
|
|
2311
|
+
let passedWeight = 0;
|
|
2312
|
+
let independentWeight = 0;
|
|
2313
|
+
const unverified = [];
|
|
2314
|
+
for (const rule of applicableRules) {
|
|
2315
|
+
const weight = CONTRACT.severityWeights[rule.severity];
|
|
2316
|
+
const verdict = judgeRegistered(rule.id, byRule, policy, nowMs, unwired);
|
|
2317
|
+
totalWeight += weight;
|
|
2318
|
+
if (!verdict.covered) {
|
|
2319
|
+
unverified.push({ ruleId: rule.id, severity: rule.severity, reason: verdict.unverified });
|
|
2320
|
+
continue;
|
|
2321
|
+
}
|
|
2322
|
+
coveredWeight += weight;
|
|
2323
|
+
if (verdict.pass) passedWeight += weight;
|
|
2324
|
+
if (verdict.independent) independentWeight += weight;
|
|
2325
|
+
}
|
|
2326
|
+
const verified = Math.round(pct(passedWeight, coveredWeight));
|
|
2327
|
+
const coverage = Math.round(pct(coveredWeight, totalWeight));
|
|
2328
|
+
return {
|
|
2329
|
+
verified,
|
|
2330
|
+
coverage,
|
|
2331
|
+
independentShare: Math.round(pct(independentWeight, coveredWeight)),
|
|
2332
|
+
grade: coverage >= CONTRACT.coverageThreshold ? scoreToGrade(verified) : "Not graded",
|
|
2333
|
+
unverified
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2336
|
+
function formatVerified(result) {
|
|
2337
|
+
if (result.grade === "Not graded") return `Not graded (coverage ${result.coverage}%)`;
|
|
2338
|
+
return `${result.grade} - ${result.verified}% verified (${result.independentShare}% independent) - coverage ${result.coverage}%`;
|
|
2339
|
+
}
|
|
2340
|
+
var NATIVE_PROVIDERS = {
|
|
2341
|
+
openai: "openAI",
|
|
2342
|
+
anthropic: "anthropic",
|
|
2343
|
+
google: "gemini",
|
|
2344
|
+
gemini: "gemini",
|
|
2345
|
+
azure: "azure"
|
|
2346
|
+
};
|
|
2347
|
+
var OPENAI_COMPATIBLE_BASE_URLS = {
|
|
2348
|
+
groq: "https://api.groq.com/openai/v1",
|
|
2349
|
+
mistral: "https://api.mistral.ai/v1"
|
|
2350
|
+
};
|
|
2351
|
+
var CATALOG_PROVIDER_IDS = {
|
|
2352
|
+
openAI: "openai",
|
|
2353
|
+
anthropic: "anthropic",
|
|
2354
|
+
gemini: "gcp.gemini",
|
|
2355
|
+
azure: "azure"
|
|
2356
|
+
};
|
|
2357
|
+
var GATEWAY_NAME = "llm";
|
|
2358
|
+
var ONE_MONTH = "30d";
|
|
2359
|
+
function envRefName(value) {
|
|
2360
|
+
return /^\$env:(.+)$/.exec(value)?.[1];
|
|
2361
|
+
}
|
|
2362
|
+
function secretEnvName(name) {
|
|
2363
|
+
return `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
|
|
2364
|
+
}
|
|
2365
|
+
function keyEnvName(apiKey) {
|
|
2366
|
+
const env = envRefName(apiKey);
|
|
2367
|
+
if (env) return env;
|
|
2368
|
+
const secret = /^\$secret:(.+)$/.exec(apiKey);
|
|
2369
|
+
if (secret) return secretEnvName(secret[1]);
|
|
2370
|
+
throw new Error(
|
|
2371
|
+
'spec.model.apiKey must be a "$env:VAR" or "$secret:name" reference; a literal key would end up in the gateway config. Replace it with "$env:<VAR>".'
|
|
2372
|
+
);
|
|
2373
|
+
}
|
|
2374
|
+
function baseUrlValue(baseUrl) {
|
|
2375
|
+
const env = envRefName(baseUrl);
|
|
2376
|
+
return env ? `$${env}` : baseUrl;
|
|
2377
|
+
}
|
|
2378
|
+
function baseUrlEnvNames({ baseUrl }) {
|
|
2379
|
+
const name = baseUrl && envRefName(baseUrl);
|
|
2380
|
+
return name ? [name] : [];
|
|
2381
|
+
}
|
|
2382
|
+
function customProvider(provider, id, baseUrl) {
|
|
2383
|
+
const anthropic = provider === "anthropic";
|
|
2384
|
+
return {
|
|
2385
|
+
provider: { custom: { providerOverride: provider, formats: [{ type: anthropic ? "messages" : "completions" }] } },
|
|
2386
|
+
params: { model: id, baseUrl, ...anthropic ? { pathPrefix: "/v1" } : {} }
|
|
2387
|
+
};
|
|
2388
|
+
}
|
|
2389
|
+
function gatewayEnvNames(manifest) {
|
|
2390
|
+
const { model } = manifest.spec;
|
|
2391
|
+
const models = [model, ...model.fallback ? [model.fallback] : []];
|
|
2392
|
+
const names = models.flatMap((m) => [
|
|
2393
|
+
keyEnvName(m.apiKey),
|
|
2394
|
+
...baseUrlEnvNames(m),
|
|
2395
|
+
...NATIVE_PROVIDERS[m.provider] === "azure" && !m.baseUrl ? [CONTRACT.env.azureResourceName] : []
|
|
2396
|
+
]);
|
|
2397
|
+
return [...new Set(names)].sort();
|
|
2398
|
+
}
|
|
2399
|
+
function providerFields({ provider, id, baseUrl }) {
|
|
2400
|
+
if (baseUrl) return customProvider(provider, id, baseUrlValue(baseUrl));
|
|
2401
|
+
const native = NATIVE_PROVIDERS[provider];
|
|
2402
|
+
if (native === "azure") {
|
|
2403
|
+
return {
|
|
2404
|
+
provider: native,
|
|
2405
|
+
params: { model: id, azureResourceName: `$${CONTRACT.env.azureResourceName}`, azureResourceType: "openAI" }
|
|
2406
|
+
};
|
|
2407
|
+
}
|
|
2408
|
+
if (native) return { provider: native, params: { model: id } };
|
|
2409
|
+
const knownBaseUrl = OPENAI_COMPATIBLE_BASE_URLS[provider];
|
|
2410
|
+
if (knownBaseUrl) return customProvider(provider, id, knownBaseUrl);
|
|
2411
|
+
const supported = [...Object.keys(NATIVE_PROVIDERS), ...Object.keys(OPENAI_COMPATIBLE_BASE_URLS)].sort();
|
|
2412
|
+
throw new Error(`spec.model.provider "${provider}" has no gateway mapping. Supported: ${supported.join(", ")}.`);
|
|
2413
|
+
}
|
|
2414
|
+
function renderModel(model) {
|
|
2415
|
+
const { provider, params } = providerFields(model);
|
|
2416
|
+
return {
|
|
2417
|
+
name: `${model.provider}/${model.id}`,
|
|
2418
|
+
visibility: "internal",
|
|
2419
|
+
provider,
|
|
2420
|
+
params: { ...params, apiKey: `$${keyEnvName(model.apiKey)}` }
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
function renderVirtualModel(models) {
|
|
2424
|
+
return {
|
|
2425
|
+
name: models[0].id,
|
|
2426
|
+
routing: {
|
|
2427
|
+
failover: {
|
|
2428
|
+
targets: models.map((m, priority) => ({ model: `${m.provider}/${m.id}`, priority }))
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
function renderRetryPolicy(model) {
|
|
2434
|
+
const attempts = model.fallback?.maxRetries;
|
|
2435
|
+
if (!attempts) return {};
|
|
2436
|
+
return {
|
|
2437
|
+
policies: [{
|
|
2438
|
+
name: { name: "llm-retry", namespace: "default" },
|
|
2439
|
+
target: { gateway: { gatewayName: "default", gatewayNamespace: "default", listenerName: GATEWAY_NAME } },
|
|
2440
|
+
policy: { retry: { attempts, codes: [500, 502, 503, 504] } }
|
|
2441
|
+
}]
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2444
|
+
function renderBudgets(spec) {
|
|
2445
|
+
const amount = spec.model.costControls?.maxMonthlyUSD;
|
|
2446
|
+
if (amount === void 0) return {};
|
|
2447
|
+
return {
|
|
2448
|
+
budgets: [{
|
|
2449
|
+
name: "monthly",
|
|
2450
|
+
limit: { unit: "USD", amount },
|
|
2451
|
+
window: { rolling: ONE_MONTH },
|
|
2452
|
+
onBudgetExceeded: "Block"
|
|
2453
|
+
}]
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2456
|
+
function renderApiKeyPolicy(manifest) {
|
|
2457
|
+
return {
|
|
2458
|
+
mode: "strict",
|
|
2459
|
+
keys: [{
|
|
2460
|
+
key: `$${CONTRACT.env.gatewayKey}`,
|
|
2461
|
+
metadata: { name: manifest.metadata.name },
|
|
2462
|
+
allowedModels: [manifest.spec.model.id],
|
|
2463
|
+
...renderBudgets(manifest.spec)
|
|
2464
|
+
}]
|
|
2465
|
+
};
|
|
2466
|
+
}
|
|
2467
|
+
function bucket(type, size, fillInterval) {
|
|
2468
|
+
return { type, maxTokens: size, tokensPerFill: size, fillInterval };
|
|
2469
|
+
}
|
|
2470
|
+
function renderRateLimits(spec) {
|
|
2471
|
+
const { requestsPerMinute, requestsPerHour, tokensPerMinute } = spec.api?.rateLimit ?? {};
|
|
2472
|
+
const buckets = [
|
|
2473
|
+
...requestsPerMinute === void 0 ? [] : [bucket("requests", requestsPerMinute, "60s")],
|
|
2474
|
+
...requestsPerHour === void 0 ? [] : [bucket("requests", requestsPerHour, "1h")],
|
|
2475
|
+
...tokensPerMinute === void 0 ? [] : [bucket("tokens", tokensPerMinute, "60s")]
|
|
2476
|
+
];
|
|
2477
|
+
return buckets.length === 0 ? {} : { localRateLimit: buckets };
|
|
2478
|
+
}
|
|
2479
|
+
function renderRuntimeConfig(spec) {
|
|
2480
|
+
return {
|
|
2481
|
+
...spec.model.costControls?.maxMonthlyUSD !== void 0 && {
|
|
2482
|
+
database: { url: CONTRACT.paths.gatewayDatabase }
|
|
2483
|
+
},
|
|
2484
|
+
// agentgateway samples no request by default and keeps prompt and completion out of the
|
|
2485
|
+
// span; the sidecar needs every LLM call with both (ADR-0030), and redacts before buffering.
|
|
2486
|
+
tracing: {
|
|
2487
|
+
otlpEndpoint: `http://${CONTRACT.podPatch.loopback}:${CONTRACT.ports.controlPlane}`,
|
|
2488
|
+
otlpProtocol: "http",
|
|
2489
|
+
path: CONTRACT.sidecarPaths.traces,
|
|
2490
|
+
randomSampling: "true",
|
|
2491
|
+
fields: { add: { "gen_ai.prompt": "llm.prompt", "gen_ai.completion": "llm.completion" } }
|
|
2492
|
+
}
|
|
2493
|
+
};
|
|
2494
|
+
}
|
|
2495
|
+
function catalogProvider(model) {
|
|
2496
|
+
const { provider } = providerFields(model);
|
|
2497
|
+
return typeof provider === "string" ? CATALOG_PROVIDER_IDS[provider] : model.provider;
|
|
2498
|
+
}
|
|
2499
|
+
function catalogRates(pricing) {
|
|
2500
|
+
return Object.fromEntries(Object.entries(pricing).filter(([, v]) => v !== void 0).map(([k, v]) => [k, String(v)]));
|
|
2501
|
+
}
|
|
2502
|
+
function renderModelCatalog(models) {
|
|
2503
|
+
const providers = {};
|
|
2504
|
+
for (const model of models) {
|
|
2505
|
+
if (!model.pricing) continue;
|
|
2506
|
+
const key = catalogProvider(model);
|
|
2507
|
+
providers[key] ??= { models: {} };
|
|
2508
|
+
providers[key].models[model.id] = { rates: catalogRates(model.pricing) };
|
|
2509
|
+
}
|
|
2510
|
+
return Object.keys(providers).length === 0 ? {} : { modelCatalog: [{ inline: { providers } }] };
|
|
2511
|
+
}
|
|
2512
|
+
function renderGatewayConfig(manifest) {
|
|
2513
|
+
const { spec } = manifest;
|
|
2514
|
+
const models = [spec.model, ...spec.model.fallback ? [spec.model.fallback] : []];
|
|
2515
|
+
const config = {
|
|
2516
|
+
config: { ...renderRuntimeConfig(spec), ...renderModelCatalog(models) },
|
|
2517
|
+
gateways: { [GATEWAY_NAME]: { port: CONTRACT.ports.gateway, bindAddress: CONTRACT.podPatch.loopback } },
|
|
2518
|
+
llm: {
|
|
2519
|
+
gateways: GATEWAY_NAME,
|
|
2520
|
+
models: models.map(renderModel),
|
|
2521
|
+
virtualModels: [renderVirtualModel(models)],
|
|
2522
|
+
policies: {
|
|
2523
|
+
apiKey: renderApiKeyPolicy(manifest),
|
|
2524
|
+
...renderRateLimits(spec)
|
|
2525
|
+
}
|
|
2526
|
+
},
|
|
2527
|
+
...renderRetryPolicy(spec.model)
|
|
2528
|
+
};
|
|
2529
|
+
return dump(config, { sortKeys: true, lineWidth: -1, noRefs: true });
|
|
2530
|
+
}
|
|
2531
|
+
function describeManifestError(err) {
|
|
2532
|
+
if (!(err instanceof Error)) return String(err);
|
|
2533
|
+
if (!err.message.startsWith("[")) {
|
|
2534
|
+
return err.message.split("\n")[0];
|
|
2535
|
+
}
|
|
2536
|
+
try {
|
|
2537
|
+
const issues = JSON.parse(err.message);
|
|
2538
|
+
return issues.map((i) => `${i.path.join(".") || "root"}: ${i.message}`).join("\n");
|
|
2539
|
+
} catch {
|
|
2540
|
+
return err.message.split("\n")[0];
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
|
|
1389
2544
|
// src/audit/rules/model.rules.ts
|
|
2545
|
+
var isLatestModelId = (id) => id === "latest" || id.endsWith(":latest");
|
|
2546
|
+
function fallbackRetryResult(model) {
|
|
2547
|
+
if (!model.fallback) return { pass: true };
|
|
2548
|
+
const { maxRetries } = model.fallback;
|
|
2549
|
+
const { policies } = renderRetryPolicy(model);
|
|
2550
|
+
const attempts = policies ? policies[0].policy.retry.attempts : 0;
|
|
2551
|
+
const fail = (message, recommendation) => ({
|
|
2552
|
+
pass: false,
|
|
2553
|
+
message,
|
|
2554
|
+
path: "/spec/model/fallback/maxRetries",
|
|
2555
|
+
recommendation,
|
|
2556
|
+
references: []
|
|
2557
|
+
});
|
|
2558
|
+
if (maxRetries === void 0) {
|
|
2559
|
+
return fail(
|
|
2560
|
+
"Fallback model has no maxRetries configured: the gateway renders no retry, so the request that triggers failover fails",
|
|
2561
|
+
"Add maxRetries: 2 to spec.model.fallback"
|
|
2562
|
+
);
|
|
2563
|
+
}
|
|
2564
|
+
if (attempts !== maxRetries) {
|
|
2565
|
+
return fail(
|
|
2566
|
+
`The rendered gateway config sets ${attempts} retry attempts, but spec.model.fallback declares maxRetries ${maxRetries}`,
|
|
2567
|
+
"Redeploy the agent so its gateway config is rendered again from agent.yaml and retries maxRetries times"
|
|
2568
|
+
);
|
|
2569
|
+
}
|
|
2570
|
+
return { pass: true };
|
|
2571
|
+
}
|
|
1390
2572
|
var modelRules = [
|
|
1391
2573
|
{
|
|
1392
2574
|
id: "MODEL-01",
|
|
@@ -1394,9 +2576,7 @@ var modelRules = [
|
|
|
1394
2576
|
title: "Model fallback declared",
|
|
1395
2577
|
description: "A fallback model should be configured to handle rate limits and errors",
|
|
1396
2578
|
severity: "high",
|
|
1397
|
-
|
|
1398
|
-
proofTool: "LiteLLM chaos test",
|
|
1399
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
|
|
2579
|
+
awaitsEvidence: { code: "no_failover_observed", phase: 2 },
|
|
1400
2580
|
check(manifest) {
|
|
1401
2581
|
const hasFallback = !!manifest.spec.model.fallback;
|
|
1402
2582
|
return {
|
|
@@ -1414,17 +2594,14 @@ var modelRules = [
|
|
|
1414
2594
|
title: "Model version pinned",
|
|
1415
2595
|
description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
|
|
1416
2596
|
severity: "medium",
|
|
1417
|
-
evidenceLevel: "probed",
|
|
1418
|
-
proofTool: "agentspec health",
|
|
1419
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1420
2597
|
check(manifest) {
|
|
1421
2598
|
const id = manifest.spec.model.id;
|
|
1422
|
-
const isLatest = id
|
|
2599
|
+
const isLatest = isLatestModelId(id);
|
|
1423
2600
|
return {
|
|
1424
2601
|
pass: !isLatest,
|
|
1425
2602
|
message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
|
|
1426
2603
|
path: "/spec/model/id",
|
|
1427
|
-
recommendation: 'Pin a specific model version, e.g. "
|
|
2604
|
+
recommendation: 'Pin a specific model version, e.g. "openai/gpt-oss-20b" or "gpt-4-0125-preview"',
|
|
1428
2605
|
references: []
|
|
1429
2606
|
};
|
|
1430
2607
|
}
|
|
@@ -1435,9 +2612,7 @@ var modelRules = [
|
|
|
1435
2612
|
title: "Cost controls declared",
|
|
1436
2613
|
description: "Monthly cost limits prevent unexpected API bills",
|
|
1437
2614
|
severity: "medium",
|
|
1438
|
-
|
|
1439
|
-
proofTool: "LiteLLM Spend Tracking",
|
|
1440
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
|
|
2615
|
+
awaitsEvidence: { code: "no_budget_evidence", phase: 2 },
|
|
1441
2616
|
check(manifest) {
|
|
1442
2617
|
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1443
2618
|
return {
|
|
@@ -1453,19 +2628,61 @@ var modelRules = [
|
|
|
1453
2628
|
id: "MODEL-04",
|
|
1454
2629
|
pack: "model-resilience",
|
|
1455
2630
|
title: "Fallback retry strategy configured",
|
|
1456
|
-
description: "Fallback should have maxRetries
|
|
2631
|
+
description: "Fallback should have maxRetries, and the rendered gateway config must retry exactly that many times",
|
|
2632
|
+
severity: "low",
|
|
2633
|
+
check(manifest) {
|
|
2634
|
+
return fallbackRetryResult(manifest.spec.model);
|
|
2635
|
+
}
|
|
2636
|
+
},
|
|
2637
|
+
{
|
|
2638
|
+
id: "MODEL-05",
|
|
2639
|
+
pack: "model-resilience",
|
|
2640
|
+
title: "Budgeted model is priced",
|
|
2641
|
+
description: "A monthly USD budget is only enforced for models the gateway can price",
|
|
2642
|
+
severity: "high",
|
|
2643
|
+
// Only the gateway knows its prices: MODEL-05 is graded from the cost on its spans (#494), never guessed from the model id (#495).
|
|
2644
|
+
check() {
|
|
2645
|
+
return { pass: true };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
];
|
|
2649
|
+
|
|
2650
|
+
// src/audit/rules/metadata.rules.ts
|
|
2651
|
+
var MAX_PRODUCTION_TEMPERATURE = 1.5;
|
|
2652
|
+
var metadataRules = [
|
|
2653
|
+
{
|
|
2654
|
+
id: "META-01",
|
|
2655
|
+
pack: "metadata-quality",
|
|
2656
|
+
title: "Agent description declared",
|
|
2657
|
+
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
1457
2658
|
severity: "low",
|
|
1458
|
-
evidenceLevel: "external",
|
|
1459
|
-
proofTool: "pytest-mockllm",
|
|
1460
|
-
proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
|
|
1461
2659
|
check(manifest) {
|
|
1462
|
-
|
|
1463
|
-
const
|
|
2660
|
+
const desc = manifest.metadata.description;
|
|
2661
|
+
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
2662
|
+
return {
|
|
2663
|
+
pass,
|
|
2664
|
+
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
2665
|
+
path: "/metadata/description",
|
|
2666
|
+
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
2667
|
+
references: []
|
|
2668
|
+
};
|
|
2669
|
+
}
|
|
2670
|
+
},
|
|
2671
|
+
{
|
|
2672
|
+
id: "META-02",
|
|
2673
|
+
pack: "metadata-quality",
|
|
2674
|
+
title: "Model temperature is production-safe",
|
|
2675
|
+
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
2676
|
+
severity: "medium",
|
|
2677
|
+
check(manifest) {
|
|
2678
|
+
const temperature = manifest.spec.model.parameters?.temperature;
|
|
2679
|
+
if (temperature === void 0) return { pass: true };
|
|
2680
|
+
const pass = temperature <= MAX_PRODUCTION_TEMPERATURE;
|
|
1464
2681
|
return {
|
|
1465
|
-
pass
|
|
1466
|
-
message:
|
|
1467
|
-
path: "/spec/model/
|
|
1468
|
-
recommendation: "
|
|
2682
|
+
pass,
|
|
2683
|
+
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above ${MAX_PRODUCTION_TEMPERATURE} produce unreliable outputs in production.`,
|
|
2684
|
+
path: "/spec/model/parameters/temperature",
|
|
2685
|
+
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
1469
2686
|
references: []
|
|
1470
2687
|
};
|
|
1471
2688
|
}
|
|
@@ -1480,9 +2697,6 @@ var securityRules = [
|
|
|
1480
2697
|
title: "Prompt injection guard: input guardrail required",
|
|
1481
2698
|
description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
|
|
1482
2699
|
severity: "high",
|
|
1483
|
-
evidenceLevel: "behavioral",
|
|
1484
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1485
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1486
2700
|
check(manifest) {
|
|
1487
2701
|
const hasGuardrail = manifest.spec.guardrails?.input?.some(
|
|
1488
2702
|
(g) => g.type === "prompt-injection" || g.type === "topic-filter"
|
|
@@ -1504,9 +2718,6 @@ var securityRules = [
|
|
|
1504
2718
|
title: "Insecure output handling: output guardrail required",
|
|
1505
2719
|
description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
|
|
1506
2720
|
severity: "high",
|
|
1507
|
-
evidenceLevel: "behavioral",
|
|
1508
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1509
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1510
2721
|
check(manifest) {
|
|
1511
2722
|
const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
|
|
1512
2723
|
return {
|
|
@@ -1526,9 +2737,6 @@ var securityRules = [
|
|
|
1526
2737
|
title: "Training/prompt data poisoning: prompt files referenced",
|
|
1527
2738
|
description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
|
|
1528
2739
|
severity: "medium",
|
|
1529
|
-
evidenceLevel: "probed",
|
|
1530
|
-
proofTool: "agentspec health",
|
|
1531
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1532
2740
|
check(manifest) {
|
|
1533
2741
|
const system = manifest.spec.prompts.system;
|
|
1534
2742
|
const usesFile = system.startsWith("$file:");
|
|
@@ -1546,21 +2754,16 @@ var securityRules = [
|
|
|
1546
2754
|
{
|
|
1547
2755
|
id: "SEC-LLM-04",
|
|
1548
2756
|
pack: "owasp-llm-top10",
|
|
1549
|
-
title: "Model DoS: rate limiting
|
|
2757
|
+
title: "Model DoS: rate limiting configured",
|
|
1550
2758
|
description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
|
|
1551
2759
|
severity: "medium",
|
|
1552
|
-
evidenceLevel: "external",
|
|
1553
|
-
proofTool: "k6 load test",
|
|
1554
|
-
proofToolUrl: "https://k6.io",
|
|
1555
2760
|
check(manifest) {
|
|
1556
|
-
const
|
|
1557
|
-
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1558
|
-
const pass = hasRateLimit || hasCostControls;
|
|
2761
|
+
const pass = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
|
|
1559
2762
|
return {
|
|
1560
2763
|
pass,
|
|
1561
|
-
message: pass ? void 0 : "No rate limiting
|
|
2764
|
+
message: pass ? void 0 : "No rate limiting declared.",
|
|
1562
2765
|
path: "/spec/api/rateLimit",
|
|
1563
|
-
recommendation: "Add spec.api.rateLimit.requestsPerMinute
|
|
2766
|
+
recommendation: "Add spec.api.rateLimit.requestsPerMinute",
|
|
1564
2767
|
references: [
|
|
1565
2768
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
|
|
1566
2769
|
]
|
|
@@ -1573,9 +2776,6 @@ var securityRules = [
|
|
|
1573
2776
|
title: "Supply chain: model provider and version pinned",
|
|
1574
2777
|
description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
|
|
1575
2778
|
severity: "medium",
|
|
1576
|
-
evidenceLevel: "probed",
|
|
1577
|
-
proofTool: "agentspec health",
|
|
1578
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1579
2779
|
check(manifest) {
|
|
1580
2780
|
const hasProvider = !!manifest.spec.model.provider;
|
|
1581
2781
|
const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
|
|
@@ -1597,9 +2797,6 @@ var securityRules = [
|
|
|
1597
2797
|
title: "Sensitive data disclosure: PII scrub in memory hygiene",
|
|
1598
2798
|
description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
|
|
1599
2799
|
severity: "critical",
|
|
1600
|
-
evidenceLevel: "external",
|
|
1601
|
-
proofTool: "Microsoft Presidio",
|
|
1602
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1603
2800
|
check(manifest) {
|
|
1604
2801
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1605
2802
|
const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1620,9 +2817,6 @@ var securityRules = [
|
|
|
1620
2817
|
title: "Insecure plugin design: tool annotations declared",
|
|
1621
2818
|
description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
|
|
1622
2819
|
severity: "medium",
|
|
1623
|
-
evidenceLevel: "external",
|
|
1624
|
-
proofTool: "Promptfoo",
|
|
1625
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
1626
2820
|
check(manifest) {
|
|
1627
2821
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1628
2822
|
const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
|
|
@@ -1643,9 +2837,7 @@ var securityRules = [
|
|
|
1643
2837
|
title: "Excessive agency: destructiveHint declared on tools",
|
|
1644
2838
|
description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
|
|
1645
2839
|
severity: "high",
|
|
1646
|
-
|
|
1647
|
-
proofTool: "Promptfoo",
|
|
1648
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
2840
|
+
awaitsEvidence: { code: "no_approval_token_evidence", phase: 5 },
|
|
1649
2841
|
check(manifest) {
|
|
1650
2842
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1651
2843
|
const allHaveDestructiveHint = manifest.spec.tools.every(
|
|
@@ -1665,21 +2857,18 @@ var securityRules = [
|
|
|
1665
2857
|
{
|
|
1666
2858
|
id: "SEC-LLM-09",
|
|
1667
2859
|
pack: "owasp-llm-top10",
|
|
1668
|
-
title: "Overreliance: evaluation
|
|
2860
|
+
title: "Overreliance: evaluation + CI gate configured",
|
|
1669
2861
|
description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
|
|
1670
2862
|
severity: "medium",
|
|
1671
|
-
evidenceLevel: "probed",
|
|
1672
|
-
proofTool: "agentspec health",
|
|
1673
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1674
2863
|
check(manifest) {
|
|
1675
|
-
const hasEval =
|
|
2864
|
+
const hasEval = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0 || (manifest.spec.evaluation?.live?.metrics?.length ?? 0) > 0;
|
|
1676
2865
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1677
2866
|
const pass = hasEval && hasCiGate;
|
|
1678
2867
|
return {
|
|
1679
2868
|
pass,
|
|
1680
|
-
message: pass ? void 0 : !hasEval ? "No evaluation
|
|
2869
|
+
message: pass ? void 0 : !hasEval ? "No evaluation datasets or live evaluation configured." : "Evaluation CI gate is not enabled.",
|
|
1681
2870
|
path: "/spec/evaluation",
|
|
1682
|
-
recommendation: "Configure spec.evaluation.
|
|
2871
|
+
recommendation: "Configure spec.evaluation.datasets (or spec.evaluation.live) and set spec.evaluation.ciGate: true",
|
|
1683
2872
|
references: [
|
|
1684
2873
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
|
|
1685
2874
|
]
|
|
@@ -1692,9 +2881,6 @@ var securityRules = [
|
|
|
1692
2881
|
title: "Model theft: API keys use $secret, not $env",
|
|
1693
2882
|
description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
|
|
1694
2883
|
severity: "high",
|
|
1695
|
-
evidenceLevel: "probed",
|
|
1696
|
-
proofTool: "agentspec health",
|
|
1697
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1698
2884
|
check(manifest) {
|
|
1699
2885
|
const apiKey = manifest.spec.model.apiKey;
|
|
1700
2886
|
const usesSecret = apiKey.startsWith("$secret:");
|
|
@@ -1710,6 +2896,18 @@ var securityRules = [
|
|
|
1710
2896
|
]
|
|
1711
2897
|
};
|
|
1712
2898
|
}
|
|
2899
|
+
},
|
|
2900
|
+
{
|
|
2901
|
+
id: "TOOL-01",
|
|
2902
|
+
pack: "owasp-llm-top10",
|
|
2903
|
+
title: "Only declared tools are called",
|
|
2904
|
+
description: "The agent must call only tools declared in spec.tools; undeclared calls are excessive agency",
|
|
2905
|
+
severity: "high",
|
|
2906
|
+
awaitsEvidence: { code: "no_calls_observed", phase: 2 },
|
|
2907
|
+
// The manifest alone cannot show an undeclared call: passes declaratively, verified only from observed spans (#152, D17).
|
|
2908
|
+
check() {
|
|
2909
|
+
return { pass: true };
|
|
2910
|
+
}
|
|
1713
2911
|
}
|
|
1714
2912
|
];
|
|
1715
2913
|
|
|
@@ -1721,9 +2919,6 @@ var memoryRules = [
|
|
|
1721
2919
|
title: "PII scrub fields declared for long-term memory",
|
|
1722
2920
|
description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
|
|
1723
2921
|
severity: "critical",
|
|
1724
|
-
evidenceLevel: "external",
|
|
1725
|
-
proofTool: "Microsoft Presidio",
|
|
1726
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1727
2922
|
check(manifest) {
|
|
1728
2923
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1729
2924
|
const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1742,9 +2937,6 @@ var memoryRules = [
|
|
|
1742
2937
|
title: "Retention policy / TTL set for all memory backends",
|
|
1743
2938
|
description: "Memory without TTL grows unbounded and increases PII exposure risk",
|
|
1744
2939
|
severity: "high",
|
|
1745
|
-
evidenceLevel: "probed",
|
|
1746
|
-
proofTool: "agentspec health",
|
|
1747
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1748
2940
|
check(manifest) {
|
|
1749
2941
|
const mem = manifest.spec.memory;
|
|
1750
2942
|
if (!mem) return { pass: true };
|
|
@@ -1769,9 +2961,6 @@ var memoryRules = [
|
|
|
1769
2961
|
title: "Audit log enabled for long-term memory",
|
|
1770
2962
|
description: "Audit logging enables compliance investigations and debugging",
|
|
1771
2963
|
severity: "medium",
|
|
1772
|
-
evidenceLevel: "probed",
|
|
1773
|
-
proofTool: "agentspec health",
|
|
1774
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1775
2964
|
check(manifest) {
|
|
1776
2965
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1777
2966
|
const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
|
|
@@ -1790,9 +2979,6 @@ var memoryRules = [
|
|
|
1790
2979
|
title: "Vector store namespace isolated",
|
|
1791
2980
|
description: "Shared vector store namespaces risk data leakage between agents",
|
|
1792
2981
|
severity: "medium",
|
|
1793
|
-
evidenceLevel: "probed",
|
|
1794
|
-
proofTool: "agentspec health",
|
|
1795
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1796
2982
|
check(manifest) {
|
|
1797
2983
|
if (!manifest.spec.memory?.vector) return { pass: true };
|
|
1798
2984
|
const hasNamespace = !!manifest.spec.memory.vector.namespace;
|
|
@@ -1811,9 +2997,6 @@ var memoryRules = [
|
|
|
1811
2997
|
title: "Short-term memory max tokens bounded",
|
|
1812
2998
|
description: "Unbounded short-term memory can cause token budget overruns",
|
|
1813
2999
|
severity: "low",
|
|
1814
|
-
evidenceLevel: "probed",
|
|
1815
|
-
proofTool: "agentspec health",
|
|
1816
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1817
3000
|
check(manifest) {
|
|
1818
3001
|
if (!manifest.spec.memory?.shortTerm) return { pass: true };
|
|
1819
3002
|
const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
|
|
@@ -1836,9 +3019,6 @@ var evaluationRules = [
|
|
|
1836
3019
|
title: "Evaluation dataset declared",
|
|
1837
3020
|
description: "At least one evaluation dataset is required for quality assurance",
|
|
1838
3021
|
severity: "medium",
|
|
1839
|
-
evidenceLevel: "probed",
|
|
1840
|
-
proofTool: "agentspec health",
|
|
1841
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1842
3022
|
check(manifest) {
|
|
1843
3023
|
const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
|
|
1844
3024
|
return {
|
|
@@ -1856,9 +3036,6 @@ var evaluationRules = [
|
|
|
1856
3036
|
title: "CI gate enabled",
|
|
1857
3037
|
description: "Evaluation CI gate prevents regressions from reaching production",
|
|
1858
3038
|
severity: "medium",
|
|
1859
|
-
evidenceLevel: "probed",
|
|
1860
|
-
proofTool: "agentspec health",
|
|
1861
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1862
3039
|
check(manifest) {
|
|
1863
3040
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1864
3041
|
return {
|
|
@@ -1873,22 +3050,22 @@ var evaluationRules = [
|
|
|
1873
3050
|
{
|
|
1874
3051
|
id: "EVAL-03",
|
|
1875
3052
|
pack: "evaluation-coverage",
|
|
1876
|
-
title: "
|
|
1877
|
-
description: "A
|
|
3053
|
+
title: "Grounded metric threshold configured",
|
|
3054
|
+
description: "A grounded threshold (minimum pass rate, 0.95 recommended) is required for production agents",
|
|
1878
3055
|
severity: "medium",
|
|
1879
|
-
evidenceLevel: "probed",
|
|
1880
|
-
proofTool: "agentspec health",
|
|
1881
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1882
3056
|
check(manifest) {
|
|
1883
|
-
const
|
|
1884
|
-
const
|
|
1885
|
-
const
|
|
1886
|
-
const
|
|
3057
|
+
const evaluation = manifest.spec.evaluation;
|
|
3058
|
+
const scopes = [evaluation, evaluation?.live];
|
|
3059
|
+
const hasGrounded = (scope) => !!scope?.metrics?.map(canonicalMetric).includes("grounded");
|
|
3060
|
+
const hasGroundedMetric = scopes.some(hasGrounded);
|
|
3061
|
+
const pass = scopes.some(
|
|
3062
|
+
(scope) => hasGrounded(scope) && Object.keys(scope?.thresholds ?? {}).some((n) => canonicalMetric(n) === "grounded")
|
|
3063
|
+
);
|
|
1887
3064
|
return {
|
|
1888
3065
|
pass,
|
|
1889
|
-
message: pass ? void 0 : !
|
|
1890
|
-
path: "/spec/evaluation/thresholds/
|
|
1891
|
-
recommendation: "Add
|
|
3066
|
+
message: pass ? void 0 : !hasGroundedMetric ? "grounded metric not in spec.evaluation.metrics or spec.evaluation.live.metrics" : "No grounded threshold next to the grounded metric",
|
|
3067
|
+
path: "/spec/evaluation/thresholds/grounded",
|
|
3068
|
+
recommendation: "Add grounded to spec.evaluation.metrics (or spec.evaluation.live.metrics) and set the matching thresholds.grounded: 0.95",
|
|
1892
3069
|
references: []
|
|
1893
3070
|
};
|
|
1894
3071
|
}
|
|
@@ -1903,9 +3080,6 @@ var observabilityRules = [
|
|
|
1903
3080
|
title: "Tracing backend declared",
|
|
1904
3081
|
description: "Agents should declare a tracing backend so runs are observable and debuggable",
|
|
1905
3082
|
severity: "medium",
|
|
1906
|
-
evidenceLevel: "probed",
|
|
1907
|
-
proofTool: "agentspec health",
|
|
1908
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1909
3083
|
check(manifest) {
|
|
1910
3084
|
const pass = !!manifest.spec.observability?.tracing?.backend;
|
|
1911
3085
|
return {
|
|
@@ -1922,9 +3096,6 @@ var observabilityRules = [
|
|
|
1922
3096
|
title: "Structured logging enabled",
|
|
1923
3097
|
description: "Structured logging makes log aggregation and searching reliable",
|
|
1924
3098
|
severity: "low",
|
|
1925
|
-
evidenceLevel: "behavioral",
|
|
1926
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1927
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1928
3099
|
check(manifest) {
|
|
1929
3100
|
const pass = manifest.spec.observability?.logging?.structured !== false;
|
|
1930
3101
|
return {
|
|
@@ -1941,9 +3112,6 @@ var observabilityRules = [
|
|
|
1941
3112
|
title: "Sensitive fields redacted from logs",
|
|
1942
3113
|
description: "Log redaction prevents secrets and PII from appearing in log aggregators",
|
|
1943
3114
|
severity: "medium",
|
|
1944
|
-
evidenceLevel: "external",
|
|
1945
|
-
proofTool: "Microsoft Presidio",
|
|
1946
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1947
3115
|
check(manifest) {
|
|
1948
3116
|
const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
|
|
1949
3117
|
return {
|
|
@@ -1956,57 +3124,8 @@ var observabilityRules = [
|
|
|
1956
3124
|
}
|
|
1957
3125
|
];
|
|
1958
3126
|
|
|
1959
|
-
// src/audit/rules/metadata.rules.ts
|
|
1960
|
-
var metadataRules = [
|
|
1961
|
-
{
|
|
1962
|
-
id: "META-01",
|
|
1963
|
-
pack: "metadata-quality",
|
|
1964
|
-
title: "Agent description declared",
|
|
1965
|
-
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
1966
|
-
severity: "low",
|
|
1967
|
-
evidenceLevel: "declarative",
|
|
1968
|
-
check(manifest) {
|
|
1969
|
-
const desc = manifest.metadata.description;
|
|
1970
|
-
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
1971
|
-
return {
|
|
1972
|
-
pass,
|
|
1973
|
-
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
1974
|
-
path: "/metadata/description",
|
|
1975
|
-
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
1976
|
-
references: []
|
|
1977
|
-
};
|
|
1978
|
-
}
|
|
1979
|
-
},
|
|
1980
|
-
{
|
|
1981
|
-
id: "META-02",
|
|
1982
|
-
pack: "metadata-quality",
|
|
1983
|
-
title: "Model temperature is production-safe",
|
|
1984
|
-
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
1985
|
-
severity: "medium",
|
|
1986
|
-
evidenceLevel: "declarative",
|
|
1987
|
-
check(manifest) {
|
|
1988
|
-
const temperature = manifest.spec.model.parameters?.temperature;
|
|
1989
|
-
if (temperature === void 0) return { pass: true };
|
|
1990
|
-
const pass = temperature <= 1.5;
|
|
1991
|
-
return {
|
|
1992
|
-
pass,
|
|
1993
|
-
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above 1.5 produce unreliable outputs in production.`,
|
|
1994
|
-
path: "/spec/model/parameters/temperature",
|
|
1995
|
-
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
1996
|
-
references: []
|
|
1997
|
-
};
|
|
1998
|
-
}
|
|
1999
|
-
}
|
|
2000
|
-
];
|
|
2001
|
-
|
|
2002
3127
|
// src/audit/index.ts
|
|
2003
|
-
var SEVERITY_WEIGHTS =
|
|
2004
|
-
critical: 4,
|
|
2005
|
-
high: 3,
|
|
2006
|
-
medium: 2,
|
|
2007
|
-
low: 1,
|
|
2008
|
-
info: 0
|
|
2009
|
-
};
|
|
3128
|
+
var SEVERITY_WEIGHTS = CONTRACT.severityWeights;
|
|
2010
3129
|
var ALL_RULES = [
|
|
2011
3130
|
...modelRules,
|
|
2012
3131
|
...securityRules,
|
|
@@ -2016,19 +3135,10 @@ var ALL_RULES = [
|
|
|
2016
3135
|
...metadataRules
|
|
2017
3136
|
];
|
|
2018
3137
|
var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
|
|
2019
|
-
|
|
2020
|
-
return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
|
|
2021
|
-
}
|
|
2022
|
-
function scoreToGrade(score) {
|
|
2023
|
-
if (score >= 90) return "A";
|
|
2024
|
-
if (score >= 75) return "B";
|
|
2025
|
-
if (score >= 60) return "C";
|
|
2026
|
-
if (score >= 45) return "D";
|
|
2027
|
-
return "F";
|
|
2028
|
-
}
|
|
3138
|
+
var COMPLIANCE_PACKS = [...new Set(ALL_RULES.map((r) => r.pack))];
|
|
2029
3139
|
function resolveActiveRules(manifest, opts) {
|
|
2030
3140
|
const manifestPacks = manifest.spec.compliance?.packs ?? [];
|
|
2031
|
-
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks :
|
|
3141
|
+
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : COMPLIANCE_PACKS);
|
|
2032
3142
|
return ALL_RULES.filter((r) => activePacks.includes(r.pack));
|
|
2033
3143
|
}
|
|
2034
3144
|
function collectSuppressions(manifest) {
|
|
@@ -2057,10 +3167,10 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2057
3167
|
const violations = [];
|
|
2058
3168
|
const packBreakdown = {};
|
|
2059
3169
|
const evidenceBreakdown = {
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
3170
|
+
independent: { passed: 0, total: 0 },
|
|
3171
|
+
"evidence-point": { passed: 0, total: 0 },
|
|
3172
|
+
offline: { passed: 0, total: 0 },
|
|
3173
|
+
declared: { passed: 0, total: 0 }
|
|
2064
3174
|
};
|
|
2065
3175
|
let totalWeight = 0;
|
|
2066
3176
|
let passedWeight = 0;
|
|
@@ -2071,19 +3181,20 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2071
3181
|
const suppressed = suppressedIds.has(rule.id);
|
|
2072
3182
|
const weight = SEVERITY_WEIGHTS[rule.severity];
|
|
2073
3183
|
const result = ruleResults.get(rule.id);
|
|
2074
|
-
const
|
|
3184
|
+
const classes = ruleClasses(rule.id);
|
|
2075
3185
|
packBreakdown[rule.pack].total += 1;
|
|
2076
|
-
|
|
3186
|
+
tally(evidenceBreakdown, classes, "total");
|
|
2077
3187
|
if (suppressed) {
|
|
2078
3188
|
packBreakdown[rule.pack].passed += 1;
|
|
2079
|
-
|
|
3189
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2080
3190
|
continue;
|
|
2081
3191
|
}
|
|
2082
3192
|
if (result.pass) {
|
|
2083
3193
|
packBreakdown[rule.pack].passed += 1;
|
|
2084
|
-
|
|
3194
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2085
3195
|
passedWeight += weight;
|
|
2086
3196
|
} else {
|
|
3197
|
+
const hint = proofHint(rule.id);
|
|
2087
3198
|
violations.push({
|
|
2088
3199
|
ruleId: rule.id,
|
|
2089
3200
|
severity: rule.severity,
|
|
@@ -2092,9 +3203,9 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2092
3203
|
path: result.path,
|
|
2093
3204
|
recommendation: result.recommendation,
|
|
2094
3205
|
references: result.references,
|
|
2095
|
-
|
|
2096
|
-
proofTool:
|
|
2097
|
-
proofToolUrl:
|
|
3206
|
+
evidenceClasses: classes,
|
|
3207
|
+
proofTool: hint?.tool,
|
|
3208
|
+
proofToolUrl: hint?.url
|
|
2098
3209
|
});
|
|
2099
3210
|
}
|
|
2100
3211
|
totalWeight += weight;
|
|
@@ -2106,24 +3217,36 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2106
3217
|
}
|
|
2107
3218
|
return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
|
|
2108
3219
|
}
|
|
2109
|
-
function
|
|
2110
|
-
const
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
const
|
|
2120
|
-
if (
|
|
3220
|
+
function tally(breakdown, classes, field) {
|
|
3221
|
+
for (const c of classes) breakdown[c][field] += 1;
|
|
3222
|
+
}
|
|
3223
|
+
var SIDECAR_GAP = { tool: "AgentSpec sidecar gap analysis", url: "https://agentspec.io/docs/concepts/probe-coverage" };
|
|
3224
|
+
var COLLECTOR_HINTS = {
|
|
3225
|
+
"sidecar-probe": SIDECAR_GAP,
|
|
3226
|
+
"sidecar-forwarder": SIDECAR_GAP
|
|
3227
|
+
};
|
|
3228
|
+
function proofHint(ruleId) {
|
|
3229
|
+
for (const { collector, status } of ruleSources(ruleId)) {
|
|
3230
|
+
const hint = COLLECTOR_HINTS[collector];
|
|
3231
|
+
if (status === "built" && hint) return { collector, ...hint };
|
|
2121
3232
|
}
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
).
|
|
2126
|
-
return {
|
|
3233
|
+
return void 0;
|
|
3234
|
+
}
|
|
3235
|
+
function explainPendingEvidence(verified, rules) {
|
|
3236
|
+
const pending = new Map(rules.filter((r) => r.awaitsEvidence).map((r) => [r.id, r.awaitsEvidence]));
|
|
3237
|
+
return {
|
|
3238
|
+
...verified,
|
|
3239
|
+
unverified: verified.unverified.map((u) => {
|
|
3240
|
+
const awaits = pending.get(u.ruleId);
|
|
3241
|
+
if (!awaits || u.reason.code !== "no_evidence") return u;
|
|
3242
|
+
return { ...u, reason: { code: awaits.code, detail: `evidence source arrives in phase ${awaits.phase}` } };
|
|
3243
|
+
})
|
|
3244
|
+
};
|
|
3245
|
+
}
|
|
3246
|
+
function summariseRuleEvidence(rules) {
|
|
3247
|
+
return Object.fromEntries(
|
|
3248
|
+
rules.map((r) => [r.id, { classes: ruleClasses(r.id), notProven: RULE_EVIDENCE[r.id]?.notProven ?? null }])
|
|
3249
|
+
);
|
|
2127
3250
|
}
|
|
2128
3251
|
function runAudit(manifest, opts = {}) {
|
|
2129
3252
|
const rules = resolveActiveRules(manifest, opts);
|
|
@@ -2133,11 +3256,21 @@ function runAudit(manifest, opts = {}) {
|
|
|
2133
3256
|
violations,
|
|
2134
3257
|
packBreakdown,
|
|
2135
3258
|
evidenceBreakdown,
|
|
2136
|
-
totalWeight,
|
|
2137
3259
|
overallScore,
|
|
2138
3260
|
categoryScores
|
|
2139
3261
|
} = computeScoring(rules, ruleResults, suppressedIds);
|
|
2140
|
-
const
|
|
3262
|
+
const applicable = rules.filter((r) => !suppressedIds.has(r.id));
|
|
3263
|
+
const applicableRules = applicable.map((r) => ({ id: r.id, severity: r.severity }));
|
|
3264
|
+
const verified = explainPendingEvidence(
|
|
3265
|
+
grade(
|
|
3266
|
+
opts.records ?? [],
|
|
3267
|
+
applicableRules,
|
|
3268
|
+
opts.trust ?? "standard",
|
|
3269
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
3270
|
+
opts.evidencePoints
|
|
3271
|
+
),
|
|
3272
|
+
applicable
|
|
3273
|
+
);
|
|
2141
3274
|
return {
|
|
2142
3275
|
agentName: manifest.metadata.name,
|
|
2143
3276
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2150,9 +3283,26 @@ function runAudit(manifest, opts = {}) {
|
|
|
2150
3283
|
totalRules: rules.length,
|
|
2151
3284
|
packBreakdown,
|
|
2152
3285
|
evidenceBreakdown,
|
|
2153
|
-
|
|
3286
|
+
verified,
|
|
3287
|
+
applicableRules,
|
|
3288
|
+
ruleEvidence: summariseRuleEvidence(rules)
|
|
2154
3289
|
};
|
|
2155
3290
|
}
|
|
3291
|
+
function canonicalJson(value) {
|
|
3292
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
3293
|
+
if (value !== null && typeof value === "object") {
|
|
3294
|
+
const obj = value;
|
|
3295
|
+
const members = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
|
|
3296
|
+
return `{${members.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
|
|
3297
|
+
}
|
|
3298
|
+
return JSON.stringify(value);
|
|
3299
|
+
}
|
|
3300
|
+
function canonicalManifestSha256(manifest) {
|
|
3301
|
+
return createHash("sha256").update(canonicalJson(manifest), "utf8").digest("hex");
|
|
3302
|
+
}
|
|
3303
|
+
function declaredManifestSha256(raw) {
|
|
3304
|
+
return canonicalManifestSha256(yaml.load(raw, { schema: yaml.JSON_SCHEMA }));
|
|
3305
|
+
}
|
|
2156
3306
|
|
|
2157
3307
|
// src/generate/index.ts
|
|
2158
3308
|
var _registry = /* @__PURE__ */ new Map();
|
|
@@ -2165,7 +3315,7 @@ function getAdapter(framework) {
|
|
|
2165
3315
|
function listAdapters() {
|
|
2166
3316
|
return [..._registry.keys()];
|
|
2167
3317
|
}
|
|
2168
|
-
function generateAdapter(manifest, framework,
|
|
3318
|
+
function generateAdapter(manifest, framework, options2) {
|
|
2169
3319
|
const adapter = _registry.get(framework.toLowerCase());
|
|
2170
3320
|
if (!adapter) {
|
|
2171
3321
|
throw new Error(
|
|
@@ -2174,7 +3324,7 @@ function generateAdapter(manifest, framework, options) {
|
|
|
2174
3324
|
Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
|
|
2175
3325
|
);
|
|
2176
3326
|
}
|
|
2177
|
-
return adapter.generate(manifest,
|
|
3327
|
+
return adapter.generate(manifest, options2);
|
|
2178
3328
|
}
|
|
2179
3329
|
|
|
2180
3330
|
// src/agent/usage-ledger.ts
|
|
@@ -2220,8 +3370,8 @@ var UsageLedger = class {
|
|
|
2220
3370
|
* Return a frozen snapshot of the current usage window.
|
|
2221
3371
|
*
|
|
2222
3372
|
* @param reset - When true (default), clears all counters and starts a new
|
|
2223
|
-
* window. Use reset=
|
|
2224
|
-
*
|
|
3373
|
+
* window. Use reset=false for inspection without draining; the heartbeat
|
|
3374
|
+
* push takes a reset=false snapshot and calls subtract() with it.
|
|
2225
3375
|
*/
|
|
2226
3376
|
snapshot(reset = true) {
|
|
2227
3377
|
const snap = buildSnapshot(this.counters, this.windowStart);
|
|
@@ -2231,39 +3381,220 @@ var UsageLedger = class {
|
|
|
2231
3381
|
}
|
|
2232
3382
|
return snap;
|
|
2233
3383
|
}
|
|
3384
|
+
/**
|
|
3385
|
+
* Remove a shipped snapshot from the ledger; a new window starts, because the control plane dedupes heartbeats on windowStartedAt.
|
|
3386
|
+
* Usage recorded after the snapshot was taken stays, so every token is
|
|
3387
|
+
* shipped exactly once even when a call finishes while a heartbeat is in flight.
|
|
3388
|
+
*/
|
|
3389
|
+
subtract(shipped) {
|
|
3390
|
+
for (const s of shipped.models) {
|
|
3391
|
+
const entry = this.counters.get(s.modelId);
|
|
3392
|
+
if (!entry) continue;
|
|
3393
|
+
entry.promptTokens -= s.promptTokens;
|
|
3394
|
+
entry.completionTokens -= s.completionTokens;
|
|
3395
|
+
entry.totalTokens -= s.totalTokens;
|
|
3396
|
+
entry.callCount -= s.callCount;
|
|
3397
|
+
if (entry.callCount <= 0) this.counters.delete(s.modelId);
|
|
3398
|
+
}
|
|
3399
|
+
this.windowStart = /* @__PURE__ */ new Date();
|
|
3400
|
+
}
|
|
2234
3401
|
};
|
|
2235
3402
|
|
|
3403
|
+
// src/runtime/evidence-points.ts
|
|
3404
|
+
var GUARDRAIL = EvidencePoint.enum.guardrail;
|
|
3405
|
+
var [MCP_LIST, MCP_CALL] = Object.keys(CONTRACT.evidencePoints.mcpClient);
|
|
3406
|
+
var [SCRUB, TTL, AUDIT] = Object.keys(CONTRACT.evidencePoints.memoryWrite);
|
|
3407
|
+
var [NAMESPACE] = Object.keys(CONTRACT.evidencePoints.vectorStore);
|
|
3408
|
+
var [TOOL_TARGET] = Object.keys(CONTRACT.evidencePoints.tool);
|
|
3409
|
+
var [LOGGING_TARGET] = Object.keys(CONTRACT.evidencePoints.logging);
|
|
3410
|
+
var keyOf = (point2, target) => `${point2}\0${target}`;
|
|
3411
|
+
var DEFAULT_KEY = /* @__PURE__ */ Symbol.for("agentspec.runtime.defaultEvidencePoints");
|
|
3412
|
+
var defaults = globalThis;
|
|
3413
|
+
var rec = (value) => value && typeof value === "object" ? value : {};
|
|
3414
|
+
var stdioServers = (spec) => (Array.isArray(rec(spec.mcp).servers) ? rec(spec.mcp).servers : []).filter((s) => s?.transport === "stdio");
|
|
3415
|
+
var isDeclared = (value) => Array.isArray(value) ? value.length > 0 : Boolean(value);
|
|
3416
|
+
function memoryWriteTargets(memory) {
|
|
3417
|
+
const hygiene = rec(memory.hygiene);
|
|
3418
|
+
const targets = /* @__PURE__ */ new Set();
|
|
3419
|
+
if (isDeclared(hygiene.piiScrubFields)) targets.add(SCRUB);
|
|
3420
|
+
if ([rec(memory.shortTerm).ttlSeconds, rec(memory.longTerm).ttlDays].some((ttl) => ttl !== void 0 && ttl !== null)) targets.add(TTL);
|
|
3421
|
+
if (memory.longTerm && hygiene.auditLog === true) targets.add(AUDIT);
|
|
3422
|
+
return targets;
|
|
3423
|
+
}
|
|
3424
|
+
function declaredTargets(spec) {
|
|
3425
|
+
const all = (point2, declared2) => new Set(declared2 ? Object.keys(CONTRACT.evidencePoints[point2] ?? {}) : []);
|
|
3426
|
+
const memory = rec(spec.memory);
|
|
3427
|
+
const logging = rec(rec(spec.observability).logging);
|
|
3428
|
+
return {
|
|
3429
|
+
[EvidencePoint.enum.memoryWrite]: memoryWriteTargets(memory),
|
|
3430
|
+
[EvidencePoint.enum.vectorStore]: new Set(rec(memory.vector).namespace ? [NAMESPACE] : []),
|
|
3431
|
+
[GUARDRAIL]: new Set(Object.keys(CONTRACT.evidencePoints[GUARDRAIL] ?? {}).filter((t) => isDeclared(rec(spec.guardrails)[t]))),
|
|
3432
|
+
[EvidencePoint.enum.tool]: all(EvidencePoint.enum.tool, (Array.isArray(spec.tools) ? spec.tools : []).some((t) => t?.type === "function")),
|
|
3433
|
+
[EvidencePoint.enum.mcpClient]: all(EvidencePoint.enum.mcpClient, stdioServers(spec).length > 0),
|
|
3434
|
+
[EvidencePoint.enum.logging]: all(EvidencePoint.enum.logging, rec(spec.observability).logging !== void 0 && logging.structured !== false)
|
|
3435
|
+
};
|
|
3436
|
+
}
|
|
3437
|
+
function applicablePoints(spec) {
|
|
3438
|
+
const declared2 = declaredTargets(spec);
|
|
3439
|
+
const applicable = /* @__PURE__ */ new Map();
|
|
3440
|
+
for (const [point2, targets] of Object.entries(CONTRACT.evidencePoints)) {
|
|
3441
|
+
for (const [target, ruleId] of Object.entries(targets)) {
|
|
3442
|
+
const listed = CONTRACT.ruleEvidence[ruleId]?.sources.some((s) => s.point === point2);
|
|
3443
|
+
if (listed && declared2[point2]?.has(target)) applicable.set(keyOf(point2, target), { point: EvidencePoint.parse(point2), target, ruleId });
|
|
3444
|
+
}
|
|
3445
|
+
}
|
|
3446
|
+
return applicable;
|
|
3447
|
+
}
|
|
3448
|
+
function createEvidencePoints(manifest) {
|
|
3449
|
+
const manifestSha256 = canonicalManifestSha256(manifest);
|
|
3450
|
+
const spec = rec(rec(manifest).spec);
|
|
3451
|
+
const applicable = applicablePoints(spec);
|
|
3452
|
+
const mcpServers = stdioServers(spec).map((s) => String(s.name));
|
|
3453
|
+
const wired = /* @__PURE__ */ new Set();
|
|
3454
|
+
const emptyWindow = () => new Map([...applicable.keys()].map((k) => [k, { total: 0, counts: { pass: 0, fail: 0, not_verifiable: 0 }, last: [] }]));
|
|
3455
|
+
let calls = emptyWindow();
|
|
3456
|
+
let windowStart = /* @__PURE__ */ new Date();
|
|
3457
|
+
let seq = 0;
|
|
3458
|
+
const shippedUpTo = /* @__PURE__ */ new WeakMap();
|
|
3459
|
+
const points = {
|
|
3460
|
+
register: (point2, target) => {
|
|
3461
|
+
wired.add(keyOf(point2, target));
|
|
3462
|
+
},
|
|
3463
|
+
applies: (point2, target) => applicable.has(keyOf(point2, target)),
|
|
3464
|
+
memorySpec: () => rec(spec.memory),
|
|
3465
|
+
declaredMcpServers: () => [...mcpServers],
|
|
3466
|
+
recordCall: (point2, target, { subject, status = "pass", reason } = {}) => {
|
|
3467
|
+
if (status !== "pass" && !reason) throw new Error(`recordCall: a ${status} call of ${point2}:${target} needs a reason`);
|
|
3468
|
+
const c = calls.get(keyOf(point2, target));
|
|
3469
|
+
if (!c) return;
|
|
3470
|
+
c.total += 1;
|
|
3471
|
+
c.counts[status] += 1;
|
|
3472
|
+
c.last.push({ at: /* @__PURE__ */ new Date(), seq: ++seq, subject, status, reason });
|
|
3473
|
+
if (c.last.length > CONTRACT.evidencePointRecordLimit) c.last.shift();
|
|
3474
|
+
},
|
|
3475
|
+
snapshot: (ttlSeconds, reset = true) => {
|
|
3476
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
3477
|
+
const lastSeq = /* @__PURE__ */ new Map();
|
|
3478
|
+
const window = { start: windowStart.toISOString(), end: windowEnd.toISOString() };
|
|
3479
|
+
const aggregates = [...applicable].map(([key, { point: point2, target, ruleId }]) => {
|
|
3480
|
+
const c = calls.get(key);
|
|
3481
|
+
const state = c.total ? "fired" : wired.has(key) ? "wired" : "not_wired";
|
|
3482
|
+
lastSeq.set(key, seq);
|
|
3483
|
+
const records = c.last.map(({ at, subject, status, reason }) => ({
|
|
3484
|
+
checkId: subject ?? `${point2}:${target}`,
|
|
3485
|
+
ruleId,
|
|
3486
|
+
status: EvidenceStatus.enum[status],
|
|
3487
|
+
...reason ? { reason } : {},
|
|
3488
|
+
source: EvidenceSource.enum["sdk-observed"],
|
|
3489
|
+
origin: EvidenceOrigin.enum.in_process,
|
|
3490
|
+
collector: Collector.enum.sdk,
|
|
3491
|
+
observedAt: at.toISOString(),
|
|
3492
|
+
window,
|
|
3493
|
+
ttlSeconds,
|
|
3494
|
+
sampleSize: c.total,
|
|
3495
|
+
traceIds: [],
|
|
3496
|
+
manifestSha256
|
|
3497
|
+
}));
|
|
3498
|
+
return { point: point2, target, state, counts: { ...c.counts }, records };
|
|
3499
|
+
});
|
|
3500
|
+
if (reset) {
|
|
3501
|
+
calls = emptyWindow();
|
|
3502
|
+
windowStart = windowEnd;
|
|
3503
|
+
} else {
|
|
3504
|
+
shippedUpTo.set(aggregates, { lastSeq, windowEnd });
|
|
3505
|
+
}
|
|
3506
|
+
return aggregates;
|
|
3507
|
+
},
|
|
3508
|
+
subtract: (shipped) => {
|
|
3509
|
+
const taken = shippedUpTo.get(shipped);
|
|
3510
|
+
if (!taken) throw new Error("EvidencePoints.subtract: pass the array snapshot() returned");
|
|
3511
|
+
for (const a of shipped) {
|
|
3512
|
+
const key = keyOf(a.point, a.target);
|
|
3513
|
+
const c = calls.get(key);
|
|
3514
|
+
c.total -= a.counts.pass + a.counts.fail + a.counts.not_verifiable;
|
|
3515
|
+
for (const status of ["pass", "fail", "not_verifiable"]) c.counts[status] -= a.counts[status];
|
|
3516
|
+
c.last = c.last.filter((l) => l.seq > taken.lastSeq.get(key));
|
|
3517
|
+
}
|
|
3518
|
+
windowStart = taken.windowEnd;
|
|
3519
|
+
}
|
|
3520
|
+
};
|
|
3521
|
+
defaults[DEFAULT_KEY] = points;
|
|
3522
|
+
return points;
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
// src/health/tool-check-id.ts
|
|
3526
|
+
function slugify(s) {
|
|
3527
|
+
return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3528
|
+
}
|
|
3529
|
+
function toolCheckId(toolName) {
|
|
3530
|
+
return `tool:${slugify(toolName)}`;
|
|
3531
|
+
}
|
|
3532
|
+
|
|
3533
|
+
// src/runtime/call-points.ts
|
|
3534
|
+
var { tool: TOOL, mcpClient: MCP_CLIENT, logging: LOGGING } = EvidencePoint.enum;
|
|
3535
|
+
var { memoryWrite: MEMORY_WRITE, vectorStore: VECTOR_STORE } = EvidencePoint.enum;
|
|
3536
|
+
var NOT_VERIFIABLE_CODE = "instrumentation_unavailable";
|
|
3537
|
+
var WRONG_NAMESPACE_CODE = "undeclared_namespace";
|
|
3538
|
+
var NO_NAMESPACE_CODE = "no_evidence";
|
|
3539
|
+
for (const code of [NOT_VERIFIABLE_CODE, WRONG_NAMESPACE_CODE, NO_NAMESPACE_CODE]) {
|
|
3540
|
+
if (!CONTRACT.reasonCodes.includes(code)) throw new Error(`contract.reasonCodes lacks '${code}'`);
|
|
3541
|
+
}
|
|
3542
|
+
new AsyncLocalStorage();
|
|
3543
|
+
|
|
2236
3544
|
// src/agent/reporter.ts
|
|
3545
|
+
var HEARTBEAT_POST_TIMEOUT_MS = 1e4;
|
|
3546
|
+
var DEFAULT_PUSH_INTERVAL_SECONDS = 30;
|
|
3547
|
+
var RECORD_TTL_INTERVALS = 2;
|
|
3548
|
+
function buildCappedBody(health, aggregates, usage) {
|
|
3549
|
+
const trimmedChecks = [...health.checks];
|
|
3550
|
+
const serialise = (checks, evidencePoints) => JSON.stringify({
|
|
3551
|
+
contract: CONTRACT.version,
|
|
3552
|
+
health: { ...health, checks },
|
|
3553
|
+
records: [],
|
|
3554
|
+
// the control plane reads the records inside each aggregate (#599)
|
|
3555
|
+
evidencePoints,
|
|
3556
|
+
usage
|
|
3557
|
+
});
|
|
3558
|
+
const size = (body2) => Buffer.byteLength(body2, "utf8");
|
|
3559
|
+
let body = serialise(trimmedChecks, aggregates);
|
|
3560
|
+
if (size(body) > CONTRACT.heartbeatMaxBytes) {
|
|
3561
|
+
aggregates = aggregates.map((a) => ({ ...a, records: [] }));
|
|
3562
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3563
|
+
}
|
|
3564
|
+
while (size(body) > CONTRACT.heartbeatMaxBytes && trimmedChecks.length > 0) {
|
|
3565
|
+
trimmedChecks.pop();
|
|
3566
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3567
|
+
}
|
|
3568
|
+
return { body, bytes: size(body) };
|
|
3569
|
+
}
|
|
2237
3570
|
var AgentSpecReporter = class {
|
|
2238
3571
|
constructor(manifest, opts = {}) {
|
|
2239
3572
|
this.manifest = manifest;
|
|
2240
3573
|
this.opts = opts;
|
|
2241
3574
|
this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
|
|
2242
3575
|
this.staleAfterMs = opts.staleAfterMs ?? 6e4;
|
|
3576
|
+
this.evidence = createEvidencePoints(opts.rawManifest ?? manifest);
|
|
2243
3577
|
}
|
|
3578
|
+
manifest;
|
|
3579
|
+
opts;
|
|
2244
3580
|
cached = null;
|
|
2245
3581
|
timer = null;
|
|
2246
3582
|
_pushTimer = null;
|
|
2247
3583
|
refreshIntervalMs;
|
|
2248
3584
|
staleAfterMs;
|
|
2249
|
-
|
|
3585
|
+
inflight = null;
|
|
3586
|
+
/** True while a heartbeat POST is in flight; a tick that finds it set sends nothing. */
|
|
3587
|
+
heartbeatInFlight = false;
|
|
2250
3588
|
stopped = false;
|
|
2251
|
-
registeredTools = /* @__PURE__ */ new Set();
|
|
2252
3589
|
/** In-process token usage accumulator. */
|
|
2253
3590
|
usage = new UsageLedger();
|
|
3591
|
+
/** Evidence points the agent's code wires (`guardrail(...)` reports here); each heartbeat sends the window's aggregates. */
|
|
3592
|
+
evidence;
|
|
2254
3593
|
/**
|
|
2255
|
-
*
|
|
2256
|
-
*
|
|
2257
|
-
* Declared tools that are not registered report status: 'fail'.
|
|
2258
|
-
*
|
|
2259
|
-
* Call this once per tool after wiring up your tool handlers:
|
|
2260
|
-
* reporter.registerTool('search-arxiv', handler)
|
|
2261
|
-
*
|
|
2262
|
-
* The handler argument is accepted but not called — it is retained solely so
|
|
2263
|
-
* callers can pass the real function object for documentation purposes.
|
|
3594
|
+
* @deprecated No-op, kept so existing callers compile. Registering a handler proves nothing about
|
|
3595
|
+
* which tools the agent calls, so the health report no longer carries `tool:<name>` checks (ADR-0017).
|
|
2264
3596
|
*/
|
|
2265
|
-
registerTool(
|
|
2266
|
-
this.registeredTools.add(name);
|
|
3597
|
+
registerTool(_name, _handler) {
|
|
2267
3598
|
}
|
|
2268
3599
|
/**
|
|
2269
3600
|
* Start background refresh. Call once during app startup.
|
|
@@ -2327,52 +3658,74 @@ var AgentSpecReporter = class {
|
|
|
2327
3658
|
*/
|
|
2328
3659
|
startPushMode(opts) {
|
|
2329
3660
|
if (this._pushTimer !== null) return;
|
|
2330
|
-
const
|
|
2331
|
-
|
|
3661
|
+
const controlPlaneUrl = opts.controlPlaneUrl || process.env[CONTRACT.env.controlPlaneUrl];
|
|
3662
|
+
if (!controlPlaneUrl) {
|
|
3663
|
+
throw new Error(
|
|
3664
|
+
`startPushMode: no control plane URL \u2014 pass controlPlaneUrl or set ${CONTRACT.env.controlPlaneUrl}`
|
|
3665
|
+
);
|
|
3666
|
+
}
|
|
3667
|
+
if (!opts.apiKey) {
|
|
3668
|
+
throw new Error("startPushMode: apiKey is required (the per-agent token from POST /api/v1/register)");
|
|
3669
|
+
}
|
|
3670
|
+
const resolved = { ...opts, controlPlaneUrl };
|
|
3671
|
+
const intervalMs = (opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * 1e3;
|
|
3672
|
+
void this._pushHeartbeat(resolved);
|
|
2332
3673
|
this._pushTimer = setInterval(() => {
|
|
2333
|
-
void this._pushHeartbeat(
|
|
3674
|
+
void this._pushHeartbeat(resolved);
|
|
2334
3675
|
}, intervalMs);
|
|
2335
3676
|
if (this._pushTimer.unref) {
|
|
2336
3677
|
this._pushTimer.unref();
|
|
2337
3678
|
}
|
|
2338
3679
|
}
|
|
2339
3680
|
async _pushHeartbeat(opts) {
|
|
3681
|
+
if (this.heartbeatInFlight) return;
|
|
3682
|
+
this.heartbeatInFlight = true;
|
|
2340
3683
|
try {
|
|
2341
3684
|
const health = await this.getReport();
|
|
2342
|
-
const gap = runAudit(this.manifest);
|
|
2343
3685
|
const usage = this.usage.snapshot(false);
|
|
2344
|
-
const
|
|
2345
|
-
|
|
2346
|
-
if (
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
}
|
|
3686
|
+
const aggregates = this.evidence.snapshot(Math.ceil((opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * RECORD_TTL_INTERVALS), false);
|
|
3687
|
+
const { body, bytes } = buildCappedBody(health, aggregates, usage);
|
|
3688
|
+
if (bytes > CONTRACT.heartbeatMaxBytes) {
|
|
3689
|
+
opts.onError?.(
|
|
3690
|
+
new Error(`Heartbeat not sent: ${bytes} bytes is over the ${CONTRACT.heartbeatMaxBytes}-byte cap with every check trimmed`)
|
|
3691
|
+
);
|
|
3692
|
+
return;
|
|
2352
3693
|
}
|
|
2353
3694
|
let res;
|
|
3695
|
+
const abort = new AbortController();
|
|
3696
|
+
const abortTimer = setTimeout(
|
|
3697
|
+
() => abort.abort(new Error(`heartbeat POST timed out after ${HEARTBEAT_POST_TIMEOUT_MS}ms`)),
|
|
3698
|
+
HEARTBEAT_POST_TIMEOUT_MS
|
|
3699
|
+
);
|
|
3700
|
+
abortTimer.unref?.();
|
|
2354
3701
|
try {
|
|
2355
|
-
res = await fetch(`${opts.controlPlaneUrl}
|
|
3702
|
+
res = await fetch(`${opts.controlPlaneUrl}${CONTRACT.controlPlanePaths.heartbeat}`, {
|
|
2356
3703
|
method: "POST",
|
|
2357
3704
|
headers: {
|
|
2358
3705
|
"Authorization": `Bearer ${opts.apiKey}`,
|
|
2359
3706
|
"Content-Type": "application/json"
|
|
2360
3707
|
},
|
|
2361
|
-
body
|
|
3708
|
+
body,
|
|
3709
|
+
signal: abort.signal
|
|
2362
3710
|
});
|
|
2363
3711
|
} catch (err) {
|
|
2364
3712
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2365
3713
|
opts.onError?.(new Error(sanitized));
|
|
2366
3714
|
return;
|
|
3715
|
+
} finally {
|
|
3716
|
+
clearTimeout(abortTimer);
|
|
2367
3717
|
}
|
|
2368
3718
|
if (res.ok) {
|
|
2369
|
-
this.usage.
|
|
3719
|
+
this.usage.subtract(usage);
|
|
3720
|
+
this.evidence.subtract(aggregates);
|
|
2370
3721
|
} else {
|
|
2371
3722
|
opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
|
|
2372
3723
|
}
|
|
2373
3724
|
} catch (err) {
|
|
2374
3725
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2375
3726
|
opts.onError?.(new Error(sanitized));
|
|
3727
|
+
} finally {
|
|
3728
|
+
this.heartbeatInFlight = false;
|
|
2376
3729
|
}
|
|
2377
3730
|
}
|
|
2378
3731
|
/** Stop sending heartbeats to the control plane. */
|
|
@@ -2386,38 +3739,26 @@ var AgentSpecReporter = class {
|
|
|
2386
3739
|
isPushModeActive() {
|
|
2387
3740
|
return this._pushTimer !== null;
|
|
2388
3741
|
}
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
this.
|
|
3742
|
+
/** Runs one refresh; concurrent callers share the in-flight promise so they all see a populated cache. */
|
|
3743
|
+
refresh() {
|
|
3744
|
+
if (this.inflight) return this.inflight;
|
|
3745
|
+
if (this.stopped) return Promise.resolve();
|
|
3746
|
+
this.inflight = this.runRefresh().finally(() => {
|
|
3747
|
+
this.inflight = null;
|
|
3748
|
+
});
|
|
3749
|
+
return this.inflight;
|
|
3750
|
+
}
|
|
3751
|
+
async runRefresh() {
|
|
2392
3752
|
try {
|
|
2393
3753
|
const report = await runHealthCheck(this.manifest, {
|
|
2394
3754
|
checkModel: true,
|
|
2395
3755
|
checkMcp: true,
|
|
2396
3756
|
checkMemory: true,
|
|
2397
|
-
checkServices: true
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
const registered = this.registeredTools.has(tool.name);
|
|
2401
|
-
return {
|
|
2402
|
-
id: `tool:${tool.name}`,
|
|
2403
|
-
category: "tool",
|
|
2404
|
-
status: registered ? "pass" : "fail",
|
|
2405
|
-
severity: "info",
|
|
2406
|
-
...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
|
|
2407
|
-
};
|
|
3757
|
+
checkServices: true,
|
|
3758
|
+
baseDir: this.opts.baseDir,
|
|
3759
|
+
rawManifest: this.opts.rawManifest
|
|
2408
3760
|
});
|
|
2409
|
-
|
|
2410
|
-
const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
|
|
2411
|
-
const enrichedReport = {
|
|
2412
|
-
...report,
|
|
2413
|
-
checks: [...report.checks, ...toolChecks],
|
|
2414
|
-
summary: {
|
|
2415
|
-
...report.summary,
|
|
2416
|
-
passed: report.summary.passed + toolPassed,
|
|
2417
|
-
failed: report.summary.failed + toolFailed
|
|
2418
|
-
}
|
|
2419
|
-
};
|
|
2420
|
-
this.cached = { report: enrichedReport, ts: Date.now() };
|
|
3761
|
+
this.cached = { report, ts: Date.now() };
|
|
2421
3762
|
} catch {
|
|
2422
3763
|
if (!this.cached) {
|
|
2423
3764
|
this.cached = {
|
|
@@ -2441,8 +3782,6 @@ var AgentSpecReporter = class {
|
|
|
2441
3782
|
} else {
|
|
2442
3783
|
this.cached = { ...this.cached, ts: Date.now() };
|
|
2443
3784
|
}
|
|
2444
|
-
} finally {
|
|
2445
|
-
this.refreshing = false;
|
|
2446
3785
|
}
|
|
2447
3786
|
}
|
|
2448
3787
|
};
|
|
@@ -2450,7 +3789,7 @@ var AgentSpecReporter = class {
|
|
|
2450
3789
|
// src/agent/adapters/fastify.ts
|
|
2451
3790
|
function agentSpecFastifyPlugin(reporter) {
|
|
2452
3791
|
return async (app) => {
|
|
2453
|
-
app.get(
|
|
3792
|
+
app.get(CONTRACT.agentPaths.health, async (_req, reply) => {
|
|
2454
3793
|
try {
|
|
2455
3794
|
const report = await reporter.getReport();
|
|
2456
3795
|
reply.status(200).send(report);
|
|
@@ -2513,6 +3852,6 @@ function agentSpecExpressRouter(reporter) {
|
|
|
2513
3852
|
return handler;
|
|
2514
3853
|
}
|
|
2515
3854
|
|
|
2516
|
-
export { AUDIT_RULE_IDS, AgentSpecReporter, LATEST_API_VERSION, ManifestSchema, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, collectEnvRefs, collectFileRefs, detectRefType, detectVersion, exportJsonSchema, generateAdapter, getAdapter,
|
|
3855
|
+
export { AUDIT_RULE_IDS, AgentSpecReporter, COMPLIANCE_PACKS, CONTRACT, ContractSchema, EVIDENCE_SHAPES, EvalResult, EvidenceOrigin, EvidenceReason, EvidenceRecord, EvidenceSource, EvidenceStatus, EvidenceWindow, Heartbeat, LATEST_API_VERSION, MAX_PRODUCTION_TEMPERATURE, MAX_TRACE_IDS, ManifestSchema, PodspecFinding, RULE_EVIDENCE, ReadinessFinding, RuleEvidenceEntry, SeveritySchema, SidecarPush, TrustPolicy, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, canonicalManifestSha256, collectEnvRefs, collectFileRefs, computeHealthStatus, declaredManifestSha256, describeManifestError, detectRefType, detectVersion, exportJsonSchema, formatVerified, gatewayEnvNames, generateAdapter, getAdapter, grade, isLatestModelId, isLatestVersion, listAdapters, loadManifest, migrateManifest, registerAdapter, renderGatewayConfig, resolveRef, resolveRefs, ruleClasses, runAudit, runHealthCheck, runServiceChecks, scoreToGrade, secretEnvName, slugify, toolCheckId, tryLoadManifest };
|
|
2517
3856
|
//# sourceMappingURL=index.js.map
|
|
2518
3857
|
//# sourceMappingURL=index.js.map
|