@agentspec/sdk 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/evidence-points-C6j4PSsQ.d.cts +1139 -0
- package/dist/evidence-points-C6j4PSsQ.d.ts +1139 -0
- package/dist/index.cjs +1733 -346
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1026 -2549
- package/dist/index.d.ts +1026 -2549
- package/dist/index.js +1684 -345
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +930 -0
- package/dist/runtime.cjs.map +1 -0
- package/dist/runtime.d.cts +63 -0
- package/dist/runtime.d.ts +63 -0
- package/dist/runtime.js +921 -0
- package/dist/runtime.js.map +1 -0
- package/package.json +15 -8
- package/schemas/evidence/v1/contract.schema.json +2078 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.n-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.pass-rate-above-one.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.bad-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.extra-field.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.fail-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.missing-ttl.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.not-verifiable-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-epoch.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-out-of-range.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-space-separated.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.sample-size-boolean.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.ttl-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-source.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-record.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.point-records-over-limit.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.with-gap.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/ReadinessFinding.bad-severity.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/SidecarPush.missing-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/TrustPolicy.lenient.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.live.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.null-dataset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.fail.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.null-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass-null-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.timezone-offset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.unknown-reason-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.with-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-null.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.evidence-points.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.full.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk-no-records.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.open-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.with-subject.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/SidecarPush.declared.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.standard.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.strict.json +1 -0
- package/schemas/grader/v1/fixtures/accepts-fractional-seconds.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-leap-day-in-year-below-100.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-more-than-6-fractional-digits.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-numeric-offset.json +62 -0
- package/schemas/grader/v1/fixtures/all-records-must-pass.json +58 -0
- package/schemas/grader/v1/fixtures/below-threshold-not-graded.json +61 -0
- package/schemas/grader/v1/fixtures/derived-rule-fails-when-a-parent-fails.json +67 -0
- package/schemas/grader/v1/fixtures/derived-rule-follows-memory-write-parent.json +43 -0
- package/schemas/grader/v1/fixtures/derived-rule-passes-with-its-parents.json +64 -0
- package/schemas/grader/v1/fixtures/derived-rule-unverified-when-a-parent-has-no-evidence.json +51 -0
- package/schemas/grader/v1/fixtures/expired-record-is-stale.json +66 -0
- package/schemas/grader/v1/fixtures/fail-beats-not-verifiable.json +62 -0
- package/schemas/grader/v1/fixtures/grade-band-b.json +62 -0
- package/schemas/grader/v1/fixtures/independent-from-listed-collector-is-admitted.json +39 -0
- package/schemas/grader/v1/fixtures/independent-from-unlisted-collector-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/low-severity-coverage-only.json +90 -0
- package/schemas/grader/v1/fixtures/malformed-point-is-ignored.json +67 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-standard.json +196 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-strict.json +196 -0
- package/schemas/grader/v1/fixtures/not-admitted-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/not-verifiable-ignored-when-one-passes.json +58 -0
- package/schemas/grader/v1/fixtures/not-verifiable-only-keeps-reason.json +71 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-fail-when-no-point-of-the-rule-is-wired.json +59 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-wired-point-pass.json +87 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-its-own-earlier-fail-when-another-point-is-wired.json +70 -0
- package/schemas/grader/v1/fixtures/not-wired-point-ignores-a-wired-point-fail-without-the-rule-id.json +69 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-a-wired-point-fail.json +79 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-independent-evidence.json +64 -0
- package/schemas/grader/v1/fixtures/not-wired-point-never-passes.json +107 -0
- package/schemas/grader/v1/fixtures/offline-record-is-not-admitted-yet.json +47 -0
- package/schemas/grader/v1/fixtures/record-without-collector-is-admitted-until-producers-set-it.json +38 -0
- package/schemas/grader/v1/fixtures/records-without-rule-id-never-graded.json +62 -0
- package/schemas/grader/v1/fixtures/rejects-compact-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-lowercase-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-impossible-date-feb-29.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-t.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-z.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-hours-above-23.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-minutes-above-59.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offsetless-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-year-zero.json +56 -0
- package/schemas/grader/v1/fixtures/rounds-coverage-before-the-threshold.json +850 -0
- package/schemas/grader/v1/fixtures/rounds-verified-before-the-band.json +674 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-admitted-standard.json +39 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-not-admitted-strict.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-observed-standard.json +58 -0
- package/schemas/grader/v1/fixtures/sdk-observed-strict.json +66 -0
- package/schemas/grader/v1/fixtures/sdk-observed-without-evidence-point-source-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-only-agent-all-points-fired.json +116 -0
- package/schemas/grader/v1/fixtures/stale-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/strict-admits-independent.json +39 -0
- package/schemas/grader/v1/fixtures/sub-millisecond-expiry-boundary.json +38 -0
- package/schemas/v1/agent.schema.json +372 -198
package/dist/index.cjs
CHANGED
|
@@ -1,15 +1,32 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var zod = require('zod');
|
|
4
|
-
var zodToJsonSchema = require('zod-to-json-schema');
|
|
5
4
|
var fs = require('fs');
|
|
6
5
|
var path = require('path');
|
|
7
6
|
var yaml = require('js-yaml');
|
|
8
7
|
var net = require('net');
|
|
8
|
+
var crypto = require('crypto');
|
|
9
|
+
var async_hooks = require('async_hooks');
|
|
9
10
|
|
|
10
|
-
function
|
|
11
|
+
function _interopNamespace(e) {
|
|
12
|
+
if (e && e.__esModule) return e;
|
|
13
|
+
var n = Object.create(null);
|
|
14
|
+
if (e) {
|
|
15
|
+
Object.keys(e).forEach(function (k) {
|
|
16
|
+
if (k !== 'default') {
|
|
17
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
18
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
get: function () { return e[k]; }
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
n.default = e;
|
|
26
|
+
return Object.freeze(n);
|
|
27
|
+
}
|
|
11
28
|
|
|
12
|
-
var
|
|
29
|
+
var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml);
|
|
13
30
|
|
|
14
31
|
// src/schema/manifest.schema.ts
|
|
15
32
|
var refOrLiteral = zod.z.string();
|
|
@@ -22,17 +39,37 @@ var MetadataSchema = zod.z.object({
|
|
|
22
39
|
author: zod.z.string().optional(),
|
|
23
40
|
license: zod.z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
|
|
24
41
|
});
|
|
42
|
+
function isHttpUrl(value) {
|
|
43
|
+
if (value.startsWith("$env:")) return true;
|
|
44
|
+
try {
|
|
45
|
+
const url = new URL(value);
|
|
46
|
+
return (url.protocol === "http:" || url.protocol === "https:") && url.hostname !== "";
|
|
47
|
+
} catch {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
var baseUrlSchema = zod.z.string().regex(/^(\$env:[A-Za-z_][A-Za-z0-9_]*|https?:\/\/.+)$/, 'must be an http(s) URL or a "$env:VAR" reference').refine(isHttpUrl, 'must be an http(s) URL with a host, or a "$env:VAR" reference').optional().describe('Override the provider endpoint with an OpenAI-compatible base URL (vLLM, Ollama, LiteLLM, an internal proxy, a mock LLM). A literal http(s) URL with a host (checked by the loader, not expressible in JSON Schema) or "$env:VAR". When set, the gateway treats the model as a custom OpenAI-compatible provider for any "provider" value except "anthropic", whose baseUrl is an Anthropic Messages endpoint without "/v1" (the gateway adds it). Omit to use the provider default.');
|
|
52
|
+
var ModelPricingSchema = zod.z.object({
|
|
53
|
+
input: zod.z.number().nonnegative().describe("USD per 1M input (prompt) tokens."),
|
|
54
|
+
output: zod.z.number().nonnegative().describe("USD per 1M output (completion) tokens."),
|
|
55
|
+
cacheRead: zod.z.number().nonnegative().optional().describe("USD per 1M tokens read from the prompt cache."),
|
|
56
|
+
cacheWrite: zod.z.number().nonnegative().optional().describe("USD per 1M tokens written to the prompt cache."),
|
|
57
|
+
reasoning: zod.z.number().nonnegative().optional().describe("USD per 1M reasoning tokens. The gateway uses the output price when omitted.")
|
|
58
|
+
}).describe("Price of this model in USD per 1M tokens. The gateway uses it instead of its built-in price, so a maxMonthlyUSD budget is spent on models it does not price (e.g. o4-mini). Omit to use the gateway's built-in price.");
|
|
25
59
|
var ModelFallbackSchema = zod.z.object({
|
|
26
60
|
provider: zod.z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
|
|
27
61
|
id: zod.z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
|
|
28
62
|
apiKey: refOrLiteral.describe(refDesc),
|
|
63
|
+
baseUrl: baseUrlSchema,
|
|
29
64
|
triggerOn: zod.z.array(zod.z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
|
|
30
|
-
maxRetries: zod.z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
|
|
65
|
+
maxRetries: zod.z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover)."),
|
|
66
|
+
pricing: ModelPricingSchema.optional()
|
|
31
67
|
});
|
|
32
68
|
var ModelSchema = zod.z.object({
|
|
33
69
|
provider: zod.z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
|
|
34
70
|
id: zod.z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
|
|
35
71
|
apiKey: refOrLiteral.describe(refDesc),
|
|
72
|
+
baseUrl: baseUrlSchema,
|
|
36
73
|
parameters: zod.z.object({
|
|
37
74
|
temperature: zod.z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
|
|
38
75
|
maxTokens: zod.z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
|
|
@@ -41,6 +78,7 @@ var ModelSchema = zod.z.object({
|
|
|
41
78
|
presencePenalty: zod.z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
|
|
42
79
|
}).optional(),
|
|
43
80
|
fallback: ModelFallbackSchema.optional(),
|
|
81
|
+
pricing: ModelPricingSchema.optional(),
|
|
44
82
|
costControls: zod.z.object({
|
|
45
83
|
maxMonthlyUSD: zod.z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
|
|
46
84
|
alertAtUSD: zod.z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
|
|
@@ -75,7 +113,12 @@ var McpServerSchema = zod.z.object({
|
|
|
75
113
|
transport: zod.z.enum(["stdio", "sse", "http"]).describe('"stdio" = spawn a local process (requires "command"), "sse" = Server-Sent Events over HTTP, "http" = streamable HTTP (requires "url").'),
|
|
76
114
|
command: zod.z.string().optional().describe('Shell command to spawn the MCP server process. Required for transport "stdio" (e.g. "npx", "python").'),
|
|
77
115
|
args: zod.z.array(zod.z.string()).optional().describe('Arguments passed to the command (e.g. ["-m", "my_mcp_server"]). Only used for transport "stdio".'),
|
|
78
|
-
|
|
116
|
+
// refOrLiteral, not z.string().url(). This field's own description promises reference syntax,
|
|
117
|
+
// and `.url()` rejected every form of it — so an MCP server whose address comes from the
|
|
118
|
+
// environment (`$env:GITHUB_MCP_URL`, which is the normal case in a cluster, where the URL is
|
|
119
|
+
// a Service name that differs per namespace) could not be declared at all. Every sibling
|
|
120
|
+
// address field in this schema already uses refOrLiteral; this one was the outlier.
|
|
121
|
+
url: refOrLiteral.optional().describe('Base URL of the remote MCP server. Required for transport "sse" or "http". ' + refDesc),
|
|
79
122
|
env: zod.z.record(zod.z.string(), refOrLiteral).optional().describe('Environment variables injected into the MCP server process. Values support reference syntax. Use "$env:API_KEY" to forward secrets without hardcoding.'),
|
|
80
123
|
healthCheck: zod.z.object({
|
|
81
124
|
timeoutSeconds: zod.z.number().int().min(1).max(60).default(5).describe("Seconds to wait for the MCP server to respond during health checks. Increase for slow-starting processes.")
|
|
@@ -151,6 +194,7 @@ var ChatEndpointSchema = zod.z.object({
|
|
|
151
194
|
var ApiSchema = zod.z.object({
|
|
152
195
|
type: zod.z.enum(["rest", "mcp", "grpc", "websocket"]),
|
|
153
196
|
port: zod.z.number().int().min(1).max(65535).optional(),
|
|
197
|
+
allowedPorts: zod.z.array(zod.z.number().int().min(1).max(65535)).optional().describe("Extra ports the agent may listen on besides the sidecar-routed API port (e.g. a platform metrics port). The sidecar reports not-ready while the agent listens on any other non-loopback port."),
|
|
154
198
|
pathPrefix: zod.z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
|
|
155
199
|
auth: zod.z.object({
|
|
156
200
|
type: zod.z.enum(["jwt", "apikey", "oauth2", "none"]),
|
|
@@ -159,7 +203,8 @@ var ApiSchema = zod.z.object({
|
|
|
159
203
|
}).optional(),
|
|
160
204
|
rateLimit: zod.z.object({
|
|
161
205
|
requestsPerMinute: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
|
|
162
|
-
requestsPerHour: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
|
|
206
|
+
requestsPerHour: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429."),
|
|
207
|
+
tokensPerMinute: zod.z.number().int().min(1).optional().describe("Maximum LLM tokens the gateway allows per minute. Once the bucket is empty, requests receive HTTP 429.")
|
|
163
208
|
}).optional(),
|
|
164
209
|
streaming: zod.z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
|
|
165
210
|
healthEndpoint: zod.z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
|
|
@@ -241,25 +286,42 @@ var EvalMetricEnum = zod.z.enum([
|
|
|
241
286
|
// RAG: do the retrieved chunks cover the ground-truth answer?
|
|
242
287
|
"bias",
|
|
243
288
|
// Does the answer exhibit demographic or political bias?
|
|
289
|
+
"grounded",
|
|
290
|
+
// Are the answer's claims supported by the retrieved context? Replaces faithfulness and hallucination.
|
|
291
|
+
"pii_leakage",
|
|
292
|
+
// Does the answer leak personal data?
|
|
293
|
+
"prompt_injection",
|
|
294
|
+
// Did the answer follow an injected instruction?
|
|
244
295
|
"custom"
|
|
245
|
-
//
|
|
296
|
+
// Rubric-graded scorer, declared in the sibling `custom: { name, rubric }`.
|
|
246
297
|
]);
|
|
247
|
-
var
|
|
298
|
+
var LiveMetricEnum = EvalMetricEnum.exclude(["context_recall", "answer_similarity"]);
|
|
299
|
+
var CustomMetricSchema = zod.z.object({
|
|
300
|
+
name: zod.z.string().min(1).describe("Name the custom metric is reported under."),
|
|
301
|
+
rubric: refOrLiteral.describe('Grading rubric. Use "$file:rubrics/<name>.md"; the loader resolves it to the file contents.')
|
|
302
|
+
});
|
|
303
|
+
var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for grounded and context_precision metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
|
|
248
304
|
var EvaluationSchema = zod.z.object({
|
|
249
|
-
framework: zod.z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
|
|
305
|
+
framework: zod.z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]).optional().describe("Deprecated: nothing reads it. validate warns when set; rejected in 0.6.0."),
|
|
306
|
+
live: zod.z.object({
|
|
307
|
+
metrics: zod.z.array(LiveMetricEnum).optional().describe("Metrics scored on live traffic."),
|
|
308
|
+
thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe("Minimum pass rate per live metric (0.0\u20131.0)."),
|
|
309
|
+
custom: CustomMetricSchema.optional()
|
|
310
|
+
}).optional().describe("Live evaluation, separate from dataset evaluation. Only live metrics are allowed."),
|
|
311
|
+
custom: CustomMetricSchema.optional().describe('Rubric for the "custom" metric on datasets.'),
|
|
250
312
|
datasets: zod.z.array(
|
|
251
313
|
zod.z.object({
|
|
252
314
|
name: zod.z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
|
|
253
315
|
path: refOrLiteral.describe(datasetPathDesc),
|
|
254
316
|
metrics: zod.z.array(EvalMetricEnum).optional().describe(
|
|
255
|
-
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["
|
|
317
|
+
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["grounded","context_recall"] while a safety dataset sets ["toxicity","bias"].'
|
|
256
318
|
)
|
|
257
319
|
})
|
|
258
320
|
).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
|
|
259
321
|
metrics: zod.z.array(EvalMetricEnum).optional().describe(
|
|
260
|
-
'Default metrics to compute across all datasets that do not declare their own metrics. "
|
|
322
|
+
'Default metrics to compute across all datasets that do not declare their own metrics. "grounded", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires spec.evaluation.custom: { name, rubric }. "faithfulness" and "hallucination" are deprecated aliases of "grounded".'
|
|
261
323
|
),
|
|
262
|
-
thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"
|
|
324
|
+
thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"grounded": 0.95, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
|
|
263
325
|
ciGate: zod.z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
|
|
264
326
|
});
|
|
265
327
|
var ObservabilitySchema = zod.z.object({
|
|
@@ -358,12 +420,608 @@ var ManifestSchema = zod.z.object({
|
|
|
358
420
|
metadata: MetadataSchema,
|
|
359
421
|
spec: SpecSchema
|
|
360
422
|
});
|
|
423
|
+
var DRAFT_07 = "http://json-schema.org/draft-07/schema#";
|
|
424
|
+
var options = {
|
|
425
|
+
target: "draft-7",
|
|
426
|
+
io: "input",
|
|
427
|
+
reused: "inline",
|
|
428
|
+
override: ({ zodSchema, jsonSchema }) => {
|
|
429
|
+
const def = zodSchema._zod.def;
|
|
430
|
+
if (def.type === "object" && jsonSchema.additionalProperties === void 0) jsonSchema.additionalProperties = false;
|
|
431
|
+
if (def.type === "record") delete jsonSchema.propertyNames;
|
|
432
|
+
if (def.type === "union" && def.discriminator && jsonSchema.oneOf) {
|
|
433
|
+
jsonSchema.anyOf = jsonSchema.oneOf;
|
|
434
|
+
delete jsonSchema.oneOf;
|
|
435
|
+
}
|
|
436
|
+
if (jsonSchema.type === "integer") {
|
|
437
|
+
if (jsonSchema.maximum === Number.MAX_SAFE_INTEGER) delete jsonSchema.maximum;
|
|
438
|
+
if (jsonSchema.minimum === Number.MIN_SAFE_INTEGER) delete jsonSchema.minimum;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
};
|
|
442
|
+
function toJsonSchema(schema) {
|
|
443
|
+
const { $schema: _, ...body } = zod.z.toJSONSchema(schema, options);
|
|
444
|
+
return { ...body, $schema: DRAFT_07 };
|
|
445
|
+
}
|
|
446
|
+
function toNamedJsonSchema(name, schema) {
|
|
447
|
+
const { $schema, ...body } = toJsonSchema(schema);
|
|
448
|
+
return { $ref: `#/definitions/${name}`, definitions: { [name]: body }, $schema };
|
|
449
|
+
}
|
|
361
450
|
function exportJsonSchema() {
|
|
362
|
-
return
|
|
363
|
-
|
|
364
|
-
|
|
451
|
+
return toNamedJsonSchema("AgentSpecManifest", ManifestSchema);
|
|
452
|
+
}
|
|
453
|
+
var EvidenceClass = zod.z.enum(["independent", "evidence-point", "offline", "declared"]);
|
|
454
|
+
var Collector = zod.z.enum([
|
|
455
|
+
"gateway-config",
|
|
456
|
+
"gateway-traffic",
|
|
457
|
+
"sidecar-probe",
|
|
458
|
+
"sidecar-forwarder",
|
|
459
|
+
"sidecar-eval",
|
|
460
|
+
"operator-webhook",
|
|
461
|
+
"operator-logs",
|
|
462
|
+
"sdk",
|
|
463
|
+
"lock",
|
|
464
|
+
"manifest"
|
|
465
|
+
]);
|
|
466
|
+
var EvidencePoint = zod.z.enum(["guardrail", "tool", "mcpClient", "memoryWrite", "vectorStore", "logging"]);
|
|
467
|
+
var Scope = zod.z.enum(["declared", "configured", "enforced", "observed", "ran-where-wired", "every-path", "effective"]);
|
|
468
|
+
var WiredBy = zod.z.enum(["none", "operator-render", "agent-code", "user-schedule", "opt-in"]);
|
|
469
|
+
var Mode = zod.z.enum(["standalone", "operator"]);
|
|
470
|
+
var BuildStatus = zod.z.enum(["built", "partial", "planned", "open"]);
|
|
471
|
+
var RuleEvidenceSource = zod.z.object({
|
|
472
|
+
class: EvidenceClass,
|
|
473
|
+
collector: Collector,
|
|
474
|
+
/** Evidence-point sources only. */
|
|
475
|
+
point: EvidencePoint.nullish(),
|
|
476
|
+
proves: Scope,
|
|
477
|
+
wiredBy: WiredBy,
|
|
478
|
+
modes: zod.z.array(Mode).min(1),
|
|
479
|
+
status: BuildStatus,
|
|
480
|
+
/** A variant such as HTTP MCP, stdio MCP, local tools or opt-in. */
|
|
481
|
+
when: zod.z.string().min(1).nullish(),
|
|
482
|
+
/** A collector that checks the counts, such as gateway calls against guardrail records. */
|
|
483
|
+
crossCheck: Collector.nullish(),
|
|
484
|
+
/** The `agentspec lock` prompt question. */
|
|
485
|
+
question: zod.z.string().min(1).nullish()
|
|
486
|
+
}).strict().refine((s) => s.class === "evidence-point" === (s.point != null), {
|
|
487
|
+
message: "an evidence-point source names its `point`, and no other class does"
|
|
488
|
+
});
|
|
489
|
+
var RuleEvidenceEntry = zod.z.object({
|
|
490
|
+
sources: zod.z.array(RuleEvidenceSource).default([]),
|
|
491
|
+
/** Rules whose verdicts decide this one (all must pass); this rule inherits their sources. */
|
|
492
|
+
derivedFrom: zod.z.array(zod.z.string().min(1)).nullish(),
|
|
493
|
+
/** What a pass does not prove, shown with the result. Free text. */
|
|
494
|
+
notProven: zod.z.string().nullish()
|
|
495
|
+
}).strict().refine((e) => e.sources.length > 0 || (e.derivedFrom?.length ?? 0) > 0, {
|
|
496
|
+
message: "an entry needs at least one source or a `derivedFrom`"
|
|
497
|
+
});
|
|
498
|
+
var BOTH = ["standalone", "operator"];
|
|
499
|
+
var CLASS_DEFAULTS = {
|
|
500
|
+
independent: { wiredBy: "none", status: "built" },
|
|
501
|
+
"evidence-point": { wiredBy: "agent-code", status: "planned" },
|
|
502
|
+
offline: { wiredBy: "user-schedule", status: "planned" },
|
|
503
|
+
declared: { wiredBy: "none", status: "built" }
|
|
504
|
+
};
|
|
505
|
+
var source = (cls, collector, proves, extra = {}) => ({ class: cls, collector, proves, modes: BOTH, ...CLASS_DEFAULTS[cls], ...extra });
|
|
506
|
+
var independent = (collector, proves, extra) => source("independent", collector, proves, extra);
|
|
507
|
+
var point = (p, proves, extra) => source("evidence-point", "sdk", proves, { point: p, ...extra });
|
|
508
|
+
var lock = (question, extra) => source("offline", "lock", "every-path", { question, ...extra });
|
|
509
|
+
var declared = (extra) => source("declared", "manifest", "declared", extra);
|
|
510
|
+
var OPERATOR_ONLY = { modes: ["operator"] };
|
|
511
|
+
var PLANNED = { status: "planned" };
|
|
512
|
+
var WRITES_BYPASS = "Writes that bypass the evidence point at runtime.";
|
|
513
|
+
var entries = {
|
|
514
|
+
"MODEL-01": { sources: [independent("gateway-config", "configured"), independent("gateway-traffic", "observed")] },
|
|
515
|
+
"MODEL-02": {
|
|
516
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
517
|
+
},
|
|
518
|
+
"MODEL-03": { sources: [independent("gateway-config", "enforced")] },
|
|
519
|
+
"MODEL-04": {
|
|
520
|
+
sources: [
|
|
521
|
+
independent("gateway-config", "enforced"),
|
|
522
|
+
lock("Do the agent's own provider clients retry more than spec.model.fallback.maxRetries, or stack retries on the gateway's?", { when: "client retries" })
|
|
523
|
+
],
|
|
524
|
+
notProven: "That the gateway's attempts match `spec.model.fallback.maxRetries` at runtime beyond the rendered config."
|
|
525
|
+
},
|
|
526
|
+
"MODEL-05": {
|
|
527
|
+
sources: [independent("gateway-traffic", "observed", { when: "agw.ai.usage.cost.total" })],
|
|
528
|
+
notProven: "Requests with no token usage: not counted."
|
|
529
|
+
},
|
|
530
|
+
"SEC-LLM-01": {
|
|
531
|
+
sources: [
|
|
532
|
+
independent("sidecar-probe", "effective", { when: "opt-in", wiredBy: "opt-in", status: "planned" }),
|
|
533
|
+
point("guardrail", "ran-where-wired"),
|
|
534
|
+
lock("Does every path that sends user input to a model run the declared input guardrails first?")
|
|
535
|
+
]
|
|
536
|
+
},
|
|
537
|
+
"SEC-LLM-02": {
|
|
538
|
+
sources: [
|
|
539
|
+
point("guardrail", "ran-where-wired"),
|
|
540
|
+
lock("Does every path that returns model output run the declared output guardrails first?")
|
|
541
|
+
],
|
|
542
|
+
notProven: "Effective: a no-op output guardrail still fires."
|
|
543
|
+
},
|
|
544
|
+
"SEC-LLM-03": { sources: [declared()] },
|
|
545
|
+
"SEC-LLM-04": {
|
|
546
|
+
sources: [independent("gateway-config", "enforced"), independent("gateway-traffic", "observed", PLANNED)],
|
|
547
|
+
notProven: "Cost: MODEL-03 covers it."
|
|
548
|
+
},
|
|
549
|
+
"SEC-LLM-05": {
|
|
550
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
551
|
+
},
|
|
552
|
+
"SEC-LLM-06": { derivedFrom: ["MEM-01"], notProven: "Same as MEM-01." },
|
|
553
|
+
"SEC-LLM-07": {
|
|
554
|
+
sources: [
|
|
555
|
+
independent("sidecar-probe", "configured", { when: "HTTP MCP" }),
|
|
556
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
557
|
+
lock("Does the agent call any MCP tool the manifest does not declare, or one that serves no annotations?"),
|
|
558
|
+
declared({ when: "local tools" })
|
|
559
|
+
]
|
|
560
|
+
},
|
|
561
|
+
"SEC-LLM-08": {
|
|
562
|
+
sources: [
|
|
563
|
+
independent("sidecar-probe", "configured", PLANNED),
|
|
564
|
+
independent("gateway-traffic", "enforced", PLANNED),
|
|
565
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" })
|
|
566
|
+
],
|
|
567
|
+
notProven: "Which tools are destructive: the server declares it; nothing checks the claim."
|
|
568
|
+
},
|
|
569
|
+
"SEC-LLM-09": { derivedFrom: ["EVAL-01", "EVAL-02"] },
|
|
570
|
+
"SEC-LLM-10": {
|
|
571
|
+
sources: [
|
|
572
|
+
independent("operator-webhook", "configured", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
573
|
+
declared({ modes: ["standalone"] })
|
|
574
|
+
]
|
|
575
|
+
},
|
|
576
|
+
"TOOL-01": {
|
|
577
|
+
sources: [
|
|
578
|
+
independent("gateway-traffic", "observed", { when: "HTTP MCP" }),
|
|
579
|
+
point("tool", "ran-where-wired", { when: "local tools" }),
|
|
580
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
581
|
+
lock("Which tools does the agent code register or call, local, HTTP MCP and stdio MCP?")
|
|
582
|
+
],
|
|
583
|
+
notProven: "Unwrapped local or stdio calls at runtime: only the lock inventory catches them."
|
|
584
|
+
},
|
|
585
|
+
"MEM-01": {
|
|
586
|
+
sources: [
|
|
587
|
+
point("memoryWrite", "enforced"),
|
|
588
|
+
lock("Does every memory write pass through the PII scrubber for the declared fields?")
|
|
589
|
+
],
|
|
590
|
+
notProven: "Full coverage: the scrubber is a regex baseline (Presidio optional in Python). " + WRITES_BYPASS
|
|
591
|
+
},
|
|
592
|
+
"MEM-02": {
|
|
593
|
+
sources: [
|
|
594
|
+
point("memoryWrite", "enforced"),
|
|
595
|
+
lock("Does every memory write apply the declared retention TTL?")
|
|
596
|
+
],
|
|
597
|
+
notProven: WRITES_BYPASS + " Backends that cannot apply a TTL (record `ttl: unsupported`): the evidence point is `not_verifiable` there and only the lock remains."
|
|
598
|
+
},
|
|
599
|
+
"MEM-03": {
|
|
600
|
+
sources: [
|
|
601
|
+
point("memoryWrite", "enforced"),
|
|
602
|
+
lock("Does every memory write honour the declared memory audit setting?")
|
|
603
|
+
],
|
|
604
|
+
notProven: WRITES_BYPASS
|
|
605
|
+
},
|
|
606
|
+
"MEM-04": {
|
|
607
|
+
sources: [
|
|
608
|
+
point("vectorStore", "ran-where-wired"),
|
|
609
|
+
lock("Does every vector store query and write use the declared namespace or tenant isolation?")
|
|
610
|
+
],
|
|
611
|
+
notProven: "Isolation: the right namespace does not prove other tenants cannot read it."
|
|
612
|
+
},
|
|
613
|
+
"MEM-05": {
|
|
614
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.usage.output_tokens" }), declared({ when: "until a call is observed" })],
|
|
615
|
+
notProven: "The tolerance for the system prompt and tool schemas is set in Phase 2, with an ADR-0015 amendment; a call without gen_ai.usage.output_tokens leaves the check not_verifiable."
|
|
616
|
+
},
|
|
617
|
+
"EVAL-01": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
618
|
+
"EVAL-02": {
|
|
619
|
+
sources: [
|
|
620
|
+
independent("sidecar-eval", "observed", PLANNED),
|
|
621
|
+
lock("Does the agent's CI run the declared evaluation gate before deploy?")
|
|
622
|
+
],
|
|
623
|
+
notProven: "That a failed gate actually blocks a deploy: CI is outside the system."
|
|
624
|
+
},
|
|
625
|
+
"EVAL-03": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
626
|
+
"OBS-01": {
|
|
627
|
+
sources: [independent("sidecar-forwarder", "observed"), independent("sidecar-probe", "configured")],
|
|
628
|
+
notProven: "That the agent exports every span: the rule is about the backend, not span coverage. agentops is not a target: it has no OTLP ingest."
|
|
629
|
+
},
|
|
630
|
+
"OBS-02": {
|
|
631
|
+
sources: [
|
|
632
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
633
|
+
point("logging", "ran-where-wired"),
|
|
634
|
+
lock("Does every log statement emit structured JSON?")
|
|
635
|
+
]
|
|
636
|
+
},
|
|
637
|
+
"OBS-03": {
|
|
638
|
+
sources: [
|
|
639
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
640
|
+
lock("Does any log statement write one of the declared sensitive fields?")
|
|
641
|
+
],
|
|
642
|
+
notProven: "Full coverage: the scanner is a regex and pattern baseline, so recall is limited."
|
|
643
|
+
},
|
|
644
|
+
"META-01": { sources: [declared()] },
|
|
645
|
+
"META-02": {
|
|
646
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.request.temperature" })],
|
|
647
|
+
notProven: "Requests that set no temperature: `not_verifiable`, not counted."
|
|
648
|
+
}
|
|
649
|
+
};
|
|
650
|
+
var RULE_EVIDENCE = Object.fromEntries(
|
|
651
|
+
Object.entries(entries).map(([id, entry]) => [id, RuleEvidenceEntry.parse(entry)])
|
|
652
|
+
);
|
|
653
|
+
function ruleSources(ruleId, registry = RULE_EVIDENCE, seen = /* @__PURE__ */ new Set()) {
|
|
654
|
+
if (!Object.hasOwn(registry, ruleId) || seen.has(ruleId)) return [];
|
|
655
|
+
const entry = registry[ruleId];
|
|
656
|
+
const visited = /* @__PURE__ */ new Set([...seen, ruleId]);
|
|
657
|
+
return [...entry.sources, ...(entry.derivedFrom ?? []).flatMap((parent) => ruleSources(parent, registry, visited))];
|
|
658
|
+
}
|
|
659
|
+
function ruleClasses(ruleId, registry = RULE_EVIDENCE) {
|
|
660
|
+
const classes = new Set(ruleSources(ruleId, registry).map((s) => s.class));
|
|
661
|
+
return EvidenceClass.options.filter((c) => classes.has(c));
|
|
662
|
+
}
|
|
663
|
+
var EvidenceStatus = zod.z.enum(["pass", "fail", "not_verifiable"]);
|
|
664
|
+
var EvidenceSource = zod.z.enum(["independent", "sdk-observed", "lower-bound", "self-declared", "declared", "offline"]);
|
|
665
|
+
var EvidenceOrigin = zod.z.enum(["ingress", "gateway", "egress_proxy", "in_process", "probe", "manifest", "judge"]);
|
|
666
|
+
var SeveritySchema = zod.z.enum(["critical", "high", "medium", "low", "info"]);
|
|
667
|
+
var TrustPolicy = zod.z.enum(["standard", "strict"]);
|
|
668
|
+
var EvidenceReason = zod.z.object({ code: zod.z.string().min(1), detail: zod.z.string().nullish() }).strict();
|
|
669
|
+
var TIMESTAMP_PATTERN = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
|
|
670
|
+
var Timestamp = zod.z.string().regex(TIMESTAMP_PATTERN);
|
|
671
|
+
var EvidenceWindow = zod.z.object({ start: Timestamp, end: Timestamp }).strict();
|
|
672
|
+
var Sha256 = zod.z.string().regex(/^[a-f0-9]{64}$/);
|
|
673
|
+
var MAX_TRACE_IDS = 20;
|
|
674
|
+
var recordFields = {
|
|
675
|
+
/** Scoped per subject: `tool:<name>`, `model:<id>`, or a bare check name for agent-wide checks. */
|
|
676
|
+
checkId: zod.z.string().min(1),
|
|
677
|
+
ruleId: zod.z.string().min(1).nullish(),
|
|
678
|
+
source: EvidenceSource,
|
|
679
|
+
origin: EvidenceOrigin,
|
|
680
|
+
/** The registry collector that produced the record. Optional until every producer sets it (#422); checking it against the collectors the registry lists for the rule is the grader's job (#423). */
|
|
681
|
+
collector: Collector.nullish(),
|
|
682
|
+
observedAt: Timestamp,
|
|
683
|
+
window: EvidenceWindow,
|
|
684
|
+
ttlSeconds: zod.z.number().int().nonnegative(),
|
|
685
|
+
sampleSize: zod.z.number().int().nonnegative(),
|
|
686
|
+
traceIds: zod.z.array(zod.z.string().min(1)).max(MAX_TRACE_IDS),
|
|
687
|
+
manifestSha256: Sha256
|
|
688
|
+
};
|
|
689
|
+
var EvidenceRecordPass = zod.z.object({ ...recordFields, status: zod.z.literal("pass"), reason: EvidenceReason.nullish() }).strict();
|
|
690
|
+
var EvidenceRecordNotPass = zod.z.object({
|
|
691
|
+
...recordFields,
|
|
692
|
+
status: zod.z.enum(["fail", "not_verifiable"]),
|
|
693
|
+
reason: EvidenceReason
|
|
694
|
+
}).strict();
|
|
695
|
+
var EvidenceRecord = zod.z.union([EvidenceRecordPass, EvidenceRecordNotPass]);
|
|
696
|
+
var PodspecFinding = zod.z.object({ code: zod.z.string().min(1), subject: zod.z.string().min(1), message: zod.z.string().min(1) }).strict();
|
|
697
|
+
var ReadinessFinding = zod.z.object({
|
|
698
|
+
code: zod.z.string().min(1),
|
|
699
|
+
severity: SeveritySchema,
|
|
700
|
+
message: zod.z.string().min(1),
|
|
701
|
+
subject: zod.z.string().nullish(),
|
|
702
|
+
ruleIds: zod.z.array(zod.z.string().min(1)).nullish()
|
|
703
|
+
}).strict();
|
|
704
|
+
var EvalResult = zod.z.object({
|
|
705
|
+
metric: zod.z.string().min(1),
|
|
706
|
+
n: zod.z.number().int().nonnegative(),
|
|
707
|
+
passRate: zod.z.number().min(0).max(1),
|
|
708
|
+
ciLow: zod.z.number().min(0).max(1),
|
|
709
|
+
ciHigh: zod.z.number().min(0).max(1),
|
|
710
|
+
threshold: zod.z.number().min(0).max(1),
|
|
711
|
+
window: EvidenceWindow,
|
|
712
|
+
judgeModel: zod.z.string().min(1),
|
|
713
|
+
rubricHash: Sha256,
|
|
714
|
+
source: EvidenceSource,
|
|
715
|
+
dataset: zod.z.string().min(1).nullish()
|
|
716
|
+
}).strict();
|
|
717
|
+
var MAX_POINT_RECORDS = 5;
|
|
718
|
+
var EvidencePointState = zod.z.enum(["fired", "wired", "not_wired"]);
|
|
719
|
+
var EvidencePointAggregate = zod.z.object({
|
|
720
|
+
point: EvidencePoint,
|
|
721
|
+
/** What the point is wired on, a key of `CONTRACT.evidencePoints[point]`. */
|
|
722
|
+
target: zod.z.string().min(1),
|
|
723
|
+
state: EvidencePointState,
|
|
724
|
+
counts: zod.z.object({
|
|
725
|
+
pass: zod.z.number().int().nonnegative(),
|
|
726
|
+
fail: zod.z.number().int().nonnegative(),
|
|
727
|
+
not_verifiable: zod.z.number().int().nonnegative()
|
|
728
|
+
}).strict(),
|
|
729
|
+
records: zod.z.array(EvidenceRecord).max(MAX_POINT_RECORDS)
|
|
730
|
+
}).strict();
|
|
731
|
+
var Passthrough = zod.z.record(zod.z.string(), zod.z.unknown());
|
|
732
|
+
var Heartbeat = zod.z.object({
|
|
733
|
+
/** Contract version of the sending SDK (`CONTRACT.version`); absent or null from older SDKs. */
|
|
734
|
+
contract: zod.z.string().regex(/^\d+\.\d+\.\d+$/).nullish(),
|
|
735
|
+
health: Passthrough,
|
|
736
|
+
records: zod.z.array(EvidenceRecord).default([]),
|
|
737
|
+
/** Every evidence point the registry gives for the manifest, wired or not; absent from older SDKs. */
|
|
738
|
+
evidencePoints: zod.z.array(EvidencePointAggregate).default([]),
|
|
739
|
+
usage: Passthrough.nullish(),
|
|
740
|
+
tools: Passthrough.nullish()
|
|
741
|
+
}).strict();
|
|
742
|
+
var SidecarPush = zod.z.object({
|
|
743
|
+
manifestSha256: Sha256,
|
|
744
|
+
declared: Passthrough.nullish(),
|
|
745
|
+
records: zod.z.array(EvidenceRecord).default([])
|
|
746
|
+
}).strict();
|
|
747
|
+
var EVIDENCE_SHAPES = { EvidenceRecord, ReadinessFinding, EvalResult, TrustPolicy, Heartbeat, SidecarPush };
|
|
748
|
+
|
|
749
|
+
// src/schema/contract.schema.ts
|
|
750
|
+
var fixed = (value) => zod.z.literal(value).default(value);
|
|
751
|
+
var REASON_CODES = [
|
|
752
|
+
"stale",
|
|
753
|
+
"source_not_admitted",
|
|
754
|
+
"source_mismatch",
|
|
755
|
+
"no_evidence",
|
|
756
|
+
"not_wired",
|
|
757
|
+
"no_calls_observed",
|
|
758
|
+
"no_failover_observed",
|
|
759
|
+
"no_budget_evidence",
|
|
760
|
+
"no_approval_token_evidence",
|
|
761
|
+
"no_retrieval_spans",
|
|
762
|
+
"unrecognised_conventions",
|
|
763
|
+
"instrumentation_unavailable",
|
|
764
|
+
"declared_env_missing",
|
|
765
|
+
"provider_key_on_agent",
|
|
766
|
+
"response_cap_exceeded",
|
|
767
|
+
"temperature_above_max",
|
|
768
|
+
"undeclared_tool",
|
|
769
|
+
"undeclared_model",
|
|
770
|
+
"unpinned_model",
|
|
771
|
+
"self_declared",
|
|
772
|
+
"unpriced_model",
|
|
773
|
+
"undeclared_namespace"
|
|
774
|
+
];
|
|
775
|
+
var ContractSchema = zod.z.object({
|
|
776
|
+
/** Contract version (major.minor.patch) a heartbeat carries: a receiver rejects another major and accepts a newer minor (ADR-0039). */
|
|
777
|
+
version: fixed("1.0.0"),
|
|
778
|
+
env: zod.z.object({
|
|
779
|
+
manifestPath: fixed("MANIFEST_PATH"),
|
|
780
|
+
upstreamUrl: fixed("UPSTREAM_URL"),
|
|
781
|
+
proxyPort: fixed("PROXY_PORT"),
|
|
782
|
+
controlPlanePort: fixed("CONTROL_PLANE_PORT"),
|
|
783
|
+
/** Control plane URL the operator webhook injects into the sidecar and agent containers; SDK push mode falls back to it. */
|
|
784
|
+
controlPlaneUrl: fixed("AGENTSPEC_CONTROL_PLANE_URL"),
|
|
785
|
+
/** Agent JWT from `POST /api/v1/register`; `agentspec register` sends it to the control plane as a Bearer token. The sidecar's own push needs the `sidecarToken` from the same call (see `agentTokenSecretSuffix`). */
|
|
786
|
+
agentKey: fixed("AGENTSPEC_API_KEY"),
|
|
787
|
+
/** Trust policy (`standard | strict`) the grader applies: set on the control plane and the sidecar, never in `agent.yaml` (ADR-0007). */
|
|
788
|
+
trustPolicy: fixed("AGENTSPEC_TRUST_POLICY"),
|
|
789
|
+
/** The webhook's PodSpec findings (the `agentspec.io/podspec-findings` annotation, JSON), injected into the sidecar by the downward API (ADR-0019). */
|
|
790
|
+
podspecFindings: fixed("AGENTSPEC_PODSPEC_FINDINGS"),
|
|
791
|
+
/** Virtual key the agent sends and the gateway accepts (ADR-0001); provider keys stay in the gateway. */
|
|
792
|
+
gatewayKey: fixed("AGENTSPEC_GATEWAY_KEY"),
|
|
793
|
+
/** Bind address and port the agent must honour so the sidecar can own `spec.api.port` (ADR-0028). */
|
|
794
|
+
agentHost: fixed("HOST"),
|
|
795
|
+
agentPort: fixed("PORT"),
|
|
796
|
+
/** Azure resource name the gateway config references (`$AZURE_OPENAI_RESOURCE_NAME`). */
|
|
797
|
+
azureResourceName: fixed("AZURE_OPENAI_RESOURCE_NAME")
|
|
798
|
+
}).strict().prefault({}),
|
|
799
|
+
/** Provider base-URL env vars, keyed by spec.model.provider. */
|
|
800
|
+
providerBaseUrlEnv: zod.z.object({
|
|
801
|
+
openai: fixed("OPENAI_BASE_URL"),
|
|
802
|
+
anthropic: fixed("ANTHROPIC_BASE_URL")
|
|
803
|
+
}).strict().prefault({}),
|
|
804
|
+
paths: zod.z.object({
|
|
805
|
+
manifestDir: fixed("/manifest"),
|
|
806
|
+
manifestFile: fixed("/manifest/agent.yaml"),
|
|
807
|
+
/** SQLite file the gateway keeps USD budget usage in; must be on a writable volume. */
|
|
808
|
+
gatewayDatabase: fixed("/gateway-data/agentgateway.db"),
|
|
809
|
+
/** Directory the init container renders the gateway config into (ADR-0025). */
|
|
810
|
+
gatewayConfigDir: fixed("/gateway"),
|
|
811
|
+
gatewayConfigFile: fixed("/gateway/config.yaml"),
|
|
812
|
+
/** Sidecar bundle inside its image; `node <entrypoint> gateway-config …` is the init container's command (ADR-0025). */
|
|
813
|
+
sidecarEntrypoint: fixed("dist/index.js"),
|
|
814
|
+
gatewayDataDir: fixed("/gateway-data")
|
|
815
|
+
}).strict().prefault({}),
|
|
816
|
+
images: zod.z.object({
|
|
817
|
+
sidecar: fixed("ghcr.io/agents-oss/agentspec-sidecar:latest"),
|
|
818
|
+
/** agentgateway, pinned (ADR-0025). */
|
|
819
|
+
gateway: fixed("cr.agentgateway.dev/agentgateway:v1.6.0")
|
|
820
|
+
}).strict().prefault({}),
|
|
821
|
+
ports: zod.z.object({
|
|
822
|
+
agentDefault: fixed(8e3),
|
|
823
|
+
/** Loopback port the agent binds once the sidecar takes over its declared port (ADR-0028). */
|
|
824
|
+
agentBind: fixed(18e3),
|
|
825
|
+
proxy: fixed(4e3),
|
|
826
|
+
controlPlane: fixed(4001),
|
|
827
|
+
/** agentgateway LLM listener on the pod loopback (ADR-0025). */
|
|
828
|
+
gateway: fixed(4002),
|
|
829
|
+
/** agentgateway's default stats and readiness listeners: wildcard-bound, not the agent's (ADR-0025). */
|
|
830
|
+
gatewayStats: fixed(15020),
|
|
831
|
+
gatewayReadiness: fixed(15021)
|
|
832
|
+
}).strict().prefault({}),
|
|
833
|
+
/** Bare paths served by the sidecar on the control-plane port (ADR-0023). */
|
|
834
|
+
sidecarPaths: zod.z.object({
|
|
835
|
+
healthReady: fixed("/health/ready"),
|
|
836
|
+
gap: fixed("/gap"),
|
|
837
|
+
explore: fixed("/explore"),
|
|
838
|
+
proof: fixed("/proof"),
|
|
839
|
+
traces: fixed("/v1/traces"),
|
|
840
|
+
/** Read-only GET of the buffered, already redacted spans the gateway copied to the sidecar. */
|
|
841
|
+
spans: fixed("/spans")
|
|
842
|
+
}).strict().prefault({}),
|
|
843
|
+
/** Bare paths an agent serves itself (SDK `/agentspec/health`); the sidecar probes them on the upstream. */
|
|
844
|
+
agentPaths: zod.z.object({
|
|
845
|
+
health: fixed("/agentspec/health")
|
|
846
|
+
}).strict().prefault({}),
|
|
847
|
+
/** Control plane API routes shared by the CLI, the sidecar push and the control plane; `{name}` is the registered agent name. */
|
|
848
|
+
controlPlanePaths: zod.z.object({
|
|
849
|
+
/** POST: registers an agent and returns its per-agent token. */
|
|
850
|
+
register: fixed("/api/v1/register"),
|
|
851
|
+
/** POST: the SDK push-mode heartbeat, authenticated with the agent token from `register`. */
|
|
852
|
+
heartbeat: fixed("/api/v1/heartbeat"),
|
|
853
|
+
declaredAudit: fixed("/api/v1/agents/{name}/declared-audit"),
|
|
854
|
+
/** POST: the sidecar's declared audit (at start and on a manifest hash change) and its own evidence records (ADR-0008). */
|
|
855
|
+
sidecarPush: fixed("/api/v1/agents/{name}/sidecar-push"),
|
|
856
|
+
/** GET: the latest stored EvidenceRecord per (ruleId, checkId, source) of an agent, as `{ records }`; read by the mcp-server. */
|
|
857
|
+
evidence: fixed("/api/v1/agents/{name}/evidence")
|
|
858
|
+
}).strict().prefault({}),
|
|
859
|
+
/** Names inside the pod patch that every injection path must use (ADR-0028). */
|
|
860
|
+
podPatch: zod.z.object({
|
|
861
|
+
sidecarContainer: fixed("agentspec-sidecar"),
|
|
862
|
+
/** Pod annotation the webhook writes its PodSpec findings to (JSON list of `{code, subject, message}`). */
|
|
863
|
+
podspecFindingsAnnotation: fixed("agentspec.io/podspec-findings"),
|
|
864
|
+
/** Pod annotation naming the agent; the webhook reads the manifest ConfigMap `<name>-manifest` from it, and `--deploy k8s` writes it. */
|
|
865
|
+
agentNameAnnotation: fixed("agentspec.io/agent-name"),
|
|
866
|
+
manifestVolume: fixed("agentspec-manifest"),
|
|
867
|
+
proxyPortName: fixed("proxy"),
|
|
868
|
+
controlPortName: fixed("control"),
|
|
869
|
+
gatewayContainer: fixed("agentspec-gateway"),
|
|
870
|
+
gatewayConfigInitContainer: fixed("agentspec-gateway-config"),
|
|
871
|
+
/** Sidecar subcommand the init container runs: `node <entrypoint> <this> <manifest> --out <path>`. */
|
|
872
|
+
gatewayConfigCommand: fixed("gateway-config"),
|
|
873
|
+
gatewayConfigVolume: fixed("agentspec-gateway-config"),
|
|
874
|
+
gatewayDataVolume: fixed("agentspec-gateway-data"),
|
|
875
|
+
/** Secret `<metadata.name><suffix>` holding the `sidecarToken` from `POST /api/v1/register`; the webhook gives it to the sidecar only, as `env.agentKey`, when a control plane URL is set. */
|
|
876
|
+
agentTokenSecretSuffix: fixed("-agentspec-token"),
|
|
877
|
+
agentTokenSecretKey: fixed("token"),
|
|
878
|
+
/** Loopback address the agent and sidecar reach each other and the gateway on. */
|
|
879
|
+
loopback: fixed("127.0.0.1"),
|
|
880
|
+
/** Gateway container sizing (ADR-0025): request 25m/32Mi, limit 128Mi. */
|
|
881
|
+
gatewayCpuRequest: fixed("25m"),
|
|
882
|
+
gatewayMemoryRequest: fixed("32Mi"),
|
|
883
|
+
gatewayMemoryLimit: fixed("128Mi")
|
|
884
|
+
}).strict().prefault({}),
|
|
885
|
+
/** Admission webhook the operator serves (ADR-0028). */
|
|
886
|
+
webhook: zod.z.object({
|
|
887
|
+
/** Path kube-apiserver posts to; kopf routes `POST /<handler id>`, so this is the handler id with a leading slash. Hyphenated: kube-apiserver rejects an underscore in a service path segment (RFC 1123). */
|
|
888
|
+
injectPath: fixed("/inject-sidecar"),
|
|
889
|
+
/** Namespace label that selects the namespaces whose pods are injected. */
|
|
890
|
+
namespaceLabel: fixed("agentspec.io/enabled")
|
|
891
|
+
}).strict().prefault({}),
|
|
892
|
+
/** Known `EvidenceRecord.reason.code` values (ADR-0006). */
|
|
893
|
+
reasonCodes: zod.z.array(zod.z.string()).default(REASON_CODES),
|
|
894
|
+
/** Severity weights for coverage and the verified score (ADR-0007). */
|
|
895
|
+
severityWeights: zod.z.object({
|
|
896
|
+
critical: fixed(4),
|
|
897
|
+
high: fixed(3),
|
|
898
|
+
medium: fixed(2),
|
|
899
|
+
low: fixed(1),
|
|
900
|
+
info: fixed(0)
|
|
901
|
+
}).strict().prefault({}),
|
|
902
|
+
/** Minimum verified percentage per grade; below `D` is F (ADR-0007). */
|
|
903
|
+
gradeBands: zod.z.object({ A: fixed(90), B: fixed(75), C: fixed(60), D: fixed(45) }).strict().prefault({}),
|
|
904
|
+
/** Severity-weighted coverage percentage below which no grade is shown (ADR-0007). */
|
|
905
|
+
coverageThreshold: fixed(60),
|
|
906
|
+
/** Rule evidence registry: what can prove each audit rule (ADR-0040). */
|
|
907
|
+
ruleEvidence: zod.z.record(zod.z.string(), RuleEvidenceEntry).default(RULE_EVIDENCE),
|
|
908
|
+
/** Evidence point → target → the rule its records serve; each rule's registry entry lists a source for that point. */
|
|
909
|
+
evidencePoints: zod.z.record(zod.z.string(), zod.z.record(zod.z.string(), zod.z.string())).default({
|
|
910
|
+
guardrail: { input: "SEC-LLM-01", output: "SEC-LLM-02" },
|
|
911
|
+
tool: { local: "TOOL-01" },
|
|
912
|
+
mcpClient: { "tools/list": "SEC-LLM-07", "tools/call": "TOOL-01" },
|
|
913
|
+
memoryWrite: { scrub: "MEM-01", ttl: "MEM-02", audit: "MEM-03" },
|
|
914
|
+
vectorStore: { namespace: "MEM-04" },
|
|
915
|
+
logging: { json: "OBS-02" }
|
|
916
|
+
}),
|
|
917
|
+
/** Records an evidence point aggregate keeps per window. */
|
|
918
|
+
evidencePointRecordLimit: fixed(MAX_POINT_RECORDS),
|
|
919
|
+
/** A heartbeat above this is trimmed to counts before it is sent. */
|
|
920
|
+
heartbeatMaxBytes: fixed(64 * 1024),
|
|
921
|
+
/** OTLP: resource attributes every exported span carries, and the standard endpoint env var. */
|
|
922
|
+
otlp: zod.z.object({
|
|
923
|
+
resourceAgentName: fixed("agentspec.agent.name"),
|
|
924
|
+
resourceManifestSha256: fixed("agentspec.manifest.sha256"),
|
|
925
|
+
endpointEnv: fixed("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
|
926
|
+
/** Resource attribute naming the span's producer, and the value agentgateway exports (its default); only such spans are gateway evidence. */
|
|
927
|
+
resourceServiceName: fixed("service.name"),
|
|
928
|
+
gatewayServiceName: fixed("agentgateway")
|
|
929
|
+
}).strict().prefault({})
|
|
930
|
+
}).strict();
|
|
931
|
+
var CONTRACT = ContractSchema.parse({});
|
|
932
|
+
|
|
933
|
+
// src/schema/evaluation.ts
|
|
934
|
+
var REPLACED_BY = {
|
|
935
|
+
hallucination: "grounded",
|
|
936
|
+
faithfulness: "grounded"
|
|
937
|
+
};
|
|
938
|
+
function canonicalMetric(metric) {
|
|
939
|
+
return REPLACED_BY[metric] ?? metric;
|
|
940
|
+
}
|
|
941
|
+
function evaluationWarnings(manifest) {
|
|
942
|
+
const evaluation = manifest.spec.evaluation;
|
|
943
|
+
if (!evaluation) return [];
|
|
944
|
+
const warnings = [];
|
|
945
|
+
if (evaluation.framework !== void 0) {
|
|
946
|
+
warnings.push(
|
|
947
|
+
"spec.evaluation.framework is deprecated: nothing reads it and AgentSpec computes evaluation evidence itself. Remove it; it is rejected in 0.6.0."
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
const declared2 = [
|
|
951
|
+
...evaluation.metrics ?? [],
|
|
952
|
+
...evaluation.live?.metrics ?? [],
|
|
953
|
+
...(evaluation.datasets ?? []).flatMap((d) => d.metrics ?? [])
|
|
954
|
+
];
|
|
955
|
+
for (const metric of new Set(declared2)) {
|
|
956
|
+
const replacement = canonicalMetric(metric);
|
|
957
|
+
if (replacement !== metric) {
|
|
958
|
+
warnings.push(
|
|
959
|
+
`spec.evaluation metric "${metric}" is deprecated and is read as "${replacement}". Rename it to "${replacement}"; it is rejected in 0.6.0.`
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
warnings.push(
|
|
964
|
+
...thresholdWarnings("spec.evaluation.thresholds", evaluation.thresholds),
|
|
965
|
+
...thresholdWarnings("spec.evaluation.live.thresholds", evaluation.live?.thresholds)
|
|
966
|
+
);
|
|
967
|
+
return warnings;
|
|
968
|
+
}
|
|
969
|
+
function thresholdWarnings(path, thresholds) {
|
|
970
|
+
return Object.keys(thresholds ?? {}).flatMap((name) => {
|
|
971
|
+
const replacement = canonicalMetric(name);
|
|
972
|
+
if (replacement === name) return [];
|
|
973
|
+
const ignored = replacement in thresholds ? ` It is ignored: ${path}.${replacement} is also declared.` : "";
|
|
974
|
+
return [
|
|
975
|
+
`${path}.${name} is deprecated and is read as ${path}.${replacement}. Rename it to "${replacement}"; it is rejected in 0.6.0.${ignored}`
|
|
976
|
+
];
|
|
365
977
|
});
|
|
366
978
|
}
|
|
979
|
+
function assertRubricRefs(parsed) {
|
|
980
|
+
const evaluation = parsed?.spec?.evaluation;
|
|
981
|
+
const live = evaluation?.live;
|
|
982
|
+
for (const [path, custom] of [
|
|
983
|
+
["spec.evaluation.custom", evaluation?.custom],
|
|
984
|
+
["spec.evaluation.live.custom", live?.custom]
|
|
985
|
+
]) {
|
|
986
|
+
const rubric = custom?.rubric;
|
|
987
|
+
if (typeof rubric === "string" && !rubric.startsWith("$file:")) {
|
|
988
|
+
throw new Error(
|
|
989
|
+
`${path}.rubric must be a "$file:<path>" reference, got "${rubric}". Fix: write it as "$file:${rubric}" (path relative to agent.yaml).`
|
|
990
|
+
);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
function normalizeEvaluation(manifest) {
|
|
995
|
+
const evaluation = manifest.spec.evaluation;
|
|
996
|
+
if (!evaluation) return manifest;
|
|
997
|
+
const metrics = (list) => list?.map(canonicalMetric);
|
|
998
|
+
const thresholds = (map) => {
|
|
999
|
+
if (!map) return map;
|
|
1000
|
+
const out = {};
|
|
1001
|
+
for (const [name, value] of Object.entries(map)) {
|
|
1002
|
+
const key = canonicalMetric(name);
|
|
1003
|
+
if (key === name || !(key in map)) out[key] = value;
|
|
1004
|
+
}
|
|
1005
|
+
return out;
|
|
1006
|
+
};
|
|
1007
|
+
return {
|
|
1008
|
+
...manifest,
|
|
1009
|
+
spec: {
|
|
1010
|
+
...manifest.spec,
|
|
1011
|
+
evaluation: {
|
|
1012
|
+
...evaluation,
|
|
1013
|
+
metrics: metrics(evaluation.metrics),
|
|
1014
|
+
thresholds: thresholds(evaluation.thresholds),
|
|
1015
|
+
datasets: evaluation.datasets?.map((d) => ({ ...d, metrics: metrics(d.metrics) })),
|
|
1016
|
+
live: evaluation.live && {
|
|
1017
|
+
...evaluation.live,
|
|
1018
|
+
metrics: metrics(evaluation.live.metrics),
|
|
1019
|
+
thresholds: thresholds(evaluation.live.thresholds)
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
367
1025
|
function detectRefType(value) {
|
|
368
1026
|
if (value.startsWith("$env:")) return "env";
|
|
369
1027
|
if (value.startsWith("$secret:")) return "secret";
|
|
@@ -409,8 +1067,9 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
409
1067
|
Use paths relative to agent.yaml location only`
|
|
410
1068
|
);
|
|
411
1069
|
}
|
|
1070
|
+
const readPath = checkedReadPath(value, absPath, opts.baseDir);
|
|
412
1071
|
try {
|
|
413
|
-
return fs.readFileSync(
|
|
1072
|
+
return fs.readFileSync(readPath, "utf-8");
|
|
414
1073
|
} catch {
|
|
415
1074
|
if (optional) return "";
|
|
416
1075
|
throw new Error(
|
|
@@ -432,6 +1091,29 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
432
1091
|
}
|
|
433
1092
|
return value;
|
|
434
1093
|
}
|
|
1094
|
+
function checkedReadPath(ref, absPath, baseDir) {
|
|
1095
|
+
let realTarget;
|
|
1096
|
+
try {
|
|
1097
|
+
realTarget = fs.realpathSync(absPath);
|
|
1098
|
+
} catch (err) {
|
|
1099
|
+
if (err.code === "ENOENT") return absPath;
|
|
1100
|
+
throw new Error(
|
|
1101
|
+
`Cannot resolve ${ref}
|
|
1102
|
+
${err.message}
|
|
1103
|
+
Fix the path: every folder in it must exist, be a folder and not loop back through symlinks`
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
const realBase = fs.realpathSync(baseDir);
|
|
1107
|
+
if (realTarget !== realBase && !realTarget.startsWith(realBase + path.sep)) {
|
|
1108
|
+
throw new Error(
|
|
1109
|
+
`Path traversal detected in ${ref}
|
|
1110
|
+
Resolved through a symlink to: ${realTarget}
|
|
1111
|
+
Which is outside baseDir: ${baseDir}
|
|
1112
|
+
Replace the symlink with a real folder inside the manifest directory, or move the file there`
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
return realTarget;
|
|
1116
|
+
}
|
|
435
1117
|
function resolveSecret(name, backend) {
|
|
436
1118
|
switch (backend) {
|
|
437
1119
|
case "env": {
|
|
@@ -458,7 +1140,7 @@ function resolveSecret(name, backend) {
|
|
|
458
1140
|
}
|
|
459
1141
|
function resolveRefs(obj, opts) {
|
|
460
1142
|
if (typeof obj === "string") {
|
|
461
|
-
return resolveRef(obj, opts
|
|
1143
|
+
return resolveRef(obj, opts);
|
|
462
1144
|
}
|
|
463
1145
|
if (Array.isArray(obj)) {
|
|
464
1146
|
return obj.map((item) => resolveRefs(item, opts));
|
|
@@ -510,18 +1192,20 @@ function loadManifest(filePath, opts = {}) {
|
|
|
510
1192
|
}
|
|
511
1193
|
let parsed;
|
|
512
1194
|
try {
|
|
513
|
-
parsed =
|
|
1195
|
+
parsed = yaml__namespace.load(raw, { schema: yaml__namespace.JSON_SCHEMA });
|
|
514
1196
|
} catch (err) {
|
|
515
1197
|
throw new Error(`Invalid YAML in ${absPath}
|
|
516
1198
|
${String(err)}`);
|
|
517
1199
|
}
|
|
1200
|
+
assertRubricRefs(parsed);
|
|
518
1201
|
const toValidate = opts.resolve === true ? resolveRefs(parsed, {
|
|
519
1202
|
baseDir,
|
|
520
1203
|
secretBackend: opts.secretBackend,
|
|
521
1204
|
failOnMissingEnv: opts.failOnMissingEnv ?? true
|
|
522
1205
|
}) : parsed;
|
|
523
|
-
const
|
|
524
|
-
|
|
1206
|
+
const declared2 = ManifestSchema.parse(toValidate);
|
|
1207
|
+
const warnings = evaluationWarnings(declared2);
|
|
1208
|
+
return { manifest: normalizeEvaluation(declared2), warnings, filePath: absPath, baseDir, raw };
|
|
525
1209
|
}
|
|
526
1210
|
function tryLoadManifest(filePath, opts = {}) {
|
|
527
1211
|
try {
|
|
@@ -568,8 +1252,8 @@ function isLatestVersion(raw) {
|
|
|
568
1252
|
function runEnvChecks(manifest, rawManifest) {
|
|
569
1253
|
const checks = [];
|
|
570
1254
|
const envRefs = collectEnvRefs(rawManifest);
|
|
571
|
-
const
|
|
572
|
-
for (const v of
|
|
1255
|
+
const declared2 = manifest.spec.requires?.envVars ?? [];
|
|
1256
|
+
for (const v of declared2) envRefs.add(v);
|
|
573
1257
|
for (const varName of envRefs) {
|
|
574
1258
|
const present = process.env[varName] !== void 0 && process.env[varName] !== "";
|
|
575
1259
|
checks.push({
|
|
@@ -606,21 +1290,53 @@ var PROVIDER_ENDPOINTS = {
|
|
|
606
1290
|
openai: "https://api.openai.com/v1/models",
|
|
607
1291
|
anthropic: "https://api.anthropic.com/v1/models",
|
|
608
1292
|
groq: "https://api.groq.com/openai/v1/models",
|
|
609
|
-
|
|
610
|
-
google: "https://generativelanguage.googleapis.com",
|
|
1293
|
+
google: "https://generativelanguage.googleapis.com/v1beta/models",
|
|
611
1294
|
mistral: "https://api.mistral.ai/v1/models",
|
|
612
|
-
cohere: "https://api.cohere.
|
|
1295
|
+
cohere: "https://api.cohere.com/v1/models",
|
|
613
1296
|
together: "https://api.together.xyz/v1/models",
|
|
614
1297
|
fireworks: "https://api.fireworks.ai/inference/v1/models"
|
|
615
1298
|
};
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
1299
|
+
var AZURE_API_VERSION = "2024-10-21";
|
|
1300
|
+
var AZURE_RESOURCE_HOSTS = [".openai.azure.com", ".cognitiveservices.azure.com", ".openai.azure.us", ".openai.azure.cn"];
|
|
1301
|
+
function isAzureResourceEndpoint(base) {
|
|
1302
|
+
const { hostname, pathname } = new URL(base);
|
|
1303
|
+
return AZURE_RESOURCE_HOSTS.some((suffix) => hostname.endsWith(suffix)) && pathname === "/";
|
|
1304
|
+
}
|
|
1305
|
+
function probeUrl(provider, baseUrl) {
|
|
1306
|
+
const key = provider.toLowerCase();
|
|
1307
|
+
const nativeUrl = PROVIDER_ENDPOINTS[key];
|
|
1308
|
+
if (!nativeUrl && !(key === "azure" && baseUrl)) return void 0;
|
|
1309
|
+
if (!baseUrl) return { url: nativeUrl, native: true };
|
|
1310
|
+
const base = resolveRef(baseUrl, { baseDir: "." }).replace(/\/+$/, "");
|
|
1311
|
+
if (key === "azure" && isAzureResourceEndpoint(base)) {
|
|
1312
|
+
return { url: `${base}/openai/models?api-version=${AZURE_API_VERSION}`, native: false, azureResource: true };
|
|
1313
|
+
}
|
|
1314
|
+
if (key === "anthropic") return { url: `${base}/v1/models`, native: false };
|
|
1315
|
+
return { url: `${base}/models`, native: false };
|
|
1316
|
+
}
|
|
1317
|
+
function unverifiableProbe(provider) {
|
|
1318
|
+
if (provider.toLowerCase() === "azure") {
|
|
619
1319
|
return {
|
|
620
|
-
|
|
621
|
-
|
|
1320
|
+
outcome: "not_verifiable",
|
|
1321
|
+
error: "azure keys are checked against your own resource endpoint, and spec.model.baseUrl is not set to name it",
|
|
1322
|
+
remediation: "Set spec.model.baseUrl to your Azure resource endpoint (https://<resource>.openai.azure.com) so the key can be probed, or verify it out of band"
|
|
622
1323
|
};
|
|
623
1324
|
}
|
|
1325
|
+
return { outcome: "not_verifiable", error: `unknown provider "${provider}"` };
|
|
1326
|
+
}
|
|
1327
|
+
async function checkModelEndpoint(provider, apiKey, baseUrl, timeoutMs = 5e3) {
|
|
1328
|
+
let target;
|
|
1329
|
+
try {
|
|
1330
|
+
target = probeUrl(provider, baseUrl);
|
|
1331
|
+
} catch (err) {
|
|
1332
|
+
return {
|
|
1333
|
+
outcome: "unreachable",
|
|
1334
|
+
error: err instanceof Error ? err.message.split("\n")[0] : String(err),
|
|
1335
|
+
remediation: `Set the variable behind spec.model baseUrl (${baseUrl}) or remove the override`
|
|
1336
|
+
};
|
|
1337
|
+
}
|
|
1338
|
+
if (!target) return unverifiableProbe(provider);
|
|
1339
|
+
const { url, native, azureResource } = target;
|
|
624
1340
|
const start = Date.now();
|
|
625
1341
|
const controller = new AbortController();
|
|
626
1342
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -634,19 +1350,30 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
634
1350
|
headers["anthropic-version"] = "2023-06-01";
|
|
635
1351
|
delete headers["Authorization"];
|
|
636
1352
|
}
|
|
1353
|
+
if (azureResource) {
|
|
1354
|
+
headers["api-key"] = apiKey;
|
|
1355
|
+
delete headers["Authorization"];
|
|
1356
|
+
}
|
|
1357
|
+
const isGemini = provider.toLowerCase() === "google" && native;
|
|
1358
|
+
if (isGemini) {
|
|
1359
|
+
headers["x-goog-api-key"] = apiKey;
|
|
1360
|
+
delete headers["Authorization"];
|
|
1361
|
+
}
|
|
637
1362
|
const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
|
|
638
1363
|
const latencyMs = Date.now() - start;
|
|
639
|
-
const
|
|
640
|
-
return {
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
1364
|
+
const geminiBadKey = isGemini && res.status === 400;
|
|
1365
|
+
if (res.status === 401 || res.status === 403 || geminiBadKey) return { outcome: "rejected", latencyMs, error: `HTTP ${res.status}` };
|
|
1366
|
+
if (res.status >= 500) return { outcome: "unreachable", latencyMs, error: `HTTP ${res.status}` };
|
|
1367
|
+
if (res.status === 404 || res.status === 405 || res.status === 410) {
|
|
1368
|
+
const { origin, pathname } = new URL(url);
|
|
1369
|
+
return { outcome: "unreachable", latencyMs, error: res.status === 404 ? "HTTP 404 (endpoint not found)" : `HTTP ${res.status} (not a models endpoint)`, remediation: `Check that ${origin}${pathname} is the provider's models endpoint${native ? "" : " (fix spec.model.baseUrl)"}` };
|
|
1370
|
+
}
|
|
1371
|
+
return { outcome: "ok", latencyMs };
|
|
645
1372
|
} catch (err) {
|
|
646
1373
|
const latencyMs = Date.now() - start;
|
|
647
1374
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
648
1375
|
return {
|
|
649
|
-
|
|
1376
|
+
outcome: "unreachable",
|
|
650
1377
|
latencyMs,
|
|
651
1378
|
error: isTimeout ? "Request timed out" : String(err)
|
|
652
1379
|
};
|
|
@@ -654,6 +1381,20 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
654
1381
|
clearTimeout(timeout);
|
|
655
1382
|
}
|
|
656
1383
|
}
|
|
1384
|
+
function modelProbeCheck(probe, fields, label, provider, keyRef) {
|
|
1385
|
+
const keyName = keyRef.startsWith("$") ? keyRef : "[api-key]";
|
|
1386
|
+
const base = { ...fields, latencyMs: probe.latencyMs };
|
|
1387
|
+
switch (probe.outcome) {
|
|
1388
|
+
case "ok":
|
|
1389
|
+
return { ...base, status: "pass" };
|
|
1390
|
+
case "not_verifiable":
|
|
1391
|
+
return { ...base, status: "not_verifiable", message: `${label} cannot be verified: ${probe.error}`, remediation: probe.remediation ?? `Use a provider the health check knows (${Object.keys(PROVIDER_ENDPOINTS).join(", ")}) or verify ${provider} out of band` };
|
|
1392
|
+
case "rejected":
|
|
1393
|
+
return { ...base, status: "fail", message: `${label} for ${provider} rejected the API key: ${probe.error}`, remediation: `Check that ${keyName} holds a valid ${provider} API key` };
|
|
1394
|
+
default:
|
|
1395
|
+
return { ...base, status: "fail", message: `${label} for ${provider} is unreachable: ${probe.error}`, remediation: probe.remediation ?? `Check that ${keyName} is set and the ${provider} API is reachable` };
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
657
1398
|
function resolveApiKey(ref) {
|
|
658
1399
|
if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
|
|
659
1400
|
if (ref.startsWith("$")) return "";
|
|
@@ -663,16 +1404,14 @@ async function runModelChecks(model) {
|
|
|
663
1404
|
const checks = [];
|
|
664
1405
|
const apiKey = resolveApiKey(model.apiKey);
|
|
665
1406
|
if (apiKey) {
|
|
666
|
-
const
|
|
667
|
-
checks.push(
|
|
668
|
-
|
|
669
|
-
category: "model",
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
|
|
675
|
-
});
|
|
1407
|
+
const probe = await checkModelEndpoint(model.provider, apiKey, model.baseUrl);
|
|
1408
|
+
checks.push(modelProbeCheck(
|
|
1409
|
+
probe,
|
|
1410
|
+
{ id: `model:${model.provider}/${model.id}`, category: "model", severity: "error" },
|
|
1411
|
+
"Model endpoint",
|
|
1412
|
+
model.provider,
|
|
1413
|
+
model.apiKey
|
|
1414
|
+
));
|
|
676
1415
|
} else {
|
|
677
1416
|
checks.push({
|
|
678
1417
|
id: `model:${model.provider}/${model.id}`,
|
|
@@ -682,32 +1421,6 @@ async function runModelChecks(model) {
|
|
|
682
1421
|
message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
|
|
683
1422
|
});
|
|
684
1423
|
}
|
|
685
|
-
if (model.fallback) {
|
|
686
|
-
const fallbackKey = resolveApiKey(model.fallback.apiKey);
|
|
687
|
-
if (fallbackKey) {
|
|
688
|
-
const { reachable, latencyMs, error } = await checkModelEndpoint(
|
|
689
|
-
model.fallback.provider,
|
|
690
|
-
fallbackKey
|
|
691
|
-
);
|
|
692
|
-
checks.push({
|
|
693
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
694
|
-
category: "model-fallback",
|
|
695
|
-
status: reachable ? "pass" : "fail",
|
|
696
|
-
severity: "warning",
|
|
697
|
-
latencyMs,
|
|
698
|
-
message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
|
|
699
|
-
remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
|
|
700
|
-
});
|
|
701
|
-
} else {
|
|
702
|
-
checks.push({
|
|
703
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
704
|
-
category: "model-fallback",
|
|
705
|
-
status: "skip",
|
|
706
|
-
severity: "warning",
|
|
707
|
-
message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
|
|
708
|
-
});
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
1424
|
return checks;
|
|
712
1425
|
}
|
|
713
1426
|
|
|
@@ -719,7 +1432,7 @@ async function runMcpChecks(servers) {
|
|
|
719
1432
|
const url = server.url;
|
|
720
1433
|
if (url && !url.startsWith("$")) {
|
|
721
1434
|
const check = await checkHttpMcpServer(server.name, url);
|
|
722
|
-
checks.push(check);
|
|
1435
|
+
checks.push(server.transport === "sse" ? withLegacySseHint(check) : check);
|
|
723
1436
|
} else {
|
|
724
1437
|
checks.push({
|
|
725
1438
|
id: `mcp:${server.name}`,
|
|
@@ -736,37 +1449,145 @@ async function runMcpChecks(servers) {
|
|
|
736
1449
|
}
|
|
737
1450
|
return checks;
|
|
738
1451
|
}
|
|
1452
|
+
var MCP_PROTOCOL_VERSION = "2025-03-26";
|
|
1453
|
+
var HANDSHAKE_TIMEOUT_MS = 5e3;
|
|
1454
|
+
var INITIALIZE_REQUEST_ID = 1;
|
|
1455
|
+
var MAX_SERVER_TEXT = 200;
|
|
1456
|
+
var PROTOCOL_VERSION_FORMAT = /^\d{4}-\d{2}-\d{2}$/;
|
|
739
1457
|
async function checkHttpMcpServer(name, url) {
|
|
740
1458
|
const controller = new AbortController();
|
|
741
|
-
const timeout = setTimeout(() => controller.abort(),
|
|
1459
|
+
const timeout = setTimeout(() => controller.abort(), HANDSHAKE_TIMEOUT_MS);
|
|
742
1460
|
const start = Date.now();
|
|
1461
|
+
const headers = {
|
|
1462
|
+
"content-type": "application/json",
|
|
1463
|
+
accept: "application/json, text/event-stream"
|
|
1464
|
+
};
|
|
743
1465
|
try {
|
|
744
|
-
const res = await fetch(url, {
|
|
1466
|
+
const res = await fetch(url, {
|
|
1467
|
+
method: "POST",
|
|
1468
|
+
headers,
|
|
1469
|
+
signal: controller.signal,
|
|
1470
|
+
body: JSON.stringify({
|
|
1471
|
+
jsonrpc: "2.0",
|
|
1472
|
+
id: INITIALIZE_REQUEST_ID,
|
|
1473
|
+
method: "initialize",
|
|
1474
|
+
params: {
|
|
1475
|
+
protocolVersion: MCP_PROTOCOL_VERSION,
|
|
1476
|
+
capabilities: {},
|
|
1477
|
+
clientInfo: { name: "agentspec-health", version: "1.0.0" }
|
|
1478
|
+
}
|
|
1479
|
+
})
|
|
1480
|
+
});
|
|
1481
|
+
if (!res.ok) {
|
|
1482
|
+
await res.body?.cancel().catch(() => void 0);
|
|
1483
|
+
return failedHandshake(name, Date.now() - start, `returned HTTP ${res.status} to initialize`);
|
|
1484
|
+
}
|
|
1485
|
+
const parsed = await readInitializeResult(res);
|
|
745
1486
|
const latencyMs = Date.now() - start;
|
|
1487
|
+
if ("error" in parsed) {
|
|
1488
|
+
return failedHandshake(name, latencyMs, parsed.error);
|
|
1489
|
+
}
|
|
1490
|
+
const result = parsed.result;
|
|
1491
|
+
const sessionId = res.headers.get("mcp-session-id");
|
|
1492
|
+
if (sessionId) headers["mcp-session-id"] = sessionId;
|
|
1493
|
+
headers["mcp-protocol-version"] = result.protocolVersion;
|
|
1494
|
+
await sendBestEffort(url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) }, controller.signal);
|
|
1495
|
+
if (sessionId) await sendBestEffort(url, { method: "DELETE", headers }, controller.signal);
|
|
1496
|
+
const serverName = clip(result.serverInfo?.name);
|
|
1497
|
+
const serverVersion = clip(result.serverInfo?.version);
|
|
1498
|
+
const info = serverName ? ` (${serverName}${serverVersion ? ` ${serverVersion}` : ""})` : "";
|
|
746
1499
|
return {
|
|
747
1500
|
id: `mcp:${name}`,
|
|
748
1501
|
category: "mcp",
|
|
749
|
-
status:
|
|
1502
|
+
status: "pass",
|
|
750
1503
|
severity: "warning",
|
|
751
1504
|
latencyMs,
|
|
752
|
-
message:
|
|
1505
|
+
message: `MCP server ${name}${info} initialized, protocolVersion ${clip(result.protocolVersion)}`
|
|
753
1506
|
};
|
|
754
1507
|
} catch (err) {
|
|
755
1508
|
const latencyMs = Date.now() - start;
|
|
756
1509
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
757
|
-
return
|
|
758
|
-
|
|
759
|
-
category: "mcp",
|
|
760
|
-
status: "fail",
|
|
761
|
-
severity: "warning",
|
|
1510
|
+
return failedHandshake(
|
|
1511
|
+
name,
|
|
762
1512
|
latencyMs,
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
};
|
|
1513
|
+
isTimeout ? "health check timed out" : `unreachable: ${redactUrlCredentials(String(err))}`
|
|
1514
|
+
);
|
|
766
1515
|
} finally {
|
|
767
1516
|
clearTimeout(timeout);
|
|
768
1517
|
}
|
|
769
1518
|
}
|
|
1519
|
+
async function sendBestEffort(url, init, signal) {
|
|
1520
|
+
await fetch(url, { ...init, signal }).then((r) => r.body?.cancel()).catch(() => void 0);
|
|
1521
|
+
}
|
|
1522
|
+
function withLegacySseHint(check) {
|
|
1523
|
+
if (check.status !== "fail") return check;
|
|
1524
|
+
return {
|
|
1525
|
+
...check,
|
|
1526
|
+
remediation: `${check.remediation ?? ""} Note: transport "sse" is probed with the streamable-HTTP handshake; legacy HTTP+SSE servers are not supported by this check.`.trim()
|
|
1527
|
+
};
|
|
1528
|
+
}
|
|
1529
|
+
function redactUrlCredentials(text) {
|
|
1530
|
+
return text.replace(/(\w+:\/\/)[^/\s]*@/g, "$1***@");
|
|
1531
|
+
}
|
|
1532
|
+
function failedHandshake(name, latencyMs, reason) {
|
|
1533
|
+
return {
|
|
1534
|
+
id: `mcp:${name}`,
|
|
1535
|
+
category: "mcp",
|
|
1536
|
+
status: "fail",
|
|
1537
|
+
severity: "warning",
|
|
1538
|
+
latencyMs,
|
|
1539
|
+
message: `MCP server ${name} ${reason}`,
|
|
1540
|
+
remediation: `Check that MCP server ${name} is running and that spec.mcp.servers[].url points at a streamable-HTTP MCP endpoint`
|
|
1541
|
+
};
|
|
1542
|
+
}
|
|
1543
|
+
async function readInitializeResult(res) {
|
|
1544
|
+
const isSse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
|
|
1545
|
+
const text = isSse ? await readSseUntilInitialize(res) : await res.text();
|
|
1546
|
+
const payloads = isSse ? splitSseEvents(text) : [text];
|
|
1547
|
+
for (const payload of payloads) {
|
|
1548
|
+
const outcome = parseInitializeFrame(payload);
|
|
1549
|
+
if (outcome) return outcome;
|
|
1550
|
+
}
|
|
1551
|
+
return { error: "did not return a valid JSON-RPC initialize result" };
|
|
1552
|
+
}
|
|
1553
|
+
function clip(value) {
|
|
1554
|
+
return typeof value === "string" ? value.slice(0, MAX_SERVER_TEXT) : "";
|
|
1555
|
+
}
|
|
1556
|
+
function parseInitializeFrame(payload) {
|
|
1557
|
+
try {
|
|
1558
|
+
const msg = JSON.parse(payload);
|
|
1559
|
+
if (msg.jsonrpc !== "2.0" || msg.id !== INITIALIZE_REQUEST_ID) return null;
|
|
1560
|
+
if (msg.result && typeof msg.result.protocolVersion === "string") {
|
|
1561
|
+
return PROTOCOL_VERSION_FORMAT.test(msg.result.protocolVersion) ? { result: msg.result } : { error: "returned a malformed protocolVersion in its initialize result" };
|
|
1562
|
+
}
|
|
1563
|
+
if (msg.error) return { error: `rejected initialize: ${clip(msg.error.message) || "JSON-RPC error"}` };
|
|
1564
|
+
} catch {
|
|
1565
|
+
}
|
|
1566
|
+
return null;
|
|
1567
|
+
}
|
|
1568
|
+
function splitSseEvents(text) {
|
|
1569
|
+
return text.split(/\r?\n\r?\n/).map(
|
|
1570
|
+
(event) => event.split(/\r?\n/).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).join("\n")
|
|
1571
|
+
);
|
|
1572
|
+
}
|
|
1573
|
+
async function readSseUntilInitialize(res) {
|
|
1574
|
+
if (!res.body) return res.text();
|
|
1575
|
+
const reader = res.body.getReader();
|
|
1576
|
+
const decoder = new TextDecoder();
|
|
1577
|
+
let text = "";
|
|
1578
|
+
try {
|
|
1579
|
+
for (; ; ) {
|
|
1580
|
+
const { done, value } = await reader.read();
|
|
1581
|
+
if (done) break;
|
|
1582
|
+
text += decoder.decode(value, { stream: true });
|
|
1583
|
+
const completeEvents = text.split(/\r?\n\r?\n/).slice(0, -1);
|
|
1584
|
+
if (completeEvents.flatMap((e) => splitSseEvents(e)).some((p) => parseInitializeFrame(p))) break;
|
|
1585
|
+
}
|
|
1586
|
+
} finally {
|
|
1587
|
+
await reader.cancel().catch(() => void 0);
|
|
1588
|
+
}
|
|
1589
|
+
return text;
|
|
1590
|
+
}
|
|
770
1591
|
async function checkStdioMcpServer(server) {
|
|
771
1592
|
const command = server.command;
|
|
772
1593
|
if (!command) {
|
|
@@ -1284,7 +2105,7 @@ function computeHealthStatus(checks) {
|
|
|
1284
2105
|
const failed = checks.filter((c) => c.status === "fail");
|
|
1285
2106
|
const warnings = checks.filter((c) => c.status === "warn");
|
|
1286
2107
|
const passed = checks.filter((c) => c.status === "pass");
|
|
1287
|
-
const skipped = checks.filter((c) => c.status === "skip");
|
|
2108
|
+
const skipped = checks.filter((c) => c.status === "skip" || c.status === "not_verifiable");
|
|
1288
2109
|
const hasErrors = failed.some((c) => c.severity === "error");
|
|
1289
2110
|
const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
|
|
1290
2111
|
const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
|
|
@@ -1301,8 +2122,10 @@ function computeHealthStatus(checks) {
|
|
|
1301
2122
|
async function runHealthCheck(manifest, opts = {}) {
|
|
1302
2123
|
const checks = [];
|
|
1303
2124
|
const rawManifest = opts.rawManifest ?? manifest;
|
|
1304
|
-
|
|
1305
|
-
|
|
2125
|
+
if (opts.checkEnv !== false) {
|
|
2126
|
+
checks.push(...runEnvChecks(manifest, rawManifest));
|
|
2127
|
+
checks.push(...await runSecretChecks());
|
|
2128
|
+
}
|
|
1306
2129
|
if (opts.baseDir) {
|
|
1307
2130
|
checks.push(...runFileChecks(rawManifest, opts.baseDir));
|
|
1308
2131
|
}
|
|
@@ -1392,7 +2215,381 @@ async function checkA2aEndpoint(name, url) {
|
|
|
1392
2215
|
}
|
|
1393
2216
|
}
|
|
1394
2217
|
|
|
2218
|
+
// src/grader/index.ts
|
|
2219
|
+
var ADMITTED = {
|
|
2220
|
+
standard: ["independent", "sdk-observed"],
|
|
2221
|
+
strict: ["independent"]
|
|
2222
|
+
};
|
|
2223
|
+
var SOURCE_CLASS = {
|
|
2224
|
+
independent: "independent",
|
|
2225
|
+
"sdk-observed": "evidence-point",
|
|
2226
|
+
offline: "offline",
|
|
2227
|
+
declared: "declared"
|
|
2228
|
+
};
|
|
2229
|
+
var pct = (part, whole) => whole === 0 ? 0 : part / whole * 100;
|
|
2230
|
+
function isCalendarDate(value) {
|
|
2231
|
+
const [y, m, d] = value.slice(0, 10).split("-").map(Number);
|
|
2232
|
+
const date = /* @__PURE__ */ new Date(0);
|
|
2233
|
+
date.setUTCFullYear(y, m - 1, d);
|
|
2234
|
+
return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
|
|
2235
|
+
}
|
|
2236
|
+
function parseTimestamp(value, field) {
|
|
2237
|
+
const ms = Date.parse(value);
|
|
2238
|
+
const valid = TIMESTAMP_PATTERN.test(value) && !Number.isNaN(ms) && isCalendarDate(value) && !value.startsWith("0000") && !value.includes("T24:");
|
|
2239
|
+
if (!valid) {
|
|
2240
|
+
throw new Error(`grade: ${field} "${value}" is not a valid RFC 3339 timestamp. Fix the producer to emit a real date and time.`);
|
|
2241
|
+
}
|
|
2242
|
+
return ms;
|
|
2243
|
+
}
|
|
2244
|
+
function isExpired(r, nowMs) {
|
|
2245
|
+
return parseTimestamp(r.observedAt, `observedAt of the record for check "${r.checkId}"`) + r.ttlSeconds * 1e3 < nowMs;
|
|
2246
|
+
}
|
|
2247
|
+
function matchesRegistry(r) {
|
|
2248
|
+
const cls = SOURCE_CLASS[r.source];
|
|
2249
|
+
if (!r.ruleId || !cls || !Object.hasOwn(RULE_EVIDENCE, r.ruleId)) return true;
|
|
2250
|
+
const listed = RULE_EVIDENCE[r.ruleId].sources.filter((s) => s.class === cls);
|
|
2251
|
+
return listed.length > 0 && (r.collector == null || listed.some((s) => s.collector === r.collector));
|
|
2252
|
+
}
|
|
2253
|
+
function unverifiedReason(admitted, stale, notAdmitted, mismatched) {
|
|
2254
|
+
const own = admitted.find((r) => r.status === "not_verifiable");
|
|
2255
|
+
if (own?.reason) return own.reason;
|
|
2256
|
+
if (stale.length > 0) return { code: "stale" };
|
|
2257
|
+
if (notAdmitted.length > 0) return { code: "source_not_admitted" };
|
|
2258
|
+
if (mismatched.length > 0) return { code: "source_mismatch" };
|
|
2259
|
+
return { code: "no_evidence" };
|
|
2260
|
+
}
|
|
2261
|
+
function judgeRule(allRecords, policy, nowMs, unwiredFails) {
|
|
2262
|
+
if (unwiredFails) allRecords = [...allRecords.filter((r) => r.source !== "sdk-observed"), ...unwiredFails];
|
|
2263
|
+
const admittedSources = ADMITTED[policy];
|
|
2264
|
+
const registryMatched = [];
|
|
2265
|
+
const mismatched = [];
|
|
2266
|
+
for (const r of allRecords) (matchesRegistry(r) ? registryMatched : mismatched).push(r);
|
|
2267
|
+
const admittedRecords = registryMatched.filter((r) => admittedSources.includes(r.source));
|
|
2268
|
+
const notAdmitted = registryMatched.filter((r) => !admittedSources.includes(r.source));
|
|
2269
|
+
const stale = admittedRecords.filter((r) => isExpired(r, nowMs));
|
|
2270
|
+
const fresh = admittedRecords.filter((r) => !isExpired(r, nowMs));
|
|
2271
|
+
const decisive = fresh.filter((r) => r.status !== "not_verifiable");
|
|
2272
|
+
if (decisive.length === 0) {
|
|
2273
|
+
const unverified = unwiredFails ? { code: "not_wired" } : unverifiedReason(fresh, stale, notAdmitted, mismatched);
|
|
2274
|
+
return { covered: false, pass: false, independent: false, unverified };
|
|
2275
|
+
}
|
|
2276
|
+
return {
|
|
2277
|
+
covered: true,
|
|
2278
|
+
pass: decisive.every((r) => r.status === "pass"),
|
|
2279
|
+
independent: decisive.every((r) => r.source === "independent")
|
|
2280
|
+
};
|
|
2281
|
+
}
|
|
2282
|
+
function unwiredRules(points) {
|
|
2283
|
+
const notWired = /* @__PURE__ */ new Set();
|
|
2284
|
+
const wiredFails = /* @__PURE__ */ new Map();
|
|
2285
|
+
for (const p of points) {
|
|
2286
|
+
const ruleId = CONTRACT.evidencePoints[p.point]?.[p.target];
|
|
2287
|
+
if (!ruleId) continue;
|
|
2288
|
+
if (p.state === "not_wired") notWired.add(ruleId);
|
|
2289
|
+
else {
|
|
2290
|
+
const fails = p.records.filter((r) => r.ruleId === ruleId && r.status === "fail");
|
|
2291
|
+
wiredFails.set(ruleId, [...wiredFails.get(ruleId) ?? [], ...fails]);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
return new Map([...notWired].map((id) => [id, wiredFails.get(id) ?? []]));
|
|
2295
|
+
}
|
|
2296
|
+
function judgeDerived(parents) {
|
|
2297
|
+
const open = parents.find((v) => !v.covered);
|
|
2298
|
+
const failed = parents.some((v) => v.covered && !v.pass);
|
|
2299
|
+
if (open && !failed) return open;
|
|
2300
|
+
const covered = parents.filter((v) => v.covered);
|
|
2301
|
+
return { covered: true, pass: !failed, independent: covered.every((v) => v.independent) };
|
|
2302
|
+
}
|
|
2303
|
+
function judgeRegistered(ruleId, byRule, policy, nowMs, unwired) {
|
|
2304
|
+
const derivedFrom = Object.hasOwn(RULE_EVIDENCE, ruleId) ? RULE_EVIDENCE[ruleId].derivedFrom : void 0;
|
|
2305
|
+
if (!derivedFrom?.length) return judgeRule(byRule.get(ruleId) ?? [], policy, nowMs, unwired.get(ruleId));
|
|
2306
|
+
return judgeDerived(derivedFrom.map((parent) => judgeRegistered(parent, byRule, policy, nowMs, unwired)));
|
|
2307
|
+
}
|
|
2308
|
+
function groupByRule(records) {
|
|
2309
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
2310
|
+
for (const r of records) {
|
|
2311
|
+
if (!r.ruleId) continue;
|
|
2312
|
+
const bucket2 = byRule.get(r.ruleId);
|
|
2313
|
+
if (bucket2) bucket2.push(r);
|
|
2314
|
+
else byRule.set(r.ruleId, [r]);
|
|
2315
|
+
}
|
|
2316
|
+
return byRule;
|
|
2317
|
+
}
|
|
2318
|
+
function scoreToGrade(score) {
|
|
2319
|
+
const { A, B, C, D } = CONTRACT.gradeBands;
|
|
2320
|
+
if (score >= A) return "A";
|
|
2321
|
+
if (score >= B) return "B";
|
|
2322
|
+
if (score >= C) return "C";
|
|
2323
|
+
if (score >= D) return "D";
|
|
2324
|
+
return "F";
|
|
2325
|
+
}
|
|
2326
|
+
function grade(records, applicableRules, policy, now, points = []) {
|
|
2327
|
+
const nowMs = parseTimestamp(now, "now");
|
|
2328
|
+
const byRule = groupByRule(records);
|
|
2329
|
+
const unwired = unwiredRules(points);
|
|
2330
|
+
let totalWeight = 0;
|
|
2331
|
+
let coveredWeight = 0;
|
|
2332
|
+
let passedWeight = 0;
|
|
2333
|
+
let independentWeight = 0;
|
|
2334
|
+
const unverified = [];
|
|
2335
|
+
for (const rule of applicableRules) {
|
|
2336
|
+
const weight = CONTRACT.severityWeights[rule.severity];
|
|
2337
|
+
const verdict = judgeRegistered(rule.id, byRule, policy, nowMs, unwired);
|
|
2338
|
+
totalWeight += weight;
|
|
2339
|
+
if (!verdict.covered) {
|
|
2340
|
+
unverified.push({ ruleId: rule.id, severity: rule.severity, reason: verdict.unverified });
|
|
2341
|
+
continue;
|
|
2342
|
+
}
|
|
2343
|
+
coveredWeight += weight;
|
|
2344
|
+
if (verdict.pass) passedWeight += weight;
|
|
2345
|
+
if (verdict.independent) independentWeight += weight;
|
|
2346
|
+
}
|
|
2347
|
+
const verified = Math.round(pct(passedWeight, coveredWeight));
|
|
2348
|
+
const coverage = Math.round(pct(coveredWeight, totalWeight));
|
|
2349
|
+
return {
|
|
2350
|
+
verified,
|
|
2351
|
+
coverage,
|
|
2352
|
+
independentShare: Math.round(pct(independentWeight, coveredWeight)),
|
|
2353
|
+
grade: coverage >= CONTRACT.coverageThreshold ? scoreToGrade(verified) : "Not graded",
|
|
2354
|
+
unverified
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
function formatVerified(result) {
|
|
2358
|
+
if (result.grade === "Not graded") return `Not graded (coverage ${result.coverage}%)`;
|
|
2359
|
+
return `${result.grade} - ${result.verified}% verified (${result.independentShare}% independent) - coverage ${result.coverage}%`;
|
|
2360
|
+
}
|
|
2361
|
+
var NATIVE_PROVIDERS = {
|
|
2362
|
+
openai: "openAI",
|
|
2363
|
+
anthropic: "anthropic",
|
|
2364
|
+
google: "gemini",
|
|
2365
|
+
gemini: "gemini",
|
|
2366
|
+
azure: "azure"
|
|
2367
|
+
};
|
|
2368
|
+
var OPENAI_COMPATIBLE_BASE_URLS = {
|
|
2369
|
+
groq: "https://api.groq.com/openai/v1",
|
|
2370
|
+
mistral: "https://api.mistral.ai/v1"
|
|
2371
|
+
};
|
|
2372
|
+
var CATALOG_PROVIDER_IDS = {
|
|
2373
|
+
openAI: "openai",
|
|
2374
|
+
anthropic: "anthropic",
|
|
2375
|
+
gemini: "gcp.gemini",
|
|
2376
|
+
azure: "azure"
|
|
2377
|
+
};
|
|
2378
|
+
var GATEWAY_NAME = "llm";
|
|
2379
|
+
var ONE_MONTH = "30d";
|
|
2380
|
+
function envRefName(value) {
|
|
2381
|
+
return /^\$env:(.+)$/.exec(value)?.[1];
|
|
2382
|
+
}
|
|
2383
|
+
function secretEnvName(name) {
|
|
2384
|
+
return `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
|
|
2385
|
+
}
|
|
2386
|
+
function keyEnvName(apiKey) {
|
|
2387
|
+
const env = envRefName(apiKey);
|
|
2388
|
+
if (env) return env;
|
|
2389
|
+
const secret = /^\$secret:(.+)$/.exec(apiKey);
|
|
2390
|
+
if (secret) return secretEnvName(secret[1]);
|
|
2391
|
+
throw new Error(
|
|
2392
|
+
'spec.model.apiKey must be a "$env:VAR" or "$secret:name" reference; a literal key would end up in the gateway config. Replace it with "$env:<VAR>".'
|
|
2393
|
+
);
|
|
2394
|
+
}
|
|
2395
|
+
function baseUrlValue(baseUrl) {
|
|
2396
|
+
const env = envRefName(baseUrl);
|
|
2397
|
+
return env ? `$${env}` : baseUrl;
|
|
2398
|
+
}
|
|
2399
|
+
function baseUrlEnvNames({ baseUrl }) {
|
|
2400
|
+
const name = baseUrl && envRefName(baseUrl);
|
|
2401
|
+
return name ? [name] : [];
|
|
2402
|
+
}
|
|
2403
|
+
function customProvider(provider, id, baseUrl) {
|
|
2404
|
+
const anthropic = provider === "anthropic";
|
|
2405
|
+
return {
|
|
2406
|
+
provider: { custom: { providerOverride: provider, formats: [{ type: anthropic ? "messages" : "completions" }] } },
|
|
2407
|
+
params: { model: id, baseUrl, ...anthropic ? { pathPrefix: "/v1" } : {} }
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
function gatewayEnvNames(manifest) {
|
|
2411
|
+
const { model } = manifest.spec;
|
|
2412
|
+
const models = [model, ...model.fallback ? [model.fallback] : []];
|
|
2413
|
+
const names = models.flatMap((m) => [
|
|
2414
|
+
keyEnvName(m.apiKey),
|
|
2415
|
+
...baseUrlEnvNames(m),
|
|
2416
|
+
...NATIVE_PROVIDERS[m.provider] === "azure" && !m.baseUrl ? [CONTRACT.env.azureResourceName] : []
|
|
2417
|
+
]);
|
|
2418
|
+
return [...new Set(names)].sort();
|
|
2419
|
+
}
|
|
2420
|
+
function providerFields({ provider, id, baseUrl }) {
|
|
2421
|
+
if (baseUrl) return customProvider(provider, id, baseUrlValue(baseUrl));
|
|
2422
|
+
const native = NATIVE_PROVIDERS[provider];
|
|
2423
|
+
if (native === "azure") {
|
|
2424
|
+
return {
|
|
2425
|
+
provider: native,
|
|
2426
|
+
params: { model: id, azureResourceName: `$${CONTRACT.env.azureResourceName}`, azureResourceType: "openAI" }
|
|
2427
|
+
};
|
|
2428
|
+
}
|
|
2429
|
+
if (native) return { provider: native, params: { model: id } };
|
|
2430
|
+
const knownBaseUrl = OPENAI_COMPATIBLE_BASE_URLS[provider];
|
|
2431
|
+
if (knownBaseUrl) return customProvider(provider, id, knownBaseUrl);
|
|
2432
|
+
const supported = [...Object.keys(NATIVE_PROVIDERS), ...Object.keys(OPENAI_COMPATIBLE_BASE_URLS)].sort();
|
|
2433
|
+
throw new Error(`spec.model.provider "${provider}" has no gateway mapping. Supported: ${supported.join(", ")}.`);
|
|
2434
|
+
}
|
|
2435
|
+
function renderModel(model) {
|
|
2436
|
+
const { provider, params } = providerFields(model);
|
|
2437
|
+
return {
|
|
2438
|
+
name: `${model.provider}/${model.id}`,
|
|
2439
|
+
visibility: "internal",
|
|
2440
|
+
provider,
|
|
2441
|
+
params: { ...params, apiKey: `$${keyEnvName(model.apiKey)}` }
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2444
|
+
function renderVirtualModel(models) {
|
|
2445
|
+
return {
|
|
2446
|
+
name: models[0].id,
|
|
2447
|
+
routing: {
|
|
2448
|
+
failover: {
|
|
2449
|
+
targets: models.map((m, priority) => ({ model: `${m.provider}/${m.id}`, priority }))
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
function renderRetryPolicy(model) {
|
|
2455
|
+
const attempts = model.fallback?.maxRetries;
|
|
2456
|
+
if (!attempts) return {};
|
|
2457
|
+
return {
|
|
2458
|
+
policies: [{
|
|
2459
|
+
name: { name: "llm-retry", namespace: "default" },
|
|
2460
|
+
target: { gateway: { gatewayName: "default", gatewayNamespace: "default", listenerName: GATEWAY_NAME } },
|
|
2461
|
+
policy: { retry: { attempts, codes: [500, 502, 503, 504] } }
|
|
2462
|
+
}]
|
|
2463
|
+
};
|
|
2464
|
+
}
|
|
2465
|
+
function renderBudgets(spec) {
|
|
2466
|
+
const amount = spec.model.costControls?.maxMonthlyUSD;
|
|
2467
|
+
if (amount === void 0) return {};
|
|
2468
|
+
return {
|
|
2469
|
+
budgets: [{
|
|
2470
|
+
name: "monthly",
|
|
2471
|
+
limit: { unit: "USD", amount },
|
|
2472
|
+
window: { rolling: ONE_MONTH },
|
|
2473
|
+
onBudgetExceeded: "Block"
|
|
2474
|
+
}]
|
|
2475
|
+
};
|
|
2476
|
+
}
|
|
2477
|
+
function renderApiKeyPolicy(manifest) {
|
|
2478
|
+
return {
|
|
2479
|
+
mode: "strict",
|
|
2480
|
+
keys: [{
|
|
2481
|
+
key: `$${CONTRACT.env.gatewayKey}`,
|
|
2482
|
+
metadata: { name: manifest.metadata.name },
|
|
2483
|
+
allowedModels: [manifest.spec.model.id],
|
|
2484
|
+
...renderBudgets(manifest.spec)
|
|
2485
|
+
}]
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2488
|
+
function bucket(type, size, fillInterval) {
|
|
2489
|
+
return { type, maxTokens: size, tokensPerFill: size, fillInterval };
|
|
2490
|
+
}
|
|
2491
|
+
function renderRateLimits(spec) {
|
|
2492
|
+
const { requestsPerMinute, requestsPerHour, tokensPerMinute } = spec.api?.rateLimit ?? {};
|
|
2493
|
+
const buckets = [
|
|
2494
|
+
...requestsPerMinute === void 0 ? [] : [bucket("requests", requestsPerMinute, "60s")],
|
|
2495
|
+
...requestsPerHour === void 0 ? [] : [bucket("requests", requestsPerHour, "1h")],
|
|
2496
|
+
...tokensPerMinute === void 0 ? [] : [bucket("tokens", tokensPerMinute, "60s")]
|
|
2497
|
+
];
|
|
2498
|
+
return buckets.length === 0 ? {} : { localRateLimit: buckets };
|
|
2499
|
+
}
|
|
2500
|
+
function renderRuntimeConfig(spec) {
|
|
2501
|
+
return {
|
|
2502
|
+
...spec.model.costControls?.maxMonthlyUSD !== void 0 && {
|
|
2503
|
+
database: { url: CONTRACT.paths.gatewayDatabase }
|
|
2504
|
+
},
|
|
2505
|
+
// agentgateway samples no request by default and keeps prompt and completion out of the
|
|
2506
|
+
// span; the sidecar needs every LLM call with both (ADR-0030), and redacts before buffering.
|
|
2507
|
+
tracing: {
|
|
2508
|
+
otlpEndpoint: `http://${CONTRACT.podPatch.loopback}:${CONTRACT.ports.controlPlane}`,
|
|
2509
|
+
otlpProtocol: "http",
|
|
2510
|
+
path: CONTRACT.sidecarPaths.traces,
|
|
2511
|
+
randomSampling: "true",
|
|
2512
|
+
fields: { add: { "gen_ai.prompt": "llm.prompt", "gen_ai.completion": "llm.completion" } }
|
|
2513
|
+
}
|
|
2514
|
+
};
|
|
2515
|
+
}
|
|
2516
|
+
function catalogProvider(model) {
|
|
2517
|
+
const { provider } = providerFields(model);
|
|
2518
|
+
return typeof provider === "string" ? CATALOG_PROVIDER_IDS[provider] : model.provider;
|
|
2519
|
+
}
|
|
2520
|
+
function catalogRates(pricing) {
|
|
2521
|
+
return Object.fromEntries(Object.entries(pricing).filter(([, v]) => v !== void 0).map(([k, v]) => [k, String(v)]));
|
|
2522
|
+
}
|
|
2523
|
+
function renderModelCatalog(models) {
|
|
2524
|
+
const providers = {};
|
|
2525
|
+
for (const model of models) {
|
|
2526
|
+
if (!model.pricing) continue;
|
|
2527
|
+
const key = catalogProvider(model);
|
|
2528
|
+
providers[key] ??= { models: {} };
|
|
2529
|
+
providers[key].models[model.id] = { rates: catalogRates(model.pricing) };
|
|
2530
|
+
}
|
|
2531
|
+
return Object.keys(providers).length === 0 ? {} : { modelCatalog: [{ inline: { providers } }] };
|
|
2532
|
+
}
|
|
2533
|
+
function renderGatewayConfig(manifest) {
|
|
2534
|
+
const { spec } = manifest;
|
|
2535
|
+
const models = [spec.model, ...spec.model.fallback ? [spec.model.fallback] : []];
|
|
2536
|
+
const config = {
|
|
2537
|
+
config: { ...renderRuntimeConfig(spec), ...renderModelCatalog(models) },
|
|
2538
|
+
gateways: { [GATEWAY_NAME]: { port: CONTRACT.ports.gateway, bindAddress: CONTRACT.podPatch.loopback } },
|
|
2539
|
+
llm: {
|
|
2540
|
+
gateways: GATEWAY_NAME,
|
|
2541
|
+
models: models.map(renderModel),
|
|
2542
|
+
virtualModels: [renderVirtualModel(models)],
|
|
2543
|
+
policies: {
|
|
2544
|
+
apiKey: renderApiKeyPolicy(manifest),
|
|
2545
|
+
...renderRateLimits(spec)
|
|
2546
|
+
}
|
|
2547
|
+
},
|
|
2548
|
+
...renderRetryPolicy(spec.model)
|
|
2549
|
+
};
|
|
2550
|
+
return yaml.dump(config, { sortKeys: true, lineWidth: -1, noRefs: true });
|
|
2551
|
+
}
|
|
2552
|
+
function describeManifestError(err) {
|
|
2553
|
+
if (!(err instanceof Error)) return String(err);
|
|
2554
|
+
if (!err.message.startsWith("[")) {
|
|
2555
|
+
return err.message.split("\n")[0];
|
|
2556
|
+
}
|
|
2557
|
+
try {
|
|
2558
|
+
const issues = JSON.parse(err.message);
|
|
2559
|
+
return issues.map((i) => `${i.path.join(".") || "root"}: ${i.message}`).join("\n");
|
|
2560
|
+
} catch {
|
|
2561
|
+
return err.message.split("\n")[0];
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
|
|
1395
2565
|
// src/audit/rules/model.rules.ts
|
|
2566
|
+
var isLatestModelId = (id) => id === "latest" || id.endsWith(":latest");
|
|
2567
|
+
function fallbackRetryResult(model) {
|
|
2568
|
+
if (!model.fallback) return { pass: true };
|
|
2569
|
+
const { maxRetries } = model.fallback;
|
|
2570
|
+
const { policies } = renderRetryPolicy(model);
|
|
2571
|
+
const attempts = policies ? policies[0].policy.retry.attempts : 0;
|
|
2572
|
+
const fail = (message, recommendation) => ({
|
|
2573
|
+
pass: false,
|
|
2574
|
+
message,
|
|
2575
|
+
path: "/spec/model/fallback/maxRetries",
|
|
2576
|
+
recommendation,
|
|
2577
|
+
references: []
|
|
2578
|
+
});
|
|
2579
|
+
if (maxRetries === void 0) {
|
|
2580
|
+
return fail(
|
|
2581
|
+
"Fallback model has no maxRetries configured: the gateway renders no retry, so the request that triggers failover fails",
|
|
2582
|
+
"Add maxRetries: 2 to spec.model.fallback"
|
|
2583
|
+
);
|
|
2584
|
+
}
|
|
2585
|
+
if (attempts !== maxRetries) {
|
|
2586
|
+
return fail(
|
|
2587
|
+
`The rendered gateway config sets ${attempts} retry attempts, but spec.model.fallback declares maxRetries ${maxRetries}`,
|
|
2588
|
+
"Redeploy the agent so its gateway config is rendered again from agent.yaml and retries maxRetries times"
|
|
2589
|
+
);
|
|
2590
|
+
}
|
|
2591
|
+
return { pass: true };
|
|
2592
|
+
}
|
|
1396
2593
|
var modelRules = [
|
|
1397
2594
|
{
|
|
1398
2595
|
id: "MODEL-01",
|
|
@@ -1400,9 +2597,7 @@ var modelRules = [
|
|
|
1400
2597
|
title: "Model fallback declared",
|
|
1401
2598
|
description: "A fallback model should be configured to handle rate limits and errors",
|
|
1402
2599
|
severity: "high",
|
|
1403
|
-
|
|
1404
|
-
proofTool: "LiteLLM chaos test",
|
|
1405
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
|
|
2600
|
+
awaitsEvidence: { code: "no_failover_observed", phase: 2 },
|
|
1406
2601
|
check(manifest) {
|
|
1407
2602
|
const hasFallback = !!manifest.spec.model.fallback;
|
|
1408
2603
|
return {
|
|
@@ -1420,17 +2615,14 @@ var modelRules = [
|
|
|
1420
2615
|
title: "Model version pinned",
|
|
1421
2616
|
description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
|
|
1422
2617
|
severity: "medium",
|
|
1423
|
-
evidenceLevel: "probed",
|
|
1424
|
-
proofTool: "agentspec health",
|
|
1425
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1426
2618
|
check(manifest) {
|
|
1427
2619
|
const id = manifest.spec.model.id;
|
|
1428
|
-
const isLatest = id
|
|
2620
|
+
const isLatest = isLatestModelId(id);
|
|
1429
2621
|
return {
|
|
1430
2622
|
pass: !isLatest,
|
|
1431
2623
|
message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
|
|
1432
2624
|
path: "/spec/model/id",
|
|
1433
|
-
recommendation: 'Pin a specific model version, e.g. "
|
|
2625
|
+
recommendation: 'Pin a specific model version, e.g. "openai/gpt-oss-20b" or "gpt-4-0125-preview"',
|
|
1434
2626
|
references: []
|
|
1435
2627
|
};
|
|
1436
2628
|
}
|
|
@@ -1441,9 +2633,7 @@ var modelRules = [
|
|
|
1441
2633
|
title: "Cost controls declared",
|
|
1442
2634
|
description: "Monthly cost limits prevent unexpected API bills",
|
|
1443
2635
|
severity: "medium",
|
|
1444
|
-
|
|
1445
|
-
proofTool: "LiteLLM Spend Tracking",
|
|
1446
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
|
|
2636
|
+
awaitsEvidence: { code: "no_budget_evidence", phase: 2 },
|
|
1447
2637
|
check(manifest) {
|
|
1448
2638
|
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1449
2639
|
return {
|
|
@@ -1459,19 +2649,61 @@ var modelRules = [
|
|
|
1459
2649
|
id: "MODEL-04",
|
|
1460
2650
|
pack: "model-resilience",
|
|
1461
2651
|
title: "Fallback retry strategy configured",
|
|
1462
|
-
description: "Fallback should have maxRetries
|
|
2652
|
+
description: "Fallback should have maxRetries, and the rendered gateway config must retry exactly that many times",
|
|
1463
2653
|
severity: "low",
|
|
1464
|
-
evidenceLevel: "external",
|
|
1465
|
-
proofTool: "pytest-mockllm",
|
|
1466
|
-
proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
|
|
1467
2654
|
check(manifest) {
|
|
1468
|
-
|
|
1469
|
-
|
|
2655
|
+
return fallbackRetryResult(manifest.spec.model);
|
|
2656
|
+
}
|
|
2657
|
+
},
|
|
2658
|
+
{
|
|
2659
|
+
id: "MODEL-05",
|
|
2660
|
+
pack: "model-resilience",
|
|
2661
|
+
title: "Budgeted model is priced",
|
|
2662
|
+
description: "A monthly USD budget is only enforced for models the gateway can price",
|
|
2663
|
+
severity: "high",
|
|
2664
|
+
// Only the gateway knows its prices: MODEL-05 is graded from the cost on its spans (#494), never guessed from the model id (#495).
|
|
2665
|
+
check() {
|
|
2666
|
+
return { pass: true };
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
];
|
|
2670
|
+
|
|
2671
|
+
// src/audit/rules/metadata.rules.ts
|
|
2672
|
+
var MAX_PRODUCTION_TEMPERATURE = 1.5;
|
|
2673
|
+
var metadataRules = [
|
|
2674
|
+
{
|
|
2675
|
+
id: "META-01",
|
|
2676
|
+
pack: "metadata-quality",
|
|
2677
|
+
title: "Agent description declared",
|
|
2678
|
+
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
2679
|
+
severity: "low",
|
|
2680
|
+
check(manifest) {
|
|
2681
|
+
const desc = manifest.metadata.description;
|
|
2682
|
+
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
2683
|
+
return {
|
|
2684
|
+
pass,
|
|
2685
|
+
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
2686
|
+
path: "/metadata/description",
|
|
2687
|
+
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
2688
|
+
references: []
|
|
2689
|
+
};
|
|
2690
|
+
}
|
|
2691
|
+
},
|
|
2692
|
+
{
|
|
2693
|
+
id: "META-02",
|
|
2694
|
+
pack: "metadata-quality",
|
|
2695
|
+
title: "Model temperature is production-safe",
|
|
2696
|
+
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
2697
|
+
severity: "medium",
|
|
2698
|
+
check(manifest) {
|
|
2699
|
+
const temperature = manifest.spec.model.parameters?.temperature;
|
|
2700
|
+
if (temperature === void 0) return { pass: true };
|
|
2701
|
+
const pass = temperature <= MAX_PRODUCTION_TEMPERATURE;
|
|
1470
2702
|
return {
|
|
1471
|
-
pass
|
|
1472
|
-
message:
|
|
1473
|
-
path: "/spec/model/
|
|
1474
|
-
recommendation: "
|
|
2703
|
+
pass,
|
|
2704
|
+
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above ${MAX_PRODUCTION_TEMPERATURE} produce unreliable outputs in production.`,
|
|
2705
|
+
path: "/spec/model/parameters/temperature",
|
|
2706
|
+
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
1475
2707
|
references: []
|
|
1476
2708
|
};
|
|
1477
2709
|
}
|
|
@@ -1486,9 +2718,6 @@ var securityRules = [
|
|
|
1486
2718
|
title: "Prompt injection guard: input guardrail required",
|
|
1487
2719
|
description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
|
|
1488
2720
|
severity: "high",
|
|
1489
|
-
evidenceLevel: "behavioral",
|
|
1490
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1491
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1492
2721
|
check(manifest) {
|
|
1493
2722
|
const hasGuardrail = manifest.spec.guardrails?.input?.some(
|
|
1494
2723
|
(g) => g.type === "prompt-injection" || g.type === "topic-filter"
|
|
@@ -1510,9 +2739,6 @@ var securityRules = [
|
|
|
1510
2739
|
title: "Insecure output handling: output guardrail required",
|
|
1511
2740
|
description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
|
|
1512
2741
|
severity: "high",
|
|
1513
|
-
evidenceLevel: "behavioral",
|
|
1514
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1515
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1516
2742
|
check(manifest) {
|
|
1517
2743
|
const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
|
|
1518
2744
|
return {
|
|
@@ -1532,9 +2758,6 @@ var securityRules = [
|
|
|
1532
2758
|
title: "Training/prompt data poisoning: prompt files referenced",
|
|
1533
2759
|
description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
|
|
1534
2760
|
severity: "medium",
|
|
1535
|
-
evidenceLevel: "probed",
|
|
1536
|
-
proofTool: "agentspec health",
|
|
1537
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1538
2761
|
check(manifest) {
|
|
1539
2762
|
const system = manifest.spec.prompts.system;
|
|
1540
2763
|
const usesFile = system.startsWith("$file:");
|
|
@@ -1552,21 +2775,16 @@ var securityRules = [
|
|
|
1552
2775
|
{
|
|
1553
2776
|
id: "SEC-LLM-04",
|
|
1554
2777
|
pack: "owasp-llm-top10",
|
|
1555
|
-
title: "Model DoS: rate limiting
|
|
2778
|
+
title: "Model DoS: rate limiting configured",
|
|
1556
2779
|
description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
|
|
1557
2780
|
severity: "medium",
|
|
1558
|
-
evidenceLevel: "external",
|
|
1559
|
-
proofTool: "k6 load test",
|
|
1560
|
-
proofToolUrl: "https://k6.io",
|
|
1561
2781
|
check(manifest) {
|
|
1562
|
-
const
|
|
1563
|
-
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1564
|
-
const pass = hasRateLimit || hasCostControls;
|
|
2782
|
+
const pass = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
|
|
1565
2783
|
return {
|
|
1566
2784
|
pass,
|
|
1567
|
-
message: pass ? void 0 : "No rate limiting
|
|
2785
|
+
message: pass ? void 0 : "No rate limiting declared.",
|
|
1568
2786
|
path: "/spec/api/rateLimit",
|
|
1569
|
-
recommendation: "Add spec.api.rateLimit.requestsPerMinute
|
|
2787
|
+
recommendation: "Add spec.api.rateLimit.requestsPerMinute",
|
|
1570
2788
|
references: [
|
|
1571
2789
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
|
|
1572
2790
|
]
|
|
@@ -1579,9 +2797,6 @@ var securityRules = [
|
|
|
1579
2797
|
title: "Supply chain: model provider and version pinned",
|
|
1580
2798
|
description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
|
|
1581
2799
|
severity: "medium",
|
|
1582
|
-
evidenceLevel: "probed",
|
|
1583
|
-
proofTool: "agentspec health",
|
|
1584
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1585
2800
|
check(manifest) {
|
|
1586
2801
|
const hasProvider = !!manifest.spec.model.provider;
|
|
1587
2802
|
const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
|
|
@@ -1603,9 +2818,6 @@ var securityRules = [
|
|
|
1603
2818
|
title: "Sensitive data disclosure: PII scrub in memory hygiene",
|
|
1604
2819
|
description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
|
|
1605
2820
|
severity: "critical",
|
|
1606
|
-
evidenceLevel: "external",
|
|
1607
|
-
proofTool: "Microsoft Presidio",
|
|
1608
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1609
2821
|
check(manifest) {
|
|
1610
2822
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1611
2823
|
const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1626,9 +2838,6 @@ var securityRules = [
|
|
|
1626
2838
|
title: "Insecure plugin design: tool annotations declared",
|
|
1627
2839
|
description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
|
|
1628
2840
|
severity: "medium",
|
|
1629
|
-
evidenceLevel: "external",
|
|
1630
|
-
proofTool: "Promptfoo",
|
|
1631
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
1632
2841
|
check(manifest) {
|
|
1633
2842
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1634
2843
|
const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
|
|
@@ -1649,9 +2858,7 @@ var securityRules = [
|
|
|
1649
2858
|
title: "Excessive agency: destructiveHint declared on tools",
|
|
1650
2859
|
description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
|
|
1651
2860
|
severity: "high",
|
|
1652
|
-
|
|
1653
|
-
proofTool: "Promptfoo",
|
|
1654
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
2861
|
+
awaitsEvidence: { code: "no_approval_token_evidence", phase: 5 },
|
|
1655
2862
|
check(manifest) {
|
|
1656
2863
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1657
2864
|
const allHaveDestructiveHint = manifest.spec.tools.every(
|
|
@@ -1671,21 +2878,18 @@ var securityRules = [
|
|
|
1671
2878
|
{
|
|
1672
2879
|
id: "SEC-LLM-09",
|
|
1673
2880
|
pack: "owasp-llm-top10",
|
|
1674
|
-
title: "Overreliance: evaluation
|
|
2881
|
+
title: "Overreliance: evaluation + CI gate configured",
|
|
1675
2882
|
description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
|
|
1676
2883
|
severity: "medium",
|
|
1677
|
-
evidenceLevel: "probed",
|
|
1678
|
-
proofTool: "agentspec health",
|
|
1679
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1680
2884
|
check(manifest) {
|
|
1681
|
-
const hasEval =
|
|
2885
|
+
const hasEval = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0 || (manifest.spec.evaluation?.live?.metrics?.length ?? 0) > 0;
|
|
1682
2886
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1683
2887
|
const pass = hasEval && hasCiGate;
|
|
1684
2888
|
return {
|
|
1685
2889
|
pass,
|
|
1686
|
-
message: pass ? void 0 : !hasEval ? "No evaluation
|
|
2890
|
+
message: pass ? void 0 : !hasEval ? "No evaluation datasets or live evaluation configured." : "Evaluation CI gate is not enabled.",
|
|
1687
2891
|
path: "/spec/evaluation",
|
|
1688
|
-
recommendation: "Configure spec.evaluation.
|
|
2892
|
+
recommendation: "Configure spec.evaluation.datasets (or spec.evaluation.live) and set spec.evaluation.ciGate: true",
|
|
1689
2893
|
references: [
|
|
1690
2894
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
|
|
1691
2895
|
]
|
|
@@ -1698,9 +2902,6 @@ var securityRules = [
|
|
|
1698
2902
|
title: "Model theft: API keys use $secret, not $env",
|
|
1699
2903
|
description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
|
|
1700
2904
|
severity: "high",
|
|
1701
|
-
evidenceLevel: "probed",
|
|
1702
|
-
proofTool: "agentspec health",
|
|
1703
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1704
2905
|
check(manifest) {
|
|
1705
2906
|
const apiKey = manifest.spec.model.apiKey;
|
|
1706
2907
|
const usesSecret = apiKey.startsWith("$secret:");
|
|
@@ -1716,6 +2917,18 @@ var securityRules = [
|
|
|
1716
2917
|
]
|
|
1717
2918
|
};
|
|
1718
2919
|
}
|
|
2920
|
+
},
|
|
2921
|
+
{
|
|
2922
|
+
id: "TOOL-01",
|
|
2923
|
+
pack: "owasp-llm-top10",
|
|
2924
|
+
title: "Only declared tools are called",
|
|
2925
|
+
description: "The agent must call only tools declared in spec.tools; undeclared calls are excessive agency",
|
|
2926
|
+
severity: "high",
|
|
2927
|
+
awaitsEvidence: { code: "no_calls_observed", phase: 2 },
|
|
2928
|
+
// The manifest alone cannot show an undeclared call: passes declaratively, verified only from observed spans (#152, D17).
|
|
2929
|
+
check() {
|
|
2930
|
+
return { pass: true };
|
|
2931
|
+
}
|
|
1719
2932
|
}
|
|
1720
2933
|
];
|
|
1721
2934
|
|
|
@@ -1727,9 +2940,6 @@ var memoryRules = [
|
|
|
1727
2940
|
title: "PII scrub fields declared for long-term memory",
|
|
1728
2941
|
description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
|
|
1729
2942
|
severity: "critical",
|
|
1730
|
-
evidenceLevel: "external",
|
|
1731
|
-
proofTool: "Microsoft Presidio",
|
|
1732
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1733
2943
|
check(manifest) {
|
|
1734
2944
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1735
2945
|
const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1748,9 +2958,6 @@ var memoryRules = [
|
|
|
1748
2958
|
title: "Retention policy / TTL set for all memory backends",
|
|
1749
2959
|
description: "Memory without TTL grows unbounded and increases PII exposure risk",
|
|
1750
2960
|
severity: "high",
|
|
1751
|
-
evidenceLevel: "probed",
|
|
1752
|
-
proofTool: "agentspec health",
|
|
1753
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1754
2961
|
check(manifest) {
|
|
1755
2962
|
const mem = manifest.spec.memory;
|
|
1756
2963
|
if (!mem) return { pass: true };
|
|
@@ -1775,9 +2982,6 @@ var memoryRules = [
|
|
|
1775
2982
|
title: "Audit log enabled for long-term memory",
|
|
1776
2983
|
description: "Audit logging enables compliance investigations and debugging",
|
|
1777
2984
|
severity: "medium",
|
|
1778
|
-
evidenceLevel: "probed",
|
|
1779
|
-
proofTool: "agentspec health",
|
|
1780
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1781
2985
|
check(manifest) {
|
|
1782
2986
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1783
2987
|
const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
|
|
@@ -1796,9 +3000,6 @@ var memoryRules = [
|
|
|
1796
3000
|
title: "Vector store namespace isolated",
|
|
1797
3001
|
description: "Shared vector store namespaces risk data leakage between agents",
|
|
1798
3002
|
severity: "medium",
|
|
1799
|
-
evidenceLevel: "probed",
|
|
1800
|
-
proofTool: "agentspec health",
|
|
1801
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1802
3003
|
check(manifest) {
|
|
1803
3004
|
if (!manifest.spec.memory?.vector) return { pass: true };
|
|
1804
3005
|
const hasNamespace = !!manifest.spec.memory.vector.namespace;
|
|
@@ -1817,9 +3018,6 @@ var memoryRules = [
|
|
|
1817
3018
|
title: "Short-term memory max tokens bounded",
|
|
1818
3019
|
description: "Unbounded short-term memory can cause token budget overruns",
|
|
1819
3020
|
severity: "low",
|
|
1820
|
-
evidenceLevel: "probed",
|
|
1821
|
-
proofTool: "agentspec health",
|
|
1822
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1823
3021
|
check(manifest) {
|
|
1824
3022
|
if (!manifest.spec.memory?.shortTerm) return { pass: true };
|
|
1825
3023
|
const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
|
|
@@ -1842,9 +3040,6 @@ var evaluationRules = [
|
|
|
1842
3040
|
title: "Evaluation dataset declared",
|
|
1843
3041
|
description: "At least one evaluation dataset is required for quality assurance",
|
|
1844
3042
|
severity: "medium",
|
|
1845
|
-
evidenceLevel: "probed",
|
|
1846
|
-
proofTool: "agentspec health",
|
|
1847
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1848
3043
|
check(manifest) {
|
|
1849
3044
|
const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
|
|
1850
3045
|
return {
|
|
@@ -1862,9 +3057,6 @@ var evaluationRules = [
|
|
|
1862
3057
|
title: "CI gate enabled",
|
|
1863
3058
|
description: "Evaluation CI gate prevents regressions from reaching production",
|
|
1864
3059
|
severity: "medium",
|
|
1865
|
-
evidenceLevel: "probed",
|
|
1866
|
-
proofTool: "agentspec health",
|
|
1867
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1868
3060
|
check(manifest) {
|
|
1869
3061
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1870
3062
|
return {
|
|
@@ -1879,22 +3071,22 @@ var evaluationRules = [
|
|
|
1879
3071
|
{
|
|
1880
3072
|
id: "EVAL-03",
|
|
1881
3073
|
pack: "evaluation-coverage",
|
|
1882
|
-
title: "
|
|
1883
|
-
description: "A
|
|
3074
|
+
title: "Grounded metric threshold configured",
|
|
3075
|
+
description: "A grounded threshold (minimum pass rate, 0.95 recommended) is required for production agents",
|
|
1884
3076
|
severity: "medium",
|
|
1885
|
-
evidenceLevel: "probed",
|
|
1886
|
-
proofTool: "agentspec health",
|
|
1887
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1888
3077
|
check(manifest) {
|
|
1889
|
-
const
|
|
1890
|
-
const
|
|
1891
|
-
const
|
|
1892
|
-
const
|
|
3078
|
+
const evaluation = manifest.spec.evaluation;
|
|
3079
|
+
const scopes = [evaluation, evaluation?.live];
|
|
3080
|
+
const hasGrounded = (scope) => !!scope?.metrics?.map(canonicalMetric).includes("grounded");
|
|
3081
|
+
const hasGroundedMetric = scopes.some(hasGrounded);
|
|
3082
|
+
const pass = scopes.some(
|
|
3083
|
+
(scope) => hasGrounded(scope) && Object.keys(scope?.thresholds ?? {}).some((n) => canonicalMetric(n) === "grounded")
|
|
3084
|
+
);
|
|
1893
3085
|
return {
|
|
1894
3086
|
pass,
|
|
1895
|
-
message: pass ? void 0 : !
|
|
1896
|
-
path: "/spec/evaluation/thresholds/
|
|
1897
|
-
recommendation: "Add
|
|
3087
|
+
message: pass ? void 0 : !hasGroundedMetric ? "grounded metric not in spec.evaluation.metrics or spec.evaluation.live.metrics" : "No grounded threshold next to the grounded metric",
|
|
3088
|
+
path: "/spec/evaluation/thresholds/grounded",
|
|
3089
|
+
recommendation: "Add grounded to spec.evaluation.metrics (or spec.evaluation.live.metrics) and set the matching thresholds.grounded: 0.95",
|
|
1898
3090
|
references: []
|
|
1899
3091
|
};
|
|
1900
3092
|
}
|
|
@@ -1909,9 +3101,6 @@ var observabilityRules = [
|
|
|
1909
3101
|
title: "Tracing backend declared",
|
|
1910
3102
|
description: "Agents should declare a tracing backend so runs are observable and debuggable",
|
|
1911
3103
|
severity: "medium",
|
|
1912
|
-
evidenceLevel: "probed",
|
|
1913
|
-
proofTool: "agentspec health",
|
|
1914
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1915
3104
|
check(manifest) {
|
|
1916
3105
|
const pass = !!manifest.spec.observability?.tracing?.backend;
|
|
1917
3106
|
return {
|
|
@@ -1928,9 +3117,6 @@ var observabilityRules = [
|
|
|
1928
3117
|
title: "Structured logging enabled",
|
|
1929
3118
|
description: "Structured logging makes log aggregation and searching reliable",
|
|
1930
3119
|
severity: "low",
|
|
1931
|
-
evidenceLevel: "behavioral",
|
|
1932
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1933
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1934
3120
|
check(manifest) {
|
|
1935
3121
|
const pass = manifest.spec.observability?.logging?.structured !== false;
|
|
1936
3122
|
return {
|
|
@@ -1947,9 +3133,6 @@ var observabilityRules = [
|
|
|
1947
3133
|
title: "Sensitive fields redacted from logs",
|
|
1948
3134
|
description: "Log redaction prevents secrets and PII from appearing in log aggregators",
|
|
1949
3135
|
severity: "medium",
|
|
1950
|
-
evidenceLevel: "external",
|
|
1951
|
-
proofTool: "Microsoft Presidio",
|
|
1952
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1953
3136
|
check(manifest) {
|
|
1954
3137
|
const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
|
|
1955
3138
|
return {
|
|
@@ -1962,57 +3145,8 @@ var observabilityRules = [
|
|
|
1962
3145
|
}
|
|
1963
3146
|
];
|
|
1964
3147
|
|
|
1965
|
-
// src/audit/rules/metadata.rules.ts
|
|
1966
|
-
var metadataRules = [
|
|
1967
|
-
{
|
|
1968
|
-
id: "META-01",
|
|
1969
|
-
pack: "metadata-quality",
|
|
1970
|
-
title: "Agent description declared",
|
|
1971
|
-
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
1972
|
-
severity: "low",
|
|
1973
|
-
evidenceLevel: "declarative",
|
|
1974
|
-
check(manifest) {
|
|
1975
|
-
const desc = manifest.metadata.description;
|
|
1976
|
-
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
1977
|
-
return {
|
|
1978
|
-
pass,
|
|
1979
|
-
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
1980
|
-
path: "/metadata/description",
|
|
1981
|
-
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
1982
|
-
references: []
|
|
1983
|
-
};
|
|
1984
|
-
}
|
|
1985
|
-
},
|
|
1986
|
-
{
|
|
1987
|
-
id: "META-02",
|
|
1988
|
-
pack: "metadata-quality",
|
|
1989
|
-
title: "Model temperature is production-safe",
|
|
1990
|
-
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
1991
|
-
severity: "medium",
|
|
1992
|
-
evidenceLevel: "declarative",
|
|
1993
|
-
check(manifest) {
|
|
1994
|
-
const temperature = manifest.spec.model.parameters?.temperature;
|
|
1995
|
-
if (temperature === void 0) return { pass: true };
|
|
1996
|
-
const pass = temperature <= 1.5;
|
|
1997
|
-
return {
|
|
1998
|
-
pass,
|
|
1999
|
-
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above 1.5 produce unreliable outputs in production.`,
|
|
2000
|
-
path: "/spec/model/parameters/temperature",
|
|
2001
|
-
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
2002
|
-
references: []
|
|
2003
|
-
};
|
|
2004
|
-
}
|
|
2005
|
-
}
|
|
2006
|
-
];
|
|
2007
|
-
|
|
2008
3148
|
// src/audit/index.ts
|
|
2009
|
-
var SEVERITY_WEIGHTS =
|
|
2010
|
-
critical: 4,
|
|
2011
|
-
high: 3,
|
|
2012
|
-
medium: 2,
|
|
2013
|
-
low: 1,
|
|
2014
|
-
info: 0
|
|
2015
|
-
};
|
|
3149
|
+
var SEVERITY_WEIGHTS = CONTRACT.severityWeights;
|
|
2016
3150
|
var ALL_RULES = [
|
|
2017
3151
|
...modelRules,
|
|
2018
3152
|
...securityRules,
|
|
@@ -2022,19 +3156,10 @@ var ALL_RULES = [
|
|
|
2022
3156
|
...metadataRules
|
|
2023
3157
|
];
|
|
2024
3158
|
var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
|
|
2025
|
-
|
|
2026
|
-
return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
|
|
2027
|
-
}
|
|
2028
|
-
function scoreToGrade(score) {
|
|
2029
|
-
if (score >= 90) return "A";
|
|
2030
|
-
if (score >= 75) return "B";
|
|
2031
|
-
if (score >= 60) return "C";
|
|
2032
|
-
if (score >= 45) return "D";
|
|
2033
|
-
return "F";
|
|
2034
|
-
}
|
|
3159
|
+
var COMPLIANCE_PACKS = [...new Set(ALL_RULES.map((r) => r.pack))];
|
|
2035
3160
|
function resolveActiveRules(manifest, opts) {
|
|
2036
3161
|
const manifestPacks = manifest.spec.compliance?.packs ?? [];
|
|
2037
|
-
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks :
|
|
3162
|
+
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : COMPLIANCE_PACKS);
|
|
2038
3163
|
return ALL_RULES.filter((r) => activePacks.includes(r.pack));
|
|
2039
3164
|
}
|
|
2040
3165
|
function collectSuppressions(manifest) {
|
|
@@ -2063,10 +3188,10 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2063
3188
|
const violations = [];
|
|
2064
3189
|
const packBreakdown = {};
|
|
2065
3190
|
const evidenceBreakdown = {
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
3191
|
+
independent: { passed: 0, total: 0 },
|
|
3192
|
+
"evidence-point": { passed: 0, total: 0 },
|
|
3193
|
+
offline: { passed: 0, total: 0 },
|
|
3194
|
+
declared: { passed: 0, total: 0 }
|
|
2070
3195
|
};
|
|
2071
3196
|
let totalWeight = 0;
|
|
2072
3197
|
let passedWeight = 0;
|
|
@@ -2077,19 +3202,20 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2077
3202
|
const suppressed = suppressedIds.has(rule.id);
|
|
2078
3203
|
const weight = SEVERITY_WEIGHTS[rule.severity];
|
|
2079
3204
|
const result = ruleResults.get(rule.id);
|
|
2080
|
-
const
|
|
3205
|
+
const classes = ruleClasses(rule.id);
|
|
2081
3206
|
packBreakdown[rule.pack].total += 1;
|
|
2082
|
-
|
|
3207
|
+
tally(evidenceBreakdown, classes, "total");
|
|
2083
3208
|
if (suppressed) {
|
|
2084
3209
|
packBreakdown[rule.pack].passed += 1;
|
|
2085
|
-
|
|
3210
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2086
3211
|
continue;
|
|
2087
3212
|
}
|
|
2088
3213
|
if (result.pass) {
|
|
2089
3214
|
packBreakdown[rule.pack].passed += 1;
|
|
2090
|
-
|
|
3215
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2091
3216
|
passedWeight += weight;
|
|
2092
3217
|
} else {
|
|
3218
|
+
const hint = proofHint(rule.id);
|
|
2093
3219
|
violations.push({
|
|
2094
3220
|
ruleId: rule.id,
|
|
2095
3221
|
severity: rule.severity,
|
|
@@ -2098,9 +3224,9 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2098
3224
|
path: result.path,
|
|
2099
3225
|
recommendation: result.recommendation,
|
|
2100
3226
|
references: result.references,
|
|
2101
|
-
|
|
2102
|
-
proofTool:
|
|
2103
|
-
proofToolUrl:
|
|
3227
|
+
evidenceClasses: classes,
|
|
3228
|
+
proofTool: hint?.tool,
|
|
3229
|
+
proofToolUrl: hint?.url
|
|
2104
3230
|
});
|
|
2105
3231
|
}
|
|
2106
3232
|
totalWeight += weight;
|
|
@@ -2112,24 +3238,36 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2112
3238
|
}
|
|
2113
3239
|
return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
|
|
2114
3240
|
}
|
|
2115
|
-
function
|
|
2116
|
-
const
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
const
|
|
2126
|
-
if (
|
|
3241
|
+
function tally(breakdown, classes, field) {
|
|
3242
|
+
for (const c of classes) breakdown[c][field] += 1;
|
|
3243
|
+
}
|
|
3244
|
+
var SIDECAR_GAP = { tool: "AgentSpec sidecar gap analysis", url: "https://agentspec.io/docs/concepts/probe-coverage" };
|
|
3245
|
+
var COLLECTOR_HINTS = {
|
|
3246
|
+
"sidecar-probe": SIDECAR_GAP,
|
|
3247
|
+
"sidecar-forwarder": SIDECAR_GAP
|
|
3248
|
+
};
|
|
3249
|
+
function proofHint(ruleId) {
|
|
3250
|
+
for (const { collector, status } of ruleSources(ruleId)) {
|
|
3251
|
+
const hint = COLLECTOR_HINTS[collector];
|
|
3252
|
+
if (status === "built" && hint) return { collector, ...hint };
|
|
2127
3253
|
}
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
).
|
|
2132
|
-
return {
|
|
3254
|
+
return void 0;
|
|
3255
|
+
}
|
|
3256
|
+
function explainPendingEvidence(verified, rules) {
|
|
3257
|
+
const pending = new Map(rules.filter((r) => r.awaitsEvidence).map((r) => [r.id, r.awaitsEvidence]));
|
|
3258
|
+
return {
|
|
3259
|
+
...verified,
|
|
3260
|
+
unverified: verified.unverified.map((u) => {
|
|
3261
|
+
const awaits = pending.get(u.ruleId);
|
|
3262
|
+
if (!awaits || u.reason.code !== "no_evidence") return u;
|
|
3263
|
+
return { ...u, reason: { code: awaits.code, detail: `evidence source arrives in phase ${awaits.phase}` } };
|
|
3264
|
+
})
|
|
3265
|
+
};
|
|
3266
|
+
}
|
|
3267
|
+
function summariseRuleEvidence(rules) {
|
|
3268
|
+
return Object.fromEntries(
|
|
3269
|
+
rules.map((r) => [r.id, { classes: ruleClasses(r.id), notProven: RULE_EVIDENCE[r.id]?.notProven ?? null }])
|
|
3270
|
+
);
|
|
2133
3271
|
}
|
|
2134
3272
|
function runAudit(manifest, opts = {}) {
|
|
2135
3273
|
const rules = resolveActiveRules(manifest, opts);
|
|
@@ -2139,11 +3277,21 @@ function runAudit(manifest, opts = {}) {
|
|
|
2139
3277
|
violations,
|
|
2140
3278
|
packBreakdown,
|
|
2141
3279
|
evidenceBreakdown,
|
|
2142
|
-
totalWeight,
|
|
2143
3280
|
overallScore,
|
|
2144
3281
|
categoryScores
|
|
2145
3282
|
} = computeScoring(rules, ruleResults, suppressedIds);
|
|
2146
|
-
const
|
|
3283
|
+
const applicable = rules.filter((r) => !suppressedIds.has(r.id));
|
|
3284
|
+
const applicableRules = applicable.map((r) => ({ id: r.id, severity: r.severity }));
|
|
3285
|
+
const verified = explainPendingEvidence(
|
|
3286
|
+
grade(
|
|
3287
|
+
opts.records ?? [],
|
|
3288
|
+
applicableRules,
|
|
3289
|
+
opts.trust ?? "standard",
|
|
3290
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
3291
|
+
opts.evidencePoints
|
|
3292
|
+
),
|
|
3293
|
+
applicable
|
|
3294
|
+
);
|
|
2147
3295
|
return {
|
|
2148
3296
|
agentName: manifest.metadata.name,
|
|
2149
3297
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2156,9 +3304,26 @@ function runAudit(manifest, opts = {}) {
|
|
|
2156
3304
|
totalRules: rules.length,
|
|
2157
3305
|
packBreakdown,
|
|
2158
3306
|
evidenceBreakdown,
|
|
2159
|
-
|
|
3307
|
+
verified,
|
|
3308
|
+
applicableRules,
|
|
3309
|
+
ruleEvidence: summariseRuleEvidence(rules)
|
|
2160
3310
|
};
|
|
2161
3311
|
}
|
|
3312
|
+
function canonicalJson(value) {
|
|
3313
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
3314
|
+
if (value !== null && typeof value === "object") {
|
|
3315
|
+
const obj = value;
|
|
3316
|
+
const members = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
|
|
3317
|
+
return `{${members.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
|
|
3318
|
+
}
|
|
3319
|
+
return JSON.stringify(value);
|
|
3320
|
+
}
|
|
3321
|
+
function canonicalManifestSha256(manifest) {
|
|
3322
|
+
return crypto.createHash("sha256").update(canonicalJson(manifest), "utf8").digest("hex");
|
|
3323
|
+
}
|
|
3324
|
+
function declaredManifestSha256(raw) {
|
|
3325
|
+
return canonicalManifestSha256(yaml__namespace.load(raw, { schema: yaml__namespace.JSON_SCHEMA }));
|
|
3326
|
+
}
|
|
2162
3327
|
|
|
2163
3328
|
// src/generate/index.ts
|
|
2164
3329
|
var _registry = /* @__PURE__ */ new Map();
|
|
@@ -2171,7 +3336,7 @@ function getAdapter(framework) {
|
|
|
2171
3336
|
function listAdapters() {
|
|
2172
3337
|
return [..._registry.keys()];
|
|
2173
3338
|
}
|
|
2174
|
-
function generateAdapter(manifest, framework,
|
|
3339
|
+
function generateAdapter(manifest, framework, options2) {
|
|
2175
3340
|
const adapter = _registry.get(framework.toLowerCase());
|
|
2176
3341
|
if (!adapter) {
|
|
2177
3342
|
throw new Error(
|
|
@@ -2180,7 +3345,7 @@ function generateAdapter(manifest, framework, options) {
|
|
|
2180
3345
|
Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
|
|
2181
3346
|
);
|
|
2182
3347
|
}
|
|
2183
|
-
return adapter.generate(manifest,
|
|
3348
|
+
return adapter.generate(manifest, options2);
|
|
2184
3349
|
}
|
|
2185
3350
|
|
|
2186
3351
|
// src/agent/usage-ledger.ts
|
|
@@ -2226,8 +3391,8 @@ var UsageLedger = class {
|
|
|
2226
3391
|
* Return a frozen snapshot of the current usage window.
|
|
2227
3392
|
*
|
|
2228
3393
|
* @param reset - When true (default), clears all counters and starts a new
|
|
2229
|
-
* window. Use reset=
|
|
2230
|
-
*
|
|
3394
|
+
* window. Use reset=false for inspection without draining; the heartbeat
|
|
3395
|
+
* push takes a reset=false snapshot and calls subtract() with it.
|
|
2231
3396
|
*/
|
|
2232
3397
|
snapshot(reset = true) {
|
|
2233
3398
|
const snap = buildSnapshot(this.counters, this.windowStart);
|
|
@@ -2237,39 +3402,220 @@ var UsageLedger = class {
|
|
|
2237
3402
|
}
|
|
2238
3403
|
return snap;
|
|
2239
3404
|
}
|
|
3405
|
+
/**
|
|
3406
|
+
* Remove a shipped snapshot from the ledger; a new window starts, because the control plane dedupes heartbeats on windowStartedAt.
|
|
3407
|
+
* Usage recorded after the snapshot was taken stays, so every token is
|
|
3408
|
+
* shipped exactly once even when a call finishes while a heartbeat is in flight.
|
|
3409
|
+
*/
|
|
3410
|
+
subtract(shipped) {
|
|
3411
|
+
for (const s of shipped.models) {
|
|
3412
|
+
const entry = this.counters.get(s.modelId);
|
|
3413
|
+
if (!entry) continue;
|
|
3414
|
+
entry.promptTokens -= s.promptTokens;
|
|
3415
|
+
entry.completionTokens -= s.completionTokens;
|
|
3416
|
+
entry.totalTokens -= s.totalTokens;
|
|
3417
|
+
entry.callCount -= s.callCount;
|
|
3418
|
+
if (entry.callCount <= 0) this.counters.delete(s.modelId);
|
|
3419
|
+
}
|
|
3420
|
+
this.windowStart = /* @__PURE__ */ new Date();
|
|
3421
|
+
}
|
|
2240
3422
|
};
|
|
2241
3423
|
|
|
3424
|
+
// src/runtime/evidence-points.ts
|
|
3425
|
+
var GUARDRAIL = EvidencePoint.enum.guardrail;
|
|
3426
|
+
var [MCP_LIST, MCP_CALL] = Object.keys(CONTRACT.evidencePoints.mcpClient);
|
|
3427
|
+
var [SCRUB, TTL, AUDIT] = Object.keys(CONTRACT.evidencePoints.memoryWrite);
|
|
3428
|
+
var [NAMESPACE] = Object.keys(CONTRACT.evidencePoints.vectorStore);
|
|
3429
|
+
var [TOOL_TARGET] = Object.keys(CONTRACT.evidencePoints.tool);
|
|
3430
|
+
var [LOGGING_TARGET] = Object.keys(CONTRACT.evidencePoints.logging);
|
|
3431
|
+
var keyOf = (point2, target) => `${point2}\0${target}`;
|
|
3432
|
+
var DEFAULT_KEY = /* @__PURE__ */ Symbol.for("agentspec.runtime.defaultEvidencePoints");
|
|
3433
|
+
var defaults = globalThis;
|
|
3434
|
+
var rec = (value) => value && typeof value === "object" ? value : {};
|
|
3435
|
+
var stdioServers = (spec) => (Array.isArray(rec(spec.mcp).servers) ? rec(spec.mcp).servers : []).filter((s) => s?.transport === "stdio");
|
|
3436
|
+
var isDeclared = (value) => Array.isArray(value) ? value.length > 0 : Boolean(value);
|
|
3437
|
+
function memoryWriteTargets(memory) {
|
|
3438
|
+
const hygiene = rec(memory.hygiene);
|
|
3439
|
+
const targets = /* @__PURE__ */ new Set();
|
|
3440
|
+
if (isDeclared(hygiene.piiScrubFields)) targets.add(SCRUB);
|
|
3441
|
+
if ([rec(memory.shortTerm).ttlSeconds, rec(memory.longTerm).ttlDays].some((ttl) => ttl !== void 0 && ttl !== null)) targets.add(TTL);
|
|
3442
|
+
if (memory.longTerm && hygiene.auditLog === true) targets.add(AUDIT);
|
|
3443
|
+
return targets;
|
|
3444
|
+
}
|
|
3445
|
+
function declaredTargets(spec) {
|
|
3446
|
+
const all = (point2, declared2) => new Set(declared2 ? Object.keys(CONTRACT.evidencePoints[point2] ?? {}) : []);
|
|
3447
|
+
const memory = rec(spec.memory);
|
|
3448
|
+
const logging = rec(rec(spec.observability).logging);
|
|
3449
|
+
return {
|
|
3450
|
+
[EvidencePoint.enum.memoryWrite]: memoryWriteTargets(memory),
|
|
3451
|
+
[EvidencePoint.enum.vectorStore]: new Set(rec(memory.vector).namespace ? [NAMESPACE] : []),
|
|
3452
|
+
[GUARDRAIL]: new Set(Object.keys(CONTRACT.evidencePoints[GUARDRAIL] ?? {}).filter((t) => isDeclared(rec(spec.guardrails)[t]))),
|
|
3453
|
+
[EvidencePoint.enum.tool]: all(EvidencePoint.enum.tool, (Array.isArray(spec.tools) ? spec.tools : []).some((t) => t?.type === "function")),
|
|
3454
|
+
[EvidencePoint.enum.mcpClient]: all(EvidencePoint.enum.mcpClient, stdioServers(spec).length > 0),
|
|
3455
|
+
[EvidencePoint.enum.logging]: all(EvidencePoint.enum.logging, rec(spec.observability).logging !== void 0 && logging.structured !== false)
|
|
3456
|
+
};
|
|
3457
|
+
}
|
|
3458
|
+
function applicablePoints(spec) {
|
|
3459
|
+
const declared2 = declaredTargets(spec);
|
|
3460
|
+
const applicable = /* @__PURE__ */ new Map();
|
|
3461
|
+
for (const [point2, targets] of Object.entries(CONTRACT.evidencePoints)) {
|
|
3462
|
+
for (const [target, ruleId] of Object.entries(targets)) {
|
|
3463
|
+
const listed = CONTRACT.ruleEvidence[ruleId]?.sources.some((s) => s.point === point2);
|
|
3464
|
+
if (listed && declared2[point2]?.has(target)) applicable.set(keyOf(point2, target), { point: EvidencePoint.parse(point2), target, ruleId });
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
3467
|
+
return applicable;
|
|
3468
|
+
}
|
|
3469
|
+
function createEvidencePoints(manifest) {
|
|
3470
|
+
const manifestSha256 = canonicalManifestSha256(manifest);
|
|
3471
|
+
const spec = rec(rec(manifest).spec);
|
|
3472
|
+
const applicable = applicablePoints(spec);
|
|
3473
|
+
const mcpServers = stdioServers(spec).map((s) => String(s.name));
|
|
3474
|
+
const wired = /* @__PURE__ */ new Set();
|
|
3475
|
+
const emptyWindow = () => new Map([...applicable.keys()].map((k) => [k, { total: 0, counts: { pass: 0, fail: 0, not_verifiable: 0 }, last: [] }]));
|
|
3476
|
+
let calls = emptyWindow();
|
|
3477
|
+
let windowStart = /* @__PURE__ */ new Date();
|
|
3478
|
+
let seq = 0;
|
|
3479
|
+
const shippedUpTo = /* @__PURE__ */ new WeakMap();
|
|
3480
|
+
const points = {
|
|
3481
|
+
register: (point2, target) => {
|
|
3482
|
+
wired.add(keyOf(point2, target));
|
|
3483
|
+
},
|
|
3484
|
+
applies: (point2, target) => applicable.has(keyOf(point2, target)),
|
|
3485
|
+
memorySpec: () => rec(spec.memory),
|
|
3486
|
+
declaredMcpServers: () => [...mcpServers],
|
|
3487
|
+
recordCall: (point2, target, { subject, status = "pass", reason } = {}) => {
|
|
3488
|
+
if (status !== "pass" && !reason) throw new Error(`recordCall: a ${status} call of ${point2}:${target} needs a reason`);
|
|
3489
|
+
const c = calls.get(keyOf(point2, target));
|
|
3490
|
+
if (!c) return;
|
|
3491
|
+
c.total += 1;
|
|
3492
|
+
c.counts[status] += 1;
|
|
3493
|
+
c.last.push({ at: /* @__PURE__ */ new Date(), seq: ++seq, subject, status, reason });
|
|
3494
|
+
if (c.last.length > CONTRACT.evidencePointRecordLimit) c.last.shift();
|
|
3495
|
+
},
|
|
3496
|
+
snapshot: (ttlSeconds, reset = true) => {
|
|
3497
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
3498
|
+
const lastSeq = /* @__PURE__ */ new Map();
|
|
3499
|
+
const window = { start: windowStart.toISOString(), end: windowEnd.toISOString() };
|
|
3500
|
+
const aggregates = [...applicable].map(([key, { point: point2, target, ruleId }]) => {
|
|
3501
|
+
const c = calls.get(key);
|
|
3502
|
+
const state = c.total ? "fired" : wired.has(key) ? "wired" : "not_wired";
|
|
3503
|
+
lastSeq.set(key, seq);
|
|
3504
|
+
const records = c.last.map(({ at, subject, status, reason }) => ({
|
|
3505
|
+
checkId: subject ?? `${point2}:${target}`,
|
|
3506
|
+
ruleId,
|
|
3507
|
+
status: EvidenceStatus.enum[status],
|
|
3508
|
+
...reason ? { reason } : {},
|
|
3509
|
+
source: EvidenceSource.enum["sdk-observed"],
|
|
3510
|
+
origin: EvidenceOrigin.enum.in_process,
|
|
3511
|
+
collector: Collector.enum.sdk,
|
|
3512
|
+
observedAt: at.toISOString(),
|
|
3513
|
+
window,
|
|
3514
|
+
ttlSeconds,
|
|
3515
|
+
sampleSize: c.total,
|
|
3516
|
+
traceIds: [],
|
|
3517
|
+
manifestSha256
|
|
3518
|
+
}));
|
|
3519
|
+
return { point: point2, target, state, counts: { ...c.counts }, records };
|
|
3520
|
+
});
|
|
3521
|
+
if (reset) {
|
|
3522
|
+
calls = emptyWindow();
|
|
3523
|
+
windowStart = windowEnd;
|
|
3524
|
+
} else {
|
|
3525
|
+
shippedUpTo.set(aggregates, { lastSeq, windowEnd });
|
|
3526
|
+
}
|
|
3527
|
+
return aggregates;
|
|
3528
|
+
},
|
|
3529
|
+
subtract: (shipped) => {
|
|
3530
|
+
const taken = shippedUpTo.get(shipped);
|
|
3531
|
+
if (!taken) throw new Error("EvidencePoints.subtract: pass the array snapshot() returned");
|
|
3532
|
+
for (const a of shipped) {
|
|
3533
|
+
const key = keyOf(a.point, a.target);
|
|
3534
|
+
const c = calls.get(key);
|
|
3535
|
+
c.total -= a.counts.pass + a.counts.fail + a.counts.not_verifiable;
|
|
3536
|
+
for (const status of ["pass", "fail", "not_verifiable"]) c.counts[status] -= a.counts[status];
|
|
3537
|
+
c.last = c.last.filter((l) => l.seq > taken.lastSeq.get(key));
|
|
3538
|
+
}
|
|
3539
|
+
windowStart = taken.windowEnd;
|
|
3540
|
+
}
|
|
3541
|
+
};
|
|
3542
|
+
defaults[DEFAULT_KEY] = points;
|
|
3543
|
+
return points;
|
|
3544
|
+
}
|
|
3545
|
+
|
|
3546
|
+
// src/health/tool-check-id.ts
|
|
3547
|
+
function slugify(s) {
|
|
3548
|
+
return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3549
|
+
}
|
|
3550
|
+
function toolCheckId(toolName) {
|
|
3551
|
+
return `tool:${slugify(toolName)}`;
|
|
3552
|
+
}
|
|
3553
|
+
|
|
3554
|
+
// src/runtime/call-points.ts
|
|
3555
|
+
var { tool: TOOL, mcpClient: MCP_CLIENT, logging: LOGGING } = EvidencePoint.enum;
|
|
3556
|
+
var { memoryWrite: MEMORY_WRITE, vectorStore: VECTOR_STORE } = EvidencePoint.enum;
|
|
3557
|
+
var NOT_VERIFIABLE_CODE = "instrumentation_unavailable";
|
|
3558
|
+
var WRONG_NAMESPACE_CODE = "undeclared_namespace";
|
|
3559
|
+
var NO_NAMESPACE_CODE = "no_evidence";
|
|
3560
|
+
for (const code of [NOT_VERIFIABLE_CODE, WRONG_NAMESPACE_CODE, NO_NAMESPACE_CODE]) {
|
|
3561
|
+
if (!CONTRACT.reasonCodes.includes(code)) throw new Error(`contract.reasonCodes lacks '${code}'`);
|
|
3562
|
+
}
|
|
3563
|
+
new async_hooks.AsyncLocalStorage();
|
|
3564
|
+
|
|
2242
3565
|
// src/agent/reporter.ts
|
|
3566
|
+
var HEARTBEAT_POST_TIMEOUT_MS = 1e4;
|
|
3567
|
+
var DEFAULT_PUSH_INTERVAL_SECONDS = 30;
|
|
3568
|
+
var RECORD_TTL_INTERVALS = 2;
|
|
3569
|
+
function buildCappedBody(health, aggregates, usage) {
|
|
3570
|
+
const trimmedChecks = [...health.checks];
|
|
3571
|
+
const serialise = (checks, evidencePoints) => JSON.stringify({
|
|
3572
|
+
contract: CONTRACT.version,
|
|
3573
|
+
health: { ...health, checks },
|
|
3574
|
+
records: [],
|
|
3575
|
+
// the control plane reads the records inside each aggregate (#599)
|
|
3576
|
+
evidencePoints,
|
|
3577
|
+
usage
|
|
3578
|
+
});
|
|
3579
|
+
const size = (body2) => Buffer.byteLength(body2, "utf8");
|
|
3580
|
+
let body = serialise(trimmedChecks, aggregates);
|
|
3581
|
+
if (size(body) > CONTRACT.heartbeatMaxBytes) {
|
|
3582
|
+
aggregates = aggregates.map((a) => ({ ...a, records: [] }));
|
|
3583
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3584
|
+
}
|
|
3585
|
+
while (size(body) > CONTRACT.heartbeatMaxBytes && trimmedChecks.length > 0) {
|
|
3586
|
+
trimmedChecks.pop();
|
|
3587
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3588
|
+
}
|
|
3589
|
+
return { body, bytes: size(body) };
|
|
3590
|
+
}
|
|
2243
3591
|
var AgentSpecReporter = class {
|
|
2244
3592
|
constructor(manifest, opts = {}) {
|
|
2245
3593
|
this.manifest = manifest;
|
|
2246
3594
|
this.opts = opts;
|
|
2247
3595
|
this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
|
|
2248
3596
|
this.staleAfterMs = opts.staleAfterMs ?? 6e4;
|
|
3597
|
+
this.evidence = createEvidencePoints(opts.rawManifest ?? manifest);
|
|
2249
3598
|
}
|
|
3599
|
+
manifest;
|
|
3600
|
+
opts;
|
|
2250
3601
|
cached = null;
|
|
2251
3602
|
timer = null;
|
|
2252
3603
|
_pushTimer = null;
|
|
2253
3604
|
refreshIntervalMs;
|
|
2254
3605
|
staleAfterMs;
|
|
2255
|
-
|
|
3606
|
+
inflight = null;
|
|
3607
|
+
/** True while a heartbeat POST is in flight; a tick that finds it set sends nothing. */
|
|
3608
|
+
heartbeatInFlight = false;
|
|
2256
3609
|
stopped = false;
|
|
2257
|
-
registeredTools = /* @__PURE__ */ new Set();
|
|
2258
3610
|
/** In-process token usage accumulator. */
|
|
2259
3611
|
usage = new UsageLedger();
|
|
3612
|
+
/** Evidence points the agent's code wires (`guardrail(...)` reports here); each heartbeat sends the window's aggregates. */
|
|
3613
|
+
evidence;
|
|
2260
3614
|
/**
|
|
2261
|
-
*
|
|
2262
|
-
*
|
|
2263
|
-
* Declared tools that are not registered report status: 'fail'.
|
|
2264
|
-
*
|
|
2265
|
-
* Call this once per tool after wiring up your tool handlers:
|
|
2266
|
-
* reporter.registerTool('search-arxiv', handler)
|
|
2267
|
-
*
|
|
2268
|
-
* The handler argument is accepted but not called — it is retained solely so
|
|
2269
|
-
* callers can pass the real function object for documentation purposes.
|
|
3615
|
+
* @deprecated No-op, kept so existing callers compile. Registering a handler proves nothing about
|
|
3616
|
+
* which tools the agent calls, so the health report no longer carries `tool:<name>` checks (ADR-0017).
|
|
2270
3617
|
*/
|
|
2271
|
-
registerTool(
|
|
2272
|
-
this.registeredTools.add(name);
|
|
3618
|
+
registerTool(_name, _handler) {
|
|
2273
3619
|
}
|
|
2274
3620
|
/**
|
|
2275
3621
|
* Start background refresh. Call once during app startup.
|
|
@@ -2333,52 +3679,74 @@ var AgentSpecReporter = class {
|
|
|
2333
3679
|
*/
|
|
2334
3680
|
startPushMode(opts) {
|
|
2335
3681
|
if (this._pushTimer !== null) return;
|
|
2336
|
-
const
|
|
2337
|
-
|
|
3682
|
+
const controlPlaneUrl = opts.controlPlaneUrl || process.env[CONTRACT.env.controlPlaneUrl];
|
|
3683
|
+
if (!controlPlaneUrl) {
|
|
3684
|
+
throw new Error(
|
|
3685
|
+
`startPushMode: no control plane URL \u2014 pass controlPlaneUrl or set ${CONTRACT.env.controlPlaneUrl}`
|
|
3686
|
+
);
|
|
3687
|
+
}
|
|
3688
|
+
if (!opts.apiKey) {
|
|
3689
|
+
throw new Error("startPushMode: apiKey is required (the per-agent token from POST /api/v1/register)");
|
|
3690
|
+
}
|
|
3691
|
+
const resolved = { ...opts, controlPlaneUrl };
|
|
3692
|
+
const intervalMs = (opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * 1e3;
|
|
3693
|
+
void this._pushHeartbeat(resolved);
|
|
2338
3694
|
this._pushTimer = setInterval(() => {
|
|
2339
|
-
void this._pushHeartbeat(
|
|
3695
|
+
void this._pushHeartbeat(resolved);
|
|
2340
3696
|
}, intervalMs);
|
|
2341
3697
|
if (this._pushTimer.unref) {
|
|
2342
3698
|
this._pushTimer.unref();
|
|
2343
3699
|
}
|
|
2344
3700
|
}
|
|
2345
3701
|
async _pushHeartbeat(opts) {
|
|
3702
|
+
if (this.heartbeatInFlight) return;
|
|
3703
|
+
this.heartbeatInFlight = true;
|
|
2346
3704
|
try {
|
|
2347
3705
|
const health = await this.getReport();
|
|
2348
|
-
const gap = runAudit(this.manifest);
|
|
2349
3706
|
const usage = this.usage.snapshot(false);
|
|
2350
|
-
const
|
|
2351
|
-
|
|
2352
|
-
if (
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
}
|
|
3707
|
+
const aggregates = this.evidence.snapshot(Math.ceil((opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * RECORD_TTL_INTERVALS), false);
|
|
3708
|
+
const { body, bytes } = buildCappedBody(health, aggregates, usage);
|
|
3709
|
+
if (bytes > CONTRACT.heartbeatMaxBytes) {
|
|
3710
|
+
opts.onError?.(
|
|
3711
|
+
new Error(`Heartbeat not sent: ${bytes} bytes is over the ${CONTRACT.heartbeatMaxBytes}-byte cap with every check trimmed`)
|
|
3712
|
+
);
|
|
3713
|
+
return;
|
|
2358
3714
|
}
|
|
2359
3715
|
let res;
|
|
3716
|
+
const abort = new AbortController();
|
|
3717
|
+
const abortTimer = setTimeout(
|
|
3718
|
+
() => abort.abort(new Error(`heartbeat POST timed out after ${HEARTBEAT_POST_TIMEOUT_MS}ms`)),
|
|
3719
|
+
HEARTBEAT_POST_TIMEOUT_MS
|
|
3720
|
+
);
|
|
3721
|
+
abortTimer.unref?.();
|
|
2360
3722
|
try {
|
|
2361
|
-
res = await fetch(`${opts.controlPlaneUrl}
|
|
3723
|
+
res = await fetch(`${opts.controlPlaneUrl}${CONTRACT.controlPlanePaths.heartbeat}`, {
|
|
2362
3724
|
method: "POST",
|
|
2363
3725
|
headers: {
|
|
2364
3726
|
"Authorization": `Bearer ${opts.apiKey}`,
|
|
2365
3727
|
"Content-Type": "application/json"
|
|
2366
3728
|
},
|
|
2367
|
-
body
|
|
3729
|
+
body,
|
|
3730
|
+
signal: abort.signal
|
|
2368
3731
|
});
|
|
2369
3732
|
} catch (err) {
|
|
2370
3733
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2371
3734
|
opts.onError?.(new Error(sanitized));
|
|
2372
3735
|
return;
|
|
3736
|
+
} finally {
|
|
3737
|
+
clearTimeout(abortTimer);
|
|
2373
3738
|
}
|
|
2374
3739
|
if (res.ok) {
|
|
2375
|
-
this.usage.
|
|
3740
|
+
this.usage.subtract(usage);
|
|
3741
|
+
this.evidence.subtract(aggregates);
|
|
2376
3742
|
} else {
|
|
2377
3743
|
opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
|
|
2378
3744
|
}
|
|
2379
3745
|
} catch (err) {
|
|
2380
3746
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2381
3747
|
opts.onError?.(new Error(sanitized));
|
|
3748
|
+
} finally {
|
|
3749
|
+
this.heartbeatInFlight = false;
|
|
2382
3750
|
}
|
|
2383
3751
|
}
|
|
2384
3752
|
/** Stop sending heartbeats to the control plane. */
|
|
@@ -2392,38 +3760,26 @@ var AgentSpecReporter = class {
|
|
|
2392
3760
|
isPushModeActive() {
|
|
2393
3761
|
return this._pushTimer !== null;
|
|
2394
3762
|
}
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
this.
|
|
3763
|
+
/** Runs one refresh; concurrent callers share the in-flight promise so they all see a populated cache. */
|
|
3764
|
+
refresh() {
|
|
3765
|
+
if (this.inflight) return this.inflight;
|
|
3766
|
+
if (this.stopped) return Promise.resolve();
|
|
3767
|
+
this.inflight = this.runRefresh().finally(() => {
|
|
3768
|
+
this.inflight = null;
|
|
3769
|
+
});
|
|
3770
|
+
return this.inflight;
|
|
3771
|
+
}
|
|
3772
|
+
async runRefresh() {
|
|
2398
3773
|
try {
|
|
2399
3774
|
const report = await runHealthCheck(this.manifest, {
|
|
2400
3775
|
checkModel: true,
|
|
2401
3776
|
checkMcp: true,
|
|
2402
3777
|
checkMemory: true,
|
|
2403
|
-
checkServices: true
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
const registered = this.registeredTools.has(tool.name);
|
|
2407
|
-
return {
|
|
2408
|
-
id: `tool:${tool.name}`,
|
|
2409
|
-
category: "tool",
|
|
2410
|
-
status: registered ? "pass" : "fail",
|
|
2411
|
-
severity: "info",
|
|
2412
|
-
...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
|
|
2413
|
-
};
|
|
3778
|
+
checkServices: true,
|
|
3779
|
+
baseDir: this.opts.baseDir,
|
|
3780
|
+
rawManifest: this.opts.rawManifest
|
|
2414
3781
|
});
|
|
2415
|
-
|
|
2416
|
-
const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
|
|
2417
|
-
const enrichedReport = {
|
|
2418
|
-
...report,
|
|
2419
|
-
checks: [...report.checks, ...toolChecks],
|
|
2420
|
-
summary: {
|
|
2421
|
-
...report.summary,
|
|
2422
|
-
passed: report.summary.passed + toolPassed,
|
|
2423
|
-
failed: report.summary.failed + toolFailed
|
|
2424
|
-
}
|
|
2425
|
-
};
|
|
2426
|
-
this.cached = { report: enrichedReport, ts: Date.now() };
|
|
3782
|
+
this.cached = { report, ts: Date.now() };
|
|
2427
3783
|
} catch {
|
|
2428
3784
|
if (!this.cached) {
|
|
2429
3785
|
this.cached = {
|
|
@@ -2447,8 +3803,6 @@ var AgentSpecReporter = class {
|
|
|
2447
3803
|
} else {
|
|
2448
3804
|
this.cached = { ...this.cached, ts: Date.now() };
|
|
2449
3805
|
}
|
|
2450
|
-
} finally {
|
|
2451
|
-
this.refreshing = false;
|
|
2452
3806
|
}
|
|
2453
3807
|
}
|
|
2454
3808
|
};
|
|
@@ -2456,7 +3810,7 @@ var AgentSpecReporter = class {
|
|
|
2456
3810
|
// src/agent/adapters/fastify.ts
|
|
2457
3811
|
function agentSpecFastifyPlugin(reporter) {
|
|
2458
3812
|
return async (app) => {
|
|
2459
|
-
app.get(
|
|
3813
|
+
app.get(CONTRACT.agentPaths.health, async (_req, reply) => {
|
|
2460
3814
|
try {
|
|
2461
3815
|
const report = await reporter.getReport();
|
|
2462
3816
|
reply.status(200).send(report);
|
|
@@ -2521,30 +3875,63 @@ function agentSpecExpressRouter(reporter) {
|
|
|
2521
3875
|
|
|
2522
3876
|
exports.AUDIT_RULE_IDS = AUDIT_RULE_IDS;
|
|
2523
3877
|
exports.AgentSpecReporter = AgentSpecReporter;
|
|
3878
|
+
exports.COMPLIANCE_PACKS = COMPLIANCE_PACKS;
|
|
3879
|
+
exports.CONTRACT = CONTRACT;
|
|
3880
|
+
exports.ContractSchema = ContractSchema;
|
|
3881
|
+
exports.EVIDENCE_SHAPES = EVIDENCE_SHAPES;
|
|
3882
|
+
exports.EvalResult = EvalResult;
|
|
3883
|
+
exports.EvidenceOrigin = EvidenceOrigin;
|
|
3884
|
+
exports.EvidenceReason = EvidenceReason;
|
|
3885
|
+
exports.EvidenceRecord = EvidenceRecord;
|
|
3886
|
+
exports.EvidenceSource = EvidenceSource;
|
|
3887
|
+
exports.EvidenceStatus = EvidenceStatus;
|
|
3888
|
+
exports.EvidenceWindow = EvidenceWindow;
|
|
3889
|
+
exports.Heartbeat = Heartbeat;
|
|
2524
3890
|
exports.LATEST_API_VERSION = LATEST_API_VERSION;
|
|
3891
|
+
exports.MAX_PRODUCTION_TEMPERATURE = MAX_PRODUCTION_TEMPERATURE;
|
|
3892
|
+
exports.MAX_TRACE_IDS = MAX_TRACE_IDS;
|
|
2525
3893
|
exports.ManifestSchema = ManifestSchema;
|
|
3894
|
+
exports.PodspecFinding = PodspecFinding;
|
|
3895
|
+
exports.RULE_EVIDENCE = RULE_EVIDENCE;
|
|
3896
|
+
exports.ReadinessFinding = ReadinessFinding;
|
|
3897
|
+
exports.RuleEvidenceEntry = RuleEvidenceEntry;
|
|
3898
|
+
exports.SeveritySchema = SeveritySchema;
|
|
3899
|
+
exports.SidecarPush = SidecarPush;
|
|
3900
|
+
exports.TrustPolicy = TrustPolicy;
|
|
2526
3901
|
exports.UsageLedger = UsageLedger;
|
|
2527
3902
|
exports.agentSpecExpressRouter = agentSpecExpressRouter;
|
|
2528
3903
|
exports.agentSpecFastifyPlugin = agentSpecFastifyPlugin;
|
|
3904
|
+
exports.canonicalManifestSha256 = canonicalManifestSha256;
|
|
2529
3905
|
exports.collectEnvRefs = collectEnvRefs;
|
|
2530
3906
|
exports.collectFileRefs = collectFileRefs;
|
|
3907
|
+
exports.computeHealthStatus = computeHealthStatus;
|
|
3908
|
+
exports.declaredManifestSha256 = declaredManifestSha256;
|
|
3909
|
+
exports.describeManifestError = describeManifestError;
|
|
2531
3910
|
exports.detectRefType = detectRefType;
|
|
2532
3911
|
exports.detectVersion = detectVersion;
|
|
2533
3912
|
exports.exportJsonSchema = exportJsonSchema;
|
|
3913
|
+
exports.formatVerified = formatVerified;
|
|
3914
|
+
exports.gatewayEnvNames = gatewayEnvNames;
|
|
2534
3915
|
exports.generateAdapter = generateAdapter;
|
|
2535
3916
|
exports.getAdapter = getAdapter;
|
|
3917
|
+
exports.grade = grade;
|
|
3918
|
+
exports.isLatestModelId = isLatestModelId;
|
|
2536
3919
|
exports.isLatestVersion = isLatestVersion;
|
|
2537
|
-
exports.isProofRecord = isProofRecord;
|
|
2538
3920
|
exports.listAdapters = listAdapters;
|
|
2539
3921
|
exports.loadManifest = loadManifest;
|
|
2540
3922
|
exports.migrateManifest = migrateManifest;
|
|
2541
3923
|
exports.registerAdapter = registerAdapter;
|
|
3924
|
+
exports.renderGatewayConfig = renderGatewayConfig;
|
|
2542
3925
|
exports.resolveRef = resolveRef;
|
|
2543
3926
|
exports.resolveRefs = resolveRefs;
|
|
3927
|
+
exports.ruleClasses = ruleClasses;
|
|
2544
3928
|
exports.runAudit = runAudit;
|
|
2545
3929
|
exports.runHealthCheck = runHealthCheck;
|
|
2546
3930
|
exports.runServiceChecks = runServiceChecks;
|
|
2547
3931
|
exports.scoreToGrade = scoreToGrade;
|
|
3932
|
+
exports.secretEnvName = secretEnvName;
|
|
3933
|
+
exports.slugify = slugify;
|
|
3934
|
+
exports.toolCheckId = toolCheckId;
|
|
2548
3935
|
exports.tryLoadManifest = tryLoadManifest;
|
|
2549
3936
|
//# sourceMappingURL=index.cjs.map
|
|
2550
3937
|
//# sourceMappingURL=index.cjs.map
|