@agentspec/sdk 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/evidence-points-C6j4PSsQ.d.cts +1139 -0
- package/dist/evidence-points-C6j4PSsQ.d.ts +1139 -0
- package/dist/index.cjs +1727 -345
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1026 -2549
- package/dist/index.d.ts +1026 -2549
- package/dist/index.js +1678 -344
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +930 -0
- package/dist/runtime.cjs.map +1 -0
- package/dist/runtime.d.cts +63 -0
- package/dist/runtime.d.ts +63 -0
- package/dist/runtime.js +921 -0
- package/dist/runtime.js.map +1 -0
- package/package.json +15 -8
- package/schemas/evidence/v1/contract.schema.json +2078 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.n-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvalResult.pass-rate-above-one.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.bad-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.extra-field.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.fail-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.missing-ttl.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.not-verifiable-without-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-epoch.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-out-of-range.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.observed-at-space-separated.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.sample-size-boolean.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.ttl-as-string.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/EvidenceRecord.unknown-source.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.bad-record.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.point-records-over-limit.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/Heartbeat.with-gap.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/ReadinessFinding.bad-severity.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/SidecarPush.missing-sha.json +1 -0
- package/schemas/evidence/v1/fixtures/drifted/TrustPolicy.lenient.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.live.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvalResult.null-dataset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.fail.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.null-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass-null-reason.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.pass.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.timezone-offset.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.unknown-reason-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/EvidenceRecord.with-collector.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-null.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.contract-version.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.evidence-points.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.full.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk-no-records.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/Heartbeat.sdk.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.open-code.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/ReadinessFinding.with-subject.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/SidecarPush.declared.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.standard.json +1 -0
- package/schemas/evidence/v1/fixtures/valid/TrustPolicy.strict.json +1 -0
- package/schemas/grader/v1/fixtures/accepts-fractional-seconds.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-leap-day-in-year-below-100.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-more-than-6-fractional-digits.json +62 -0
- package/schemas/grader/v1/fixtures/accepts-numeric-offset.json +62 -0
- package/schemas/grader/v1/fixtures/all-records-must-pass.json +58 -0
- package/schemas/grader/v1/fixtures/below-threshold-not-graded.json +61 -0
- package/schemas/grader/v1/fixtures/derived-rule-fails-when-a-parent-fails.json +67 -0
- package/schemas/grader/v1/fixtures/derived-rule-follows-memory-write-parent.json +43 -0
- package/schemas/grader/v1/fixtures/derived-rule-passes-with-its-parents.json +64 -0
- package/schemas/grader/v1/fixtures/derived-rule-unverified-when-a-parent-has-no-evidence.json +51 -0
- package/schemas/grader/v1/fixtures/expired-record-is-stale.json +66 -0
- package/schemas/grader/v1/fixtures/fail-beats-not-verifiable.json +62 -0
- package/schemas/grader/v1/fixtures/grade-band-b.json +62 -0
- package/schemas/grader/v1/fixtures/independent-from-listed-collector-is-admitted.json +39 -0
- package/schemas/grader/v1/fixtures/independent-from-unlisted-collector-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/low-severity-coverage-only.json +90 -0
- package/schemas/grader/v1/fixtures/malformed-point-is-ignored.json +67 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-standard.json +196 -0
- package/schemas/grader/v1/fixtures/never-admitted-sources-strict.json +196 -0
- package/schemas/grader/v1/fixtures/not-admitted-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/not-verifiable-ignored-when-one-passes.json +58 -0
- package/schemas/grader/v1/fixtures/not-verifiable-only-keeps-reason.json +71 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-fail-when-no-point-of-the-rule-is-wired.json +59 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-a-wired-point-pass.json +87 -0
- package/schemas/grader/v1/fixtures/not-wired-point-drops-its-own-earlier-fail-when-another-point-is-wired.json +70 -0
- package/schemas/grader/v1/fixtures/not-wired-point-ignores-a-wired-point-fail-without-the-rule-id.json +69 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-a-wired-point-fail.json +79 -0
- package/schemas/grader/v1/fixtures/not-wired-point-keeps-independent-evidence.json +64 -0
- package/schemas/grader/v1/fixtures/not-wired-point-never-passes.json +107 -0
- package/schemas/grader/v1/fixtures/offline-record-is-not-admitted-yet.json +47 -0
- package/schemas/grader/v1/fixtures/record-without-collector-is-admitted-until-producers-set-it.json +38 -0
- package/schemas/grader/v1/fixtures/records-without-rule-id-never-graded.json +62 -0
- package/schemas/grader/v1/fixtures/rejects-compact-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-lowercase-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-hour-24.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-impossible-date-feb-29.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-t.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-lowercase-z.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-hours-above-23.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offset-minutes-above-59.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-offsetless-timestamp.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-space-separator.json +56 -0
- package/schemas/grader/v1/fixtures/rejects-year-zero.json +56 -0
- package/schemas/grader/v1/fixtures/rounds-coverage-before-the-threshold.json +850 -0
- package/schemas/grader/v1/fixtures/rounds-verified-before-the-band.json +674 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-admitted-standard.json +39 -0
- package/schemas/grader/v1/fixtures/sdk-observed-from-evidence-point-rule-is-not-admitted-strict.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-observed-standard.json +58 -0
- package/schemas/grader/v1/fixtures/sdk-observed-strict.json +66 -0
- package/schemas/grader/v1/fixtures/sdk-observed-without-evidence-point-source-is-rejected.json +47 -0
- package/schemas/grader/v1/fixtures/sdk-only-agent-all-points-fired.json +116 -0
- package/schemas/grader/v1/fixtures/stale-outranks-mismatch.json +64 -0
- package/schemas/grader/v1/fixtures/strict-admits-independent.json +39 -0
- package/schemas/grader/v1/fixtures/sub-millisecond-expiry-boundary.json +38 -0
- package/schemas/v1/agent.schema.json +372 -198
package/dist/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import {
|
|
3
|
-
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { readFileSync, realpathSync, existsSync } from 'fs';
|
|
4
3
|
import { resolve, sep, dirname } from 'path';
|
|
5
|
-
import yaml from 'js-yaml';
|
|
4
|
+
import * as yaml from 'js-yaml';
|
|
5
|
+
import { dump } from 'js-yaml';
|
|
6
6
|
import { isIP } from 'net';
|
|
7
|
+
import { createHash } from 'crypto';
|
|
8
|
+
import { AsyncLocalStorage } from 'async_hooks';
|
|
7
9
|
|
|
8
10
|
// src/schema/manifest.schema.ts
|
|
9
11
|
var refOrLiteral = z.string();
|
|
@@ -16,17 +18,37 @@ var MetadataSchema = z.object({
|
|
|
16
18
|
author: z.string().optional(),
|
|
17
19
|
license: z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
|
|
18
20
|
});
|
|
21
|
+
function isHttpUrl(value) {
|
|
22
|
+
if (value.startsWith("$env:")) return true;
|
|
23
|
+
try {
|
|
24
|
+
const url = new URL(value);
|
|
25
|
+
return (url.protocol === "http:" || url.protocol === "https:") && url.hostname !== "";
|
|
26
|
+
} catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
var baseUrlSchema = z.string().regex(/^(\$env:[A-Za-z_][A-Za-z0-9_]*|https?:\/\/.+)$/, 'must be an http(s) URL or a "$env:VAR" reference').refine(isHttpUrl, 'must be an http(s) URL with a host, or a "$env:VAR" reference').optional().describe('Override the provider endpoint with an OpenAI-compatible base URL (vLLM, Ollama, LiteLLM, an internal proxy, a mock LLM). A literal http(s) URL with a host (checked by the loader, not expressible in JSON Schema) or "$env:VAR". When set, the gateway treats the model as a custom OpenAI-compatible provider for any "provider" value except "anthropic", whose baseUrl is an Anthropic Messages endpoint without "/v1" (the gateway adds it). Omit to use the provider default.');
|
|
31
|
+
var ModelPricingSchema = z.object({
|
|
32
|
+
input: z.number().nonnegative().describe("USD per 1M input (prompt) tokens."),
|
|
33
|
+
output: z.number().nonnegative().describe("USD per 1M output (completion) tokens."),
|
|
34
|
+
cacheRead: z.number().nonnegative().optional().describe("USD per 1M tokens read from the prompt cache."),
|
|
35
|
+
cacheWrite: z.number().nonnegative().optional().describe("USD per 1M tokens written to the prompt cache."),
|
|
36
|
+
reasoning: z.number().nonnegative().optional().describe("USD per 1M reasoning tokens. The gateway uses the output price when omitted.")
|
|
37
|
+
}).describe("Price of this model in USD per 1M tokens. The gateway uses it instead of its built-in price, so a maxMonthlyUSD budget is spent on models it does not price (e.g. o4-mini). Omit to use the gateway's built-in price.");
|
|
19
38
|
var ModelFallbackSchema = z.object({
|
|
20
39
|
provider: z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
|
|
21
40
|
id: z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
|
|
22
41
|
apiKey: refOrLiteral.describe(refDesc),
|
|
42
|
+
baseUrl: baseUrlSchema,
|
|
23
43
|
triggerOn: z.array(z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
|
|
24
|
-
maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
|
|
44
|
+
maxRetries: z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover)."),
|
|
45
|
+
pricing: ModelPricingSchema.optional()
|
|
25
46
|
});
|
|
26
47
|
var ModelSchema = z.object({
|
|
27
48
|
provider: z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
|
|
28
49
|
id: z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
|
|
29
50
|
apiKey: refOrLiteral.describe(refDesc),
|
|
51
|
+
baseUrl: baseUrlSchema,
|
|
30
52
|
parameters: z.object({
|
|
31
53
|
temperature: z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
|
|
32
54
|
maxTokens: z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
|
|
@@ -35,6 +57,7 @@ var ModelSchema = z.object({
|
|
|
35
57
|
presencePenalty: z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
|
|
36
58
|
}).optional(),
|
|
37
59
|
fallback: ModelFallbackSchema.optional(),
|
|
60
|
+
pricing: ModelPricingSchema.optional(),
|
|
38
61
|
costControls: z.object({
|
|
39
62
|
maxMonthlyUSD: z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
|
|
40
63
|
alertAtUSD: z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
|
|
@@ -150,6 +173,7 @@ var ChatEndpointSchema = z.object({
|
|
|
150
173
|
var ApiSchema = z.object({
|
|
151
174
|
type: z.enum(["rest", "mcp", "grpc", "websocket"]),
|
|
152
175
|
port: z.number().int().min(1).max(65535).optional(),
|
|
176
|
+
allowedPorts: z.array(z.number().int().min(1).max(65535)).optional().describe("Extra ports the agent may listen on besides the sidecar-routed API port (e.g. a platform metrics port). The sidecar reports not-ready while the agent listens on any other non-loopback port."),
|
|
153
177
|
pathPrefix: z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
|
|
154
178
|
auth: z.object({
|
|
155
179
|
type: z.enum(["jwt", "apikey", "oauth2", "none"]),
|
|
@@ -158,7 +182,8 @@ var ApiSchema = z.object({
|
|
|
158
182
|
}).optional(),
|
|
159
183
|
rateLimit: z.object({
|
|
160
184
|
requestsPerMinute: z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
|
|
161
|
-
requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
|
|
185
|
+
requestsPerHour: z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429."),
|
|
186
|
+
tokensPerMinute: z.number().int().min(1).optional().describe("Maximum LLM tokens the gateway allows per minute. Once the bucket is empty, requests receive HTTP 429.")
|
|
162
187
|
}).optional(),
|
|
163
188
|
streaming: z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
|
|
164
189
|
healthEndpoint: z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
|
|
@@ -240,25 +265,42 @@ var EvalMetricEnum = z.enum([
|
|
|
240
265
|
// RAG: do the retrieved chunks cover the ground-truth answer?
|
|
241
266
|
"bias",
|
|
242
267
|
// Does the answer exhibit demographic or political bias?
|
|
268
|
+
"grounded",
|
|
269
|
+
// Are the answer's claims supported by the retrieved context? Replaces faithfulness and hallucination.
|
|
270
|
+
"pii_leakage",
|
|
271
|
+
// Does the answer leak personal data?
|
|
272
|
+
"prompt_injection",
|
|
273
|
+
// Did the answer follow an injected instruction?
|
|
243
274
|
"custom"
|
|
244
|
-
//
|
|
275
|
+
// Rubric-graded scorer, declared in the sibling `custom: { name, rubric }`.
|
|
245
276
|
]);
|
|
246
|
-
var
|
|
277
|
+
var LiveMetricEnum = EvalMetricEnum.exclude(["context_recall", "answer_similarity"]);
|
|
278
|
+
var CustomMetricSchema = z.object({
|
|
279
|
+
name: z.string().min(1).describe("Name the custom metric is reported under."),
|
|
280
|
+
rubric: refOrLiteral.describe('Grading rubric. Use "$file:rubrics/<name>.md"; the loader resolves it to the file contents.')
|
|
281
|
+
});
|
|
282
|
+
var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for grounded and context_precision metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
|
|
247
283
|
var EvaluationSchema = z.object({
|
|
248
|
-
framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
|
|
284
|
+
framework: z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]).optional().describe("Deprecated: nothing reads it. validate warns when set; rejected in 0.6.0."),
|
|
285
|
+
live: z.object({
|
|
286
|
+
metrics: z.array(LiveMetricEnum).optional().describe("Metrics scored on live traffic."),
|
|
287
|
+
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe("Minimum pass rate per live metric (0.0\u20131.0)."),
|
|
288
|
+
custom: CustomMetricSchema.optional()
|
|
289
|
+
}).optional().describe("Live evaluation, separate from dataset evaluation. Only live metrics are allowed."),
|
|
290
|
+
custom: CustomMetricSchema.optional().describe('Rubric for the "custom" metric on datasets.'),
|
|
249
291
|
datasets: z.array(
|
|
250
292
|
z.object({
|
|
251
293
|
name: z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
|
|
252
294
|
path: refOrLiteral.describe(datasetPathDesc),
|
|
253
295
|
metrics: z.array(EvalMetricEnum).optional().describe(
|
|
254
|
-
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["
|
|
296
|
+
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["grounded","context_recall"] while a safety dataset sets ["toxicity","bias"].'
|
|
255
297
|
)
|
|
256
298
|
})
|
|
257
299
|
).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
|
|
258
300
|
metrics: z.array(EvalMetricEnum).optional().describe(
|
|
259
|
-
'Default metrics to compute across all datasets that do not declare their own metrics. "
|
|
301
|
+
'Default metrics to compute across all datasets that do not declare their own metrics. "grounded", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires spec.evaluation.custom: { name, rubric }. "faithfulness" and "hallucination" are deprecated aliases of "grounded".'
|
|
260
302
|
),
|
|
261
|
-
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"
|
|
303
|
+
thresholds: z.record(z.string(), z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"grounded": 0.95, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
|
|
262
304
|
ciGate: z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
|
|
263
305
|
});
|
|
264
306
|
var ObservabilitySchema = z.object({
|
|
@@ -357,12 +399,608 @@ var ManifestSchema = z.object({
|
|
|
357
399
|
metadata: MetadataSchema,
|
|
358
400
|
spec: SpecSchema
|
|
359
401
|
});
|
|
402
|
+
var DRAFT_07 = "http://json-schema.org/draft-07/schema#";
|
|
403
|
+
var options = {
|
|
404
|
+
target: "draft-7",
|
|
405
|
+
io: "input",
|
|
406
|
+
reused: "inline",
|
|
407
|
+
override: ({ zodSchema, jsonSchema }) => {
|
|
408
|
+
const def = zodSchema._zod.def;
|
|
409
|
+
if (def.type === "object" && jsonSchema.additionalProperties === void 0) jsonSchema.additionalProperties = false;
|
|
410
|
+
if (def.type === "record") delete jsonSchema.propertyNames;
|
|
411
|
+
if (def.type === "union" && def.discriminator && jsonSchema.oneOf) {
|
|
412
|
+
jsonSchema.anyOf = jsonSchema.oneOf;
|
|
413
|
+
delete jsonSchema.oneOf;
|
|
414
|
+
}
|
|
415
|
+
if (jsonSchema.type === "integer") {
|
|
416
|
+
if (jsonSchema.maximum === Number.MAX_SAFE_INTEGER) delete jsonSchema.maximum;
|
|
417
|
+
if (jsonSchema.minimum === Number.MIN_SAFE_INTEGER) delete jsonSchema.minimum;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
function toJsonSchema(schema) {
|
|
422
|
+
const { $schema: _, ...body } = z.toJSONSchema(schema, options);
|
|
423
|
+
return { ...body, $schema: DRAFT_07 };
|
|
424
|
+
}
|
|
425
|
+
function toNamedJsonSchema(name, schema) {
|
|
426
|
+
const { $schema, ...body } = toJsonSchema(schema);
|
|
427
|
+
return { $ref: `#/definitions/${name}`, definitions: { [name]: body }, $schema };
|
|
428
|
+
}
|
|
360
429
|
function exportJsonSchema() {
|
|
361
|
-
return
|
|
362
|
-
|
|
363
|
-
|
|
430
|
+
return toNamedJsonSchema("AgentSpecManifest", ManifestSchema);
|
|
431
|
+
}
|
|
432
|
+
var EvidenceClass = z.enum(["independent", "evidence-point", "offline", "declared"]);
|
|
433
|
+
var Collector = z.enum([
|
|
434
|
+
"gateway-config",
|
|
435
|
+
"gateway-traffic",
|
|
436
|
+
"sidecar-probe",
|
|
437
|
+
"sidecar-forwarder",
|
|
438
|
+
"sidecar-eval",
|
|
439
|
+
"operator-webhook",
|
|
440
|
+
"operator-logs",
|
|
441
|
+
"sdk",
|
|
442
|
+
"lock",
|
|
443
|
+
"manifest"
|
|
444
|
+
]);
|
|
445
|
+
var EvidencePoint = z.enum(["guardrail", "tool", "mcpClient", "memoryWrite", "vectorStore", "logging"]);
|
|
446
|
+
var Scope = z.enum(["declared", "configured", "enforced", "observed", "ran-where-wired", "every-path", "effective"]);
|
|
447
|
+
var WiredBy = z.enum(["none", "operator-render", "agent-code", "user-schedule", "opt-in"]);
|
|
448
|
+
var Mode = z.enum(["standalone", "operator"]);
|
|
449
|
+
var BuildStatus = z.enum(["built", "partial", "planned", "open"]);
|
|
450
|
+
var RuleEvidenceSource = z.object({
|
|
451
|
+
class: EvidenceClass,
|
|
452
|
+
collector: Collector,
|
|
453
|
+
/** Evidence-point sources only. */
|
|
454
|
+
point: EvidencePoint.nullish(),
|
|
455
|
+
proves: Scope,
|
|
456
|
+
wiredBy: WiredBy,
|
|
457
|
+
modes: z.array(Mode).min(1),
|
|
458
|
+
status: BuildStatus,
|
|
459
|
+
/** A variant such as HTTP MCP, stdio MCP, local tools or opt-in. */
|
|
460
|
+
when: z.string().min(1).nullish(),
|
|
461
|
+
/** A collector that checks the counts, such as gateway calls against guardrail records. */
|
|
462
|
+
crossCheck: Collector.nullish(),
|
|
463
|
+
/** The `agentspec lock` prompt question. */
|
|
464
|
+
question: z.string().min(1).nullish()
|
|
465
|
+
}).strict().refine((s) => s.class === "evidence-point" === (s.point != null), {
|
|
466
|
+
message: "an evidence-point source names its `point`, and no other class does"
|
|
467
|
+
});
|
|
468
|
+
var RuleEvidenceEntry = z.object({
|
|
469
|
+
sources: z.array(RuleEvidenceSource).default([]),
|
|
470
|
+
/** Rules whose verdicts decide this one (all must pass); this rule inherits their sources. */
|
|
471
|
+
derivedFrom: z.array(z.string().min(1)).nullish(),
|
|
472
|
+
/** What a pass does not prove, shown with the result. Free text. */
|
|
473
|
+
notProven: z.string().nullish()
|
|
474
|
+
}).strict().refine((e) => e.sources.length > 0 || (e.derivedFrom?.length ?? 0) > 0, {
|
|
475
|
+
message: "an entry needs at least one source or a `derivedFrom`"
|
|
476
|
+
});
|
|
477
|
+
var BOTH = ["standalone", "operator"];
|
|
478
|
+
var CLASS_DEFAULTS = {
|
|
479
|
+
independent: { wiredBy: "none", status: "built" },
|
|
480
|
+
"evidence-point": { wiredBy: "agent-code", status: "planned" },
|
|
481
|
+
offline: { wiredBy: "user-schedule", status: "planned" },
|
|
482
|
+
declared: { wiredBy: "none", status: "built" }
|
|
483
|
+
};
|
|
484
|
+
var source = (cls, collector, proves, extra = {}) => ({ class: cls, collector, proves, modes: BOTH, ...CLASS_DEFAULTS[cls], ...extra });
|
|
485
|
+
var independent = (collector, proves, extra) => source("independent", collector, proves, extra);
|
|
486
|
+
var point = (p, proves, extra) => source("evidence-point", "sdk", proves, { point: p, ...extra });
|
|
487
|
+
var lock = (question, extra) => source("offline", "lock", "every-path", { question, ...extra });
|
|
488
|
+
var declared = (extra) => source("declared", "manifest", "declared", extra);
|
|
489
|
+
var OPERATOR_ONLY = { modes: ["operator"] };
|
|
490
|
+
var PLANNED = { status: "planned" };
|
|
491
|
+
var WRITES_BYPASS = "Writes that bypass the evidence point at runtime.";
|
|
492
|
+
var entries = {
|
|
493
|
+
"MODEL-01": { sources: [independent("gateway-config", "configured"), independent("gateway-traffic", "observed")] },
|
|
494
|
+
"MODEL-02": {
|
|
495
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
496
|
+
},
|
|
497
|
+
"MODEL-03": { sources: [independent("gateway-config", "enforced")] },
|
|
498
|
+
"MODEL-04": {
|
|
499
|
+
sources: [
|
|
500
|
+
independent("gateway-config", "enforced"),
|
|
501
|
+
lock("Do the agent's own provider clients retry more than spec.model.fallback.maxRetries, or stack retries on the gateway's?", { when: "client retries" })
|
|
502
|
+
],
|
|
503
|
+
notProven: "That the gateway's attempts match `spec.model.fallback.maxRetries` at runtime beyond the rendered config."
|
|
504
|
+
},
|
|
505
|
+
"MODEL-05": {
|
|
506
|
+
sources: [independent("gateway-traffic", "observed", { when: "agw.ai.usage.cost.total" })],
|
|
507
|
+
notProven: "Requests with no token usage: not counted."
|
|
508
|
+
},
|
|
509
|
+
"SEC-LLM-01": {
|
|
510
|
+
sources: [
|
|
511
|
+
independent("sidecar-probe", "effective", { when: "opt-in", wiredBy: "opt-in", status: "planned" }),
|
|
512
|
+
point("guardrail", "ran-where-wired"),
|
|
513
|
+
lock("Does every path that sends user input to a model run the declared input guardrails first?")
|
|
514
|
+
]
|
|
515
|
+
},
|
|
516
|
+
"SEC-LLM-02": {
|
|
517
|
+
sources: [
|
|
518
|
+
point("guardrail", "ran-where-wired"),
|
|
519
|
+
lock("Does every path that returns model output run the declared output guardrails first?")
|
|
520
|
+
],
|
|
521
|
+
notProven: "Effective: a no-op output guardrail still fires."
|
|
522
|
+
},
|
|
523
|
+
"SEC-LLM-03": { sources: [declared()] },
|
|
524
|
+
"SEC-LLM-04": {
|
|
525
|
+
sources: [independent("gateway-config", "enforced"), independent("gateway-traffic", "observed", PLANNED)],
|
|
526
|
+
notProven: "Cost: MODEL-03 covers it."
|
|
527
|
+
},
|
|
528
|
+
"SEC-LLM-05": {
|
|
529
|
+
sources: [independent("gateway-traffic", "observed"), independent("gateway-config", "configured", PLANNED)]
|
|
530
|
+
},
|
|
531
|
+
"SEC-LLM-06": { derivedFrom: ["MEM-01"], notProven: "Same as MEM-01." },
|
|
532
|
+
"SEC-LLM-07": {
|
|
533
|
+
sources: [
|
|
534
|
+
independent("sidecar-probe", "configured", { when: "HTTP MCP" }),
|
|
535
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
536
|
+
lock("Does the agent call any MCP tool the manifest does not declare, or one that serves no annotations?"),
|
|
537
|
+
declared({ when: "local tools" })
|
|
538
|
+
]
|
|
539
|
+
},
|
|
540
|
+
"SEC-LLM-08": {
|
|
541
|
+
sources: [
|
|
542
|
+
independent("sidecar-probe", "configured", PLANNED),
|
|
543
|
+
independent("gateway-traffic", "enforced", PLANNED),
|
|
544
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" })
|
|
545
|
+
],
|
|
546
|
+
notProven: "Which tools are destructive: the server declares it; nothing checks the claim."
|
|
547
|
+
},
|
|
548
|
+
"SEC-LLM-09": { derivedFrom: ["EVAL-01", "EVAL-02"] },
|
|
549
|
+
"SEC-LLM-10": {
|
|
550
|
+
sources: [
|
|
551
|
+
independent("operator-webhook", "configured", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
552
|
+
declared({ modes: ["standalone"] })
|
|
553
|
+
]
|
|
554
|
+
},
|
|
555
|
+
"TOOL-01": {
|
|
556
|
+
sources: [
|
|
557
|
+
independent("gateway-traffic", "observed", { when: "HTTP MCP" }),
|
|
558
|
+
point("tool", "ran-where-wired", { when: "local tools" }),
|
|
559
|
+
point("mcpClient", "ran-where-wired", { when: "stdio MCP" }),
|
|
560
|
+
lock("Which tools does the agent code register or call, local, HTTP MCP and stdio MCP?")
|
|
561
|
+
],
|
|
562
|
+
notProven: "Unwrapped local or stdio calls at runtime: only the lock inventory catches them."
|
|
563
|
+
},
|
|
564
|
+
"MEM-01": {
|
|
565
|
+
sources: [
|
|
566
|
+
point("memoryWrite", "enforced"),
|
|
567
|
+
lock("Does every memory write pass through the PII scrubber for the declared fields?")
|
|
568
|
+
],
|
|
569
|
+
notProven: "Full coverage: the scrubber is a regex baseline (Presidio optional in Python). " + WRITES_BYPASS
|
|
570
|
+
},
|
|
571
|
+
"MEM-02": {
|
|
572
|
+
sources: [
|
|
573
|
+
point("memoryWrite", "enforced"),
|
|
574
|
+
lock("Does every memory write apply the declared retention TTL?")
|
|
575
|
+
],
|
|
576
|
+
notProven: WRITES_BYPASS + " Backends that cannot apply a TTL (record `ttl: unsupported`): the evidence point is `not_verifiable` there and only the lock remains."
|
|
577
|
+
},
|
|
578
|
+
"MEM-03": {
|
|
579
|
+
sources: [
|
|
580
|
+
point("memoryWrite", "enforced"),
|
|
581
|
+
lock("Does every memory write honour the declared memory audit setting?")
|
|
582
|
+
],
|
|
583
|
+
notProven: WRITES_BYPASS
|
|
584
|
+
},
|
|
585
|
+
"MEM-04": {
|
|
586
|
+
sources: [
|
|
587
|
+
point("vectorStore", "ran-where-wired"),
|
|
588
|
+
lock("Does every vector store query and write use the declared namespace or tenant isolation?")
|
|
589
|
+
],
|
|
590
|
+
notProven: "Isolation: the right namespace does not prove other tenants cannot read it."
|
|
591
|
+
},
|
|
592
|
+
"MEM-05": {
|
|
593
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.usage.output_tokens" }), declared({ when: "until a call is observed" })],
|
|
594
|
+
notProven: "The tolerance for the system prompt and tool schemas is set in Phase 2, with an ADR-0015 amendment; a call without gen_ai.usage.output_tokens leaves the check not_verifiable."
|
|
595
|
+
},
|
|
596
|
+
"EVAL-01": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
597
|
+
"EVAL-02": {
|
|
598
|
+
sources: [
|
|
599
|
+
independent("sidecar-eval", "observed", PLANNED),
|
|
600
|
+
lock("Does the agent's CI run the declared evaluation gate before deploy?")
|
|
601
|
+
],
|
|
602
|
+
notProven: "That a failed gate actually blocks a deploy: CI is outside the system."
|
|
603
|
+
},
|
|
604
|
+
"EVAL-03": { sources: [independent("sidecar-eval", "observed", PLANNED)] },
|
|
605
|
+
"OBS-01": {
|
|
606
|
+
sources: [independent("sidecar-forwarder", "observed"), independent("sidecar-probe", "configured")],
|
|
607
|
+
notProven: "That the agent exports every span: the rule is about the backend, not span coverage. agentops is not a target: it has no OTLP ingest."
|
|
608
|
+
},
|
|
609
|
+
"OBS-02": {
|
|
610
|
+
sources: [
|
|
611
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
612
|
+
point("logging", "ran-where-wired"),
|
|
613
|
+
lock("Does every log statement emit structured JSON?")
|
|
614
|
+
]
|
|
615
|
+
},
|
|
616
|
+
"OBS-03": {
|
|
617
|
+
sources: [
|
|
618
|
+
independent("operator-logs", "observed", { ...OPERATOR_ONLY, ...PLANNED }),
|
|
619
|
+
lock("Does any log statement write one of the declared sensitive fields?")
|
|
620
|
+
],
|
|
621
|
+
notProven: "Full coverage: the scanner is a regex and pattern baseline, so recall is limited."
|
|
622
|
+
},
|
|
623
|
+
"META-01": { sources: [declared()] },
|
|
624
|
+
"META-02": {
|
|
625
|
+
sources: [independent("gateway-traffic", "observed", { when: "gen_ai.request.temperature" })],
|
|
626
|
+
notProven: "Requests that set no temperature: `not_verifiable`, not counted."
|
|
627
|
+
}
|
|
628
|
+
};
|
|
629
|
+
var RULE_EVIDENCE = Object.fromEntries(
|
|
630
|
+
Object.entries(entries).map(([id, entry]) => [id, RuleEvidenceEntry.parse(entry)])
|
|
631
|
+
);
|
|
632
|
+
function ruleSources(ruleId, registry = RULE_EVIDENCE, seen = /* @__PURE__ */ new Set()) {
|
|
633
|
+
if (!Object.hasOwn(registry, ruleId) || seen.has(ruleId)) return [];
|
|
634
|
+
const entry = registry[ruleId];
|
|
635
|
+
const visited = /* @__PURE__ */ new Set([...seen, ruleId]);
|
|
636
|
+
return [...entry.sources, ...(entry.derivedFrom ?? []).flatMap((parent) => ruleSources(parent, registry, visited))];
|
|
637
|
+
}
|
|
638
|
+
function ruleClasses(ruleId, registry = RULE_EVIDENCE) {
|
|
639
|
+
const classes = new Set(ruleSources(ruleId, registry).map((s) => s.class));
|
|
640
|
+
return EvidenceClass.options.filter((c) => classes.has(c));
|
|
641
|
+
}
|
|
642
|
+
var EvidenceStatus = z.enum(["pass", "fail", "not_verifiable"]);
|
|
643
|
+
var EvidenceSource = z.enum(["independent", "sdk-observed", "lower-bound", "self-declared", "declared", "offline"]);
|
|
644
|
+
var EvidenceOrigin = z.enum(["ingress", "gateway", "egress_proxy", "in_process", "probe", "manifest", "judge"]);
|
|
645
|
+
var SeveritySchema = z.enum(["critical", "high", "medium", "low", "info"]);
|
|
646
|
+
var TrustPolicy = z.enum(["standard", "strict"]);
|
|
647
|
+
var EvidenceReason = z.object({ code: z.string().min(1), detail: z.string().nullish() }).strict();
|
|
648
|
+
var TIMESTAMP_PATTERN = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
|
|
649
|
+
var Timestamp = z.string().regex(TIMESTAMP_PATTERN);
|
|
650
|
+
var EvidenceWindow = z.object({ start: Timestamp, end: Timestamp }).strict();
|
|
651
|
+
var Sha256 = z.string().regex(/^[a-f0-9]{64}$/);
|
|
652
|
+
var MAX_TRACE_IDS = 20;
|
|
653
|
+
var recordFields = {
|
|
654
|
+
/** Scoped per subject: `tool:<name>`, `model:<id>`, or a bare check name for agent-wide checks. */
|
|
655
|
+
checkId: z.string().min(1),
|
|
656
|
+
ruleId: z.string().min(1).nullish(),
|
|
657
|
+
source: EvidenceSource,
|
|
658
|
+
origin: EvidenceOrigin,
|
|
659
|
+
/** The registry collector that produced the record. Optional until every producer sets it (#422); checking it against the collectors the registry lists for the rule is the grader's job (#423). */
|
|
660
|
+
collector: Collector.nullish(),
|
|
661
|
+
observedAt: Timestamp,
|
|
662
|
+
window: EvidenceWindow,
|
|
663
|
+
ttlSeconds: z.number().int().nonnegative(),
|
|
664
|
+
sampleSize: z.number().int().nonnegative(),
|
|
665
|
+
traceIds: z.array(z.string().min(1)).max(MAX_TRACE_IDS),
|
|
666
|
+
manifestSha256: Sha256
|
|
667
|
+
};
|
|
668
|
+
var EvidenceRecordPass = z.object({ ...recordFields, status: z.literal("pass"), reason: EvidenceReason.nullish() }).strict();
|
|
669
|
+
var EvidenceRecordNotPass = z.object({
|
|
670
|
+
...recordFields,
|
|
671
|
+
status: z.enum(["fail", "not_verifiable"]),
|
|
672
|
+
reason: EvidenceReason
|
|
673
|
+
}).strict();
|
|
674
|
+
var EvidenceRecord = z.union([EvidenceRecordPass, EvidenceRecordNotPass]);
|
|
675
|
+
var PodspecFinding = z.object({ code: z.string().min(1), subject: z.string().min(1), message: z.string().min(1) }).strict();
|
|
676
|
+
var ReadinessFinding = z.object({
|
|
677
|
+
code: z.string().min(1),
|
|
678
|
+
severity: SeveritySchema,
|
|
679
|
+
message: z.string().min(1),
|
|
680
|
+
subject: z.string().nullish(),
|
|
681
|
+
ruleIds: z.array(z.string().min(1)).nullish()
|
|
682
|
+
}).strict();
|
|
683
|
+
var EvalResult = z.object({
|
|
684
|
+
metric: z.string().min(1),
|
|
685
|
+
n: z.number().int().nonnegative(),
|
|
686
|
+
passRate: z.number().min(0).max(1),
|
|
687
|
+
ciLow: z.number().min(0).max(1),
|
|
688
|
+
ciHigh: z.number().min(0).max(1),
|
|
689
|
+
threshold: z.number().min(0).max(1),
|
|
690
|
+
window: EvidenceWindow,
|
|
691
|
+
judgeModel: z.string().min(1),
|
|
692
|
+
rubricHash: Sha256,
|
|
693
|
+
source: EvidenceSource,
|
|
694
|
+
dataset: z.string().min(1).nullish()
|
|
695
|
+
}).strict();
|
|
696
|
+
var MAX_POINT_RECORDS = 5;
|
|
697
|
+
var EvidencePointState = z.enum(["fired", "wired", "not_wired"]);
|
|
698
|
+
var EvidencePointAggregate = z.object({
|
|
699
|
+
point: EvidencePoint,
|
|
700
|
+
/** What the point is wired on, a key of `CONTRACT.evidencePoints[point]`. */
|
|
701
|
+
target: z.string().min(1),
|
|
702
|
+
state: EvidencePointState,
|
|
703
|
+
counts: z.object({
|
|
704
|
+
pass: z.number().int().nonnegative(),
|
|
705
|
+
fail: z.number().int().nonnegative(),
|
|
706
|
+
not_verifiable: z.number().int().nonnegative()
|
|
707
|
+
}).strict(),
|
|
708
|
+
records: z.array(EvidenceRecord).max(MAX_POINT_RECORDS)
|
|
709
|
+
}).strict();
|
|
710
|
+
var Passthrough = z.record(z.string(), z.unknown());
|
|
711
|
+
var Heartbeat = z.object({
|
|
712
|
+
/** Contract version of the sending SDK (`CONTRACT.version`); absent or null from older SDKs. */
|
|
713
|
+
contract: z.string().regex(/^\d+\.\d+\.\d+$/).nullish(),
|
|
714
|
+
health: Passthrough,
|
|
715
|
+
records: z.array(EvidenceRecord).default([]),
|
|
716
|
+
/** Every evidence point the registry gives for the manifest, wired or not; absent from older SDKs. */
|
|
717
|
+
evidencePoints: z.array(EvidencePointAggregate).default([]),
|
|
718
|
+
usage: Passthrough.nullish(),
|
|
719
|
+
tools: Passthrough.nullish()
|
|
720
|
+
}).strict();
|
|
721
|
+
var SidecarPush = z.object({
|
|
722
|
+
manifestSha256: Sha256,
|
|
723
|
+
declared: Passthrough.nullish(),
|
|
724
|
+
records: z.array(EvidenceRecord).default([])
|
|
725
|
+
}).strict();
|
|
726
|
+
var EVIDENCE_SHAPES = { EvidenceRecord, ReadinessFinding, EvalResult, TrustPolicy, Heartbeat, SidecarPush };
|
|
727
|
+
|
|
728
|
+
// src/schema/contract.schema.ts
|
|
729
|
+
var fixed = (value) => z.literal(value).default(value);
|
|
730
|
+
var REASON_CODES = [
|
|
731
|
+
"stale",
|
|
732
|
+
"source_not_admitted",
|
|
733
|
+
"source_mismatch",
|
|
734
|
+
"no_evidence",
|
|
735
|
+
"not_wired",
|
|
736
|
+
"no_calls_observed",
|
|
737
|
+
"no_failover_observed",
|
|
738
|
+
"no_budget_evidence",
|
|
739
|
+
"no_approval_token_evidence",
|
|
740
|
+
"no_retrieval_spans",
|
|
741
|
+
"unrecognised_conventions",
|
|
742
|
+
"instrumentation_unavailable",
|
|
743
|
+
"declared_env_missing",
|
|
744
|
+
"provider_key_on_agent",
|
|
745
|
+
"response_cap_exceeded",
|
|
746
|
+
"temperature_above_max",
|
|
747
|
+
"undeclared_tool",
|
|
748
|
+
"undeclared_model",
|
|
749
|
+
"unpinned_model",
|
|
750
|
+
"self_declared",
|
|
751
|
+
"unpriced_model",
|
|
752
|
+
"undeclared_namespace"
|
|
753
|
+
];
|
|
754
|
+
var ContractSchema = z.object({
|
|
755
|
+
/** Contract version (major.minor.patch) a heartbeat carries: a receiver rejects another major and accepts a newer minor (ADR-0039). */
|
|
756
|
+
version: fixed("1.0.0"),
|
|
757
|
+
env: z.object({
|
|
758
|
+
manifestPath: fixed("MANIFEST_PATH"),
|
|
759
|
+
upstreamUrl: fixed("UPSTREAM_URL"),
|
|
760
|
+
proxyPort: fixed("PROXY_PORT"),
|
|
761
|
+
controlPlanePort: fixed("CONTROL_PLANE_PORT"),
|
|
762
|
+
/** Control plane URL the operator webhook injects into the sidecar and agent containers; SDK push mode falls back to it. */
|
|
763
|
+
controlPlaneUrl: fixed("AGENTSPEC_CONTROL_PLANE_URL"),
|
|
764
|
+
/** Agent JWT from `POST /api/v1/register`; `agentspec register` sends it to the control plane as a Bearer token. The sidecar's own push needs the `sidecarToken` from the same call (see `agentTokenSecretSuffix`). */
|
|
765
|
+
agentKey: fixed("AGENTSPEC_API_KEY"),
|
|
766
|
+
/** Trust policy (`standard | strict`) the grader applies: set on the control plane and the sidecar, never in `agent.yaml` (ADR-0007). */
|
|
767
|
+
trustPolicy: fixed("AGENTSPEC_TRUST_POLICY"),
|
|
768
|
+
/** The webhook's PodSpec findings (the `agentspec.io/podspec-findings` annotation, JSON), injected into the sidecar by the downward API (ADR-0019). */
|
|
769
|
+
podspecFindings: fixed("AGENTSPEC_PODSPEC_FINDINGS"),
|
|
770
|
+
/** Virtual key the agent sends and the gateway accepts (ADR-0001); provider keys stay in the gateway. */
|
|
771
|
+
gatewayKey: fixed("AGENTSPEC_GATEWAY_KEY"),
|
|
772
|
+
/** Bind address and port the agent must honour so the sidecar can own `spec.api.port` (ADR-0028). */
|
|
773
|
+
agentHost: fixed("HOST"),
|
|
774
|
+
agentPort: fixed("PORT"),
|
|
775
|
+
/** Azure resource name the gateway config references (`$AZURE_OPENAI_RESOURCE_NAME`). */
|
|
776
|
+
azureResourceName: fixed("AZURE_OPENAI_RESOURCE_NAME")
|
|
777
|
+
}).strict().prefault({}),
|
|
778
|
+
/** Provider base-URL env vars, keyed by spec.model.provider. */
|
|
779
|
+
providerBaseUrlEnv: z.object({
|
|
780
|
+
openai: fixed("OPENAI_BASE_URL"),
|
|
781
|
+
anthropic: fixed("ANTHROPIC_BASE_URL")
|
|
782
|
+
}).strict().prefault({}),
|
|
783
|
+
paths: z.object({
|
|
784
|
+
manifestDir: fixed("/manifest"),
|
|
785
|
+
manifestFile: fixed("/manifest/agent.yaml"),
|
|
786
|
+
/** SQLite file the gateway keeps USD budget usage in; must be on a writable volume. */
|
|
787
|
+
gatewayDatabase: fixed("/gateway-data/agentgateway.db"),
|
|
788
|
+
/** Directory the init container renders the gateway config into (ADR-0025). */
|
|
789
|
+
gatewayConfigDir: fixed("/gateway"),
|
|
790
|
+
gatewayConfigFile: fixed("/gateway/config.yaml"),
|
|
791
|
+
/** Sidecar bundle inside its image; `node <entrypoint> gateway-config …` is the init container's command (ADR-0025). */
|
|
792
|
+
sidecarEntrypoint: fixed("dist/index.js"),
|
|
793
|
+
gatewayDataDir: fixed("/gateway-data")
|
|
794
|
+
}).strict().prefault({}),
|
|
795
|
+
images: z.object({
|
|
796
|
+
sidecar: fixed("ghcr.io/agents-oss/agentspec-sidecar:latest"),
|
|
797
|
+
/** agentgateway, pinned (ADR-0025). */
|
|
798
|
+
gateway: fixed("cr.agentgateway.dev/agentgateway:v1.6.0")
|
|
799
|
+
}).strict().prefault({}),
|
|
800
|
+
ports: z.object({
|
|
801
|
+
agentDefault: fixed(8e3),
|
|
802
|
+
/** Loopback port the agent binds once the sidecar takes over its declared port (ADR-0028). */
|
|
803
|
+
agentBind: fixed(18e3),
|
|
804
|
+
proxy: fixed(4e3),
|
|
805
|
+
controlPlane: fixed(4001),
|
|
806
|
+
/** agentgateway LLM listener on the pod loopback (ADR-0025). */
|
|
807
|
+
gateway: fixed(4002),
|
|
808
|
+
/** agentgateway's default stats and readiness listeners: wildcard-bound, not the agent's (ADR-0025). */
|
|
809
|
+
gatewayStats: fixed(15020),
|
|
810
|
+
gatewayReadiness: fixed(15021)
|
|
811
|
+
}).strict().prefault({}),
|
|
812
|
+
/** Bare paths served by the sidecar on the control-plane port (ADR-0023). */
|
|
813
|
+
sidecarPaths: z.object({
|
|
814
|
+
healthReady: fixed("/health/ready"),
|
|
815
|
+
gap: fixed("/gap"),
|
|
816
|
+
explore: fixed("/explore"),
|
|
817
|
+
proof: fixed("/proof"),
|
|
818
|
+
traces: fixed("/v1/traces"),
|
|
819
|
+
/** Read-only GET of the buffered, already redacted spans the gateway copied to the sidecar. */
|
|
820
|
+
spans: fixed("/spans")
|
|
821
|
+
}).strict().prefault({}),
|
|
822
|
+
/** Bare paths an agent serves itself (SDK `/agentspec/health`); the sidecar probes them on the upstream. */
|
|
823
|
+
agentPaths: z.object({
|
|
824
|
+
health: fixed("/agentspec/health")
|
|
825
|
+
}).strict().prefault({}),
|
|
826
|
+
/** Control plane API routes shared by the CLI, the sidecar push and the control plane; `{name}` is the registered agent name. */
|
|
827
|
+
controlPlanePaths: z.object({
|
|
828
|
+
/** POST: registers an agent and returns its per-agent token. */
|
|
829
|
+
register: fixed("/api/v1/register"),
|
|
830
|
+
/** POST: the SDK push-mode heartbeat, authenticated with the agent token from `register`. */
|
|
831
|
+
heartbeat: fixed("/api/v1/heartbeat"),
|
|
832
|
+
declaredAudit: fixed("/api/v1/agents/{name}/declared-audit"),
|
|
833
|
+
/** POST: the sidecar's declared audit (at start and on a manifest hash change) and its own evidence records (ADR-0008). */
|
|
834
|
+
sidecarPush: fixed("/api/v1/agents/{name}/sidecar-push"),
|
|
835
|
+
/** GET: the latest stored EvidenceRecord per (ruleId, checkId, source) of an agent, as `{ records }`; read by the mcp-server. */
|
|
836
|
+
evidence: fixed("/api/v1/agents/{name}/evidence")
|
|
837
|
+
}).strict().prefault({}),
|
|
838
|
+
/** Names inside the pod patch that every injection path must use (ADR-0028). */
|
|
839
|
+
podPatch: z.object({
|
|
840
|
+
sidecarContainer: fixed("agentspec-sidecar"),
|
|
841
|
+
/** Pod annotation the webhook writes its PodSpec findings to (JSON list of `{code, subject, message}`). */
|
|
842
|
+
podspecFindingsAnnotation: fixed("agentspec.io/podspec-findings"),
|
|
843
|
+
/** Pod annotation naming the agent; the webhook reads the manifest ConfigMap `<name>-manifest` from it, and `--deploy k8s` writes it. */
|
|
844
|
+
agentNameAnnotation: fixed("agentspec.io/agent-name"),
|
|
845
|
+
manifestVolume: fixed("agentspec-manifest"),
|
|
846
|
+
proxyPortName: fixed("proxy"),
|
|
847
|
+
controlPortName: fixed("control"),
|
|
848
|
+
gatewayContainer: fixed("agentspec-gateway"),
|
|
849
|
+
gatewayConfigInitContainer: fixed("agentspec-gateway-config"),
|
|
850
|
+
/** Sidecar subcommand the init container runs: `node <entrypoint> <this> <manifest> --out <path>`. */
|
|
851
|
+
gatewayConfigCommand: fixed("gateway-config"),
|
|
852
|
+
gatewayConfigVolume: fixed("agentspec-gateway-config"),
|
|
853
|
+
gatewayDataVolume: fixed("agentspec-gateway-data"),
|
|
854
|
+
/** Secret `<metadata.name><suffix>` holding the `sidecarToken` from `POST /api/v1/register`; the webhook gives it to the sidecar only, as `env.agentKey`, when a control plane URL is set. */
|
|
855
|
+
agentTokenSecretSuffix: fixed("-agentspec-token"),
|
|
856
|
+
agentTokenSecretKey: fixed("token"),
|
|
857
|
+
/** Loopback address the agent and sidecar reach each other and the gateway on. */
|
|
858
|
+
loopback: fixed("127.0.0.1"),
|
|
859
|
+
/** Gateway container sizing (ADR-0025): request 25m/32Mi, limit 128Mi. */
|
|
860
|
+
gatewayCpuRequest: fixed("25m"),
|
|
861
|
+
gatewayMemoryRequest: fixed("32Mi"),
|
|
862
|
+
gatewayMemoryLimit: fixed("128Mi")
|
|
863
|
+
}).strict().prefault({}),
|
|
864
|
+
/** Admission webhook the operator serves (ADR-0028). */
|
|
865
|
+
webhook: z.object({
|
|
866
|
+
/** Path kube-apiserver posts to; kopf routes `POST /<handler id>`, so this is the handler id with a leading slash. Hyphenated: kube-apiserver rejects an underscore in a service path segment (RFC 1123). */
|
|
867
|
+
injectPath: fixed("/inject-sidecar"),
|
|
868
|
+
/** Namespace label that selects the namespaces whose pods are injected. */
|
|
869
|
+
namespaceLabel: fixed("agentspec.io/enabled")
|
|
870
|
+
}).strict().prefault({}),
|
|
871
|
+
/** Known `EvidenceRecord.reason.code` values (ADR-0006). */
|
|
872
|
+
reasonCodes: z.array(z.string()).default(REASON_CODES),
|
|
873
|
+
/** Severity weights for coverage and the verified score (ADR-0007). */
|
|
874
|
+
severityWeights: z.object({
|
|
875
|
+
critical: fixed(4),
|
|
876
|
+
high: fixed(3),
|
|
877
|
+
medium: fixed(2),
|
|
878
|
+
low: fixed(1),
|
|
879
|
+
info: fixed(0)
|
|
880
|
+
}).strict().prefault({}),
|
|
881
|
+
/** Minimum verified percentage per grade; below `D` is F (ADR-0007). */
|
|
882
|
+
gradeBands: z.object({ A: fixed(90), B: fixed(75), C: fixed(60), D: fixed(45) }).strict().prefault({}),
|
|
883
|
+
/** Severity-weighted coverage percentage below which no grade is shown (ADR-0007). */
|
|
884
|
+
coverageThreshold: fixed(60),
|
|
885
|
+
/** Rule evidence registry: what can prove each audit rule (ADR-0040). */
|
|
886
|
+
ruleEvidence: z.record(z.string(), RuleEvidenceEntry).default(RULE_EVIDENCE),
|
|
887
|
+
/** Evidence point → target → the rule its records serve; each rule's registry entry lists a source for that point. */
|
|
888
|
+
evidencePoints: z.record(z.string(), z.record(z.string(), z.string())).default({
|
|
889
|
+
guardrail: { input: "SEC-LLM-01", output: "SEC-LLM-02" },
|
|
890
|
+
tool: { local: "TOOL-01" },
|
|
891
|
+
mcpClient: { "tools/list": "SEC-LLM-07", "tools/call": "TOOL-01" },
|
|
892
|
+
memoryWrite: { scrub: "MEM-01", ttl: "MEM-02", audit: "MEM-03" },
|
|
893
|
+
vectorStore: { namespace: "MEM-04" },
|
|
894
|
+
logging: { json: "OBS-02" }
|
|
895
|
+
}),
|
|
896
|
+
/** Records an evidence point aggregate keeps per window. */
|
|
897
|
+
evidencePointRecordLimit: fixed(MAX_POINT_RECORDS),
|
|
898
|
+
/** A heartbeat above this is trimmed to counts before it is sent. */
|
|
899
|
+
heartbeatMaxBytes: fixed(64 * 1024),
|
|
900
|
+
/** OTLP: resource attributes every exported span carries, and the standard endpoint env var. */
|
|
901
|
+
otlp: z.object({
|
|
902
|
+
resourceAgentName: fixed("agentspec.agent.name"),
|
|
903
|
+
resourceManifestSha256: fixed("agentspec.manifest.sha256"),
|
|
904
|
+
endpointEnv: fixed("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
|
905
|
+
/** Resource attribute naming the span's producer, and the value agentgateway exports (its default); only such spans are gateway evidence. */
|
|
906
|
+
resourceServiceName: fixed("service.name"),
|
|
907
|
+
gatewayServiceName: fixed("agentgateway")
|
|
908
|
+
}).strict().prefault({})
|
|
909
|
+
}).strict();
|
|
910
|
+
var CONTRACT = ContractSchema.parse({});
|
|
911
|
+
|
|
912
|
+
// src/schema/evaluation.ts
|
|
913
|
+
var REPLACED_BY = {
|
|
914
|
+
hallucination: "grounded",
|
|
915
|
+
faithfulness: "grounded"
|
|
916
|
+
};
|
|
917
|
+
function canonicalMetric(metric) {
|
|
918
|
+
return REPLACED_BY[metric] ?? metric;
|
|
919
|
+
}
|
|
920
|
+
function evaluationWarnings(manifest) {
|
|
921
|
+
const evaluation = manifest.spec.evaluation;
|
|
922
|
+
if (!evaluation) return [];
|
|
923
|
+
const warnings = [];
|
|
924
|
+
if (evaluation.framework !== void 0) {
|
|
925
|
+
warnings.push(
|
|
926
|
+
"spec.evaluation.framework is deprecated: nothing reads it and AgentSpec computes evaluation evidence itself. Remove it; it is rejected in 0.6.0."
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
const declared2 = [
|
|
930
|
+
...evaluation.metrics ?? [],
|
|
931
|
+
...evaluation.live?.metrics ?? [],
|
|
932
|
+
...(evaluation.datasets ?? []).flatMap((d) => d.metrics ?? [])
|
|
933
|
+
];
|
|
934
|
+
for (const metric of new Set(declared2)) {
|
|
935
|
+
const replacement = canonicalMetric(metric);
|
|
936
|
+
if (replacement !== metric) {
|
|
937
|
+
warnings.push(
|
|
938
|
+
`spec.evaluation metric "${metric}" is deprecated and is read as "${replacement}". Rename it to "${replacement}"; it is rejected in 0.6.0.`
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
warnings.push(
|
|
943
|
+
...thresholdWarnings("spec.evaluation.thresholds", evaluation.thresholds),
|
|
944
|
+
...thresholdWarnings("spec.evaluation.live.thresholds", evaluation.live?.thresholds)
|
|
945
|
+
);
|
|
946
|
+
return warnings;
|
|
947
|
+
}
|
|
948
|
+
function thresholdWarnings(path, thresholds) {
|
|
949
|
+
return Object.keys(thresholds ?? {}).flatMap((name) => {
|
|
950
|
+
const replacement = canonicalMetric(name);
|
|
951
|
+
if (replacement === name) return [];
|
|
952
|
+
const ignored = replacement in thresholds ? ` It is ignored: ${path}.${replacement} is also declared.` : "";
|
|
953
|
+
return [
|
|
954
|
+
`${path}.${name} is deprecated and is read as ${path}.${replacement}. Rename it to "${replacement}"; it is rejected in 0.6.0.${ignored}`
|
|
955
|
+
];
|
|
364
956
|
});
|
|
365
957
|
}
|
|
958
|
+
function assertRubricRefs(parsed) {
|
|
959
|
+
const evaluation = parsed?.spec?.evaluation;
|
|
960
|
+
const live = evaluation?.live;
|
|
961
|
+
for (const [path, custom] of [
|
|
962
|
+
["spec.evaluation.custom", evaluation?.custom],
|
|
963
|
+
["spec.evaluation.live.custom", live?.custom]
|
|
964
|
+
]) {
|
|
965
|
+
const rubric = custom?.rubric;
|
|
966
|
+
if (typeof rubric === "string" && !rubric.startsWith("$file:")) {
|
|
967
|
+
throw new Error(
|
|
968
|
+
`${path}.rubric must be a "$file:<path>" reference, got "${rubric}". Fix: write it as "$file:${rubric}" (path relative to agent.yaml).`
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
function normalizeEvaluation(manifest) {
|
|
974
|
+
const evaluation = manifest.spec.evaluation;
|
|
975
|
+
if (!evaluation) return manifest;
|
|
976
|
+
const metrics = (list) => list?.map(canonicalMetric);
|
|
977
|
+
const thresholds = (map) => {
|
|
978
|
+
if (!map) return map;
|
|
979
|
+
const out = {};
|
|
980
|
+
for (const [name, value] of Object.entries(map)) {
|
|
981
|
+
const key = canonicalMetric(name);
|
|
982
|
+
if (key === name || !(key in map)) out[key] = value;
|
|
983
|
+
}
|
|
984
|
+
return out;
|
|
985
|
+
};
|
|
986
|
+
return {
|
|
987
|
+
...manifest,
|
|
988
|
+
spec: {
|
|
989
|
+
...manifest.spec,
|
|
990
|
+
evaluation: {
|
|
991
|
+
...evaluation,
|
|
992
|
+
metrics: metrics(evaluation.metrics),
|
|
993
|
+
thresholds: thresholds(evaluation.thresholds),
|
|
994
|
+
datasets: evaluation.datasets?.map((d) => ({ ...d, metrics: metrics(d.metrics) })),
|
|
995
|
+
live: evaluation.live && {
|
|
996
|
+
...evaluation.live,
|
|
997
|
+
metrics: metrics(evaluation.live.metrics),
|
|
998
|
+
thresholds: thresholds(evaluation.live.thresholds)
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
366
1004
|
function detectRefType(value) {
|
|
367
1005
|
if (value.startsWith("$env:")) return "env";
|
|
368
1006
|
if (value.startsWith("$secret:")) return "secret";
|
|
@@ -408,8 +1046,9 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
408
1046
|
Use paths relative to agent.yaml location only`
|
|
409
1047
|
);
|
|
410
1048
|
}
|
|
1049
|
+
const readPath = checkedReadPath(value, absPath, opts.baseDir);
|
|
411
1050
|
try {
|
|
412
|
-
return readFileSync(
|
|
1051
|
+
return readFileSync(readPath, "utf-8");
|
|
413
1052
|
} catch {
|
|
414
1053
|
if (optional) return "";
|
|
415
1054
|
throw new Error(
|
|
@@ -431,6 +1070,29 @@ function resolveRef(value, opts, { optional = false } = {}) {
|
|
|
431
1070
|
}
|
|
432
1071
|
return value;
|
|
433
1072
|
}
|
|
1073
|
+
function checkedReadPath(ref, absPath, baseDir) {
|
|
1074
|
+
let realTarget;
|
|
1075
|
+
try {
|
|
1076
|
+
realTarget = realpathSync(absPath);
|
|
1077
|
+
} catch (err) {
|
|
1078
|
+
if (err.code === "ENOENT") return absPath;
|
|
1079
|
+
throw new Error(
|
|
1080
|
+
`Cannot resolve ${ref}
|
|
1081
|
+
${err.message}
|
|
1082
|
+
Fix the path: every folder in it must exist, be a folder and not loop back through symlinks`
|
|
1083
|
+
);
|
|
1084
|
+
}
|
|
1085
|
+
const realBase = realpathSync(baseDir);
|
|
1086
|
+
if (realTarget !== realBase && !realTarget.startsWith(realBase + sep)) {
|
|
1087
|
+
throw new Error(
|
|
1088
|
+
`Path traversal detected in ${ref}
|
|
1089
|
+
Resolved through a symlink to: ${realTarget}
|
|
1090
|
+
Which is outside baseDir: ${baseDir}
|
|
1091
|
+
Replace the symlink with a real folder inside the manifest directory, or move the file there`
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
return realTarget;
|
|
1095
|
+
}
|
|
434
1096
|
function resolveSecret(name, backend) {
|
|
435
1097
|
switch (backend) {
|
|
436
1098
|
case "env": {
|
|
@@ -457,7 +1119,7 @@ function resolveSecret(name, backend) {
|
|
|
457
1119
|
}
|
|
458
1120
|
function resolveRefs(obj, opts) {
|
|
459
1121
|
if (typeof obj === "string") {
|
|
460
|
-
return resolveRef(obj, opts
|
|
1122
|
+
return resolveRef(obj, opts);
|
|
461
1123
|
}
|
|
462
1124
|
if (Array.isArray(obj)) {
|
|
463
1125
|
return obj.map((item) => resolveRefs(item, opts));
|
|
@@ -514,13 +1176,15 @@ function loadManifest(filePath, opts = {}) {
|
|
|
514
1176
|
throw new Error(`Invalid YAML in ${absPath}
|
|
515
1177
|
${String(err)}`);
|
|
516
1178
|
}
|
|
1179
|
+
assertRubricRefs(parsed);
|
|
517
1180
|
const toValidate = opts.resolve === true ? resolveRefs(parsed, {
|
|
518
1181
|
baseDir,
|
|
519
1182
|
secretBackend: opts.secretBackend,
|
|
520
1183
|
failOnMissingEnv: opts.failOnMissingEnv ?? true
|
|
521
1184
|
}) : parsed;
|
|
522
|
-
const
|
|
523
|
-
|
|
1185
|
+
const declared2 = ManifestSchema.parse(toValidate);
|
|
1186
|
+
const warnings = evaluationWarnings(declared2);
|
|
1187
|
+
return { manifest: normalizeEvaluation(declared2), warnings, filePath: absPath, baseDir, raw };
|
|
524
1188
|
}
|
|
525
1189
|
function tryLoadManifest(filePath, opts = {}) {
|
|
526
1190
|
try {
|
|
@@ -567,8 +1231,8 @@ function isLatestVersion(raw) {
|
|
|
567
1231
|
function runEnvChecks(manifest, rawManifest) {
|
|
568
1232
|
const checks = [];
|
|
569
1233
|
const envRefs = collectEnvRefs(rawManifest);
|
|
570
|
-
const
|
|
571
|
-
for (const v of
|
|
1234
|
+
const declared2 = manifest.spec.requires?.envVars ?? [];
|
|
1235
|
+
for (const v of declared2) envRefs.add(v);
|
|
572
1236
|
for (const varName of envRefs) {
|
|
573
1237
|
const present = process.env[varName] !== void 0 && process.env[varName] !== "";
|
|
574
1238
|
checks.push({
|
|
@@ -605,21 +1269,53 @@ var PROVIDER_ENDPOINTS = {
|
|
|
605
1269
|
openai: "https://api.openai.com/v1/models",
|
|
606
1270
|
anthropic: "https://api.anthropic.com/v1/models",
|
|
607
1271
|
groq: "https://api.groq.com/openai/v1/models",
|
|
608
|
-
|
|
609
|
-
google: "https://generativelanguage.googleapis.com",
|
|
1272
|
+
google: "https://generativelanguage.googleapis.com/v1beta/models",
|
|
610
1273
|
mistral: "https://api.mistral.ai/v1/models",
|
|
611
|
-
cohere: "https://api.cohere.
|
|
1274
|
+
cohere: "https://api.cohere.com/v1/models",
|
|
612
1275
|
together: "https://api.together.xyz/v1/models",
|
|
613
1276
|
fireworks: "https://api.fireworks.ai/inference/v1/models"
|
|
614
1277
|
};
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
1278
|
+
var AZURE_API_VERSION = "2024-10-21";
|
|
1279
|
+
var AZURE_RESOURCE_HOSTS = [".openai.azure.com", ".cognitiveservices.azure.com", ".openai.azure.us", ".openai.azure.cn"];
|
|
1280
|
+
function isAzureResourceEndpoint(base) {
|
|
1281
|
+
const { hostname, pathname } = new URL(base);
|
|
1282
|
+
return AZURE_RESOURCE_HOSTS.some((suffix) => hostname.endsWith(suffix)) && pathname === "/";
|
|
1283
|
+
}
|
|
1284
|
+
function probeUrl(provider, baseUrl) {
|
|
1285
|
+
const key = provider.toLowerCase();
|
|
1286
|
+
const nativeUrl = PROVIDER_ENDPOINTS[key];
|
|
1287
|
+
if (!nativeUrl && !(key === "azure" && baseUrl)) return void 0;
|
|
1288
|
+
if (!baseUrl) return { url: nativeUrl, native: true };
|
|
1289
|
+
const base = resolveRef(baseUrl, { baseDir: "." }).replace(/\/+$/, "");
|
|
1290
|
+
if (key === "azure" && isAzureResourceEndpoint(base)) {
|
|
1291
|
+
return { url: `${base}/openai/models?api-version=${AZURE_API_VERSION}`, native: false, azureResource: true };
|
|
1292
|
+
}
|
|
1293
|
+
if (key === "anthropic") return { url: `${base}/v1/models`, native: false };
|
|
1294
|
+
return { url: `${base}/models`, native: false };
|
|
1295
|
+
}
|
|
1296
|
+
function unverifiableProbe(provider) {
|
|
1297
|
+
if (provider.toLowerCase() === "azure") {
|
|
618
1298
|
return {
|
|
619
|
-
|
|
620
|
-
|
|
1299
|
+
outcome: "not_verifiable",
|
|
1300
|
+
error: "azure keys are checked against your own resource endpoint, and spec.model.baseUrl is not set to name it",
|
|
1301
|
+
remediation: "Set spec.model.baseUrl to your Azure resource endpoint (https://<resource>.openai.azure.com) so the key can be probed, or verify it out of band"
|
|
621
1302
|
};
|
|
622
1303
|
}
|
|
1304
|
+
return { outcome: "not_verifiable", error: `unknown provider "${provider}"` };
|
|
1305
|
+
}
|
|
1306
|
+
async function checkModelEndpoint(provider, apiKey, baseUrl, timeoutMs = 5e3) {
|
|
1307
|
+
let target;
|
|
1308
|
+
try {
|
|
1309
|
+
target = probeUrl(provider, baseUrl);
|
|
1310
|
+
} catch (err) {
|
|
1311
|
+
return {
|
|
1312
|
+
outcome: "unreachable",
|
|
1313
|
+
error: err instanceof Error ? err.message.split("\n")[0] : String(err),
|
|
1314
|
+
remediation: `Set the variable behind spec.model baseUrl (${baseUrl}) or remove the override`
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
if (!target) return unverifiableProbe(provider);
|
|
1318
|
+
const { url, native, azureResource } = target;
|
|
623
1319
|
const start = Date.now();
|
|
624
1320
|
const controller = new AbortController();
|
|
625
1321
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -633,19 +1329,30 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
633
1329
|
headers["anthropic-version"] = "2023-06-01";
|
|
634
1330
|
delete headers["Authorization"];
|
|
635
1331
|
}
|
|
1332
|
+
if (azureResource) {
|
|
1333
|
+
headers["api-key"] = apiKey;
|
|
1334
|
+
delete headers["Authorization"];
|
|
1335
|
+
}
|
|
1336
|
+
const isGemini = provider.toLowerCase() === "google" && native;
|
|
1337
|
+
if (isGemini) {
|
|
1338
|
+
headers["x-goog-api-key"] = apiKey;
|
|
1339
|
+
delete headers["Authorization"];
|
|
1340
|
+
}
|
|
636
1341
|
const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
|
|
637
1342
|
const latencyMs = Date.now() - start;
|
|
638
|
-
const
|
|
639
|
-
return {
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
1343
|
+
const geminiBadKey = isGemini && res.status === 400;
|
|
1344
|
+
if (res.status === 401 || res.status === 403 || geminiBadKey) return { outcome: "rejected", latencyMs, error: `HTTP ${res.status}` };
|
|
1345
|
+
if (res.status >= 500) return { outcome: "unreachable", latencyMs, error: `HTTP ${res.status}` };
|
|
1346
|
+
if (res.status === 404 || res.status === 405 || res.status === 410) {
|
|
1347
|
+
const { origin, pathname } = new URL(url);
|
|
1348
|
+
return { outcome: "unreachable", latencyMs, error: res.status === 404 ? "HTTP 404 (endpoint not found)" : `HTTP ${res.status} (not a models endpoint)`, remediation: `Check that ${origin}${pathname} is the provider's models endpoint${native ? "" : " (fix spec.model.baseUrl)"}` };
|
|
1349
|
+
}
|
|
1350
|
+
return { outcome: "ok", latencyMs };
|
|
644
1351
|
} catch (err) {
|
|
645
1352
|
const latencyMs = Date.now() - start;
|
|
646
1353
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
647
1354
|
return {
|
|
648
|
-
|
|
1355
|
+
outcome: "unreachable",
|
|
649
1356
|
latencyMs,
|
|
650
1357
|
error: isTimeout ? "Request timed out" : String(err)
|
|
651
1358
|
};
|
|
@@ -653,6 +1360,20 @@ async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
|
653
1360
|
clearTimeout(timeout);
|
|
654
1361
|
}
|
|
655
1362
|
}
|
|
1363
|
+
function modelProbeCheck(probe, fields, label, provider, keyRef) {
|
|
1364
|
+
const keyName = keyRef.startsWith("$") ? keyRef : "[api-key]";
|
|
1365
|
+
const base = { ...fields, latencyMs: probe.latencyMs };
|
|
1366
|
+
switch (probe.outcome) {
|
|
1367
|
+
case "ok":
|
|
1368
|
+
return { ...base, status: "pass" };
|
|
1369
|
+
case "not_verifiable":
|
|
1370
|
+
return { ...base, status: "not_verifiable", message: `${label} cannot be verified: ${probe.error}`, remediation: probe.remediation ?? `Use a provider the health check knows (${Object.keys(PROVIDER_ENDPOINTS).join(", ")}) or verify ${provider} out of band` };
|
|
1371
|
+
case "rejected":
|
|
1372
|
+
return { ...base, status: "fail", message: `${label} for ${provider} rejected the API key: ${probe.error}`, remediation: `Check that ${keyName} holds a valid ${provider} API key` };
|
|
1373
|
+
default:
|
|
1374
|
+
return { ...base, status: "fail", message: `${label} for ${provider} is unreachable: ${probe.error}`, remediation: probe.remediation ?? `Check that ${keyName} is set and the ${provider} API is reachable` };
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
656
1377
|
function resolveApiKey(ref) {
|
|
657
1378
|
if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
|
|
658
1379
|
if (ref.startsWith("$")) return "";
|
|
@@ -662,16 +1383,14 @@ async function runModelChecks(model) {
|
|
|
662
1383
|
const checks = [];
|
|
663
1384
|
const apiKey = resolveApiKey(model.apiKey);
|
|
664
1385
|
if (apiKey) {
|
|
665
|
-
const
|
|
666
|
-
checks.push(
|
|
667
|
-
|
|
668
|
-
category: "model",
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
|
|
674
|
-
});
|
|
1386
|
+
const probe = await checkModelEndpoint(model.provider, apiKey, model.baseUrl);
|
|
1387
|
+
checks.push(modelProbeCheck(
|
|
1388
|
+
probe,
|
|
1389
|
+
{ id: `model:${model.provider}/${model.id}`, category: "model", severity: "error" },
|
|
1390
|
+
"Model endpoint",
|
|
1391
|
+
model.provider,
|
|
1392
|
+
model.apiKey
|
|
1393
|
+
));
|
|
675
1394
|
} else {
|
|
676
1395
|
checks.push({
|
|
677
1396
|
id: `model:${model.provider}/${model.id}`,
|
|
@@ -681,32 +1400,6 @@ async function runModelChecks(model) {
|
|
|
681
1400
|
message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
|
|
682
1401
|
});
|
|
683
1402
|
}
|
|
684
|
-
if (model.fallback) {
|
|
685
|
-
const fallbackKey = resolveApiKey(model.fallback.apiKey);
|
|
686
|
-
if (fallbackKey) {
|
|
687
|
-
const { reachable, latencyMs, error } = await checkModelEndpoint(
|
|
688
|
-
model.fallback.provider,
|
|
689
|
-
fallbackKey
|
|
690
|
-
);
|
|
691
|
-
checks.push({
|
|
692
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
693
|
-
category: "model-fallback",
|
|
694
|
-
status: reachable ? "pass" : "fail",
|
|
695
|
-
severity: "warning",
|
|
696
|
-
latencyMs,
|
|
697
|
-
message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
|
|
698
|
-
remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
|
|
699
|
-
});
|
|
700
|
-
} else {
|
|
701
|
-
checks.push({
|
|
702
|
-
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
703
|
-
category: "model-fallback",
|
|
704
|
-
status: "skip",
|
|
705
|
-
severity: "warning",
|
|
706
|
-
message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
|
|
707
|
-
});
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
1403
|
return checks;
|
|
711
1404
|
}
|
|
712
1405
|
|
|
@@ -718,7 +1411,7 @@ async function runMcpChecks(servers) {
|
|
|
718
1411
|
const url = server.url;
|
|
719
1412
|
if (url && !url.startsWith("$")) {
|
|
720
1413
|
const check = await checkHttpMcpServer(server.name, url);
|
|
721
|
-
checks.push(check);
|
|
1414
|
+
checks.push(server.transport === "sse" ? withLegacySseHint(check) : check);
|
|
722
1415
|
} else {
|
|
723
1416
|
checks.push({
|
|
724
1417
|
id: `mcp:${server.name}`,
|
|
@@ -735,37 +1428,145 @@ async function runMcpChecks(servers) {
|
|
|
735
1428
|
}
|
|
736
1429
|
return checks;
|
|
737
1430
|
}
|
|
1431
|
+
var MCP_PROTOCOL_VERSION = "2025-03-26";
|
|
1432
|
+
var HANDSHAKE_TIMEOUT_MS = 5e3;
|
|
1433
|
+
var INITIALIZE_REQUEST_ID = 1;
|
|
1434
|
+
var MAX_SERVER_TEXT = 200;
|
|
1435
|
+
var PROTOCOL_VERSION_FORMAT = /^\d{4}-\d{2}-\d{2}$/;
|
|
738
1436
|
async function checkHttpMcpServer(name, url) {
|
|
739
1437
|
const controller = new AbortController();
|
|
740
|
-
const timeout = setTimeout(() => controller.abort(),
|
|
1438
|
+
const timeout = setTimeout(() => controller.abort(), HANDSHAKE_TIMEOUT_MS);
|
|
741
1439
|
const start = Date.now();
|
|
1440
|
+
const headers = {
|
|
1441
|
+
"content-type": "application/json",
|
|
1442
|
+
accept: "application/json, text/event-stream"
|
|
1443
|
+
};
|
|
742
1444
|
try {
|
|
743
|
-
const res = await fetch(url, {
|
|
1445
|
+
const res = await fetch(url, {
|
|
1446
|
+
method: "POST",
|
|
1447
|
+
headers,
|
|
1448
|
+
signal: controller.signal,
|
|
1449
|
+
body: JSON.stringify({
|
|
1450
|
+
jsonrpc: "2.0",
|
|
1451
|
+
id: INITIALIZE_REQUEST_ID,
|
|
1452
|
+
method: "initialize",
|
|
1453
|
+
params: {
|
|
1454
|
+
protocolVersion: MCP_PROTOCOL_VERSION,
|
|
1455
|
+
capabilities: {},
|
|
1456
|
+
clientInfo: { name: "agentspec-health", version: "1.0.0" }
|
|
1457
|
+
}
|
|
1458
|
+
})
|
|
1459
|
+
});
|
|
1460
|
+
if (!res.ok) {
|
|
1461
|
+
await res.body?.cancel().catch(() => void 0);
|
|
1462
|
+
return failedHandshake(name, Date.now() - start, `returned HTTP ${res.status} to initialize`);
|
|
1463
|
+
}
|
|
1464
|
+
const parsed = await readInitializeResult(res);
|
|
744
1465
|
const latencyMs = Date.now() - start;
|
|
1466
|
+
if ("error" in parsed) {
|
|
1467
|
+
return failedHandshake(name, latencyMs, parsed.error);
|
|
1468
|
+
}
|
|
1469
|
+
const result = parsed.result;
|
|
1470
|
+
const sessionId = res.headers.get("mcp-session-id");
|
|
1471
|
+
if (sessionId) headers["mcp-session-id"] = sessionId;
|
|
1472
|
+
headers["mcp-protocol-version"] = result.protocolVersion;
|
|
1473
|
+
await sendBestEffort(url, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) }, controller.signal);
|
|
1474
|
+
if (sessionId) await sendBestEffort(url, { method: "DELETE", headers }, controller.signal);
|
|
1475
|
+
const serverName = clip(result.serverInfo?.name);
|
|
1476
|
+
const serverVersion = clip(result.serverInfo?.version);
|
|
1477
|
+
const info = serverName ? ` (${serverName}${serverVersion ? ` ${serverVersion}` : ""})` : "";
|
|
745
1478
|
return {
|
|
746
1479
|
id: `mcp:${name}`,
|
|
747
1480
|
category: "mcp",
|
|
748
|
-
status:
|
|
1481
|
+
status: "pass",
|
|
749
1482
|
severity: "warning",
|
|
750
1483
|
latencyMs,
|
|
751
|
-
message:
|
|
1484
|
+
message: `MCP server ${name}${info} initialized, protocolVersion ${clip(result.protocolVersion)}`
|
|
752
1485
|
};
|
|
753
1486
|
} catch (err) {
|
|
754
1487
|
const latencyMs = Date.now() - start;
|
|
755
1488
|
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
756
|
-
return
|
|
757
|
-
|
|
758
|
-
category: "mcp",
|
|
759
|
-
status: "fail",
|
|
760
|
-
severity: "warning",
|
|
1489
|
+
return failedHandshake(
|
|
1490
|
+
name,
|
|
761
1491
|
latencyMs,
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
};
|
|
1492
|
+
isTimeout ? "health check timed out" : `unreachable: ${redactUrlCredentials(String(err))}`
|
|
1493
|
+
);
|
|
765
1494
|
} finally {
|
|
766
1495
|
clearTimeout(timeout);
|
|
767
1496
|
}
|
|
768
1497
|
}
|
|
1498
|
+
async function sendBestEffort(url, init, signal) {
|
|
1499
|
+
await fetch(url, { ...init, signal }).then((r) => r.body?.cancel()).catch(() => void 0);
|
|
1500
|
+
}
|
|
1501
|
+
function withLegacySseHint(check) {
|
|
1502
|
+
if (check.status !== "fail") return check;
|
|
1503
|
+
return {
|
|
1504
|
+
...check,
|
|
1505
|
+
remediation: `${check.remediation ?? ""} Note: transport "sse" is probed with the streamable-HTTP handshake; legacy HTTP+SSE servers are not supported by this check.`.trim()
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
function redactUrlCredentials(text) {
|
|
1509
|
+
return text.replace(/(\w+:\/\/)[^/\s]*@/g, "$1***@");
|
|
1510
|
+
}
|
|
1511
|
+
function failedHandshake(name, latencyMs, reason) {
|
|
1512
|
+
return {
|
|
1513
|
+
id: `mcp:${name}`,
|
|
1514
|
+
category: "mcp",
|
|
1515
|
+
status: "fail",
|
|
1516
|
+
severity: "warning",
|
|
1517
|
+
latencyMs,
|
|
1518
|
+
message: `MCP server ${name} ${reason}`,
|
|
1519
|
+
remediation: `Check that MCP server ${name} is running and that spec.mcp.servers[].url points at a streamable-HTTP MCP endpoint`
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
async function readInitializeResult(res) {
|
|
1523
|
+
const isSse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
|
|
1524
|
+
const text = isSse ? await readSseUntilInitialize(res) : await res.text();
|
|
1525
|
+
const payloads = isSse ? splitSseEvents(text) : [text];
|
|
1526
|
+
for (const payload of payloads) {
|
|
1527
|
+
const outcome = parseInitializeFrame(payload);
|
|
1528
|
+
if (outcome) return outcome;
|
|
1529
|
+
}
|
|
1530
|
+
return { error: "did not return a valid JSON-RPC initialize result" };
|
|
1531
|
+
}
|
|
1532
|
+
function clip(value) {
|
|
1533
|
+
return typeof value === "string" ? value.slice(0, MAX_SERVER_TEXT) : "";
|
|
1534
|
+
}
|
|
1535
|
+
function parseInitializeFrame(payload) {
|
|
1536
|
+
try {
|
|
1537
|
+
const msg = JSON.parse(payload);
|
|
1538
|
+
if (msg.jsonrpc !== "2.0" || msg.id !== INITIALIZE_REQUEST_ID) return null;
|
|
1539
|
+
if (msg.result && typeof msg.result.protocolVersion === "string") {
|
|
1540
|
+
return PROTOCOL_VERSION_FORMAT.test(msg.result.protocolVersion) ? { result: msg.result } : { error: "returned a malformed protocolVersion in its initialize result" };
|
|
1541
|
+
}
|
|
1542
|
+
if (msg.error) return { error: `rejected initialize: ${clip(msg.error.message) || "JSON-RPC error"}` };
|
|
1543
|
+
} catch {
|
|
1544
|
+
}
|
|
1545
|
+
return null;
|
|
1546
|
+
}
|
|
1547
|
+
function splitSseEvents(text) {
|
|
1548
|
+
return text.split(/\r?\n\r?\n/).map(
|
|
1549
|
+
(event) => event.split(/\r?\n/).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).join("\n")
|
|
1550
|
+
);
|
|
1551
|
+
}
|
|
1552
|
+
async function readSseUntilInitialize(res) {
|
|
1553
|
+
if (!res.body) return res.text();
|
|
1554
|
+
const reader = res.body.getReader();
|
|
1555
|
+
const decoder = new TextDecoder();
|
|
1556
|
+
let text = "";
|
|
1557
|
+
try {
|
|
1558
|
+
for (; ; ) {
|
|
1559
|
+
const { done, value } = await reader.read();
|
|
1560
|
+
if (done) break;
|
|
1561
|
+
text += decoder.decode(value, { stream: true });
|
|
1562
|
+
const completeEvents = text.split(/\r?\n\r?\n/).slice(0, -1);
|
|
1563
|
+
if (completeEvents.flatMap((e) => splitSseEvents(e)).some((p) => parseInitializeFrame(p))) break;
|
|
1564
|
+
}
|
|
1565
|
+
} finally {
|
|
1566
|
+
await reader.cancel().catch(() => void 0);
|
|
1567
|
+
}
|
|
1568
|
+
return text;
|
|
1569
|
+
}
|
|
769
1570
|
async function checkStdioMcpServer(server) {
|
|
770
1571
|
const command = server.command;
|
|
771
1572
|
if (!command) {
|
|
@@ -1283,7 +2084,7 @@ function computeHealthStatus(checks) {
|
|
|
1283
2084
|
const failed = checks.filter((c) => c.status === "fail");
|
|
1284
2085
|
const warnings = checks.filter((c) => c.status === "warn");
|
|
1285
2086
|
const passed = checks.filter((c) => c.status === "pass");
|
|
1286
|
-
const skipped = checks.filter((c) => c.status === "skip");
|
|
2087
|
+
const skipped = checks.filter((c) => c.status === "skip" || c.status === "not_verifiable");
|
|
1287
2088
|
const hasErrors = failed.some((c) => c.severity === "error");
|
|
1288
2089
|
const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
|
|
1289
2090
|
const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
|
|
@@ -1300,8 +2101,10 @@ function computeHealthStatus(checks) {
|
|
|
1300
2101
|
async function runHealthCheck(manifest, opts = {}) {
|
|
1301
2102
|
const checks = [];
|
|
1302
2103
|
const rawManifest = opts.rawManifest ?? manifest;
|
|
1303
|
-
|
|
1304
|
-
|
|
2104
|
+
if (opts.checkEnv !== false) {
|
|
2105
|
+
checks.push(...runEnvChecks(manifest, rawManifest));
|
|
2106
|
+
checks.push(...await runSecretChecks());
|
|
2107
|
+
}
|
|
1305
2108
|
if (opts.baseDir) {
|
|
1306
2109
|
checks.push(...runFileChecks(rawManifest, opts.baseDir));
|
|
1307
2110
|
}
|
|
@@ -1391,7 +2194,381 @@ async function checkA2aEndpoint(name, url) {
|
|
|
1391
2194
|
}
|
|
1392
2195
|
}
|
|
1393
2196
|
|
|
2197
|
+
// src/grader/index.ts
|
|
2198
|
+
var ADMITTED = {
|
|
2199
|
+
standard: ["independent", "sdk-observed"],
|
|
2200
|
+
strict: ["independent"]
|
|
2201
|
+
};
|
|
2202
|
+
var SOURCE_CLASS = {
|
|
2203
|
+
independent: "independent",
|
|
2204
|
+
"sdk-observed": "evidence-point",
|
|
2205
|
+
offline: "offline",
|
|
2206
|
+
declared: "declared"
|
|
2207
|
+
};
|
|
2208
|
+
var pct = (part, whole) => whole === 0 ? 0 : part / whole * 100;
|
|
2209
|
+
function isCalendarDate(value) {
|
|
2210
|
+
const [y, m, d] = value.slice(0, 10).split("-").map(Number);
|
|
2211
|
+
const date = /* @__PURE__ */ new Date(0);
|
|
2212
|
+
date.setUTCFullYear(y, m - 1, d);
|
|
2213
|
+
return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
|
|
2214
|
+
}
|
|
2215
|
+
function parseTimestamp(value, field) {
|
|
2216
|
+
const ms = Date.parse(value);
|
|
2217
|
+
const valid = TIMESTAMP_PATTERN.test(value) && !Number.isNaN(ms) && isCalendarDate(value) && !value.startsWith("0000") && !value.includes("T24:");
|
|
2218
|
+
if (!valid) {
|
|
2219
|
+
throw new Error(`grade: ${field} "${value}" is not a valid RFC 3339 timestamp. Fix the producer to emit a real date and time.`);
|
|
2220
|
+
}
|
|
2221
|
+
return ms;
|
|
2222
|
+
}
|
|
2223
|
+
function isExpired(r, nowMs) {
|
|
2224
|
+
return parseTimestamp(r.observedAt, `observedAt of the record for check "${r.checkId}"`) + r.ttlSeconds * 1e3 < nowMs;
|
|
2225
|
+
}
|
|
2226
|
+
function matchesRegistry(r) {
|
|
2227
|
+
const cls = SOURCE_CLASS[r.source];
|
|
2228
|
+
if (!r.ruleId || !cls || !Object.hasOwn(RULE_EVIDENCE, r.ruleId)) return true;
|
|
2229
|
+
const listed = RULE_EVIDENCE[r.ruleId].sources.filter((s) => s.class === cls);
|
|
2230
|
+
return listed.length > 0 && (r.collector == null || listed.some((s) => s.collector === r.collector));
|
|
2231
|
+
}
|
|
2232
|
+
function unverifiedReason(admitted, stale, notAdmitted, mismatched) {
|
|
2233
|
+
const own = admitted.find((r) => r.status === "not_verifiable");
|
|
2234
|
+
if (own?.reason) return own.reason;
|
|
2235
|
+
if (stale.length > 0) return { code: "stale" };
|
|
2236
|
+
if (notAdmitted.length > 0) return { code: "source_not_admitted" };
|
|
2237
|
+
if (mismatched.length > 0) return { code: "source_mismatch" };
|
|
2238
|
+
return { code: "no_evidence" };
|
|
2239
|
+
}
|
|
2240
|
+
function judgeRule(allRecords, policy, nowMs, unwiredFails) {
|
|
2241
|
+
if (unwiredFails) allRecords = [...allRecords.filter((r) => r.source !== "sdk-observed"), ...unwiredFails];
|
|
2242
|
+
const admittedSources = ADMITTED[policy];
|
|
2243
|
+
const registryMatched = [];
|
|
2244
|
+
const mismatched = [];
|
|
2245
|
+
for (const r of allRecords) (matchesRegistry(r) ? registryMatched : mismatched).push(r);
|
|
2246
|
+
const admittedRecords = registryMatched.filter((r) => admittedSources.includes(r.source));
|
|
2247
|
+
const notAdmitted = registryMatched.filter((r) => !admittedSources.includes(r.source));
|
|
2248
|
+
const stale = admittedRecords.filter((r) => isExpired(r, nowMs));
|
|
2249
|
+
const fresh = admittedRecords.filter((r) => !isExpired(r, nowMs));
|
|
2250
|
+
const decisive = fresh.filter((r) => r.status !== "not_verifiable");
|
|
2251
|
+
if (decisive.length === 0) {
|
|
2252
|
+
const unverified = unwiredFails ? { code: "not_wired" } : unverifiedReason(fresh, stale, notAdmitted, mismatched);
|
|
2253
|
+
return { covered: false, pass: false, independent: false, unverified };
|
|
2254
|
+
}
|
|
2255
|
+
return {
|
|
2256
|
+
covered: true,
|
|
2257
|
+
pass: decisive.every((r) => r.status === "pass"),
|
|
2258
|
+
independent: decisive.every((r) => r.source === "independent")
|
|
2259
|
+
};
|
|
2260
|
+
}
|
|
2261
|
+
function unwiredRules(points) {
|
|
2262
|
+
const notWired = /* @__PURE__ */ new Set();
|
|
2263
|
+
const wiredFails = /* @__PURE__ */ new Map();
|
|
2264
|
+
for (const p of points) {
|
|
2265
|
+
const ruleId = CONTRACT.evidencePoints[p.point]?.[p.target];
|
|
2266
|
+
if (!ruleId) continue;
|
|
2267
|
+
if (p.state === "not_wired") notWired.add(ruleId);
|
|
2268
|
+
else {
|
|
2269
|
+
const fails = p.records.filter((r) => r.ruleId === ruleId && r.status === "fail");
|
|
2270
|
+
wiredFails.set(ruleId, [...wiredFails.get(ruleId) ?? [], ...fails]);
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
return new Map([...notWired].map((id) => [id, wiredFails.get(id) ?? []]));
|
|
2274
|
+
}
|
|
2275
|
+
function judgeDerived(parents) {
|
|
2276
|
+
const open = parents.find((v) => !v.covered);
|
|
2277
|
+
const failed = parents.some((v) => v.covered && !v.pass);
|
|
2278
|
+
if (open && !failed) return open;
|
|
2279
|
+
const covered = parents.filter((v) => v.covered);
|
|
2280
|
+
return { covered: true, pass: !failed, independent: covered.every((v) => v.independent) };
|
|
2281
|
+
}
|
|
2282
|
+
function judgeRegistered(ruleId, byRule, policy, nowMs, unwired) {
|
|
2283
|
+
const derivedFrom = Object.hasOwn(RULE_EVIDENCE, ruleId) ? RULE_EVIDENCE[ruleId].derivedFrom : void 0;
|
|
2284
|
+
if (!derivedFrom?.length) return judgeRule(byRule.get(ruleId) ?? [], policy, nowMs, unwired.get(ruleId));
|
|
2285
|
+
return judgeDerived(derivedFrom.map((parent) => judgeRegistered(parent, byRule, policy, nowMs, unwired)));
|
|
2286
|
+
}
|
|
2287
|
+
function groupByRule(records) {
|
|
2288
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
2289
|
+
for (const r of records) {
|
|
2290
|
+
if (!r.ruleId) continue;
|
|
2291
|
+
const bucket2 = byRule.get(r.ruleId);
|
|
2292
|
+
if (bucket2) bucket2.push(r);
|
|
2293
|
+
else byRule.set(r.ruleId, [r]);
|
|
2294
|
+
}
|
|
2295
|
+
return byRule;
|
|
2296
|
+
}
|
|
2297
|
+
function scoreToGrade(score) {
|
|
2298
|
+
const { A, B, C, D } = CONTRACT.gradeBands;
|
|
2299
|
+
if (score >= A) return "A";
|
|
2300
|
+
if (score >= B) return "B";
|
|
2301
|
+
if (score >= C) return "C";
|
|
2302
|
+
if (score >= D) return "D";
|
|
2303
|
+
return "F";
|
|
2304
|
+
}
|
|
2305
|
+
function grade(records, applicableRules, policy, now, points = []) {
|
|
2306
|
+
const nowMs = parseTimestamp(now, "now");
|
|
2307
|
+
const byRule = groupByRule(records);
|
|
2308
|
+
const unwired = unwiredRules(points);
|
|
2309
|
+
let totalWeight = 0;
|
|
2310
|
+
let coveredWeight = 0;
|
|
2311
|
+
let passedWeight = 0;
|
|
2312
|
+
let independentWeight = 0;
|
|
2313
|
+
const unverified = [];
|
|
2314
|
+
for (const rule of applicableRules) {
|
|
2315
|
+
const weight = CONTRACT.severityWeights[rule.severity];
|
|
2316
|
+
const verdict = judgeRegistered(rule.id, byRule, policy, nowMs, unwired);
|
|
2317
|
+
totalWeight += weight;
|
|
2318
|
+
if (!verdict.covered) {
|
|
2319
|
+
unverified.push({ ruleId: rule.id, severity: rule.severity, reason: verdict.unverified });
|
|
2320
|
+
continue;
|
|
2321
|
+
}
|
|
2322
|
+
coveredWeight += weight;
|
|
2323
|
+
if (verdict.pass) passedWeight += weight;
|
|
2324
|
+
if (verdict.independent) independentWeight += weight;
|
|
2325
|
+
}
|
|
2326
|
+
const verified = Math.round(pct(passedWeight, coveredWeight));
|
|
2327
|
+
const coverage = Math.round(pct(coveredWeight, totalWeight));
|
|
2328
|
+
return {
|
|
2329
|
+
verified,
|
|
2330
|
+
coverage,
|
|
2331
|
+
independentShare: Math.round(pct(independentWeight, coveredWeight)),
|
|
2332
|
+
grade: coverage >= CONTRACT.coverageThreshold ? scoreToGrade(verified) : "Not graded",
|
|
2333
|
+
unverified
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2336
|
+
function formatVerified(result) {
|
|
2337
|
+
if (result.grade === "Not graded") return `Not graded (coverage ${result.coverage}%)`;
|
|
2338
|
+
return `${result.grade} - ${result.verified}% verified (${result.independentShare}% independent) - coverage ${result.coverage}%`;
|
|
2339
|
+
}
|
|
2340
|
+
var NATIVE_PROVIDERS = {
|
|
2341
|
+
openai: "openAI",
|
|
2342
|
+
anthropic: "anthropic",
|
|
2343
|
+
google: "gemini",
|
|
2344
|
+
gemini: "gemini",
|
|
2345
|
+
azure: "azure"
|
|
2346
|
+
};
|
|
2347
|
+
var OPENAI_COMPATIBLE_BASE_URLS = {
|
|
2348
|
+
groq: "https://api.groq.com/openai/v1",
|
|
2349
|
+
mistral: "https://api.mistral.ai/v1"
|
|
2350
|
+
};
|
|
2351
|
+
var CATALOG_PROVIDER_IDS = {
|
|
2352
|
+
openAI: "openai",
|
|
2353
|
+
anthropic: "anthropic",
|
|
2354
|
+
gemini: "gcp.gemini",
|
|
2355
|
+
azure: "azure"
|
|
2356
|
+
};
|
|
2357
|
+
var GATEWAY_NAME = "llm";
|
|
2358
|
+
var ONE_MONTH = "30d";
|
|
2359
|
+
function envRefName(value) {
|
|
2360
|
+
return /^\$env:(.+)$/.exec(value)?.[1];
|
|
2361
|
+
}
|
|
2362
|
+
function secretEnvName(name) {
|
|
2363
|
+
return `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
|
|
2364
|
+
}
|
|
2365
|
+
function keyEnvName(apiKey) {
|
|
2366
|
+
const env = envRefName(apiKey);
|
|
2367
|
+
if (env) return env;
|
|
2368
|
+
const secret = /^\$secret:(.+)$/.exec(apiKey);
|
|
2369
|
+
if (secret) return secretEnvName(secret[1]);
|
|
2370
|
+
throw new Error(
|
|
2371
|
+
'spec.model.apiKey must be a "$env:VAR" or "$secret:name" reference; a literal key would end up in the gateway config. Replace it with "$env:<VAR>".'
|
|
2372
|
+
);
|
|
2373
|
+
}
|
|
2374
|
+
function baseUrlValue(baseUrl) {
|
|
2375
|
+
const env = envRefName(baseUrl);
|
|
2376
|
+
return env ? `$${env}` : baseUrl;
|
|
2377
|
+
}
|
|
2378
|
+
function baseUrlEnvNames({ baseUrl }) {
|
|
2379
|
+
const name = baseUrl && envRefName(baseUrl);
|
|
2380
|
+
return name ? [name] : [];
|
|
2381
|
+
}
|
|
2382
|
+
function customProvider(provider, id, baseUrl) {
|
|
2383
|
+
const anthropic = provider === "anthropic";
|
|
2384
|
+
return {
|
|
2385
|
+
provider: { custom: { providerOverride: provider, formats: [{ type: anthropic ? "messages" : "completions" }] } },
|
|
2386
|
+
params: { model: id, baseUrl, ...anthropic ? { pathPrefix: "/v1" } : {} }
|
|
2387
|
+
};
|
|
2388
|
+
}
|
|
2389
|
+
function gatewayEnvNames(manifest) {
|
|
2390
|
+
const { model } = manifest.spec;
|
|
2391
|
+
const models = [model, ...model.fallback ? [model.fallback] : []];
|
|
2392
|
+
const names = models.flatMap((m) => [
|
|
2393
|
+
keyEnvName(m.apiKey),
|
|
2394
|
+
...baseUrlEnvNames(m),
|
|
2395
|
+
...NATIVE_PROVIDERS[m.provider] === "azure" && !m.baseUrl ? [CONTRACT.env.azureResourceName] : []
|
|
2396
|
+
]);
|
|
2397
|
+
return [...new Set(names)].sort();
|
|
2398
|
+
}
|
|
2399
|
+
function providerFields({ provider, id, baseUrl }) {
|
|
2400
|
+
if (baseUrl) return customProvider(provider, id, baseUrlValue(baseUrl));
|
|
2401
|
+
const native = NATIVE_PROVIDERS[provider];
|
|
2402
|
+
if (native === "azure") {
|
|
2403
|
+
return {
|
|
2404
|
+
provider: native,
|
|
2405
|
+
params: { model: id, azureResourceName: `$${CONTRACT.env.azureResourceName}`, azureResourceType: "openAI" }
|
|
2406
|
+
};
|
|
2407
|
+
}
|
|
2408
|
+
if (native) return { provider: native, params: { model: id } };
|
|
2409
|
+
const knownBaseUrl = OPENAI_COMPATIBLE_BASE_URLS[provider];
|
|
2410
|
+
if (knownBaseUrl) return customProvider(provider, id, knownBaseUrl);
|
|
2411
|
+
const supported = [...Object.keys(NATIVE_PROVIDERS), ...Object.keys(OPENAI_COMPATIBLE_BASE_URLS)].sort();
|
|
2412
|
+
throw new Error(`spec.model.provider "${provider}" has no gateway mapping. Supported: ${supported.join(", ")}.`);
|
|
2413
|
+
}
|
|
2414
|
+
function renderModel(model) {
|
|
2415
|
+
const { provider, params } = providerFields(model);
|
|
2416
|
+
return {
|
|
2417
|
+
name: `${model.provider}/${model.id}`,
|
|
2418
|
+
visibility: "internal",
|
|
2419
|
+
provider,
|
|
2420
|
+
params: { ...params, apiKey: `$${keyEnvName(model.apiKey)}` }
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
function renderVirtualModel(models) {
|
|
2424
|
+
return {
|
|
2425
|
+
name: models[0].id,
|
|
2426
|
+
routing: {
|
|
2427
|
+
failover: {
|
|
2428
|
+
targets: models.map((m, priority) => ({ model: `${m.provider}/${m.id}`, priority }))
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
function renderRetryPolicy(model) {
|
|
2434
|
+
const attempts = model.fallback?.maxRetries;
|
|
2435
|
+
if (!attempts) return {};
|
|
2436
|
+
return {
|
|
2437
|
+
policies: [{
|
|
2438
|
+
name: { name: "llm-retry", namespace: "default" },
|
|
2439
|
+
target: { gateway: { gatewayName: "default", gatewayNamespace: "default", listenerName: GATEWAY_NAME } },
|
|
2440
|
+
policy: { retry: { attempts, codes: [500, 502, 503, 504] } }
|
|
2441
|
+
}]
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2444
|
+
function renderBudgets(spec) {
|
|
2445
|
+
const amount = spec.model.costControls?.maxMonthlyUSD;
|
|
2446
|
+
if (amount === void 0) return {};
|
|
2447
|
+
return {
|
|
2448
|
+
budgets: [{
|
|
2449
|
+
name: "monthly",
|
|
2450
|
+
limit: { unit: "USD", amount },
|
|
2451
|
+
window: { rolling: ONE_MONTH },
|
|
2452
|
+
onBudgetExceeded: "Block"
|
|
2453
|
+
}]
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2456
|
+
function renderApiKeyPolicy(manifest) {
|
|
2457
|
+
return {
|
|
2458
|
+
mode: "strict",
|
|
2459
|
+
keys: [{
|
|
2460
|
+
key: `$${CONTRACT.env.gatewayKey}`,
|
|
2461
|
+
metadata: { name: manifest.metadata.name },
|
|
2462
|
+
allowedModels: [manifest.spec.model.id],
|
|
2463
|
+
...renderBudgets(manifest.spec)
|
|
2464
|
+
}]
|
|
2465
|
+
};
|
|
2466
|
+
}
|
|
2467
|
+
function bucket(type, size, fillInterval) {
|
|
2468
|
+
return { type, maxTokens: size, tokensPerFill: size, fillInterval };
|
|
2469
|
+
}
|
|
2470
|
+
function renderRateLimits(spec) {
|
|
2471
|
+
const { requestsPerMinute, requestsPerHour, tokensPerMinute } = spec.api?.rateLimit ?? {};
|
|
2472
|
+
const buckets = [
|
|
2473
|
+
...requestsPerMinute === void 0 ? [] : [bucket("requests", requestsPerMinute, "60s")],
|
|
2474
|
+
...requestsPerHour === void 0 ? [] : [bucket("requests", requestsPerHour, "1h")],
|
|
2475
|
+
...tokensPerMinute === void 0 ? [] : [bucket("tokens", tokensPerMinute, "60s")]
|
|
2476
|
+
];
|
|
2477
|
+
return buckets.length === 0 ? {} : { localRateLimit: buckets };
|
|
2478
|
+
}
|
|
2479
|
+
function renderRuntimeConfig(spec) {
|
|
2480
|
+
return {
|
|
2481
|
+
...spec.model.costControls?.maxMonthlyUSD !== void 0 && {
|
|
2482
|
+
database: { url: CONTRACT.paths.gatewayDatabase }
|
|
2483
|
+
},
|
|
2484
|
+
// agentgateway samples no request by default and keeps prompt and completion out of the
|
|
2485
|
+
// span; the sidecar needs every LLM call with both (ADR-0030), and redacts before buffering.
|
|
2486
|
+
tracing: {
|
|
2487
|
+
otlpEndpoint: `http://${CONTRACT.podPatch.loopback}:${CONTRACT.ports.controlPlane}`,
|
|
2488
|
+
otlpProtocol: "http",
|
|
2489
|
+
path: CONTRACT.sidecarPaths.traces,
|
|
2490
|
+
randomSampling: "true",
|
|
2491
|
+
fields: { add: { "gen_ai.prompt": "llm.prompt", "gen_ai.completion": "llm.completion" } }
|
|
2492
|
+
}
|
|
2493
|
+
};
|
|
2494
|
+
}
|
|
2495
|
+
function catalogProvider(model) {
|
|
2496
|
+
const { provider } = providerFields(model);
|
|
2497
|
+
return typeof provider === "string" ? CATALOG_PROVIDER_IDS[provider] : model.provider;
|
|
2498
|
+
}
|
|
2499
|
+
function catalogRates(pricing) {
|
|
2500
|
+
return Object.fromEntries(Object.entries(pricing).filter(([, v]) => v !== void 0).map(([k, v]) => [k, String(v)]));
|
|
2501
|
+
}
|
|
2502
|
+
function renderModelCatalog(models) {
|
|
2503
|
+
const providers = {};
|
|
2504
|
+
for (const model of models) {
|
|
2505
|
+
if (!model.pricing) continue;
|
|
2506
|
+
const key = catalogProvider(model);
|
|
2507
|
+
providers[key] ??= { models: {} };
|
|
2508
|
+
providers[key].models[model.id] = { rates: catalogRates(model.pricing) };
|
|
2509
|
+
}
|
|
2510
|
+
return Object.keys(providers).length === 0 ? {} : { modelCatalog: [{ inline: { providers } }] };
|
|
2511
|
+
}
|
|
2512
|
+
function renderGatewayConfig(manifest) {
|
|
2513
|
+
const { spec } = manifest;
|
|
2514
|
+
const models = [spec.model, ...spec.model.fallback ? [spec.model.fallback] : []];
|
|
2515
|
+
const config = {
|
|
2516
|
+
config: { ...renderRuntimeConfig(spec), ...renderModelCatalog(models) },
|
|
2517
|
+
gateways: { [GATEWAY_NAME]: { port: CONTRACT.ports.gateway, bindAddress: CONTRACT.podPatch.loopback } },
|
|
2518
|
+
llm: {
|
|
2519
|
+
gateways: GATEWAY_NAME,
|
|
2520
|
+
models: models.map(renderModel),
|
|
2521
|
+
virtualModels: [renderVirtualModel(models)],
|
|
2522
|
+
policies: {
|
|
2523
|
+
apiKey: renderApiKeyPolicy(manifest),
|
|
2524
|
+
...renderRateLimits(spec)
|
|
2525
|
+
}
|
|
2526
|
+
},
|
|
2527
|
+
...renderRetryPolicy(spec.model)
|
|
2528
|
+
};
|
|
2529
|
+
return dump(config, { sortKeys: true, lineWidth: -1, noRefs: true });
|
|
2530
|
+
}
|
|
2531
|
+
function describeManifestError(err) {
|
|
2532
|
+
if (!(err instanceof Error)) return String(err);
|
|
2533
|
+
if (!err.message.startsWith("[")) {
|
|
2534
|
+
return err.message.split("\n")[0];
|
|
2535
|
+
}
|
|
2536
|
+
try {
|
|
2537
|
+
const issues = JSON.parse(err.message);
|
|
2538
|
+
return issues.map((i) => `${i.path.join(".") || "root"}: ${i.message}`).join("\n");
|
|
2539
|
+
} catch {
|
|
2540
|
+
return err.message.split("\n")[0];
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
|
|
1394
2544
|
// src/audit/rules/model.rules.ts
|
|
2545
|
+
var isLatestModelId = (id) => id === "latest" || id.endsWith(":latest");
|
|
2546
|
+
function fallbackRetryResult(model) {
|
|
2547
|
+
if (!model.fallback) return { pass: true };
|
|
2548
|
+
const { maxRetries } = model.fallback;
|
|
2549
|
+
const { policies } = renderRetryPolicy(model);
|
|
2550
|
+
const attempts = policies ? policies[0].policy.retry.attempts : 0;
|
|
2551
|
+
const fail = (message, recommendation) => ({
|
|
2552
|
+
pass: false,
|
|
2553
|
+
message,
|
|
2554
|
+
path: "/spec/model/fallback/maxRetries",
|
|
2555
|
+
recommendation,
|
|
2556
|
+
references: []
|
|
2557
|
+
});
|
|
2558
|
+
if (maxRetries === void 0) {
|
|
2559
|
+
return fail(
|
|
2560
|
+
"Fallback model has no maxRetries configured: the gateway renders no retry, so the request that triggers failover fails",
|
|
2561
|
+
"Add maxRetries: 2 to spec.model.fallback"
|
|
2562
|
+
);
|
|
2563
|
+
}
|
|
2564
|
+
if (attempts !== maxRetries) {
|
|
2565
|
+
return fail(
|
|
2566
|
+
`The rendered gateway config sets ${attempts} retry attempts, but spec.model.fallback declares maxRetries ${maxRetries}`,
|
|
2567
|
+
"Redeploy the agent so its gateway config is rendered again from agent.yaml and retries maxRetries times"
|
|
2568
|
+
);
|
|
2569
|
+
}
|
|
2570
|
+
return { pass: true };
|
|
2571
|
+
}
|
|
1395
2572
|
var modelRules = [
|
|
1396
2573
|
{
|
|
1397
2574
|
id: "MODEL-01",
|
|
@@ -1399,9 +2576,7 @@ var modelRules = [
|
|
|
1399
2576
|
title: "Model fallback declared",
|
|
1400
2577
|
description: "A fallback model should be configured to handle rate limits and errors",
|
|
1401
2578
|
severity: "high",
|
|
1402
|
-
|
|
1403
|
-
proofTool: "LiteLLM chaos test",
|
|
1404
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
|
|
2579
|
+
awaitsEvidence: { code: "no_failover_observed", phase: 2 },
|
|
1405
2580
|
check(manifest) {
|
|
1406
2581
|
const hasFallback = !!manifest.spec.model.fallback;
|
|
1407
2582
|
return {
|
|
@@ -1419,17 +2594,14 @@ var modelRules = [
|
|
|
1419
2594
|
title: "Model version pinned",
|
|
1420
2595
|
description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
|
|
1421
2596
|
severity: "medium",
|
|
1422
|
-
evidenceLevel: "probed",
|
|
1423
|
-
proofTool: "agentspec health",
|
|
1424
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1425
2597
|
check(manifest) {
|
|
1426
2598
|
const id = manifest.spec.model.id;
|
|
1427
|
-
const isLatest = id
|
|
2599
|
+
const isLatest = isLatestModelId(id);
|
|
1428
2600
|
return {
|
|
1429
2601
|
pass: !isLatest,
|
|
1430
2602
|
message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
|
|
1431
2603
|
path: "/spec/model/id",
|
|
1432
|
-
recommendation: 'Pin a specific model version, e.g. "
|
|
2604
|
+
recommendation: 'Pin a specific model version, e.g. "openai/gpt-oss-20b" or "gpt-4-0125-preview"',
|
|
1433
2605
|
references: []
|
|
1434
2606
|
};
|
|
1435
2607
|
}
|
|
@@ -1440,9 +2612,7 @@ var modelRules = [
|
|
|
1440
2612
|
title: "Cost controls declared",
|
|
1441
2613
|
description: "Monthly cost limits prevent unexpected API bills",
|
|
1442
2614
|
severity: "medium",
|
|
1443
|
-
|
|
1444
|
-
proofTool: "LiteLLM Spend Tracking",
|
|
1445
|
-
proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
|
|
2615
|
+
awaitsEvidence: { code: "no_budget_evidence", phase: 2 },
|
|
1446
2616
|
check(manifest) {
|
|
1447
2617
|
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1448
2618
|
return {
|
|
@@ -1458,19 +2628,61 @@ var modelRules = [
|
|
|
1458
2628
|
id: "MODEL-04",
|
|
1459
2629
|
pack: "model-resilience",
|
|
1460
2630
|
title: "Fallback retry strategy configured",
|
|
1461
|
-
description: "Fallback should have maxRetries
|
|
2631
|
+
description: "Fallback should have maxRetries, and the rendered gateway config must retry exactly that many times",
|
|
2632
|
+
severity: "low",
|
|
2633
|
+
check(manifest) {
|
|
2634
|
+
return fallbackRetryResult(manifest.spec.model);
|
|
2635
|
+
}
|
|
2636
|
+
},
|
|
2637
|
+
{
|
|
2638
|
+
id: "MODEL-05",
|
|
2639
|
+
pack: "model-resilience",
|
|
2640
|
+
title: "Budgeted model is priced",
|
|
2641
|
+
description: "A monthly USD budget is only enforced for models the gateway can price",
|
|
2642
|
+
severity: "high",
|
|
2643
|
+
// Only the gateway knows its prices: MODEL-05 is graded from the cost on its spans (#494), never guessed from the model id (#495).
|
|
2644
|
+
check() {
|
|
2645
|
+
return { pass: true };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
];
|
|
2649
|
+
|
|
2650
|
+
// src/audit/rules/metadata.rules.ts
|
|
2651
|
+
var MAX_PRODUCTION_TEMPERATURE = 1.5;
|
|
2652
|
+
var metadataRules = [
|
|
2653
|
+
{
|
|
2654
|
+
id: "META-01",
|
|
2655
|
+
pack: "metadata-quality",
|
|
2656
|
+
title: "Agent description declared",
|
|
2657
|
+
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
1462
2658
|
severity: "low",
|
|
1463
|
-
evidenceLevel: "external",
|
|
1464
|
-
proofTool: "pytest-mockllm",
|
|
1465
|
-
proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
|
|
1466
2659
|
check(manifest) {
|
|
1467
|
-
|
|
1468
|
-
const
|
|
2660
|
+
const desc = manifest.metadata.description;
|
|
2661
|
+
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
2662
|
+
return {
|
|
2663
|
+
pass,
|
|
2664
|
+
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
2665
|
+
path: "/metadata/description",
|
|
2666
|
+
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
2667
|
+
references: []
|
|
2668
|
+
};
|
|
2669
|
+
}
|
|
2670
|
+
},
|
|
2671
|
+
{
|
|
2672
|
+
id: "META-02",
|
|
2673
|
+
pack: "metadata-quality",
|
|
2674
|
+
title: "Model temperature is production-safe",
|
|
2675
|
+
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
2676
|
+
severity: "medium",
|
|
2677
|
+
check(manifest) {
|
|
2678
|
+
const temperature = manifest.spec.model.parameters?.temperature;
|
|
2679
|
+
if (temperature === void 0) return { pass: true };
|
|
2680
|
+
const pass = temperature <= MAX_PRODUCTION_TEMPERATURE;
|
|
1469
2681
|
return {
|
|
1470
|
-
pass
|
|
1471
|
-
message:
|
|
1472
|
-
path: "/spec/model/
|
|
1473
|
-
recommendation: "
|
|
2682
|
+
pass,
|
|
2683
|
+
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above ${MAX_PRODUCTION_TEMPERATURE} produce unreliable outputs in production.`,
|
|
2684
|
+
path: "/spec/model/parameters/temperature",
|
|
2685
|
+
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
1474
2686
|
references: []
|
|
1475
2687
|
};
|
|
1476
2688
|
}
|
|
@@ -1485,9 +2697,6 @@ var securityRules = [
|
|
|
1485
2697
|
title: "Prompt injection guard: input guardrail required",
|
|
1486
2698
|
description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
|
|
1487
2699
|
severity: "high",
|
|
1488
|
-
evidenceLevel: "behavioral",
|
|
1489
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1490
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1491
2700
|
check(manifest) {
|
|
1492
2701
|
const hasGuardrail = manifest.spec.guardrails?.input?.some(
|
|
1493
2702
|
(g) => g.type === "prompt-injection" || g.type === "topic-filter"
|
|
@@ -1509,9 +2718,6 @@ var securityRules = [
|
|
|
1509
2718
|
title: "Insecure output handling: output guardrail required",
|
|
1510
2719
|
description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
|
|
1511
2720
|
severity: "high",
|
|
1512
|
-
evidenceLevel: "behavioral",
|
|
1513
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1514
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1515
2721
|
check(manifest) {
|
|
1516
2722
|
const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
|
|
1517
2723
|
return {
|
|
@@ -1531,9 +2737,6 @@ var securityRules = [
|
|
|
1531
2737
|
title: "Training/prompt data poisoning: prompt files referenced",
|
|
1532
2738
|
description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
|
|
1533
2739
|
severity: "medium",
|
|
1534
|
-
evidenceLevel: "probed",
|
|
1535
|
-
proofTool: "agentspec health",
|
|
1536
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1537
2740
|
check(manifest) {
|
|
1538
2741
|
const system = manifest.spec.prompts.system;
|
|
1539
2742
|
const usesFile = system.startsWith("$file:");
|
|
@@ -1551,21 +2754,16 @@ var securityRules = [
|
|
|
1551
2754
|
{
|
|
1552
2755
|
id: "SEC-LLM-04",
|
|
1553
2756
|
pack: "owasp-llm-top10",
|
|
1554
|
-
title: "Model DoS: rate limiting
|
|
2757
|
+
title: "Model DoS: rate limiting configured",
|
|
1555
2758
|
description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
|
|
1556
2759
|
severity: "medium",
|
|
1557
|
-
evidenceLevel: "external",
|
|
1558
|
-
proofTool: "k6 load test",
|
|
1559
|
-
proofToolUrl: "https://k6.io",
|
|
1560
2760
|
check(manifest) {
|
|
1561
|
-
const
|
|
1562
|
-
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1563
|
-
const pass = hasRateLimit || hasCostControls;
|
|
2761
|
+
const pass = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
|
|
1564
2762
|
return {
|
|
1565
2763
|
pass,
|
|
1566
|
-
message: pass ? void 0 : "No rate limiting
|
|
2764
|
+
message: pass ? void 0 : "No rate limiting declared.",
|
|
1567
2765
|
path: "/spec/api/rateLimit",
|
|
1568
|
-
recommendation: "Add spec.api.rateLimit.requestsPerMinute
|
|
2766
|
+
recommendation: "Add spec.api.rateLimit.requestsPerMinute",
|
|
1569
2767
|
references: [
|
|
1570
2768
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
|
|
1571
2769
|
]
|
|
@@ -1578,9 +2776,6 @@ var securityRules = [
|
|
|
1578
2776
|
title: "Supply chain: model provider and version pinned",
|
|
1579
2777
|
description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
|
|
1580
2778
|
severity: "medium",
|
|
1581
|
-
evidenceLevel: "probed",
|
|
1582
|
-
proofTool: "agentspec health",
|
|
1583
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1584
2779
|
check(manifest) {
|
|
1585
2780
|
const hasProvider = !!manifest.spec.model.provider;
|
|
1586
2781
|
const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
|
|
@@ -1602,9 +2797,6 @@ var securityRules = [
|
|
|
1602
2797
|
title: "Sensitive data disclosure: PII scrub in memory hygiene",
|
|
1603
2798
|
description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
|
|
1604
2799
|
severity: "critical",
|
|
1605
|
-
evidenceLevel: "external",
|
|
1606
|
-
proofTool: "Microsoft Presidio",
|
|
1607
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1608
2800
|
check(manifest) {
|
|
1609
2801
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1610
2802
|
const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1625,9 +2817,6 @@ var securityRules = [
|
|
|
1625
2817
|
title: "Insecure plugin design: tool annotations declared",
|
|
1626
2818
|
description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
|
|
1627
2819
|
severity: "medium",
|
|
1628
|
-
evidenceLevel: "external",
|
|
1629
|
-
proofTool: "Promptfoo",
|
|
1630
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
1631
2820
|
check(manifest) {
|
|
1632
2821
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1633
2822
|
const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
|
|
@@ -1648,9 +2837,7 @@ var securityRules = [
|
|
|
1648
2837
|
title: "Excessive agency: destructiveHint declared on tools",
|
|
1649
2838
|
description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
|
|
1650
2839
|
severity: "high",
|
|
1651
|
-
|
|
1652
|
-
proofTool: "Promptfoo",
|
|
1653
|
-
proofToolUrl: "https://promptfoo.dev",
|
|
2840
|
+
awaitsEvidence: { code: "no_approval_token_evidence", phase: 5 },
|
|
1654
2841
|
check(manifest) {
|
|
1655
2842
|
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1656
2843
|
const allHaveDestructiveHint = manifest.spec.tools.every(
|
|
@@ -1670,21 +2857,18 @@ var securityRules = [
|
|
|
1670
2857
|
{
|
|
1671
2858
|
id: "SEC-LLM-09",
|
|
1672
2859
|
pack: "owasp-llm-top10",
|
|
1673
|
-
title: "Overreliance: evaluation
|
|
2860
|
+
title: "Overreliance: evaluation + CI gate configured",
|
|
1674
2861
|
description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
|
|
1675
2862
|
severity: "medium",
|
|
1676
|
-
evidenceLevel: "probed",
|
|
1677
|
-
proofTool: "agentspec health",
|
|
1678
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1679
2863
|
check(manifest) {
|
|
1680
|
-
const hasEval =
|
|
2864
|
+
const hasEval = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0 || (manifest.spec.evaluation?.live?.metrics?.length ?? 0) > 0;
|
|
1681
2865
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1682
2866
|
const pass = hasEval && hasCiGate;
|
|
1683
2867
|
return {
|
|
1684
2868
|
pass,
|
|
1685
|
-
message: pass ? void 0 : !hasEval ? "No evaluation
|
|
2869
|
+
message: pass ? void 0 : !hasEval ? "No evaluation datasets or live evaluation configured." : "Evaluation CI gate is not enabled.",
|
|
1686
2870
|
path: "/spec/evaluation",
|
|
1687
|
-
recommendation: "Configure spec.evaluation.
|
|
2871
|
+
recommendation: "Configure spec.evaluation.datasets (or spec.evaluation.live) and set spec.evaluation.ciGate: true",
|
|
1688
2872
|
references: [
|
|
1689
2873
|
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
|
|
1690
2874
|
]
|
|
@@ -1697,9 +2881,6 @@ var securityRules = [
|
|
|
1697
2881
|
title: "Model theft: API keys use $secret, not $env",
|
|
1698
2882
|
description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
|
|
1699
2883
|
severity: "high",
|
|
1700
|
-
evidenceLevel: "probed",
|
|
1701
|
-
proofTool: "agentspec health",
|
|
1702
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1703
2884
|
check(manifest) {
|
|
1704
2885
|
const apiKey = manifest.spec.model.apiKey;
|
|
1705
2886
|
const usesSecret = apiKey.startsWith("$secret:");
|
|
@@ -1715,6 +2896,18 @@ var securityRules = [
|
|
|
1715
2896
|
]
|
|
1716
2897
|
};
|
|
1717
2898
|
}
|
|
2899
|
+
},
|
|
2900
|
+
{
|
|
2901
|
+
id: "TOOL-01",
|
|
2902
|
+
pack: "owasp-llm-top10",
|
|
2903
|
+
title: "Only declared tools are called",
|
|
2904
|
+
description: "The agent must call only tools declared in spec.tools; undeclared calls are excessive agency",
|
|
2905
|
+
severity: "high",
|
|
2906
|
+
awaitsEvidence: { code: "no_calls_observed", phase: 2 },
|
|
2907
|
+
// The manifest alone cannot show an undeclared call: passes declaratively, verified only from observed spans (#152, D17).
|
|
2908
|
+
check() {
|
|
2909
|
+
return { pass: true };
|
|
2910
|
+
}
|
|
1718
2911
|
}
|
|
1719
2912
|
];
|
|
1720
2913
|
|
|
@@ -1726,9 +2919,6 @@ var memoryRules = [
|
|
|
1726
2919
|
title: "PII scrub fields declared for long-term memory",
|
|
1727
2920
|
description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
|
|
1728
2921
|
severity: "critical",
|
|
1729
|
-
evidenceLevel: "external",
|
|
1730
|
-
proofTool: "Microsoft Presidio",
|
|
1731
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1732
2922
|
check(manifest) {
|
|
1733
2923
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1734
2924
|
const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
@@ -1747,9 +2937,6 @@ var memoryRules = [
|
|
|
1747
2937
|
title: "Retention policy / TTL set for all memory backends",
|
|
1748
2938
|
description: "Memory without TTL grows unbounded and increases PII exposure risk",
|
|
1749
2939
|
severity: "high",
|
|
1750
|
-
evidenceLevel: "probed",
|
|
1751
|
-
proofTool: "agentspec health",
|
|
1752
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1753
2940
|
check(manifest) {
|
|
1754
2941
|
const mem = manifest.spec.memory;
|
|
1755
2942
|
if (!mem) return { pass: true };
|
|
@@ -1774,9 +2961,6 @@ var memoryRules = [
|
|
|
1774
2961
|
title: "Audit log enabled for long-term memory",
|
|
1775
2962
|
description: "Audit logging enables compliance investigations and debugging",
|
|
1776
2963
|
severity: "medium",
|
|
1777
|
-
evidenceLevel: "probed",
|
|
1778
|
-
proofTool: "agentspec health",
|
|
1779
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1780
2964
|
check(manifest) {
|
|
1781
2965
|
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1782
2966
|
const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
|
|
@@ -1795,9 +2979,6 @@ var memoryRules = [
|
|
|
1795
2979
|
title: "Vector store namespace isolated",
|
|
1796
2980
|
description: "Shared vector store namespaces risk data leakage between agents",
|
|
1797
2981
|
severity: "medium",
|
|
1798
|
-
evidenceLevel: "probed",
|
|
1799
|
-
proofTool: "agentspec health",
|
|
1800
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1801
2982
|
check(manifest) {
|
|
1802
2983
|
if (!manifest.spec.memory?.vector) return { pass: true };
|
|
1803
2984
|
const hasNamespace = !!manifest.spec.memory.vector.namespace;
|
|
@@ -1816,9 +2997,6 @@ var memoryRules = [
|
|
|
1816
2997
|
title: "Short-term memory max tokens bounded",
|
|
1817
2998
|
description: "Unbounded short-term memory can cause token budget overruns",
|
|
1818
2999
|
severity: "low",
|
|
1819
|
-
evidenceLevel: "probed",
|
|
1820
|
-
proofTool: "agentspec health",
|
|
1821
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1822
3000
|
check(manifest) {
|
|
1823
3001
|
if (!manifest.spec.memory?.shortTerm) return { pass: true };
|
|
1824
3002
|
const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
|
|
@@ -1841,9 +3019,6 @@ var evaluationRules = [
|
|
|
1841
3019
|
title: "Evaluation dataset declared",
|
|
1842
3020
|
description: "At least one evaluation dataset is required for quality assurance",
|
|
1843
3021
|
severity: "medium",
|
|
1844
|
-
evidenceLevel: "probed",
|
|
1845
|
-
proofTool: "agentspec health",
|
|
1846
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1847
3022
|
check(manifest) {
|
|
1848
3023
|
const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
|
|
1849
3024
|
return {
|
|
@@ -1861,9 +3036,6 @@ var evaluationRules = [
|
|
|
1861
3036
|
title: "CI gate enabled",
|
|
1862
3037
|
description: "Evaluation CI gate prevents regressions from reaching production",
|
|
1863
3038
|
severity: "medium",
|
|
1864
|
-
evidenceLevel: "probed",
|
|
1865
|
-
proofTool: "agentspec health",
|
|
1866
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1867
3039
|
check(manifest) {
|
|
1868
3040
|
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1869
3041
|
return {
|
|
@@ -1878,22 +3050,22 @@ var evaluationRules = [
|
|
|
1878
3050
|
{
|
|
1879
3051
|
id: "EVAL-03",
|
|
1880
3052
|
pack: "evaluation-coverage",
|
|
1881
|
-
title: "
|
|
1882
|
-
description: "A
|
|
3053
|
+
title: "Grounded metric threshold configured",
|
|
3054
|
+
description: "A grounded threshold (minimum pass rate, 0.95 recommended) is required for production agents",
|
|
1883
3055
|
severity: "medium",
|
|
1884
|
-
evidenceLevel: "probed",
|
|
1885
|
-
proofTool: "agentspec health",
|
|
1886
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1887
3056
|
check(manifest) {
|
|
1888
|
-
const
|
|
1889
|
-
const
|
|
1890
|
-
const
|
|
1891
|
-
const
|
|
3057
|
+
const evaluation = manifest.spec.evaluation;
|
|
3058
|
+
const scopes = [evaluation, evaluation?.live];
|
|
3059
|
+
const hasGrounded = (scope) => !!scope?.metrics?.map(canonicalMetric).includes("grounded");
|
|
3060
|
+
const hasGroundedMetric = scopes.some(hasGrounded);
|
|
3061
|
+
const pass = scopes.some(
|
|
3062
|
+
(scope) => hasGrounded(scope) && Object.keys(scope?.thresholds ?? {}).some((n) => canonicalMetric(n) === "grounded")
|
|
3063
|
+
);
|
|
1892
3064
|
return {
|
|
1893
3065
|
pass,
|
|
1894
|
-
message: pass ? void 0 : !
|
|
1895
|
-
path: "/spec/evaluation/thresholds/
|
|
1896
|
-
recommendation: "Add
|
|
3066
|
+
message: pass ? void 0 : !hasGroundedMetric ? "grounded metric not in spec.evaluation.metrics or spec.evaluation.live.metrics" : "No grounded threshold next to the grounded metric",
|
|
3067
|
+
path: "/spec/evaluation/thresholds/grounded",
|
|
3068
|
+
recommendation: "Add grounded to spec.evaluation.metrics (or spec.evaluation.live.metrics) and set the matching thresholds.grounded: 0.95",
|
|
1897
3069
|
references: []
|
|
1898
3070
|
};
|
|
1899
3071
|
}
|
|
@@ -1908,9 +3080,6 @@ var observabilityRules = [
|
|
|
1908
3080
|
title: "Tracing backend declared",
|
|
1909
3081
|
description: "Agents should declare a tracing backend so runs are observable and debuggable",
|
|
1910
3082
|
severity: "medium",
|
|
1911
|
-
evidenceLevel: "probed",
|
|
1912
|
-
proofTool: "agentspec health",
|
|
1913
|
-
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1914
3083
|
check(manifest) {
|
|
1915
3084
|
const pass = !!manifest.spec.observability?.tracing?.backend;
|
|
1916
3085
|
return {
|
|
@@ -1927,9 +3096,6 @@ var observabilityRules = [
|
|
|
1927
3096
|
title: "Structured logging enabled",
|
|
1928
3097
|
description: "Structured logging makes log aggregation and searching reliable",
|
|
1929
3098
|
severity: "low",
|
|
1930
|
-
evidenceLevel: "behavioral",
|
|
1931
|
-
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1932
|
-
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1933
3099
|
check(manifest) {
|
|
1934
3100
|
const pass = manifest.spec.observability?.logging?.structured !== false;
|
|
1935
3101
|
return {
|
|
@@ -1946,9 +3112,6 @@ var observabilityRules = [
|
|
|
1946
3112
|
title: "Sensitive fields redacted from logs",
|
|
1947
3113
|
description: "Log redaction prevents secrets and PII from appearing in log aggregators",
|
|
1948
3114
|
severity: "medium",
|
|
1949
|
-
evidenceLevel: "external",
|
|
1950
|
-
proofTool: "Microsoft Presidio",
|
|
1951
|
-
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1952
3115
|
check(manifest) {
|
|
1953
3116
|
const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
|
|
1954
3117
|
return {
|
|
@@ -1961,57 +3124,8 @@ var observabilityRules = [
|
|
|
1961
3124
|
}
|
|
1962
3125
|
];
|
|
1963
3126
|
|
|
1964
|
-
// src/audit/rules/metadata.rules.ts
|
|
1965
|
-
var metadataRules = [
|
|
1966
|
-
{
|
|
1967
|
-
id: "META-01",
|
|
1968
|
-
pack: "metadata-quality",
|
|
1969
|
-
title: "Agent description declared",
|
|
1970
|
-
description: "Agents should have a non-empty description for fleet discovery and documentation",
|
|
1971
|
-
severity: "low",
|
|
1972
|
-
evidenceLevel: "declarative",
|
|
1973
|
-
check(manifest) {
|
|
1974
|
-
const desc = manifest.metadata.description;
|
|
1975
|
-
const pass = typeof desc === "string" && desc.trim().length > 0;
|
|
1976
|
-
return {
|
|
1977
|
-
pass,
|
|
1978
|
-
message: pass ? void 0 : "No description declared in metadata. Agents without descriptions are hard to discover in a fleet.",
|
|
1979
|
-
path: "/metadata/description",
|
|
1980
|
-
recommendation: "Add a concise metadata.description explaining what the agent does",
|
|
1981
|
-
references: []
|
|
1982
|
-
};
|
|
1983
|
-
}
|
|
1984
|
-
},
|
|
1985
|
-
{
|
|
1986
|
-
id: "META-02",
|
|
1987
|
-
pack: "metadata-quality",
|
|
1988
|
-
title: "Model temperature is production-safe",
|
|
1989
|
-
description: "Temperature above 1.5 causes unpredictable outputs that are unsuitable for production",
|
|
1990
|
-
severity: "medium",
|
|
1991
|
-
evidenceLevel: "declarative",
|
|
1992
|
-
check(manifest) {
|
|
1993
|
-
const temperature = manifest.spec.model.parameters?.temperature;
|
|
1994
|
-
if (temperature === void 0) return { pass: true };
|
|
1995
|
-
const pass = temperature <= 1.5;
|
|
1996
|
-
return {
|
|
1997
|
-
pass,
|
|
1998
|
-
message: pass ? void 0 : `Model temperature is ${temperature} \u2014 values above 1.5 produce unreliable outputs in production.`,
|
|
1999
|
-
path: "/spec/model/parameters/temperature",
|
|
2000
|
-
recommendation: "Set spec.model.parameters.temperature to 1.0 or lower for production agents",
|
|
2001
|
-
references: []
|
|
2002
|
-
};
|
|
2003
|
-
}
|
|
2004
|
-
}
|
|
2005
|
-
];
|
|
2006
|
-
|
|
2007
3127
|
// src/audit/index.ts
|
|
2008
|
-
var SEVERITY_WEIGHTS =
|
|
2009
|
-
critical: 4,
|
|
2010
|
-
high: 3,
|
|
2011
|
-
medium: 2,
|
|
2012
|
-
low: 1,
|
|
2013
|
-
info: 0
|
|
2014
|
-
};
|
|
3128
|
+
var SEVERITY_WEIGHTS = CONTRACT.severityWeights;
|
|
2015
3129
|
var ALL_RULES = [
|
|
2016
3130
|
...modelRules,
|
|
2017
3131
|
...securityRules,
|
|
@@ -2021,19 +3135,10 @@ var ALL_RULES = [
|
|
|
2021
3135
|
...metadataRules
|
|
2022
3136
|
];
|
|
2023
3137
|
var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
|
|
2024
|
-
|
|
2025
|
-
return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
|
|
2026
|
-
}
|
|
2027
|
-
function scoreToGrade(score) {
|
|
2028
|
-
if (score >= 90) return "A";
|
|
2029
|
-
if (score >= 75) return "B";
|
|
2030
|
-
if (score >= 60) return "C";
|
|
2031
|
-
if (score >= 45) return "D";
|
|
2032
|
-
return "F";
|
|
2033
|
-
}
|
|
3138
|
+
var COMPLIANCE_PACKS = [...new Set(ALL_RULES.map((r) => r.pack))];
|
|
2034
3139
|
function resolveActiveRules(manifest, opts) {
|
|
2035
3140
|
const manifestPacks = manifest.spec.compliance?.packs ?? [];
|
|
2036
|
-
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks :
|
|
3141
|
+
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : COMPLIANCE_PACKS);
|
|
2037
3142
|
return ALL_RULES.filter((r) => activePacks.includes(r.pack));
|
|
2038
3143
|
}
|
|
2039
3144
|
function collectSuppressions(manifest) {
|
|
@@ -2062,10 +3167,10 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2062
3167
|
const violations = [];
|
|
2063
3168
|
const packBreakdown = {};
|
|
2064
3169
|
const evidenceBreakdown = {
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
3170
|
+
independent: { passed: 0, total: 0 },
|
|
3171
|
+
"evidence-point": { passed: 0, total: 0 },
|
|
3172
|
+
offline: { passed: 0, total: 0 },
|
|
3173
|
+
declared: { passed: 0, total: 0 }
|
|
2069
3174
|
};
|
|
2070
3175
|
let totalWeight = 0;
|
|
2071
3176
|
let passedWeight = 0;
|
|
@@ -2076,19 +3181,20 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2076
3181
|
const suppressed = suppressedIds.has(rule.id);
|
|
2077
3182
|
const weight = SEVERITY_WEIGHTS[rule.severity];
|
|
2078
3183
|
const result = ruleResults.get(rule.id);
|
|
2079
|
-
const
|
|
3184
|
+
const classes = ruleClasses(rule.id);
|
|
2080
3185
|
packBreakdown[rule.pack].total += 1;
|
|
2081
|
-
|
|
3186
|
+
tally(evidenceBreakdown, classes, "total");
|
|
2082
3187
|
if (suppressed) {
|
|
2083
3188
|
packBreakdown[rule.pack].passed += 1;
|
|
2084
|
-
|
|
3189
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2085
3190
|
continue;
|
|
2086
3191
|
}
|
|
2087
3192
|
if (result.pass) {
|
|
2088
3193
|
packBreakdown[rule.pack].passed += 1;
|
|
2089
|
-
|
|
3194
|
+
tally(evidenceBreakdown, classes, "passed");
|
|
2090
3195
|
passedWeight += weight;
|
|
2091
3196
|
} else {
|
|
3197
|
+
const hint = proofHint(rule.id);
|
|
2092
3198
|
violations.push({
|
|
2093
3199
|
ruleId: rule.id,
|
|
2094
3200
|
severity: rule.severity,
|
|
@@ -2097,9 +3203,9 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2097
3203
|
path: result.path,
|
|
2098
3204
|
recommendation: result.recommendation,
|
|
2099
3205
|
references: result.references,
|
|
2100
|
-
|
|
2101
|
-
proofTool:
|
|
2102
|
-
proofToolUrl:
|
|
3206
|
+
evidenceClasses: classes,
|
|
3207
|
+
proofTool: hint?.tool,
|
|
3208
|
+
proofToolUrl: hint?.url
|
|
2103
3209
|
});
|
|
2104
3210
|
}
|
|
2105
3211
|
totalWeight += weight;
|
|
@@ -2111,24 +3217,36 @@ function computeScoring(rules, ruleResults, suppressedIds) {
|
|
|
2111
3217
|
}
|
|
2112
3218
|
return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
|
|
2113
3219
|
}
|
|
2114
|
-
function
|
|
2115
|
-
const
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
const
|
|
2125
|
-
if (
|
|
3220
|
+
function tally(breakdown, classes, field) {
|
|
3221
|
+
for (const c of classes) breakdown[c][field] += 1;
|
|
3222
|
+
}
|
|
3223
|
+
var SIDECAR_GAP = { tool: "AgentSpec sidecar gap analysis", url: "https://agentspec.io/docs/concepts/probe-coverage" };
|
|
3224
|
+
var COLLECTOR_HINTS = {
|
|
3225
|
+
"sidecar-probe": SIDECAR_GAP,
|
|
3226
|
+
"sidecar-forwarder": SIDECAR_GAP
|
|
3227
|
+
};
|
|
3228
|
+
function proofHint(ruleId) {
|
|
3229
|
+
for (const { collector, status } of ruleSources(ruleId)) {
|
|
3230
|
+
const hint = COLLECTOR_HINTS[collector];
|
|
3231
|
+
if (status === "built" && hint) return { collector, ...hint };
|
|
2126
3232
|
}
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
).
|
|
2131
|
-
return {
|
|
3233
|
+
return void 0;
|
|
3234
|
+
}
|
|
3235
|
+
function explainPendingEvidence(verified, rules) {
|
|
3236
|
+
const pending = new Map(rules.filter((r) => r.awaitsEvidence).map((r) => [r.id, r.awaitsEvidence]));
|
|
3237
|
+
return {
|
|
3238
|
+
...verified,
|
|
3239
|
+
unverified: verified.unverified.map((u) => {
|
|
3240
|
+
const awaits = pending.get(u.ruleId);
|
|
3241
|
+
if (!awaits || u.reason.code !== "no_evidence") return u;
|
|
3242
|
+
return { ...u, reason: { code: awaits.code, detail: `evidence source arrives in phase ${awaits.phase}` } };
|
|
3243
|
+
})
|
|
3244
|
+
};
|
|
3245
|
+
}
|
|
3246
|
+
function summariseRuleEvidence(rules) {
|
|
3247
|
+
return Object.fromEntries(
|
|
3248
|
+
rules.map((r) => [r.id, { classes: ruleClasses(r.id), notProven: RULE_EVIDENCE[r.id]?.notProven ?? null }])
|
|
3249
|
+
);
|
|
2132
3250
|
}
|
|
2133
3251
|
function runAudit(manifest, opts = {}) {
|
|
2134
3252
|
const rules = resolveActiveRules(manifest, opts);
|
|
@@ -2138,11 +3256,21 @@ function runAudit(manifest, opts = {}) {
|
|
|
2138
3256
|
violations,
|
|
2139
3257
|
packBreakdown,
|
|
2140
3258
|
evidenceBreakdown,
|
|
2141
|
-
totalWeight,
|
|
2142
3259
|
overallScore,
|
|
2143
3260
|
categoryScores
|
|
2144
3261
|
} = computeScoring(rules, ruleResults, suppressedIds);
|
|
2145
|
-
const
|
|
3262
|
+
const applicable = rules.filter((r) => !suppressedIds.has(r.id));
|
|
3263
|
+
const applicableRules = applicable.map((r) => ({ id: r.id, severity: r.severity }));
|
|
3264
|
+
const verified = explainPendingEvidence(
|
|
3265
|
+
grade(
|
|
3266
|
+
opts.records ?? [],
|
|
3267
|
+
applicableRules,
|
|
3268
|
+
opts.trust ?? "standard",
|
|
3269
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
3270
|
+
opts.evidencePoints
|
|
3271
|
+
),
|
|
3272
|
+
applicable
|
|
3273
|
+
);
|
|
2146
3274
|
return {
|
|
2147
3275
|
agentName: manifest.metadata.name,
|
|
2148
3276
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2155,9 +3283,26 @@ function runAudit(manifest, opts = {}) {
|
|
|
2155
3283
|
totalRules: rules.length,
|
|
2156
3284
|
packBreakdown,
|
|
2157
3285
|
evidenceBreakdown,
|
|
2158
|
-
|
|
3286
|
+
verified,
|
|
3287
|
+
applicableRules,
|
|
3288
|
+
ruleEvidence: summariseRuleEvidence(rules)
|
|
2159
3289
|
};
|
|
2160
3290
|
}
|
|
3291
|
+
function canonicalJson(value) {
|
|
3292
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
3293
|
+
if (value !== null && typeof value === "object") {
|
|
3294
|
+
const obj = value;
|
|
3295
|
+
const members = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
|
|
3296
|
+
return `{${members.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
|
|
3297
|
+
}
|
|
3298
|
+
return JSON.stringify(value);
|
|
3299
|
+
}
|
|
3300
|
+
function canonicalManifestSha256(manifest) {
|
|
3301
|
+
return createHash("sha256").update(canonicalJson(manifest), "utf8").digest("hex");
|
|
3302
|
+
}
|
|
3303
|
+
function declaredManifestSha256(raw) {
|
|
3304
|
+
return canonicalManifestSha256(yaml.load(raw, { schema: yaml.JSON_SCHEMA }));
|
|
3305
|
+
}
|
|
2161
3306
|
|
|
2162
3307
|
// src/generate/index.ts
|
|
2163
3308
|
var _registry = /* @__PURE__ */ new Map();
|
|
@@ -2170,7 +3315,7 @@ function getAdapter(framework) {
|
|
|
2170
3315
|
function listAdapters() {
|
|
2171
3316
|
return [..._registry.keys()];
|
|
2172
3317
|
}
|
|
2173
|
-
function generateAdapter(manifest, framework,
|
|
3318
|
+
function generateAdapter(manifest, framework, options2) {
|
|
2174
3319
|
const adapter = _registry.get(framework.toLowerCase());
|
|
2175
3320
|
if (!adapter) {
|
|
2176
3321
|
throw new Error(
|
|
@@ -2179,7 +3324,7 @@ function generateAdapter(manifest, framework, options) {
|
|
|
2179
3324
|
Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
|
|
2180
3325
|
);
|
|
2181
3326
|
}
|
|
2182
|
-
return adapter.generate(manifest,
|
|
3327
|
+
return adapter.generate(manifest, options2);
|
|
2183
3328
|
}
|
|
2184
3329
|
|
|
2185
3330
|
// src/agent/usage-ledger.ts
|
|
@@ -2225,8 +3370,8 @@ var UsageLedger = class {
|
|
|
2225
3370
|
* Return a frozen snapshot of the current usage window.
|
|
2226
3371
|
*
|
|
2227
3372
|
* @param reset - When true (default), clears all counters and starts a new
|
|
2228
|
-
* window. Use reset=
|
|
2229
|
-
*
|
|
3373
|
+
* window. Use reset=false for inspection without draining; the heartbeat
|
|
3374
|
+
* push takes a reset=false snapshot and calls subtract() with it.
|
|
2230
3375
|
*/
|
|
2231
3376
|
snapshot(reset = true) {
|
|
2232
3377
|
const snap = buildSnapshot(this.counters, this.windowStart);
|
|
@@ -2236,39 +3381,220 @@ var UsageLedger = class {
|
|
|
2236
3381
|
}
|
|
2237
3382
|
return snap;
|
|
2238
3383
|
}
|
|
3384
|
+
/**
|
|
3385
|
+
* Remove a shipped snapshot from the ledger; a new window starts, because the control plane dedupes heartbeats on windowStartedAt.
|
|
3386
|
+
* Usage recorded after the snapshot was taken stays, so every token is
|
|
3387
|
+
* shipped exactly once even when a call finishes while a heartbeat is in flight.
|
|
3388
|
+
*/
|
|
3389
|
+
subtract(shipped) {
|
|
3390
|
+
for (const s of shipped.models) {
|
|
3391
|
+
const entry = this.counters.get(s.modelId);
|
|
3392
|
+
if (!entry) continue;
|
|
3393
|
+
entry.promptTokens -= s.promptTokens;
|
|
3394
|
+
entry.completionTokens -= s.completionTokens;
|
|
3395
|
+
entry.totalTokens -= s.totalTokens;
|
|
3396
|
+
entry.callCount -= s.callCount;
|
|
3397
|
+
if (entry.callCount <= 0) this.counters.delete(s.modelId);
|
|
3398
|
+
}
|
|
3399
|
+
this.windowStart = /* @__PURE__ */ new Date();
|
|
3400
|
+
}
|
|
2239
3401
|
};
|
|
2240
3402
|
|
|
3403
|
+
// src/runtime/evidence-points.ts
|
|
3404
|
+
var GUARDRAIL = EvidencePoint.enum.guardrail;
|
|
3405
|
+
var [MCP_LIST, MCP_CALL] = Object.keys(CONTRACT.evidencePoints.mcpClient);
|
|
3406
|
+
var [SCRUB, TTL, AUDIT] = Object.keys(CONTRACT.evidencePoints.memoryWrite);
|
|
3407
|
+
var [NAMESPACE] = Object.keys(CONTRACT.evidencePoints.vectorStore);
|
|
3408
|
+
var [TOOL_TARGET] = Object.keys(CONTRACT.evidencePoints.tool);
|
|
3409
|
+
var [LOGGING_TARGET] = Object.keys(CONTRACT.evidencePoints.logging);
|
|
3410
|
+
var keyOf = (point2, target) => `${point2}\0${target}`;
|
|
3411
|
+
var DEFAULT_KEY = /* @__PURE__ */ Symbol.for("agentspec.runtime.defaultEvidencePoints");
|
|
3412
|
+
var defaults = globalThis;
|
|
3413
|
+
var rec = (value) => value && typeof value === "object" ? value : {};
|
|
3414
|
+
var stdioServers = (spec) => (Array.isArray(rec(spec.mcp).servers) ? rec(spec.mcp).servers : []).filter((s) => s?.transport === "stdio");
|
|
3415
|
+
var isDeclared = (value) => Array.isArray(value) ? value.length > 0 : Boolean(value);
|
|
3416
|
+
function memoryWriteTargets(memory) {
|
|
3417
|
+
const hygiene = rec(memory.hygiene);
|
|
3418
|
+
const targets = /* @__PURE__ */ new Set();
|
|
3419
|
+
if (isDeclared(hygiene.piiScrubFields)) targets.add(SCRUB);
|
|
3420
|
+
if ([rec(memory.shortTerm).ttlSeconds, rec(memory.longTerm).ttlDays].some((ttl) => ttl !== void 0 && ttl !== null)) targets.add(TTL);
|
|
3421
|
+
if (memory.longTerm && hygiene.auditLog === true) targets.add(AUDIT);
|
|
3422
|
+
return targets;
|
|
3423
|
+
}
|
|
3424
|
+
function declaredTargets(spec) {
|
|
3425
|
+
const all = (point2, declared2) => new Set(declared2 ? Object.keys(CONTRACT.evidencePoints[point2] ?? {}) : []);
|
|
3426
|
+
const memory = rec(spec.memory);
|
|
3427
|
+
const logging = rec(rec(spec.observability).logging);
|
|
3428
|
+
return {
|
|
3429
|
+
[EvidencePoint.enum.memoryWrite]: memoryWriteTargets(memory),
|
|
3430
|
+
[EvidencePoint.enum.vectorStore]: new Set(rec(memory.vector).namespace ? [NAMESPACE] : []),
|
|
3431
|
+
[GUARDRAIL]: new Set(Object.keys(CONTRACT.evidencePoints[GUARDRAIL] ?? {}).filter((t) => isDeclared(rec(spec.guardrails)[t]))),
|
|
3432
|
+
[EvidencePoint.enum.tool]: all(EvidencePoint.enum.tool, (Array.isArray(spec.tools) ? spec.tools : []).some((t) => t?.type === "function")),
|
|
3433
|
+
[EvidencePoint.enum.mcpClient]: all(EvidencePoint.enum.mcpClient, stdioServers(spec).length > 0),
|
|
3434
|
+
[EvidencePoint.enum.logging]: all(EvidencePoint.enum.logging, rec(spec.observability).logging !== void 0 && logging.structured !== false)
|
|
3435
|
+
};
|
|
3436
|
+
}
|
|
3437
|
+
function applicablePoints(spec) {
|
|
3438
|
+
const declared2 = declaredTargets(spec);
|
|
3439
|
+
const applicable = /* @__PURE__ */ new Map();
|
|
3440
|
+
for (const [point2, targets] of Object.entries(CONTRACT.evidencePoints)) {
|
|
3441
|
+
for (const [target, ruleId] of Object.entries(targets)) {
|
|
3442
|
+
const listed = CONTRACT.ruleEvidence[ruleId]?.sources.some((s) => s.point === point2);
|
|
3443
|
+
if (listed && declared2[point2]?.has(target)) applicable.set(keyOf(point2, target), { point: EvidencePoint.parse(point2), target, ruleId });
|
|
3444
|
+
}
|
|
3445
|
+
}
|
|
3446
|
+
return applicable;
|
|
3447
|
+
}
|
|
3448
|
+
function createEvidencePoints(manifest) {
|
|
3449
|
+
const manifestSha256 = canonicalManifestSha256(manifest);
|
|
3450
|
+
const spec = rec(rec(manifest).spec);
|
|
3451
|
+
const applicable = applicablePoints(spec);
|
|
3452
|
+
const mcpServers = stdioServers(spec).map((s) => String(s.name));
|
|
3453
|
+
const wired = /* @__PURE__ */ new Set();
|
|
3454
|
+
const emptyWindow = () => new Map([...applicable.keys()].map((k) => [k, { total: 0, counts: { pass: 0, fail: 0, not_verifiable: 0 }, last: [] }]));
|
|
3455
|
+
let calls = emptyWindow();
|
|
3456
|
+
let windowStart = /* @__PURE__ */ new Date();
|
|
3457
|
+
let seq = 0;
|
|
3458
|
+
const shippedUpTo = /* @__PURE__ */ new WeakMap();
|
|
3459
|
+
const points = {
|
|
3460
|
+
register: (point2, target) => {
|
|
3461
|
+
wired.add(keyOf(point2, target));
|
|
3462
|
+
},
|
|
3463
|
+
applies: (point2, target) => applicable.has(keyOf(point2, target)),
|
|
3464
|
+
memorySpec: () => rec(spec.memory),
|
|
3465
|
+
declaredMcpServers: () => [...mcpServers],
|
|
3466
|
+
recordCall: (point2, target, { subject, status = "pass", reason } = {}) => {
|
|
3467
|
+
if (status !== "pass" && !reason) throw new Error(`recordCall: a ${status} call of ${point2}:${target} needs a reason`);
|
|
3468
|
+
const c = calls.get(keyOf(point2, target));
|
|
3469
|
+
if (!c) return;
|
|
3470
|
+
c.total += 1;
|
|
3471
|
+
c.counts[status] += 1;
|
|
3472
|
+
c.last.push({ at: /* @__PURE__ */ new Date(), seq: ++seq, subject, status, reason });
|
|
3473
|
+
if (c.last.length > CONTRACT.evidencePointRecordLimit) c.last.shift();
|
|
3474
|
+
},
|
|
3475
|
+
snapshot: (ttlSeconds, reset = true) => {
|
|
3476
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
3477
|
+
const lastSeq = /* @__PURE__ */ new Map();
|
|
3478
|
+
const window = { start: windowStart.toISOString(), end: windowEnd.toISOString() };
|
|
3479
|
+
const aggregates = [...applicable].map(([key, { point: point2, target, ruleId }]) => {
|
|
3480
|
+
const c = calls.get(key);
|
|
3481
|
+
const state = c.total ? "fired" : wired.has(key) ? "wired" : "not_wired";
|
|
3482
|
+
lastSeq.set(key, seq);
|
|
3483
|
+
const records = c.last.map(({ at, subject, status, reason }) => ({
|
|
3484
|
+
checkId: subject ?? `${point2}:${target}`,
|
|
3485
|
+
ruleId,
|
|
3486
|
+
status: EvidenceStatus.enum[status],
|
|
3487
|
+
...reason ? { reason } : {},
|
|
3488
|
+
source: EvidenceSource.enum["sdk-observed"],
|
|
3489
|
+
origin: EvidenceOrigin.enum.in_process,
|
|
3490
|
+
collector: Collector.enum.sdk,
|
|
3491
|
+
observedAt: at.toISOString(),
|
|
3492
|
+
window,
|
|
3493
|
+
ttlSeconds,
|
|
3494
|
+
sampleSize: c.total,
|
|
3495
|
+
traceIds: [],
|
|
3496
|
+
manifestSha256
|
|
3497
|
+
}));
|
|
3498
|
+
return { point: point2, target, state, counts: { ...c.counts }, records };
|
|
3499
|
+
});
|
|
3500
|
+
if (reset) {
|
|
3501
|
+
calls = emptyWindow();
|
|
3502
|
+
windowStart = windowEnd;
|
|
3503
|
+
} else {
|
|
3504
|
+
shippedUpTo.set(aggregates, { lastSeq, windowEnd });
|
|
3505
|
+
}
|
|
3506
|
+
return aggregates;
|
|
3507
|
+
},
|
|
3508
|
+
subtract: (shipped) => {
|
|
3509
|
+
const taken = shippedUpTo.get(shipped);
|
|
3510
|
+
if (!taken) throw new Error("EvidencePoints.subtract: pass the array snapshot() returned");
|
|
3511
|
+
for (const a of shipped) {
|
|
3512
|
+
const key = keyOf(a.point, a.target);
|
|
3513
|
+
const c = calls.get(key);
|
|
3514
|
+
c.total -= a.counts.pass + a.counts.fail + a.counts.not_verifiable;
|
|
3515
|
+
for (const status of ["pass", "fail", "not_verifiable"]) c.counts[status] -= a.counts[status];
|
|
3516
|
+
c.last = c.last.filter((l) => l.seq > taken.lastSeq.get(key));
|
|
3517
|
+
}
|
|
3518
|
+
windowStart = taken.windowEnd;
|
|
3519
|
+
}
|
|
3520
|
+
};
|
|
3521
|
+
defaults[DEFAULT_KEY] = points;
|
|
3522
|
+
return points;
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
// src/health/tool-check-id.ts
|
|
3526
|
+
function slugify(s) {
|
|
3527
|
+
return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3528
|
+
}
|
|
3529
|
+
function toolCheckId(toolName) {
|
|
3530
|
+
return `tool:${slugify(toolName)}`;
|
|
3531
|
+
}
|
|
3532
|
+
|
|
3533
|
+
// src/runtime/call-points.ts
|
|
3534
|
+
var { tool: TOOL, mcpClient: MCP_CLIENT, logging: LOGGING } = EvidencePoint.enum;
|
|
3535
|
+
var { memoryWrite: MEMORY_WRITE, vectorStore: VECTOR_STORE } = EvidencePoint.enum;
|
|
3536
|
+
var NOT_VERIFIABLE_CODE = "instrumentation_unavailable";
|
|
3537
|
+
var WRONG_NAMESPACE_CODE = "undeclared_namespace";
|
|
3538
|
+
var NO_NAMESPACE_CODE = "no_evidence";
|
|
3539
|
+
for (const code of [NOT_VERIFIABLE_CODE, WRONG_NAMESPACE_CODE, NO_NAMESPACE_CODE]) {
|
|
3540
|
+
if (!CONTRACT.reasonCodes.includes(code)) throw new Error(`contract.reasonCodes lacks '${code}'`);
|
|
3541
|
+
}
|
|
3542
|
+
new AsyncLocalStorage();
|
|
3543
|
+
|
|
2241
3544
|
// src/agent/reporter.ts
|
|
3545
|
+
var HEARTBEAT_POST_TIMEOUT_MS = 1e4;
|
|
3546
|
+
var DEFAULT_PUSH_INTERVAL_SECONDS = 30;
|
|
3547
|
+
var RECORD_TTL_INTERVALS = 2;
|
|
3548
|
+
function buildCappedBody(health, aggregates, usage) {
|
|
3549
|
+
const trimmedChecks = [...health.checks];
|
|
3550
|
+
const serialise = (checks, evidencePoints) => JSON.stringify({
|
|
3551
|
+
contract: CONTRACT.version,
|
|
3552
|
+
health: { ...health, checks },
|
|
3553
|
+
records: [],
|
|
3554
|
+
// the control plane reads the records inside each aggregate (#599)
|
|
3555
|
+
evidencePoints,
|
|
3556
|
+
usage
|
|
3557
|
+
});
|
|
3558
|
+
const size = (body2) => Buffer.byteLength(body2, "utf8");
|
|
3559
|
+
let body = serialise(trimmedChecks, aggregates);
|
|
3560
|
+
if (size(body) > CONTRACT.heartbeatMaxBytes) {
|
|
3561
|
+
aggregates = aggregates.map((a) => ({ ...a, records: [] }));
|
|
3562
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3563
|
+
}
|
|
3564
|
+
while (size(body) > CONTRACT.heartbeatMaxBytes && trimmedChecks.length > 0) {
|
|
3565
|
+
trimmedChecks.pop();
|
|
3566
|
+
body = serialise(trimmedChecks, aggregates);
|
|
3567
|
+
}
|
|
3568
|
+
return { body, bytes: size(body) };
|
|
3569
|
+
}
|
|
2242
3570
|
var AgentSpecReporter = class {
|
|
2243
3571
|
constructor(manifest, opts = {}) {
|
|
2244
3572
|
this.manifest = manifest;
|
|
2245
3573
|
this.opts = opts;
|
|
2246
3574
|
this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
|
|
2247
3575
|
this.staleAfterMs = opts.staleAfterMs ?? 6e4;
|
|
3576
|
+
this.evidence = createEvidencePoints(opts.rawManifest ?? manifest);
|
|
2248
3577
|
}
|
|
3578
|
+
manifest;
|
|
3579
|
+
opts;
|
|
2249
3580
|
cached = null;
|
|
2250
3581
|
timer = null;
|
|
2251
3582
|
_pushTimer = null;
|
|
2252
3583
|
refreshIntervalMs;
|
|
2253
3584
|
staleAfterMs;
|
|
2254
|
-
|
|
3585
|
+
inflight = null;
|
|
3586
|
+
/** True while a heartbeat POST is in flight; a tick that finds it set sends nothing. */
|
|
3587
|
+
heartbeatInFlight = false;
|
|
2255
3588
|
stopped = false;
|
|
2256
|
-
registeredTools = /* @__PURE__ */ new Set();
|
|
2257
3589
|
/** In-process token usage accumulator. */
|
|
2258
3590
|
usage = new UsageLedger();
|
|
3591
|
+
/** Evidence points the agent's code wires (`guardrail(...)` reports here); each heartbeat sends the window's aggregates. */
|
|
3592
|
+
evidence;
|
|
2259
3593
|
/**
|
|
2260
|
-
*
|
|
2261
|
-
*
|
|
2262
|
-
* Declared tools that are not registered report status: 'fail'.
|
|
2263
|
-
*
|
|
2264
|
-
* Call this once per tool after wiring up your tool handlers:
|
|
2265
|
-
* reporter.registerTool('search-arxiv', handler)
|
|
2266
|
-
*
|
|
2267
|
-
* The handler argument is accepted but not called — it is retained solely so
|
|
2268
|
-
* callers can pass the real function object for documentation purposes.
|
|
3594
|
+
* @deprecated No-op, kept so existing callers compile. Registering a handler proves nothing about
|
|
3595
|
+
* which tools the agent calls, so the health report no longer carries `tool:<name>` checks (ADR-0017).
|
|
2269
3596
|
*/
|
|
2270
|
-
registerTool(
|
|
2271
|
-
this.registeredTools.add(name);
|
|
3597
|
+
registerTool(_name, _handler) {
|
|
2272
3598
|
}
|
|
2273
3599
|
/**
|
|
2274
3600
|
* Start background refresh. Call once during app startup.
|
|
@@ -2332,52 +3658,74 @@ var AgentSpecReporter = class {
|
|
|
2332
3658
|
*/
|
|
2333
3659
|
startPushMode(opts) {
|
|
2334
3660
|
if (this._pushTimer !== null) return;
|
|
2335
|
-
const
|
|
2336
|
-
|
|
3661
|
+
const controlPlaneUrl = opts.controlPlaneUrl || process.env[CONTRACT.env.controlPlaneUrl];
|
|
3662
|
+
if (!controlPlaneUrl) {
|
|
3663
|
+
throw new Error(
|
|
3664
|
+
`startPushMode: no control plane URL \u2014 pass controlPlaneUrl or set ${CONTRACT.env.controlPlaneUrl}`
|
|
3665
|
+
);
|
|
3666
|
+
}
|
|
3667
|
+
if (!opts.apiKey) {
|
|
3668
|
+
throw new Error("startPushMode: apiKey is required (the per-agent token from POST /api/v1/register)");
|
|
3669
|
+
}
|
|
3670
|
+
const resolved = { ...opts, controlPlaneUrl };
|
|
3671
|
+
const intervalMs = (opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * 1e3;
|
|
3672
|
+
void this._pushHeartbeat(resolved);
|
|
2337
3673
|
this._pushTimer = setInterval(() => {
|
|
2338
|
-
void this._pushHeartbeat(
|
|
3674
|
+
void this._pushHeartbeat(resolved);
|
|
2339
3675
|
}, intervalMs);
|
|
2340
3676
|
if (this._pushTimer.unref) {
|
|
2341
3677
|
this._pushTimer.unref();
|
|
2342
3678
|
}
|
|
2343
3679
|
}
|
|
2344
3680
|
async _pushHeartbeat(opts) {
|
|
3681
|
+
if (this.heartbeatInFlight) return;
|
|
3682
|
+
this.heartbeatInFlight = true;
|
|
2345
3683
|
try {
|
|
2346
3684
|
const health = await this.getReport();
|
|
2347
|
-
const gap = runAudit(this.manifest);
|
|
2348
3685
|
const usage = this.usage.snapshot(false);
|
|
2349
|
-
const
|
|
2350
|
-
|
|
2351
|
-
if (
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
}
|
|
3686
|
+
const aggregates = this.evidence.snapshot(Math.ceil((opts.intervalSeconds ?? DEFAULT_PUSH_INTERVAL_SECONDS) * RECORD_TTL_INTERVALS), false);
|
|
3687
|
+
const { body, bytes } = buildCappedBody(health, aggregates, usage);
|
|
3688
|
+
if (bytes > CONTRACT.heartbeatMaxBytes) {
|
|
3689
|
+
opts.onError?.(
|
|
3690
|
+
new Error(`Heartbeat not sent: ${bytes} bytes is over the ${CONTRACT.heartbeatMaxBytes}-byte cap with every check trimmed`)
|
|
3691
|
+
);
|
|
3692
|
+
return;
|
|
2357
3693
|
}
|
|
2358
3694
|
let res;
|
|
3695
|
+
const abort = new AbortController();
|
|
3696
|
+
const abortTimer = setTimeout(
|
|
3697
|
+
() => abort.abort(new Error(`heartbeat POST timed out after ${HEARTBEAT_POST_TIMEOUT_MS}ms`)),
|
|
3698
|
+
HEARTBEAT_POST_TIMEOUT_MS
|
|
3699
|
+
);
|
|
3700
|
+
abortTimer.unref?.();
|
|
2359
3701
|
try {
|
|
2360
|
-
res = await fetch(`${opts.controlPlaneUrl}
|
|
3702
|
+
res = await fetch(`${opts.controlPlaneUrl}${CONTRACT.controlPlanePaths.heartbeat}`, {
|
|
2361
3703
|
method: "POST",
|
|
2362
3704
|
headers: {
|
|
2363
3705
|
"Authorization": `Bearer ${opts.apiKey}`,
|
|
2364
3706
|
"Content-Type": "application/json"
|
|
2365
3707
|
},
|
|
2366
|
-
body
|
|
3708
|
+
body,
|
|
3709
|
+
signal: abort.signal
|
|
2367
3710
|
});
|
|
2368
3711
|
} catch (err) {
|
|
2369
3712
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2370
3713
|
opts.onError?.(new Error(sanitized));
|
|
2371
3714
|
return;
|
|
3715
|
+
} finally {
|
|
3716
|
+
clearTimeout(abortTimer);
|
|
2372
3717
|
}
|
|
2373
3718
|
if (res.ok) {
|
|
2374
|
-
this.usage.
|
|
3719
|
+
this.usage.subtract(usage);
|
|
3720
|
+
this.evidence.subtract(aggregates);
|
|
2375
3721
|
} else {
|
|
2376
3722
|
opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
|
|
2377
3723
|
}
|
|
2378
3724
|
} catch (err) {
|
|
2379
3725
|
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2380
3726
|
opts.onError?.(new Error(sanitized));
|
|
3727
|
+
} finally {
|
|
3728
|
+
this.heartbeatInFlight = false;
|
|
2381
3729
|
}
|
|
2382
3730
|
}
|
|
2383
3731
|
/** Stop sending heartbeats to the control plane. */
|
|
@@ -2391,38 +3739,26 @@ var AgentSpecReporter = class {
|
|
|
2391
3739
|
isPushModeActive() {
|
|
2392
3740
|
return this._pushTimer !== null;
|
|
2393
3741
|
}
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
this.
|
|
3742
|
+
/** Runs one refresh; concurrent callers share the in-flight promise so they all see a populated cache. */
|
|
3743
|
+
refresh() {
|
|
3744
|
+
if (this.inflight) return this.inflight;
|
|
3745
|
+
if (this.stopped) return Promise.resolve();
|
|
3746
|
+
this.inflight = this.runRefresh().finally(() => {
|
|
3747
|
+
this.inflight = null;
|
|
3748
|
+
});
|
|
3749
|
+
return this.inflight;
|
|
3750
|
+
}
|
|
3751
|
+
async runRefresh() {
|
|
2397
3752
|
try {
|
|
2398
3753
|
const report = await runHealthCheck(this.manifest, {
|
|
2399
3754
|
checkModel: true,
|
|
2400
3755
|
checkMcp: true,
|
|
2401
3756
|
checkMemory: true,
|
|
2402
|
-
checkServices: true
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
const registered = this.registeredTools.has(tool.name);
|
|
2406
|
-
return {
|
|
2407
|
-
id: `tool:${tool.name}`,
|
|
2408
|
-
category: "tool",
|
|
2409
|
-
status: registered ? "pass" : "fail",
|
|
2410
|
-
severity: "info",
|
|
2411
|
-
...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
|
|
2412
|
-
};
|
|
3757
|
+
checkServices: true,
|
|
3758
|
+
baseDir: this.opts.baseDir,
|
|
3759
|
+
rawManifest: this.opts.rawManifest
|
|
2413
3760
|
});
|
|
2414
|
-
|
|
2415
|
-
const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
|
|
2416
|
-
const enrichedReport = {
|
|
2417
|
-
...report,
|
|
2418
|
-
checks: [...report.checks, ...toolChecks],
|
|
2419
|
-
summary: {
|
|
2420
|
-
...report.summary,
|
|
2421
|
-
passed: report.summary.passed + toolPassed,
|
|
2422
|
-
failed: report.summary.failed + toolFailed
|
|
2423
|
-
}
|
|
2424
|
-
};
|
|
2425
|
-
this.cached = { report: enrichedReport, ts: Date.now() };
|
|
3761
|
+
this.cached = { report, ts: Date.now() };
|
|
2426
3762
|
} catch {
|
|
2427
3763
|
if (!this.cached) {
|
|
2428
3764
|
this.cached = {
|
|
@@ -2446,8 +3782,6 @@ var AgentSpecReporter = class {
|
|
|
2446
3782
|
} else {
|
|
2447
3783
|
this.cached = { ...this.cached, ts: Date.now() };
|
|
2448
3784
|
}
|
|
2449
|
-
} finally {
|
|
2450
|
-
this.refreshing = false;
|
|
2451
3785
|
}
|
|
2452
3786
|
}
|
|
2453
3787
|
};
|
|
@@ -2455,7 +3789,7 @@ var AgentSpecReporter = class {
|
|
|
2455
3789
|
// src/agent/adapters/fastify.ts
|
|
2456
3790
|
function agentSpecFastifyPlugin(reporter) {
|
|
2457
3791
|
return async (app) => {
|
|
2458
|
-
app.get(
|
|
3792
|
+
app.get(CONTRACT.agentPaths.health, async (_req, reply) => {
|
|
2459
3793
|
try {
|
|
2460
3794
|
const report = await reporter.getReport();
|
|
2461
3795
|
reply.status(200).send(report);
|
|
@@ -2518,6 +3852,6 @@ function agentSpecExpressRouter(reporter) {
|
|
|
2518
3852
|
return handler;
|
|
2519
3853
|
}
|
|
2520
3854
|
|
|
2521
|
-
export { AUDIT_RULE_IDS, AgentSpecReporter, LATEST_API_VERSION, ManifestSchema, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, collectEnvRefs, collectFileRefs, detectRefType, detectVersion, exportJsonSchema, generateAdapter, getAdapter,
|
|
3855
|
+
export { AUDIT_RULE_IDS, AgentSpecReporter, COMPLIANCE_PACKS, CONTRACT, ContractSchema, EVIDENCE_SHAPES, EvalResult, EvidenceOrigin, EvidenceReason, EvidenceRecord, EvidenceSource, EvidenceStatus, EvidenceWindow, Heartbeat, LATEST_API_VERSION, MAX_PRODUCTION_TEMPERATURE, MAX_TRACE_IDS, ManifestSchema, PodspecFinding, RULE_EVIDENCE, ReadinessFinding, RuleEvidenceEntry, SeveritySchema, SidecarPush, TrustPolicy, UsageLedger, agentSpecExpressRouter, agentSpecFastifyPlugin, canonicalManifestSha256, collectEnvRefs, collectFileRefs, computeHealthStatus, declaredManifestSha256, describeManifestError, detectRefType, detectVersion, exportJsonSchema, formatVerified, gatewayEnvNames, generateAdapter, getAdapter, grade, isLatestModelId, isLatestVersion, listAdapters, loadManifest, migrateManifest, registerAdapter, renderGatewayConfig, resolveRef, resolveRefs, ruleClasses, runAudit, runHealthCheck, runServiceChecks, scoreToGrade, secretEnvName, slugify, toolCheckId, tryLoadManifest };
|
|
2522
3856
|
//# sourceMappingURL=index.js.map
|
|
2523
3857
|
//# sourceMappingURL=index.js.map
|