@agentspec/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2404 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +3257 -0
- package/dist/index.d.ts +3257 -0
- package/dist/index.js +2373 -0
- package/dist/index.js.map +1 -0
- package/package.json +50 -0
- package/schemas/schemas/v1/agent.schema.json +1382 -0
- package/schemas/schemas/v1/generate-schema.ts +16 -0
- package/schemas/v1/agent.schema.json +1382 -0
- package/schemas/v1/generate-schema.ts +16 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,2404 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var zod = require('zod');
|
|
4
|
+
var zodToJsonSchema = require('zod-to-json-schema');
|
|
5
|
+
var fs = require('fs');
|
|
6
|
+
var path = require('path');
|
|
7
|
+
var yaml = require('js-yaml');
|
|
8
|
+
|
|
9
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
10
|
+
|
|
11
|
+
var yaml__default = /*#__PURE__*/_interopDefault(yaml);
|
|
12
|
+
|
|
13
|
+
// src/schema/manifest.schema.ts
|
|
14
|
+
var refOrLiteral = zod.z.string();
|
|
15
|
+
var refDesc = 'Supports reference syntax: "$env:VAR_NAME" (env var), "$secret:name" (secret manager), "$file:path" (file relative to agent.yaml), or a literal string value.';
|
|
16
|
+
var MetadataSchema = zod.z.object({
|
|
17
|
+
name: zod.z.string().min(1).regex(/^[a-z0-9-]+$/, "Must be a lowercase slug"),
|
|
18
|
+
version: zod.z.string().regex(/^\d+\.\d+\.\d+$/, "Must be semver (e.g. 1.0.0)"),
|
|
19
|
+
description: zod.z.string().min(1),
|
|
20
|
+
tags: zod.z.array(zod.z.string()).optional().describe('Free-form tags for filtering and discovery (e.g. ["finance", "rag", "internal"]).'),
|
|
21
|
+
author: zod.z.string().optional(),
|
|
22
|
+
license: zod.z.string().optional().describe('SPDX license identifier (e.g. "MIT", "Apache-2.0", "proprietary").')
|
|
23
|
+
});
|
|
24
|
+
var ModelFallbackSchema = zod.z.object({
|
|
25
|
+
provider: zod.z.string().describe('LLM provider for the fallback model. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock".'),
|
|
26
|
+
id: zod.z.string().describe('Model ID for the fallback (e.g. "gpt-4o-mini", "claude-haiku-4-5-20251001"). Must be a valid ID for the declared provider.'),
|
|
27
|
+
apiKey: refOrLiteral.describe(refDesc),
|
|
28
|
+
triggerOn: zod.z.array(zod.z.enum(["rate_limit", "timeout", "error_5xx", "error_4xx"])).optional().describe("Conditions that trigger failover to this model. Omit to use the fallback for all errors."),
|
|
29
|
+
maxRetries: zod.z.number().int().min(0).max(10).optional().describe("Number of retry attempts before failing over. Defaults to 0 (immediate failover).")
|
|
30
|
+
});
|
|
31
|
+
var ModelSchema = zod.z.object({
|
|
32
|
+
provider: zod.z.string().describe('LLM provider. Common values: "openai", "anthropic", "groq", "google", "mistral", "azure", "bedrock". Determines the API client and base URL used.'),
|
|
33
|
+
id: zod.z.string().describe('Exact model ID as accepted by the provider API (e.g. "gpt-4o", "claude-sonnet-4-6", "llama-3.1-70b-versatile"). Check provider docs for valid IDs.'),
|
|
34
|
+
apiKey: refOrLiteral.describe(refDesc),
|
|
35
|
+
parameters: zod.z.object({
|
|
36
|
+
temperature: zod.z.number().min(0).max(2).optional().describe("Sampling temperature. 0 = deterministic/focused, 1 = balanced, 2 = highly creative. Most providers recommend 0\u20131 for production agents."),
|
|
37
|
+
maxTokens: zod.z.number().int().min(1).optional().describe("Maximum tokens in the model response. Controls cost and latency. Check model context window limits."),
|
|
38
|
+
topP: zod.z.number().min(0).max(1).optional().describe("Nucleus sampling threshold. The model samples from the smallest set of tokens whose cumulative probability exceeds topP. Alternative to temperature \u2014 do not set both."),
|
|
39
|
+
frequencyPenalty: zod.z.number().optional().describe("Penalises tokens that have already appeared frequently in the output (-2.0\u20132.0). Positive values reduce repetition."),
|
|
40
|
+
presencePenalty: zod.z.number().optional().describe("Penalises tokens that appear at all in the output so far (-2.0\u20132.0). Positive values encourage the model to introduce new topics.")
|
|
41
|
+
}).optional(),
|
|
42
|
+
fallback: ModelFallbackSchema.optional(),
|
|
43
|
+
costControls: zod.z.object({
|
|
44
|
+
maxMonthlyUSD: zod.z.number().positive().optional().describe("Hard monthly spend cap in USD. Requests are rejected once this limit is reached."),
|
|
45
|
+
alertAtUSD: zod.z.number().positive().optional().describe("Send an alert when cumulative monthly spend crosses this USD threshold. Must be less than maxMonthlyUSD.")
|
|
46
|
+
}).optional()
|
|
47
|
+
});
|
|
48
|
+
var PromptVariableSchema = zod.z.object({
|
|
49
|
+
name: zod.z.string().describe('Variable name used in the prompt template (e.g. "user_name" for {{user_name}}).'),
|
|
50
|
+
value: refOrLiteral.describe(refDesc)
|
|
51
|
+
});
|
|
52
|
+
var PromptsSchema = zod.z.object({
|
|
53
|
+
system: refOrLiteral.describe('System prompt. Use "$file:prompts/system.md" to load from a file (recommended for long prompts), or provide an inline string. ' + refDesc),
|
|
54
|
+
fallback: zod.z.string().optional().describe('Inline fallback system prompt used if the primary system prompt file cannot be loaded. Only applies when "system" is a $file: reference.'),
|
|
55
|
+
hotReload: zod.z.boolean().optional().default(false).describe("When true, the system prompt file is re-read on every request instead of being cached at startup. Useful for rapid iteration without restarts."),
|
|
56
|
+
variables: zod.z.array(PromptVariableSchema).optional().describe("Template variables injected into the system prompt at runtime. Values support reference syntax for secrets and env vars.")
|
|
57
|
+
});
|
|
58
|
+
var ToolAnnotationsSchema = zod.z.object({
|
|
59
|
+
readOnlyHint: zod.z.boolean().optional().describe("MCP hint: tool does not modify state. Allows clients to show a read-only badge and skip confirmation prompts."),
|
|
60
|
+
destructiveHint: zod.z.boolean().optional().describe('MCP hint: tool may perform irreversible actions (delete, overwrite). Triggers human-in-the-loop approval when humanInTheLoop.approvalRequired includes "before-destructive-tool".'),
|
|
61
|
+
idempotentHint: zod.z.boolean().optional().describe("MCP hint: calling the tool multiple times with the same inputs produces the same result. Allows safe retries."),
|
|
62
|
+
openWorldHint: zod.z.boolean().optional().describe("MCP hint: tool interacts with systems outside the agent's controlled environment (e.g. the internet). Affects trust level in tool outputs.")
|
|
63
|
+
});
|
|
64
|
+
var ToolSchema = zod.z.object({
|
|
65
|
+
name: zod.z.string().min(1).regex(/^[a-z0-9-]+$/, "Tool name must be a lowercase slug"),
|
|
66
|
+
type: zod.z.enum(["function", "mcp", "builtin"]).describe('"function" = local code module, "mcp" = registered via an MCP server in spec.mcp, "builtin" = provided by the runtime (e.g. code-interpreter, web-search).'),
|
|
67
|
+
description: zod.z.string().describe("Human-readable description of what the tool does and when the agent should use it. This text is included in the tool-call prompt."),
|
|
68
|
+
module: zod.z.string().optional().describe('Path to the module that implements this tool. Use "$file:tools/my_tool.py" syntax. Only required for type "function".'),
|
|
69
|
+
function: zod.z.string().optional().describe('Name of the callable function exported from the module (e.g. "search", "run_query"). Only required for type "function".'),
|
|
70
|
+
annotations: ToolAnnotationsSchema.optional()
|
|
71
|
+
});
|
|
72
|
+
var McpServerSchema = zod.z.object({
|
|
73
|
+
name: zod.z.string(),
|
|
74
|
+
transport: zod.z.enum(["stdio", "sse", "http"]).describe('"stdio" = spawn a local process (requires "command"), "sse" = Server-Sent Events over HTTP, "http" = streamable HTTP (requires "url").'),
|
|
75
|
+
command: zod.z.string().optional().describe('Shell command to spawn the MCP server process. Required for transport "stdio" (e.g. "npx", "python").'),
|
|
76
|
+
args: zod.z.array(zod.z.string()).optional().describe('Arguments passed to the command (e.g. ["-m", "my_mcp_server"]). Only used for transport "stdio".'),
|
|
77
|
+
url: zod.z.string().url().optional().describe('Base URL of the remote MCP server. Required for transport "sse" or "http". ' + refDesc),
|
|
78
|
+
env: zod.z.record(zod.z.string(), refOrLiteral).optional().describe('Environment variables injected into the MCP server process. Values support reference syntax. Use "$env:API_KEY" to forward secrets without hardcoding.'),
|
|
79
|
+
healthCheck: zod.z.object({
|
|
80
|
+
timeoutSeconds: zod.z.number().int().min(1).max(60).default(5).describe("Seconds to wait for the MCP server to respond during health checks. Increase for slow-starting processes.")
|
|
81
|
+
}).optional()
|
|
82
|
+
});
|
|
83
|
+
var McpSchema = zod.z.object({
|
|
84
|
+
servers: zod.z.array(McpServerSchema)
|
|
85
|
+
});
|
|
86
|
+
var ShortTermMemorySchema = zod.z.object({
|
|
87
|
+
backend: zod.z.enum(["in-memory", "redis", "sqlite"]),
|
|
88
|
+
maxTurns: zod.z.number().int().min(1).optional().describe("Maximum number of conversation turns (user + assistant message pairs) to retain in context. Older turns are evicted first. Use with maxTokens for dual-bounded windows."),
|
|
89
|
+
maxTokens: zod.z.number().int().min(1).optional().describe("Maximum total tokens across all retained messages. When exceeded, oldest turns are evicted. Required by MEM-05 compliance rule."),
|
|
90
|
+
ttlSeconds: zod.z.number().int().min(0).optional().describe("Time-to-live in seconds for memory entries. 0 = session-scoped (cleared when connection closes). Omitting means no expiry \u2014 increases PII exposure risk (MEM-02)."),
|
|
91
|
+
connection: refOrLiteral.optional().describe("Connection string for redis or sqlite backends. " + refDesc)
|
|
92
|
+
});
|
|
93
|
+
var LongTermMemorySchema = zod.z.object({
|
|
94
|
+
backend: zod.z.enum(["postgres", "sqlite", "mongodb"]),
|
|
95
|
+
connectionString: refOrLiteral.describe("Database connection URI. " + refDesc),
|
|
96
|
+
table: zod.z.string().optional().describe('Table or collection name where agent memories are stored. Defaults to "agent_memories" if omitted. Use a per-agent name to avoid cross-agent data leakage.'),
|
|
97
|
+
ttlDays: zod.z.number().int().min(1).optional().describe("Time-to-live in days for long-term memory records. Omitting means no expiry \u2014 a compliance risk (MEM-02). Required by GDPR Article 5 data minimisation principle.")
|
|
98
|
+
});
|
|
99
|
+
var VectorMemorySchema = zod.z.object({
|
|
100
|
+
backend: zod.z.enum(["pgvector", "pinecone", "weaviate", "qdrant", "chroma"]),
|
|
101
|
+
connectionString: refOrLiteral.optional().describe("Connection URI for self-hosted backends (pgvector, weaviate, qdrant, chroma). " + refDesc),
|
|
102
|
+
apiKey: refOrLiteral.optional().describe("API key for managed backends (Pinecone, Weaviate Cloud). " + refDesc),
|
|
103
|
+
dimension: zod.z.number().int().min(1).describe("Embedding vector dimension. Must exactly match the output dimension of your embedding model (e.g. 1536 for text-embedding-3-small, 768 for nomic-embed-text). Mismatch causes insertion errors."),
|
|
104
|
+
topK: zod.z.number().int().min(1).optional().describe("Number of nearest-neighbour results to retrieve per similarity search. Higher values improve recall at the cost of latency and context token usage."),
|
|
105
|
+
namespace: zod.z.string().optional().describe("Logical partition within the vector store. Required by MEM-04 to prevent cross-agent data leakage. Recommended value: the agent name (spec.metadata.name).")
|
|
106
|
+
});
|
|
107
|
+
var MemoryHygieneSchema = zod.z.object({
|
|
108
|
+
piiScrubFields: zod.z.array(zod.z.string()).optional().describe(
|
|
109
|
+
'PII field names to scrub before writing to long-term or vector memory. Required when longTerm is set (MEM-01). Use recogniser names compatible with your scrubbing library (e.g. Microsoft Presidio entity types): "email", "phone_number", "credit_card", "iban", "ssn", "bank_account", "passport", "drivers_license", "person_name", "date_of_birth", "address", "ip_address", "api_key", "password", "medical_record".'
|
|
110
|
+
),
|
|
111
|
+
auditLog: zod.z.boolean().optional().default(false).describe("When true, all memory read and write operations are emitted as structured audit log events. Required for long-term memory (MEM-03) and HIPAA/SOC2 compliance."),
|
|
112
|
+
retentionPolicy: zod.z.string().optional().describe(
|
|
113
|
+
`Named governance or compliance retention policy applied to this agent's memory. Informational label \u2014 must be consistent with ttlSeconds/ttlDays on the backend configs. Common values: "session" (cleared on disconnect), "ephemeral" (never persisted), "30d", "90d", "1y", "persistent" (explicit no-expiry opt-in), "gdpr-30d" (GDPR Art.5 minimum), "hipaa-6y" (HIPAA 45 CFR \xA7164.530), "pci-dss-1y" (PCI DSS Req.10.7).`
|
|
114
|
+
)
|
|
115
|
+
});
|
|
116
|
+
var MemorySchema = zod.z.object({
|
|
117
|
+
shortTerm: ShortTermMemorySchema.optional(),
|
|
118
|
+
longTerm: LongTermMemorySchema.optional(),
|
|
119
|
+
vector: VectorMemorySchema.optional(),
|
|
120
|
+
hygiene: MemoryHygieneSchema.optional()
|
|
121
|
+
});
|
|
122
|
+
var SubagentRefSchema = zod.z.union([
|
|
123
|
+
zod.z.object({ agentspec: zod.z.string() }),
|
|
124
|
+
// local file path
|
|
125
|
+
zod.z.object({
|
|
126
|
+
a2a: zod.z.object({
|
|
127
|
+
url: refOrLiteral,
|
|
128
|
+
auth: zod.z.object({
|
|
129
|
+
type: zod.z.enum(["bearer", "apikey", "none"]),
|
|
130
|
+
token: refOrLiteral.optional(),
|
|
131
|
+
header: zod.z.string().optional().describe('HTTP header name for the API key (e.g. "X-API-Key"). Only used when auth type is "apikey".')
|
|
132
|
+
}).optional()
|
|
133
|
+
})
|
|
134
|
+
})
|
|
135
|
+
]);
|
|
136
|
+
var SubagentSchema = zod.z.object({
|
|
137
|
+
name: zod.z.string(),
|
|
138
|
+
ref: SubagentRefSchema,
|
|
139
|
+
invocation: zod.z.enum(["parallel", "sequential", "on-demand"]).describe('"parallel" = invoked concurrently with other subagents, "sequential" = invoked in order after previous subagents complete, "on-demand" = only invoked when the agent decides it is needed (triggered by LLM or triggerKeywords).'),
|
|
140
|
+
passContext: zod.z.boolean().optional().default(false).describe("When true, the current conversation thread (messages so far) is forwarded to the subagent as context. Use with care \u2014 increases token usage and may leak sensitive conversation history."),
|
|
141
|
+
triggerKeywords: zod.z.array(zod.z.string()).optional().describe('Keywords in user messages that automatically trigger this subagent (only applies when invocation is "on-demand"). Case-insensitive substring match.')
|
|
142
|
+
});
|
|
143
|
+
var ChatEndpointSchema = zod.z.object({
|
|
144
|
+
path: zod.z.string().optional().default("/v1/chat").describe('URL path for the chat endpoint (e.g. "/v1/chat", "/api/chat"). Must start with "/".'),
|
|
145
|
+
protocol: zod.z.enum(["openai-compatible", "custom"]).optional().default("openai-compatible").describe('"openai-compatible" = request/response shape matches the OpenAI Chat Completions API (enables drop-in compatibility with OpenAI SDKs). "custom" = proprietary shape defined by the agent.'),
|
|
146
|
+
streaming: zod.z.boolean().optional().default(true).describe("When true, the endpoint supports Server-Sent Events streaming responses. Clients receive tokens as they are generated."),
|
|
147
|
+
sessionMode: zod.z.enum(["stateful", "stateless"]).optional().default("stateful").describe('"stateful" = conversation history is persisted per thread (requires threadIdHeader). "stateless" = each request is independent; no memory across calls.'),
|
|
148
|
+
threadIdHeader: zod.z.string().optional().default("X-Thread-Id").describe('HTTP request header used to identify the conversation thread for stateful session routing (e.g. "X-Thread-Id", "X-Session-Id").')
|
|
149
|
+
});
|
|
150
|
+
var ApiSchema = zod.z.object({
|
|
151
|
+
type: zod.z.enum(["rest", "mcp", "grpc", "websocket"]),
|
|
152
|
+
port: zod.z.number().int().min(1).max(65535).optional(),
|
|
153
|
+
pathPrefix: zod.z.string().optional().describe('URL path prefix applied to all API routes (e.g. "/api/v1"). Useful when the agent runs behind a reverse proxy.'),
|
|
154
|
+
auth: zod.z.object({
|
|
155
|
+
type: zod.z.enum(["jwt", "apikey", "oauth2", "none"]),
|
|
156
|
+
jwksUri: refOrLiteral.optional().describe('JWKS endpoint URL used to fetch public keys for JWT signature verification (e.g. "https://your-auth.example.com/.well-known/jwks.json"). ' + refDesc),
|
|
157
|
+
header: zod.z.string().optional().describe('HTTP header name carrying the credential (e.g. "Authorization" for JWT bearer, "X-API-Key" for apikey auth). Defaults to "Authorization".')
|
|
158
|
+
}).optional(),
|
|
159
|
+
rateLimit: zod.z.object({
|
|
160
|
+
requestsPerMinute: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per minute. Excess requests receive HTTP 429."),
|
|
161
|
+
requestsPerHour: zod.z.number().int().min(1).optional().describe("Maximum requests allowed per client per hour. Excess requests receive HTTP 429.")
|
|
162
|
+
}).optional(),
|
|
163
|
+
streaming: zod.z.boolean().optional().default(false).describe("When true, the API supports streaming responses (SSE). Set true for conversational agents to reduce perceived latency."),
|
|
164
|
+
healthEndpoint: zod.z.string().optional().describe('URL path for the health check endpoint (e.g. "/health", "/ready"). Exposes agentspec health report. Defaults to "/agentspec/health" when using the SDK.'),
|
|
165
|
+
metricsEndpoint: zod.z.string().optional().describe('URL path where Prometheus-format metrics are exposed (e.g. "/metrics"). Required for observability.metrics to be scraped.'),
|
|
166
|
+
corsOrigins: zod.z.array(zod.z.string()).optional().describe('Allowed CORS origins for browser clients (e.g. ["https://app.example.com", "http://localhost:3000"]). Use ["*"] only for fully public APIs.'),
|
|
167
|
+
/** Declares the conversational chat endpoint for this agent */
|
|
168
|
+
chatEndpoint: ChatEndpointSchema.optional()
|
|
169
|
+
});
|
|
170
|
+
var SkillSchema = zod.z.object({
|
|
171
|
+
id: zod.z.string().describe('Skill identifier as registered in the runtime skill registry (e.g. "code-interpreter", "image-gen").'),
|
|
172
|
+
version: zod.z.string().optional().describe("Pinned skill version. Omit to use the latest available version. Semver range or exact version string.")
|
|
173
|
+
});
|
|
174
|
+
var InputGuardrailSchema = zod.z.discriminatedUnion("type", [
|
|
175
|
+
zod.z.object({
|
|
176
|
+
type: zod.z.literal("topic-filter"),
|
|
177
|
+
blockedTopics: zod.z.array(zod.z.string()).describe('Topics or keywords that should be blocked in user input (e.g. ["competitor-names", "legal-advice"]). Case-insensitive substring match against the full message.'),
|
|
178
|
+
action: zod.z.enum(["reject", "warn", "log"]).describe('"reject" = refuse the request and return an error, "warn" = continue but emit a warning event, "log" = silently log the violation.'),
|
|
179
|
+
message: zod.z.string().optional().describe("Custom error message returned to the user when a topic is blocked. Defaults to a generic refusal message.")
|
|
180
|
+
}),
|
|
181
|
+
zod.z.object({
|
|
182
|
+
type: zod.z.literal("pii-detector"),
|
|
183
|
+
action: zod.z.enum(["scrub", "reject", "warn"]).describe('"scrub" = replace detected PII with placeholders before passing to the model, "reject" = refuse the request, "warn" = pass through but emit a warning event.'),
|
|
184
|
+
fields: zod.z.array(zod.z.string()).optional().describe("Specific PII entity types to detect. When omitted, all PII types supported by the detection library are scanned. Use the same entity names as in spec.memory.hygiene.piiScrubFields.")
|
|
185
|
+
}),
|
|
186
|
+
zod.z.object({
|
|
187
|
+
type: zod.z.literal("prompt-injection"),
|
|
188
|
+
action: zod.z.enum(["reject", "warn"]),
|
|
189
|
+
sensitivity: zod.z.enum(["low", "medium", "high"]).optional().describe('Detection sensitivity. "low" = only block obvious injection patterns, "medium" = balanced (default), "high" = aggressive \u2014 may produce false positives on legitimate complex instructions.')
|
|
190
|
+
}),
|
|
191
|
+
zod.z.object({
|
|
192
|
+
type: zod.z.literal("custom"),
|
|
193
|
+
module: zod.z.string().describe('Path to the module implementing the guardrail function. Use "$file:guardrails/my_check.py" syntax.'),
|
|
194
|
+
function: zod.z.string().describe("Name of the guardrail function exported from the module. Signature: (input: str) -> None (raise to block) or (input: str) -> str (return scrubbed value)."),
|
|
195
|
+
action: zod.z.enum(["reject", "warn", "log"])
|
|
196
|
+
})
|
|
197
|
+
]);
|
|
198
|
+
var OutputGuardrailSchema = zod.z.discriminatedUnion("type", [
|
|
199
|
+
zod.z.object({
|
|
200
|
+
type: zod.z.literal("hallucination-detector"),
|
|
201
|
+
threshold: zod.z.number().min(0).max(1).describe("Confidence score below which a response is considered hallucinated (0.0\u20131.0). 0.0 = block everything, 1.0 = block nothing. Typical value: 0.5\u20130.7."),
|
|
202
|
+
action: zod.z.enum(["reject", "retry", "warn"]).describe('"reject" = return error to user, "retry" = re-invoke the model (up to maxRetries), "warn" = return response with a warning annotation.'),
|
|
203
|
+
maxRetries: zod.z.number().int().min(1).max(5).optional().describe('Maximum re-generation attempts before giving up. Only used when action is "retry". Defaults to 1.')
|
|
204
|
+
}),
|
|
205
|
+
zod.z.object({
|
|
206
|
+
type: zod.z.literal("toxicity-filter"),
|
|
207
|
+
threshold: zod.z.number().min(0).max(1).describe("Toxicity score above which a response is blocked (0.0\u20131.0). 0.0 = block all output, 1.0 = block nothing. Typical production value: 0.7\u20130.85."),
|
|
208
|
+
action: zod.z.enum(["reject", "warn"])
|
|
209
|
+
}),
|
|
210
|
+
zod.z.object({
|
|
211
|
+
type: zod.z.literal("pii-detector"),
|
|
212
|
+
action: zod.z.enum(["scrub", "reject", "warn"]).describe('"scrub" = replace PII in the model output before returning to the user, "reject" = discard the response and return an error, "warn" = return with a warning annotation.'),
|
|
213
|
+
fields: zod.z.array(zod.z.string()).optional().describe("Specific PII entity types to detect in model output. Omit to scan all types. Use the same entity names as in spec.memory.hygiene.piiScrubFields.")
|
|
214
|
+
}),
|
|
215
|
+
zod.z.object({
|
|
216
|
+
type: zod.z.literal("custom"),
|
|
217
|
+
module: zod.z.string().describe('Path to the module implementing the output guardrail. Use "$file:guardrails/output_check.py" syntax.'),
|
|
218
|
+
function: zod.z.string().describe("Name of the guardrail function. Signature: (output: str) -> None (raise to block) or (output: str) -> str (return transformed value)."),
|
|
219
|
+
action: zod.z.enum(["reject", "warn", "log"])
|
|
220
|
+
})
|
|
221
|
+
]);
|
|
222
|
+
var GuardrailsSchema = zod.z.object({
|
|
223
|
+
input: zod.z.array(InputGuardrailSchema).optional().describe("Guardrails applied to user input before it is sent to the model. Evaluated in order \u2014 first blocking rule wins."),
|
|
224
|
+
output: zod.z.array(OutputGuardrailSchema).optional().describe("Guardrails applied to model output before it is returned to the user. Evaluated in order \u2014 first blocking rule wins.")
|
|
225
|
+
});
|
|
226
|
+
var EvalMetricEnum = zod.z.enum([
|
|
227
|
+
"faithfulness",
|
|
228
|
+
// RAG: does the answer only use information from the retrieved context?
|
|
229
|
+
"answer_relevancy",
|
|
230
|
+
// Is the answer relevant to the question?
|
|
231
|
+
"answer_similarity",
|
|
232
|
+
// Semantic similarity between actual and expected output (embedding-based).
|
|
233
|
+
"hallucination",
|
|
234
|
+
// Does the answer contain claims not grounded in context or facts?
|
|
235
|
+
"toxicity",
|
|
236
|
+
// Does the answer contain harmful, offensive, or unsafe content?
|
|
237
|
+
"context_precision",
|
|
238
|
+
// RAG: are the retrieved chunks actually relevant to the question?
|
|
239
|
+
"context_recall",
|
|
240
|
+
// RAG: do the retrieved chunks cover the ground-truth answer?
|
|
241
|
+
"bias",
|
|
242
|
+
// Does the answer exhibit demographic or political bias?
|
|
243
|
+
"custom"
|
|
244
|
+
// Custom scorer defined in the evaluation framework config.
|
|
245
|
+
]);
|
|
246
|
+
var datasetPathDesc = 'Path to the evaluation dataset file (JSONL format). Use "$file:evals/my_dataset.jsonl" syntax. Each line is a JSON object with the following fields:\n input (required) \u2013 user query sent to the agent.\n expected (required) \u2013 expected output; used for string_match and answer_similarity scoring.\n context (optional) \u2013 string[] of retrieved chunks the agent actually used. Required for faithfulness, context_precision, hallucination metrics.\n reference_contexts (optional) \u2013 string[] of ground-truth relevant chunks. Required for context_recall metric.\n tags (optional) \u2013 string[] labels for filtering with --tag.\n metadata (optional) \u2013 arbitrary key/value pairs for reporting (e.g. {"difficulty": "hard"}).\nExample: {"input":"What is RAG?","expected":"Retrieval Augmented Generation","context":["RAG combines retrieval..."],"tags":["rag"]}';
|
|
247
|
+
var EvaluationSchema = zod.z.object({
|
|
248
|
+
framework: zod.z.enum(["deepeval", "braintrust", "langsmith", "ragas", "custom"]),
|
|
249
|
+
datasets: zod.z.array(
|
|
250
|
+
zod.z.object({
|
|
251
|
+
name: zod.z.string().describe('Unique dataset identifier, referenced by "agentspec evaluate --dataset <name>".'),
|
|
252
|
+
path: refOrLiteral.describe(datasetPathDesc),
|
|
253
|
+
metrics: zod.z.array(EvalMetricEnum).optional().describe(
|
|
254
|
+
'Metrics to compute for this specific dataset. Overrides (narrows) spec.evaluation.metrics for this dataset only. When omitted, the global spec.evaluation.metrics list is used. Example: a RAG dataset sets ["faithfulness","context_recall"] while a safety dataset sets ["toxicity","bias"].'
|
|
255
|
+
)
|
|
256
|
+
})
|
|
257
|
+
).optional().describe("Evaluation datasets. Each entry maps a name to a JSONL file and optionally scopes which metrics to compute."),
|
|
258
|
+
metrics: zod.z.array(EvalMetricEnum).optional().describe(
|
|
259
|
+
'Default metrics to compute across all datasets that do not declare their own metrics. "faithfulness", "context_precision", "context_recall" require the "context" field in dataset samples. "context_recall" additionally requires "reference_contexts". "custom" requires a scorer registered in the evaluation framework config.'
|
|
260
|
+
),
|
|
261
|
+
thresholds: zod.z.record(zod.z.string(), zod.z.number().min(0).max(1)).optional().describe('Pass/fail thresholds per metric (0.0\u20131.0). Key = metric name, value = minimum required score. Example: {"faithfulness": 0.8, "answer_relevancy": 0.75}. Used by ciGate and "agentspec evaluate".'),
|
|
262
|
+
ciGate: zod.z.boolean().optional().default(false).describe('When true, "agentspec evaluate" exits with code 1 if any metric falls below its threshold. Blocks CI pipelines on evaluation regression (EVAL-01).')
|
|
263
|
+
});
|
|
264
|
+
var ObservabilitySchema = zod.z.object({
|
|
265
|
+
tracing: zod.z.object({
|
|
266
|
+
backend: zod.z.enum([
|
|
267
|
+
"langfuse",
|
|
268
|
+
"langsmith",
|
|
269
|
+
"agentops",
|
|
270
|
+
"otel",
|
|
271
|
+
"honeycomb",
|
|
272
|
+
"datadog"
|
|
273
|
+
]),
|
|
274
|
+
endpoint: refOrLiteral.optional().describe('Tracing collector endpoint URL. Required for "otel", "honeycomb", and self-hosted backends. ' + refDesc),
|
|
275
|
+
publicKey: refOrLiteral.optional().describe("Public/project API key for the tracing backend (e.g. Langfuse public key). " + refDesc),
|
|
276
|
+
secretKey: refOrLiteral.optional().describe('Secret API key for the tracing backend (e.g. Langfuse secret key). Always use "$secret:" or "$env:" \u2014 never hardcode. ' + refDesc),
|
|
277
|
+
sampleRate: zod.z.number().min(0).max(1).optional().default(1).describe("Fraction of requests to trace (0.0\u20131.0). 1.0 = trace every request (default), 0.1 = trace 10% for high-traffic cost savings. Set to 1.0 during development.")
|
|
278
|
+
}).optional(),
|
|
279
|
+
metrics: zod.z.object({
|
|
280
|
+
backend: zod.z.enum(["opentelemetry", "prometheus", "datadog"]),
|
|
281
|
+
endpoint: refOrLiteral.optional().describe("Metrics export endpoint (e.g. OTLP endpoint for OpenTelemetry, Datadog agent URL). " + refDesc),
|
|
282
|
+
serviceName: zod.z.string().optional().describe("Service name used to label all emitted metrics and traces. Defaults to metadata.name. Use a consistent name across deployments for accurate dashboards.")
|
|
283
|
+
}).optional(),
|
|
284
|
+
logging: zod.z.object({
|
|
285
|
+
level: zod.z.enum(["debug", "info", "warn", "error"]).optional().default("info"),
|
|
286
|
+
structured: zod.z.boolean().optional().default(true).describe("When true, logs are emitted as JSON objects (structured logging). Required for log aggregation platforms (Datadog, Splunk, CloudWatch). Set false only for local development readability."),
|
|
287
|
+
redactFields: zod.z.array(zod.z.string()).optional().describe('Field names to redact from structured log output before writing to log sinks. Use the same values as spec.memory.hygiene.piiScrubFields plus any application-specific sensitive keys (e.g. ["password", "credit_card", "session_token"]).')
|
|
288
|
+
}).optional()
|
|
289
|
+
});
|
|
290
|
+
var ComplianceSuppressionSchema = zod.z.object({
|
|
291
|
+
rule: zod.z.string().describe('Rule ID to suppress (e.g. "SEC-LLM-03", "MEM-02"). Must match an existing compliance rule ID.'),
|
|
292
|
+
reason: zod.z.string().describe('Human-readable justification for suppressing this rule. Required for audit trails. Example: "Hallucination detection handled by external Lakera Guard proxy."'),
|
|
293
|
+
approvedBy: zod.z.string().optional().describe('Name or identifier of the person who approved this suppression (e.g. "security-team", "alice@example.com"). Used for compliance sign-off tracking.'),
|
|
294
|
+
expires: zod.z.string().optional().describe('ISO 8601 date (YYYY-MM-DD) when this suppression expires and should be re-evaluated (e.g. "2025-12-31"). Suppressions without an expiry are flagged in audits.')
|
|
295
|
+
});
|
|
296
|
+
var ComplianceSchema = zod.z.object({
|
|
297
|
+
packs: zod.z.array(
|
|
298
|
+
zod.z.enum([
|
|
299
|
+
"owasp-llm-top10",
|
|
300
|
+
"memory-hygiene",
|
|
301
|
+
"model-resilience",
|
|
302
|
+
"evaluation-coverage",
|
|
303
|
+
"observability"
|
|
304
|
+
])
|
|
305
|
+
).optional().describe('Compliance rule packs to evaluate during "agentspec audit". Each pack groups related rules. Omit to run all packs.'),
|
|
306
|
+
suppressions: zod.z.array(ComplianceSuppressionSchema).optional().describe("Rules to suppress in audit output. Each suppression requires a reason and optional expiry date. Over-suppression is itself flagged."),
|
|
307
|
+
auditSchedule: zod.z.enum(["daily", "weekly", "monthly", "on-change"]).optional().describe('How often the compliance audit should be re-run in CI. "on-change" = re-audit on every manifest change (recommended). Informational \u2014 enforcement depends on your CI configuration.')
|
|
308
|
+
});
|
|
309
|
+
var RequiresSchema = zod.z.object({
|
|
310
|
+
envVars: zod.z.array(zod.z.string()).optional().describe('Environment variable names that must be set at agent startup. The SDK validates these at load time and throws a descriptive error if any are missing (e.g. ["OPENAI_API_KEY", "DATABASE_URL"]).'),
|
|
311
|
+
services: zod.z.array(
|
|
312
|
+
zod.z.object({
|
|
313
|
+
type: zod.z.enum(["postgres", "redis", "mysql", "mongodb", "elasticsearch"]),
|
|
314
|
+
connection: refOrLiteral.describe("Connection URI for this service. " + refDesc)
|
|
315
|
+
})
|
|
316
|
+
).optional().describe("External services that must be reachable at startup. The SDK health check verifies TCP connectivity to each service."),
|
|
317
|
+
minimumMemoryMB: zod.z.number().int().min(128).optional().describe("Minimum RAM required to run the agent in megabytes (e.g. 512, 2048). Used by orchestrators for resource scheduling. Does not enforce a limit \u2014 purely declarative."),
|
|
318
|
+
nodeVersion: zod.z.string().optional().describe('Minimum Node.js version required, as a semver range or exact version (e.g. ">=20.0.0", "20.x"). Used by CI and orchestrators for compatibility checks.'),
|
|
319
|
+
pythonVersion: zod.z.string().optional().describe('Minimum Python version required, as a semver range or exact version (e.g. ">=3.11", "3.12"). Used by CI and orchestrators for compatibility checks.')
|
|
320
|
+
});
|
|
321
|
+
var HumanInTheLoopSchema = zod.z.object({
|
|
322
|
+
enabled: zod.z.boolean().optional().default(false),
|
|
323
|
+
approvalRequired: zod.z.array(
|
|
324
|
+
zod.z.enum([
|
|
325
|
+
"before-destructive-tool",
|
|
326
|
+
"before-external-call",
|
|
327
|
+
"on-low-confidence",
|
|
328
|
+
"on-high-cost",
|
|
329
|
+
"always"
|
|
330
|
+
])
|
|
331
|
+
).optional().describe('Conditions that pause execution and request human approval. "before-destructive-tool" = any tool with destructiveHint=true, "on-low-confidence" = model confidence below a threshold, "on-high-cost" = single call exceeds cost controls.'),
|
|
332
|
+
timeoutSeconds: zod.z.number().int().min(1).max(3600).optional().default(300).describe("Seconds to wait for human approval before executing the timeoutAction. Default 300s (5 minutes). Max 3600s (1 hour)."),
|
|
333
|
+
timeoutAction: zod.z.enum(["reject", "proceed", "escalate"]).optional().default("reject").describe('"reject" = abort the action and return an error (default, safest), "proceed" = continue without approval (use only for low-risk actions), "escalate" = forward to a higher-level approver or on-call channel.'),
|
|
334
|
+
notifyVia: zod.z.array(zod.z.enum(["slack", "email", "webhook", "console"])).optional().describe("Channels used to notify humans when approval is required. Multiple channels can be specified; all are notified simultaneously."),
|
|
335
|
+
webhookUrl: refOrLiteral.optional().describe('URL to POST approval requests to when notifyVia includes "webhook". The payload contains the pending action and a callback token. ' + refDesc)
|
|
336
|
+
});
|
|
337
|
+
var SpecSchema = zod.z.object({
|
|
338
|
+
model: ModelSchema,
|
|
339
|
+
prompts: PromptsSchema,
|
|
340
|
+
tools: zod.z.array(ToolSchema).optional(),
|
|
341
|
+
mcp: McpSchema.optional(),
|
|
342
|
+
memory: MemorySchema.optional(),
|
|
343
|
+
subagents: zod.z.array(SubagentSchema).optional(),
|
|
344
|
+
api: ApiSchema.optional(),
|
|
345
|
+
skills: zod.z.array(SkillSchema).optional(),
|
|
346
|
+
guardrails: GuardrailsSchema.optional(),
|
|
347
|
+
humanInTheLoop: HumanInTheLoopSchema.optional(),
|
|
348
|
+
evaluation: EvaluationSchema.optional(),
|
|
349
|
+
observability: ObservabilitySchema.optional(),
|
|
350
|
+
compliance: ComplianceSchema.optional(),
|
|
351
|
+
requires: RequiresSchema.optional()
|
|
352
|
+
});
|
|
353
|
+
var ManifestSchema = zod.z.object({
|
|
354
|
+
apiVersion: zod.z.literal("agentspec.io/v1"),
|
|
355
|
+
kind: zod.z.literal("AgentSpec"),
|
|
356
|
+
metadata: MetadataSchema,
|
|
357
|
+
spec: SpecSchema
|
|
358
|
+
});
|
|
359
|
+
function exportJsonSchema() {
|
|
360
|
+
return zodToJsonSchema.zodToJsonSchema(ManifestSchema, {
|
|
361
|
+
name: "AgentSpecManifest",
|
|
362
|
+
$refStrategy: "none"
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
function detectRefType(value) {
|
|
366
|
+
if (value.startsWith("$env:")) return "env";
|
|
367
|
+
if (value.startsWith("$secret:")) return "secret";
|
|
368
|
+
if (value.startsWith("$file:")) return "file";
|
|
369
|
+
if (value.startsWith("$func:")) return "func";
|
|
370
|
+
return "literal";
|
|
371
|
+
}
|
|
372
|
+
var BUILTINS = {
|
|
373
|
+
now_iso: () => (/* @__PURE__ */ new Date()).toISOString(),
|
|
374
|
+
now_unix: () => String(Math.floor(Date.now() / 1e3)),
|
|
375
|
+
now_date: () => (/* @__PURE__ */ new Date()).toISOString().split("T")[0]
|
|
376
|
+
};
|
|
377
|
+
function resolveRef(value, opts, { optional = false } = {}) {
|
|
378
|
+
const refType = detectRefType(value);
|
|
379
|
+
if (refType === "literal") return value;
|
|
380
|
+
if (refType === "env") {
|
|
381
|
+
const varName = value.slice("$env:".length);
|
|
382
|
+
const resolved = process.env[varName];
|
|
383
|
+
if (resolved === void 0) {
|
|
384
|
+
if (optional || opts.failOnMissingEnv === false) return "";
|
|
385
|
+
throw new Error(
|
|
386
|
+
`Missing environment variable: ${varName}
|
|
387
|
+
Referenced as ${value}
|
|
388
|
+
Set ${varName} in your .env file or environment`
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
return resolved;
|
|
392
|
+
}
|
|
393
|
+
if (refType === "secret") {
|
|
394
|
+
const secretName = value.slice("$secret:".length);
|
|
395
|
+
const backend = opts.secretBackend ?? (process.env["AGENTSPEC_SECRET_BACKEND"] || "env");
|
|
396
|
+
return resolveSecret(secretName, backend);
|
|
397
|
+
}
|
|
398
|
+
if (refType === "file") {
|
|
399
|
+
const filePath = value.slice("$file:".length);
|
|
400
|
+
const absPath = path.resolve(opts.baseDir, filePath);
|
|
401
|
+
const normalizedBase = path.resolve(opts.baseDir) + path.sep;
|
|
402
|
+
if (!absPath.startsWith(normalizedBase) && absPath !== path.resolve(opts.baseDir)) {
|
|
403
|
+
throw new Error(
|
|
404
|
+
`Path traversal detected in ${value}
|
|
405
|
+
Resolved to: ${absPath}
|
|
406
|
+
Which is outside baseDir: ${opts.baseDir}
|
|
407
|
+
Use paths relative to agent.yaml location only`
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
try {
|
|
411
|
+
return fs.readFileSync(absPath, "utf-8");
|
|
412
|
+
} catch {
|
|
413
|
+
if (optional) return "";
|
|
414
|
+
throw new Error(
|
|
415
|
+
`Cannot read file referenced by ${value}
|
|
416
|
+
Resolved path: ${absPath}`
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
if (refType === "func") {
|
|
421
|
+
const funcName = value.slice("$func:".length);
|
|
422
|
+
const fn = BUILTINS[funcName];
|
|
423
|
+
if (!fn) {
|
|
424
|
+
throw new Error(
|
|
425
|
+
`Unknown $func: ${funcName}
|
|
426
|
+
Available functions: ${Object.keys(BUILTINS).join(", ")}`
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
return fn();
|
|
430
|
+
}
|
|
431
|
+
return value;
|
|
432
|
+
}
|
|
433
|
+
function resolveSecret(name, backend) {
|
|
434
|
+
switch (backend) {
|
|
435
|
+
case "env": {
|
|
436
|
+
const envKey = `AGENTSPEC_SECRET_${name.toUpperCase().replace(/-/g, "_")}`;
|
|
437
|
+
const val = process.env[envKey];
|
|
438
|
+
if (!val) {
|
|
439
|
+
throw new Error(
|
|
440
|
+
`Secret "${name}" not found.
|
|
441
|
+
In 'env' backend, set environment variable: ${envKey}`
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
return val;
|
|
445
|
+
}
|
|
446
|
+
case "vault":
|
|
447
|
+
case "aws":
|
|
448
|
+
case "gcp":
|
|
449
|
+
case "azure":
|
|
450
|
+
throw new Error(
|
|
451
|
+
`Secret backend "${backend}" requires async resolution.
|
|
452
|
+
Use loadManifestWithSecrets() which pre-fetches all secrets,
|
|
453
|
+
or set AGENTSPEC_SECRET_BACKEND=env and map secrets to env vars.`
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
function resolveRefs(obj, opts) {
|
|
458
|
+
if (typeof obj === "string") {
|
|
459
|
+
return resolveRef(obj, opts, { optional: true });
|
|
460
|
+
}
|
|
461
|
+
if (Array.isArray(obj)) {
|
|
462
|
+
return obj.map((item) => resolveRefs(item, opts));
|
|
463
|
+
}
|
|
464
|
+
if (obj !== null && typeof obj === "object") {
|
|
465
|
+
const result = {};
|
|
466
|
+
for (const [key, val] of Object.entries(obj)) {
|
|
467
|
+
result[key] = resolveRefs(val, opts);
|
|
468
|
+
}
|
|
469
|
+
return result;
|
|
470
|
+
}
|
|
471
|
+
return obj;
|
|
472
|
+
}
|
|
473
|
+
function collectEnvRefs(obj, refs = /* @__PURE__ */ new Set()) {
|
|
474
|
+
if (typeof obj === "string" && obj.startsWith("$env:")) {
|
|
475
|
+
refs.add(obj.slice("$env:".length));
|
|
476
|
+
} else if (Array.isArray(obj)) {
|
|
477
|
+
for (const item of obj) collectEnvRefs(item, refs);
|
|
478
|
+
} else if (obj !== null && typeof obj === "object") {
|
|
479
|
+
for (const val of Object.values(obj)) {
|
|
480
|
+
collectEnvRefs(val, refs);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return refs;
|
|
484
|
+
}
|
|
485
|
+
function collectFileRefs(obj, refs = /* @__PURE__ */ new Set()) {
|
|
486
|
+
if (typeof obj === "string" && obj.startsWith("$file:")) {
|
|
487
|
+
refs.add(obj.slice("$file:".length));
|
|
488
|
+
} else if (Array.isArray(obj)) {
|
|
489
|
+
for (const item of obj) collectFileRefs(item, refs);
|
|
490
|
+
} else if (obj !== null && typeof obj === "object") {
|
|
491
|
+
for (const val of Object.values(obj)) {
|
|
492
|
+
collectFileRefs(val, refs);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
return refs;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// src/loader/index.ts
|
|
499
|
+
function loadManifest(filePath, opts = {}) {
|
|
500
|
+
const absPath = path.resolve(filePath);
|
|
501
|
+
const baseDir = path.dirname(absPath);
|
|
502
|
+
let raw;
|
|
503
|
+
try {
|
|
504
|
+
raw = fs.readFileSync(absPath, "utf-8");
|
|
505
|
+
} catch (err) {
|
|
506
|
+
throw new Error(`Cannot read manifest: ${absPath}
|
|
507
|
+
${String(err)}`);
|
|
508
|
+
}
|
|
509
|
+
let parsed;
|
|
510
|
+
try {
|
|
511
|
+
parsed = yaml__default.default.load(raw);
|
|
512
|
+
} catch (err) {
|
|
513
|
+
throw new Error(`Invalid YAML in ${absPath}
|
|
514
|
+
${String(err)}`);
|
|
515
|
+
}
|
|
516
|
+
const toValidate = opts.resolve === true ? resolveRefs(parsed, {
|
|
517
|
+
baseDir,
|
|
518
|
+
secretBackend: opts.secretBackend,
|
|
519
|
+
failOnMissingEnv: opts.failOnMissingEnv ?? true
|
|
520
|
+
}) : parsed;
|
|
521
|
+
const manifest = ManifestSchema.parse(toValidate);
|
|
522
|
+
return { manifest, filePath: absPath, baseDir, raw };
|
|
523
|
+
}
|
|
524
|
+
function tryLoadManifest(filePath, opts = {}) {
|
|
525
|
+
try {
|
|
526
|
+
return { ok: true, data: loadManifest(filePath, opts) };
|
|
527
|
+
} catch (err) {
|
|
528
|
+
return { ok: false, error: err instanceof Error ? err : new Error(String(err)) };
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// src/loader/migrations/v1alpha1-to-v1.ts
|
|
533
|
+
var v1alpha1ToV1 = {
|
|
534
|
+
from: "agentspec/v1alpha1",
|
|
535
|
+
to: "agentspec.io/v1",
|
|
536
|
+
migrate(raw) {
|
|
537
|
+
return {
|
|
538
|
+
...raw,
|
|
539
|
+
apiVersion: "agentspec.io/v1",
|
|
540
|
+
kind: raw.kind ?? "AgentSpec"
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
};
|
|
544
|
+
|
|
545
|
+
// src/loader/migrations/index.ts
|
|
546
|
+
var migrations = [v1alpha1ToV1];
|
|
547
|
+
var LATEST_API_VERSION = "agentspec.io/v1";
|
|
548
|
+
function migrateManifest(raw) {
|
|
549
|
+
let current = raw;
|
|
550
|
+
const migrationsApplied = [];
|
|
551
|
+
for (const m of migrations) {
|
|
552
|
+
const version = current.apiVersion ?? "";
|
|
553
|
+
if (version === m.from || version.startsWith(m.from)) {
|
|
554
|
+
current = m.migrate(current);
|
|
555
|
+
migrationsApplied.push(`${m.from} \u2192 ${m.to}`);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
return { result: current, migrationsApplied };
|
|
559
|
+
}
|
|
560
|
+
function detectVersion(raw) {
|
|
561
|
+
return raw.apiVersion ?? "unknown";
|
|
562
|
+
}
|
|
563
|
+
function isLatestVersion(raw) {
|
|
564
|
+
return detectVersion(raw) === LATEST_API_VERSION;
|
|
565
|
+
}
|
|
566
|
+
function runEnvChecks(manifest, rawManifest) {
|
|
567
|
+
const checks = [];
|
|
568
|
+
const envRefs = collectEnvRefs(rawManifest);
|
|
569
|
+
const declared = manifest.spec.requires?.envVars ?? [];
|
|
570
|
+
for (const v of declared) envRefs.add(v);
|
|
571
|
+
for (const varName of envRefs) {
|
|
572
|
+
const present = process.env[varName] !== void 0 && process.env[varName] !== "";
|
|
573
|
+
checks.push({
|
|
574
|
+
id: `env:${varName}`,
|
|
575
|
+
category: "env",
|
|
576
|
+
status: present ? "pass" : "fail",
|
|
577
|
+
severity: "error",
|
|
578
|
+
message: present ? void 0 : `Environment variable ${varName} is not set`,
|
|
579
|
+
remediation: present ? void 0 : `Set ${varName} in your .env file or environment`
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
return checks;
|
|
583
|
+
}
|
|
584
|
+
function runFileChecks(rawManifest, baseDir) {
|
|
585
|
+
const checks = [];
|
|
586
|
+
const fileRefs = collectFileRefs(rawManifest);
|
|
587
|
+
for (const filePath of fileRefs) {
|
|
588
|
+
const absPath = path.resolve(baseDir, filePath);
|
|
589
|
+
const exists = fs.existsSync(absPath);
|
|
590
|
+
checks.push({
|
|
591
|
+
id: `file:${filePath}`,
|
|
592
|
+
category: "file",
|
|
593
|
+
status: exists ? "pass" : "fail",
|
|
594
|
+
severity: "error",
|
|
595
|
+
message: exists ? void 0 : `Referenced file not found: ${filePath}`,
|
|
596
|
+
remediation: exists ? void 0 : `Create the file at ${absPath} or update the $file: reference`
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
return checks;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// src/health/checks/model.check.ts
|
|
603
|
+
var PROVIDER_ENDPOINTS = {
|
|
604
|
+
openai: "https://api.openai.com/v1/models",
|
|
605
|
+
anthropic: "https://api.anthropic.com/v1/models",
|
|
606
|
+
groq: "https://api.groq.com/openai/v1/models",
|
|
607
|
+
azure: "https://management.azure.com",
|
|
608
|
+
google: "https://generativelanguage.googleapis.com",
|
|
609
|
+
mistral: "https://api.mistral.ai/v1/models",
|
|
610
|
+
cohere: "https://api.cohere.ai/v1",
|
|
611
|
+
together: "https://api.together.xyz/v1/models",
|
|
612
|
+
fireworks: "https://api.fireworks.ai/inference/v1/models"
|
|
613
|
+
};
|
|
614
|
+
async function checkModelEndpoint(provider, apiKey, timeoutMs = 5e3) {
|
|
615
|
+
const url = PROVIDER_ENDPOINTS[provider.toLowerCase()];
|
|
616
|
+
if (!url) {
|
|
617
|
+
return {
|
|
618
|
+
reachable: true
|
|
619
|
+
// Unknown provider — assume reachable, can't check
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
const start = Date.now();
|
|
623
|
+
const controller = new AbortController();
|
|
624
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
625
|
+
try {
|
|
626
|
+
const headers = {
|
|
627
|
+
"Authorization": `Bearer ${apiKey}`,
|
|
628
|
+
"Content-Type": "application/json"
|
|
629
|
+
};
|
|
630
|
+
if (provider.toLowerCase() === "anthropic") {
|
|
631
|
+
headers["x-api-key"] = apiKey;
|
|
632
|
+
headers["anthropic-version"] = "2023-06-01";
|
|
633
|
+
delete headers["Authorization"];
|
|
634
|
+
}
|
|
635
|
+
const res = await fetch(url, { method: "GET", headers, signal: controller.signal });
|
|
636
|
+
const latencyMs = Date.now() - start;
|
|
637
|
+
const reachable = res.status < 500;
|
|
638
|
+
return {
|
|
639
|
+
reachable,
|
|
640
|
+
latencyMs,
|
|
641
|
+
error: !reachable ? `HTTP ${res.status}` : void 0
|
|
642
|
+
};
|
|
643
|
+
} catch (err) {
|
|
644
|
+
const latencyMs = Date.now() - start;
|
|
645
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
646
|
+
return {
|
|
647
|
+
reachable: false,
|
|
648
|
+
latencyMs,
|
|
649
|
+
error: isTimeout ? "Request timed out" : String(err)
|
|
650
|
+
};
|
|
651
|
+
} finally {
|
|
652
|
+
clearTimeout(timeout);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
function resolveApiKey(ref) {
|
|
656
|
+
if (ref.startsWith("$env:")) return process.env[ref.slice(5)] ?? "";
|
|
657
|
+
if (ref.startsWith("$")) return "";
|
|
658
|
+
return ref;
|
|
659
|
+
}
|
|
660
|
+
async function runModelChecks(model) {
|
|
661
|
+
const checks = [];
|
|
662
|
+
const apiKey = resolveApiKey(model.apiKey);
|
|
663
|
+
if (apiKey) {
|
|
664
|
+
const { reachable, latencyMs, error } = await checkModelEndpoint(model.provider, apiKey);
|
|
665
|
+
checks.push({
|
|
666
|
+
id: `model:${model.provider}/${model.id}`,
|
|
667
|
+
category: "model",
|
|
668
|
+
status: reachable ? "pass" : "fail",
|
|
669
|
+
severity: "error",
|
|
670
|
+
latencyMs,
|
|
671
|
+
message: reachable ? void 0 : `Model endpoint for ${model.provider} is unreachable: ${error}`,
|
|
672
|
+
remediation: reachable ? void 0 : `Check that ${model.apiKey.startsWith("$") ? model.apiKey : "[api-key]"} is set and the ${model.provider} API is reachable`
|
|
673
|
+
});
|
|
674
|
+
} else {
|
|
675
|
+
checks.push({
|
|
676
|
+
id: `model:${model.provider}/${model.id}`,
|
|
677
|
+
category: "model",
|
|
678
|
+
status: "skip",
|
|
679
|
+
severity: "error",
|
|
680
|
+
message: `Cannot check model endpoint: API key reference not resolved (${model.apiKey})`
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
if (model.fallback) {
|
|
684
|
+
const fallbackKey = resolveApiKey(model.fallback.apiKey);
|
|
685
|
+
if (fallbackKey) {
|
|
686
|
+
const { reachable, latencyMs, error } = await checkModelEndpoint(
|
|
687
|
+
model.fallback.provider,
|
|
688
|
+
fallbackKey
|
|
689
|
+
);
|
|
690
|
+
checks.push({
|
|
691
|
+
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
692
|
+
category: "model-fallback",
|
|
693
|
+
status: reachable ? "pass" : "fail",
|
|
694
|
+
severity: "warning",
|
|
695
|
+
latencyMs,
|
|
696
|
+
message: reachable ? void 0 : `Fallback model endpoint unreachable: ${error}`,
|
|
697
|
+
remediation: reachable ? void 0 : `Check that ${model.fallback.apiKey.startsWith("$") ? model.fallback.apiKey : "[api-key]"} is set and ${model.fallback.provider} API is reachable`
|
|
698
|
+
});
|
|
699
|
+
} else {
|
|
700
|
+
checks.push({
|
|
701
|
+
id: `model-fallback:${model.fallback.provider}/${model.fallback.id}`,
|
|
702
|
+
category: "model-fallback",
|
|
703
|
+
status: "skip",
|
|
704
|
+
severity: "warning",
|
|
705
|
+
message: `Cannot check fallback model: API key not resolved (${model.fallback.apiKey})`
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
return checks;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// src/health/checks/mcp.check.ts
|
|
713
|
+
async function runMcpChecks(servers) {
|
|
714
|
+
const checks = [];
|
|
715
|
+
for (const server of servers) {
|
|
716
|
+
if (server.transport === "sse" || server.transport === "http") {
|
|
717
|
+
const url = server.url;
|
|
718
|
+
if (url && !url.startsWith("$")) {
|
|
719
|
+
const check = await checkHttpMcpServer(server.name, url);
|
|
720
|
+
checks.push(check);
|
|
721
|
+
} else {
|
|
722
|
+
checks.push({
|
|
723
|
+
id: `mcp:${server.name}`,
|
|
724
|
+
category: "mcp",
|
|
725
|
+
status: "skip",
|
|
726
|
+
severity: "warning",
|
|
727
|
+
message: `Cannot check MCP server ${server.name}: URL not resolved`
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
} else if (server.transport === "stdio") {
|
|
731
|
+
const check = await checkStdioMcpServer(server);
|
|
732
|
+
checks.push(check);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return checks;
|
|
736
|
+
}
|
|
737
|
+
async function checkHttpMcpServer(name, url) {
|
|
738
|
+
const controller = new AbortController();
|
|
739
|
+
const timeout = setTimeout(() => controller.abort(), 5e3);
|
|
740
|
+
const start = Date.now();
|
|
741
|
+
try {
|
|
742
|
+
const res = await fetch(url, { signal: controller.signal });
|
|
743
|
+
const latencyMs = Date.now() - start;
|
|
744
|
+
return {
|
|
745
|
+
id: `mcp:${name}`,
|
|
746
|
+
category: "mcp",
|
|
747
|
+
status: res.ok || res.status === 404 ? "pass" : "fail",
|
|
748
|
+
severity: "warning",
|
|
749
|
+
latencyMs,
|
|
750
|
+
message: res.status >= 500 ? `MCP server ${name} returned HTTP ${res.status}` : void 0
|
|
751
|
+
};
|
|
752
|
+
} catch (err) {
|
|
753
|
+
const latencyMs = Date.now() - start;
|
|
754
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
755
|
+
return {
|
|
756
|
+
id: `mcp:${name}`,
|
|
757
|
+
category: "mcp",
|
|
758
|
+
status: "fail",
|
|
759
|
+
severity: "warning",
|
|
760
|
+
latencyMs,
|
|
761
|
+
message: isTimeout ? `MCP server ${name} health check timed out` : `MCP server ${name} unreachable: ${String(err)}`,
|
|
762
|
+
remediation: `Check that the MCP server ${name} is running and accessible`
|
|
763
|
+
};
|
|
764
|
+
} finally {
|
|
765
|
+
clearTimeout(timeout);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
async function checkStdioMcpServer(server) {
|
|
769
|
+
const command = server.command;
|
|
770
|
+
if (!command) {
|
|
771
|
+
return {
|
|
772
|
+
id: `mcp:${server.name}`,
|
|
773
|
+
category: "mcp",
|
|
774
|
+
status: "skip",
|
|
775
|
+
severity: "warning",
|
|
776
|
+
message: `MCP server ${server.name} has no command specified`
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
if (!/^[a-zA-Z0-9._-]+$/.test(command)) {
|
|
780
|
+
return {
|
|
781
|
+
id: `mcp:${server.name}`,
|
|
782
|
+
category: "mcp",
|
|
783
|
+
status: "fail",
|
|
784
|
+
severity: "warning",
|
|
785
|
+
message: `MCP server ${server.name} has unsafe command name: "${command}"`,
|
|
786
|
+
remediation: "MCP server command must match /^[a-zA-Z0-9._-]+$/"
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
try {
|
|
790
|
+
const { execFileSync } = await import('child_process');
|
|
791
|
+
const whichBin = process.platform === "win32" ? "where" : "which";
|
|
792
|
+
execFileSync(whichBin, [command], { stdio: "ignore" });
|
|
793
|
+
return {
|
|
794
|
+
id: `mcp:${server.name}`,
|
|
795
|
+
category: "mcp",
|
|
796
|
+
status: "pass",
|
|
797
|
+
severity: "warning",
|
|
798
|
+
message: void 0
|
|
799
|
+
};
|
|
800
|
+
} catch {
|
|
801
|
+
return {
|
|
802
|
+
id: `mcp:${server.name}`,
|
|
803
|
+
category: "mcp",
|
|
804
|
+
status: "fail",
|
|
805
|
+
severity: "warning",
|
|
806
|
+
message: `MCP server ${server.name} command not found: ${command}`,
|
|
807
|
+
remediation: `Install ${command} or add it to PATH`
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// src/health/checks/memory.check.ts
|
|
813
|
+
async function runMemoryChecks(memory) {
|
|
814
|
+
const checks = [];
|
|
815
|
+
if (memory.shortTerm) {
|
|
816
|
+
const st = memory.shortTerm;
|
|
817
|
+
const connection = st.connection;
|
|
818
|
+
if (st.backend === "redis" && connection && !connection.startsWith("$")) {
|
|
819
|
+
checks.push(await checkRedis(connection));
|
|
820
|
+
} else if (st.backend === "in-memory") {
|
|
821
|
+
checks.push({
|
|
822
|
+
id: "memory.shortTerm:in-memory",
|
|
823
|
+
category: "memory",
|
|
824
|
+
status: "pass",
|
|
825
|
+
severity: "warning",
|
|
826
|
+
message: void 0
|
|
827
|
+
});
|
|
828
|
+
} else {
|
|
829
|
+
checks.push({
|
|
830
|
+
id: `memory.shortTerm:${st.backend}`,
|
|
831
|
+
category: "memory",
|
|
832
|
+
status: "skip",
|
|
833
|
+
severity: "warning",
|
|
834
|
+
message: `Cannot check ${st.backend} connection: connection string not resolved`
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
if (memory.longTerm) {
|
|
839
|
+
const lt = memory.longTerm;
|
|
840
|
+
const conn = lt.connectionString;
|
|
841
|
+
if (!conn.startsWith("$")) {
|
|
842
|
+
if (lt.backend === "postgres") {
|
|
843
|
+
checks.push(await checkPostgres(conn));
|
|
844
|
+
} else {
|
|
845
|
+
checks.push({
|
|
846
|
+
id: `memory.longTerm:${lt.backend}`,
|
|
847
|
+
category: "memory",
|
|
848
|
+
status: "skip",
|
|
849
|
+
severity: "warning",
|
|
850
|
+
message: `${lt.backend} connectivity check not yet supported`
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
} else {
|
|
854
|
+
checks.push({
|
|
855
|
+
id: `memory.longTerm:${lt.backend}`,
|
|
856
|
+
category: "memory",
|
|
857
|
+
status: "skip",
|
|
858
|
+
severity: "warning",
|
|
859
|
+
message: `Cannot check ${lt.backend}: connection string not resolved (${conn})`
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
if (memory.vector) {
|
|
864
|
+
const v = memory.vector;
|
|
865
|
+
const conn = v.connectionString ?? v.apiKey;
|
|
866
|
+
if (conn && !conn.startsWith("$")) {
|
|
867
|
+
if (v.backend === "pgvector") {
|
|
868
|
+
checks.push(await checkPostgres(conn, "memory.vector:pgvector"));
|
|
869
|
+
} else {
|
|
870
|
+
checks.push({
|
|
871
|
+
id: `memory.vector:${v.backend}`,
|
|
872
|
+
category: "memory",
|
|
873
|
+
status: "skip",
|
|
874
|
+
severity: "warning",
|
|
875
|
+
message: `${v.backend} connectivity check not yet supported`
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
} else {
|
|
879
|
+
checks.push({
|
|
880
|
+
id: `memory.vector:${v.backend}`,
|
|
881
|
+
category: "memory",
|
|
882
|
+
status: "skip",
|
|
883
|
+
severity: "warning",
|
|
884
|
+
message: `Cannot check ${v.backend}: connection not resolved`
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
return checks;
|
|
889
|
+
}
|
|
890
|
+
async function checkRedis(url) {
|
|
891
|
+
const start = Date.now();
|
|
892
|
+
try {
|
|
893
|
+
const { createConnection } = await import('net');
|
|
894
|
+
const parsed = new URL(url);
|
|
895
|
+
const port = parseInt(parsed.port || "6379", 10);
|
|
896
|
+
const host = parsed.hostname || "localhost";
|
|
897
|
+
await new Promise((resolve5, reject) => {
|
|
898
|
+
const socket = createConnection({ port, host }, () => {
|
|
899
|
+
socket.destroy();
|
|
900
|
+
resolve5();
|
|
901
|
+
});
|
|
902
|
+
const timer = setTimeout(() => {
|
|
903
|
+
socket.destroy();
|
|
904
|
+
reject(new Error("timeout"));
|
|
905
|
+
}, 3e3);
|
|
906
|
+
socket.on("error", (err) => {
|
|
907
|
+
clearTimeout(timer);
|
|
908
|
+
reject(err);
|
|
909
|
+
});
|
|
910
|
+
socket.on("connect", () => clearTimeout(timer));
|
|
911
|
+
});
|
|
912
|
+
return {
|
|
913
|
+
id: "memory.shortTerm:redis",
|
|
914
|
+
category: "memory",
|
|
915
|
+
status: "pass",
|
|
916
|
+
severity: "warning",
|
|
917
|
+
latencyMs: Date.now() - start
|
|
918
|
+
};
|
|
919
|
+
} catch (err) {
|
|
920
|
+
return {
|
|
921
|
+
id: "memory.shortTerm:redis",
|
|
922
|
+
category: "memory",
|
|
923
|
+
status: "fail",
|
|
924
|
+
severity: "warning",
|
|
925
|
+
latencyMs: Date.now() - start,
|
|
926
|
+
message: `Redis not reachable: ${String(err)}`,
|
|
927
|
+
remediation: "Check REDIS_URL and ensure your Redis instance is running"
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
async function checkPostgres(connStr, id = "memory.longTerm:postgres") {
|
|
932
|
+
const start = Date.now();
|
|
933
|
+
try {
|
|
934
|
+
const parsed = new URL(connStr);
|
|
935
|
+
const port = parseInt(parsed.port || "5432", 10);
|
|
936
|
+
const host = parsed.hostname || "localhost";
|
|
937
|
+
const { createConnection } = await import('net');
|
|
938
|
+
await new Promise((resolve5, reject) => {
|
|
939
|
+
const socket = createConnection({ port, host }, () => {
|
|
940
|
+
socket.destroy();
|
|
941
|
+
resolve5();
|
|
942
|
+
});
|
|
943
|
+
const timer = setTimeout(() => {
|
|
944
|
+
socket.destroy();
|
|
945
|
+
reject(new Error("timeout"));
|
|
946
|
+
}, 3e3);
|
|
947
|
+
socket.on("error", (err) => {
|
|
948
|
+
clearTimeout(timer);
|
|
949
|
+
reject(err);
|
|
950
|
+
});
|
|
951
|
+
socket.on("connect", () => clearTimeout(timer));
|
|
952
|
+
});
|
|
953
|
+
return {
|
|
954
|
+
id,
|
|
955
|
+
category: "memory",
|
|
956
|
+
status: "pass",
|
|
957
|
+
severity: "warning",
|
|
958
|
+
latencyMs: Date.now() - start
|
|
959
|
+
};
|
|
960
|
+
} catch (err) {
|
|
961
|
+
return {
|
|
962
|
+
id,
|
|
963
|
+
category: "memory",
|
|
964
|
+
status: "fail",
|
|
965
|
+
severity: "warning",
|
|
966
|
+
latencyMs: Date.now() - start,
|
|
967
|
+
message: `PostgreSQL not reachable: ${String(err)}`,
|
|
968
|
+
remediation: "Check DATABASE_URL and ensure your PostgreSQL instance is running"
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
// src/health/checks/secret.check.ts
|
|
974
|
+
async function runSecretChecks() {
|
|
975
|
+
const backend = process.env["AGENTSPEC_SECRET_BACKEND"] ?? "env";
|
|
976
|
+
if (backend === "env") {
|
|
977
|
+
return [
|
|
978
|
+
{
|
|
979
|
+
id: "secret:backend",
|
|
980
|
+
category: "env",
|
|
981
|
+
status: "pass",
|
|
982
|
+
severity: "info",
|
|
983
|
+
message: "Secret backend is env \u2014 no external secret manager required"
|
|
984
|
+
}
|
|
985
|
+
];
|
|
986
|
+
}
|
|
987
|
+
if (backend === "vault") {
|
|
988
|
+
return [await checkVault()];
|
|
989
|
+
}
|
|
990
|
+
if (backend === "aws") {
|
|
991
|
+
return [await checkAws()];
|
|
992
|
+
}
|
|
993
|
+
return [
|
|
994
|
+
{
|
|
995
|
+
id: "secret:backend",
|
|
996
|
+
category: "env",
|
|
997
|
+
status: "skip",
|
|
998
|
+
severity: "info",
|
|
999
|
+
message: `Secret backend '${backend}' reachability check not yet supported \u2014 verify manually`
|
|
1000
|
+
}
|
|
1001
|
+
];
|
|
1002
|
+
}
|
|
1003
|
+
async function checkVault() {
|
|
1004
|
+
const vaultAddr = process.env["VAULT_ADDR"];
|
|
1005
|
+
if (!vaultAddr) {
|
|
1006
|
+
return {
|
|
1007
|
+
id: "secret:vault",
|
|
1008
|
+
category: "env",
|
|
1009
|
+
status: "skip",
|
|
1010
|
+
severity: "warning",
|
|
1011
|
+
message: "AGENTSPEC_SECRET_BACKEND=vault but VAULT_ADDR is not set",
|
|
1012
|
+
remediation: "Set VAULT_ADDR to your Vault server address (e.g. https://vault.example.com)"
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
const controller = new AbortController();
|
|
1016
|
+
const timer = setTimeout(() => controller.abort(), 3e3);
|
|
1017
|
+
const start = Date.now();
|
|
1018
|
+
try {
|
|
1019
|
+
const url = `${vaultAddr.replace(/\/$/, "")}/v1/sys/health`;
|
|
1020
|
+
const res = await fetch(url, { signal: controller.signal });
|
|
1021
|
+
const ok = res.status < 600;
|
|
1022
|
+
return {
|
|
1023
|
+
id: "secret:vault",
|
|
1024
|
+
category: "env",
|
|
1025
|
+
status: ok ? "pass" : "fail",
|
|
1026
|
+
severity: "error",
|
|
1027
|
+
latencyMs: Date.now() - start,
|
|
1028
|
+
message: ok ? void 0 : `Vault returned unexpected status ${res.status}`
|
|
1029
|
+
};
|
|
1030
|
+
} catch (err) {
|
|
1031
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
1032
|
+
return {
|
|
1033
|
+
id: "secret:vault",
|
|
1034
|
+
category: "env",
|
|
1035
|
+
status: "fail",
|
|
1036
|
+
severity: "error",
|
|
1037
|
+
latencyMs: Date.now() - start,
|
|
1038
|
+
message: isTimeout ? `Vault health check timed out (VAULT_ADDR=${vaultAddr})` : `Vault not reachable at ${vaultAddr}: ${String(err)}`,
|
|
1039
|
+
remediation: "Check that VAULT_ADDR is correct and the Vault server is running"
|
|
1040
|
+
};
|
|
1041
|
+
} finally {
|
|
1042
|
+
clearTimeout(timer);
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
async function checkAws() {
|
|
1046
|
+
const stsEndpoint = "https://sts.amazonaws.com";
|
|
1047
|
+
const controller = new AbortController();
|
|
1048
|
+
const timer = setTimeout(() => controller.abort(), 3e3);
|
|
1049
|
+
const start = Date.now();
|
|
1050
|
+
try {
|
|
1051
|
+
await fetch(stsEndpoint, { signal: controller.signal });
|
|
1052
|
+
return {
|
|
1053
|
+
id: "secret:aws",
|
|
1054
|
+
category: "env",
|
|
1055
|
+
status: "pass",
|
|
1056
|
+
severity: "info",
|
|
1057
|
+
latencyMs: Date.now() - start,
|
|
1058
|
+
message: "AWS STS endpoint is reachable"
|
|
1059
|
+
};
|
|
1060
|
+
} catch (err) {
|
|
1061
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
1062
|
+
return {
|
|
1063
|
+
id: "secret:aws",
|
|
1064
|
+
category: "env",
|
|
1065
|
+
status: "fail",
|
|
1066
|
+
severity: "error",
|
|
1067
|
+
latencyMs: Date.now() - start,
|
|
1068
|
+
message: isTimeout ? "AWS STS endpoint check timed out" : `AWS STS endpoint unreachable: ${String(err)}`,
|
|
1069
|
+
remediation: "Check network connectivity and AWS credentials configuration"
|
|
1070
|
+
};
|
|
1071
|
+
} finally {
|
|
1072
|
+
clearTimeout(timer);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
// src/health/checks/service.check.ts
|
|
1077
|
+
var TIMEOUT_MS = 3e3;
|
|
1078
|
+
async function runServiceChecks(services) {
|
|
1079
|
+
return Promise.all(services.map((svc) => checkService(svc)));
|
|
1080
|
+
}
|
|
1081
|
+
async function checkService(svc) {
|
|
1082
|
+
const id = `service:${svc.type}`;
|
|
1083
|
+
const connection = resolveConnection(svc.connection);
|
|
1084
|
+
if (!connection) {
|
|
1085
|
+
return {
|
|
1086
|
+
id,
|
|
1087
|
+
category: "service",
|
|
1088
|
+
status: "skip",
|
|
1089
|
+
severity: "warning",
|
|
1090
|
+
message: `Cannot check ${svc.type}: connection string not resolved (${svc.connection})`
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
const parsed = parseConnectionUrl(connection);
|
|
1094
|
+
if (!parsed) {
|
|
1095
|
+
return {
|
|
1096
|
+
id,
|
|
1097
|
+
category: "service",
|
|
1098
|
+
status: "skip",
|
|
1099
|
+
severity: "warning",
|
|
1100
|
+
message: `Cannot check ${svc.type}: unrecognised connection string format`
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
const hostRisk = classifyHost(parsed.host);
|
|
1104
|
+
if (hostRisk) {
|
|
1105
|
+
return {
|
|
1106
|
+
id,
|
|
1107
|
+
category: "service",
|
|
1108
|
+
status: "skip",
|
|
1109
|
+
severity: "warning",
|
|
1110
|
+
message: `Cannot check ${svc.type}: ${hostRisk}`
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
if (!isSupportedType(svc.type)) {
|
|
1114
|
+
return {
|
|
1115
|
+
id,
|
|
1116
|
+
category: "service",
|
|
1117
|
+
status: "skip",
|
|
1118
|
+
severity: "info",
|
|
1119
|
+
message: `TCP check not implemented for service type "${svc.type}"`
|
|
1120
|
+
};
|
|
1121
|
+
}
|
|
1122
|
+
return tcpCheck(id, parsed.host, parsed.port);
|
|
1123
|
+
}
|
|
1124
|
+
function resolveConnection(connection) {
|
|
1125
|
+
if (!connection.startsWith("$")) return connection;
|
|
1126
|
+
const envMatch = connection.match(/^\$env:(.+)$/);
|
|
1127
|
+
if (envMatch) {
|
|
1128
|
+
return process.env[envMatch[1]] ?? null;
|
|
1129
|
+
}
|
|
1130
|
+
return null;
|
|
1131
|
+
}
|
|
1132
|
+
function parseConnectionUrl(connection) {
|
|
1133
|
+
try {
|
|
1134
|
+
const url = new URL(connection);
|
|
1135
|
+
const host = url.hostname || "localhost";
|
|
1136
|
+
const defaultPort = getDefaultPort(url.protocol.replace(":", ""));
|
|
1137
|
+
const port = url.port ? parseInt(url.port, 10) : defaultPort;
|
|
1138
|
+
if (!port || isNaN(port) || port < 1 || port > 65535) return null;
|
|
1139
|
+
return { host, port };
|
|
1140
|
+
} catch {
|
|
1141
|
+
const colonIdx = connection.lastIndexOf(":");
|
|
1142
|
+
if (colonIdx > 0) {
|
|
1143
|
+
const host = connection.slice(0, colonIdx);
|
|
1144
|
+
const port = parseInt(connection.slice(colonIdx + 1), 10);
|
|
1145
|
+
if (!isNaN(port) && port >= 1 && port <= 65535) return { host, port };
|
|
1146
|
+
}
|
|
1147
|
+
return null;
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
function classifyHost(host) {
|
|
1151
|
+
const h = host.replace(/^\[(.+)\]$/, "$1").toLowerCase();
|
|
1152
|
+
if (h === "0.0.0.0") return "unspecified address 0.0.0.0 is not a valid service host";
|
|
1153
|
+
if (h === "::1" || h === "localhost") return "loopback address is not probed from service checks";
|
|
1154
|
+
if (h.startsWith("127.")) return "IPv4 loopback (127.x.x.x) is not probed from service checks";
|
|
1155
|
+
if (h.startsWith("169.254.")) return "link-local address (169.254.x.x) blocked to prevent instance-metadata SSRF";
|
|
1156
|
+
if (h.startsWith("fe80:")) return "IPv6 link-local (fe80::/10) blocked to prevent SSRF";
|
|
1157
|
+
return null;
|
|
1158
|
+
}
|
|
1159
|
+
function getDefaultPort(scheme) {
|
|
1160
|
+
const ports = {
|
|
1161
|
+
redis: 6379,
|
|
1162
|
+
rediss: 6379,
|
|
1163
|
+
postgres: 5432,
|
|
1164
|
+
postgresql: 5432,
|
|
1165
|
+
mysql: 3306,
|
|
1166
|
+
mongodb: 27017,
|
|
1167
|
+
elasticsearch: 9200
|
|
1168
|
+
};
|
|
1169
|
+
return ports[scheme] ?? 0;
|
|
1170
|
+
}
|
|
1171
|
+
function isSupportedType(type) {
|
|
1172
|
+
return ["redis", "postgres", "mysql", "mongodb", "elasticsearch"].includes(type);
|
|
1173
|
+
}
|
|
1174
|
+
async function tcpCheck(id, host, port) {
|
|
1175
|
+
const start = Date.now();
|
|
1176
|
+
try {
|
|
1177
|
+
const { createConnection } = await import('net');
|
|
1178
|
+
await new Promise((resolve5, reject) => {
|
|
1179
|
+
const socket = createConnection({ host, port });
|
|
1180
|
+
const timer = setTimeout(() => {
|
|
1181
|
+
socket.destroy();
|
|
1182
|
+
reject(new Error(`TCP connection timed out after ${TIMEOUT_MS}ms`));
|
|
1183
|
+
}, TIMEOUT_MS);
|
|
1184
|
+
socket.on("connect", () => {
|
|
1185
|
+
clearTimeout(timer);
|
|
1186
|
+
socket.destroy();
|
|
1187
|
+
resolve5();
|
|
1188
|
+
});
|
|
1189
|
+
socket.on("error", (err) => {
|
|
1190
|
+
clearTimeout(timer);
|
|
1191
|
+
reject(err);
|
|
1192
|
+
});
|
|
1193
|
+
});
|
|
1194
|
+
return {
|
|
1195
|
+
id,
|
|
1196
|
+
category: "service",
|
|
1197
|
+
status: "pass",
|
|
1198
|
+
severity: "info",
|
|
1199
|
+
latencyMs: Date.now() - start
|
|
1200
|
+
};
|
|
1201
|
+
} catch (err) {
|
|
1202
|
+
return {
|
|
1203
|
+
id,
|
|
1204
|
+
category: "service",
|
|
1205
|
+
status: "fail",
|
|
1206
|
+
severity: "warning",
|
|
1207
|
+
latencyMs: Date.now() - start,
|
|
1208
|
+
message: `Service unreachable at ${host}:${port} \u2014 ${String(err)}`,
|
|
1209
|
+
remediation: `Check the connection string for ${id} and ensure the service is running and accessible`
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
// src/health/index.ts
|
|
1215
|
+
async function runSubagentChecks(manifest, opts) {
|
|
1216
|
+
const checks = [];
|
|
1217
|
+
if (!manifest.spec.subagents) return checks;
|
|
1218
|
+
for (const sub of manifest.spec.subagents) {
|
|
1219
|
+
const ref = sub.ref;
|
|
1220
|
+
if ("agentspec" in ref && opts.baseDir) {
|
|
1221
|
+
const subPath = path.resolve(opts.baseDir, ref.agentspec);
|
|
1222
|
+
const exists = fs.existsSync(subPath);
|
|
1223
|
+
checks.push({
|
|
1224
|
+
id: `subagent:${sub.name}`,
|
|
1225
|
+
category: "subagent",
|
|
1226
|
+
status: exists ? "pass" : "fail",
|
|
1227
|
+
severity: "warning",
|
|
1228
|
+
message: exists ? void 0 : `Sub-agent manifest not found: ${ref.agentspec}`,
|
|
1229
|
+
remediation: exists ? void 0 : `Create the sub-agent manifest at ${subPath}`
|
|
1230
|
+
});
|
|
1231
|
+
} else if ("a2a" in ref) {
|
|
1232
|
+
const rawUrl = ref.a2a.url;
|
|
1233
|
+
if (!rawUrl.startsWith("$")) {
|
|
1234
|
+
const check = await checkA2aEndpoint(sub.name, rawUrl);
|
|
1235
|
+
checks.push(check);
|
|
1236
|
+
} else {
|
|
1237
|
+
checks.push({
|
|
1238
|
+
id: `subagent:${sub.name}`,
|
|
1239
|
+
category: "subagent",
|
|
1240
|
+
status: "skip",
|
|
1241
|
+
severity: "warning",
|
|
1242
|
+
message: `Cannot check A2A endpoint: URL not resolved (${rawUrl})`
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
return checks;
|
|
1248
|
+
}
|
|
1249
|
+
function runEvalChecks(manifest, opts) {
|
|
1250
|
+
const checks = [];
|
|
1251
|
+
if (!manifest.spec.evaluation?.datasets || !opts.baseDir) return checks;
|
|
1252
|
+
for (const dataset of manifest.spec.evaluation.datasets) {
|
|
1253
|
+
const path$1 = dataset.path;
|
|
1254
|
+
if (!path$1.startsWith("$")) {
|
|
1255
|
+
const absPath = path.resolve(opts.baseDir, path$1);
|
|
1256
|
+
const exists = fs.existsSync(absPath);
|
|
1257
|
+
checks.push({
|
|
1258
|
+
id: `eval:dataset:${dataset.name}`,
|
|
1259
|
+
category: "eval",
|
|
1260
|
+
status: exists ? "pass" : "fail",
|
|
1261
|
+
severity: "info",
|
|
1262
|
+
message: exists ? void 0 : `Eval dataset not found: ${path$1}`,
|
|
1263
|
+
remediation: exists ? void 0 : `Create the dataset file at ${absPath}`
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
return checks;
|
|
1268
|
+
}
|
|
1269
|
+
function computeHealthStatus(checks) {
|
|
1270
|
+
const failed = checks.filter((c) => c.status === "fail");
|
|
1271
|
+
const warnings = checks.filter((c) => c.status === "warn");
|
|
1272
|
+
const passed = checks.filter((c) => c.status === "pass");
|
|
1273
|
+
const skipped = checks.filter((c) => c.status === "skip");
|
|
1274
|
+
const hasErrors = failed.some((c) => c.severity === "error");
|
|
1275
|
+
const hasWarnings = failed.some((c) => c.severity === "warning") || warnings.length > 0;
|
|
1276
|
+
const status = hasErrors ? "unhealthy" : hasWarnings ? "degraded" : "healthy";
|
|
1277
|
+
return {
|
|
1278
|
+
status,
|
|
1279
|
+
summary: {
|
|
1280
|
+
passed: passed.length,
|
|
1281
|
+
failed: failed.length,
|
|
1282
|
+
warnings: warnings.length,
|
|
1283
|
+
skipped: skipped.length
|
|
1284
|
+
}
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
async function runHealthCheck(manifest, opts = {}) {
|
|
1288
|
+
const checks = [];
|
|
1289
|
+
const rawManifest = opts.rawManifest ?? manifest;
|
|
1290
|
+
checks.push(...runEnvChecks(manifest, rawManifest));
|
|
1291
|
+
checks.push(...await runSecretChecks());
|
|
1292
|
+
if (opts.baseDir) {
|
|
1293
|
+
checks.push(...runFileChecks(rawManifest, opts.baseDir));
|
|
1294
|
+
}
|
|
1295
|
+
if (opts.checkModel !== false && manifest.spec.model) {
|
|
1296
|
+
checks.push(...await runModelChecks(manifest.spec.model));
|
|
1297
|
+
}
|
|
1298
|
+
if (opts.checkMcp !== false && manifest.spec.mcp?.servers?.length) {
|
|
1299
|
+
checks.push(...await runMcpChecks(manifest.spec.mcp.servers));
|
|
1300
|
+
}
|
|
1301
|
+
if (opts.checkMemory !== false && manifest.spec.memory) {
|
|
1302
|
+
checks.push(...await runMemoryChecks(manifest.spec.memory));
|
|
1303
|
+
}
|
|
1304
|
+
if (opts.checkServices !== false && manifest.spec.requires?.services?.length) {
|
|
1305
|
+
checks.push(...await runServiceChecks(manifest.spec.requires.services));
|
|
1306
|
+
}
|
|
1307
|
+
checks.push(...await runSubagentChecks(manifest, opts));
|
|
1308
|
+
checks.push(...runEvalChecks(manifest, opts));
|
|
1309
|
+
const { status, summary } = computeHealthStatus(checks);
|
|
1310
|
+
return {
|
|
1311
|
+
agentName: manifest.metadata.name,
|
|
1312
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1313
|
+
status,
|
|
1314
|
+
summary,
|
|
1315
|
+
checks
|
|
1316
|
+
};
|
|
1317
|
+
}
|
|
1318
|
+
async function checkA2aEndpoint(name, url) {
|
|
1319
|
+
let parsed;
|
|
1320
|
+
try {
|
|
1321
|
+
parsed = new URL(url);
|
|
1322
|
+
} catch {
|
|
1323
|
+
return {
|
|
1324
|
+
id: `subagent:${name}`,
|
|
1325
|
+
category: "subagent",
|
|
1326
|
+
status: "fail",
|
|
1327
|
+
severity: "warning",
|
|
1328
|
+
message: `A2A endpoint for ${name} has an invalid URL`,
|
|
1329
|
+
remediation: `Set ref.a2a.url to a valid http:// or https:// URL`
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
1333
|
+
return {
|
|
1334
|
+
id: `subagent:${name}`,
|
|
1335
|
+
category: "subagent",
|
|
1336
|
+
status: "fail",
|
|
1337
|
+
severity: "warning",
|
|
1338
|
+
message: `A2A endpoint for ${name} uses disallowed scheme "${parsed.protocol}"`,
|
|
1339
|
+
remediation: `Use an http:// or https:// URL for ref.a2a.url`
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
1342
|
+
const host = parsed.hostname.replace(/^\[(.+)\]$/, "$1");
|
|
1343
|
+
if (host === "localhost" || host === "127.0.0.1" || host === "::1" || host.startsWith("169.254.") || // link-local (AWS metadata, etc.)
|
|
1344
|
+
host.startsWith("fe80:")) {
|
|
1345
|
+
return {
|
|
1346
|
+
id: `subagent:${name}`,
|
|
1347
|
+
category: "subagent",
|
|
1348
|
+
status: "skip",
|
|
1349
|
+
severity: "warning",
|
|
1350
|
+
message: `Skipping A2A check for ${name} \u2014 loopback/link-local address not checked`
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
const controller = new AbortController();
|
|
1354
|
+
const timeout = setTimeout(() => controller.abort(), 5e3);
|
|
1355
|
+
const start = Date.now();
|
|
1356
|
+
try {
|
|
1357
|
+
await fetch(url, { signal: controller.signal });
|
|
1358
|
+
return {
|
|
1359
|
+
id: `subagent:${name}`,
|
|
1360
|
+
category: "subagent",
|
|
1361
|
+
status: "pass",
|
|
1362
|
+
severity: "warning",
|
|
1363
|
+
latencyMs: Date.now() - start
|
|
1364
|
+
};
|
|
1365
|
+
} catch (err) {
|
|
1366
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
1367
|
+
return {
|
|
1368
|
+
id: `subagent:${name}`,
|
|
1369
|
+
category: "subagent",
|
|
1370
|
+
status: "fail",
|
|
1371
|
+
severity: "warning",
|
|
1372
|
+
latencyMs: Date.now() - start,
|
|
1373
|
+
message: isTimeout ? `A2A endpoint for ${name} timed out` : `A2A endpoint for ${name} unreachable: ${String(err)}`,
|
|
1374
|
+
remediation: `Check that the A2A endpoint is reachable`
|
|
1375
|
+
};
|
|
1376
|
+
} finally {
|
|
1377
|
+
clearTimeout(timeout);
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
// src/audit/rules/model.rules.ts
|
|
1382
|
+
var modelRules = [
|
|
1383
|
+
{
|
|
1384
|
+
id: "MODEL-01",
|
|
1385
|
+
pack: "model-resilience",
|
|
1386
|
+
title: "Model fallback declared",
|
|
1387
|
+
description: "A fallback model should be configured to handle rate limits and errors",
|
|
1388
|
+
severity: "high",
|
|
1389
|
+
evidenceLevel: "external",
|
|
1390
|
+
proofTool: "LiteLLM chaos test",
|
|
1391
|
+
proofToolUrl: "https://docs.litellm.ai/docs/proxy/reliability#mock-testing-fallbacks",
|
|
1392
|
+
check(manifest) {
|
|
1393
|
+
const hasFallback = !!manifest.spec.model.fallback;
|
|
1394
|
+
return {
|
|
1395
|
+
pass: hasFallback,
|
|
1396
|
+
message: hasFallback ? void 0 : "No fallback model declared. Configure spec.model.fallback",
|
|
1397
|
+
path: "/spec/model/fallback",
|
|
1398
|
+
recommendation: "Add a fallback model with triggerOn: [rate_limit, timeout, error_5xx]",
|
|
1399
|
+
references: []
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
},
|
|
1403
|
+
{
|
|
1404
|
+
id: "MODEL-02",
|
|
1405
|
+
pack: "model-resilience",
|
|
1406
|
+
title: "Model version pinned",
|
|
1407
|
+
description: 'Model ID should not use "latest" \u2014 pin a specific version for reproducibility',
|
|
1408
|
+
severity: "medium",
|
|
1409
|
+
evidenceLevel: "probed",
|
|
1410
|
+
proofTool: "agentspec health",
|
|
1411
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1412
|
+
check(manifest) {
|
|
1413
|
+
const id = manifest.spec.model.id;
|
|
1414
|
+
const isLatest = id === "latest" || id.endsWith(":latest");
|
|
1415
|
+
return {
|
|
1416
|
+
pass: !isLatest,
|
|
1417
|
+
message: isLatest ? `Model ID "${id}" uses "latest" \u2014 this is not reproducible` : void 0,
|
|
1418
|
+
path: "/spec/model/id",
|
|
1419
|
+
recommendation: 'Pin a specific model version, e.g. "llama-3.3-70b-versatile" or "gpt-4-0125-preview"',
|
|
1420
|
+
references: []
|
|
1421
|
+
};
|
|
1422
|
+
}
|
|
1423
|
+
},
|
|
1424
|
+
{
|
|
1425
|
+
id: "MODEL-03",
|
|
1426
|
+
pack: "model-resilience",
|
|
1427
|
+
title: "Cost controls declared",
|
|
1428
|
+
description: "Monthly cost limits prevent unexpected API bills",
|
|
1429
|
+
severity: "medium",
|
|
1430
|
+
evidenceLevel: "external",
|
|
1431
|
+
proofTool: "LiteLLM Spend Tracking",
|
|
1432
|
+
proofToolUrl: "https://docs.litellm.ai/docs/proxy/cost_tracking",
|
|
1433
|
+
check(manifest) {
|
|
1434
|
+
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1435
|
+
return {
|
|
1436
|
+
pass: hasCostControls,
|
|
1437
|
+
message: hasCostControls ? void 0 : "No cost controls declared. Configure spec.model.costControls",
|
|
1438
|
+
path: "/spec/model/costControls",
|
|
1439
|
+
recommendation: "Add costControls.maxMonthlyUSD and costControls.alertAtUSD",
|
|
1440
|
+
references: []
|
|
1441
|
+
};
|
|
1442
|
+
}
|
|
1443
|
+
},
|
|
1444
|
+
{
|
|
1445
|
+
id: "MODEL-04",
|
|
1446
|
+
pack: "model-resilience",
|
|
1447
|
+
title: "Fallback retry strategy configured",
|
|
1448
|
+
description: "Fallback should have maxRetries to avoid infinite loops",
|
|
1449
|
+
severity: "low",
|
|
1450
|
+
evidenceLevel: "external",
|
|
1451
|
+
proofTool: "pytest-mockllm",
|
|
1452
|
+
proofToolUrl: "https://pypi.org/project/pytest-mockllm/",
|
|
1453
|
+
check(manifest) {
|
|
1454
|
+
if (!manifest.spec.model.fallback) return { pass: true };
|
|
1455
|
+
const hasRetries = manifest.spec.model.fallback.maxRetries !== void 0;
|
|
1456
|
+
return {
|
|
1457
|
+
pass: hasRetries,
|
|
1458
|
+
message: hasRetries ? void 0 : "Fallback model has no maxRetries configured",
|
|
1459
|
+
path: "/spec/model/fallback/maxRetries",
|
|
1460
|
+
recommendation: "Add maxRetries: 2 to spec.model.fallback",
|
|
1461
|
+
references: []
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
];
|
|
1466
|
+
|
|
1467
|
+
// src/audit/rules/security.rules.ts
|
|
1468
|
+
var securityRules = [
|
|
1469
|
+
{
|
|
1470
|
+
id: "SEC-LLM-01",
|
|
1471
|
+
pack: "owasp-llm-top10",
|
|
1472
|
+
title: "Prompt injection guard: input guardrail required",
|
|
1473
|
+
description: "OWASP LLM Top 10 #1: Agents should have input guardrails to detect prompt injection",
|
|
1474
|
+
severity: "high",
|
|
1475
|
+
evidenceLevel: "behavioral",
|
|
1476
|
+
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1477
|
+
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1478
|
+
check(manifest) {
|
|
1479
|
+
const hasGuardrail = manifest.spec.guardrails?.input?.some(
|
|
1480
|
+
(g) => g.type === "prompt-injection" || g.type === "topic-filter"
|
|
1481
|
+
) ?? false;
|
|
1482
|
+
return {
|
|
1483
|
+
pass: hasGuardrail,
|
|
1484
|
+
message: hasGuardrail ? void 0 : "No input guardrail configured. Prompt injection attacks are unmitigated.",
|
|
1485
|
+
path: "/spec/guardrails/input",
|
|
1486
|
+
recommendation: "Add a prompt-injection guardrail to spec.guardrails.input",
|
|
1487
|
+
references: [
|
|
1488
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm01-prompt-injection"
|
|
1489
|
+
]
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
},
|
|
1493
|
+
{
|
|
1494
|
+
id: "SEC-LLM-02",
|
|
1495
|
+
pack: "owasp-llm-top10",
|
|
1496
|
+
title: "Insecure output handling: output guardrail required",
|
|
1497
|
+
description: "OWASP LLM Top 10 #2: Agents should validate and sanitize LLM outputs",
|
|
1498
|
+
severity: "high",
|
|
1499
|
+
evidenceLevel: "behavioral",
|
|
1500
|
+
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1501
|
+
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1502
|
+
check(manifest) {
|
|
1503
|
+
const hasGuardrail = (manifest.spec.guardrails?.output?.length ?? 0) > 0;
|
|
1504
|
+
return {
|
|
1505
|
+
pass: hasGuardrail,
|
|
1506
|
+
message: hasGuardrail ? void 0 : "No output guardrail configured. LLM outputs are not sanitized.",
|
|
1507
|
+
path: "/spec/guardrails/output",
|
|
1508
|
+
recommendation: "Add a toxicity-filter or hallucination-detector to spec.guardrails.output",
|
|
1509
|
+
references: [
|
|
1510
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm02-insecure-output-handling"
|
|
1511
|
+
]
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
},
|
|
1515
|
+
{
|
|
1516
|
+
id: "SEC-LLM-03",
|
|
1517
|
+
pack: "owasp-llm-top10",
|
|
1518
|
+
title: "Training/prompt data poisoning: prompt files referenced",
|
|
1519
|
+
description: "OWASP LLM Top 10 #3: Prompt content should be loaded from versioned files, not inlined in the manifest",
|
|
1520
|
+
severity: "medium",
|
|
1521
|
+
evidenceLevel: "probed",
|
|
1522
|
+
proofTool: "agentspec health",
|
|
1523
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1524
|
+
check(manifest) {
|
|
1525
|
+
const system = manifest.spec.prompts.system;
|
|
1526
|
+
const usesFile = system.startsWith("$file:");
|
|
1527
|
+
return {
|
|
1528
|
+
pass: usesFile,
|
|
1529
|
+
message: usesFile ? void 0 : "System prompt is inlined in the manifest rather than loaded from a versioned file. Inline prompts bypass version control and integrity checks.",
|
|
1530
|
+
path: "/spec/prompts/system",
|
|
1531
|
+
recommendation: "Use $file:prompts/system.md to load prompt from a versioned file that can be reviewed and audited.",
|
|
1532
|
+
references: [
|
|
1533
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm03-training-data-poisoning"
|
|
1534
|
+
]
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
},
|
|
1538
|
+
{
|
|
1539
|
+
id: "SEC-LLM-04",
|
|
1540
|
+
pack: "owasp-llm-top10",
|
|
1541
|
+
title: "Model DoS: rate limiting + cost controls declared",
|
|
1542
|
+
description: "OWASP LLM Top 10 #4: Protect against denial-of-service via rate limiting",
|
|
1543
|
+
severity: "medium",
|
|
1544
|
+
evidenceLevel: "external",
|
|
1545
|
+
proofTool: "k6 load test",
|
|
1546
|
+
proofToolUrl: "https://k6.io",
|
|
1547
|
+
check(manifest) {
|
|
1548
|
+
const hasRateLimit = !!manifest.spec.api?.rateLimit?.requestsPerMinute;
|
|
1549
|
+
const hasCostControls = !!manifest.spec.model.costControls?.maxMonthlyUSD;
|
|
1550
|
+
const pass = hasRateLimit || hasCostControls;
|
|
1551
|
+
return {
|
|
1552
|
+
pass,
|
|
1553
|
+
message: pass ? void 0 : "No rate limiting or cost controls declared.",
|
|
1554
|
+
path: "/spec/api/rateLimit",
|
|
1555
|
+
recommendation: "Add spec.api.rateLimit.requestsPerMinute and spec.model.costControls.maxMonthlyUSD",
|
|
1556
|
+
references: [
|
|
1557
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm04-model-denial-of-service"
|
|
1558
|
+
]
|
|
1559
|
+
};
|
|
1560
|
+
}
|
|
1561
|
+
},
|
|
1562
|
+
{
|
|
1563
|
+
id: "SEC-LLM-05",
|
|
1564
|
+
pack: "owasp-llm-top10",
|
|
1565
|
+
title: "Supply chain: model provider and version pinned",
|
|
1566
|
+
description: "OWASP LLM Top 10 #5: Pin model versions to prevent supply chain drift",
|
|
1567
|
+
severity: "medium",
|
|
1568
|
+
evidenceLevel: "probed",
|
|
1569
|
+
proofTool: "agentspec health",
|
|
1570
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1571
|
+
check(manifest) {
|
|
1572
|
+
const hasProvider = !!manifest.spec.model.provider;
|
|
1573
|
+
const hasVersion = !!manifest.spec.model.id && manifest.spec.model.id !== "latest";
|
|
1574
|
+
const pass = hasProvider && hasVersion;
|
|
1575
|
+
return {
|
|
1576
|
+
pass,
|
|
1577
|
+
message: pass ? void 0 : "Model provider or version not pinned.",
|
|
1578
|
+
path: "/spec/model",
|
|
1579
|
+
recommendation: "Specify both spec.model.provider and a specific spec.model.id",
|
|
1580
|
+
references: [
|
|
1581
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm05-supply-chain-vulnerabilities"
|
|
1582
|
+
]
|
|
1583
|
+
};
|
|
1584
|
+
}
|
|
1585
|
+
},
|
|
1586
|
+
{
|
|
1587
|
+
id: "SEC-LLM-06",
|
|
1588
|
+
pack: "owasp-llm-top10",
|
|
1589
|
+
title: "Sensitive data disclosure: PII scrub in memory hygiene",
|
|
1590
|
+
description: "OWASP LLM Top 10 #6: Agents storing long-term memory must scrub PII",
|
|
1591
|
+
severity: "critical",
|
|
1592
|
+
evidenceLevel: "external",
|
|
1593
|
+
proofTool: "Microsoft Presidio",
|
|
1594
|
+
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1595
|
+
check(manifest) {
|
|
1596
|
+
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1597
|
+
const hasPiiScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
1598
|
+
return {
|
|
1599
|
+
pass: hasPiiScrub,
|
|
1600
|
+
message: hasPiiScrub ? void 0 : "Long-term memory declared without piiScrubFields \u2014 PII may be persisted.",
|
|
1601
|
+
path: "/spec/memory/hygiene/piiScrubFields",
|
|
1602
|
+
recommendation: "Add spec.memory.hygiene.piiScrubFields with sensitive fields like [ssn, credit_card, bank_account]",
|
|
1603
|
+
references: [
|
|
1604
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm06-sensitive-information-disclosure"
|
|
1605
|
+
]
|
|
1606
|
+
};
|
|
1607
|
+
}
|
|
1608
|
+
},
|
|
1609
|
+
{
|
|
1610
|
+
id: "SEC-LLM-07",
|
|
1611
|
+
pack: "owasp-llm-top10",
|
|
1612
|
+
title: "Insecure plugin design: tool annotations declared",
|
|
1613
|
+
description: "OWASP LLM Top 10 #7: Tools should declare their access scope via annotations",
|
|
1614
|
+
severity: "medium",
|
|
1615
|
+
evidenceLevel: "external",
|
|
1616
|
+
proofTool: "Promptfoo",
|
|
1617
|
+
proofToolUrl: "https://promptfoo.dev",
|
|
1618
|
+
check(manifest) {
|
|
1619
|
+
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1620
|
+
const allAnnotated = manifest.spec.tools.every((t) => t.annotations !== void 0);
|
|
1621
|
+
return {
|
|
1622
|
+
pass: allAnnotated,
|
|
1623
|
+
message: allAnnotated ? void 0 : "Some tools lack annotations (readOnlyHint, destructiveHint).",
|
|
1624
|
+
path: "/spec/tools",
|
|
1625
|
+
recommendation: "Add annotations to each tool: { readOnlyHint: boolean, destructiveHint: boolean }",
|
|
1626
|
+
references: [
|
|
1627
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm07-insecure-plugin-design"
|
|
1628
|
+
]
|
|
1629
|
+
};
|
|
1630
|
+
}
|
|
1631
|
+
},
|
|
1632
|
+
{
|
|
1633
|
+
id: "SEC-LLM-08",
|
|
1634
|
+
pack: "owasp-llm-top10",
|
|
1635
|
+
title: "Excessive agency: destructiveHint declared on tools",
|
|
1636
|
+
description: "OWASP LLM Top 10 #8: Destructive tools must be explicitly flagged to limit agent autonomy",
|
|
1637
|
+
severity: "high",
|
|
1638
|
+
evidenceLevel: "external",
|
|
1639
|
+
proofTool: "Promptfoo",
|
|
1640
|
+
proofToolUrl: "https://promptfoo.dev",
|
|
1641
|
+
check(manifest) {
|
|
1642
|
+
if (!manifest.spec.tools?.length) return { pass: true };
|
|
1643
|
+
const allHaveDestructiveHint = manifest.spec.tools.every(
|
|
1644
|
+
(t) => t.annotations?.destructiveHint !== void 0
|
|
1645
|
+
);
|
|
1646
|
+
return {
|
|
1647
|
+
pass: allHaveDestructiveHint,
|
|
1648
|
+
message: allHaveDestructiveHint ? void 0 : "Some tools do not declare destructiveHint \u2014 excessive agency risk.",
|
|
1649
|
+
path: "/spec/tools",
|
|
1650
|
+
recommendation: "Add annotations.destructiveHint: true|false to every tool",
|
|
1651
|
+
references: [
|
|
1652
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm08-excessive-agency"
|
|
1653
|
+
]
|
|
1654
|
+
};
|
|
1655
|
+
}
|
|
1656
|
+
},
|
|
1657
|
+
{
|
|
1658
|
+
id: "SEC-LLM-09",
|
|
1659
|
+
pack: "owasp-llm-top10",
|
|
1660
|
+
title: "Overreliance: evaluation framework + CI gate configured",
|
|
1661
|
+
description: "OWASP LLM Top 10 #9: Evaluation with a CI gate reduces overreliance on LLM outputs",
|
|
1662
|
+
severity: "medium",
|
|
1663
|
+
evidenceLevel: "probed",
|
|
1664
|
+
proofTool: "agentspec health",
|
|
1665
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1666
|
+
check(manifest) {
|
|
1667
|
+
const hasEval = !!manifest.spec.evaluation?.framework;
|
|
1668
|
+
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1669
|
+
const pass = hasEval && hasCiGate;
|
|
1670
|
+
return {
|
|
1671
|
+
pass,
|
|
1672
|
+
message: pass ? void 0 : !hasEval ? "No evaluation framework configured." : "Evaluation CI gate is not enabled.",
|
|
1673
|
+
path: "/spec/evaluation",
|
|
1674
|
+
recommendation: "Configure spec.evaluation.framework and set spec.evaluation.ciGate: true",
|
|
1675
|
+
references: [
|
|
1676
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm09-overreliance"
|
|
1677
|
+
]
|
|
1678
|
+
};
|
|
1679
|
+
}
|
|
1680
|
+
},
|
|
1681
|
+
{
|
|
1682
|
+
id: "SEC-LLM-10",
|
|
1683
|
+
pack: "owasp-llm-top10",
|
|
1684
|
+
title: "Model theft: API keys use $secret, not $env",
|
|
1685
|
+
description: "OWASP LLM Top 10 #10: API keys should use $secret: not $env: to go through a secret manager",
|
|
1686
|
+
severity: "high",
|
|
1687
|
+
evidenceLevel: "probed",
|
|
1688
|
+
proofTool: "agentspec health",
|
|
1689
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1690
|
+
check(manifest) {
|
|
1691
|
+
const apiKey = manifest.spec.model.apiKey;
|
|
1692
|
+
const usesSecret = apiKey.startsWith("$secret:");
|
|
1693
|
+
const usesEnv = apiKey.startsWith("$env:");
|
|
1694
|
+
const isLiteral = !apiKey.startsWith("$");
|
|
1695
|
+
return {
|
|
1696
|
+
pass: usesSecret,
|
|
1697
|
+
message: isLiteral ? "Model API key appears to be a hardcoded literal value \u2014 credential exposure risk." : usesEnv ? `Model API key uses $env: (${apiKey}). Prefer $secret: for production deployments.` : void 0,
|
|
1698
|
+
path: "/spec/model/apiKey",
|
|
1699
|
+
recommendation: "Use $secret:groq-api-key instead of $env:GROQ_API_KEY for production deployments",
|
|
1700
|
+
references: [
|
|
1701
|
+
"https://owasp.org/www-project-top-10-for-large-language-model-applications/#llm10-model-theft"
|
|
1702
|
+
]
|
|
1703
|
+
};
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
];
|
|
1707
|
+
|
|
1708
|
+
// src/audit/rules/memory.rules.ts
|
|
1709
|
+
var memoryRules = [
|
|
1710
|
+
{
|
|
1711
|
+
id: "MEM-01",
|
|
1712
|
+
pack: "memory-hygiene",
|
|
1713
|
+
title: "PII scrub fields declared for long-term memory",
|
|
1714
|
+
description: "Agents with long-term memory must declare which fields to scrub for PII compliance",
|
|
1715
|
+
severity: "critical",
|
|
1716
|
+
evidenceLevel: "external",
|
|
1717
|
+
proofTool: "Microsoft Presidio",
|
|
1718
|
+
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1719
|
+
check(manifest) {
|
|
1720
|
+
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1721
|
+
const hasScrub = (manifest.spec.memory.hygiene?.piiScrubFields?.length ?? 0) > 0;
|
|
1722
|
+
return {
|
|
1723
|
+
pass: hasScrub,
|
|
1724
|
+
message: hasScrub ? void 0 : "Long-term memory enabled without PII scrub fields",
|
|
1725
|
+
path: "/spec/memory/hygiene/piiScrubFields",
|
|
1726
|
+
recommendation: "Add spec.memory.hygiene.piiScrubFields: [ssn, credit_card, bank_account]",
|
|
1727
|
+
references: []
|
|
1728
|
+
};
|
|
1729
|
+
}
|
|
1730
|
+
},
|
|
1731
|
+
{
|
|
1732
|
+
id: "MEM-02",
|
|
1733
|
+
pack: "memory-hygiene",
|
|
1734
|
+
title: "Retention policy / TTL set for all memory backends",
|
|
1735
|
+
description: "Memory without TTL grows unbounded and increases PII exposure risk",
|
|
1736
|
+
severity: "high",
|
|
1737
|
+
evidenceLevel: "probed",
|
|
1738
|
+
proofTool: "agentspec health",
|
|
1739
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1740
|
+
check(manifest) {
|
|
1741
|
+
const mem = manifest.spec.memory;
|
|
1742
|
+
if (!mem) return { pass: true };
|
|
1743
|
+
const stOk = !mem.shortTerm || mem.shortTerm.ttlSeconds !== void 0;
|
|
1744
|
+
const ltOk = !mem.longTerm || mem.longTerm.ttlDays !== void 0;
|
|
1745
|
+
const pass = stOk && ltOk;
|
|
1746
|
+
return {
|
|
1747
|
+
pass,
|
|
1748
|
+
message: pass ? void 0 : [
|
|
1749
|
+
!stOk ? "Short-term memory has no ttlSeconds" : "",
|
|
1750
|
+
!ltOk ? "Long-term memory has no ttlDays" : ""
|
|
1751
|
+
].filter(Boolean).join("; "),
|
|
1752
|
+
path: "/spec/memory",
|
|
1753
|
+
recommendation: "Set spec.memory.shortTerm.ttlSeconds and spec.memory.longTerm.ttlDays",
|
|
1754
|
+
references: []
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
},
|
|
1758
|
+
{
|
|
1759
|
+
id: "MEM-03",
|
|
1760
|
+
pack: "memory-hygiene",
|
|
1761
|
+
title: "Audit log enabled for long-term memory",
|
|
1762
|
+
description: "Audit logging enables compliance investigations and debugging",
|
|
1763
|
+
severity: "medium",
|
|
1764
|
+
evidenceLevel: "probed",
|
|
1765
|
+
proofTool: "agentspec health",
|
|
1766
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1767
|
+
check(manifest) {
|
|
1768
|
+
if (!manifest.spec.memory?.longTerm) return { pass: true };
|
|
1769
|
+
const hasAuditLog = manifest.spec.memory.hygiene?.auditLog === true;
|
|
1770
|
+
return {
|
|
1771
|
+
pass: hasAuditLog,
|
|
1772
|
+
message: hasAuditLog ? void 0 : "Long-term memory enabled without audit log",
|
|
1773
|
+
path: "/spec/memory/hygiene/auditLog",
|
|
1774
|
+
recommendation: "Set spec.memory.hygiene.auditLog: true",
|
|
1775
|
+
references: []
|
|
1776
|
+
};
|
|
1777
|
+
}
|
|
1778
|
+
},
|
|
1779
|
+
{
|
|
1780
|
+
id: "MEM-04",
|
|
1781
|
+
pack: "memory-hygiene",
|
|
1782
|
+
title: "Vector store namespace isolated",
|
|
1783
|
+
description: "Shared vector store namespaces risk data leakage between agents",
|
|
1784
|
+
severity: "medium",
|
|
1785
|
+
evidenceLevel: "probed",
|
|
1786
|
+
proofTool: "agentspec health",
|
|
1787
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1788
|
+
check(manifest) {
|
|
1789
|
+
if (!manifest.spec.memory?.vector) return { pass: true };
|
|
1790
|
+
const hasNamespace = !!manifest.spec.memory.vector.namespace;
|
|
1791
|
+
return {
|
|
1792
|
+
pass: hasNamespace,
|
|
1793
|
+
message: hasNamespace ? void 0 : "Vector store has no namespace \u2014 may share space with other agents",
|
|
1794
|
+
path: "/spec/memory/vector/namespace",
|
|
1795
|
+
recommendation: `Add spec.memory.vector.namespace: "${manifest.metadata.name}"`,
|
|
1796
|
+
references: []
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
},
|
|
1800
|
+
{
|
|
1801
|
+
id: "MEM-05",
|
|
1802
|
+
pack: "memory-hygiene",
|
|
1803
|
+
title: "Short-term memory max tokens bounded",
|
|
1804
|
+
description: "Unbounded short-term memory can cause token budget overruns",
|
|
1805
|
+
severity: "low",
|
|
1806
|
+
evidenceLevel: "probed",
|
|
1807
|
+
proofTool: "agentspec health",
|
|
1808
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1809
|
+
check(manifest) {
|
|
1810
|
+
if (!manifest.spec.memory?.shortTerm) return { pass: true };
|
|
1811
|
+
const hasBound = manifest.spec.memory.shortTerm.maxTokens !== void 0;
|
|
1812
|
+
return {
|
|
1813
|
+
pass: hasBound,
|
|
1814
|
+
message: hasBound ? void 0 : "Short-term memory has no maxTokens bound",
|
|
1815
|
+
path: "/spec/memory/shortTerm/maxTokens",
|
|
1816
|
+
recommendation: "Set spec.memory.shortTerm.maxTokens: 8000",
|
|
1817
|
+
references: []
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
];
|
|
1822
|
+
|
|
1823
|
+
// src/audit/rules/evaluation.rules.ts
|
|
1824
|
+
var evaluationRules = [
|
|
1825
|
+
{
|
|
1826
|
+
id: "EVAL-01",
|
|
1827
|
+
pack: "evaluation-coverage",
|
|
1828
|
+
title: "Evaluation dataset declared",
|
|
1829
|
+
description: "At least one evaluation dataset is required for quality assurance",
|
|
1830
|
+
severity: "medium",
|
|
1831
|
+
evidenceLevel: "probed",
|
|
1832
|
+
proofTool: "agentspec health",
|
|
1833
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1834
|
+
check(manifest) {
|
|
1835
|
+
const hasDataset = (manifest.spec.evaluation?.datasets?.length ?? 0) > 0;
|
|
1836
|
+
return {
|
|
1837
|
+
pass: hasDataset,
|
|
1838
|
+
message: hasDataset ? void 0 : "No evaluation datasets declared",
|
|
1839
|
+
path: "/spec/evaluation/datasets",
|
|
1840
|
+
recommendation: "Add spec.evaluation.datasets with at least one .jsonl dataset",
|
|
1841
|
+
references: []
|
|
1842
|
+
};
|
|
1843
|
+
}
|
|
1844
|
+
},
|
|
1845
|
+
{
|
|
1846
|
+
id: "EVAL-02",
|
|
1847
|
+
pack: "evaluation-coverage",
|
|
1848
|
+
title: "CI gate enabled",
|
|
1849
|
+
description: "Evaluation CI gate prevents regressions from reaching production",
|
|
1850
|
+
severity: "medium",
|
|
1851
|
+
evidenceLevel: "probed",
|
|
1852
|
+
proofTool: "agentspec health",
|
|
1853
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1854
|
+
check(manifest) {
|
|
1855
|
+
const hasCiGate = manifest.spec.evaluation?.ciGate === true;
|
|
1856
|
+
return {
|
|
1857
|
+
pass: hasCiGate,
|
|
1858
|
+
message: hasCiGate ? void 0 : "Evaluation CI gate is not enabled",
|
|
1859
|
+
path: "/spec/evaluation/ciGate",
|
|
1860
|
+
recommendation: "Set spec.evaluation.ciGate: true",
|
|
1861
|
+
references: []
|
|
1862
|
+
};
|
|
1863
|
+
}
|
|
1864
|
+
},
|
|
1865
|
+
{
|
|
1866
|
+
id: "EVAL-03",
|
|
1867
|
+
pack: "evaluation-coverage",
|
|
1868
|
+
title: "Hallucination metric threshold configured",
|
|
1869
|
+
description: "A hallucination threshold below 0.1 is required for production agents",
|
|
1870
|
+
severity: "medium",
|
|
1871
|
+
evidenceLevel: "probed",
|
|
1872
|
+
proofTool: "agentspec health",
|
|
1873
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1874
|
+
check(manifest) {
|
|
1875
|
+
const metrics = manifest.spec.evaluation?.metrics ?? [];
|
|
1876
|
+
const hasHallucinationMetric = metrics.includes("hallucination");
|
|
1877
|
+
const threshold = manifest.spec.evaluation?.thresholds?.["hallucination"];
|
|
1878
|
+
const pass = hasHallucinationMetric && threshold !== void 0;
|
|
1879
|
+
return {
|
|
1880
|
+
pass,
|
|
1881
|
+
message: pass ? void 0 : !hasHallucinationMetric ? "hallucination metric not in spec.evaluation.metrics" : "No hallucination threshold configured",
|
|
1882
|
+
path: "/spec/evaluation/thresholds/hallucination",
|
|
1883
|
+
recommendation: "Add hallucination to spec.evaluation.metrics and set spec.evaluation.thresholds.hallucination: 0.05",
|
|
1884
|
+
references: []
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
];
|
|
1889
|
+
|
|
1890
|
+
// src/audit/rules/observability.rules.ts
|
|
1891
|
+
var observabilityRules = [
|
|
1892
|
+
{
|
|
1893
|
+
id: "OBS-01",
|
|
1894
|
+
pack: "observability",
|
|
1895
|
+
title: "Tracing backend declared",
|
|
1896
|
+
description: "Agents should declare a tracing backend so runs are observable and debuggable",
|
|
1897
|
+
severity: "medium",
|
|
1898
|
+
evidenceLevel: "probed",
|
|
1899
|
+
proofTool: "agentspec health",
|
|
1900
|
+
proofToolUrl: "https://agentspec.io/docs/reference/cli#agentspec-health",
|
|
1901
|
+
check(manifest) {
|
|
1902
|
+
const pass = !!manifest.spec.observability?.tracing?.backend;
|
|
1903
|
+
return {
|
|
1904
|
+
pass,
|
|
1905
|
+
message: pass ? void 0 : "No tracing backend configured \u2014 agent runs are unobservable.",
|
|
1906
|
+
path: "/spec/observability/tracing/backend",
|
|
1907
|
+
recommendation: "Add spec.observability.tracing.backend (langfuse, langsmith, agentops, otel)"
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1910
|
+
},
|
|
1911
|
+
{
|
|
1912
|
+
id: "OBS-02",
|
|
1913
|
+
pack: "observability",
|
|
1914
|
+
title: "Structured logging enabled",
|
|
1915
|
+
description: "Structured logging makes log aggregation and searching reliable",
|
|
1916
|
+
severity: "low",
|
|
1917
|
+
evidenceLevel: "behavioral",
|
|
1918
|
+
proofTool: "AgentSpec EventPush + sidecar gap analysis",
|
|
1919
|
+
proofToolUrl: "https://agentspec.io/docs/concepts/probe-coverage",
|
|
1920
|
+
check(manifest) {
|
|
1921
|
+
const pass = manifest.spec.observability?.logging?.structured !== false;
|
|
1922
|
+
return {
|
|
1923
|
+
pass,
|
|
1924
|
+
message: pass ? void 0 : "Structured logging is disabled.",
|
|
1925
|
+
path: "/spec/observability/logging/structured",
|
|
1926
|
+
recommendation: "Set spec.observability.logging.structured: true"
|
|
1927
|
+
};
|
|
1928
|
+
}
|
|
1929
|
+
},
|
|
1930
|
+
{
|
|
1931
|
+
id: "OBS-03",
|
|
1932
|
+
pack: "observability",
|
|
1933
|
+
title: "Sensitive fields redacted from logs",
|
|
1934
|
+
description: "Log redaction prevents secrets and PII from appearing in log aggregators",
|
|
1935
|
+
severity: "medium",
|
|
1936
|
+
evidenceLevel: "external",
|
|
1937
|
+
proofTool: "Microsoft Presidio",
|
|
1938
|
+
proofToolUrl: "https://microsoft.github.io/presidio/",
|
|
1939
|
+
check(manifest) {
|
|
1940
|
+
const pass = (manifest.spec.observability?.logging?.redactFields?.length ?? 0) > 0;
|
|
1941
|
+
return {
|
|
1942
|
+
pass,
|
|
1943
|
+
message: pass ? void 0 : "No log redaction fields declared \u2014 sensitive data may appear in logs.",
|
|
1944
|
+
path: "/spec/observability/logging/redactFields",
|
|
1945
|
+
recommendation: "Add spec.observability.logging.redactFields: [api_key, account_number, ...]"
|
|
1946
|
+
};
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
];
|
|
1950
|
+
|
|
1951
|
+
// src/audit/index.ts
|
|
1952
|
+
var SEVERITY_WEIGHTS = {
|
|
1953
|
+
critical: 4,
|
|
1954
|
+
high: 3,
|
|
1955
|
+
medium: 2,
|
|
1956
|
+
low: 1,
|
|
1957
|
+
info: 0
|
|
1958
|
+
};
|
|
1959
|
+
var ALL_RULES = [
|
|
1960
|
+
...modelRules,
|
|
1961
|
+
...securityRules,
|
|
1962
|
+
...memoryRules,
|
|
1963
|
+
...evaluationRules,
|
|
1964
|
+
...observabilityRules
|
|
1965
|
+
];
|
|
1966
|
+
var AUDIT_RULE_IDS = new Set(ALL_RULES.map((r) => r.id));
|
|
1967
|
+
function isProofRecord(r) {
|
|
1968
|
+
return typeof r === "object" && r !== null && typeof r["ruleId"] === "string" && typeof r["verifiedBy"] === "string" && typeof r["method"] === "string";
|
|
1969
|
+
}
|
|
1970
|
+
function scoreToGrade(score) {
|
|
1971
|
+
if (score >= 90) return "A";
|
|
1972
|
+
if (score >= 75) return "B";
|
|
1973
|
+
if (score >= 60) return "C";
|
|
1974
|
+
if (score >= 45) return "D";
|
|
1975
|
+
return "F";
|
|
1976
|
+
}
|
|
1977
|
+
function resolveActiveRules(manifest, opts) {
|
|
1978
|
+
const manifestPacks = manifest.spec.compliance?.packs ?? [];
|
|
1979
|
+
const activePacks = opts.packs ?? (opts.useManifestPacks !== false && manifestPacks.length > 0 ? manifestPacks : [...new Set(ALL_RULES.map((r) => r.pack))]);
|
|
1980
|
+
return ALL_RULES.filter((r) => activePacks.includes(r.pack));
|
|
1981
|
+
}
|
|
1982
|
+
function collectSuppressions(manifest) {
|
|
1983
|
+
const suppressions = manifest.spec.compliance?.suppressions?.map((s) => ({
|
|
1984
|
+
ruleId: s.rule,
|
|
1985
|
+
reason: s.reason,
|
|
1986
|
+
approvedBy: s.approvedBy,
|
|
1987
|
+
expires: s.expires
|
|
1988
|
+
})) ?? [];
|
|
1989
|
+
const suppressedIds = new Set(
|
|
1990
|
+
suppressions.filter((s) => !s.expires || new Date(s.expires) > /* @__PURE__ */ new Date()).map((s) => s.ruleId)
|
|
1991
|
+
);
|
|
1992
|
+
return { suppressions, suppressedIds };
|
|
1993
|
+
}
|
|
1994
|
+
function executeRuleChecks(rules, manifest, suppressedIds) {
|
|
1995
|
+
const ruleResults = /* @__PURE__ */ new Map();
|
|
1996
|
+
for (const rule of rules) {
|
|
1997
|
+
ruleResults.set(
|
|
1998
|
+
rule.id,
|
|
1999
|
+
suppressedIds.has(rule.id) ? { pass: true } : rule.check(manifest)
|
|
2000
|
+
);
|
|
2001
|
+
}
|
|
2002
|
+
return ruleResults;
|
|
2003
|
+
}
|
|
2004
|
+
function computeScoring(rules, ruleResults, suppressedIds) {
|
|
2005
|
+
const violations = [];
|
|
2006
|
+
const packBreakdown = {};
|
|
2007
|
+
const evidenceBreakdown = {
|
|
2008
|
+
declarative: { passed: 0, total: 0 },
|
|
2009
|
+
probed: { passed: 0, total: 0 },
|
|
2010
|
+
behavioral: { passed: 0, total: 0 },
|
|
2011
|
+
external: { passed: 0, total: 0 }
|
|
2012
|
+
};
|
|
2013
|
+
let totalWeight = 0;
|
|
2014
|
+
let passedWeight = 0;
|
|
2015
|
+
for (const rule of rules) {
|
|
2016
|
+
if (!packBreakdown[rule.pack]) {
|
|
2017
|
+
packBreakdown[rule.pack] = { passed: 0, total: 0 };
|
|
2018
|
+
}
|
|
2019
|
+
const suppressed = suppressedIds.has(rule.id);
|
|
2020
|
+
const weight = SEVERITY_WEIGHTS[rule.severity];
|
|
2021
|
+
const result = ruleResults.get(rule.id);
|
|
2022
|
+
const tier = evidenceBreakdown[rule.evidenceLevel];
|
|
2023
|
+
packBreakdown[rule.pack].total += 1;
|
|
2024
|
+
tier.total += 1;
|
|
2025
|
+
if (suppressed) {
|
|
2026
|
+
packBreakdown[rule.pack].passed += 1;
|
|
2027
|
+
tier.passed += 1;
|
|
2028
|
+
continue;
|
|
2029
|
+
}
|
|
2030
|
+
if (result.pass) {
|
|
2031
|
+
packBreakdown[rule.pack].passed += 1;
|
|
2032
|
+
tier.passed += 1;
|
|
2033
|
+
passedWeight += weight;
|
|
2034
|
+
} else {
|
|
2035
|
+
violations.push({
|
|
2036
|
+
ruleId: rule.id,
|
|
2037
|
+
severity: rule.severity,
|
|
2038
|
+
title: rule.title,
|
|
2039
|
+
message: result.message ?? `Rule ${rule.id} failed`,
|
|
2040
|
+
path: result.path,
|
|
2041
|
+
recommendation: result.recommendation,
|
|
2042
|
+
references: result.references,
|
|
2043
|
+
evidenceLevel: rule.evidenceLevel,
|
|
2044
|
+
proofTool: rule.proofTool,
|
|
2045
|
+
proofToolUrl: rule.proofToolUrl
|
|
2046
|
+
});
|
|
2047
|
+
}
|
|
2048
|
+
totalWeight += weight;
|
|
2049
|
+
}
|
|
2050
|
+
const overallScore = totalWeight === 0 ? 100 : Math.round(passedWeight / totalWeight * 100);
|
|
2051
|
+
const categoryScores = {};
|
|
2052
|
+
for (const [pack, { passed, total }] of Object.entries(packBreakdown)) {
|
|
2053
|
+
categoryScores[pack] = total === 0 ? 100 : Math.round(passed / total * 100);
|
|
2054
|
+
}
|
|
2055
|
+
return { violations, packBreakdown, evidenceBreakdown, totalWeight, overallScore, categoryScores };
|
|
2056
|
+
}
|
|
2057
|
+
function computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, proofRecords) {
|
|
2058
|
+
const now = /* @__PURE__ */ new Date();
|
|
2059
|
+
const proofSet = new Set(
|
|
2060
|
+
proofRecords.filter((r) => !r.expiresAt || new Date(r.expiresAt) > now).map((r) => r.ruleId)
|
|
2061
|
+
);
|
|
2062
|
+
let provedPassedWeight = 0;
|
|
2063
|
+
for (const rule of rules) {
|
|
2064
|
+
if (suppressedIds.has(rule.id)) continue;
|
|
2065
|
+
const result = ruleResults.get(rule.id);
|
|
2066
|
+
const weight = SEVERITY_WEIGHTS[rule.severity];
|
|
2067
|
+
const isProved = rule.evidenceLevel === "probed" && result.pass || rule.evidenceLevel === "behavioral" && result.pass || rule.evidenceLevel === "external" && proofSet.has(rule.id);
|
|
2068
|
+
if (isProved) provedPassedWeight += weight;
|
|
2069
|
+
}
|
|
2070
|
+
const provedScore = totalWeight === 0 ? 100 : Math.round(provedPassedWeight / totalWeight * 100);
|
|
2071
|
+
const pendingProofCount = rules.filter(
|
|
2072
|
+
(r) => r.evidenceLevel === "external" && !suppressedIds.has(r.id) && ruleResults.get(r.id).pass && !proofSet.has(r.id)
|
|
2073
|
+
).length;
|
|
2074
|
+
return { provedScore, provedGrade: scoreToGrade(provedScore), pendingProofCount };
|
|
2075
|
+
}
|
|
2076
|
+
function runAudit(manifest, opts = {}) {
|
|
2077
|
+
const rules = resolveActiveRules(manifest, opts);
|
|
2078
|
+
const { suppressions, suppressedIds } = collectSuppressions(manifest);
|
|
2079
|
+
const ruleResults = executeRuleChecks(rules, manifest, suppressedIds);
|
|
2080
|
+
const {
|
|
2081
|
+
violations,
|
|
2082
|
+
packBreakdown,
|
|
2083
|
+
evidenceBreakdown,
|
|
2084
|
+
totalWeight,
|
|
2085
|
+
overallScore,
|
|
2086
|
+
categoryScores
|
|
2087
|
+
} = computeScoring(rules, ruleResults, suppressedIds);
|
|
2088
|
+
const proved = opts.proofRecords !== void 0 ? computeProvedScore(rules, ruleResults, suppressedIds, totalWeight, opts.proofRecords) : {};
|
|
2089
|
+
return {
|
|
2090
|
+
agentName: manifest.metadata.name,
|
|
2091
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2092
|
+
overallScore,
|
|
2093
|
+
grade: scoreToGrade(overallScore),
|
|
2094
|
+
categoryScores,
|
|
2095
|
+
violations,
|
|
2096
|
+
suppressions,
|
|
2097
|
+
passedRules: rules.length - violations.length,
|
|
2098
|
+
totalRules: rules.length,
|
|
2099
|
+
packBreakdown,
|
|
2100
|
+
evidenceBreakdown,
|
|
2101
|
+
...proved
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
// src/generate/index.ts
|
|
2106
|
+
var _registry = /* @__PURE__ */ new Map();
|
|
2107
|
+
function registerAdapter(adapter) {
|
|
2108
|
+
_registry.set(adapter.framework.toLowerCase(), adapter);
|
|
2109
|
+
}
|
|
2110
|
+
function getAdapter(framework) {
|
|
2111
|
+
return _registry.get(framework.toLowerCase());
|
|
2112
|
+
}
|
|
2113
|
+
function listAdapters() {
|
|
2114
|
+
return [..._registry.keys()];
|
|
2115
|
+
}
|
|
2116
|
+
function generateAdapter(manifest, framework, options) {
|
|
2117
|
+
const adapter = _registry.get(framework.toLowerCase());
|
|
2118
|
+
if (!adapter) {
|
|
2119
|
+
throw new Error(
|
|
2120
|
+
`No adapter registered for framework: ${framework}
|
|
2121
|
+
Available: ${listAdapters().join(", ") || "none"}
|
|
2122
|
+
Install an adapter package, e.g: npm install @agentspec/adapter-langgraph`
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
2125
|
+
return adapter.generate(manifest, options);
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
// src/agent/reporter.ts
|
|
2129
|
+
var AgentSpecReporter = class {
|
|
2130
|
+
constructor(manifest, opts = {}) {
|
|
2131
|
+
this.manifest = manifest;
|
|
2132
|
+
this.opts = opts;
|
|
2133
|
+
this.refreshIntervalMs = opts.refreshIntervalMs ?? 3e4;
|
|
2134
|
+
this.staleAfterMs = opts.staleAfterMs ?? 6e4;
|
|
2135
|
+
}
|
|
2136
|
+
cached = null;
|
|
2137
|
+
timer = null;
|
|
2138
|
+
_pushTimer = null;
|
|
2139
|
+
refreshIntervalMs;
|
|
2140
|
+
staleAfterMs;
|
|
2141
|
+
refreshing = false;
|
|
2142
|
+
stopped = false;
|
|
2143
|
+
registeredTools = /* @__PURE__ */ new Set();
|
|
2144
|
+
/**
|
|
2145
|
+
* Register a tool handler by name.
|
|
2146
|
+
* Only registered tools report status: 'pass' in the live health check.
|
|
2147
|
+
* Declared tools that are not registered report status: 'fail'.
|
|
2148
|
+
*
|
|
2149
|
+
* Call this once per tool after wiring up your tool handlers:
|
|
2150
|
+
* reporter.registerTool('search-arxiv', handler)
|
|
2151
|
+
*
|
|
2152
|
+
* The handler argument is accepted but not called — it is retained solely so
|
|
2153
|
+
* callers can pass the real function object for documentation purposes.
|
|
2154
|
+
*/
|
|
2155
|
+
registerTool(name, _handler) {
|
|
2156
|
+
this.registeredTools.add(name);
|
|
2157
|
+
}
|
|
2158
|
+
/**
|
|
2159
|
+
* Start background refresh. Call once during app startup.
|
|
2160
|
+
* The first refresh runs immediately (non-blocking).
|
|
2161
|
+
*/
|
|
2162
|
+
start() {
|
|
2163
|
+
if (this.timer !== null) return;
|
|
2164
|
+
void this.refresh();
|
|
2165
|
+
this.timer = setInterval(() => {
|
|
2166
|
+
void this.refresh();
|
|
2167
|
+
}, this.refreshIntervalMs);
|
|
2168
|
+
if (this.timer.unref) {
|
|
2169
|
+
this.timer.unref();
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
/**
|
|
2173
|
+
* Stop background refresh. Call during graceful shutdown.
|
|
2174
|
+
* After stop(), getReport() will no longer trigger new checks.
|
|
2175
|
+
*/
|
|
2176
|
+
stop() {
|
|
2177
|
+
this.stopped = true;
|
|
2178
|
+
if (this.timer !== null) {
|
|
2179
|
+
clearInterval(this.timer);
|
|
2180
|
+
this.timer = null;
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
/**
|
|
2184
|
+
* Returns the cached HealthReport.
|
|
2185
|
+
* If no cached result exists or the cache is stale, runs checks synchronously.
|
|
2186
|
+
* After stop() is called, returns the last cached result (no new checks run).
|
|
2187
|
+
*/
|
|
2188
|
+
async getReport() {
|
|
2189
|
+
const now = Date.now();
|
|
2190
|
+
if (this.cached && (this.stopped || now - this.cached.ts < this.staleAfterMs)) {
|
|
2191
|
+
return this.cached.report;
|
|
2192
|
+
}
|
|
2193
|
+
await this.refresh();
|
|
2194
|
+
return this.cached.report;
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Standard HTTP handler. Mount at GET /agentspec/health.
|
|
2198
|
+
* Returns 200 with the HealthReport as JSON.
|
|
2199
|
+
*
|
|
2200
|
+
* Compatible with any framework that accepts (req, res) handlers
|
|
2201
|
+
* with res.status() and res.json().
|
|
2202
|
+
*/
|
|
2203
|
+
httpHandler() {
|
|
2204
|
+
return async (_req, res) => {
|
|
2205
|
+
try {
|
|
2206
|
+
const report = await this.getReport();
|
|
2207
|
+
res.status(200).json(report);
|
|
2208
|
+
} catch (err) {
|
|
2209
|
+
res.status(500).json({ error: String(err) });
|
|
2210
|
+
}
|
|
2211
|
+
};
|
|
2212
|
+
}
|
|
2213
|
+
/**
|
|
2214
|
+
* Start sending heartbeats to the control plane.
|
|
2215
|
+
* Fires immediately, then repeats every `intervalSeconds` (default 30).
|
|
2216
|
+
* Idempotent — calling twice does not create a second timer.
|
|
2217
|
+
*/
|
|
2218
|
+
startPushMode(opts) {
|
|
2219
|
+
if (this._pushTimer !== null) return;
|
|
2220
|
+
const intervalMs = (opts.intervalSeconds ?? 30) * 1e3;
|
|
2221
|
+
void this._pushHeartbeat(opts);
|
|
2222
|
+
this._pushTimer = setInterval(() => {
|
|
2223
|
+
void this._pushHeartbeat(opts);
|
|
2224
|
+
}, intervalMs);
|
|
2225
|
+
if (this._pushTimer.unref) {
|
|
2226
|
+
this._pushTimer.unref();
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
async _pushHeartbeat(opts) {
|
|
2230
|
+
try {
|
|
2231
|
+
const health = await this.getReport();
|
|
2232
|
+
const gap = runAudit(this.manifest);
|
|
2233
|
+
const trimmedChecks = [...health.checks];
|
|
2234
|
+
let body = JSON.stringify({ health, gap });
|
|
2235
|
+
if (body.length > 65536) {
|
|
2236
|
+
while (trimmedChecks.length > 0) {
|
|
2237
|
+
trimmedChecks.pop();
|
|
2238
|
+
body = JSON.stringify({ health: { ...health, checks: trimmedChecks }, gap });
|
|
2239
|
+
if (body.length <= 65536) break;
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
let res;
|
|
2243
|
+
try {
|
|
2244
|
+
res = await fetch(`${opts.controlPlaneUrl}/api/v1/heartbeat`, {
|
|
2245
|
+
method: "POST",
|
|
2246
|
+
headers: {
|
|
2247
|
+
"Authorization": `Bearer ${opts.apiKey}`,
|
|
2248
|
+
"Content-Type": "application/json"
|
|
2249
|
+
},
|
|
2250
|
+
body
|
|
2251
|
+
});
|
|
2252
|
+
} catch (err) {
|
|
2253
|
+
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2254
|
+
opts.onError?.(new Error(sanitized));
|
|
2255
|
+
return;
|
|
2256
|
+
}
|
|
2257
|
+
if (!res.ok) {
|
|
2258
|
+
opts.onError?.(new Error(`Heartbeat failed: HTTP ${res.status}`));
|
|
2259
|
+
}
|
|
2260
|
+
} catch (err) {
|
|
2261
|
+
const sanitized = String(err).split(opts.apiKey).join("[REDACTED]");
|
|
2262
|
+
opts.onError?.(new Error(sanitized));
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
/** Stop sending heartbeats to the control plane. */
|
|
2266
|
+
stopPushMode() {
|
|
2267
|
+
if (this._pushTimer !== null) {
|
|
2268
|
+
clearInterval(this._pushTimer);
|
|
2269
|
+
this._pushTimer = null;
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
/** Returns true if heartbeats are currently being sent. */
|
|
2273
|
+
isPushModeActive() {
|
|
2274
|
+
return this._pushTimer !== null;
|
|
2275
|
+
}
|
|
2276
|
+
async refresh() {
|
|
2277
|
+
if (this.stopped || this.refreshing) return;
|
|
2278
|
+
this.refreshing = true;
|
|
2279
|
+
try {
|
|
2280
|
+
const report = await runHealthCheck(this.manifest, {
|
|
2281
|
+
checkModel: true,
|
|
2282
|
+
checkMcp: true,
|
|
2283
|
+
checkMemory: true,
|
|
2284
|
+
checkServices: true
|
|
2285
|
+
});
|
|
2286
|
+
const toolChecks = (this.manifest.spec.tools ?? []).map((tool) => {
|
|
2287
|
+
const registered = this.registeredTools.has(tool.name);
|
|
2288
|
+
return {
|
|
2289
|
+
id: `tool:${tool.name}`,
|
|
2290
|
+
category: "tool",
|
|
2291
|
+
status: registered ? "pass" : "fail",
|
|
2292
|
+
severity: "info",
|
|
2293
|
+
...registered ? {} : { message: "Handler not registered \u2014 call reporter.registerTool()" }
|
|
2294
|
+
};
|
|
2295
|
+
});
|
|
2296
|
+
const toolPassed = toolChecks.filter((c) => c.status === "pass").length;
|
|
2297
|
+
const toolFailed = toolChecks.filter((c) => c.status === "fail").length;
|
|
2298
|
+
const enrichedReport = {
|
|
2299
|
+
...report,
|
|
2300
|
+
checks: [...report.checks, ...toolChecks],
|
|
2301
|
+
summary: {
|
|
2302
|
+
...report.summary,
|
|
2303
|
+
passed: report.summary.passed + toolPassed,
|
|
2304
|
+
failed: report.summary.failed + toolFailed
|
|
2305
|
+
}
|
|
2306
|
+
};
|
|
2307
|
+
this.cached = { report: enrichedReport, ts: Date.now() };
|
|
2308
|
+
} catch {
|
|
2309
|
+
if (!this.cached) {
|
|
2310
|
+
this.cached = {
|
|
2311
|
+
ts: Date.now(),
|
|
2312
|
+
report: {
|
|
2313
|
+
agentName: this.manifest.metadata.name,
|
|
2314
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2315
|
+
status: "unhealthy",
|
|
2316
|
+
summary: { passed: 0, failed: 1, warnings: 0, skipped: 0 },
|
|
2317
|
+
checks: [
|
|
2318
|
+
{
|
|
2319
|
+
id: "reporter:internal-error",
|
|
2320
|
+
category: "env",
|
|
2321
|
+
status: "fail",
|
|
2322
|
+
severity: "error",
|
|
2323
|
+
message: "AgentSpecReporter failed to run health checks"
|
|
2324
|
+
}
|
|
2325
|
+
]
|
|
2326
|
+
}
|
|
2327
|
+
};
|
|
2328
|
+
} else {
|
|
2329
|
+
this.cached = { ...this.cached, ts: Date.now() };
|
|
2330
|
+
}
|
|
2331
|
+
} finally {
|
|
2332
|
+
this.refreshing = false;
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
};
|
|
2336
|
+
|
|
2337
|
+
// src/agent/adapters/fastify.ts
|
|
2338
|
+
function agentSpecFastifyPlugin(reporter) {
|
|
2339
|
+
return async (app) => {
|
|
2340
|
+
app.get("/agentspec/health", async (_req, reply) => {
|
|
2341
|
+
try {
|
|
2342
|
+
const report = await reporter.getReport();
|
|
2343
|
+
reply.status(200).send(report);
|
|
2344
|
+
} catch (err) {
|
|
2345
|
+
reply.status(500).send({ error: String(err) });
|
|
2346
|
+
}
|
|
2347
|
+
});
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
// src/agent/adapters/express.ts
|
|
2352
|
+
function agentSpecExpressRouter(reporter) {
|
|
2353
|
+
const handler = async (req, res, next) => {
|
|
2354
|
+
const method = req.method?.toUpperCase() ?? "GET";
|
|
2355
|
+
const path = req.path ?? req.url ?? "";
|
|
2356
|
+
if (method !== "GET" || path !== "/health") {
|
|
2357
|
+
next();
|
|
2358
|
+
return;
|
|
2359
|
+
}
|
|
2360
|
+
try {
|
|
2361
|
+
const report = await reporter.getReport();
|
|
2362
|
+
res.status(200).json(report);
|
|
2363
|
+
} catch (err) {
|
|
2364
|
+
try {
|
|
2365
|
+
res.status(500).json({ error: String(err) });
|
|
2366
|
+
} catch {
|
|
2367
|
+
next(err);
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
};
|
|
2371
|
+
handler._routes = [
|
|
2372
|
+
{ path: "/health" }
|
|
2373
|
+
];
|
|
2374
|
+
return handler;
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
exports.AUDIT_RULE_IDS = AUDIT_RULE_IDS;
|
|
2378
|
+
exports.AgentSpecReporter = AgentSpecReporter;
|
|
2379
|
+
exports.LATEST_API_VERSION = LATEST_API_VERSION;
|
|
2380
|
+
exports.ManifestSchema = ManifestSchema;
|
|
2381
|
+
exports.agentSpecExpressRouter = agentSpecExpressRouter;
|
|
2382
|
+
exports.agentSpecFastifyPlugin = agentSpecFastifyPlugin;
|
|
2383
|
+
exports.collectEnvRefs = collectEnvRefs;
|
|
2384
|
+
exports.collectFileRefs = collectFileRefs;
|
|
2385
|
+
exports.detectRefType = detectRefType;
|
|
2386
|
+
exports.detectVersion = detectVersion;
|
|
2387
|
+
exports.exportJsonSchema = exportJsonSchema;
|
|
2388
|
+
exports.generateAdapter = generateAdapter;
|
|
2389
|
+
exports.getAdapter = getAdapter;
|
|
2390
|
+
exports.isLatestVersion = isLatestVersion;
|
|
2391
|
+
exports.isProofRecord = isProofRecord;
|
|
2392
|
+
exports.listAdapters = listAdapters;
|
|
2393
|
+
exports.loadManifest = loadManifest;
|
|
2394
|
+
exports.migrateManifest = migrateManifest;
|
|
2395
|
+
exports.registerAdapter = registerAdapter;
|
|
2396
|
+
exports.resolveRef = resolveRef;
|
|
2397
|
+
exports.resolveRefs = resolveRefs;
|
|
2398
|
+
exports.runAudit = runAudit;
|
|
2399
|
+
exports.runHealthCheck = runHealthCheck;
|
|
2400
|
+
exports.runServiceChecks = runServiceChecks;
|
|
2401
|
+
exports.scoreToGrade = scoreToGrade;
|
|
2402
|
+
exports.tryLoadManifest = tryLoadManifest;
|
|
2403
|
+
//# sourceMappingURL=index.cjs.map
|
|
2404
|
+
//# sourceMappingURL=index.cjs.map
|