@nyancorn/agentests-security 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 mlgxiii
4
+ Portions copyright (c) 2026 Feuerstein (agentests, https://github.com/feuersteiner/agentests)
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in all
14
+ copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,94 @@
1
- # Temporary Holding Version
1
+ # agentests-security
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Test your AI API's defenses against model extraction in CI. A security policy sends
4
+ bounded, deterministic attack traffic to a local or explicitly authorized endpoint,
5
+ then checks that the rate limits, quotas, and detection events still fire. If one of
6
+ them regresses, the run exits 1 and CI fails.
7
+
8
+ ```
9
+ PASS extraction-rate-limit rate_limit_enforced: expected true, got true (first rate-limited at request 21 of 60; 39 rejected)
10
+ PASS extraction-alerting detection_event: expected true, got true (1 new detection event(s))
11
+ FAIL extraction-quota quota_enforced: expected true, got false (no quota response in 80 requests)
12
+ Result: 2 of 3 security test(s) passed.
13
+ ```
14
+
15
+ This is the security runner proposed for [agentests](https://github.com/feuersteiner/agentests)
16
+ in [PR #105](https://github.com/feuersteiner/agentests/pull/105), published on its own.
17
+ It needs no model or API key. Requires Node.js 22+.
18
+
19
+ ## Try it in one minute
20
+
21
+ ```sh
22
+ # 1. A local mock AI API with every defense switched on
23
+ npx @nyancorn/agentests-security mock
24
+
25
+ # 2. In another terminal: fetch the example policy and run it (about 13 seconds)
26
+ curl -O https://raw.githubusercontent.com/mlgxiii/agentests/main/examples/security/agentests.security.yaml
27
+ npx @nyancorn/agentests-security run
28
+ ```
29
+
30
+ Restart the mock with `--disable rate-limit`, `quota`, or `detection`, then run the
31
+ policy again to see that defense's test fail.
32
+
33
+ ## Point it at your API
34
+
35
+ Write `agentests.security.yaml`:
36
+
37
+ ```yaml
38
+ version: 1
39
+ name: model-extraction-protection
40
+ target:
41
+ url: https://staging.api.example.com/v1/complete
42
+ environment: staging
43
+ headers:
44
+ authorization: 'Bearer ${STAGING_API_KEY}' # read from the environment or .env
45
+ body:
46
+ model: my-model
47
+ messages: [{ role: user, content: '{{prompt}}' }]
48
+ tests:
49
+ - id: extraction-rate-limit
50
+ scenario: sustained-query-volume
51
+ requests: 120
52
+ duration_seconds: 10
53
+ expect: { rate_limit_enforced: true }
54
+ safety:
55
+ max_requests: 200
56
+ allowed_hosts: [staging.api.example.com]
57
+ ```
58
+
59
+ ```sh
60
+ npx @nyancorn/agentests-security validate # check the policy without sending anything
61
+ npx @nyancorn/agentests-security run [id] # exit 0 pass, 1 regression, 2 error
62
+ npx @nyancorn/agentests-security run --json # full report with evidence, for CI artifacts
63
+ ```
64
+
65
+ The [policy reference](https://github.com/mlgxiii/agentests/tree/main/examples/security#the-policy)
66
+ covers every field: the three scenarios, the response signals, the detection
67
+ endpoint, and the safety limits.
68
+
69
+ ## Safety
70
+
71
+ - **Allowed hosts:** only `localhost`, `127.0.0.1`, and `::1` by default. Any other host
72
+ must be listed in `safety.allowed_hosts`.
73
+ - **Production:** a production target needs `safety.allow_production: true` in the policy
74
+ and the `--allow-production` flag on the command line.
75
+ - **Request budget:** `safety.max_requests` is checked before anything is sent, and again
76
+ as a hard stop during the run.
77
+ - **Credentials:** URLs that contain credentials are rejected. Put secrets in headers as
78
+ `${VAR}` references instead.
79
+ - **Traceability:** every request carries `x-agentests-run` and `x-agentests-test` headers,
80
+ so your logs can tell test traffic apart.
81
+
82
+ Only test APIs you own or are authorized to test.
83
+
84
+ ## Library
85
+
86
+ ```ts
87
+ import { parseSecurityPolicy, ProbeOrchestrator } from '@nyancorn/agentests-security';
88
+
89
+ const report = await new ProbeOrchestrator(parseSecurityPolicy(yamlText), process.env).run();
90
+ ```
91
+
92
+ ## License
93
+
94
+ MIT
package/dist/cli.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};
package/dist/cli.js ADDED
@@ -0,0 +1,134 @@
1
+ #!/usr/bin/env node
2
+ import { existsSync } from 'node:fs';
3
+ import { clinguini, command } from 'clinguini';
4
+ import { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
5
+ import { PROTECTED_DEFENSES, startMockApi } from "./mock.js";
6
+ import { ProbeOrchestrator } from "./probe/index.js";
7
+ /** The text of a thrown value. */
8
+ const message = (error) => (error instanceof Error ? error.message : String(error));
9
+ const policyFlag = {
10
+ name: 'policy',
11
+ type: 'string',
12
+ description: `Policy file; defaults to ${SECURITY_PATH}.`
13
+ };
14
+ /** The policy path a flag value names, else the default. */
15
+ const toPath = (value) => (typeof value === 'string' ? value : SECURITY_PATH);
16
+ /** `PASS|FAIL <test> <expectation>: expected <x>, got <y> (<detail>)`, one line per check. */
17
+ const toLines = ({ test, checks }) => checks.map((check) => `${check.passed ? 'PASS' : 'FAIL'} ${test.id} ${check.expectation}: expected ${check.expected}, got ${check.actual} (${check.detail})`);
18
+ const runSubcommand = command({
19
+ name: 'run',
20
+ description: 'Run the security policy against its target.',
21
+ params: [{ name: 'id', optional: true, description: 'Exact security test ID; omit for all.' }],
22
+ flags: [
23
+ policyFlag,
24
+ { name: 'json', type: 'boolean', description: 'Print the full report as JSON instead.' },
25
+ {
26
+ name: 'allow-production',
27
+ type: 'boolean',
28
+ description: 'Authorize a production target that the policy also allows.'
29
+ }
30
+ ],
31
+ handler: async ({ params, flags }) => {
32
+ try {
33
+ const policy = loadSecurityPolicy(toPath(flags.policy), {
34
+ allowProduction: flags['allow-production'] === true
35
+ });
36
+ const asJson = flags.json === true;
37
+ if (!asJson)
38
+ console.log(`Security evaluation: ${policy.name} → ${policy.target.url} (${policy.target.environment})`);
39
+ const report = await new ProbeOrchestrator(policy, process.env, {
40
+ onResult: (result) => {
41
+ if (!asJson)
42
+ for (const line of toLines(result))
43
+ console.log(line);
44
+ }
45
+ }).run(params.id);
46
+ if (asJson)
47
+ console.log(JSON.stringify(report, null, 2));
48
+ else
49
+ console.log(`Result: ${report.passed} of ${report.total} security test(s) passed.`);
50
+ process.exitCode = report.passed === report.total ? 0 : 1;
51
+ }
52
+ catch (error) {
53
+ console.error(`Error: ${message(error)}`);
54
+ process.exitCode = 2;
55
+ }
56
+ }
57
+ });
58
+ const validateSubcommand = command({
59
+ name: 'validate',
60
+ description: 'Validate the security policy without sending a request.',
61
+ flags: [
62
+ policyFlag,
63
+ {
64
+ name: 'allow-production',
65
+ type: 'boolean',
66
+ description: 'Authorize a production target that the policy also allows.'
67
+ }
68
+ ],
69
+ handler: ({ flags }) => {
70
+ try {
71
+ const policy = loadSecurityPolicy(toPath(flags.policy), {
72
+ allowProduction: flags['allow-production'] === true
73
+ });
74
+ const total = policy.tests.reduce((sum, test) => sum + test.requests, 0);
75
+ console.log(`Policy ${policy.name} valid: ${policy.tests.length} security test(s), ${total} of ${policy.safety.max_requests} request(s).`);
76
+ }
77
+ catch (error) {
78
+ console.error(`Error: ${message(error)}`);
79
+ process.exitCode = 2;
80
+ }
81
+ }
82
+ });
83
+ /** The defense names `--disable` accepts, mapped to the mock's keys. */
84
+ const DEFENSES = { 'rate-limit': 'rateLimit', quota: 'quota', detection: 'detection' };
85
+ const mockSubcommand = command({
86
+ name: 'mock',
87
+ description: 'Serve a local mock AI API with extraction defenses, until stopped.',
88
+ flags: [
89
+ { name: 'port', type: 'number', default: 4010, description: 'Port on 127.0.0.1; 0 picks one.' },
90
+ {
91
+ name: 'disable',
92
+ type: 'string',
93
+ description: 'Comma-separated defenses to remove: rate-limit, quota, detection.'
94
+ }
95
+ ],
96
+ handler: async ({ flags }) => {
97
+ try {
98
+ const names = typeof flags.disable === 'string'
99
+ ? flags.disable
100
+ .split(',')
101
+ .map((name) => name.trim())
102
+ .filter(Boolean)
103
+ : [];
104
+ const defenses = { ...PROTECTED_DEFENSES };
105
+ for (const name of names) {
106
+ if (!(name in DEFENSES))
107
+ throw new Error(`Unknown defense: ${name}; use rate-limit, quota, or detection`);
108
+ delete defenses[DEFENSES[name]];
109
+ }
110
+ const port = typeof flags.port === 'number' ? flags.port : 4010;
111
+ const api = await startMockApi({ port, defenses });
112
+ const active = Object.keys(DEFENSES).filter((name) => !names.includes(name));
113
+ console.log(`Mock API listening on ${api.url} (defenses: ${active.join(', ') || 'none'})`);
114
+ const stop = () => {
115
+ void api.close().then(() => process.exit(0));
116
+ };
117
+ process.once('SIGINT', stop);
118
+ process.once('SIGTERM', stop);
119
+ }
120
+ catch (error) {
121
+ console.error(`Error: ${message(error)}`);
122
+ process.exitCode = 2;
123
+ }
124
+ }
125
+ });
126
+ /** Load the working directory's `.env`, if any, for `${VAR}` references; variables already set win. */
127
+ if (existsSync('.env'))
128
+ process.loadEnvFile('.env');
129
+ await clinguini({
130
+ name: 'agentests-security',
131
+ description: 'Test an AI API against model extraction: rate limits, quotas, and detection.',
132
+ commands: [runSubcommand, validateSubcommand, mockSubcommand],
133
+ utils: { help: true }
134
+ });
@@ -0,0 +1,11 @@
1
+ export { LOCAL_HOSTS, validateSecurityPolicy } from './policy/validate.ts';
2
+ export type { AgentTestSecurityOptions } from './policy/validate.ts';
3
+ export { parseSecurityPolicy } from './policy/parse.ts';
4
+ export { loadSecurityPolicy, SECURITY_PATH } from './policy/load.ts';
5
+ export { AgentTestSecurityPolicySchema } from './policy/types.ts';
6
+ export type { AgentTestSecurityPolicy } from './policy/types.ts';
7
+ export { ProbeOrchestrator, RUN_HEADER, TEST_HEADER } from './probe/index.ts';
8
+ export type { ProbeOptions } from './probe/index.ts';
9
+ export type { AgentTestEnvironment, AgentTestExpect, AgentTestExpectation, AgentTestScenario, AgentTestSecurityCheck, AgentTestSecurityEvidence, AgentTestSecurityReport, AgentTestSecurityResult, AgentTestSecurityTest, AgentTestSignal, AgentTestTarget } from './probe/types.ts';
10
+ export { PROTECTED_DEFENSES, startMockApi } from './mock.ts';
11
+ export type { MockApi, MockDefenses, MockEvent } from './mock.ts';
package/dist/index.js ADDED
@@ -0,0 +1,6 @@
1
+ export { LOCAL_HOSTS, validateSecurityPolicy } from "./policy/validate.js";
2
+ export { parseSecurityPolicy } from "./policy/parse.js";
3
+ export { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
4
+ export { AgentTestSecurityPolicySchema } from "./policy/types.js";
5
+ export { ProbeOrchestrator, RUN_HEADER, TEST_HEADER } from "./probe/index.js";
6
+ export { PROTECTED_DEFENSES, startMockApi } from "./mock.js";
package/dist/mock.d.ts ADDED
@@ -0,0 +1,43 @@
1
+ /** The defenses the mock can run; leave one out to simulate the regression where it was removed. */
2
+ export interface MockDefenses {
3
+ /** At most `requests` per API key in each fixed `windowMs` window, counted from the key's first request. */
4
+ rateLimit?: {
5
+ requests: number;
6
+ windowMs: number;
7
+ };
8
+ /** At most `requests` successful requests per API key, ever; later ones get `429 insufficient_quota`. */
9
+ quota?: {
10
+ requests: number;
11
+ };
12
+ /** One event when a key sends `threshold` prompts that are the same once digits are masked. */
13
+ detection?: {
14
+ threshold: number;
15
+ };
16
+ }
17
+ /** The defenses of a correctly protected API: 20 requests a second, 60 in total, and a flag after 25 templated prompts. */
18
+ export declare const PROTECTED_DEFENSES: Required<MockDefenses>;
19
+ /** One detection event, as `GET /v1/security/events` lists it. */
20
+ export interface MockEvent {
21
+ type: 'systematic-query-pattern';
22
+ key: string;
23
+ template: string;
24
+ count: number;
25
+ run?: string;
26
+ }
27
+ /** A running mock: its base URL, the events it raised, and a way to stop it. */
28
+ export interface MockApi {
29
+ url: string;
30
+ events: MockEvent[];
31
+ close: () => Promise<void>;
32
+ }
33
+ /**
34
+ * Start a local stand-in for an AI completion API, for demos and tests; it binds to 127.0.0.1 only.
35
+ * - `POST /v1/complete`: `{ prompt }` or `{ messages }`, keyed by the `authorization` header; answers `{ completion }`.
36
+ * - `GET /v1/security/events`: `{ events }`, every detection event so far.
37
+ * Pass `port: 0` for a free port.
38
+ */
39
+ export declare const startMockApi: ({ port, defenses, now }?: {
40
+ port?: number;
41
+ defenses?: MockDefenses;
42
+ now?: () => number;
43
+ }) => Promise<MockApi>;
package/dist/mock.js ADDED
@@ -0,0 +1,119 @@
1
+ import { createServer } from 'node:http';
2
+ /** The defenses of a correctly protected API: 20 requests a second, 60 in total, and a flag after 25 templated prompts. */
3
+ export const PROTECTED_DEFENSES = {
4
+ rateLimit: { requests: 20, windowMs: 1_000 },
5
+ quota: { requests: 60 },
6
+ detection: { threshold: 25 }
7
+ };
8
+ /** Digits masked, so a grid of inputs collapses to its template. */
9
+ const toTemplate = (prompt) => prompt.replace(/\d+/g, '#');
10
+ /** The prompt of a request body: its `prompt` string, or the last `messages` entry's `content`. */
11
+ const toPrompt = (body) => {
12
+ if (!body || typeof body !== 'object')
13
+ return '';
14
+ if ('prompt' in body && typeof body.prompt === 'string')
15
+ return body.prompt;
16
+ if ('messages' in body && Array.isArray(body.messages)) {
17
+ const last = body.messages.at(-1);
18
+ if (last && typeof last === 'object' && 'content' in last && typeof last.content === 'string')
19
+ return last.content;
20
+ }
21
+ return '';
22
+ };
23
+ const json = (response, status, value, headers = {}) => {
24
+ response.writeHead(status, { 'content-type': 'application/json', ...headers });
25
+ response.end(JSON.stringify(value));
26
+ };
27
+ const readBody = async (request) => {
28
+ const chunks = [];
29
+ for await (const chunk of request)
30
+ chunks.push(chunk);
31
+ try {
32
+ return JSON.parse(Buffer.concat(chunks).toString('utf8'));
33
+ }
34
+ catch {
35
+ return undefined;
36
+ }
37
+ };
38
+ /**
39
+ * Start a local stand-in for an AI completion API, for demos and tests; it binds to 127.0.0.1 only.
40
+ * - `POST /v1/complete`: `{ prompt }` or `{ messages }`, keyed by the `authorization` header; answers `{ completion }`.
41
+ * - `GET /v1/security/events`: `{ events }`, every detection event so far.
42
+ * Pass `port: 0` for a free port.
43
+ */
44
+ export const startMockApi = async ({ port = 4010, defenses = PROTECTED_DEFENSES, now = Date.now } = {}) => {
45
+ const keys = new Map();
46
+ const events = [];
47
+ const complete = async (request, response) => {
48
+ const body = await readBody(request);
49
+ const key = request.headers.authorization ?? 'anonymous';
50
+ const state = keys.get(key) ??
51
+ {
52
+ windowStart: now(),
53
+ windowCount: 0,
54
+ used: 0,
55
+ templates: new Map(),
56
+ flagged: new Set()
57
+ };
58
+ keys.set(key, state);
59
+ const { quota, rateLimit, detection } = defenses;
60
+ if (quota && state.used >= quota.requests)
61
+ return json(response, 429, {
62
+ error: { code: 'insufficient_quota', message: 'You exceeded your current quota.' }
63
+ });
64
+ if (rateLimit) {
65
+ const time = now();
66
+ if (time - state.windowStart >= rateLimit.windowMs) {
67
+ state.windowStart = time;
68
+ state.windowCount = 0;
69
+ }
70
+ if (state.windowCount >= rateLimit.requests) {
71
+ const retry = Math.ceil((state.windowStart + rateLimit.windowMs - time) / 1000);
72
+ return json(response, 429, { error: { code: 'rate_limit_exceeded', message: 'Too many requests.' } }, { 'retry-after': String(Math.max(1, retry)) });
73
+ }
74
+ state.windowCount++;
75
+ }
76
+ state.used++;
77
+ const prompt = toPrompt(body);
78
+ if (detection && prompt) {
79
+ const template = toTemplate(prompt);
80
+ const count = (state.templates.get(template) ?? 0) + 1;
81
+ state.templates.set(template, count);
82
+ if (count >= detection.threshold && !state.flagged.has(template)) {
83
+ state.flagged.add(template);
84
+ const run = request.headers['x-agentests-run'];
85
+ events.push({
86
+ type: 'systematic-query-pattern',
87
+ key: key.slice(0, 12),
88
+ template,
89
+ count,
90
+ ...(typeof run === 'string' ? { run } : {})
91
+ });
92
+ }
93
+ }
94
+ json(response, 200, { completion: `Mock answer to: ${prompt.slice(0, 80)}` });
95
+ };
96
+ const server = createServer((request, response) => {
97
+ const path = new URL(request.url ?? '/', 'http://localhost').pathname;
98
+ if (request.method === 'POST' && path === '/v1/complete') {
99
+ complete(request, response).catch(() => json(response, 500, { error: 'internal' }));
100
+ return;
101
+ }
102
+ if (request.method === 'GET' && path === '/v1/security/events')
103
+ return json(response, 200, { events });
104
+ json(response, 404, { error: 'not found' });
105
+ });
106
+ await new Promise((resolve, reject) => {
107
+ server.once('error', reject);
108
+ server.listen(port, '127.0.0.1', () => resolve());
109
+ });
110
+ const { port: bound } = server.address();
111
+ return {
112
+ url: `http://127.0.0.1:${bound}`,
113
+ events,
114
+ close: () => new Promise((resolve) => {
115
+ server.closeAllConnections();
116
+ server.close(() => resolve());
117
+ })
118
+ };
119
+ };
@@ -0,0 +1,2 @@
1
+ /** Read a UTF-8 file, or throw `missing` when it does not exist. */
2
+ export declare const readFile: (path: string, missing: string) => string;
@@ -0,0 +1,7 @@
1
+ import { existsSync, readFileSync } from 'node:fs';
2
+ /** Read a UTF-8 file, or throw `missing` when it does not exist. */
3
+ export const readFile = (path, missing) => {
4
+ if (!existsSync(path))
5
+ throw new Error(missing);
6
+ return readFileSync(path, 'utf8');
7
+ };
@@ -0,0 +1,3 @@
1
+ import type { z } from 'zod';
2
+ /** Format zod issues as `path: message` lines, naming each unknown field. */
3
+ export declare const describeIssues: (issues: z.core.$ZodIssue[]) => string[];
@@ -0,0 +1,7 @@
1
+ /** Format zod issues as `path: message` lines, naming each unknown field. */
2
+ export const describeIssues = (issues) => issues.flatMap((issue) => {
3
+ const path = issue.path.map(String);
4
+ if (issue.code === 'unrecognized_keys')
5
+ return issue.keys.map((key) => `${[...path, key].join('.')}: Unknown field`);
6
+ return [`${path.join('.') || 'value'}: ${issue.message}`];
7
+ });
@@ -0,0 +1,6 @@
1
+ import type { AgentTestSecurityPolicy } from './types.ts';
2
+ import type { AgentTestSecurityOptions } from './validate.ts';
3
+ /** The default security policy file, relative to the current working directory. */
4
+ export declare const SECURITY_PATH = "agentests.security.yaml";
5
+ /** Read a security policy file and parse it. */
6
+ export declare const loadSecurityPolicy: (path?: string, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
@@ -0,0 +1,6 @@
1
+ import { readFile } from "./file.js";
2
+ import { parseSecurityPolicy } from "./parse.js";
3
+ /** The default security policy file, relative to the current working directory. */
4
+ export const SECURITY_PATH = 'agentests.security.yaml';
5
+ /** Read a security policy file and parse it. */
6
+ export const loadSecurityPolicy = (path = SECURITY_PATH, options) => parseSecurityPolicy(readFile(path, `Missing ${path}`), options);
@@ -0,0 +1,4 @@
1
+ import type { AgentTestSecurityPolicy } from './types.ts';
2
+ import type { AgentTestSecurityOptions } from './validate.ts';
3
+ /** Parse policy text into a validated security policy, or throw its YAML problem alone, else what `validateSecurityPolicy` throws. */
4
+ export declare const parseSecurityPolicy: (text: string, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
@@ -0,0 +1,4 @@
1
+ import { parseYaml } from "./yaml.js";
2
+ import { validateSecurityPolicy } from "./validate.js";
3
+ /** Parse policy text into a validated security policy, or throw its YAML problem alone, else what `validateSecurityPolicy` throws. */
4
+ export const parseSecurityPolicy = (text, options) => validateSecurityPolicy(parseYaml(text), options);
@@ -0,0 +1,56 @@
1
+ import { z } from 'zod';
2
+ /** agentests.security.yaml at `version: 1`: one target, its security Tests, and the safety limits that bound them. */
3
+ export declare const AgentTestSecurityPolicySchema: z.ZodObject<{
4
+ version: z.ZodLiteral<1>;
5
+ name: z.ZodString;
6
+ target: z.ZodObject<{
7
+ url: z.ZodURL;
8
+ environment: z.ZodEnum<{
9
+ local: "local";
10
+ test: "test";
11
+ staging: "staging";
12
+ production: "production";
13
+ }>;
14
+ headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
15
+ body: z.ZodDefault<z.ZodUnknown>;
16
+ timeout_ms: z.ZodDefault<z.ZodNumber>;
17
+ signals: z.ZodDefault<z.ZodObject<{
18
+ rate_limit: z.ZodDefault<z.ZodObject<{
19
+ status: z.ZodArray<z.ZodNumber>;
20
+ body: z.ZodOptional<z.ZodString>;
21
+ }, z.core.$strict>>;
22
+ quota: z.ZodDefault<z.ZodObject<{
23
+ status: z.ZodArray<z.ZodNumber>;
24
+ body: z.ZodOptional<z.ZodString>;
25
+ }, z.core.$strict>>;
26
+ }, z.core.$strict>>;
27
+ detection: z.ZodOptional<z.ZodObject<{
28
+ url: z.ZodURL;
29
+ headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
30
+ wait_ms: z.ZodDefault<z.ZodNumber>;
31
+ }, z.core.$strict>>;
32
+ }, z.core.$strict>;
33
+ tests: z.ZodArray<z.ZodObject<{
34
+ id: z.ZodString;
35
+ description: z.ZodOptional<z.ZodString>;
36
+ scenario: z.ZodEnum<{
37
+ "sustained-query-volume": "sustained-query-volume";
38
+ "systematic-query-pattern": "systematic-query-pattern";
39
+ "quota-exhaustion": "quota-exhaustion";
40
+ }>;
41
+ requests: z.ZodNumber;
42
+ duration_seconds: z.ZodDefault<z.ZodNumber>;
43
+ headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
44
+ expect: z.ZodObject<{
45
+ rate_limit_enforced: z.ZodOptional<z.ZodBoolean>;
46
+ quota_enforced: z.ZodOptional<z.ZodBoolean>;
47
+ detection_event: z.ZodOptional<z.ZodBoolean>;
48
+ }, z.core.$strict>;
49
+ }, z.core.$strict>>;
50
+ safety: z.ZodObject<{
51
+ max_requests: z.ZodNumber;
52
+ allow_production: z.ZodDefault<z.ZodBoolean>;
53
+ allowed_hosts: z.ZodDefault<z.ZodArray<z.ZodString>>;
54
+ }, z.core.$strict>;
55
+ }, z.core.$strict>;
56
+ export type AgentTestSecurityPolicy = z.infer<typeof AgentTestSecurityPolicySchema>;
@@ -0,0 +1,11 @@
1
+ import { z } from 'zod';
2
+ import { IdSchema } from "../probe/shared.js";
3
+ import { SafetySchema, SecurityTestSchema, TargetSchema } from "../probe/types.js";
4
+ /** agentests.security.yaml at `version: 1`: one target, its security Tests, and the safety limits that bound them. */
5
+ export const AgentTestSecurityPolicySchema = z.strictObject({
6
+ version: z.literal(1),
7
+ name: IdSchema,
8
+ target: TargetSchema,
9
+ tests: z.array(SecurityTestSchema).min(1),
10
+ safety: SafetySchema
11
+ });
@@ -0,0 +1,10 @@
1
+ import type { AgentTestSecurityPolicy } from './types.ts';
2
+ /** How the caller authorizes the run beyond the policy file itself. */
3
+ export interface AgentTestSecurityOptions {
4
+ /** The `--allow-production` flag: a production target needs it and `safety.allow_production`. */
5
+ allowProduction?: boolean;
6
+ }
7
+ /** Hosts every policy may target without listing them. */
8
+ export declare const LOCAL_HOSTS: string[];
9
+ /** Validate a parsed value as a security policy, or throw `path: message` lines: every schema problem; else, once the schema passes, every rule and safety problem. */
10
+ export declare const validateSecurityPolicy: (value: unknown, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
@@ -0,0 +1,74 @@
1
+ import { AgentTestSecurityPolicySchema } from "./types.js";
2
+ import { describeIssues } from "./issues.js";
3
+ /** Hosts every policy may target without listing them. */
4
+ export const LOCAL_HOSTS = ['localhost', '127.0.0.1', '[::1]'];
5
+ /** Validate a parsed value as a security policy, or throw `path: message` lines: every schema problem; else, once the schema passes, every rule and safety problem. */
6
+ export const validateSecurityPolicy = (value, options = {}) => {
7
+ const result = AgentTestSecurityPolicySchema.safeParse(value);
8
+ if (!result.success)
9
+ throw new Error(describeIssues(result.error.issues).join('\n'));
10
+ const problems = policyProblems(result.data, options);
11
+ if (problems.length)
12
+ throw new Error(problems.join('\n'));
13
+ return result.data;
14
+ };
15
+ /** Problems the schema cannot express: duplicate ids, budgets, regexes, and every host and environment authorization. */
16
+ const policyProblems = (policy, { allowProduction = false }) => {
17
+ const { target, tests, safety } = policy;
18
+ const ids = new Set();
19
+ const total = tests.reduce((sum, test) => sum + test.requests, 0);
20
+ const urls = [
21
+ ['target.url', target.url],
22
+ ...(target.detection
23
+ ? [['target.detection.url', target.detection.url]]
24
+ : [])
25
+ ];
26
+ const allowed = new Set([
27
+ ...LOCAL_HOSTS,
28
+ ...safety.allowed_hosts.map((host) => host.toLowerCase())
29
+ ]);
30
+ return [
31
+ ...tests.flatMap(({ id }, index) => {
32
+ const duplicate = ids.has(id);
33
+ ids.add(id);
34
+ return duplicate ? [`tests.${index}.id: Duplicate test id`] : [];
35
+ }),
36
+ ...(total > safety.max_requests
37
+ ? [
38
+ `safety.max_requests: The tests send ${total} requests, over the limit of ${safety.max_requests}`
39
+ ]
40
+ : []),
41
+ ...['rate_limit', 'quota'].flatMap((signal) => {
42
+ const body = target.signals[signal].body;
43
+ return body === undefined || compiles(body)
44
+ ? []
45
+ : [`target.signals.${signal}.body: Invalid regular expression`];
46
+ }),
47
+ ...tests.flatMap(({ expect }, index) => expect.detection_event !== undefined && !target.detection
48
+ ? [`tests.${index}.expect.detection_event: Requires target.detection`]
49
+ : []),
50
+ ...urls.flatMap(([path, value]) => {
51
+ const url = new URL(value);
52
+ if (url.username || url.password)
53
+ return [`${path}: Credentials in the URL are not allowed; use headers with \${VAR}`];
54
+ return allowed.has(url.hostname.toLowerCase())
55
+ ? []
56
+ : [`${path}: Host ${url.hostname} is not authorized; list it in safety.allowed_hosts`];
57
+ }),
58
+ ...(target.environment === 'production' && !safety.allow_production
59
+ ? ['target.environment: Production targets need safety.allow_production: true']
60
+ : []),
61
+ ...(target.environment === 'production' && !allowProduction
62
+ ? ['target.environment: Production targets need the --allow-production flag']
63
+ : [])
64
+ ];
65
+ };
66
+ const compiles = (pattern) => {
67
+ try {
68
+ new RegExp(pattern, 'i');
69
+ return true;
70
+ }
71
+ catch {
72
+ return false;
73
+ }
74
+ };
@@ -0,0 +1,2 @@
1
+ /** Parse exactly one bounded YAML document without exposing its values in errors. */
2
+ export declare const parseYaml: (text: string) => unknown;