@nyancorn/agentests-security 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +93 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +134 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +6 -0
- package/dist/mock.d.ts +43 -0
- package/dist/mock.js +119 -0
- package/dist/policy/file.d.ts +2 -0
- package/dist/policy/file.js +7 -0
- package/dist/policy/issues.d.ts +3 -0
- package/dist/policy/issues.js +7 -0
- package/dist/policy/load.d.ts +6 -0
- package/dist/policy/load.js +6 -0
- package/dist/policy/parse.d.ts +4 -0
- package/dist/policy/parse.js +4 -0
- package/dist/policy/types.d.ts +56 -0
- package/dist/policy/types.js +11 -0
- package/dist/policy/validate.d.ts +10 -0
- package/dist/policy/validate.js +74 -0
- package/dist/policy/yaml.d.ts +2 -0
- package/dist/policy/yaml.js +25 -0
- package/dist/probe/http.d.ts +17 -0
- package/dist/probe/http.js +68 -0
- package/dist/probe/index.d.ts +24 -0
- package/dist/probe/index.js +132 -0
- package/dist/probe/scenarios.d.ts +8 -0
- package/dist/probe/scenarios.js +58 -0
- package/dist/probe/shared.d.ts +5 -0
- package/dist/probe/shared.js +9 -0
- package/dist/probe/types.d.ts +150 -0
- package/dist/probe/types.js +67 -0
- package/dist/probe/utils.d.ts +4 -0
- package/dist/probe/utils.js +9 -0
- package/dist/probe/verdict.d.ts +3 -0
- package/dist/probe/verdict.js +37 -0
- package/package.json +57 -5
package/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mlgxiii
|
|
4
|
+
Portions copyright (c) 2026 Feuerstein (agentests, https://github.com/feuersteiner/agentests)
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,94 @@
|
|
|
1
|
-
#
|
|
1
|
+
# agentests-security
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Test your AI API's defenses against model extraction in CI. A security policy sends
|
|
4
|
+
bounded, deterministic attack traffic to a local or explicitly authorized endpoint,
|
|
5
|
+
then checks that the rate limits, quotas, and detection events still fire. If one of
|
|
6
|
+
them regresses, the run exits 1 and CI fails.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
PASS extraction-rate-limit rate_limit_enforced: expected true, got true (first rate-limited at request 21 of 60; 39 rejected)
|
|
10
|
+
PASS extraction-alerting detection_event: expected true, got true (1 new detection event(s))
|
|
11
|
+
FAIL extraction-quota quota_enforced: expected true, got false (no quota response in 80 requests)
|
|
12
|
+
Result: 2 of 3 security test(s) passed.
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
This is the security runner proposed for [agentests](https://github.com/feuersteiner/agentests)
|
|
16
|
+
in [PR #105](https://github.com/feuersteiner/agentests/pull/105), published on its own.
|
|
17
|
+
It needs no model or API key. Requires Node.js 22+.
|
|
18
|
+
|
|
19
|
+
## Try it in one minute
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
# 1. A local mock AI API with every defense switched on
|
|
23
|
+
npx @nyancorn/agentests-security mock
|
|
24
|
+
|
|
25
|
+
# 2. In another terminal: fetch the example policy and run it (about 13 seconds)
|
|
26
|
+
curl -O https://raw.githubusercontent.com/mlgxiii/agentests/main/examples/security/agentests.security.yaml
|
|
27
|
+
npx @nyancorn/agentests-security run
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Restart the mock with `--disable rate-limit`, `quota`, or `detection`, then run the
|
|
31
|
+
policy again to see that defense's test fail.
|
|
32
|
+
|
|
33
|
+
## Point it at your API
|
|
34
|
+
|
|
35
|
+
Write `agentests.security.yaml`:
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
version: 1
|
|
39
|
+
name: model-extraction-protection
|
|
40
|
+
target:
|
|
41
|
+
url: https://staging.api.example.com/v1/complete
|
|
42
|
+
environment: staging
|
|
43
|
+
headers:
|
|
44
|
+
authorization: 'Bearer ${STAGING_API_KEY}' # read from the environment or .env
|
|
45
|
+
body:
|
|
46
|
+
model: my-model
|
|
47
|
+
messages: [{ role: user, content: '{{prompt}}' }]
|
|
48
|
+
tests:
|
|
49
|
+
- id: extraction-rate-limit
|
|
50
|
+
scenario: sustained-query-volume
|
|
51
|
+
requests: 120
|
|
52
|
+
duration_seconds: 10
|
|
53
|
+
expect: { rate_limit_enforced: true }
|
|
54
|
+
safety:
|
|
55
|
+
max_requests: 200
|
|
56
|
+
allowed_hosts: [staging.api.example.com]
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
npx @nyancorn/agentests-security validate # check the policy without sending anything
|
|
61
|
+
npx @nyancorn/agentests-security run [id] # exit 0 pass, 1 regression, 2 error
|
|
62
|
+
npx @nyancorn/agentests-security run --json # full report with evidence, for CI artifacts
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The [policy reference](https://github.com/mlgxiii/agentests/tree/main/examples/security#the-policy)
|
|
66
|
+
covers every field: the three scenarios, the response signals, the detection
|
|
67
|
+
endpoint, and the safety limits.
|
|
68
|
+
|
|
69
|
+
## Safety
|
|
70
|
+
|
|
71
|
+
- **Allowed hosts:** only `localhost`, `127.0.0.1`, and `::1` by default. Any other host
|
|
72
|
+
must be listed in `safety.allowed_hosts`.
|
|
73
|
+
- **Production:** a production target needs `safety.allow_production: true` in the policy
|
|
74
|
+
and the `--allow-production` flag on the command line.
|
|
75
|
+
- **Request budget:** `safety.max_requests` is checked before anything is sent, and again
|
|
76
|
+
as a hard stop during the run.
|
|
77
|
+
- **Credentials:** URLs that contain credentials are rejected. Put secrets in headers as
|
|
78
|
+
`${VAR}` references instead.
|
|
79
|
+
- **Traceability:** every request carries `x-agentests-run` and `x-agentests-test` headers,
|
|
80
|
+
so your logs can tell test traffic apart.
|
|
81
|
+
|
|
82
|
+
Only test APIs you own or are authorized to test.
|
|
83
|
+
|
|
84
|
+
## Library
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
import { parseSecurityPolicy, ProbeOrchestrator } from '@nyancorn/agentests-security';
|
|
88
|
+
|
|
89
|
+
const report = await new ProbeOrchestrator(parseSecurityPolicy(yamlText), process.env).run();
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## License
|
|
93
|
+
|
|
94
|
+
MIT
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { clinguini, command } from 'clinguini';
|
|
4
|
+
import { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
|
|
5
|
+
import { PROTECTED_DEFENSES, startMockApi } from "./mock.js";
|
|
6
|
+
import { ProbeOrchestrator } from "./probe/index.js";
|
|
7
|
+
/** The text of a thrown value. */
|
|
8
|
+
const message = (error) => (error instanceof Error ? error.message : String(error));
|
|
9
|
+
const policyFlag = {
|
|
10
|
+
name: 'policy',
|
|
11
|
+
type: 'string',
|
|
12
|
+
description: `Policy file; defaults to ${SECURITY_PATH}.`
|
|
13
|
+
};
|
|
14
|
+
/** The policy path a flag value names, else the default. */
|
|
15
|
+
const toPath = (value) => (typeof value === 'string' ? value : SECURITY_PATH);
|
|
16
|
+
/** `PASS|FAIL <test> <expectation>: expected <x>, got <y> (<detail>)`, one line per check. */
|
|
17
|
+
const toLines = ({ test, checks }) => checks.map((check) => `${check.passed ? 'PASS' : 'FAIL'} ${test.id} ${check.expectation}: expected ${check.expected}, got ${check.actual} (${check.detail})`);
|
|
18
|
+
const runSubcommand = command({
|
|
19
|
+
name: 'run',
|
|
20
|
+
description: 'Run the security policy against its target.',
|
|
21
|
+
params: [{ name: 'id', optional: true, description: 'Exact security test ID; omit for all.' }],
|
|
22
|
+
flags: [
|
|
23
|
+
policyFlag,
|
|
24
|
+
{ name: 'json', type: 'boolean', description: 'Print the full report as JSON instead.' },
|
|
25
|
+
{
|
|
26
|
+
name: 'allow-production',
|
|
27
|
+
type: 'boolean',
|
|
28
|
+
description: 'Authorize a production target that the policy also allows.'
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
handler: async ({ params, flags }) => {
|
|
32
|
+
try {
|
|
33
|
+
const policy = loadSecurityPolicy(toPath(flags.policy), {
|
|
34
|
+
allowProduction: flags['allow-production'] === true
|
|
35
|
+
});
|
|
36
|
+
const asJson = flags.json === true;
|
|
37
|
+
if (!asJson)
|
|
38
|
+
console.log(`Security evaluation: ${policy.name} → ${policy.target.url} (${policy.target.environment})`);
|
|
39
|
+
const report = await new ProbeOrchestrator(policy, process.env, {
|
|
40
|
+
onResult: (result) => {
|
|
41
|
+
if (!asJson)
|
|
42
|
+
for (const line of toLines(result))
|
|
43
|
+
console.log(line);
|
|
44
|
+
}
|
|
45
|
+
}).run(params.id);
|
|
46
|
+
if (asJson)
|
|
47
|
+
console.log(JSON.stringify(report, null, 2));
|
|
48
|
+
else
|
|
49
|
+
console.log(`Result: ${report.passed} of ${report.total} security test(s) passed.`);
|
|
50
|
+
process.exitCode = report.passed === report.total ? 0 : 1;
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
console.error(`Error: ${message(error)}`);
|
|
54
|
+
process.exitCode = 2;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
const validateSubcommand = command({
|
|
59
|
+
name: 'validate',
|
|
60
|
+
description: 'Validate the security policy without sending a request.',
|
|
61
|
+
flags: [
|
|
62
|
+
policyFlag,
|
|
63
|
+
{
|
|
64
|
+
name: 'allow-production',
|
|
65
|
+
type: 'boolean',
|
|
66
|
+
description: 'Authorize a production target that the policy also allows.'
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
handler: ({ flags }) => {
|
|
70
|
+
try {
|
|
71
|
+
const policy = loadSecurityPolicy(toPath(flags.policy), {
|
|
72
|
+
allowProduction: flags['allow-production'] === true
|
|
73
|
+
});
|
|
74
|
+
const total = policy.tests.reduce((sum, test) => sum + test.requests, 0);
|
|
75
|
+
console.log(`Policy ${policy.name} valid: ${policy.tests.length} security test(s), ${total} of ${policy.safety.max_requests} request(s).`);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
console.error(`Error: ${message(error)}`);
|
|
79
|
+
process.exitCode = 2;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
/** The defense names `--disable` accepts, mapped to the mock's keys. */
|
|
84
|
+
const DEFENSES = { 'rate-limit': 'rateLimit', quota: 'quota', detection: 'detection' };
|
|
85
|
+
const mockSubcommand = command({
|
|
86
|
+
name: 'mock',
|
|
87
|
+
description: 'Serve a local mock AI API with extraction defenses, until stopped.',
|
|
88
|
+
flags: [
|
|
89
|
+
{ name: 'port', type: 'number', default: 4010, description: 'Port on 127.0.0.1; 0 picks one.' },
|
|
90
|
+
{
|
|
91
|
+
name: 'disable',
|
|
92
|
+
type: 'string',
|
|
93
|
+
description: 'Comma-separated defenses to remove: rate-limit, quota, detection.'
|
|
94
|
+
}
|
|
95
|
+
],
|
|
96
|
+
handler: async ({ flags }) => {
|
|
97
|
+
try {
|
|
98
|
+
const names = typeof flags.disable === 'string'
|
|
99
|
+
? flags.disable
|
|
100
|
+
.split(',')
|
|
101
|
+
.map((name) => name.trim())
|
|
102
|
+
.filter(Boolean)
|
|
103
|
+
: [];
|
|
104
|
+
const defenses = { ...PROTECTED_DEFENSES };
|
|
105
|
+
for (const name of names) {
|
|
106
|
+
if (!(name in DEFENSES))
|
|
107
|
+
throw new Error(`Unknown defense: ${name}; use rate-limit, quota, or detection`);
|
|
108
|
+
delete defenses[DEFENSES[name]];
|
|
109
|
+
}
|
|
110
|
+
const port = typeof flags.port === 'number' ? flags.port : 4010;
|
|
111
|
+
const api = await startMockApi({ port, defenses });
|
|
112
|
+
const active = Object.keys(DEFENSES).filter((name) => !names.includes(name));
|
|
113
|
+
console.log(`Mock API listening on ${api.url} (defenses: ${active.join(', ') || 'none'})`);
|
|
114
|
+
const stop = () => {
|
|
115
|
+
void api.close().then(() => process.exit(0));
|
|
116
|
+
};
|
|
117
|
+
process.once('SIGINT', stop);
|
|
118
|
+
process.once('SIGTERM', stop);
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
console.error(`Error: ${message(error)}`);
|
|
122
|
+
process.exitCode = 2;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
/** Load the working directory's `.env`, if any, for `${VAR}` references; variables already set win. */
|
|
127
|
+
if (existsSync('.env'))
|
|
128
|
+
process.loadEnvFile('.env');
|
|
129
|
+
await clinguini({
|
|
130
|
+
name: 'agentests-security',
|
|
131
|
+
description: 'Test an AI API against model extraction: rate limits, quotas, and detection.',
|
|
132
|
+
commands: [runSubcommand, validateSubcommand, mockSubcommand],
|
|
133
|
+
utils: { help: true }
|
|
134
|
+
});
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { LOCAL_HOSTS, validateSecurityPolicy } from './policy/validate.ts';
|
|
2
|
+
export type { AgentTestSecurityOptions } from './policy/validate.ts';
|
|
3
|
+
export { parseSecurityPolicy } from './policy/parse.ts';
|
|
4
|
+
export { loadSecurityPolicy, SECURITY_PATH } from './policy/load.ts';
|
|
5
|
+
export { AgentTestSecurityPolicySchema } from './policy/types.ts';
|
|
6
|
+
export type { AgentTestSecurityPolicy } from './policy/types.ts';
|
|
7
|
+
export { ProbeOrchestrator, RUN_HEADER, TEST_HEADER } from './probe/index.ts';
|
|
8
|
+
export type { ProbeOptions } from './probe/index.ts';
|
|
9
|
+
export type { AgentTestEnvironment, AgentTestExpect, AgentTestExpectation, AgentTestScenario, AgentTestSecurityCheck, AgentTestSecurityEvidence, AgentTestSecurityReport, AgentTestSecurityResult, AgentTestSecurityTest, AgentTestSignal, AgentTestTarget } from './probe/types.ts';
|
|
10
|
+
export { PROTECTED_DEFENSES, startMockApi } from './mock.ts';
|
|
11
|
+
export type { MockApi, MockDefenses, MockEvent } from './mock.ts';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { LOCAL_HOSTS, validateSecurityPolicy } from "./policy/validate.js";
|
|
2
|
+
export { parseSecurityPolicy } from "./policy/parse.js";
|
|
3
|
+
export { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
|
|
4
|
+
export { AgentTestSecurityPolicySchema } from "./policy/types.js";
|
|
5
|
+
export { ProbeOrchestrator, RUN_HEADER, TEST_HEADER } from "./probe/index.js";
|
|
6
|
+
export { PROTECTED_DEFENSES, startMockApi } from "./mock.js";
|
package/dist/mock.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/** The defenses the mock can run; leave one out to simulate the regression where it was removed. */
|
|
2
|
+
export interface MockDefenses {
|
|
3
|
+
/** At most `requests` per API key in each fixed `windowMs` window, counted from the key's first request. */
|
|
4
|
+
rateLimit?: {
|
|
5
|
+
requests: number;
|
|
6
|
+
windowMs: number;
|
|
7
|
+
};
|
|
8
|
+
/** At most `requests` successful requests per API key, ever; later ones get `429 insufficient_quota`. */
|
|
9
|
+
quota?: {
|
|
10
|
+
requests: number;
|
|
11
|
+
};
|
|
12
|
+
/** One event when a key sends `threshold` prompts that are the same once digits are masked. */
|
|
13
|
+
detection?: {
|
|
14
|
+
threshold: number;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
/** The defenses of a correctly protected API: 20 requests a second, 60 in total, and a flag after 25 templated prompts. */
|
|
18
|
+
export declare const PROTECTED_DEFENSES: Required<MockDefenses>;
|
|
19
|
+
/** One detection event, as `GET /v1/security/events` lists it. */
|
|
20
|
+
export interface MockEvent {
|
|
21
|
+
type: 'systematic-query-pattern';
|
|
22
|
+
key: string;
|
|
23
|
+
template: string;
|
|
24
|
+
count: number;
|
|
25
|
+
run?: string;
|
|
26
|
+
}
|
|
27
|
+
/** A running mock: its base URL, the events it raised, and a way to stop it. */
|
|
28
|
+
export interface MockApi {
|
|
29
|
+
url: string;
|
|
30
|
+
events: MockEvent[];
|
|
31
|
+
close: () => Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Start a local stand-in for an AI completion API, for demos and tests; it binds to 127.0.0.1 only.
|
|
35
|
+
* - `POST /v1/complete`: `{ prompt }` or `{ messages }`, keyed by the `authorization` header; answers `{ completion }`.
|
|
36
|
+
* - `GET /v1/security/events`: `{ events }`, every detection event so far.
|
|
37
|
+
* Pass `port: 0` for a free port.
|
|
38
|
+
*/
|
|
39
|
+
export declare const startMockApi: ({ port, defenses, now }?: {
|
|
40
|
+
port?: number;
|
|
41
|
+
defenses?: MockDefenses;
|
|
42
|
+
now?: () => number;
|
|
43
|
+
}) => Promise<MockApi>;
|
package/dist/mock.js
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { createServer } from 'node:http';
|
|
2
|
+
/** The defenses of a correctly protected API: 20 requests a second, 60 in total, and a flag after 25 templated prompts. */
|
|
3
|
+
export const PROTECTED_DEFENSES = {
|
|
4
|
+
rateLimit: { requests: 20, windowMs: 1_000 },
|
|
5
|
+
quota: { requests: 60 },
|
|
6
|
+
detection: { threshold: 25 }
|
|
7
|
+
};
|
|
8
|
+
/** Digits masked, so a grid of inputs collapses to its template. */
|
|
9
|
+
const toTemplate = (prompt) => prompt.replace(/\d+/g, '#');
|
|
10
|
+
/** The prompt of a request body: its `prompt` string, or the last `messages` entry's `content`. */
|
|
11
|
+
const toPrompt = (body) => {
|
|
12
|
+
if (!body || typeof body !== 'object')
|
|
13
|
+
return '';
|
|
14
|
+
if ('prompt' in body && typeof body.prompt === 'string')
|
|
15
|
+
return body.prompt;
|
|
16
|
+
if ('messages' in body && Array.isArray(body.messages)) {
|
|
17
|
+
const last = body.messages.at(-1);
|
|
18
|
+
if (last && typeof last === 'object' && 'content' in last && typeof last.content === 'string')
|
|
19
|
+
return last.content;
|
|
20
|
+
}
|
|
21
|
+
return '';
|
|
22
|
+
};
|
|
23
|
+
const json = (response, status, value, headers = {}) => {
|
|
24
|
+
response.writeHead(status, { 'content-type': 'application/json', ...headers });
|
|
25
|
+
response.end(JSON.stringify(value));
|
|
26
|
+
};
|
|
27
|
+
const readBody = async (request) => {
|
|
28
|
+
const chunks = [];
|
|
29
|
+
for await (const chunk of request)
|
|
30
|
+
chunks.push(chunk);
|
|
31
|
+
try {
|
|
32
|
+
return JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Start a local stand-in for an AI completion API, for demos and tests; it binds to 127.0.0.1 only.
|
|
40
|
+
* - `POST /v1/complete`: `{ prompt }` or `{ messages }`, keyed by the `authorization` header; answers `{ completion }`.
|
|
41
|
+
* - `GET /v1/security/events`: `{ events }`, every detection event so far.
|
|
42
|
+
* Pass `port: 0` for a free port.
|
|
43
|
+
*/
|
|
44
|
+
export const startMockApi = async ({ port = 4010, defenses = PROTECTED_DEFENSES, now = Date.now } = {}) => {
|
|
45
|
+
const keys = new Map();
|
|
46
|
+
const events = [];
|
|
47
|
+
const complete = async (request, response) => {
|
|
48
|
+
const body = await readBody(request);
|
|
49
|
+
const key = request.headers.authorization ?? 'anonymous';
|
|
50
|
+
const state = keys.get(key) ??
|
|
51
|
+
{
|
|
52
|
+
windowStart: now(),
|
|
53
|
+
windowCount: 0,
|
|
54
|
+
used: 0,
|
|
55
|
+
templates: new Map(),
|
|
56
|
+
flagged: new Set()
|
|
57
|
+
};
|
|
58
|
+
keys.set(key, state);
|
|
59
|
+
const { quota, rateLimit, detection } = defenses;
|
|
60
|
+
if (quota && state.used >= quota.requests)
|
|
61
|
+
return json(response, 429, {
|
|
62
|
+
error: { code: 'insufficient_quota', message: 'You exceeded your current quota.' }
|
|
63
|
+
});
|
|
64
|
+
if (rateLimit) {
|
|
65
|
+
const time = now();
|
|
66
|
+
if (time - state.windowStart >= rateLimit.windowMs) {
|
|
67
|
+
state.windowStart = time;
|
|
68
|
+
state.windowCount = 0;
|
|
69
|
+
}
|
|
70
|
+
if (state.windowCount >= rateLimit.requests) {
|
|
71
|
+
const retry = Math.ceil((state.windowStart + rateLimit.windowMs - time) / 1000);
|
|
72
|
+
return json(response, 429, { error: { code: 'rate_limit_exceeded', message: 'Too many requests.' } }, { 'retry-after': String(Math.max(1, retry)) });
|
|
73
|
+
}
|
|
74
|
+
state.windowCount++;
|
|
75
|
+
}
|
|
76
|
+
state.used++;
|
|
77
|
+
const prompt = toPrompt(body);
|
|
78
|
+
if (detection && prompt) {
|
|
79
|
+
const template = toTemplate(prompt);
|
|
80
|
+
const count = (state.templates.get(template) ?? 0) + 1;
|
|
81
|
+
state.templates.set(template, count);
|
|
82
|
+
if (count >= detection.threshold && !state.flagged.has(template)) {
|
|
83
|
+
state.flagged.add(template);
|
|
84
|
+
const run = request.headers['x-agentests-run'];
|
|
85
|
+
events.push({
|
|
86
|
+
type: 'systematic-query-pattern',
|
|
87
|
+
key: key.slice(0, 12),
|
|
88
|
+
template,
|
|
89
|
+
count,
|
|
90
|
+
...(typeof run === 'string' ? { run } : {})
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
json(response, 200, { completion: `Mock answer to: ${prompt.slice(0, 80)}` });
|
|
95
|
+
};
|
|
96
|
+
const server = createServer((request, response) => {
|
|
97
|
+
const path = new URL(request.url ?? '/', 'http://localhost').pathname;
|
|
98
|
+
if (request.method === 'POST' && path === '/v1/complete') {
|
|
99
|
+
complete(request, response).catch(() => json(response, 500, { error: 'internal' }));
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (request.method === 'GET' && path === '/v1/security/events')
|
|
103
|
+
return json(response, 200, { events });
|
|
104
|
+
json(response, 404, { error: 'not found' });
|
|
105
|
+
});
|
|
106
|
+
await new Promise((resolve, reject) => {
|
|
107
|
+
server.once('error', reject);
|
|
108
|
+
server.listen(port, '127.0.0.1', () => resolve());
|
|
109
|
+
});
|
|
110
|
+
const { port: bound } = server.address();
|
|
111
|
+
return {
|
|
112
|
+
url: `http://127.0.0.1:${bound}`,
|
|
113
|
+
events,
|
|
114
|
+
close: () => new Promise((resolve) => {
|
|
115
|
+
server.closeAllConnections();
|
|
116
|
+
server.close(() => resolve());
|
|
117
|
+
})
|
|
118
|
+
};
|
|
119
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Format zod issues as `path: message` lines, naming each unknown field. */
|
|
2
|
+
export const describeIssues = (issues) => issues.flatMap((issue) => {
|
|
3
|
+
const path = issue.path.map(String);
|
|
4
|
+
if (issue.code === 'unrecognized_keys')
|
|
5
|
+
return issue.keys.map((key) => `${[...path, key].join('.')}: Unknown field`);
|
|
6
|
+
return [`${path.join('.') || 'value'}: ${issue.message}`];
|
|
7
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { AgentTestSecurityPolicy } from './types.ts';
|
|
2
|
+
import type { AgentTestSecurityOptions } from './validate.ts';
|
|
3
|
+
/** The default security policy file, relative to the current working directory. */
|
|
4
|
+
export declare const SECURITY_PATH = "agentests.security.yaml";
|
|
5
|
+
/** Read a security policy file and parse it. */
|
|
6
|
+
export declare const loadSecurityPolicy: (path?: string, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { readFile } from "./file.js";
|
|
2
|
+
import { parseSecurityPolicy } from "./parse.js";
|
|
3
|
+
/** The default security policy file, relative to the current working directory. */
|
|
4
|
+
export const SECURITY_PATH = 'agentests.security.yaml';
|
|
5
|
+
/** Read a security policy file and parse it. */
|
|
6
|
+
export const loadSecurityPolicy = (path = SECURITY_PATH, options) => parseSecurityPolicy(readFile(path, `Missing ${path}`), options);
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { AgentTestSecurityPolicy } from './types.ts';
|
|
2
|
+
import type { AgentTestSecurityOptions } from './validate.ts';
|
|
3
|
+
/** Parse policy text into a validated security policy, or throw its YAML problem alone, else what `validateSecurityPolicy` throws. */
|
|
4
|
+
export declare const parseSecurityPolicy: (text: string, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { parseYaml } from "./yaml.js";
|
|
2
|
+
import { validateSecurityPolicy } from "./validate.js";
|
|
3
|
+
/** Parse policy text into a validated security policy, or throw its YAML problem alone, else what `validateSecurityPolicy` throws. */
|
|
4
|
+
export const parseSecurityPolicy = (text, options) => validateSecurityPolicy(parseYaml(text), options);
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/** agentests.security.yaml at `version: 1`: one target, its security Tests, and the safety limits that bound them. */
|
|
3
|
+
export declare const AgentTestSecurityPolicySchema: z.ZodObject<{
|
|
4
|
+
version: z.ZodLiteral<1>;
|
|
5
|
+
name: z.ZodString;
|
|
6
|
+
target: z.ZodObject<{
|
|
7
|
+
url: z.ZodURL;
|
|
8
|
+
environment: z.ZodEnum<{
|
|
9
|
+
local: "local";
|
|
10
|
+
test: "test";
|
|
11
|
+
staging: "staging";
|
|
12
|
+
production: "production";
|
|
13
|
+
}>;
|
|
14
|
+
headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
15
|
+
body: z.ZodDefault<z.ZodUnknown>;
|
|
16
|
+
timeout_ms: z.ZodDefault<z.ZodNumber>;
|
|
17
|
+
signals: z.ZodDefault<z.ZodObject<{
|
|
18
|
+
rate_limit: z.ZodDefault<z.ZodObject<{
|
|
19
|
+
status: z.ZodArray<z.ZodNumber>;
|
|
20
|
+
body: z.ZodOptional<z.ZodString>;
|
|
21
|
+
}, z.core.$strict>>;
|
|
22
|
+
quota: z.ZodDefault<z.ZodObject<{
|
|
23
|
+
status: z.ZodArray<z.ZodNumber>;
|
|
24
|
+
body: z.ZodOptional<z.ZodString>;
|
|
25
|
+
}, z.core.$strict>>;
|
|
26
|
+
}, z.core.$strict>>;
|
|
27
|
+
detection: z.ZodOptional<z.ZodObject<{
|
|
28
|
+
url: z.ZodURL;
|
|
29
|
+
headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
30
|
+
wait_ms: z.ZodDefault<z.ZodNumber>;
|
|
31
|
+
}, z.core.$strict>>;
|
|
32
|
+
}, z.core.$strict>;
|
|
33
|
+
tests: z.ZodArray<z.ZodObject<{
|
|
34
|
+
id: z.ZodString;
|
|
35
|
+
description: z.ZodOptional<z.ZodString>;
|
|
36
|
+
scenario: z.ZodEnum<{
|
|
37
|
+
"sustained-query-volume": "sustained-query-volume";
|
|
38
|
+
"systematic-query-pattern": "systematic-query-pattern";
|
|
39
|
+
"quota-exhaustion": "quota-exhaustion";
|
|
40
|
+
}>;
|
|
41
|
+
requests: z.ZodNumber;
|
|
42
|
+
duration_seconds: z.ZodDefault<z.ZodNumber>;
|
|
43
|
+
headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
44
|
+
expect: z.ZodObject<{
|
|
45
|
+
rate_limit_enforced: z.ZodOptional<z.ZodBoolean>;
|
|
46
|
+
quota_enforced: z.ZodOptional<z.ZodBoolean>;
|
|
47
|
+
detection_event: z.ZodOptional<z.ZodBoolean>;
|
|
48
|
+
}, z.core.$strict>;
|
|
49
|
+
}, z.core.$strict>>;
|
|
50
|
+
safety: z.ZodObject<{
|
|
51
|
+
max_requests: z.ZodNumber;
|
|
52
|
+
allow_production: z.ZodDefault<z.ZodBoolean>;
|
|
53
|
+
allowed_hosts: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
54
|
+
}, z.core.$strict>;
|
|
55
|
+
}, z.core.$strict>;
|
|
56
|
+
export type AgentTestSecurityPolicy = z.infer<typeof AgentTestSecurityPolicySchema>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { IdSchema } from "../probe/shared.js";
|
|
3
|
+
import { SafetySchema, SecurityTestSchema, TargetSchema } from "../probe/types.js";
|
|
4
|
+
/** agentests.security.yaml at `version: 1`: one target, its security Tests, and the safety limits that bound them. */
|
|
5
|
+
export const AgentTestSecurityPolicySchema = z.strictObject({
|
|
6
|
+
version: z.literal(1),
|
|
7
|
+
name: IdSchema,
|
|
8
|
+
target: TargetSchema,
|
|
9
|
+
tests: z.array(SecurityTestSchema).min(1),
|
|
10
|
+
safety: SafetySchema
|
|
11
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AgentTestSecurityPolicy } from './types.ts';
|
|
2
|
+
/** How the caller authorizes the run beyond the policy file itself. */
|
|
3
|
+
export interface AgentTestSecurityOptions {
|
|
4
|
+
/** The `--allow-production` flag: a production target needs it and `safety.allow_production`. */
|
|
5
|
+
allowProduction?: boolean;
|
|
6
|
+
}
|
|
7
|
+
/** Hosts every policy may target without listing them. */
|
|
8
|
+
export declare const LOCAL_HOSTS: string[];
|
|
9
|
+
/** Validate a parsed value as a security policy, or throw `path: message` lines: every schema problem; else, once the schema passes, every rule and safety problem. */
|
|
10
|
+
export declare const validateSecurityPolicy: (value: unknown, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { AgentTestSecurityPolicySchema } from "./types.js";
|
|
2
|
+
import { describeIssues } from "./issues.js";
|
|
3
|
+
/** Hosts every policy may target without listing them. */
|
|
4
|
+
export const LOCAL_HOSTS = ['localhost', '127.0.0.1', '[::1]'];
|
|
5
|
+
/** Validate a parsed value as a security policy, or throw `path: message` lines: every schema problem; else, once the schema passes, every rule and safety problem. */
|
|
6
|
+
export const validateSecurityPolicy = (value, options = {}) => {
|
|
7
|
+
const result = AgentTestSecurityPolicySchema.safeParse(value);
|
|
8
|
+
if (!result.success)
|
|
9
|
+
throw new Error(describeIssues(result.error.issues).join('\n'));
|
|
10
|
+
const problems = policyProblems(result.data, options);
|
|
11
|
+
if (problems.length)
|
|
12
|
+
throw new Error(problems.join('\n'));
|
|
13
|
+
return result.data;
|
|
14
|
+
};
|
|
15
|
+
/** Problems the schema cannot express: duplicate ids, budgets, regexes, and every host and environment authorization. */
|
|
16
|
+
const policyProblems = (policy, { allowProduction = false }) => {
|
|
17
|
+
const { target, tests, safety } = policy;
|
|
18
|
+
const ids = new Set();
|
|
19
|
+
const total = tests.reduce((sum, test) => sum + test.requests, 0);
|
|
20
|
+
const urls = [
|
|
21
|
+
['target.url', target.url],
|
|
22
|
+
...(target.detection
|
|
23
|
+
? [['target.detection.url', target.detection.url]]
|
|
24
|
+
: [])
|
|
25
|
+
];
|
|
26
|
+
const allowed = new Set([
|
|
27
|
+
...LOCAL_HOSTS,
|
|
28
|
+
...safety.allowed_hosts.map((host) => host.toLowerCase())
|
|
29
|
+
]);
|
|
30
|
+
return [
|
|
31
|
+
...tests.flatMap(({ id }, index) => {
|
|
32
|
+
const duplicate = ids.has(id);
|
|
33
|
+
ids.add(id);
|
|
34
|
+
return duplicate ? [`tests.${index}.id: Duplicate test id`] : [];
|
|
35
|
+
}),
|
|
36
|
+
...(total > safety.max_requests
|
|
37
|
+
? [
|
|
38
|
+
`safety.max_requests: The tests send ${total} requests, over the limit of ${safety.max_requests}`
|
|
39
|
+
]
|
|
40
|
+
: []),
|
|
41
|
+
...['rate_limit', 'quota'].flatMap((signal) => {
|
|
42
|
+
const body = target.signals[signal].body;
|
|
43
|
+
return body === undefined || compiles(body)
|
|
44
|
+
? []
|
|
45
|
+
: [`target.signals.${signal}.body: Invalid regular expression`];
|
|
46
|
+
}),
|
|
47
|
+
...tests.flatMap(({ expect }, index) => expect.detection_event !== undefined && !target.detection
|
|
48
|
+
? [`tests.${index}.expect.detection_event: Requires target.detection`]
|
|
49
|
+
: []),
|
|
50
|
+
...urls.flatMap(([path, value]) => {
|
|
51
|
+
const url = new URL(value);
|
|
52
|
+
if (url.username || url.password)
|
|
53
|
+
return [`${path}: Credentials in the URL are not allowed; use headers with \${VAR}`];
|
|
54
|
+
return allowed.has(url.hostname.toLowerCase())
|
|
55
|
+
? []
|
|
56
|
+
: [`${path}: Host ${url.hostname} is not authorized; list it in safety.allowed_hosts`];
|
|
57
|
+
}),
|
|
58
|
+
...(target.environment === 'production' && !safety.allow_production
|
|
59
|
+
? ['target.environment: Production targets need safety.allow_production: true']
|
|
60
|
+
: []),
|
|
61
|
+
...(target.environment === 'production' && !allowProduction
|
|
62
|
+
? ['target.environment: Production targets need the --allow-production flag']
|
|
63
|
+
: [])
|
|
64
|
+
];
|
|
65
|
+
};
|
|
66
|
+
const compiles = (pattern) => {
|
|
67
|
+
try {
|
|
68
|
+
new RegExp(pattern, 'i');
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
};
|