@nyancorn/agentests-security 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,40 @@ All notable changes are listed here. The format follows
4
4
  [Keep a Changelog](https://keepachangelog.com), and the project follows
5
5
  [semantic versioning](https://semver.org) as described in [STABILITY.md](STABILITY.md).
6
6
 
7
+ ## [1.1.0] - 2026-11-18
8
+
9
+ Hardening of the runner. Every 1.0 policy is still valid.
10
+
11
+ ### Added
12
+
13
+ - **`expect.rate_limit_within` and `expect.quota_within`:** require a defense to fire by a
14
+ given request, not just at some point in the test.
15
+ - **`target.detection.run_field`:** count only the detection events tagged with this run's
16
+ `x-agentests-run` id, so other traffic can't make a test pass. Detection headers accept
17
+ `{{run_id}}`. The report's evidence says whether detection was correlated
18
+ (`detectionCorrelated`). The mock API tags its events with `run`.
19
+ - **`max_error_rate` per test (default 0.05):** a test whose responses are mostly errors
20
+ matching no signal fails as inconclusive, with the reason in `result.inconclusive`. The
21
+ evidence counts these responses in `errors`.
22
+ - **`safety.allow_insecure_http`:** accept plain HTTP to a remote host without a warning.
23
+ - **`warnings` in the report**, and `Warning:` lines on stderr from `run` and `validate`.
24
+ - **`securityPolicyWarnings(policy)`** in the library.
25
+
26
+ ### Changed
27
+
28
+ - **Errors no longer pass:** a test that expects a defense to be absent
29
+ (`…: false`) used to pass when the API answered with errors. It now fails as inconclusive
30
+ when more than 5% of responses are errors matching no signal. Raise `max_error_rate` to
31
+ keep the old behaviour.
32
+ - **Responses are read with a cap:** 64 KiB per completion response, with the rest of a
33
+ stream cancelled, and 4 MiB per detection response, beyond which the run stops with an error.
34
+ - **Secrets are redacted** from the report and from error messages.
35
+
36
+ ### Deprecated
37
+
38
+ - **Plain HTTP to a remote host:** it is a warning in 1.x and will be an error in 2.0
39
+ unless `safety.allow_insecure_http: true` is set.
40
+
7
41
  ## [1.0.0] - 2026-11-18
8
42
 
9
43
  First stable release.
package/README.md CHANGED
@@ -68,6 +68,15 @@ npx @nyancorn/agentests-security run [id] # exit 0 pass, 1 regression, 2 e
68
68
  npx @nyancorn/agentests-security run --json # full report with evidence, for CI artifacts
69
69
  ```
70
70
 
71
+ ### Stricter checks (1.1)
72
+
73
+ | Field | What it does |
74
+ | --- | --- |
75
+ | `expect.rate_limit_within: 25` | The rate limit must fire by request 25, not just eventually. Needs `rate_limit_enforced: true`. |
76
+ | `expect.quota_within: 60` | The same for the quota. Needs `quota_enforced: true`. |
77
+ | `max_error_rate: 0.05` (per test, default) | If more than 5% of responses are errors that match no signal, such as 500s, the test fails as **inconclusive**. A broken API no longer passes `rate_limit_enforced: false`. |
78
+ | `target.detection.run_field: run` | Count only detection events whose `run` field equals this run's `x-agentests-run` id. Without it, any new event counts, even one caused by other traffic. Detection headers can also use `{{run_id}}`, for APIs that filter on the server side. |
79
+
71
80
  The [policy reference](https://github.com/mlgxiii/agentests/tree/main/examples/security#the-policy)
72
81
  covers every field: the three scenarios, the response signals, the detection
73
82
  endpoint, and the safety limits.
@@ -82,6 +91,14 @@ endpoint, and the safety limits.
82
91
  as a hard stop during the run.
83
92
  - **Credentials:** URLs that contain credentials are rejected. Put secrets in headers as
84
93
  `${VAR}` references instead.
94
+ - **TLS:** plain `http://` to a host other than localhost prints a warning, because it sends
95
+ your API key unencrypted. From 2.0 it will be an error. Set `safety.allow_insecure_http: true`
96
+ to accept it, for example on a private network.
97
+ - **Redaction:** the values of `${VAR}` secrets, literal values of headers such as
98
+ `authorization` or `x-api-key`, and URL parameters such as `api_key` or `token` are replaced
99
+ with `[REDACTED]` in the report and in error messages.
100
+ - **Bounded responses:** at most 64 KiB of each response is read, and the rest of a streaming
101
+ answer is cancelled. A detection endpoint returning more than 4 MiB is an error.
85
102
  - **Traceability:** every request carries `x-agentests-run` and `x-agentests-test` headers,
86
103
  so your logs can tell test traffic apart.
87
104
 
package/dist/cli.js CHANGED
@@ -4,8 +4,10 @@ import { clinguini, command } from 'clinguini';
4
4
  import { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
5
5
  import { PROTECTED_DEFENSES, startMockApi } from "./mock.js";
6
6
  import { ProbeOrchestrator } from "./probe/index.js";
7
+ import { redactUrl } from "./probe/redact.js";
8
+ import { securityPolicyWarnings } from "./policy/validate.js";
7
9
  /** The text of a thrown value. */
8
- const message = (error) => (error instanceof Error ? error.message : String(error));
10
+ const message = (error) => error instanceof Error ? error.message : String(error);
9
11
  const policyFlag = {
10
12
  name: 'policy',
11
13
  type: 'string',
@@ -13,8 +15,16 @@ const policyFlag = {
13
15
  };
14
16
  /** The policy path a flag value names, else the default. */
15
17
  const toPath = (value) => (typeof value === 'string' ? value : SECURITY_PATH);
16
- /** `PASS|FAIL <test> <expectation>: expected <x>, got <y> (<detail>)`, one line per check. */
17
- const toLines = ({ test, checks }) => checks.map((check) => `${check.passed ? 'PASS' : 'FAIL'} ${test.id} ${check.expectation}: expected ${check.expected}, got ${check.actual} (${check.detail})`);
18
+ /** `PASS|FAIL <test> <expectation>: expected <x>, got <y> (<detail>)`, one line per check, then a FAIL line if inconclusive. */
19
+ const toLines = ({ test, checks, inconclusive }) => [
20
+ ...checks.map((check) => `${check.passed ? 'PASS' : 'FAIL'} ${test.id} ${check.expectation}: expected ${check.expected}, got ${check.actual} (${check.detail})`),
21
+ ...(inconclusive === undefined ? [] : [`FAIL ${test.id} inconclusive: ${inconclusive}`])
22
+ ];
23
+ /** Print each policy warning to stderr, so `--json` output stays parseable. */
24
+ const warn = (policy) => {
25
+ for (const warning of securityPolicyWarnings(policy))
26
+ console.error(`Warning: ${warning}`);
27
+ };
18
28
  const runSubcommand = command({
19
29
  name: 'run',
20
30
  description: 'Run the security policy against its target.',
@@ -34,8 +44,9 @@ const runSubcommand = command({
34
44
  allowProduction: flags['allow-production'] === true
35
45
  });
36
46
  const asJson = flags.json === true;
47
+ warn(policy);
37
48
  if (!asJson)
38
- console.log(`Security evaluation: ${policy.name} → ${policy.target.url} (${policy.target.environment})`);
49
+ console.log(`Security evaluation: ${policy.name} → ${redactUrl(policy.target.url)} (${policy.target.environment})`);
39
50
  const report = await new ProbeOrchestrator(policy, process.env, {
40
51
  onResult: (result) => {
41
52
  if (!asJson)
@@ -72,6 +83,7 @@ const validateSubcommand = command({
72
83
  allowProduction: flags['allow-production'] === true
73
84
  });
74
85
  const total = policy.tests.reduce((sum, test) => sum + test.requests, 0);
86
+ warn(policy);
75
87
  console.log(`Policy ${policy.name} valid: ${policy.tests.length} security test(s), ${total} of ${policy.safety.max_requests} request(s).`);
76
88
  }
77
89
  catch (error) {
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export { LOCAL_HOSTS, validateSecurityPolicy } from './policy/validate.ts';
1
+ export { LOCAL_HOSTS, securityPolicyWarnings, validateSecurityPolicy } from './policy/validate.ts';
2
2
  export type { AgentTestSecurityOptions } from './policy/validate.ts';
3
3
  export { parseSecurityPolicy } from './policy/parse.ts';
4
4
  export { loadSecurityPolicy, SECURITY_PATH } from './policy/load.ts';
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- export { LOCAL_HOSTS, validateSecurityPolicy } from "./policy/validate.js";
1
+ export { LOCAL_HOSTS, securityPolicyWarnings, validateSecurityPolicy } from "./policy/validate.js";
2
2
  export { parseSecurityPolicy } from "./policy/parse.js";
3
3
  export { loadSecurityPolicy, SECURITY_PATH } from "./policy/load.js";
4
4
  export { AgentTestSecurityPolicySchema } from "./policy/types.js";
@@ -28,6 +28,7 @@ export declare const AgentTestSecurityPolicySchema: z.ZodObject<{
28
28
  url: z.ZodURL;
29
29
  headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
30
30
  wait_ms: z.ZodDefault<z.ZodNumber>;
31
+ run_field: z.ZodOptional<z.ZodString>;
31
32
  }, z.core.$strict>>;
32
33
  }, z.core.$strict>;
33
34
  tests: z.ZodArray<z.ZodObject<{
@@ -41,16 +42,20 @@ export declare const AgentTestSecurityPolicySchema: z.ZodObject<{
41
42
  requests: z.ZodNumber;
42
43
  duration_seconds: z.ZodDefault<z.ZodNumber>;
43
44
  headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
45
+ max_error_rate: z.ZodDefault<z.ZodNumber>;
44
46
  expect: z.ZodObject<{
45
47
  rate_limit_enforced: z.ZodOptional<z.ZodBoolean>;
46
48
  quota_enforced: z.ZodOptional<z.ZodBoolean>;
47
49
  detection_event: z.ZodOptional<z.ZodBoolean>;
50
+ rate_limit_within: z.ZodOptional<z.ZodNumber>;
51
+ quota_within: z.ZodOptional<z.ZodNumber>;
48
52
  }, z.core.$strict>;
49
53
  }, z.core.$strict>>;
50
54
  safety: z.ZodObject<{
51
55
  max_requests: z.ZodNumber;
52
56
  allow_production: z.ZodDefault<z.ZodBoolean>;
53
57
  allowed_hosts: z.ZodDefault<z.ZodArray<z.ZodString>>;
58
+ allow_insecure_http: z.ZodDefault<z.ZodBoolean>;
54
59
  }, z.core.$strict>;
55
60
  }, z.core.$strict>;
56
61
  export type AgentTestSecurityPolicy = z.infer<typeof AgentTestSecurityPolicySchema>;
@@ -8,3 +8,8 @@ export interface AgentTestSecurityOptions {
8
8
  export declare const LOCAL_HOSTS: string[];
9
9
  /** Validate a parsed value as a security policy, or throw `path: message` lines: every schema problem; else, once the schema passes, every rule and safety problem. */
10
10
  export declare const validateSecurityPolicy: (value: unknown, options?: AgentTestSecurityOptions) => AgentTestSecurityPolicy;
11
+ /**
12
+ * Problems that are not errors in 1.x but will be in 2.0: a plain `http://` URL to a host outside `LOCAL_HOSTS`,
13
+ * which sends the API key unencrypted, unless `safety.allow_insecure_http` accepts it.
14
+ */
15
+ export declare const securityPolicyWarnings: (policy: AgentTestSecurityPolicy) => string[];
@@ -47,6 +47,21 @@ const policyProblems = (policy, { allowProduction = false }) => {
47
47
  ...tests.flatMap(({ expect }, index) => expect.detection_event !== undefined && !target.detection
48
48
  ? [`tests.${index}.expect.detection_event: Requires target.detection`]
49
49
  : []),
50
+ ...tests.flatMap(({ expect, requests }, index) => [
51
+ ['rate_limit_within', 'rate_limit_enforced'],
52
+ ['quota_within', 'quota_enforced']
53
+ ].flatMap(([within, enforced]) => {
54
+ const limit = expect[within];
55
+ if (limit === undefined)
56
+ return [];
57
+ if (expect[enforced] !== true)
58
+ return [`tests.${index}.expect.${within}: Requires ${enforced}: true`];
59
+ return limit > requests
60
+ ? [
61
+ `tests.${index}.expect.${within}: ${limit} is more than the Test's ${requests} requests`
62
+ ]
63
+ : [];
64
+ })),
50
65
  ...urls.flatMap(([path, value]) => {
51
66
  const url = new URL(value);
52
67
  if (url.username || url.password)
@@ -63,6 +78,29 @@ const policyProblems = (policy, { allowProduction = false }) => {
63
78
  : [])
64
79
  ];
65
80
  };
81
+ /**
82
+ * Problems that are not errors in 1.x but will be in 2.0: a plain `http://` URL to a host outside `LOCAL_HOSTS`,
83
+ * which sends the API key unencrypted, unless `safety.allow_insecure_http` accepts it.
84
+ */
85
+ export const securityPolicyWarnings = (policy) => {
86
+ const { target, safety } = policy;
87
+ if (safety.allow_insecure_http)
88
+ return [];
89
+ const urls = [
90
+ ['target.url', target.url],
91
+ ...(target.detection
92
+ ? [['target.detection.url', target.detection.url]]
93
+ : [])
94
+ ];
95
+ return urls.flatMap(([path, value]) => {
96
+ const url = new URL(value);
97
+ return url.protocol === 'http:' && !LOCAL_HOSTS.includes(url.hostname.toLowerCase())
98
+ ? [
99
+ `${path}: Plain HTTP to ${url.hostname} sends credentials unencrypted; use https, or set safety.allow_insecure_http: true. This becomes an error in 2.0.`
100
+ ]
101
+ : [];
102
+ });
103
+ };
66
104
  const compiles = (pattern) => {
67
105
  try {
68
106
  new RegExp(pattern, 'i');
@@ -11,7 +11,19 @@ export interface ProbeResponse {
11
11
  status: number;
12
12
  body: string;
13
13
  }
14
+ /** The most of a completion response the runner reads; the rest of a long or endless stream is cancelled. */
15
+ export declare const MAX_BODY: number;
16
+ /** The most of a detection events response the runner reads; a larger list should be filtered by the API. */
17
+ export declare const MAX_EVENTS_BODY: number;
18
+ /**
19
+ * Read at most `limit` bytes of a body and cancel the rest, so a streaming or oversized response cannot hold
20
+ * the run or its memory. Returns the text and whether it was cut.
21
+ */
22
+ export declare const readCapped: (response: Response, limit: number) => Promise<{
23
+ text: string;
24
+ truncated: boolean;
25
+ }>;
14
26
  /** POST one JSON request; a network failure or timeout throws, so an unreachable target is an error, never a pass or a fail. */
15
27
  export declare const send: (fetcher: typeof fetch, url: string, headers: Record<string, string>, body: unknown, timeoutMs: number) => Promise<ProbeResponse>;
16
- /** The number of detection events the endpoint reports: a JSON array's length, or the length of an object's `events` array. */
17
- export declare const countEvents: (fetcher: typeof fetch, url: string, headers: Record<string, string>, timeoutMs: number) => Promise<number>;
28
+ /** The detection events the endpoint reports: a JSON array, or an object's `events` array. */
29
+ export declare const readEvents: (fetcher: typeof fetch, url: string, headers: Record<string, string>, timeoutMs: number) => Promise<unknown[]>;
@@ -20,7 +20,50 @@ export const expandValue = (value, env, placeholders) => {
20
20
  };
21
21
  /** `expand` applied to each header value. */
22
22
  export const expandHeaders = (headers, env, placeholders = {}) => Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, expand(value, env, placeholders)]));
23
- const MAX_BODY = 64 * 1024;
23
+ /** The most of a completion response the runner reads; the rest of a long or endless stream is cancelled. */
24
+ export const MAX_BODY = 64 * 1024;
25
+ /** The most of a detection events response the runner reads; a larger list should be filtered by the API. */
26
+ export const MAX_EVENTS_BODY = 4 * 1024 * 1024;
27
+ /**
28
+ * Read at most `limit` bytes of a body and cancel the rest, so a streaming or oversized response cannot hold
29
+ * the run or its memory. Returns the text and whether it was cut.
30
+ */
31
+ export const readCapped = async (response, limit) => {
32
+ if (!response.body)
33
+ return { text: '', truncated: false };
34
+ const reader = response.body.getReader();
35
+ const chunks = [];
36
+ let size = 0;
37
+ let truncated = false;
38
+ try {
39
+ while (true) {
40
+ const { done, value } = await reader.read();
41
+ if (done)
42
+ break;
43
+ const room = limit - size;
44
+ if (value.byteLength >= room) {
45
+ chunks.push(value.subarray(0, room));
46
+ size += room;
47
+ truncated = value.byteLength > room || !(await reader.read()).done;
48
+ break;
49
+ }
50
+ chunks.push(value);
51
+ size += value.byteLength;
52
+ }
53
+ }
54
+ finally {
55
+ if (truncated)
56
+ await reader.cancel().catch(() => undefined);
57
+ reader.releaseLock();
58
+ }
59
+ const bytes = new Uint8Array(size);
60
+ let offset = 0;
61
+ for (const chunk of chunks) {
62
+ bytes.set(chunk, offset);
63
+ offset += chunk.byteLength;
64
+ }
65
+ return { text: new TextDecoder().decode(bytes), truncated };
66
+ };
24
67
  /** POST one JSON request; a network failure or timeout throws, so an unreachable target is an error, never a pass or a fail. */
25
68
  export const send = async (fetcher, url, headers, body, timeoutMs) => {
26
69
  try {
@@ -31,38 +74,46 @@ export const send = async (fetcher, url, headers, body, timeoutMs) => {
31
74
  signal: AbortSignal.timeout(timeoutMs),
32
75
  redirect: 'manual'
33
76
  });
34
- const text = await response.text();
35
- return { status: response.status, body: text.slice(0, MAX_BODY) };
77
+ const { text } = await readCapped(response, MAX_BODY);
78
+ return { status: response.status, body: text };
36
79
  }
37
80
  catch (error) {
38
81
  throw new Error(`${url} unreachable: ${message(error)}`, { cause: error });
39
82
  }
40
83
  };
41
- /** The number of detection events the endpoint reports: a JSON array's length, or the length of an object's `events` array. */
42
- export const countEvents = async (fetcher, url, headers, timeoutMs) => {
43
- let response;
84
+ /** The detection events the endpoint reports: a JSON array, or an object's `events` array. */
85
+ export const readEvents = async (fetcher, url, headers, timeoutMs) => {
86
+ let text;
44
87
  try {
45
- response = await fetcher(url, {
88
+ const response = await fetcher(url, {
46
89
  headers: { accept: 'application/json', ...headers },
47
90
  signal: AbortSignal.timeout(timeoutMs),
48
91
  redirect: 'manual'
49
92
  });
93
+ if (!response.ok) {
94
+ await response.body?.cancel().catch(() => undefined);
95
+ throw new Error(`${url} returned ${response.status}`);
96
+ }
97
+ const read = await readCapped(response, MAX_EVENTS_BODY);
98
+ if (read.truncated)
99
+ throw new Error(`${url} returned more than ${MAX_EVENTS_BODY / 1024 / 1024} MiB; filter events by run or time on the API side`);
100
+ text = read.text;
50
101
  }
51
102
  catch (error) {
103
+ if (error instanceof Error && error.message.startsWith(url))
104
+ throw error;
52
105
  throw new Error(`${url} unreachable: ${message(error)}`, { cause: error });
53
106
  }
54
- if (!response.ok)
55
- throw new Error(`${url} returned ${response.status}`);
56
107
  let value;
57
108
  try {
58
- value = await response.json();
109
+ value = JSON.parse(text);
59
110
  }
60
111
  catch {
61
112
  throw new Error(`${url} did not return JSON`);
62
113
  }
63
114
  if (Array.isArray(value))
64
- return value.length;
115
+ return value;
65
116
  if (value && typeof value === 'object' && 'events' in value && Array.isArray(value.events))
66
- return value.events.length;
117
+ return value.events;
67
118
  throw new Error(`${url} returned neither an array nor an object with an events array`);
68
119
  };
@@ -19,6 +19,9 @@ export declare const TEST_HEADER = "x-agentests-test";
19
19
  export declare class ProbeOrchestrator {
20
20
  #private;
21
21
  constructor(policy: AgentTestSecurityPolicy, env: Record<string, string | undefined>, options?: ProbeOptions);
22
- /** Run every Test in policy order, or only the one with `id`; throws on an unknown id or an unreachable target. */
22
+ /**
23
+ * Run every Test in policy order, or only the one with `id`; throws on an unknown id or an unreachable target.
24
+ * Secrets the policy sends never appear in the report or in a thrown message.
25
+ */
23
26
  readonly run: (id?: string) => Promise<AgentTestSecurityReport>;
24
27
  }
@@ -1,8 +1,10 @@
1
1
  import { randomUUID } from 'node:crypto';
2
- import { countEvents, expandHeaders, expandValue, send } from "./http.js";
2
+ import { expandHeaders, expandValue, readEvents, send } from "./http.js";
3
3
  import { scenarioPrompt } from "./scenarios.js";
4
- import { sleep } from "./utils.js";
5
- import { judge } from "./verdict.js";
4
+ import { policySecrets, redactTest, redactText, redactUrl } from "./redact.js";
5
+ import { message, sleep } from "./utils.js";
6
+ import { securityPolicyWarnings } from "../policy/validate.js";
7
+ import { inconclusive, judge } from "./verdict.js";
6
8
  /** Headers sent on every request, so the target's logs can tell test traffic apart. */
7
9
  export const RUN_HEADER = 'x-agentests-run';
8
10
  export const TEST_HEADER = 'x-agentests-test';
@@ -32,8 +34,20 @@ export class ProbeOrchestrator {
32
34
  this.#runId = options.runId ?? randomUUID();
33
35
  this.#onResult = options.onResult;
34
36
  }
35
- /** Run every Test in policy order, or only the one with `id`; throws on an unknown id or an unreachable target. */
37
+ /**
38
+ * Run every Test in policy order, or only the one with `id`; throws on an unknown id or an unreachable target.
39
+ * Secrets the policy sends never appear in the report or in a thrown message.
40
+ */
36
41
  run = async (id) => {
42
+ const secrets = policySecrets(this.#policy, this.#env);
43
+ try {
44
+ return await this.#run(id);
45
+ }
46
+ catch (error) {
47
+ throw new Error(redactText(message(error), secrets));
48
+ }
49
+ };
50
+ #run = async (id) => {
37
51
  const { name, target, tests } = this.#policy;
38
52
  const selected = id === undefined ? tests : tests.filter((test) => test.id === id);
39
53
  if (!selected.length)
@@ -43,27 +57,48 @@ export class ProbeOrchestrator {
43
57
  for (const test of selected) {
44
58
  const evidence = await this.#probe(test);
45
59
  const checks = judge(test.expect, evidence);
46
- const result = { test, passed: checks.every((check) => check.passed), checks, evidence };
60
+ const reason = inconclusive(test, evidence);
61
+ const result = {
62
+ test: redactTest(test),
63
+ passed: reason === undefined && checks.every((check) => check.passed),
64
+ ...(reason === undefined ? {} : { inconclusive: reason }),
65
+ checks,
66
+ evidence
67
+ };
47
68
  results.push(result);
48
69
  this.#onResult?.(result);
49
70
  }
50
71
  return {
51
72
  policy: name,
52
73
  runId: this.#runId,
53
- target: { url: target.url, environment: target.environment },
74
+ target: { url: redactUrl(target.url), environment: target.environment },
54
75
  passed: results.filter((result) => result.passed).length,
55
76
  total: results.length,
56
- results
77
+ results,
78
+ warnings: securityPolicyWarnings(this.#policy)
57
79
  };
58
80
  };
81
+ /** This run's share of the detection events: all of them, or with `run_field` only those carrying this run's id. */
82
+ #countEvents = async (headers) => {
83
+ const { detection, timeout_ms } = this.#policy.target;
84
+ if (!detection)
85
+ return 0;
86
+ const events = await readEvents(this.#fetch, detection.url, headers, timeout_ms);
87
+ const field = detection.run_field;
88
+ if (field === undefined)
89
+ return events.length;
90
+ return events.filter((event) => !!event &&
91
+ typeof event === 'object' &&
92
+ String(event[field]) === this.#runId).length;
93
+ };
59
94
  /** Send one Test's scenario and collect its evidence. */
60
95
  #probe = async (test) => {
61
96
  const { target } = this.#policy;
62
97
  const detection = test.expect.detection_event !== undefined ? target.detection : undefined;
63
- const detectionHeaders = detection ? expandHeaders(detection.headers, this.#env) : {};
64
- const before = detection
65
- ? await countEvents(this.#fetch, detection.url, detectionHeaders, target.timeout_ms)
66
- : 0;
98
+ const detectionHeaders = detection
99
+ ? expandHeaders(detection.headers, this.#env, { run_id: this.#runId })
100
+ : {};
101
+ const before = detection ? await this.#countEvents(detectionHeaders) : 0;
67
102
  const interval = (test.duration_seconds * 1000) / test.requests;
68
103
  const evidence = {
69
104
  sent: 0,
@@ -72,6 +107,7 @@ export class ProbeOrchestrator {
72
107
  rateLimited: 0,
73
108
  quotaRejected: 0,
74
109
  succeededAfterQuota: 0,
110
+ errors: 0,
75
111
  elapsedMs: 0
76
112
  };
77
113
  const start = this.#now();
@@ -100,12 +136,13 @@ export class ProbeOrchestrator {
100
136
  }
101
137
  if (detection) {
102
138
  const deadline = this.#now() + detection.wait_ms;
103
- let after = await countEvents(this.#fetch, detection.url, detectionHeaders, target.timeout_ms);
139
+ let after = await this.#countEvents(detectionHeaders);
104
140
  while (after <= before && this.#now() < deadline) {
105
141
  await this.#sleep(Math.min(250, deadline - this.#now()));
106
- after = await countEvents(this.#fetch, detection.url, detectionHeaders, target.timeout_ms);
142
+ after = await this.#countEvents(detectionHeaders);
107
143
  }
108
144
  evidence.detectionEvents = Math.max(0, after - before);
145
+ evidence.detectionCorrelated = detection.run_field !== undefined;
109
146
  }
110
147
  evidence.elapsedMs = this.#now() - start;
111
148
  return evidence;
@@ -128,5 +165,7 @@ export class ProbeOrchestrator {
128
165
  if (evidence.firstQuota !== undefined)
129
166
  evidence.succeededAfterQuota++;
130
167
  }
168
+ else
169
+ evidence.errors++;
131
170
  };
132
171
  }
@@ -0,0 +1,12 @@
1
+ import type { AgentTestSecurityPolicy } from '../policy/types.ts';
2
+ import type { AgentTestSecurityTest } from './types.ts';
3
+ /** What every redacted value becomes. */
4
+ export declare const REDACTED = "[REDACTED]";
5
+ /** The environment values a policy sends, plus literal values of its sensitive headers: what must never be printed. */
6
+ export declare const policySecrets: (policy: AgentTestSecurityPolicy, env: Record<string, string | undefined>) => string[];
7
+ /** A URL with the values of its sensitive query parameters redacted; anything unparsable is returned as is. */
8
+ export declare const redactUrl: (value: string) => string;
9
+ /** `text` with every secret, and every sensitive query parameter of every URL in it, redacted. */
10
+ export declare const redactText: (text: string, secrets: string[]) => string;
11
+ /** A Test as the report shows it: sensitive header values redacted unless they are only `${VAR}` references. */
12
+ export declare const redactTest: (test: AgentTestSecurityTest) => AgentTestSecurityTest;
@@ -0,0 +1,66 @@
1
+ /** What every redacted value becomes. */
2
+ export const REDACTED = '[REDACTED]';
3
+ /** Header and query parameter names whose values are treated as secrets. */
4
+ const SENSITIVE = /auth|key|token|secret|password|cookie|session|signature|^sig$/i;
5
+ /** Shorter values are left alone: redacting `1` or `abc` would mangle every message. */
6
+ const MIN_SECRET = 6;
7
+ /** The names of every `${VAR}` a value refers to. */
8
+ const references = (value) => {
9
+ if (typeof value === 'string')
10
+ return [...value.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g)].map((match) => match[1]);
11
+ if (Array.isArray(value))
12
+ return value.flatMap(references);
13
+ if (value && typeof value === 'object')
14
+ return Object.values(value).flatMap(references);
15
+ return [];
16
+ };
17
+ /** The environment values a policy sends, plus literal values of its sensitive headers: what must never be printed. */
18
+ export const policySecrets = (policy, env) => {
19
+ const { target, tests } = policy;
20
+ const headerSets = [
21
+ target.headers,
22
+ target.detection?.headers ?? {},
23
+ ...tests.map((t) => t.headers)
24
+ ];
25
+ const names = references([target.body, ...headerSets]);
26
+ const literals = headerSets.flatMap((headers) => Object.entries(headers)
27
+ .filter(([name, value]) => SENSITIVE.test(name) && !references(value).length)
28
+ .map(([, value]) => value.replace(/^(Bearer|Basic|Token)\s+/i, '')));
29
+ const values = [...names.map((name) => env[name] ?? ''), ...literals];
30
+ return [...new Set(values.filter((value) => value.length >= MIN_SECRET))].sort((a, b) => b.length - a.length);
31
+ };
32
+ /** A URL with the values of its sensitive query parameters redacted; anything unparsable is returned as is. */
33
+ export const redactUrl = (value) => {
34
+ let url;
35
+ try {
36
+ url = new URL(value);
37
+ }
38
+ catch {
39
+ return value;
40
+ }
41
+ let changed = false;
42
+ for (const name of [...url.searchParams.keys()])
43
+ if (SENSITIVE.test(name)) {
44
+ url.searchParams.set(name, REDACTED);
45
+ changed = true;
46
+ }
47
+ return changed ? url.toString() : value;
48
+ };
49
+ /** `text` with every secret, and every sensitive query parameter of every URL in it, redacted. */
50
+ export const redactText = (text, secrets) => secrets
51
+ .reduce((result, secret) => result.split(secret).join(REDACTED), text)
52
+ .replace(/https?:\/\/[^\s"'<>]+/g, redactUrl);
53
+ /** A Test as the report shows it: sensitive header values redacted unless they are only `${VAR}` references. */
54
+ export const redactTest = (test) => ({
55
+ ...test,
56
+ headers: Object.fromEntries(Object.entries(test.headers).map(([name, value]) => [
57
+ name,
58
+ SENSITIVE.test(name) &&
59
+ value
60
+ .replace(/\$\{[A-Za-z_][A-Za-z0-9_]*\}/g, '')
61
+ .replace(/^(Bearer|Basic|Token)\s*/i, '')
62
+ .trim()
63
+ ? REDACTED
64
+ : value
65
+ ]))
66
+ });
@@ -39,6 +39,7 @@ export declare const DetectionSchema: z.ZodObject<{
39
39
  url: z.ZodURL;
40
40
  headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
41
41
  wait_ms: z.ZodDefault<z.ZodNumber>;
42
+ run_field: z.ZodOptional<z.ZodString>;
42
43
  }, z.core.$strict>;
43
44
  export type AgentTestDetection = z.infer<typeof DetectionSchema>;
44
45
  /** The API under test. Requests are POSTs of `body`, a JSON template whose strings may hold `{{prompt}}`, `{{index}}`, `{{scenario}}`, and `{{run_id}}`. */
@@ -67,17 +68,20 @@ export declare const TargetSchema: z.ZodObject<{
67
68
  url: z.ZodURL;
68
69
  headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
69
70
  wait_ms: z.ZodDefault<z.ZodNumber>;
71
+ run_field: z.ZodOptional<z.ZodString>;
70
72
  }, z.core.$strict>>;
71
73
  }, z.core.$strict>;
72
74
  export type AgentTestTarget = z.infer<typeof TargetSchema>;
73
- /** What a security Test expects of the defenses; at least one field. */
75
+ /** What a security Test expects of the defenses; at least one of the three expectations. */
74
76
  export declare const ExpectSchema: z.ZodObject<{
75
77
  rate_limit_enforced: z.ZodOptional<z.ZodBoolean>;
76
78
  quota_enforced: z.ZodOptional<z.ZodBoolean>;
77
79
  detection_event: z.ZodOptional<z.ZodBoolean>;
80
+ rate_limit_within: z.ZodOptional<z.ZodNumber>;
81
+ quota_within: z.ZodOptional<z.ZodNumber>;
78
82
  }, z.core.$strict>;
79
83
  export type AgentTestExpect = z.infer<typeof ExpectSchema>;
80
- export type AgentTestExpectation = keyof AgentTestExpect;
84
+ export type AgentTestExpectation = 'rate_limit_enforced' | 'quota_enforced' | 'detection_event';
81
85
  /** One bounded attack simulation and its expectations; `duration_seconds` spreads the requests evenly, 0 sends them back to back. */
82
86
  export declare const SecurityTestSchema: z.ZodObject<{
83
87
  id: z.ZodString;
@@ -90,10 +94,13 @@ export declare const SecurityTestSchema: z.ZodObject<{
90
94
  requests: z.ZodNumber;
91
95
  duration_seconds: z.ZodDefault<z.ZodNumber>;
92
96
  headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
97
+ max_error_rate: z.ZodDefault<z.ZodNumber>;
93
98
  expect: z.ZodObject<{
94
99
  rate_limit_enforced: z.ZodOptional<z.ZodBoolean>;
95
100
  quota_enforced: z.ZodOptional<z.ZodBoolean>;
96
101
  detection_event: z.ZodOptional<z.ZodBoolean>;
102
+ rate_limit_within: z.ZodOptional<z.ZodNumber>;
103
+ quota_within: z.ZodOptional<z.ZodNumber>;
97
104
  }, z.core.$strict>;
98
105
  }, z.core.$strict>;
99
106
  export type AgentTestSecurityTest = z.infer<typeof SecurityTestSchema>;
@@ -102,6 +109,7 @@ export declare const SafetySchema: z.ZodObject<{
102
109
  max_requests: z.ZodNumber;
103
110
  allow_production: z.ZodDefault<z.ZodBoolean>;
104
111
  allowed_hosts: z.ZodDefault<z.ZodArray<z.ZodString>>;
112
+ allow_insecure_http: z.ZodDefault<z.ZodBoolean>;
105
113
  }, z.core.$strict>;
106
114
  export type AgentTestSafety = z.infer<typeof SafetySchema>;
107
115
  /** The result of one expectation: what was expected, what the evidence shows, and why. */
@@ -125,14 +133,20 @@ export interface AgentTestSecurityEvidence {
125
133
  firstQuota?: number;
126
134
  /** Successful responses after the first quota response: a quota that lets traffic through again is not enforced. */
127
135
  succeededAfterQuota: number;
136
+ /** Non-2xx responses that match no signal, such as 500s: evidence about the target, not its defenses. */
137
+ errors: number;
128
138
  /** New detection events seen during the scenario; absent without `target.detection`. */
129
139
  detectionEvents?: number;
140
+ /** Whether `detectionEvents` counts only this run's events (`target.detection.run_field`); absent without detection. */
141
+ detectionCorrelated?: boolean;
130
142
  elapsedMs: number;
131
143
  }
132
144
  /** One security Test with its checks, evidence, and overall pass. */
133
145
  export interface AgentTestSecurityResult {
134
146
  test: AgentTestSecurityTest;
135
147
  passed: boolean;
148
+ /** Why the evidence cannot be trusted, such as too many errors; set only then, and the Test fails. */
149
+ inconclusive?: string;
136
150
  checks: AgentTestSecurityCheck[];
137
151
  evidence: AgentTestSecurityEvidence;
138
152
  }
@@ -147,4 +161,6 @@ export interface AgentTestSecurityReport {
147
161
  passed: number;
148
162
  total: number;
149
163
  results: AgentTestSecurityResult[];
164
+ /** Policy problems that are not errors yet, such as plain HTTP to a remote host. */
165
+ warnings: string[];
150
166
  }
@@ -25,7 +25,9 @@ export const DetectionSchema = z.strictObject({
25
25
  url: z.url({ protocol: /^https?$/ }),
26
26
  headers: HeadersSchema.default({}),
27
27
  /** How long to keep polling for a new event after the scenario ends. */
28
- wait_ms: z.number().int().min(0).max(60_000).default(0)
28
+ wait_ms: z.number().int().min(0).max(60_000).default(0),
29
+ /** The event field holding the run id the API saw in `x-agentests-run`; when set, only this run's events count. */
30
+ run_field: TextSchema.optional()
29
31
  });
30
32
  /** The API under test. Requests are POSTs of `body`, a JSON template whose strings may hold `{{prompt}}`, `{{index}}`, `{{scenario}}`, and `{{run_id}}`. */
31
33
  export const TargetSchema = z.strictObject({
@@ -40,14 +42,20 @@ export const TargetSchema = z.strictObject({
40
42
  }),
41
43
  detection: DetectionSchema.optional()
42
44
  });
43
- /** What a security Test expects of the defenses; at least one field. */
45
+ /** What a security Test expects of the defenses; at least one of the three expectations. */
44
46
  export const ExpectSchema = z
45
47
  .strictObject({
46
48
  rate_limit_enforced: z.boolean().optional(),
47
49
  quota_enforced: z.boolean().optional(),
48
- detection_event: z.boolean().optional()
50
+ detection_event: z.boolean().optional(),
51
+ /** With `rate_limit_enforced: true`: the first rate-limited response must come by this request. */
52
+ rate_limit_within: z.number().int().min(1).optional(),
53
+ /** With `quota_enforced: true`: the first quota response must come by this request. */
54
+ quota_within: z.number().int().min(1).optional()
49
55
  })
50
- .refine((value) => Object.keys(value).length > 0, 'Expected at least one expectation');
56
+ .refine((value) => value.rate_limit_enforced !== undefined ||
57
+ value.quota_enforced !== undefined ||
58
+ value.detection_event !== undefined, 'Expected at least one expectation');
51
59
  /** One bounded attack simulation and its expectations; `duration_seconds` spreads the requests evenly, 0 sends them back to back. */
52
60
  export const SecurityTestSchema = z.strictObject({
53
61
  id: IdSchema,
@@ -57,11 +65,15 @@ export const SecurityTestSchema = z.strictObject({
57
65
  duration_seconds: z.number().min(0).max(3_600).default(0),
58
66
  /** Merged over the target's headers for this Test only, e.g. its own API key. */
59
67
  headers: HeadersSchema.default({}),
68
+ /** The largest share of responses that may be errors matching no signal before the Test is inconclusive, and fails. */
69
+ max_error_rate: z.number().min(0).max(1).default(0.05),
60
70
  expect: ExpectSchema
61
71
  });
62
72
  /** Hard limits the runner never exceeds. Hosts other than localhost need listing; production needs `allow_production` and the `--allow-production` flag. */
63
73
  export const SafetySchema = z.strictObject({
64
74
  max_requests: z.number().int().min(1).max(10_000),
65
75
  allow_production: z.boolean().default(false),
66
- allowed_hosts: z.array(TextSchema).default([])
76
+ allowed_hosts: z.array(TextSchema).default([]),
77
+ /** Accept plain `http://` to a non-local host. Without it, such a URL is a warning in 1.x and an error from 2.0. */
78
+ allow_insecure_http: z.boolean().default(false)
67
79
  });
@@ -1,3 +1,5 @@
1
- import type { AgentTestExpect, AgentTestSecurityCheck, AgentTestSecurityEvidence } from './types.ts';
1
+ import type { AgentTestSecurityTest, AgentTestExpect, AgentTestSecurityCheck, AgentTestSecurityEvidence } from './types.ts';
2
2
  /** One check per expectation the Test sets, in a fixed order. */
3
3
  export declare const judge: (expect: AgentTestExpect, evidence: AgentTestSecurityEvidence) => AgentTestSecurityCheck[];
4
+ /** Why a Test's evidence says more about the target's errors than its defenses, or undefined when it is sound. */
5
+ export declare const inconclusive: ({ max_error_rate }: AgentTestSecurityTest, { sent, errors, statuses }: AgentTestSecurityEvidence) => string | undefined;
@@ -1,17 +1,29 @@
1
1
  /** What the evidence shows for one expectation, and the sentence explaining it. */
2
- const observe = (expectation, evidence) => {
2
+ const observe = (expectation, evidence, within) => {
3
3
  const { sent } = evidence;
4
4
  switch (expectation) {
5
- case 'rate_limit_enforced':
6
- return evidence.firstRateLimit === undefined
7
- ? { actual: false, detail: `no rate-limited response in ${sent} requests` }
8
- : {
9
- actual: true,
10
- detail: `first rate-limited at request ${evidence.firstRateLimit} of ${sent}; ${evidence.rateLimited} rejected`
5
+ case 'rate_limit_enforced': {
6
+ const first = evidence.firstRateLimit;
7
+ if (first === undefined)
8
+ return { actual: false, detail: `no rate-limited response in ${sent} requests` };
9
+ if (within !== undefined && first > within)
10
+ return {
11
+ actual: false,
12
+ detail: `first rate-limited at request ${first}, after the limit of ${within}`
11
13
  };
14
+ return {
15
+ actual: true,
16
+ detail: `first rate-limited at request ${first} of ${sent}; ${evidence.rateLimited} rejected`
17
+ };
18
+ }
12
19
  case 'quota_enforced':
13
20
  if (evidence.firstQuota === undefined)
14
21
  return { actual: false, detail: `no quota response in ${sent} requests` };
22
+ if (within !== undefined && evidence.firstQuota > within)
23
+ return {
24
+ actual: false,
25
+ detail: `quota hit at request ${evidence.firstQuota}, after the limit of ${within}`
26
+ };
15
27
  if (evidence.succeededAfterQuota > 0)
16
28
  return {
17
29
  actual: false,
@@ -23,7 +35,8 @@ const observe = (expectation, evidence) => {
23
35
  };
24
36
  case 'detection_event': {
25
37
  const events = evidence.detectionEvents ?? 0;
26
- return { actual: events > 0, detail: `${events} new detection event(s)` };
38
+ const scope = evidence.detectionCorrelated ? ' for this run' : '';
39
+ return { actual: events > 0, detail: `${events} new detection event(s)${scope}` };
27
40
  }
28
41
  }
29
42
  };
@@ -32,6 +45,21 @@ export const judge = (expect, evidence) => ['rate_limit_enforced', 'quota_enforc
32
45
  const expected = expect[expectation];
33
46
  if (expected === undefined)
34
47
  return [];
35
- const { actual, detail } = observe(expectation, evidence);
48
+ const within = expectation === 'rate_limit_enforced'
49
+ ? expect.rate_limit_within
50
+ : expectation === 'quota_enforced'
51
+ ? expect.quota_within
52
+ : undefined;
53
+ const { actual, detail } = observe(expectation, evidence, within);
36
54
  return [{ expectation, expected, actual, passed: actual === expected, detail }];
37
55
  });
56
+ /** Why a Test's evidence says more about the target's errors than its defenses, or undefined when it is sound. */
57
+ export const inconclusive = ({ max_error_rate }, { sent, errors, statuses }) => {
58
+ if (!sent || errors / sent <= max_error_rate)
59
+ return undefined;
60
+ const codes = Object.entries(statuses)
61
+ .filter(([status]) => !/^2/.test(status))
62
+ .map(([status, count]) => `${status}×${count}`)
63
+ .join(', ');
64
+ return `${errors} of ${sent} responses were errors matching no signal (non-2xx: ${codes}), over max_error_rate ${max_error_rate}`;
65
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nyancorn/agentests-security",
3
- "version": "1.0.0",
3
+ "version": "1.1.0",
4
4
  "description": "Test an AI API's defenses against model extraction in CI: rate limits, quotas, and detection, as bounded, deterministic security policies.",
5
5
  "keywords": [
6
6
  "ai-security",
@@ -149,6 +149,9 @@
149
149
  "type": "integer",
150
150
  "minimum": 0,
151
151
  "maximum": 60000
152
+ },
153
+ "run_field": {
154
+ "type": "string"
152
155
  }
153
156
  },
154
157
  "required": [
@@ -206,6 +209,12 @@
206
209
  "type": "string"
207
210
  }
208
211
  },
212
+ "max_error_rate": {
213
+ "default": 0.05,
214
+ "type": "number",
215
+ "minimum": 0,
216
+ "maximum": 1
217
+ },
209
218
  "expect": {
210
219
  "type": "object",
211
220
  "properties": {
@@ -217,6 +226,16 @@
217
226
  },
218
227
  "detection_event": {
219
228
  "type": "boolean"
229
+ },
230
+ "rate_limit_within": {
231
+ "type": "integer",
232
+ "minimum": 1,
233
+ "maximum": 9007199254740991
234
+ },
235
+ "quota_within": {
236
+ "type": "integer",
237
+ "minimum": 1,
238
+ "maximum": 9007199254740991
220
239
  }
221
240
  },
222
241
  "additionalProperties": false
@@ -249,6 +268,10 @@
249
268
  "items": {
250
269
  "type": "string"
251
270
  }
271
+ },
272
+ "allow_insecure_http": {
273
+ "default": false,
274
+ "type": "boolean"
252
275
  }
253
276
  },
254
277
  "required": [