@kirrosh/zond 0.12.7 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/package.json +1 -1
- package/src/core/generator/guide-builder.ts +53 -1
- package/src/core/generator/serializer.ts +46 -10
- package/src/core/parser/schema.ts +51 -4
- package/src/core/parser/types.ts +27 -0
- package/src/core/parser/variables.ts +1 -0
- package/src/core/runner/assertions.ts +126 -0
- package/src/core/runner/executor.ts +120 -2
- package/src/core/runner/expr-eval.ts +41 -0
- package/src/core/runner/transforms.ts +65 -0
package/CHANGELOG.md
CHANGED
|
@@ -50,6 +50,16 @@ All notable changes to this project will be documented in this file.
|
|
|
50
50
|
|
|
51
51
|
### Added
|
|
52
52
|
|
|
53
|
+
- **Extended YAML test format** — 12 new assertion operators, flow control, and data transforms:
|
|
54
|
+
- **Assertion operators**: `not_equals`, `not_contains`, `gte`, `lte`, `length`, `length_gt/gte/lt/lte`
|
|
55
|
+
- **Array assertions**: `each` (every element matches), `contains_item` (at least one matches), `set_equals` (order-independent comparison)
|
|
56
|
+
- **Flow control**: `skip_if` (conditional skip with expression evaluator), `retry_until` (retry with condition/max_attempts/delay_ms), `for_each` (iterate over array)
|
|
57
|
+
- **Data transforms**: `set` steps with directives — `concat`, `append`, `length`, `get`, `first`, `map_field`
|
|
58
|
+
- **Generator**: `{{$isoTimestamp}}` — ISO 8601 timestamp string
|
|
59
|
+
- **Expression evaluator**: supports `==`, `!=`, `>`, `<`, `>=`, `<=` for skip_if/retry_until conditions
|
|
60
|
+
- Guide-builder YAML cheatsheet updated with all new features
|
|
61
|
+
- Full backward compatibility — all existing tests continue to work unchanged
|
|
62
|
+
|
|
53
63
|
- **MCP feedback improvements**
|
|
54
64
|
- `diagnose_failure` now includes `response_headers` in failure output (e.g. `X-Ably-ErrorMessage`)
|
|
55
65
|
- `generate_tests_guide`: annotates `any`-typed request bodies with a warning comment
|
package/package.json
CHANGED
|
@@ -120,7 +120,7 @@ Use \`json:\` for JSON request bodies. Do NOT use \`body:\` — it is not a vali
|
|
|
120
120
|
For form-encoded: use \`form:\` instead of \`json:\`.
|
|
121
121
|
|
|
122
122
|
### Built-in generators
|
|
123
|
-
\`{{$uuid}}\`, \`{{$randomInt}}\`, \`{{$timestamp}}\`, \`{{$randomName}}\`, \`{{$randomEmail}}\`, \`{{$randomString}}\`
|
|
123
|
+
\`{{$uuid}}\`, \`{{$randomInt}}\`, \`{{$timestamp}}\`, \`{{$isoTimestamp}}\`, \`{{$randomName}}\`, \`{{$randomEmail}}\`, \`{{$randomString}}\`
|
|
124
124
|
|
|
125
125
|
### Variable capture & interpolation
|
|
126
126
|
\`\`\`yaml
|
|
@@ -139,6 +139,58 @@ For form-encoded: use \`form:\` instead of \`json:\`.
|
|
|
139
139
|
id: { equals: "{{created_id}}" }
|
|
140
140
|
\`\`\`
|
|
141
141
|
|
|
142
|
+
### Array assertions
|
|
143
|
+
\`\`\`yaml
|
|
144
|
+
items:
|
|
145
|
+
each: # every element must match
|
|
146
|
+
status: { not_equals: "deleted" }
|
|
147
|
+
id: { type: integer }
|
|
148
|
+
|
|
149
|
+
items:
|
|
150
|
+
contains_item: # at least one element matches
|
|
151
|
+
name: { contains: "test" }
|
|
152
|
+
|
|
153
|
+
ids:
|
|
154
|
+
set_equals: [1, 2, 3] # same elements, order-independent
|
|
155
|
+
\`\`\`
|
|
156
|
+
|
|
157
|
+
### Flow control
|
|
158
|
+
\`\`\`yaml
|
|
159
|
+
# skip_if — skip step when condition is true (after variable substitution)
|
|
160
|
+
- name: Delete only if exists
|
|
161
|
+
DELETE: /items/{{item_id}}
|
|
162
|
+
skip_if: "{{item_id}} == 0"
|
|
163
|
+
expect:
|
|
164
|
+
status: 204
|
|
165
|
+
|
|
166
|
+
# retry_until — repeat request until condition met
|
|
167
|
+
- name: Wait for processing
|
|
168
|
+
GET: /jobs/{{job_id}}
|
|
169
|
+
retry_until:
|
|
170
|
+
condition: "{{status}} == completed"
|
|
171
|
+
max_attempts: 5
|
|
172
|
+
delay_ms: 1000
|
|
173
|
+
expect:
|
|
174
|
+
status: 200
|
|
175
|
+
|
|
176
|
+
# for_each — repeat step for each item in array
|
|
177
|
+
- name: Delete item
|
|
178
|
+
DELETE: /items/{{id}}
|
|
179
|
+
for_each:
|
|
180
|
+
var: id
|
|
181
|
+
in: "{{item_ids}}"
|
|
182
|
+
expect:
|
|
183
|
+
status: [200, 204]
|
|
184
|
+
|
|
185
|
+
# set — transform variables without HTTP request
|
|
186
|
+
- name: Extract IDs
|
|
187
|
+
set:
|
|
188
|
+
all_ids: { map_field: ["{{items}}", "id"] }
|
|
189
|
+
count: { length: "{{items}}" }
|
|
190
|
+
first_item: { first: "{{items}}" }
|
|
191
|
+
merged: { concat: ["{{list_a}}", "{{list_b}}"] }
|
|
192
|
+
\`\`\`
|
|
193
|
+
|
|
142
194
|
### Coverage matching
|
|
143
195
|
Use spec paths with \`{param}\` placeholders in the path for coverage to match:
|
|
144
196
|
- Spec says \`GET /products/{id}\` → write \`GET: /products/1\` (hardcode the value)
|
|
@@ -93,17 +93,53 @@ export function serializeSuite(suite: RawSuite): string {
|
|
|
93
93
|
serializeValue(test.query, 3, lines);
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
96
|
+
// skip_if
|
|
97
|
+
if (test.skip_if) {
|
|
98
|
+
lines.push(` skip_if: ${yamlScalar(String(test.skip_if))}`);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// retry_until
|
|
102
|
+
if (test.retry_until && typeof test.retry_until === "object") {
|
|
103
|
+
const rt = test.retry_until as Record<string, unknown>;
|
|
104
|
+
lines.push(" retry_until:");
|
|
105
|
+
if (rt.condition !== undefined) lines.push(` condition: ${yamlScalar(String(rt.condition))}`);
|
|
106
|
+
if (rt.max_attempts !== undefined) lines.push(` max_attempts: ${rt.max_attempts}`);
|
|
107
|
+
if (rt.delay_ms !== undefined) lines.push(` delay_ms: ${rt.delay_ms}`);
|
|
100
108
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
109
|
+
|
|
110
|
+
// for_each
|
|
111
|
+
if (test.for_each && typeof test.for_each === "object") {
|
|
112
|
+
const fe = test.for_each as Record<string, unknown>;
|
|
113
|
+
lines.push(" for_each:");
|
|
114
|
+
if (fe.var !== undefined) lines.push(` var: ${yamlScalar(String(fe.var))}`);
|
|
115
|
+
if (fe.in !== undefined) lines.push(` in: ${yamlScalar(String(fe.in))}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// set
|
|
119
|
+
if (test.set && typeof test.set === "object") {
|
|
120
|
+
lines.push(" set:");
|
|
121
|
+
serializeValue(test.set, 3, lines);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// expect
|
|
125
|
+
const hasExpect = test.expect && (test.expect.status !== undefined || test.expect.body);
|
|
126
|
+
if (hasExpect) {
|
|
127
|
+
lines.push(" expect:");
|
|
128
|
+
if (test.expect.status !== undefined) {
|
|
129
|
+
lines.push(` status: ${test.expect.status}`);
|
|
130
|
+
}
|
|
131
|
+
if (test.expect.body) {
|
|
132
|
+
lines.push(" body:");
|
|
133
|
+
for (const [key, rule] of Object.entries(test.expect.body)) {
|
|
134
|
+
lines.push(` ${key}:`);
|
|
135
|
+
for (const [rk, rv] of Object.entries(rule)) {
|
|
136
|
+
if (typeof rv === "object" && rv !== null) {
|
|
137
|
+
lines.push(` ${rk}:`);
|
|
138
|
+
serializeValue(rv, 6, lines);
|
|
139
|
+
} else {
|
|
140
|
+
lines.push(` ${rk}: ${yamlScalar(String(rv))}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
107
143
|
}
|
|
108
144
|
}
|
|
109
145
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type { TestSuite, TestStep, AssertionRule, TestStepExpect, SuiteConfig } from "./types.ts";
|
|
2
|
+
import type { TestSuite, TestStep, AssertionRule, TestStepExpect, SuiteConfig, RetryUntil, ForEach } from "./types.ts";
|
|
3
3
|
|
|
4
4
|
const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const;
|
|
5
5
|
|
|
@@ -26,11 +26,19 @@ function extractMethodAndPath(raw: unknown): unknown {
|
|
|
26
26
|
return { ...rest, method: foundMethod, path };
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
// set-only step: no HTTP method required
|
|
30
|
+
if (obj.set && !obj.method) {
|
|
31
|
+
return { ...obj, method: "GET", path: "" };
|
|
32
|
+
}
|
|
33
|
+
|
|
29
34
|
return raw;
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
const ASSERTION_KEYS = new Set([
|
|
33
|
-
"capture", "type", "equals", "
|
|
38
|
+
"capture", "type", "equals", "not_equals", "contains", "not_contains",
|
|
39
|
+
"matches", "gt", "lt", "gte", "lte", "exists",
|
|
40
|
+
"length", "length_gt", "length_gte", "length_lt", "length_lte",
|
|
41
|
+
"each", "contains_item", "set_equals",
|
|
34
42
|
]);
|
|
35
43
|
|
|
36
44
|
/**
|
|
@@ -68,7 +76,7 @@ export function flattenBodyAssertions(body: Record<string, unknown>): Record<str
|
|
|
68
76
|
return result;
|
|
69
77
|
}
|
|
70
78
|
|
|
71
|
-
const
|
|
79
|
+
const AssertionRuleSchemaInner: z.ZodType<AssertionRule> = z.preprocess(
|
|
72
80
|
(val) => {
|
|
73
81
|
if (typeof val === "string") return { type: val };
|
|
74
82
|
if (val === null || val === undefined) return { exists: true };
|
|
@@ -86,14 +94,28 @@ const AssertionRuleSchema: z.ZodType<AssertionRule> = z.preprocess(
|
|
|
86
94
|
capture: z.string().optional(),
|
|
87
95
|
type: z.enum(["string", "integer", "number", "boolean", "array", "object"]).optional(),
|
|
88
96
|
equals: z.unknown().optional(),
|
|
97
|
+
not_equals: z.unknown().optional(),
|
|
89
98
|
contains: z.string().optional(),
|
|
99
|
+
not_contains: z.string().optional(),
|
|
90
100
|
matches: z.string().optional(),
|
|
91
101
|
gt: z.number().optional(),
|
|
92
102
|
lt: z.number().optional(),
|
|
103
|
+
gte: z.number().optional(),
|
|
104
|
+
lte: z.number().optional(),
|
|
93
105
|
exists: z.boolean().optional(),
|
|
106
|
+
length: z.number().int().optional(),
|
|
107
|
+
length_gt: z.number().int().optional(),
|
|
108
|
+
length_gte: z.number().int().optional(),
|
|
109
|
+
length_lt: z.number().int().optional(),
|
|
110
|
+
length_lte: z.number().int().optional(),
|
|
111
|
+
each: z.record(z.string(), z.lazy(() => AssertionRuleSchemaInner)).optional(),
|
|
112
|
+
contains_item: z.record(z.string(), z.lazy(() => AssertionRuleSchemaInner)).optional(),
|
|
113
|
+
set_equals: z.unknown().optional(),
|
|
94
114
|
}),
|
|
95
115
|
) as z.ZodType<AssertionRule>;
|
|
96
116
|
|
|
117
|
+
const AssertionRuleSchema = AssertionRuleSchemaInner;
|
|
118
|
+
|
|
97
119
|
const TestStepExpectSchema: z.ZodType<TestStepExpect> = z.preprocess(
|
|
98
120
|
(val) => {
|
|
99
121
|
if (typeof val !== "object" || val === null) return val;
|
|
@@ -117,8 +139,29 @@ const TestStepExpectSchema: z.ZodType<TestStepExpect> = z.preprocess(
|
|
|
117
139
|
}),
|
|
118
140
|
) as z.ZodType<TestStepExpect>;
|
|
119
141
|
|
|
142
|
+
const RetryUntilSchema: z.ZodType<RetryUntil> = z.object({
|
|
143
|
+
condition: z.string(),
|
|
144
|
+
max_attempts: z.number().int().positive(),
|
|
145
|
+
delay_ms: z.number().int().nonnegative(),
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
const ForEachSchema: z.ZodType<ForEach> = z.object({
|
|
149
|
+
var: z.string(),
|
|
150
|
+
in: z.unknown(),
|
|
151
|
+
});
|
|
152
|
+
|
|
120
153
|
const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
|
|
121
|
-
|
|
154
|
+
(raw) => {
|
|
155
|
+
const obj = extractMethodAndPath(raw);
|
|
156
|
+
// Make expect optional for set-only steps
|
|
157
|
+
if (typeof obj === "object" && obj !== null) {
|
|
158
|
+
const o = obj as Record<string, unknown>;
|
|
159
|
+
if (o.set && !o.expect) {
|
|
160
|
+
o.expect = {};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return obj;
|
|
164
|
+
},
|
|
122
165
|
z.object({
|
|
123
166
|
name: z.string(),
|
|
124
167
|
method: z.enum(HTTP_METHODS),
|
|
@@ -128,6 +171,10 @@ const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
|
|
|
128
171
|
form: z.record(z.string(), z.string()).optional(),
|
|
129
172
|
query: z.record(z.string(), z.string()).optional(),
|
|
130
173
|
expect: TestStepExpectSchema,
|
|
174
|
+
skip_if: z.string().optional(),
|
|
175
|
+
retry_until: RetryUntilSchema.optional(),
|
|
176
|
+
for_each: ForEachSchema.optional(),
|
|
177
|
+
set: z.record(z.string(), z.unknown()).optional(),
|
|
131
178
|
}),
|
|
132
179
|
) as z.ZodType<TestStep>;
|
|
133
180
|
|
package/src/core/parser/types.ts
CHANGED
|
@@ -4,11 +4,23 @@ export interface AssertionRule {
|
|
|
4
4
|
capture?: string;
|
|
5
5
|
type?: "string" | "integer" | "number" | "boolean" | "array" | "object";
|
|
6
6
|
equals?: unknown;
|
|
7
|
+
not_equals?: unknown;
|
|
7
8
|
contains?: string;
|
|
9
|
+
not_contains?: string;
|
|
8
10
|
matches?: string;
|
|
9
11
|
gt?: number;
|
|
10
12
|
lt?: number;
|
|
13
|
+
gte?: number;
|
|
14
|
+
lte?: number;
|
|
11
15
|
exists?: boolean;
|
|
16
|
+
length?: number;
|
|
17
|
+
length_gt?: number;
|
|
18
|
+
length_gte?: number;
|
|
19
|
+
length_lt?: number;
|
|
20
|
+
length_lte?: number;
|
|
21
|
+
each?: Record<string, AssertionRule>;
|
|
22
|
+
contains_item?: Record<string, AssertionRule>;
|
|
23
|
+
set_equals?: unknown;
|
|
12
24
|
}
|
|
13
25
|
|
|
14
26
|
export interface TestStepExpect {
|
|
@@ -18,6 +30,17 @@ export interface TestStepExpect {
|
|
|
18
30
|
duration?: number;
|
|
19
31
|
}
|
|
20
32
|
|
|
33
|
+
export interface RetryUntil {
|
|
34
|
+
condition: string;
|
|
35
|
+
max_attempts: number;
|
|
36
|
+
delay_ms: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ForEach {
|
|
40
|
+
var: string;
|
|
41
|
+
in: unknown;
|
|
42
|
+
}
|
|
43
|
+
|
|
21
44
|
export interface TestStep {
|
|
22
45
|
name: string;
|
|
23
46
|
method: HttpMethod;
|
|
@@ -27,6 +50,10 @@ export interface TestStep {
|
|
|
27
50
|
form?: Record<string, string>;
|
|
28
51
|
query?: Record<string, string>;
|
|
29
52
|
expect: TestStepExpect;
|
|
53
|
+
skip_if?: string;
|
|
54
|
+
retry_until?: RetryUntil;
|
|
55
|
+
for_each?: ForEach;
|
|
56
|
+
set?: Record<string, unknown>;
|
|
30
57
|
}
|
|
31
58
|
|
|
32
59
|
export interface SuiteConfig {
|
|
@@ -19,6 +19,7 @@ function randomChars(len: number): string {
|
|
|
19
19
|
export const GENERATORS: Record<string, () => string | number> = {
|
|
20
20
|
"$uuid": () => crypto.randomUUID(),
|
|
21
21
|
"$timestamp": () => Math.floor(Date.now() / 1000),
|
|
22
|
+
"$isoTimestamp": () => new Date().toISOString(),
|
|
22
23
|
"$randomName": () => randomFrom(NAMES),
|
|
23
24
|
"$randomEmail": () => `${randomChars(8).toLowerCase()}@test.com`,
|
|
24
25
|
"$randomInt": () => Math.floor(Math.random() * 10000),
|
|
@@ -90,6 +90,132 @@ function checkRule(path: string, rule: AssertionRule, actual: unknown): Assertio
|
|
|
90
90
|
});
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
+
if (rule.not_equals !== undefined) {
|
|
94
|
+
results.push({
|
|
95
|
+
field, rule: `not_equals ${JSON.stringify(rule.not_equals)}`,
|
|
96
|
+
passed: !deepEquals(actual, rule.not_equals), actual, expected: rule.not_equals,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (rule.not_contains !== undefined) {
|
|
101
|
+
const passed = typeof actual === "string" && !actual.includes(rule.not_contains);
|
|
102
|
+
results.push({
|
|
103
|
+
field, rule: `not_contains "${rule.not_contains}"`,
|
|
104
|
+
passed, actual, expected: rule.not_contains,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (rule.gte !== undefined) {
|
|
109
|
+
const passed = typeof actual === "number" && actual >= rule.gte;
|
|
110
|
+
results.push({
|
|
111
|
+
field, rule: `gte ${rule.gte}`,
|
|
112
|
+
passed, actual, expected: rule.gte,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (rule.lte !== undefined) {
|
|
117
|
+
const passed = typeof actual === "number" && actual <= rule.lte;
|
|
118
|
+
results.push({
|
|
119
|
+
field, rule: `lte ${rule.lte}`,
|
|
120
|
+
passed, actual, expected: rule.lte,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (rule.length !== undefined) {
|
|
125
|
+
const hasLength = (Array.isArray(actual) || typeof actual === "string");
|
|
126
|
+
const passed = hasLength && (actual as string | unknown[]).length === rule.length;
|
|
127
|
+
results.push({
|
|
128
|
+
field, rule: `length ${rule.length}`,
|
|
129
|
+
passed, actual: hasLength ? (actual as string | unknown[]).length : describeType(actual), expected: rule.length,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (rule.length_gt !== undefined) {
|
|
134
|
+
const hasLength = (Array.isArray(actual) || typeof actual === "string");
|
|
135
|
+
const passed = hasLength && (actual as string | unknown[]).length > rule.length_gt;
|
|
136
|
+
results.push({
|
|
137
|
+
field, rule: `length_gt ${rule.length_gt}`,
|
|
138
|
+
passed, actual: hasLength ? (actual as string | unknown[]).length : describeType(actual), expected: rule.length_gt,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (rule.length_gte !== undefined) {
|
|
143
|
+
const hasLength = (Array.isArray(actual) || typeof actual === "string");
|
|
144
|
+
const passed = hasLength && (actual as string | unknown[]).length >= rule.length_gte;
|
|
145
|
+
results.push({
|
|
146
|
+
field, rule: `length_gte ${rule.length_gte}`,
|
|
147
|
+
passed, actual: hasLength ? (actual as string | unknown[]).length : describeType(actual), expected: rule.length_gte,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (rule.length_lt !== undefined) {
|
|
152
|
+
const hasLength = (Array.isArray(actual) || typeof actual === "string");
|
|
153
|
+
const passed = hasLength && (actual as string | unknown[]).length < rule.length_lt;
|
|
154
|
+
results.push({
|
|
155
|
+
field, rule: `length_lt ${rule.length_lt}`,
|
|
156
|
+
passed, actual: hasLength ? (actual as string | unknown[]).length : describeType(actual), expected: rule.length_lt,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (rule.length_lte !== undefined) {
|
|
161
|
+
const hasLength = (Array.isArray(actual) || typeof actual === "string");
|
|
162
|
+
const passed = hasLength && (actual as string | unknown[]).length <= rule.length_lte;
|
|
163
|
+
results.push({
|
|
164
|
+
field, rule: `length_lte ${rule.length_lte}`,
|
|
165
|
+
passed, actual: hasLength ? (actual as string | unknown[]).length : describeType(actual), expected: rule.length_lte,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (rule.each !== undefined) {
|
|
170
|
+
if (!Array.isArray(actual)) {
|
|
171
|
+
results.push({ field, rule: "each", passed: false, actual: describeType(actual), expected: "array" });
|
|
172
|
+
} else {
|
|
173
|
+
for (let i = 0; i < actual.length; i++) {
|
|
174
|
+
for (const [subPath, subRule] of Object.entries(rule.each)) {
|
|
175
|
+
const subActual = getByPath(actual[i], subPath);
|
|
176
|
+
const subResults = checkRule(`${path}[${i}].${subPath}`, subRule, subActual);
|
|
177
|
+
results.push(...subResults);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (rule.contains_item !== undefined) {
|
|
184
|
+
if (!Array.isArray(actual)) {
|
|
185
|
+
results.push({ field, rule: "contains_item", passed: false, actual: describeType(actual), expected: "array" });
|
|
186
|
+
} else {
|
|
187
|
+
const found = actual.some((item) => {
|
|
188
|
+
for (const [subPath, subRule] of Object.entries(rule.contains_item!)) {
|
|
189
|
+
const subActual = getByPath(item, subPath);
|
|
190
|
+
const subResults = checkRule("", subRule, subActual);
|
|
191
|
+
if (subResults.some(r => !r.passed)) return false;
|
|
192
|
+
}
|
|
193
|
+
return true;
|
|
194
|
+
});
|
|
195
|
+
results.push({
|
|
196
|
+
field, rule: "contains_item",
|
|
197
|
+
passed: found, actual: `array(${actual.length})`, expected: "at least one matching item",
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (rule.set_equals !== undefined) {
|
|
203
|
+
if (!Array.isArray(actual) || !Array.isArray(rule.set_equals)) {
|
|
204
|
+
results.push({
|
|
205
|
+
field, rule: "set_equals",
|
|
206
|
+
passed: false, actual: describeType(actual), expected: "both must be arrays",
|
|
207
|
+
});
|
|
208
|
+
} else {
|
|
209
|
+
const actualSet = new Set(actual.map(v => JSON.stringify(v)));
|
|
210
|
+
const expectedSet = new Set((rule.set_equals as unknown[]).map(v => JSON.stringify(v)));
|
|
211
|
+
const passed = actualSet.size === expectedSet.size && [...actualSet].every(v => expectedSet.has(v));
|
|
212
|
+
results.push({
|
|
213
|
+
field, rule: "set_equals",
|
|
214
|
+
passed, actual, expected: rule.set_equals,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
93
219
|
return results;
|
|
94
220
|
}
|
|
95
221
|
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import type { TestSuite, Environment } from "../parser/types.ts";
|
|
1
|
+
import type { TestSuite, TestStep, Environment } from "../parser/types.ts";
|
|
2
2
|
import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
|
|
3
3
|
import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
|
|
4
4
|
import { executeRequest, type FetchOptions } from "./http-client.ts";
|
|
5
5
|
import { checkAssertions, extractCaptures } from "./assertions.ts";
|
|
6
|
+
import { evaluateExpr } from "./expr-eval.ts";
|
|
7
|
+
import { applyTransform } from "./transforms.ts";
|
|
6
8
|
|
|
7
9
|
function buildUrl(baseUrl: string | undefined, path: string, query?: Record<string, string>): string {
|
|
8
10
|
let url = baseUrl ? `${baseUrl.replace(/\/+$/, "")}${path}` : path;
|
|
@@ -38,7 +40,61 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
38
40
|
follow_redirects: suite.config.follow_redirects,
|
|
39
41
|
};
|
|
40
42
|
|
|
41
|
-
|
|
43
|
+
// Expand steps lazily (for_each needs current variables)
|
|
44
|
+
let stepIndex = 0;
|
|
45
|
+
const rawSteps = [...suite.tests];
|
|
46
|
+
|
|
47
|
+
while (stepIndex < rawSteps.length) {
|
|
48
|
+
const step = rawSteps[stepIndex]!;
|
|
49
|
+
stepIndex++;
|
|
50
|
+
|
|
51
|
+
// Expand for_each: insert expanded steps and skip current
|
|
52
|
+
if (step.for_each) {
|
|
53
|
+
const resolvedIn = substituteDeep(step.for_each.in, variables);
|
|
54
|
+
const items = Array.isArray(resolvedIn) ? resolvedIn : [];
|
|
55
|
+
const expanded: TestStep[] = [];
|
|
56
|
+
for (const item of items) {
|
|
57
|
+
const { for_each: _, ...rest } = step;
|
|
58
|
+
expanded.push({ ...rest, name: `${step.name} [${step.for_each.var}=${JSON.stringify(item)}]` } as TestStep);
|
|
59
|
+
// We'll inject the variable right before executing each expanded step
|
|
60
|
+
// Store the var assignment via a set field
|
|
61
|
+
}
|
|
62
|
+
// Insert expanded steps at current position
|
|
63
|
+
rawSteps.splice(stepIndex, 0, ...expanded);
|
|
64
|
+
// Set the for_each variable for each expanded step
|
|
65
|
+
for (let i = 0; i < items.length; i++) {
|
|
66
|
+
const expandedStep = rawSteps[stepIndex + i]!;
|
|
67
|
+
// Temporarily inject into variables when we reach this step
|
|
68
|
+
// We need a way to pass the variable — use a hidden _for_each_vars
|
|
69
|
+
(expandedStep as Record<string, unknown>).__for_each_var = { key: step.for_each.var, value: items[i] };
|
|
70
|
+
}
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Inject for_each variable if present
|
|
75
|
+
const forEachData = (step as Record<string, unknown>).__for_each_var as { key: string; value: unknown } | undefined;
|
|
76
|
+
if (forEachData) {
|
|
77
|
+
variables[forEachData.key] = forEachData.value;
|
|
78
|
+
delete (step as Record<string, unknown>).__for_each_var;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Handle set-only steps (no HTTP request)
|
|
82
|
+
if (step.set && step.path === "") {
|
|
83
|
+
for (const [key, rawDirective] of Object.entries(step.set)) {
|
|
84
|
+
const substituted = substituteDeep(rawDirective, variables);
|
|
85
|
+
variables[key] = applyTransform(substituted);
|
|
86
|
+
}
|
|
87
|
+
steps.push({
|
|
88
|
+
name: step.name,
|
|
89
|
+
status: "pass",
|
|
90
|
+
duration_ms: 0,
|
|
91
|
+
request: { method: "", url: "", headers: {} },
|
|
92
|
+
assertions: [],
|
|
93
|
+
captures: {},
|
|
94
|
+
});
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
|
|
42
98
|
// Skip check: if step references a failed capture variable, skip it
|
|
43
99
|
const referencedVars = extractVariableReferences(step);
|
|
44
100
|
const missingCapture = referencedVars.find((v) => failedCaptures.has(v));
|
|
@@ -47,6 +103,15 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
47
103
|
continue;
|
|
48
104
|
}
|
|
49
105
|
|
|
106
|
+
// skip_if evaluation
|
|
107
|
+
if (step.skip_if) {
|
|
108
|
+
const exprAfterSubst = String(substituteString(step.skip_if, variables));
|
|
109
|
+
if (evaluateExpr(exprAfterSubst)) {
|
|
110
|
+
steps.push(makeSkippedResult(step.name, `Skipped: ${step.skip_if}`));
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
50
115
|
// Substitute variables
|
|
51
116
|
const resolved = substituteStep(step, variables);
|
|
52
117
|
|
|
@@ -104,6 +169,59 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
104
169
|
continue;
|
|
105
170
|
}
|
|
106
171
|
|
|
172
|
+
// retry_until wrapper
|
|
173
|
+
if (step.retry_until) {
|
|
174
|
+
const rt = step.retry_until;
|
|
175
|
+
let lastStepResult: StepResult | undefined;
|
|
176
|
+
for (let attempt = 0; attempt < rt.max_attempts; attempt++) {
|
|
177
|
+
try {
|
|
178
|
+
const response = await executeRequest(request, fetchOptions);
|
|
179
|
+
const captures = extractCaptures(resolved.expect.body, response.body_parsed);
|
|
180
|
+
const assertions = checkAssertions(resolved.expect, response);
|
|
181
|
+
const allPassed = assertions.every((a) => a.passed);
|
|
182
|
+
|
|
183
|
+
lastStepResult = {
|
|
184
|
+
name: step.name,
|
|
185
|
+
status: allPassed ? "pass" : "fail",
|
|
186
|
+
duration_ms: response.duration_ms,
|
|
187
|
+
request,
|
|
188
|
+
response,
|
|
189
|
+
assertions,
|
|
190
|
+
captures,
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// Evaluate condition with response context
|
|
194
|
+
const condVars: Record<string, unknown> = { ...variables, ...captures, status: response.status };
|
|
195
|
+
if (response.body_parsed && typeof response.body_parsed === "object") {
|
|
196
|
+
for (const [k, v] of Object.entries(response.body_parsed as Record<string, unknown>)) {
|
|
197
|
+
condVars[k] = v;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
const condStr = String(substituteString(rt.condition, condVars));
|
|
201
|
+
if (evaluateExpr(condStr)) {
|
|
202
|
+
Object.assign(variables, captures);
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (attempt < rt.max_attempts - 1) {
|
|
207
|
+
await new Promise((resolve) => setTimeout(resolve, rt.delay_ms));
|
|
208
|
+
}
|
|
209
|
+
} catch (err) {
|
|
210
|
+
lastStepResult = {
|
|
211
|
+
name: step.name,
|
|
212
|
+
status: "error",
|
|
213
|
+
duration_ms: 0,
|
|
214
|
+
request,
|
|
215
|
+
assertions: [],
|
|
216
|
+
captures: {},
|
|
217
|
+
error: err instanceof Error ? err.message : String(err),
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (lastStepResult) steps.push(lastStepResult);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
|
|
107
225
|
try {
|
|
108
226
|
const response = await executeRequest(request, fetchOptions);
|
|
109
227
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const OPERATORS = ["!=", "==", ">=", "<=", ">", "<"] as const;
|
|
2
|
+
|
|
3
|
+
export function evaluateExpr(expr: string): boolean {
|
|
4
|
+
const trimmed = expr.trim();
|
|
5
|
+
if (trimmed === "") return false;
|
|
6
|
+
|
|
7
|
+
for (const op of OPERATORS) {
|
|
8
|
+
const idx = trimmed.indexOf(op);
|
|
9
|
+
if (idx !== -1) {
|
|
10
|
+
const left = trimmed.slice(0, idx).trim();
|
|
11
|
+
const right = trimmed.slice(idx + op.length).trim();
|
|
12
|
+
return compareValues(left, right, op);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// No operator — truthiness
|
|
17
|
+
return isTruthy(trimmed);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function compareValues(left: string, right: string, op: string): boolean {
|
|
21
|
+
const lNum = Number(left);
|
|
22
|
+
const rNum = Number(right);
|
|
23
|
+
const numeric = !isNaN(lNum) && !isNaN(rNum) && left !== "" && right !== "";
|
|
24
|
+
|
|
25
|
+
switch (op) {
|
|
26
|
+
case "==": return numeric ? lNum === rNum : left === right;
|
|
27
|
+
case "!=": return numeric ? lNum !== rNum : left !== right;
|
|
28
|
+
case ">": return numeric ? lNum > rNum : left > right;
|
|
29
|
+
case "<": return numeric ? lNum < rNum : left < right;
|
|
30
|
+
case ">=": return numeric ? lNum >= rNum : left >= right;
|
|
31
|
+
case "<=": return numeric ? lNum <= rNum : left <= right;
|
|
32
|
+
default: return false;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function isTruthy(value: string): boolean {
|
|
37
|
+
if (value === "" || value === "0" || value === "false" || value === "null" || value === "undefined") {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
const DIRECTIVES = new Set(["concat", "append", "length", "get", "first", "map_field"]);
|
|
2
|
+
|
|
3
|
+
export function applyTransform(directive: unknown): unknown {
|
|
4
|
+
if (typeof directive !== "object" || directive === null || Array.isArray(directive)) {
|
|
5
|
+
return directive;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const obj = directive as Record<string, unknown>;
|
|
9
|
+
const keys = Object.keys(obj);
|
|
10
|
+
|
|
11
|
+
if (keys.length !== 1 || !DIRECTIVES.has(keys[0]!)) {
|
|
12
|
+
return directive;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const op = keys[0]!;
|
|
16
|
+
const arg = obj[op];
|
|
17
|
+
|
|
18
|
+
switch (op) {
|
|
19
|
+
case "concat": {
|
|
20
|
+
if (!Array.isArray(arg)) return directive;
|
|
21
|
+
const result: unknown[] = [];
|
|
22
|
+
for (const item of arg) {
|
|
23
|
+
if (Array.isArray(item)) result.push(...item);
|
|
24
|
+
else result.push(item);
|
|
25
|
+
}
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
case "append": {
|
|
29
|
+
if (!Array.isArray(arg) || arg.length < 2) return directive;
|
|
30
|
+
const arr = Array.isArray(arg[0]) ? [...arg[0]] : [];
|
|
31
|
+
return [...arr, ...arg.slice(1)];
|
|
32
|
+
}
|
|
33
|
+
case "length": {
|
|
34
|
+
if (Array.isArray(arg)) return arg.length;
|
|
35
|
+
if (typeof arg === "string") return arg.length;
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
case "get": {
|
|
39
|
+
if (!Array.isArray(arg) || arg.length < 2) return directive;
|
|
40
|
+
const [source, index] = arg;
|
|
41
|
+
if (Array.isArray(source) && typeof index === "number") return source[index];
|
|
42
|
+
if (typeof source === "object" && source !== null && typeof index === "string") {
|
|
43
|
+
return (source as Record<string, unknown>)[index];
|
|
44
|
+
}
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
case "first": {
|
|
48
|
+
if (Array.isArray(arg)) return arg[0];
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
case "map_field": {
|
|
52
|
+
if (!Array.isArray(arg) || arg.length < 2) return directive;
|
|
53
|
+
const [items, field] = arg;
|
|
54
|
+
if (!Array.isArray(items) || typeof field !== "string") return directive;
|
|
55
|
+
return items.map((item) => {
|
|
56
|
+
if (typeof item === "object" && item !== null) {
|
|
57
|
+
return (item as Record<string, unknown>)[field];
|
|
58
|
+
}
|
|
59
|
+
return undefined;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
default:
|
|
63
|
+
return directive;
|
|
64
|
+
}
|
|
65
|
+
}
|