omniqa 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +3 -2
- package/dist/cli.js.map +1 -1
- package/dist/commands/coverage.d.ts +13 -0
- package/dist/commands/coverage.d.ts.map +1 -1
- package/dist/commands/coverage.js +35 -6
- package/dist/commands/coverage.js.map +1 -1
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +6 -2
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/write.d.ts +7 -3
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +11 -6
- package/dist/commands/write.js.map +1 -1
- package/dist/coverage-page.d.ts +6 -2
- package/dist/coverage-page.d.ts.map +1 -1
- package/dist/coverage-page.js +110 -44
- package/dist/coverage-page.js.map +1 -1
- package/dist/mcp/tools.js +1 -1
- package/dist/mcp/tools.js.map +1 -1
- package/dist/page.d.ts +20 -5
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +73 -21
- package/dist/page.js.map +1 -1
- package/dist/plan-page.d.ts +8 -4
- package/dist/plan-page.d.ts.map +1 -1
- package/dist/plan-page.js +134 -32
- package/dist/plan-page.js.map +1 -1
- package/dist/planner.d.ts +11 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +28 -1
- package/dist/planner.js.map +1 -1
- package/dist/writer.d.ts +23 -5
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +62 -27
- package/dist/writer.js.map +1 -1
- package/package.json +3 -3
- package/skills/review-plan.md +3 -2
- package/skills/review-tests.md +4 -4
- package/skills/test-plan.md +3 -2
- package/skills/write-tests.md +6 -3
package/dist/writer.js
CHANGED
|
@@ -65,12 +65,13 @@ export function renderE2eTest(c, steps, fixme) {
|
|
|
65
65
|
const SHORTHAND = new Set(["get", "post", "put", "patch", "delete", "head"]);
|
|
66
66
|
/** `{ "email": fields["email"] }`: request fields read from the account at run time. */
|
|
67
67
|
const fieldsExpr = (fields) => `{ ${Object.entries(fields).map(([k, v]) => `${q(k)}: fields[${q(v)}]`).join(", ")} }`;
|
|
68
|
-
/** A request on the api fixture: its JSON body, the account's fields merged over it. */
|
|
69
|
-
function apiCall(r) {
|
|
68
|
+
/** A request on the api fixture: its JSON body, the account's fields merged over it; `asIs` keeps a redirect as the answer (maxRedirects: 0). */
|
|
69
|
+
function apiCall(r, asIs = false) {
|
|
70
70
|
const method = r.method.toLowerCase();
|
|
71
71
|
const overrides = r.fields === undefined ? undefined : fieldsExpr(r.fields);
|
|
72
72
|
const data = overrides === undefined ? r.body : r.body === undefined ? overrides : `{ ...${r.body}, ${overrides.slice(2, -2)} }`;
|
|
73
|
-
|
|
73
|
+
const options = [...(data === undefined ? [] : [`data: ${data}`]), ...(asIs ? ["maxRedirects: 0"] : [])];
|
|
74
|
+
return SHORTHAND.has(method) ? `api.${method}(${q(r.path)}${options.length === 0 ? "" : `, { ${options.join(", ")} }`})` : `api.fetch(${q(r.path)}, { ${[`method: ${q(r.method)}`, ...options].join(", ")} })`;
|
|
74
75
|
}
|
|
75
76
|
/** The cookie checks after a request: the names it set are on the context, the ones it cleared are not. */
|
|
76
77
|
const cookieChecks = (cookies, set = [], cleared = []) => [...(set.length === 0 ? [] : [` expect(${cookies}).toEqual(expect.arrayContaining([${set.map(q).join(", ")}]));`]), ...cleared.map((name) => ` expect(${cookies}).not.toContain(${q(name)});`)];
|
|
@@ -81,8 +82,9 @@ export function renderApiTest(c, spec, fixme) {
|
|
|
81
82
|
...(spec.fields === undefined && !before.some((b) => b.fields !== undefined) ? [] : [` const fields = accountFields(${q(spec.account ?? spec.actor)});`]),
|
|
82
83
|
// The before requests are plain steps: each request and its observed status, then the cookies it set or cleared.
|
|
83
84
|
...before.flatMap((b) => [` expect((await ${apiCall(b)}).status()).toBe(${b.status});`, ...cookieChecks("(await api.storageState()).cookies.map((c) => c.name)", b.cookiesSet, b.cookiesCleared)]),
|
|
84
|
-
spec.signUp === true ? ` const { response } = await apiSignUp(api${spec.fields === undefined ? "" : `, ${fieldsExpr(spec.fields)}`});` : ` const response = await ${apiCall(spec)};`,
|
|
85
|
+
spec.signUp === true ? ` const { response } = await apiSignUp(api${spec.fields === undefined ? "" : `, ${fieldsExpr(spec.fields)}`});` : ` const response = await ${apiCall(spec, spec.redirectsTo !== undefined)};`,
|
|
85
86
|
` expect(response.status()).toBe(${spec.status});`,
|
|
87
|
+
...(spec.redirectsTo === undefined ? [] : [` expect(response.headers()["location"]).toMatch(${regexOf(spec.redirectsTo)});`]),
|
|
86
88
|
...(spec.contentType === undefined ? [] : [` expect(response.headers()["content-type"]).toMatch(${regexOf(spec.contentType)});`]),
|
|
87
89
|
...(spec.keys === undefined ? [] : [` expect(Object.keys(await response.json()).toSorted()).toEqual([${[...spec.keys].toSorted().map(q).join(", ")}]);`]),
|
|
88
90
|
...Object.entries(spec.values ?? {}).map(([k, v]) => ` expect((await response.json())[${q(k)}]).toContain(${q(v)});`),
|
|
@@ -292,7 +294,12 @@ async function openApi(baseUrl, o) {
|
|
|
292
294
|
}
|
|
293
295
|
/** A request's JSON body: the literal it was given, the account's values merged over it. */
|
|
294
296
|
const dataOf = (r) => (r.body === undefined && r.extra === undefined ? undefined : { ...(r.body === undefined ? {} : JSON.parse(r.body)), ...r.extra });
|
|
295
|
-
/**
|
|
297
|
+
/**
|
|
298
|
+
* One request as the actor (signed in through the login spec first when given): the status, the content type, the top-level keys and the cookie names; never the body.
|
|
299
|
+
* A redirect is the answer and is not followed, so a guest sent to the login page is observed turned away (302, where it points) instead of as the login page's 200;
|
|
300
|
+
* through a page (`mode: page`) the redirect is followed, as a page's fetch hides it. The login, the before requests and a sign-up follow theirs.
|
|
301
|
+
* ponytail: where a redirect points is the path of its Location; one to another host is told by its path alone.
|
|
302
|
+
*/
|
|
296
303
|
export async function captureApi(baseUrl, spec, login, o = {}) {
|
|
297
304
|
const { api, close } = await openApi(baseUrl, o);
|
|
298
305
|
const redact = o.redact ?? ((s) => s);
|
|
@@ -319,7 +326,7 @@ export async function captureApi(baseUrl, spec, login, o = {}) {
|
|
|
319
326
|
const before = await names();
|
|
320
327
|
const data = dataOf(spec);
|
|
321
328
|
let signed;
|
|
322
|
-
const response = o.signUp === undefined ? await api.fetch(spec.path, { method: spec.method, ...(data === undefined ? {} : { data }) }) : (signed = await signUpWith(api, o.signUp.spec, o.signUp.overrides, o.signUp.where)).response;
|
|
329
|
+
const response = o.signUp === undefined ? await api.fetch(spec.path, { method: spec.method, ...(data === undefined ? {} : { data }), ...(o.mode === "page" ? {} : { maxRedirects: 0 }) }) : (signed = await signUpWith(api, o.signUp.spec, o.signUp.overrides, o.signUp.where)).response;
|
|
323
330
|
// A sign-up's generated credentials never reach an excerpt either.
|
|
324
331
|
const hide = redactor(signed === undefined ? [] : [signed.email, signed.password, signed.username]);
|
|
325
332
|
const show = (s) => hide(redact(s));
|
|
@@ -344,7 +351,10 @@ export async function captureApi(baseUrl, spec, login, o = {}) {
|
|
|
344
351
|
}
|
|
345
352
|
const after = await names();
|
|
346
353
|
const [teardown] = await tearDown(api);
|
|
347
|
-
|
|
354
|
+
const requested = new URL(spec.path, baseUrl);
|
|
355
|
+
const location = response.status() >= 300 && response.status() < 400 ? response.headers()["location"] : undefined;
|
|
356
|
+
const redirectsTo = location !== undefined && URL.canParse(location, requested) ? new URL(location, requested).pathname : undefined;
|
|
357
|
+
return { status: response.status(), ...(redirectsTo === undefined ? {} : { redirectsTo }), contentType, keys, cookies: after, cookiesSet: after.filter((n) => !before.includes(n)), cookiesCleared: before.filter((n) => !after.includes(n)), excerpt: excerptOf(contentType, text, show), ...(teardown === undefined ? {} : { teardown }), ...(values === undefined ? {} : { values }), ...(prior.length === 0 ? {} : { before: prior }) };
|
|
348
358
|
}
|
|
349
359
|
finally {
|
|
350
360
|
// A request that failed after a sign-up still deletes its account.
|
|
@@ -397,10 +407,17 @@ export async function runCase(cwd, env, o) {
|
|
|
397
407
|
return { code, error: `the run left no report (omniqa test exited ${code}): ${firstLine(err.message)}` };
|
|
398
408
|
}
|
|
399
409
|
}
|
|
400
|
-
/**
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
410
|
+
/**
|
|
411
|
+
* The writer's decisions per round: the knowledge reads, the session steps, write_test. A flow of ten steps takes about 25 and one of
|
|
412
|
+
* twenty about 40, so a round ends on its own well before this; the case's cap (CASE_BUDGET) is the limit that binds a runaway round.
|
|
413
|
+
*/
|
|
414
|
+
export const WRITER_BUDGET = 48;
|
|
415
|
+
/**
|
|
416
|
+
* Model calls one case may spend in all (counted as WriteResult.calls: the writer, the session, the first runs, the reviewer); at the
|
|
417
|
+
* cap the case ends (needs-human with the last draft the reviewer judged, else failed) and the run goes on. A written case costs about 35 and a round that writes nothing up to 60, but a flow of ten
|
|
418
|
+
* steps costs about 70 a round once its first run plans every step again (agentic runs): the cap leaves such a flow a second round.
|
|
419
|
+
*/
|
|
420
|
+
export const CASE_BUDGET = 160;
|
|
404
421
|
/** Executor actions per session step: one intent is one to three actions; a looping executor is cut off here, not at the agent's 25 (seen on the stage: 25 calls on one step, 84 in a round). */
|
|
405
422
|
export const SESSION_STEP_BUDGET = 8;
|
|
406
423
|
export const WRITER_MAX_TOKENS = 8_000;
|
|
@@ -509,9 +526,11 @@ const detail = (ob) => (ob.excerpt === "" ? "" : `; the answer: ${ob.excerpt}`);
|
|
|
509
526
|
const cookieChanges = (set, cleared) => `${(set?.length ?? 0) === 0 ? "" : `; sets cookies ${set.join(", ")}`}${(cleared?.length ?? 0) === 0 ? "" : `; clears cookies ${cleared.join(", ")}`}`;
|
|
510
527
|
/** `; error contains "…"`: an error answer's asserted values, for the round line and the reviewer. */
|
|
511
528
|
const valueChecks = (values) => Object.entries(values ?? {}).map(([k, v]) => `; ${k} contains ${q(v)}`).join("");
|
|
529
|
+
/** `, redirects to /login`: where a redirect the request did not follow points, for the round line and the reviewer. */
|
|
530
|
+
const redirect = (to) => (to === undefined ? "" : `, redirects to ${to}`);
|
|
512
531
|
/** `POST /login → 200; sets cookies user; then `: the before requests, for the round line and the reviewer. */
|
|
513
532
|
const priorRequests = (spec) => (spec.before ?? []).map((b) => `${b.method} ${b.path} → ${b.status}${cookieChanges(b.cookiesSet, b.cookiesCleared)}; then `).join("");
|
|
514
|
-
const describeObserved = (spec, ob) => `${priorRequests(spec)}${spec.method.toUpperCase()} ${spec.path} as ${spec.actor}${spec.signUp === true ? (spec.fields === undefined ? " (a fresh sign-up)" : ` (a sign-up with ${spec.account}'s ${Object.values(spec.fields).join(", ")})`) : ""}: ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : `, keys ${ob.keys.join(", ")}`}${cookieChanges(ob.cookiesSet, ob.cookiesCleared)}${valueChecks(spec.values)}`;
|
|
533
|
+
const describeObserved = (spec, ob) => `${priorRequests(spec)}${spec.method.toUpperCase()} ${spec.path} as ${spec.actor}${spec.signUp === true ? (spec.fields === undefined ? " (a fresh sign-up)" : ` (a sign-up with ${spec.account}'s ${Object.values(spec.fields).join(", ")})`) : ""}: ${ob.status} ${ob.contentType || "no content type"}${redirect(ob.redirectsTo)}${ob.keys.length === 0 ? "" : `, keys ${ob.keys.join(", ")}`}${cookieChanges(ob.cookiesSet, ob.cookiesCleared)}${valueChecks(spec.values)}`;
|
|
515
534
|
/** A request's body must be a JSON object literal. */
|
|
516
535
|
function refuseBody(label, b) {
|
|
517
536
|
if (b === undefined)
|
|
@@ -594,6 +613,8 @@ export async function citedKnowledge(knowledgeDir, refs) {
|
|
|
594
613
|
}
|
|
595
614
|
return out;
|
|
596
615
|
}
|
|
616
|
+
/** The accounts a `login` step, open_session's `account` and an API actor can name, for the writer's task and the reviewer's facts: an actor product.yaml lacks signs in as the case states it, typed. */
|
|
617
|
+
export const accountsLine = (product) => `Accounts of product.yaml: ${Object.keys(product.accounts).join(", ") || "none"}`;
|
|
597
618
|
export function caseFacts(s) {
|
|
598
619
|
const c = s.case;
|
|
599
620
|
return [
|
|
@@ -948,7 +969,7 @@ export async function createWriter(o) {
|
|
|
948
969
|
if (b.status >= 400)
|
|
949
970
|
throw new Error(`refused: before[${i + 1}] ${before[i].method} ${before[i].path} answered ${b.status}${b.excerpt === "" ? "" : `; the answer: ${b.excerpt}`}; a request before the main one prepares it and must succeed`);
|
|
950
971
|
if (ob.status !== expected)
|
|
951
|
-
throw new Error(`refused: ${method} ${path} as ${actor} answered ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : ` (keys: ${ob.keys.join(", ")})`}, not ${expected}${detail(ob)}; use the observed status or change the request`);
|
|
972
|
+
throw new Error(`refused: ${method} ${path} as ${actor} answered ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : ` (keys: ${ob.keys.join(", ")})`}${ob.redirectsTo === undefined ? "" : `, a redirect to ${ob.redirectsTo}`}, not ${expected}${detail(ob)}; use the observed status or change the request`);
|
|
952
973
|
// An error answer's values: the observed text, or the part the writer picked, never an account value, an id or a token.
|
|
953
974
|
let asserted;
|
|
954
975
|
if (values !== undefined) {
|
|
@@ -968,7 +989,7 @@ export async function createWriter(o) {
|
|
|
968
989
|
asserted[k] = text;
|
|
969
990
|
}
|
|
970
991
|
}
|
|
971
|
-
const spec = { method, path, actor, ...(body === undefined ? {} : { body }), ...(viaSignUp ? { signUp: true } : {}), ...(account === undefined || (fields === undefined && !before.some((b) => b.fields !== undefined)) ? {} : { account }), ...(fields === undefined ? {} : { fields }), ...(before.length === 0 ? {} : { before: before.map((b, i) => ({ method: b.method, path: b.path, ...(b.body === undefined ? {} : { body: b.body }), ...(b.fields === undefined ? {} : { fields: b.fields }), status: answered[i].status, ...(answered[i].cookiesSet.length === 0 ? {} : { cookiesSet: answered[i].cookiesSet }), ...(answered[i].cookiesCleared.length === 0 ? {} : { cookiesCleared: answered[i].cookiesCleared }) })) }), status: ob.status, ...(ob.contentType === "" ? {} : { contentType: ob.contentType }), ...(ob.keys.length === 0 ? {} : { keys: ob.keys }), ...(ob.cookiesSet.length === 0 ? {} : { cookiesSet: ob.cookiesSet }), ...(ob.cookiesCleared.length === 0 ? {} : { cookiesCleared: ob.cookiesCleared }), ...(asserted === undefined ? {} : { values: asserted }) };
|
|
992
|
+
const spec = { method, path, actor, ...(body === undefined ? {} : { body }), ...(viaSignUp ? { signUp: true } : {}), ...(account === undefined || (fields === undefined && !before.some((b) => b.fields !== undefined)) ? {} : { account }), ...(fields === undefined ? {} : { fields }), ...(before.length === 0 ? {} : { before: before.map((b, i) => ({ method: b.method, path: b.path, ...(b.body === undefined ? {} : { body: b.body }), ...(b.fields === undefined ? {} : { fields: b.fields }), status: answered[i].status, ...(answered[i].cookiesSet.length === 0 ? {} : { cookiesSet: answered[i].cookiesSet }), ...(answered[i].cookiesCleared.length === 0 ? {} : { cookiesCleared: answered[i].cookiesCleared }) })) }), status: ob.status, ...(ob.redirectsTo === undefined ? {} : { redirectsTo: ob.redirectsTo }), ...(ob.contentType === "" ? {} : { contentType: ob.contentType }), ...(ob.keys.length === 0 ? {} : { keys: ob.keys }), ...(ob.cookiesSet.length === 0 ? {} : { cookiesSet: ob.cookiesSet }), ...(ob.cookiesCleared.length === 0 ? {} : { cookiesCleared: ob.cookiesCleared }), ...(asserted === undefined ? {} : { values: asserted }) };
|
|
972
993
|
await writeBlock(c, file, renderApiTest(c, spec), "api");
|
|
973
994
|
const observed = describeObserved(spec, ob);
|
|
974
995
|
draft = { kind: "api", file, rel, block: renderApiTest(c, spec), steps: [], recorded: 0, spec, observed, note: observed };
|
|
@@ -1078,7 +1099,7 @@ export async function createWriter(o) {
|
|
|
1078
1099
|
},
|
|
1079
1100
|
{
|
|
1080
1101
|
name: "write_api_test",
|
|
1081
|
-
description: "Write the case's deterministic API test into tests/api/<suite>.api.ts: the request as the actor (guest, or an account signed in through _auth.ts), the status (default: what api/endpoints.md lists for that actor), and the content type and top-level JSON keys observed when the request is made now; for an error answer, values asserts what one or two top-level fields say (the message of a credentials error): the observed text, or the part of it you pass. A request on the sign-up endpoint of _auth.ts goes through apiSignUp with generated credentials (pass status); fields with account send an account's values (its email for the already-registered case) read at run time, never written. before makes requests first, in order, on the test's own context, and the main request carries their cookies: the login itself when the case is about it (POST /login with fields and account), an item put in the cart before the cart is read; each is asserted on its observed status and the cookies it set or cleared, and since they change what the main request answers, pass status. Refused when a before request answers 4xx or 5xx, when the observed status differs, when a mutating method is used by a read-only case or on a read-only environment, when a value is missing, absent from the answer or carries an account value, an id or a token, or when a test of that title exists elsewhere. Placeholders in a path ({n}, {id}) must be concrete values.",
|
|
1102
|
+
description: "Write the case's deterministic API test into tests/api/<suite>.api.ts: the request as the actor (guest, or an account signed in through _auth.ts), the status (default: what api/endpoints.md lists for that actor), and the content type and top-level JSON keys observed when the request is made now; a redirect is the answer and is not followed: the test asserts its own status and the path of its Location (pass the redirect's status; where requests go through a page, api.mode page, redirects are followed); for an error answer, values asserts what one or two top-level fields say (the message of a credentials error): the observed text, or the part of it you pass. A request on the sign-up endpoint of _auth.ts goes through apiSignUp with generated credentials (pass status); fields with account send an account's values (its email for the already-registered case) read at run time, never written. before makes requests first, in order, on the test's own context, and the main request carries their cookies: the login itself when the case is about it (POST /login with fields and account), an item put in the cart before the cart is read; each is asserted on its observed status and the cookies it set or cleared, and since they change what the main request answers, pass status. Refused when a before request answers 4xx or 5xx, when the observed status differs, when a mutating method is used by a read-only case or on a read-only environment, when a value is missing, absent from the answer or carries an account value, an id or a token, or when a test of that title exists elsewhere. Placeholders in a path ({n}, {id}) must be concrete values.",
|
|
1082
1103
|
schema: object({ method: str("GET, POST, PUT, PATCH or DELETE"), path: str("A concrete path, e.g. /api/v1/subscription"), actor: str("guest, or a product.yaml account name"), body: str("A JSON object literal to send (POST, PUT, PATCH); empty otherwise"), account: str("With fields (here or in before): the product.yaml account whose values they take (default: the actor)"), fields: { type: "object", description: 'Request field → account field sent from accountFields at run time, e.g. {"email": "email"}' }, status: int("The expected status; default: the one api/endpoints.md lists for the actor"), values: { type: "object", description: 'For an error answer (4xx or 5xx): top-level field → the text it must contain, "" for its whole observed value, e.g. {"error": ""}; one or two fields' }, before: { type: "array", items: object({ method: str("GET, POST, PUT, PATCH or DELETE"), path: str("A concrete same-origin path, e.g. /login"), body: str("A JSON object literal to send; empty otherwise"), fields: { type: "object", description: 'Request field → field of the account named by account, sent from accountFields at run time, e.g. {"email": "email", "password": "password"}' } }, ["method", "path"]), description: "Requests made first, in order, on the same context, e.g. the login a case is about: [{\"method\": \"POST\", \"path\": \"/login\", \"fields\": {\"email\": \"email\", \"password\": \"password\"}}] with account" } }, ["method", "path", "actor"]),
|
|
1083
1104
|
run: writeApiTest,
|
|
1084
1105
|
},
|
|
@@ -1093,6 +1114,7 @@ export async function createWriter(o) {
|
|
|
1093
1114
|
`Write the test for case ${c.id} of suite ${s.suite.slug} (${s.suite.title}; suite tiers ${s.suite.tiers.join(", ")}): ${c.title}`,
|
|
1094
1115
|
`Type ${c.type}; priority ${c.priority}; tiers ${c.tiers.join(", ")}; ${c.mutating ? "mutating" : "read-only"}; preconditions: ${c.preconditions || "none"}.`,
|
|
1095
1116
|
`Knowledge: ${knowledgeLine(s)}. Routes: ${pages.join("; ") || "none"}.`,
|
|
1117
|
+
`${accountsLine(product)}.`,
|
|
1096
1118
|
c.type === "api"
|
|
1097
1119
|
? `The request is made now and on the first run against ${o.envName} (${o.environment.url}), the ${o.environment.tier} tier${o.environment.mutations ? "" : ", which is read-only: no POST, PUT, PATCH or DELETE"}.${c.tiers.includes("production") && !productionWritable ? " The case also runs on the read-only production tier: GET only." : ""}`
|
|
1098
1120
|
: `The session and the first run happen on ${o.envName} (${o.environment.url}), the ${o.environment.tier} tier.${c.tiers.includes("production") && !productionWritable ? " Production is read-only: this case may only follow links and read (a do that clicks a link is fine; no login, no typing, no button)." : ""}`,
|
|
@@ -1145,10 +1167,11 @@ export async function createWriter(o) {
|
|
|
1145
1167
|
const facts = [
|
|
1146
1168
|
caseFacts(s),
|
|
1147
1169
|
productSummary(product).split("\n")[1],
|
|
1170
|
+
accountsLine(product),
|
|
1148
1171
|
...(cited.length === 0 ? [] : [`Knowledge the case cites (at most ${CITED_MAX} characters a section, ${CITED_TOTAL} in all):`, ...cited.map(hide)]),
|
|
1149
1172
|
`Test (${d.rel}):`,
|
|
1150
1173
|
d.block,
|
|
1151
|
-
...(d.kind === "api" ? [`Observed on ${o.envName}: ${priorRequests(d.spec)}${d.spec.method} ${d.spec.path} as ${d.spec.actor} → ${d.spec.status} ${d.spec.contentType ?? "no content type"}; keys: ${d.spec.keys?.join(", ") ?? "none"}${cookieChanges(d.spec.cookiesSet, d.spec.cookiesCleared)}${valueChecks(d.spec.values)}`] : []),
|
|
1174
|
+
...(d.kind === "api" ? [`Observed on ${o.envName}: ${priorRequests(d.spec)}${d.spec.method} ${d.spec.path} as ${d.spec.actor} → ${d.spec.status} ${d.spec.contentType ?? "no content type"}${redirect(d.spec.redirectsTo)}; keys: ${d.spec.keys?.join(", ") ?? "none"}${cookieChanges(d.spec.cookiesSet, d.spec.cookiesCleared)}${valueChecks(d.spec.values)}`] : []),
|
|
1152
1175
|
...helpersWritten.flatMap((h) => [`${h.label} written this round (${AUTH_FILE}): ${h.check}`, h.source.trimEnd()]),
|
|
1153
1176
|
`First run on ${o.envName} (${project}): passed in ${secs(run.ms)}, ${plural(run.calls, "model call")}`,
|
|
1154
1177
|
...(d.kind === "api" ? [] : [`Recorded steps (${relative(cwd, recordingPath(testDir, d.file))}):`, ...(await recordedLines(s.case, d))]),
|
|
@@ -1181,12 +1204,23 @@ export async function createWriter(o) {
|
|
|
1181
1204
|
let round = 0;
|
|
1182
1205
|
let roundLine = "";
|
|
1183
1206
|
let previous;
|
|
1207
|
+
let judged;
|
|
1184
1208
|
const own = suiteFile(testDir, s.suite.slug, c.type);
|
|
1185
1209
|
const sidecar = recordingPath(testDir, own);
|
|
1186
1210
|
// What a stop before the review restores: the file and its recording as they were, so no unapproved draft is left behind.
|
|
1187
1211
|
const untouched = { test: await readFile(own, "utf8").catch(() => undefined), recording: await readFile(sidecar, "utf8").catch(() => undefined) };
|
|
1188
1212
|
const existing = findTest(untouched.test ?? "", c.id) === undefined ? undefined : relative(cwd, own);
|
|
1189
1213
|
const result = (status, extra = {}) => ({ id: c.id, status, rounds: round, calls: by.writer + by.session + by.run + by.reviewer, by, ms: Date.now() - t0, ...extra });
|
|
1214
|
+
/** The case ends for a human: the draft becomes a test.fixme under the reason (it may quote what a run showed and is committed with the test, so it is redacted); the plan row and the state say needs-human. */
|
|
1215
|
+
const forHuman = async (d, reason) => {
|
|
1216
|
+
const feedback = hide(reason);
|
|
1217
|
+
await writeBlock(c, d.file, d.kind === "api" ? renderApiTest(c, d.spec, feedback) : renderE2eTest(c, d.steps, feedback), d.kind);
|
|
1218
|
+
await setStatus(c.id, "needs-human", d.rel);
|
|
1219
|
+
const r = result("needs-human", { file: d.rel, feedback });
|
|
1220
|
+
await recordTests(cwd, { needsHuman: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "needs-human", file: d.rel, reason: feedback } });
|
|
1221
|
+
out(`${c.id}: needs a human: ${firstLine(feedback)} (${d.rel}, ${plural(r.calls, "model call")}, ${secs(r.ms)})`);
|
|
1222
|
+
return r;
|
|
1223
|
+
};
|
|
1190
1224
|
try {
|
|
1191
1225
|
const reviewed = await withReview({
|
|
1192
1226
|
maxRounds: WRITER_ROUNDS,
|
|
@@ -1227,21 +1261,14 @@ export async function createWriter(o) {
|
|
|
1227
1261
|
const used0 = budget.used;
|
|
1228
1262
|
const verdict = await review(s, d, run);
|
|
1229
1263
|
by.reviewer += budget.used - used0;
|
|
1264
|
+
judged = { draft: d, feedback: verdict.feedback || "the reviewer did not approve", test: await readFile(own, "utf8").catch(() => undefined), recording: await readFile(sidecar, "utf8").catch(() => undefined) };
|
|
1230
1265
|
out(` ${c.id} round ${round}: ${roundLine}; first run ${run.passed ? "passed" : "failed"} (${plural(run.calls, "model call")}${run.passed ? "" : `: ${run.error}`}); reviewer ${verdict.needsHuman === true ? "needs a human" : verdict.verdict} — ${firstLine(verdict.feedback || verdict.evidence)}`);
|
|
1231
1266
|
return verdict;
|
|
1232
1267
|
},
|
|
1233
1268
|
});
|
|
1234
1269
|
const d = reviewed.draft;
|
|
1235
|
-
if (reviewed.needsHuman)
|
|
1236
|
-
|
|
1237
|
-
const feedback = hide(reviewed.verdict.feedback || "the reviewer did not approve");
|
|
1238
|
-
await writeBlock(c, d.file, d.kind === "api" ? renderApiTest(c, d.spec, feedback) : renderE2eTest(c, d.steps, feedback), d.kind);
|
|
1239
|
-
await setStatus(c.id, "needs-human", d.rel);
|
|
1240
|
-
const r = result("needs-human", { file: d.rel, feedback });
|
|
1241
|
-
await recordTests(cwd, { needsHuman: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "needs-human", file: d.rel, reason: feedback } });
|
|
1242
|
-
out(`${c.id}: needs a human: ${firstLine(feedback)} (${d.rel}, ${plural(r.calls, "model call")}, ${secs(r.ms)})`);
|
|
1243
|
-
return r;
|
|
1244
|
-
}
|
|
1270
|
+
if (reviewed.needsHuman)
|
|
1271
|
+
return await forHuman(d, reviewed.verdict.feedback || "the reviewer did not approve");
|
|
1245
1272
|
await setStatus(c.id, "written", d.rel);
|
|
1246
1273
|
const r = result("written", { file: d.rel });
|
|
1247
1274
|
await recordTests(cwd, { written: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "written", file: d.rel } });
|
|
@@ -1260,7 +1287,15 @@ export async function createWriter(o) {
|
|
|
1260
1287
|
throw err;
|
|
1261
1288
|
}
|
|
1262
1289
|
const why = firstLine(err.message);
|
|
1263
|
-
// A
|
|
1290
|
+
// A later round that ends without a test (the writer wrote none, the case's cap was reached) keeps the last draft the reviewer
|
|
1291
|
+
// judged, as the rounds running out do: the file and its recording as that draft's first run left them, a later unreviewed draft gone.
|
|
1292
|
+
const kept = judged;
|
|
1293
|
+
if (kept !== undefined) {
|
|
1294
|
+
await restore(own, kept.test);
|
|
1295
|
+
await restore(sidecar, kept.recording);
|
|
1296
|
+
return forHuman(kept.draft, `${why}; the reviewer on the draft kept: ${kept.feedback}`);
|
|
1297
|
+
}
|
|
1298
|
+
// A case that ends before any review leaves no draft: the file and its recording go back to what they were.
|
|
1264
1299
|
if (draft !== undefined || previous !== undefined) {
|
|
1265
1300
|
await restore(own, untouched.test);
|
|
1266
1301
|
await restore(sidecar, untouched.recording);
|