omniqa 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/writer.js CHANGED
@@ -65,12 +65,13 @@ export function renderE2eTest(c, steps, fixme) {
65
65
  const SHORTHAND = new Set(["get", "post", "put", "patch", "delete", "head"]);
66
66
  /** `{ "email": fields["email"] }`: request fields read from the account at run time. */
67
67
  const fieldsExpr = (fields) => `{ ${Object.entries(fields).map(([k, v]) => `${q(k)}: fields[${q(v)}]`).join(", ")} }`;
68
- /** A request on the api fixture: its JSON body, the account's fields merged over it. */
69
- function apiCall(r) {
68
+ /** A request on the api fixture: its JSON body, the account's fields merged over it; `asIs` keeps a redirect as the answer (maxRedirects: 0). */
69
+ function apiCall(r, asIs = false) {
70
70
  const method = r.method.toLowerCase();
71
71
  const overrides = r.fields === undefined ? undefined : fieldsExpr(r.fields);
72
72
  const data = overrides === undefined ? r.body : r.body === undefined ? overrides : `{ ...${r.body}, ${overrides.slice(2, -2)} }`;
73
- return SHORTHAND.has(method) ? `api.${method}(${q(r.path)}${data === undefined ? "" : `, { data: ${data} }`})` : `api.fetch(${q(r.path)}, { method: ${q(r.method)}${data === undefined ? "" : `, data: ${data}`} })`;
73
+ const options = [...(data === undefined ? [] : [`data: ${data}`]), ...(asIs ? ["maxRedirects: 0"] : [])];
74
+ return SHORTHAND.has(method) ? `api.${method}(${q(r.path)}${options.length === 0 ? "" : `, { ${options.join(", ")} }`})` : `api.fetch(${q(r.path)}, { ${[`method: ${q(r.method)}`, ...options].join(", ")} })`;
74
75
  }
75
76
  /** The cookie checks after a request: the names it set are on the context, the ones it cleared are not. */
76
77
  const cookieChecks = (cookies, set = [], cleared = []) => [...(set.length === 0 ? [] : [` expect(${cookies}).toEqual(expect.arrayContaining([${set.map(q).join(", ")}]));`]), ...cleared.map((name) => ` expect(${cookies}).not.toContain(${q(name)});`)];
@@ -81,8 +82,9 @@ export function renderApiTest(c, spec, fixme) {
81
82
  ...(spec.fields === undefined && !before.some((b) => b.fields !== undefined) ? [] : [` const fields = accountFields(${q(spec.account ?? spec.actor)});`]),
82
83
  // The before requests are plain steps: each request and its observed status, then the cookies it set or cleared.
83
84
  ...before.flatMap((b) => [` expect((await ${apiCall(b)}).status()).toBe(${b.status});`, ...cookieChecks("(await api.storageState()).cookies.map((c) => c.name)", b.cookiesSet, b.cookiesCleared)]),
84
- spec.signUp === true ? ` const { response } = await apiSignUp(api${spec.fields === undefined ? "" : `, ${fieldsExpr(spec.fields)}`});` : ` const response = await ${apiCall(spec)};`,
85
+ spec.signUp === true ? ` const { response } = await apiSignUp(api${spec.fields === undefined ? "" : `, ${fieldsExpr(spec.fields)}`});` : ` const response = await ${apiCall(spec, spec.redirectsTo !== undefined)};`,
85
86
  ` expect(response.status()).toBe(${spec.status});`,
87
+ ...(spec.redirectsTo === undefined ? [] : [` expect(response.headers()["location"]).toMatch(${regexOf(spec.redirectsTo)});`]),
86
88
  ...(spec.contentType === undefined ? [] : [` expect(response.headers()["content-type"]).toMatch(${regexOf(spec.contentType)});`]),
87
89
  ...(spec.keys === undefined ? [] : [` expect(Object.keys(await response.json()).toSorted()).toEqual([${[...spec.keys].toSorted().map(q).join(", ")}]);`]),
88
90
  ...Object.entries(spec.values ?? {}).map(([k, v]) => ` expect((await response.json())[${q(k)}]).toContain(${q(v)});`),
@@ -292,7 +294,12 @@ async function openApi(baseUrl, o) {
292
294
  }
293
295
  /** A request's JSON body: the literal it was given, the account's values merged over it. */
294
296
  const dataOf = (r) => (r.body === undefined && r.extra === undefined ? undefined : { ...(r.body === undefined ? {} : JSON.parse(r.body)), ...r.extra });
295
- /** One request as the actor (signed in through the login spec first when given): the status, the content type, the top-level keys and the cookie names; never the body. */
297
+ /**
298
+ * One request as the actor (signed in through the login spec first when given): the status, the content type, the top-level keys and the cookie names; never the body.
299
+ * A redirect is the answer and is not followed, so a guest sent to the login page is observed turned away (302, where it points) instead of as the login page's 200;
300
+ * through a page (`mode: page`) the redirect is followed, as a page's fetch hides it. The login, the before requests and a sign-up follow theirs.
301
+ * ponytail: where a redirect points is the path of its Location; one to another host is told by its path alone.
302
+ */
296
303
  export async function captureApi(baseUrl, spec, login, o = {}) {
297
304
  const { api, close } = await openApi(baseUrl, o);
298
305
  const redact = o.redact ?? ((s) => s);
@@ -319,7 +326,7 @@ export async function captureApi(baseUrl, spec, login, o = {}) {
319
326
  const before = await names();
320
327
  const data = dataOf(spec);
321
328
  let signed;
322
- const response = o.signUp === undefined ? await api.fetch(spec.path, { method: spec.method, ...(data === undefined ? {} : { data }) }) : (signed = await signUpWith(api, o.signUp.spec, o.signUp.overrides, o.signUp.where)).response;
329
+ const response = o.signUp === undefined ? await api.fetch(spec.path, { method: spec.method, ...(data === undefined ? {} : { data }), ...(o.mode === "page" ? {} : { maxRedirects: 0 }) }) : (signed = await signUpWith(api, o.signUp.spec, o.signUp.overrides, o.signUp.where)).response;
323
330
  // A sign-up's generated credentials never reach an excerpt either.
324
331
  const hide = redactor(signed === undefined ? [] : [signed.email, signed.password, signed.username]);
325
332
  const show = (s) => hide(redact(s));
@@ -344,7 +351,10 @@ export async function captureApi(baseUrl, spec, login, o = {}) {
344
351
  }
345
352
  const after = await names();
346
353
  const [teardown] = await tearDown(api);
347
- return { status: response.status(), contentType, keys, cookies: after, cookiesSet: after.filter((n) => !before.includes(n)), cookiesCleared: before.filter((n) => !after.includes(n)), excerpt: excerptOf(contentType, text, show), ...(teardown === undefined ? {} : { teardown }), ...(values === undefined ? {} : { values }), ...(prior.length === 0 ? {} : { before: prior }) };
354
+ const requested = new URL(spec.path, baseUrl);
355
+ const location = response.status() >= 300 && response.status() < 400 ? response.headers()["location"] : undefined;
356
+ const redirectsTo = location !== undefined && URL.canParse(location, requested) ? new URL(location, requested).pathname : undefined;
357
+ return { status: response.status(), ...(redirectsTo === undefined ? {} : { redirectsTo }), contentType, keys, cookies: after, cookiesSet: after.filter((n) => !before.includes(n)), cookiesCleared: before.filter((n) => !after.includes(n)), excerpt: excerptOf(contentType, text, show), ...(teardown === undefined ? {} : { teardown }), ...(values === undefined ? {} : { values }), ...(prior.length === 0 ? {} : { before: prior }) };
348
358
  }
349
359
  finally {
350
360
  // A request that failed after a sign-up still deletes its account.
@@ -397,10 +407,17 @@ export async function runCase(cwd, env, o) {
397
407
  return { code, error: `the run left no report (omniqa test exited ${code}): ${firstLine(err.message)}` };
398
408
  }
399
409
  }
400
- /** The writer's decisions per round: the knowledge reads, the session steps, write_test. */
401
- export const WRITER_BUDGET = 32;
402
- /** Model calls one case may spend in all (counted as WriteResult.calls: the writer, the session, the first runs, the reviewer); at the cap the case fails and the run goes on (a written case costs about 35, a round that writes nothing up to 60). */
403
- export const CASE_BUDGET = 80;
410
+ /**
411
+ * The writer's decisions per round: the knowledge reads, the session steps, write_test. A flow of ten steps takes about 25 and one of
412
+ * twenty about 40, so a round ends on its own well before this; the case's cap (CASE_BUDGET) is the limit that binds a runaway round.
413
+ */
414
+ export const WRITER_BUDGET = 48;
415
+ /**
416
+ * Model calls one case may spend in all (counted as WriteResult.calls: the writer, the session, the first runs, the reviewer); at the
417
+ * cap the case ends (needs-human with the last draft the reviewer judged, else failed) and the run goes on. A written case costs about 35 and a round that writes nothing up to 60, but a flow of ten
418
+ * steps costs about 70 a round once its first run plans every step again (agentic runs): the cap leaves such a flow a second round.
419
+ */
420
+ export const CASE_BUDGET = 160;
404
421
  /** Executor actions per session step: one intent is one to three actions; a looping executor is cut off here, not at the agent's 25 (seen on the stage: 25 calls on one step, 84 in a round). */
405
422
  export const SESSION_STEP_BUDGET = 8;
406
423
  export const WRITER_MAX_TOKENS = 8_000;
@@ -509,9 +526,11 @@ const detail = (ob) => (ob.excerpt === "" ? "" : `; the answer: ${ob.excerpt}`);
509
526
  const cookieChanges = (set, cleared) => `${(set?.length ?? 0) === 0 ? "" : `; sets cookies ${set.join(", ")}`}${(cleared?.length ?? 0) === 0 ? "" : `; clears cookies ${cleared.join(", ")}`}`;
510
527
  /** `; error contains "…"`: an error answer's asserted values, for the round line and the reviewer. */
511
528
  const valueChecks = (values) => Object.entries(values ?? {}).map(([k, v]) => `; ${k} contains ${q(v)}`).join("");
529
+ /** `, redirects to /login`: where a redirect the request did not follow points, for the round line and the reviewer. */
530
+ const redirect = (to) => (to === undefined ? "" : `, redirects to ${to}`);
512
531
  /** `POST /login → 200; sets cookies user; then `: the before requests, for the round line and the reviewer. */
513
532
  const priorRequests = (spec) => (spec.before ?? []).map((b) => `${b.method} ${b.path} → ${b.status}${cookieChanges(b.cookiesSet, b.cookiesCleared)}; then `).join("");
514
- const describeObserved = (spec, ob) => `${priorRequests(spec)}${spec.method.toUpperCase()} ${spec.path} as ${spec.actor}${spec.signUp === true ? (spec.fields === undefined ? " (a fresh sign-up)" : ` (a sign-up with ${spec.account}'s ${Object.values(spec.fields).join(", ")})`) : ""}: ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : `, keys ${ob.keys.join(", ")}`}${cookieChanges(ob.cookiesSet, ob.cookiesCleared)}${valueChecks(spec.values)}`;
533
+ const describeObserved = (spec, ob) => `${priorRequests(spec)}${spec.method.toUpperCase()} ${spec.path} as ${spec.actor}${spec.signUp === true ? (spec.fields === undefined ? " (a fresh sign-up)" : ` (a sign-up with ${spec.account}'s ${Object.values(spec.fields).join(", ")})`) : ""}: ${ob.status} ${ob.contentType || "no content type"}${redirect(ob.redirectsTo)}${ob.keys.length === 0 ? "" : `, keys ${ob.keys.join(", ")}`}${cookieChanges(ob.cookiesSet, ob.cookiesCleared)}${valueChecks(spec.values)}`;
515
534
  /** A request's body must be a JSON object literal. */
516
535
  function refuseBody(label, b) {
517
536
  if (b === undefined)
@@ -594,6 +613,8 @@ export async function citedKnowledge(knowledgeDir, refs) {
594
613
  }
595
614
  return out;
596
615
  }
616
+ /** The accounts a `login` step, open_session's `account` and an API actor can name, for the writer's task and the reviewer's facts: an actor product.yaml lacks signs in as the case states it, typed. */
617
+ export const accountsLine = (product) => `Accounts of product.yaml: ${Object.keys(product.accounts).join(", ") || "none"}`;
597
618
  export function caseFacts(s) {
598
619
  const c = s.case;
599
620
  return [
@@ -948,7 +969,7 @@ export async function createWriter(o) {
948
969
  if (b.status >= 400)
949
970
  throw new Error(`refused: before[${i + 1}] ${before[i].method} ${before[i].path} answered ${b.status}${b.excerpt === "" ? "" : `; the answer: ${b.excerpt}`}; a request before the main one prepares it and must succeed`);
950
971
  if (ob.status !== expected)
951
- throw new Error(`refused: ${method} ${path} as ${actor} answered ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : ` (keys: ${ob.keys.join(", ")})`}, not ${expected}${detail(ob)}; use the observed status or change the request`);
972
+ throw new Error(`refused: ${method} ${path} as ${actor} answered ${ob.status} ${ob.contentType || "no content type"}${ob.keys.length === 0 ? "" : ` (keys: ${ob.keys.join(", ")})`}${ob.redirectsTo === undefined ? "" : `, a redirect to ${ob.redirectsTo}`}, not ${expected}${detail(ob)}; use the observed status or change the request`);
952
973
  // An error answer's values: the observed text, or the part the writer picked, never an account value, an id or a token.
953
974
  let asserted;
954
975
  if (values !== undefined) {
@@ -968,7 +989,7 @@ export async function createWriter(o) {
968
989
  asserted[k] = text;
969
990
  }
970
991
  }
971
- const spec = { method, path, actor, ...(body === undefined ? {} : { body }), ...(viaSignUp ? { signUp: true } : {}), ...(account === undefined || (fields === undefined && !before.some((b) => b.fields !== undefined)) ? {} : { account }), ...(fields === undefined ? {} : { fields }), ...(before.length === 0 ? {} : { before: before.map((b, i) => ({ method: b.method, path: b.path, ...(b.body === undefined ? {} : { body: b.body }), ...(b.fields === undefined ? {} : { fields: b.fields }), status: answered[i].status, ...(answered[i].cookiesSet.length === 0 ? {} : { cookiesSet: answered[i].cookiesSet }), ...(answered[i].cookiesCleared.length === 0 ? {} : { cookiesCleared: answered[i].cookiesCleared }) })) }), status: ob.status, ...(ob.contentType === "" ? {} : { contentType: ob.contentType }), ...(ob.keys.length === 0 ? {} : { keys: ob.keys }), ...(ob.cookiesSet.length === 0 ? {} : { cookiesSet: ob.cookiesSet }), ...(ob.cookiesCleared.length === 0 ? {} : { cookiesCleared: ob.cookiesCleared }), ...(asserted === undefined ? {} : { values: asserted }) };
992
+ const spec = { method, path, actor, ...(body === undefined ? {} : { body }), ...(viaSignUp ? { signUp: true } : {}), ...(account === undefined || (fields === undefined && !before.some((b) => b.fields !== undefined)) ? {} : { account }), ...(fields === undefined ? {} : { fields }), ...(before.length === 0 ? {} : { before: before.map((b, i) => ({ method: b.method, path: b.path, ...(b.body === undefined ? {} : { body: b.body }), ...(b.fields === undefined ? {} : { fields: b.fields }), status: answered[i].status, ...(answered[i].cookiesSet.length === 0 ? {} : { cookiesSet: answered[i].cookiesSet }), ...(answered[i].cookiesCleared.length === 0 ? {} : { cookiesCleared: answered[i].cookiesCleared }) })) }), status: ob.status, ...(ob.redirectsTo === undefined ? {} : { redirectsTo: ob.redirectsTo }), ...(ob.contentType === "" ? {} : { contentType: ob.contentType }), ...(ob.keys.length === 0 ? {} : { keys: ob.keys }), ...(ob.cookiesSet.length === 0 ? {} : { cookiesSet: ob.cookiesSet }), ...(ob.cookiesCleared.length === 0 ? {} : { cookiesCleared: ob.cookiesCleared }), ...(asserted === undefined ? {} : { values: asserted }) };
972
993
  await writeBlock(c, file, renderApiTest(c, spec), "api");
973
994
  const observed = describeObserved(spec, ob);
974
995
  draft = { kind: "api", file, rel, block: renderApiTest(c, spec), steps: [], recorded: 0, spec, observed, note: observed };
@@ -1078,7 +1099,7 @@ export async function createWriter(o) {
1078
1099
  },
1079
1100
  {
1080
1101
  name: "write_api_test",
1081
- description: "Write the case's deterministic API test into tests/api/<suite>.api.ts: the request as the actor (guest, or an account signed in through _auth.ts), the status (default: what api/endpoints.md lists for that actor), and the content type and top-level JSON keys observed when the request is made now; for an error answer, values asserts what one or two top-level fields say (the message of a credentials error): the observed text, or the part of it you pass. A request on the sign-up endpoint of _auth.ts goes through apiSignUp with generated credentials (pass status); fields with account send an account's values (its email for the already-registered case) read at run time, never written. before makes requests first, in order, on the test's own context, and the main request carries their cookies: the login itself when the case is about it (POST /login with fields and account), an item put in the cart before the cart is read; each is asserted on its observed status and the cookies it set or cleared, and since they change what the main request answers, pass status. Refused when a before request answers 4xx or 5xx, when the observed status differs, when a mutating method is used by a read-only case or on a read-only environment, when a value is missing, absent from the answer or carries an account value, an id or a token, or when a test of that title exists elsewhere. Placeholders in a path ({n}, {id}) must be concrete values.",
1102
+ description: "Write the case's deterministic API test into tests/api/<suite>.api.ts: the request as the actor (guest, or an account signed in through _auth.ts), the status (default: what api/endpoints.md lists for that actor), and the content type and top-level JSON keys observed when the request is made now; a redirect is the answer and is not followed: the test asserts its own status and the path of its Location (pass the redirect's status; where requests go through a page, api.mode page, redirects are followed); for an error answer, values asserts what one or two top-level fields say (the message of a credentials error): the observed text, or the part of it you pass. A request on the sign-up endpoint of _auth.ts goes through apiSignUp with generated credentials (pass status); fields with account send an account's values (its email for the already-registered case) read at run time, never written. before makes requests first, in order, on the test's own context, and the main request carries their cookies: the login itself when the case is about it (POST /login with fields and account), an item put in the cart before the cart is read; each is asserted on its observed status and the cookies it set or cleared, and since they change what the main request answers, pass status. Refused when a before request answers 4xx or 5xx, when the observed status differs, when a mutating method is used by a read-only case or on a read-only environment, when a value is missing, absent from the answer or carries an account value, an id or a token, or when a test of that title exists elsewhere. Placeholders in a path ({n}, {id}) must be concrete values.",
1082
1103
  schema: object({ method: str("GET, POST, PUT, PATCH or DELETE"), path: str("A concrete path, e.g. /api/v1/subscription"), actor: str("guest, or a product.yaml account name"), body: str("A JSON object literal to send (POST, PUT, PATCH); empty otherwise"), account: str("With fields (here or in before): the product.yaml account whose values they take (default: the actor)"), fields: { type: "object", description: 'Request field → account field sent from accountFields at run time, e.g. {"email": "email"}' }, status: int("The expected status; default: the one api/endpoints.md lists for the actor"), values: { type: "object", description: 'For an error answer (4xx or 5xx): top-level field → the text it must contain, "" for its whole observed value, e.g. {"error": ""}; one or two fields' }, before: { type: "array", items: object({ method: str("GET, POST, PUT, PATCH or DELETE"), path: str("A concrete same-origin path, e.g. /login"), body: str("A JSON object literal to send; empty otherwise"), fields: { type: "object", description: 'Request field → field of the account named by account, sent from accountFields at run time, e.g. {"email": "email", "password": "password"}' } }, ["method", "path"]), description: "Requests made first, in order, on the same context, e.g. the login a case is about: [{\"method\": \"POST\", \"path\": \"/login\", \"fields\": {\"email\": \"email\", \"password\": \"password\"}}] with account" } }, ["method", "path", "actor"]),
1083
1104
  run: writeApiTest,
1084
1105
  },
@@ -1093,6 +1114,7 @@ export async function createWriter(o) {
1093
1114
  `Write the test for case ${c.id} of suite ${s.suite.slug} (${s.suite.title}; suite tiers ${s.suite.tiers.join(", ")}): ${c.title}`,
1094
1115
  `Type ${c.type}; priority ${c.priority}; tiers ${c.tiers.join(", ")}; ${c.mutating ? "mutating" : "read-only"}; preconditions: ${c.preconditions || "none"}.`,
1095
1116
  `Knowledge: ${knowledgeLine(s)}. Routes: ${pages.join("; ") || "none"}.`,
1117
+ `${accountsLine(product)}.`,
1096
1118
  c.type === "api"
1097
1119
  ? `The request is made now and on the first run against ${o.envName} (${o.environment.url}), the ${o.environment.tier} tier${o.environment.mutations ? "" : ", which is read-only: no POST, PUT, PATCH or DELETE"}.${c.tiers.includes("production") && !productionWritable ? " The case also runs on the read-only production tier: GET only." : ""}`
1098
1120
  : `The session and the first run happen on ${o.envName} (${o.environment.url}), the ${o.environment.tier} tier.${c.tiers.includes("production") && !productionWritable ? " Production is read-only: this case may only follow links and read (a do that clicks a link is fine; no login, no typing, no button)." : ""}`,
@@ -1145,10 +1167,11 @@ export async function createWriter(o) {
1145
1167
  const facts = [
1146
1168
  caseFacts(s),
1147
1169
  productSummary(product).split("\n")[1],
1170
+ accountsLine(product),
1148
1171
  ...(cited.length === 0 ? [] : [`Knowledge the case cites (at most ${CITED_MAX} characters a section, ${CITED_TOTAL} in all):`, ...cited.map(hide)]),
1149
1172
  `Test (${d.rel}):`,
1150
1173
  d.block,
1151
- ...(d.kind === "api" ? [`Observed on ${o.envName}: ${priorRequests(d.spec)}${d.spec.method} ${d.spec.path} as ${d.spec.actor} → ${d.spec.status} ${d.spec.contentType ?? "no content type"}; keys: ${d.spec.keys?.join(", ") ?? "none"}${cookieChanges(d.spec.cookiesSet, d.spec.cookiesCleared)}${valueChecks(d.spec.values)}`] : []),
1174
+ ...(d.kind === "api" ? [`Observed on ${o.envName}: ${priorRequests(d.spec)}${d.spec.method} ${d.spec.path} as ${d.spec.actor} → ${d.spec.status} ${d.spec.contentType ?? "no content type"}${redirect(d.spec.redirectsTo)}; keys: ${d.spec.keys?.join(", ") ?? "none"}${cookieChanges(d.spec.cookiesSet, d.spec.cookiesCleared)}${valueChecks(d.spec.values)}`] : []),
1152
1175
  ...helpersWritten.flatMap((h) => [`${h.label} written this round (${AUTH_FILE}): ${h.check}`, h.source.trimEnd()]),
1153
1176
  `First run on ${o.envName} (${project}): passed in ${secs(run.ms)}, ${plural(run.calls, "model call")}`,
1154
1177
  ...(d.kind === "api" ? [] : [`Recorded steps (${relative(cwd, recordingPath(testDir, d.file))}):`, ...(await recordedLines(s.case, d))]),
@@ -1181,12 +1204,23 @@ export async function createWriter(o) {
1181
1204
  let round = 0;
1182
1205
  let roundLine = "";
1183
1206
  let previous;
1207
+ let judged;
1184
1208
  const own = suiteFile(testDir, s.suite.slug, c.type);
1185
1209
  const sidecar = recordingPath(testDir, own);
1186
1210
  // What a stop before the review restores: the file and its recording as they were, so no unapproved draft is left behind.
1187
1211
  const untouched = { test: await readFile(own, "utf8").catch(() => undefined), recording: await readFile(sidecar, "utf8").catch(() => undefined) };
1188
1212
  const existing = findTest(untouched.test ?? "", c.id) === undefined ? undefined : relative(cwd, own);
1189
1213
  const result = (status, extra = {}) => ({ id: c.id, status, rounds: round, calls: by.writer + by.session + by.run + by.reviewer, by, ms: Date.now() - t0, ...extra });
1214
+ /** The case ends for a human: the draft becomes a test.fixme under the reason (it may quote what a run showed and is committed with the test, so it is redacted); the plan row and the state say needs-human. */
1215
+ const forHuman = async (d, reason) => {
1216
+ const feedback = hide(reason);
1217
+ await writeBlock(c, d.file, d.kind === "api" ? renderApiTest(c, d.spec, feedback) : renderE2eTest(c, d.steps, feedback), d.kind);
1218
+ await setStatus(c.id, "needs-human", d.rel);
1219
+ const r = result("needs-human", { file: d.rel, feedback });
1220
+ await recordTests(cwd, { needsHuman: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "needs-human", file: d.rel, reason: feedback } });
1221
+ out(`${c.id}: needs a human: ${firstLine(feedback)} (${d.rel}, ${plural(r.calls, "model call")}, ${secs(r.ms)})`);
1222
+ return r;
1223
+ };
1190
1224
  try {
1191
1225
  const reviewed = await withReview({
1192
1226
  maxRounds: WRITER_ROUNDS,
@@ -1227,21 +1261,14 @@ export async function createWriter(o) {
1227
1261
  const used0 = budget.used;
1228
1262
  const verdict = await review(s, d, run);
1229
1263
  by.reviewer += budget.used - used0;
1264
+ judged = { draft: d, feedback: verdict.feedback || "the reviewer did not approve", test: await readFile(own, "utf8").catch(() => undefined), recording: await readFile(sidecar, "utf8").catch(() => undefined) };
1230
1265
  out(` ${c.id} round ${round}: ${roundLine}; first run ${run.passed ? "passed" : "failed"} (${plural(run.calls, "model call")}${run.passed ? "" : `: ${run.error}`}); reviewer ${verdict.needsHuman === true ? "needs a human" : verdict.verdict} — ${firstLine(verdict.feedback || verdict.evidence)}`);
1231
1266
  return verdict;
1232
1267
  },
1233
1268
  });
1234
1269
  const d = reviewed.draft;
1235
- if (reviewed.needsHuman) {
1236
- // The feedback may quote what a run showed: the fixme comment is committed with the test, so it is redacted.
1237
- const feedback = hide(reviewed.verdict.feedback || "the reviewer did not approve");
1238
- await writeBlock(c, d.file, d.kind === "api" ? renderApiTest(c, d.spec, feedback) : renderE2eTest(c, d.steps, feedback), d.kind);
1239
- await setStatus(c.id, "needs-human", d.rel);
1240
- const r = result("needs-human", { file: d.rel, feedback });
1241
- await recordTests(cwd, { needsHuman: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "needs-human", file: d.rel, reason: feedback } });
1242
- out(`${c.id}: needs a human: ${firstLine(feedback)} (${d.rel}, ${plural(r.calls, "model call")}, ${secs(r.ms)})`);
1243
- return r;
1244
- }
1270
+ if (reviewed.needsHuman)
1271
+ return await forHuman(d, reviewed.verdict.feedback || "the reviewer did not approve");
1245
1272
  await setStatus(c.id, "written", d.rel);
1246
1273
  const r = result("written", { file: d.rel });
1247
1274
  await recordTests(cwd, { written: 1, modelCalls: r.calls, ms: r.ms, case: { id: c.id, status: "written", file: d.rel } });
@@ -1260,7 +1287,15 @@ export async function createWriter(o) {
1260
1287
  throw err;
1261
1288
  }
1262
1289
  const why = firstLine(err.message);
1263
- // A case that ends without an approval leaves no draft: the file and its recording go back to what they were.
1290
+ // A later round that ends without a test (the writer wrote none, the case's cap was reached) keeps the last draft the reviewer
1291
+ // judged, as the rounds running out do: the file and its recording as that draft's first run left them, a later unreviewed draft gone.
1292
+ const kept = judged;
1293
+ if (kept !== undefined) {
1294
+ await restore(own, kept.test);
1295
+ await restore(sidecar, kept.recording);
1296
+ return forHuman(kept.draft, `${why}; the reviewer on the draft kept: ${kept.feedback}`);
1297
+ }
1298
+ // A case that ends before any review leaves no draft: the file and its recording go back to what they were.
1264
1299
  if (draft !== undefined || previous !== undefined) {
1265
1300
  await restore(own, untouched.test);
1266
1301
  await restore(sidecar, untouched.recording);