@skyramp/mcp 0.4.1-rc.1 → 0.4.1-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
  2. package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
  3. package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
  4. package/build/prompts/enhance-assertions/sharedAssertionRules.js +41 -22
  5. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
  6. package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
  7. package/build/prompts/test-recommendation/test-recommendation-prompt.js +8 -2
  8. package/build/prompts/testbot/testbot-prompts.js +12 -7
  9. package/build/recommendation/registerPlan.d.ts +5 -1
  10. package/build/recommendation/registerPlan.js +5 -0
  11. package/build/recommendation/types.d.ts +25 -2
  12. package/build/recommendation/verifierContracts.d.ts +16 -4
  13. package/build/recommendation/verifierContracts.js +20 -4
  14. package/build/recommendation/verifiers/coverage.d.ts +10 -0
  15. package/build/recommendation/verifiers/coverage.js +144 -22
  16. package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
  17. package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
  18. package/build/recommendation/verifiers/existingCoverage.js +53 -0
  19. package/build/recommendation/verifiers/expectedValueSourced.js +111 -14
  20. package/build/services/TestGenerationService.js +3 -1
  21. package/build/tools/code-refactor/codeReuseTool.js +1 -1
  22. package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
  23. package/build/tools/code-refactor/reuse-state.d.ts +85 -7
  24. package/build/tools/code-refactor/reuse-state.js +239 -34
  25. package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
  26. package/build/tools/code-refactor/utils-verify-gates.js +103 -11
  27. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
  28. package/build/tools/submitReportTool.js +259 -38
  29. package/build/tools/test-management/actionsTool.js +5 -0
  30. package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
  31. package/build/tools/test-management/analyzeChangesTool.js +55 -2
  32. package/build/tools/test-management/registerTestPlanTool.d.ts +2 -1
  33. package/build/tools/test-management/registerTestPlanTool.js +29 -14
  34. package/build/types/ReuseOutcome.d.ts +73 -7
  35. package/build/types/TestAnalysis.d.ts +6 -0
  36. package/build/types/TestbotReport.d.ts +15 -1
  37. package/build/utils/AnalysisStateManager.d.ts +7 -1
  38. package/build/utils/AnalysisStateManager.js +5 -1
  39. package/build/utils/assertion-verify/api-shared-lints.js +71 -34
  40. package/build/utils/assertion-verify/format.js +2 -2
  41. package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
  42. package/build/utils/assertion-verify/helper-imports.js +119 -27
  43. package/build/utils/assertion-verify/lint-types.d.ts +31 -2
  44. package/build/utils/assertion-verify/lint-types.js +66 -0
  45. package/build/utils/assertion-verify/metrics.d.ts +13 -0
  46. package/build/utils/assertion-verify/metrics.js +16 -0
  47. package/build/utils/assertion-verify/verify.d.ts +11 -6
  48. package/build/utils/assertion-verify/verify.js +56 -15
  49. package/build/utils/canonicalJson.d.ts +11 -0
  50. package/build/utils/canonicalJson.js +17 -0
  51. package/build/utils/utils-verify/action-key.d.ts +27 -0
  52. package/build/utils/utils-verify/action-key.js +292 -0
  53. package/build/utils/utils-verify/allow.d.ts +8 -1
  54. package/build/utils/utils-verify/allow.js +14 -1
  55. package/build/utils/utils-verify/call-sites.d.ts +76 -8
  56. package/build/utils/utils-verify/call-sites.js +256 -70
  57. package/build/utils/utils-verify/language-spec.d.ts +3 -2
  58. package/build/utils/utils-verify/parse.d.ts +22 -3
  59. package/build/utils/utils-verify/parse.js +123 -52
  60. package/build/utils/utils-verify/verify.d.ts +33 -3
  61. package/build/utils/utils-verify/verify.js +126 -12
  62. package/build/utils/workspaceAuth.d.ts +59 -19
  63. package/build/utils/workspaceAuth.js +228 -31
  64. package/package.json +1 -1
  65. package/plugin/prompts/generate-tests/execution-plan.md +1 -1
  66. package/plugin/prompts/generate-tests/generation.md +1 -0
  67. package/plugin/prompts/plan-tests.md +33 -16
@@ -8,7 +8,7 @@ const specificRules = [
8
8
  "When an ID was sent in the request body, path, or query, assert its exact value in the response. Since the sent value is already known, asserting only that it is non-null is not sufficient.",
9
9
  "Reserve `is not None` / `not.toBeNull()` for server-generated IDs only.",
10
10
  "Apply the same rule to all other fields and to array items: use inline request body values as the expected values, not setup helper data.",
11
- "Every assertion that can be derived from `expected_response_body`, the inline request body, or path/query literals must appear in this file — do not rely on the integration test to cover it.",
11
+ "The behaviour this test claims must be proven from `expected_response_body`, the inline request body, or path/query literals — do not rely on the integration test to cover it.",
12
12
  ],
13
13
  examples: [
14
14
  {
@@ -21,6 +21,7 @@ const specificRules = [
21
21
  const SCOPE = `### Scope
22
22
  - Only modify test functions — do not touch \`beforeAll\`, \`afterAll\`, or any setup or teardown helper.
23
23
  - Only add assertions clearly supported by \`expected_response_body\`, inline request / path / query literals, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
24
+ - A response no behaviour in this file depends on needs its status code and nothing more.
24
25
  - Add new assertions immediately after the existing status-code assertion — do not move or remove anything.
25
26
  - Do not reference \`beforeAll\` / \`afterAll\` provisioning data in any assertion — every assertion value must come from the inline request body, path, query, prior response, or \`expected_response_body\`.`;
26
27
  export function getContractProviderAssertionsPrompt(testFile, enhanceType) {
@@ -9,7 +9,7 @@ const specificRules = [
9
9
  "After any POST, PATCH, DELETE, sort, reorder, or bulk operation, re-assert the chained, stable, and computed values on the follow-up read step: a follow-up GET after DELETE should return 404 or show the item absent; a follow-up GET after PATCH should assert the new value; a follow-up GET after sort/reorder should confirm the new ordering with chained IDs.",
10
10
  "After a POST creates a resource and a GET retrieves the collection, assert that the created item appears in the list by its chained ID with its exact stable fields — do not stop at a null-check on the array or the id alone.",
11
11
  "When one response field describes a count or summary of a collection returned by a related endpoint (for example, `active_session_count` from `/users/me` and the array length from `/users/me/sessions`), assert that relationship by comparing both extracted values — never hardcode the same number in two places.",
12
- "Every assertion that can be derived from the request and response must appear in this file. If a test name claims a state change (such as \"Updates X\" or \"Increments Y\"), include an assertion that proves that state change across the relevant calls — do not rely on the contract test to cover it.",
12
+ 'A state change the test is for ("Updates X", "Increments Y" from the recommendation it came from, or from the test name when that name says more than `test_integration`) must be proven across the calls that carry it — do not rely on the contract test to cover it.',
13
13
  ],
14
14
  examples: [
15
15
  {
@@ -29,6 +29,7 @@ await skyramp.sendRequest(\`/orders/\${orderId}\`);`,
29
29
  const SCOPE = `### Scope
30
30
  - Apply to every \`send_request\` / \`sendRequest\` call that returns a body.
31
31
  - Only add assertions clearly supported by the request body, prior response values, field names, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
32
+ - A response no behaviour in this file depends on needs its status code and nothing more.
32
33
  - Add new assertions immediately after the existing status-code assertion — do not move or remove anything.`;
33
34
  export function getIntegrationAssertionsPrompt(testFile, enhanceType) {
34
35
  return getAssertionsPrompt(specificRules, SCOPE, testFile, enhanceType);
@@ -1,5 +1,5 @@
1
1
  export type EnhanceType = "generation" | "maintenance";
2
- export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
2
+ export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
3
3
  export declare function maintenanceTaskSuffix(enhanceType: EnhanceType): string;
4
4
  export interface AssertionEnrichmentExample {
5
5
  language: string;
@@ -1,16 +1,33 @@
1
1
  import { getPersonaPrefix } from "../personas.js";
2
- export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
2
+ export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
3
3
  export function maintenanceTaskSuffix(enhanceType) {
4
4
  return enhanceType === "maintenance" ? ` ${MAINTENANCE_SCOPE_NOTE}` : "";
5
5
  }
6
6
  const SHARED_RULES = [
7
+ {
8
+ title: "Assert the behaviour under test",
9
+ description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support; a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced.",
10
+ subPoints: [
11
+ "Name the behaviour, then ask what value in the response changes when it breaks. Assert that value exactly.",
12
+ 'When the test claims a state change ("Updates X", "Increments Y", "rejects Z"), the assertion must prove the change, not merely that the call succeeded.',
13
+ "When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running service still returns the old value. The test is expected to fail until the code is fixed.",
14
+ ],
15
+ examples: [
16
+ {
17
+ language: "javascript",
18
+ code: `// "PATCH /orders applies the coupon discount" — the discount is the behaviour.
19
+ expect(getResponseValue(patchResponse, "discount_amount"), 'discount applied').toBe(20);
20
+ expect(getResponseValue(patchResponse, "total_amount"), 'total after discount').toBe(80);`,
21
+ },
22
+ ],
23
+ },
7
24
  {
8
25
  title: "Echo-back the request fields",
9
26
  description: "For every field returned unchanged from the request body, assert the exact sent value.",
10
27
  subPoints: [
11
28
  "`is not None` / `not.toBeNull()` is only acceptable when the value is genuinely unknown — for server-generated timestamps or opaque IDs. This rule does not apply to computed fields.",
12
29
  "When a response field's value equals the value sent in the request body, path, or query, assert that exact value rather than a null-check — the sent value is known and reproducible.",
13
- "Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted. Asserting only the id on a multi-key resource response is not sufficient.",
30
+ "Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted. When the behaviour is that a resource was created with the fields sent, the id alone does not prove it — assert those fields.",
14
31
  "Range matchers like `toBeGreaterThanOrEqual(0)` and type-only checks like `typeof X === 'number'` are not acceptable for fields whose exact values are known.",
15
32
  "Assert the exact status code from the recorded trace or `expected_response_body` — for example, `201` for resource-creation endpoints. Action-style endpoints with verb path segments like `/run`, `/trigger`, `/execute`, `/deploy`, `/submit`, `/start`, `/stop`, or `/cancel` often use `200`, but assert the recorded or expected code. Permissive status matchers are never acceptable.",
16
33
  ],
@@ -34,7 +51,7 @@ expect(response.statusCode).toBe(201);`,
34
51
  "Also apply the array-validation rule to any errors array.",
35
52
  "Asserting only the status code is never sufficient when a body is present.",
36
53
  "For no-body responses such as a successful DELETE (204): assert the status code only.",
37
- "For a 2xx error body, assert the exact status code, the error field's exact value, and that the resource fields carry no value (`toBeNull()` / `is None` — the SDK helper reads a missing path and an explicit null the same way, so this asserts \"no payload\", not \"key absent\"; use `checkSchema` when the two must be told apart). Never assert the success shape (a non-empty collection, `error` absent) against a response the recorded trace or `expected_response_body` shows returning an error or no-match body — classify the response by what this request returns, not by the outcome the test was meant to check for. The one exception is a test the plan declared `expected.outcome: fail` (typically a `bug_caught` or `requirement_conflict` candidate; the category is only the fallback when nothing was declared): it asserts the intended behaviour established by the diff, the PR statement, or the code's evident intent — keep that assertion and let the test fail.",
54
+ 'For a 2xx error body, assert the exact status code, the error field\'s exact value, and that the resource fields carry no value (`toBeNull()` / `is None` — the SDK helper reads a missing path and an explicit null the same way, so this asserts "no payload", not "key absent"; use `checkSchema` when the two must be told apart). Never assert the success shape (a non-empty collection, `error` absent) against a response the recorded trace or `expected_response_body` shows returning an error or no-match body — classify the response by what this request returns, not by the outcome the test was meant to check for. The one exception is a test the plan declared `expected.outcome: fail` (typically a `bug_caught` or `requirement_conflict` candidate; the category is only the fallback when nothing was declared): it asserts the intended behaviour established by the diff, the PR statement, or the code\'s evident intent — keep that assertion and let the test fail.',
38
55
  ],
39
56
  examples: [
40
57
  {
@@ -60,16 +77,6 @@ assert skyramp.get_response_value(record_post_response, "insertedId") is None`,
60
77
  },
61
78
  ],
62
79
  },
63
- {
64
- title: "Value ranges",
65
- description: "For numeric fields where a realistic range is inferable from the field name, domain, or OpenAPI schema (`minimum` / `maximum`), assert the value falls within the expected range.",
66
- examples: [
67
- {
68
- language: "javascript",
69
- code: `expect(getResponseValue(productsPostResponse, "price")).toBeGreaterThanOrEqual(0);`,
70
- },
71
- ],
72
- },
73
80
  {
74
81
  title: "Specific known values",
75
82
  description: "For enum or status fields where only one outcome is valid for this flow, assert the exact expected value.",
@@ -117,8 +124,8 @@ expect(getValue(response, "data.0")).toBeNull();`,
117
124
  ],
118
125
  },
119
126
  {
120
- title: "Format for server-generated fields",
121
- description: "For fields whose exact value varies across runs — UUIDs, auto-incremented IDs, ISO timestamps, IP addresses, emails, and URLs — assert the format or pattern rather than the exact value.",
127
+ title: "Fields whose value varies across runs",
128
+ description: "For a field whose exact value varies across runs — a UUID, an auto-incremented ID, an ISO timestamp, an IP address, an email, a URL, or a number with no knowable exact value — assert its format or its range instead of its value, and only when the behaviour under test depends on that field. A generated id the test chains onward needs a format check; the same id on a response nothing reads again needs no assertion at all.",
122
129
  subPoints: [
123
130
  "UUID matches `/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i`.",
124
131
  "Auto-incremented ID is greater than 0.",
@@ -128,6 +135,7 @@ expect(getValue(response, "data.0")).toBeNull();`,
128
135
  "URL starts with `http://` or `https://`.",
129
136
  "Use `not.toBeNull()` / `is not None` only for truly opaque random tokens where no format is recognizable.",
130
137
  "Use format info from the OpenAPI schema (`format: uuid`, `format: date-time`) to identify these fields.",
138
+ "A numeric field with no knowable exact value takes a range from the field name, the domain, or the OpenAPI `minimum` / `maximum` — a range is the weakest assertion in this rule set, so use it only where no exact value exists and the behaviour depends on the field.",
131
139
  ],
132
140
  examples: [
133
141
  {
@@ -139,9 +147,8 @@ expect(getResponseValue(productsPostResponse, "created_at")).toMatch(/^\\d{4}-\\
139
147
  },
140
148
  {
141
149
  title: "Use exact recorded JSON paths",
142
- description: "Do not add object-wrapper prefixes or nested paths that are not present in the actual response body. Use the exact JSON path from the recorded response body.",
150
+ description: 'Read every field at the path the recorded body actually uses. A wrapper prefix the response does not have makes the assertion read null and the test fail for a reason that is not the behaviour if the body is flat (`{"id": "acc_...", "tax_exempt": true}`), the path is `"tax_exempt"`, never `"account.tax_exempt"`.',
143
151
  subPoints: [
144
- `If the body is flat (\`{"id": "acc_...", "tax_exempt": true}\`), use \`"tax_exempt"\`, not \`"account.tax_exempt"\`.`,
145
152
  "Only use dot notation when the recorded response genuinely nests the field under that key.",
146
153
  ],
147
154
  examples: [
@@ -216,9 +223,18 @@ export function getAssertionsPrompt(specificRules, scope, testFile, enhanceType)
216
223
  .join(",\n");
217
224
  return `${getPersonaPrefix()}Your task is to enhance response body assertions in the given test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
218
225
 
226
+ ### What to assert
227
+ Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour, and the values it depends on, exactly. Leave the rest of the response alone.
228
+
229
+ A field belongs in the test when a wrong value in it would mean the behaviour is broken. A field does not belong when its value can change for reasons that have nothing to do with the behaviour — a generated id, a timestamp, an unrelated sibling field. Asserting those does not make the test stronger. It makes the test fail for reasons the test is not about, and it buries the one assertion a reader needs to understand what the test is for.
230
+
231
+ So the rules below say how to assert a value, not how many values to assert. Apply a rule to the fields the behaviour depends on. \`[]\` is the correct answer for a rule whose fields this test does not depend on.
232
+
233
+ If nothing in the response distinguishes the behaviour working from the behaviour broken, the test is exercising the wrong request or reading the wrong response. Say so; do not pad it with assertions on whatever else the body carries.
234
+
219
235
  ### Pre-Edit Assertion Analysis
220
- Before editing the given file, you must output a \`<thinking>\` block. The aim of the \`<thinking>\` block is to analyze each in-scope response in the given test file and output a JSON array that ensures no assertion rule is overlooked. The JSON array should match the template below — every assertion rule title must appear as a key in the \`rule_checklist\`, even when the value is \`[]\`.
221
- 1. Scan the given test file and expected responses based on the test recommendations for the code change tested.
236
+ Before editing the given file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the responses that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion rule title must appear as a key in the \`rule_checklist\`, with \`[]\` where the rule does not apply.
237
+ 1. For each test function in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from. Generated test functions are usually named \`test_integration\` or \`testUi\`, which claim nothing — use the name only when it is specific enough to add something, and never treat a generic one as evidence the test has no single behaviour. Then scan the given test file and the expected responses for the code change tested.
222
238
  2. Classify each response first by its response status type and then assign the applicable assertion rules to the response.
223
239
  1. Success with body (2xx with a response body): all assertion rules below may apply — echo-back of request fields, computed response fields, array / items validation, and chained values across steps.
224
240
  2. Success with no body (200/202/204 with an empty body — e.g. logout/cancel/submit action endpoints): assert the status code only. Also apply chained-values rules if a follow-up step uses this response's ID.
@@ -227,12 +243,14 @@ Before editing the given file, you must output a \`<thinking>\` block. The aim o
227
243
  3. For each in-scope response, output one JSON object using the template below. The output is an array — one object per in-scope response.
228
244
  - \`step\`: the HTTP method, path, and response variable name for this request (e.g. \`POST /products → products_POST_response\`).
229
245
  - \`response_status\`: one of \`success\`, \`no_body\`, or \`error\` based on the classification in step 2.
230
- - \`rule_checklist\`: an object that MUST contain every rule title below as a key. For each rule, the value is an array of assertion lines you will add for this response under that rule. Use \`[]\` only when the rule does not apply to this response — every key must still be present. This forces you to consider every rule for every response.
246
+ - \`behaviour_under_test\`: the one-line behaviour from step 1 that this response is evidence for. When a response is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
247
+ - \`rule_checklist\`: an object that MUST contain every rule title below as a key. For each rule, the value is the assertion lines you will add for this response under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's fields are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
231
248
 
232
249
  \`\`\`json
233
250
  [{
234
251
  "step": "<METHOD> <path> → <responseVar>",
235
252
  "response_status": "success | no_body | error",
253
+ "behaviour_under_test": "<the one-line behaviour this response is evidence for, or none>",
236
254
  "rule_checklist": {
237
255
  ${ruleChecklistKeys}
238
256
  }
@@ -251,7 +269,8 @@ ${scope}
251
269
  - Do not access response fields via dict syntax (\`response["field"]\`) or attribute access (\`response.field\`) — always use the SDK helper.
252
270
  - Do not assert \`not.toBeNull()\` / \`is not None\` on a field whose exact value is in the request body or a prior response.
253
271
  - Do not assert an exact value you learned only from a request you sent yourself while writing the test. Take the value from the diff, from what the request determines, or from a response this test received earlier.
254
- - Do not skip body assertions citing genuinely unpredictable fields every assertable field still needs an assertion.
272
+ - Do not assert a field only because the body carries it. If a wrong value there would not mean the behaviour under test is broken, leave it alone.
273
+ - Do not leave a test whose assertions would all still pass with the behaviour under test broken.
255
274
  - Do not use permissive status matchers (\`.toMatch(/^2/)\`, \`.toBeGreaterThanOrEqual(200)\`, \`checkStatusCode(response, '20x')\`).
256
275
  - Do not use shape-only or type-only assertions as a substitute for exact value validation. Forbidden patterns: \`Array.isArray(...)\`, \`typeof X === '...'\`, \`X instanceof Array\`, \`Object.keys(X).length > 0\`. When a field's value comes from one of those sources, assert it exactly.
257
276
  - Do not use shape-only, containment-only, range-only, or weak-length as the sole assertion on a populated array.
@@ -261,5 +280,5 @@ ${scope}
261
280
  - Do not remove existing assertions.
262
281
 
263
282
  ### Verification of Assertions
264
- After adding all assertion lines in the given test file, verify that every applicable rule has been applied correctly to each in-scope response. If any are missing or weakly applied, fix them before completing.`;
283
+ After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.`;
265
284
  }
@@ -51,6 +51,8 @@ expect(trigger).to_be_focused()`,
51
51
  "Anchor on the removed attribute+value or role+name, never on text or position — a text-based absence check breaks when unrelated siblings change.",
52
52
  "Do not add a guard for an element whose whole page or route was removed — that is maintenance of the tests that covered it, not a new assertion; a spec that opens a deleted route must be anchored on a surviving page first (a retained element asserted on a live page in the same spec) — a spec whose only visited page is the dead URL is not a valid guard.",
53
53
  "A renamed test id or accessible name (a removed `aria-label` included) is not a removed element: assert the element under its new identifier and never assert the retired identifier `toBeHidden()` — it passes today and turns into a false failure the day the name is reused.",
54
+ "An element that exists only inside a control the page has to open — an `option` of a `select`, an item of a closed menu or dropdown — takes `toHaveCount(0)`, not `toBeHidden()`. A closed control's contents have no bounding box, so `toBeHidden()` passes whether the element was removed or not, and the guard stays green the day it comes back. If the spec opens the control before asserting, the element would have had a box had it survived, so assert both.",
55
+ "The positive companion for that guard is counted or read, never `toBeVisible`: the same missing bounding box that makes `toBeHidden()` a false green on the removed entry makes `toBeVisible()` a false red on the one that stayed. Assert a retained sibling with `toHaveCount(1)` or `toHaveText`, or assert the control itself, which does have a box.",
54
56
  ],
55
57
  examples: [
56
58
  {
@@ -83,7 +85,7 @@ expect(page.get_by_test_id('navbar-products')).to_have_text('Products')`,
83
85
  },
84
86
  {
85
87
  title: "Tests target the changed behavior introduced by the pull request",
86
- description: "At least one assertion targets the changed behavior on populated or updated state, not only empty or zero state.",
88
+ description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support: a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced. Target it on populated or updated state, not only on empty or zero state, and assert it on the page where the behaviour shows. When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running app still renders the old value; the test is expected to fail until the code is fixed.",
87
89
  examples: [],
88
90
  },
89
91
  {
@@ -222,6 +224,9 @@ export function getUIAssertionsPrompt(testFile, enhanceType) {
222
224
  const categoryTemplate = renderAssertionCategoriesTemplate(UI_ASSERTION_CATEGORIES);
223
225
  return `${getPersonaPrefix()}Your task is to enhance assertions for the given UI test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
224
226
 
227
+ ### What to assert
228
+ Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour on the page where it shows, and the rendered values it depends on, exactly. A page the test only passes through, and an element the behaviour does not touch, need no assertion: asserting them makes the test fail for reasons the test is not about and buries what the test is for. The rules below say how to assert a value, not how many values to assert.
229
+
225
230
  ### First Check
226
231
  If the generated test file has no \`expect()\` assertions, you MUST manually add them before anything else. Use \`import { expect } from '@skyramp/skyramp';\` — never from \`@playwright/test\`. If an existing import pulls \`expect\` from \`@playwright/test\`, move it to \`@skyramp/skyramp\` (keep \`test\` on the playwright line).
227
232
 
@@ -328,25 +333,28 @@ if (await existingSecret.isVisible().catch(() => false)) {
328
333
  \`\`\`
329
334
 
330
335
  ### Pre-Edit Assertion Analysis
331
- Before editing the given test file, you must output a \`<thinking>\` block. The aim of the \`<thinking>\` block is to analyze each in-scope item (action, selector, or captured network response) in the given test file and output a JSON array that ensures no assertion rule is overlooked. The JSON array should match the template below — every assertion category and every rule title under it must appear as a key, even when the value is \`[]\`.
332
- 1. Selector inventory list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use only selectors from this list. An assertion the recorder dropped because it failed during recording may use the selector of the element named in that recording's blueprint capture. Do not invent \`data-testid\` values, role names, or aria attributes. Also note captured network responses, repeated element patterns, exact rendered text/value/attribute from trace/source, and existing \`toBeVisible()\` assertions whose exact text is knowable.
333
- 2. ProcessReplay Identify Fix or Add walk through these three steps explicitly.
336
+ Before editing the given test file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the items that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion category and every rule title under it must appear as a key, with \`[]\` where the rule does not apply.
337
+ 1. For each test in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from, and name the page where that behaviour shows. Generated specs are usually named \`testUi\`, which claims nothing use the name only when it is specific enough to add something.
338
+ 2. Selector inventory list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use only selectors from this list. An assertion the recorder dropped because it failed during recording may use the selector of the element named in that recording's blueprint capture. Do not invent \`data-testid\` values, role names, or aria attributes. Also note captured network responses, repeated element patterns, exact rendered text/value/attribute from trace/source, and existing \`toBeVisible()\` assertions whose exact text is knowable.
339
+ 3. Process — Replay → Identify → Fix or Add — walk through these three steps explicitly.
334
340
  a. Replay the scenario mentally. At each state-changing action (form submit, item add/edit/delete), ask: "What is the EXPECTED outcome based on the action performed?"
335
341
  b. Identify expectation mismatches. If the recorded trace shows a result that contradicts the action (e.g. removing 1 of 2 items but the page shows 3, submitting a form but getting a blank page, editing a field but the old value persists), that is an app bug the test should catch. List every mismatch you find.
336
342
  c. Fix or add assertions for each mismatch.
337
343
  - If an existing assertion uses the wrong (buggy) value, edit it to assert the correct expected value.
338
344
  - If no assertion exists for the buggy behavior, add one immediately after the action that triggers it.
339
- 3. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
345
+ 4. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
340
346
  - Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
341
347
  - Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
342
348
  - Post-edit State applies whenever EITHER (a) the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), OR (b) the PR changes interactive behavior — in case (b), add the triggering action and assert the before/after transition (see "Exercise the changed interaction"). Mark NOT APPLICABLE only when the PR changes no interactive behavior and the test has no state-changing action.
343
- 4. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
349
+ 5. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
344
350
  - \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
345
- - \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is an array of assertion lines you will add for this item under that rule. Use \`[]\` only when the rule does not apply to this item — every category key and every rule key must still be present. This forces you to consider every rule for every item.
351
+ - \`behaviour_under_test\`: the one-line behaviour from step 1 that this item is evidence for. When an item is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
352
+ - \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is the assertion lines you will add for this item under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's elements are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
346
353
 
347
354
  \`\`\`json
348
355
  [{
349
356
  "action_or_selector_or_response": "<selector | action | response>",
357
+ "behaviour_under_test": "<the one-line behaviour this item is evidence for, or none>",
350
358
  "assertion_categories": {
351
359
  ${categoryTemplate}
352
360
  }
@@ -354,7 +362,7 @@ ${categoryTemplate}
354
362
  \`\`\`
355
363
 
356
364
  ### Assertion Rules with Examples
357
- Most-violated patterns — apply every time. (1) repeated elements → \`toHaveCount(N)\` + per-item \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` (Collection / repeated UI elements). (2) post-action state → \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\`/\`toBeChecked\`, not \`toBeVisible\` when exact value is knowable (Post-action visible state). (3) routing/URL-only tests → at least one rendered-element exact text (Positive-path companion).
365
+ Most-violated patterns — whenever the behaviour under test involves one of these, this is the shape it takes. (1) repeated elements → \`toHaveCount(N)\` + per-item \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` (Collection / repeated UI elements). (2) post-action state → \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\`/\`toBeChecked\`, not \`toBeVisible\` when exact value is knowable (Post-action visible state). (3) routing/URL-only tests → at least one rendered-element exact text (Positive-path companion).
358
366
  Strength order. \`toHaveCount\`/\`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` > \`toContainText\` (only when string is genuinely dynamic) > \`toBeVisible\` (only when presence is the actual test).
359
367
 
360
368
  ${renderCategories(UI_ASSERTION_CATEGORIES)}
@@ -373,7 +381,7 @@ ${renderCategories(UI_ASSERTION_CATEGORIES)}
373
381
  - Do not assert buggy/error text as expected (unless the intended UX is an error state)
374
382
 
375
383
  ### Verification of Assertions
376
- After adding all assertion lines in the given test file, verify that every applicable rule has been applied correctly to each in-scope action, selector, and captured response. If any are missing or weakly applied, fix them before completing.
384
+ After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.
377
385
 
378
386
  The goal is tests that FAIL when the app has bugs, not tests that simply replay what happened.
379
387
  `;
@@ -1,14 +1,12 @@
1
1
  import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
2
2
  import { sectionBody } from "../promptAssets.js";
3
3
  import { readPromptAsset } from "../promptAssets.js";
4
- import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
5
4
  import { resolveServiceDetailsRef } from "../../utils/utils.js";
6
5
  import { PromptPlan } from "./promptPlan.js";
7
6
  import { buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, buildFinishChecks, } from "./recommendationSections.js";
8
7
  const EXECUTION_PLAN_MD = readPromptAsset("generate-tests/execution-plan.md");
9
8
  function _execEnrichBody(ctx) {
10
9
  return fillPlaceholders(sectionBody(EXECUTION_PLAN_MD, "## Parameter grounding"), {
11
- authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
12
10
  enrichSurface: ctx.isDiffScope ? "the diff" : "the analysed surface",
13
11
  testPatternGuidelines: buildTestPatternGuidelines(),
14
12
  });
@@ -1,9 +1,10 @@
1
1
  import { AnalysisScope, isDiff, } from "../../types/RepositoryAnalysis.js";
2
- import { WorkspaceAuthType, getDefaultAuthHeader } from "../../utils/workspaceAuth.js";
2
+ import { WorkspaceAuthType, getDefaultAuthHeader, AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
3
3
  import { logger } from "../../utils/logger.js";
4
4
  import { buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildToolWorkflows, buildVerificationChecklist, } from "./recommendationSections.js";
5
5
  import { buildExecutionPlan, EXEC_STEP_ENRICH } from "./diffExecutionPlan.js";
6
6
  import { readPromptAsset } from "../promptAssets.js";
7
+ import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
7
8
  import { TASK_GENERATE, taskRef, } from "./recommendationShared.js";
8
9
  // Re-export for backward compatibility (tests and external callers import this from this module)
9
10
  function formatTestLocations(locs) {
@@ -25,7 +26,12 @@ const SKYRAMP_TEST_FILE_PATTERN = /(?:_test|_smoke|_contract|_fuzz|_integration|
25
26
  * one text and nothing to drift from. `isDiffScope` is unused here — the
26
27
  * procedure reads the same for a diff and for a repository. */
27
28
  function planningInstruction(_isDiffScope) {
28
- return readPromptAsset("plan-tests.md").trim();
29
+ // The procedure names the auth patterns. BOTH renderers of this asset fill the
30
+ // placeholder — this lane's and the testbot lane's — or one of them ships the
31
+ // literal `{authMiddlewarePatterns}` to the model.
32
+ return fillPlaceholders(readPromptAsset("plan-tests.md").trim(), {
33
+ authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
34
+ });
29
35
  }
30
36
  export function buildRecommendationPrompt(analysis, analysisScope = AnalysisScope.FullRepo, prContext, workspaceAuthHeader, workspaceAuthType, workspaceAuthScheme, sessionId) {
31
37
  const isDiffScope = isDiff(analysisScope);
@@ -5,6 +5,7 @@ import { buildPathParamGuidance, } from "../test-recommendation/recommendationSe
5
5
  import { setReportLanguage } from "../../utils/reportLanguage.js";
6
6
  import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
7
7
  import { recordPullRequestText } from "../../recommendation/pullRequestText.js";
8
+ import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
8
9
  import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, TESTBOT_TASK1_MAINTAIN_LABELS, TESTBOT_TASK1_MULTIREPO_LABELS, TESTBOT_TASK1_STEP_LABELS, orderedLabels, stepSubRef, taskRef, } from "../test-recommendation/recommendationShared.js";
9
10
  import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
10
11
  import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, isSkillsLoaded, } from "../../utils/featureFlags.js";
@@ -59,24 +60,28 @@ const PLAN_TESTS_MD = readPromptAsset("plan-tests.md").trim();
59
60
  // rendered neither it nor a copy, so the agent planned without the claims step and
60
61
  // without the backward surface map. Rendered here, never retyped.
61
62
  //
62
- // It sits OUTSIDE Task 1: the procedure is a twelve-item numbered list, Task 1's
63
+ // It sits OUTSIDE Task 1: the procedure is a numbered list, Task 1's
63
64
  // own steps are numbered too, and nested inside Task 1 the two share one ordinal
64
65
  // space (`referenceIntegrity.test.ts` reads every `N.` there as a Task 1 step).
65
66
  //
66
- // Two steps run elsewhere in this lane — step 7's maintenance update is Task 1's
67
- // tool step, step 10's registration happens in Task 2 after it. The note states that
68
- // order; the markdown stays the source.
67
+ // Two steps run elsewhere in this lane — the maintenance update in **Group the
68
+ // changes into behaviours and decide each** is Task 1's tool step, and **Register the
69
+ // whole plan in one call to `skyramp_register_test_plan`** happens in Task 2 after it.
70
+ // The note states that order; the markdown stays the source.
69
71
  //
70
72
  // The count sentence sits HERE, beside the register instruction, because the agent
71
73
  // reads the URI first and plans straight after: on run 34005367281 it read
72
74
  // `maxGenerate=3` as this run's budget and left its fourth planned test unwritten.
73
- const planProcedureBlock = `${section(PLAN_TESTS_MD, "## Terms")}
75
+ // The procedure names the auth patterns. BOTH renderers of this asset fill the
76
+ // placeholder — this lane's and the recommendation lane's — or one of them ships
77
+ // the literal `{authMiddlewarePatterns}` to the model.
78
+ const planProcedureBlock = fillPlaceholders(`${section(PLAN_TESTS_MD, "## Terms")}
74
79
 
75
80
  ${section(PLAN_TESTS_MD, "## The procedure")}
76
81
 
77
82
  **In this run:** ${taskRef(TASK_ANALYZE_MAINTAIN)}'s maintenance step performs the maintenance update of an existing test, with the maintenance tools — a test those tools update is not a plan planned test. Register the plan in ${taskRef(TASK_GENERATE)}, after that step. Nothing caps how many planned tests the plan holds: \`maxGenerate\` and \`maxRecommendations\` arrive in this run's URI, and the server accepts both and applies neither — no check reads either one, so a count you see there is not a budget.
78
83
 
79
- ${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}`;
84
+ ${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}`, { authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR });
80
85
  // The checks on the test file itself. One home — `generation.md` — rendered by
81
86
  // this lane at the end of generation and by the IDE lane after its generation
82
87
  // rules, so neither carries a retyped copy.
@@ -460,7 +465,7 @@ ${CONTRACT_MODE_GUIDANCE}
460
465
  - \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
461
466
  - \`skyramp_ui_test_generation\`: set \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript Playwright only — the default; leave \`codeReuse\` unset for other languages)
462
467
  - **\`browser_assert\`**: the \`browser_navigate\` result states the rule this tool has to satisfy on every page you reach. Call multiple assertions in the same tool call batch when checking independent elements.
463
- - **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
468
+ - **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. When the removed element lived inside a control the page has to open — an \`option\` of a \`select\`, an item of a closed menu — open that control before asserting: a closed control's contents are never visible, so a hidden assertion recorded against a shut one passes whether the element was removed or not. \`browser_assert_hidden\` stays the tool either way, because \`browser_assert\` needs a snapshot \`ref\` and a removed element has none; where the control cannot be opened, say so in \`reasoning\` so the assertion step turns that guard into a count. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
464
469
  - **Renames are not removals — on every PR, whatever \`uiContext.removedElements\` holds**: when a test id, \`aria-label\` or other accessible name changed, assert the element by its new identifier and leave the retired identifier alone (no \`browser_assert_hidden\` on it). A rename PR removes nothing, so this rule applies when the removed list is empty.
465
470
  - **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
466
471
  - **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
@@ -1,4 +1,4 @@
1
- import { Objection, ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlannedTest, VerifyContext } from "./types.js";
1
+ import { Objection, ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlanMaintenance, PlannedTest, VerifyContext } from "./types.js";
2
2
  /** The plan as stored for the run. Every registration produces one; the newest
3
3
  * replaces the last. */
4
4
  export interface Plan {
@@ -10,6 +10,10 @@ export interface Plan {
10
10
  * test that proves it. */
11
11
  defects: PlanDefect[];
12
12
  plannedTests: PlannedTest[];
13
+ /** Existing tests this run edits instead of planning new ones. Stored because
14
+ * `coverage` counts the changes they cite, so a reader of the plan has to be
15
+ * able to see which coverage came from maintenance rather than a new test. */
16
+ maintains?: PlanMaintenance[];
13
17
  /** Objections the agent answered, with the answers. */
14
18
  answeredObjections: Array<{
15
19
  objection: Objection;
@@ -110,6 +110,11 @@ export function registerPlan(registration, ctx, previous) {
110
110
  changes,
111
111
  defects,
112
112
  plannedTests,
113
+ // Only when declared: the field is optional, and writing `[]` on every plan
114
+ // would change the stored shape of every run that maintains nothing.
115
+ ...(Array.isArray(registration.maintains) && registration.maintains.length > 0
116
+ ? { maintains: registration.maintains }
117
+ : {}),
113
118
  answeredObjections,
114
119
  unverifiedCloses,
115
120
  openObjections,
@@ -41,9 +41,15 @@ export interface PlanChange {
41
41
  * test asserts a value the plan decided, not the value the running app
42
42
  * happened to return — the one source that always agrees with a defect. */
43
43
  expectedValue?: string | number | boolean | null;
44
- /** Where the agent read `expectedValue`: `pr-description`, `spec:<path>`,
45
- * `convention:<file:line>`, or `code`. */
44
+ /** Where the agent read `expectedValue` OR `derived` — the check reads it for
45
+ * either one: `pr-description`, `spec:<path>`, `convention:<file:line>`, or
46
+ * `code`. */
46
47
  expectedFrom?: string;
48
+ /** The rule the value follows from, in the source's words, when the source
49
+ * states a rule and not a value. Stands in for `expectedValue`: the test
50
+ * computes the number from what it sent, so nothing pins one the source
51
+ * never wrote. */
52
+ derived?: string;
47
53
  }>;
48
54
  }
49
55
  /** One defect the code review found, as the agent read it. Nothing on the server
@@ -124,6 +130,20 @@ export interface ObjectionAnswer {
124
130
  * An objection about an untested change closes only with one of these. */
125
131
  blocker?: string;
126
132
  }
133
+ /** An existing test this run maintains instead of planning a new one, and the
134
+ * declared changes that maintenance covers. A removal already covered by a spec
135
+ * is maintained, not duplicated, so the plan needs a way to say "this change is
136
+ * tested by a file I am editing" — without it, `coverage:change:` is unanswerable
137
+ * for a correct maintenance decision, and it closes only with a `blocker`. */
138
+ export interface PlanMaintenance {
139
+ /** Repository-relative path of the existing test. Only its presence in the
140
+ * checkout is checked, the same rule `declarations.existingTests` follows. */
141
+ file: string;
142
+ /** The edit this run makes to it, in one sentence. */
143
+ differsBy: string;
144
+ /** The ids of the declared changes this maintenance covers. */
145
+ changes: string[];
146
+ }
127
147
  export interface PlanInput {
128
148
  /** The changes the diff makes. Each planned test cites the ones it tests. */
129
149
  changes: PlanChange[];
@@ -132,6 +152,9 @@ export interface PlanInput {
132
152
  * objection the agent answers. */
133
153
  defects: PlanDefect[];
134
154
  plannedTests: PlannedTest[];
155
+ /** Existing tests this run edits rather than replacing with new ones. Coverage
156
+ * counts the changes these cite, so a maintenance-only plan is a complete plan. */
157
+ maintains?: PlanMaintenance[];
135
158
  answers: ObjectionAnswer[];
136
159
  /** The tool owns this and overwrites whatever the caller sends. */
137
160
  registrationNumber: number;
@@ -106,8 +106,16 @@ export declare const EXISTING_COVERAGE_CONTRACT: {
106
106
  readonly message: "This planned test names an existing test file that is not in the checkout.";
107
107
  readonly suggestion: "Name the file repository-relative, spelled as the analysis lists it. Drop the entry if it names a test you did not read — naming none is an answer too.";
108
108
  };
109
+ readonly maintainedTestMissing: {
110
+ readonly message: "`maintains` names an existing test file that is not in the checkout.";
111
+ readonly suggestion: "Name the file repository-relative, spelled as the analysis lists it. A maintenance entry is what makes its changes count as covered, so it has to name a file this run can edit.";
112
+ };
113
+ readonly maintainedUnknownChange: {
114
+ readonly message: "A `maintains` entry cites a change id that this plan does not declare.";
115
+ readonly suggestion: "Spell the change's `id` exactly as your own `changes` list gives it — the match is exact apart from surrounding space. Declare the change if it is missing.";
116
+ };
109
117
  };
110
- readonly suggestion: "Which existing tests did you read on this change, and how does this planned test differ from them?";
118
+ readonly suggestion: "Which existing tests did you read on this change, how does this planned test differ from them, and which changes does the maintenance you are doing cover?";
111
119
  };
112
120
  export declare const EXPECTED_OUTCOME_CONTRACT: {
113
121
  readonly id: "expectedOutcome";
@@ -150,7 +158,7 @@ export declare const COVERAGE_CONTRACT: {
150
158
  readonly objections: {
151
159
  readonly change: {
152
160
  readonly message: "This declared change has no test in the plan.";
153
- readonly suggestion: "Plan a test that exercises it. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
161
+ readonly suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
154
162
  };
155
163
  readonly noChanges: {
156
164
  readonly message: "The plan declares no changes, so nothing says what the pull request must make different.";
@@ -218,11 +226,15 @@ export declare const EXPECTED_VALUE_SOURCED_CONTRACT: {
218
226
  };
219
227
  readonly quoteMissing: {
220
228
  readonly message: "This case reads its value from the pull request, and its change does not quote the sentence that states the rule.";
221
- readonly suggestion: "Put the sentence from the title or description on the change as `quote`, as written there. Derive this case's value from that rule; the value itself need not appear in the pull request.";
229
+ readonly suggestion: "Put the sentence from the title or description on the change as `quote`, as written there. If that sentence states the rule and not the number, put the rule in `derived` and leave `expectedValue` out.";
230
+ };
231
+ readonly valueNotQuoted: {
232
+ readonly message: "This case reads its value from the pull request, and the pull request does not state this value.";
233
+ readonly suggestion: "Quote the sentence that states it, or, if the pull request states a rule, put the rule in `derived` and leave `expectedValue` out; the test computes the value from what it sent.";
222
234
  };
223
235
  readonly failWithoutValue: {
224
236
  readonly message: "This test expects to fail, and no case on the changes it cites states the value it must assert.";
225
- readonly suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If no source states the value, answer with what the test asserts instead and where that comes from.";
237
+ readonly suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If the source states a rule rather than a number, put the rule in `derived` instead. If no source states either, answer with what the test asserts instead and where that comes from.";
226
238
  };
227
239
  };
228
240
  readonly suggestion: "Where does the correct value for this case come from?";