@skyramp/mcp 0.4.1-rc.1 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +41 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +8 -2
- package/build/prompts/testbot/testbot-prompts.js +12 -7
- package/build/recommendation/registerPlan.d.ts +5 -1
- package/build/recommendation/registerPlan.js +5 -0
- package/build/recommendation/types.d.ts +25 -2
- package/build/recommendation/verifierContracts.d.ts +16 -4
- package/build/recommendation/verifierContracts.js +20 -4
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +111 -14
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +259 -38
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +2 -1
- package/build/tools/test-management/registerTestPlanTool.js +29 -14
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +15 -1
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +71 -34
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +1 -1
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +33 -16
|
@@ -8,7 +8,7 @@ const specificRules = [
|
|
|
8
8
|
"When an ID was sent in the request body, path, or query, assert its exact value in the response. Since the sent value is already known, asserting only that it is non-null is not sufficient.",
|
|
9
9
|
"Reserve `is not None` / `not.toBeNull()` for server-generated IDs only.",
|
|
10
10
|
"Apply the same rule to all other fields and to array items: use inline request body values as the expected values, not setup helper data.",
|
|
11
|
-
"
|
|
11
|
+
"The behaviour this test claims must be proven from `expected_response_body`, the inline request body, or path/query literals — do not rely on the integration test to cover it.",
|
|
12
12
|
],
|
|
13
13
|
examples: [
|
|
14
14
|
{
|
|
@@ -21,6 +21,7 @@ const specificRules = [
|
|
|
21
21
|
const SCOPE = `### Scope
|
|
22
22
|
- Only modify test functions — do not touch \`beforeAll\`, \`afterAll\`, or any setup or teardown helper.
|
|
23
23
|
- Only add assertions clearly supported by \`expected_response_body\`, inline request / path / query literals, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
|
|
24
|
+
- A response no behaviour in this file depends on needs its status code and nothing more.
|
|
24
25
|
- Add new assertions immediately after the existing status-code assertion — do not move or remove anything.
|
|
25
26
|
- Do not reference \`beforeAll\` / \`afterAll\` provisioning data in any assertion — every assertion value must come from the inline request body, path, query, prior response, or \`expected_response_body\`.`;
|
|
26
27
|
export function getContractProviderAssertionsPrompt(testFile, enhanceType) {
|
|
@@ -9,7 +9,7 @@ const specificRules = [
|
|
|
9
9
|
"After any POST, PATCH, DELETE, sort, reorder, or bulk operation, re-assert the chained, stable, and computed values on the follow-up read step: a follow-up GET after DELETE should return 404 or show the item absent; a follow-up GET after PATCH should assert the new value; a follow-up GET after sort/reorder should confirm the new ordering with chained IDs.",
|
|
10
10
|
"After a POST creates a resource and a GET retrieves the collection, assert that the created item appears in the list by its chained ID with its exact stable fields — do not stop at a null-check on the array or the id alone.",
|
|
11
11
|
"When one response field describes a count or summary of a collection returned by a related endpoint (for example, `active_session_count` from `/users/me` and the array length from `/users/me/sessions`), assert that relationship by comparing both extracted values — never hardcode the same number in two places.",
|
|
12
|
-
|
|
12
|
+
'A state change the test is for ("Updates X", "Increments Y" — from the recommendation it came from, or from the test name when that name says more than `test_integration`) must be proven across the calls that carry it — do not rely on the contract test to cover it.',
|
|
13
13
|
],
|
|
14
14
|
examples: [
|
|
15
15
|
{
|
|
@@ -29,6 +29,7 @@ await skyramp.sendRequest(\`/orders/\${orderId}\`);`,
|
|
|
29
29
|
const SCOPE = `### Scope
|
|
30
30
|
- Apply to every \`send_request\` / \`sendRequest\` call that returns a body.
|
|
31
31
|
- Only add assertions clearly supported by the request body, prior response values, field names, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
|
|
32
|
+
- A response no behaviour in this file depends on needs its status code and nothing more.
|
|
32
33
|
- Add new assertions immediately after the existing status-code assertion — do not move or remove anything.`;
|
|
33
34
|
export function getIntegrationAssertionsPrompt(testFile, enhanceType) {
|
|
34
35
|
return getAssertionsPrompt(specificRules, SCOPE, testFile, enhanceType);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type EnhanceType = "generation" | "maintenance";
|
|
2
|
-
export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
|
|
2
|
+
export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
|
|
3
3
|
export declare function maintenanceTaskSuffix(enhanceType: EnhanceType): string;
|
|
4
4
|
export interface AssertionEnrichmentExample {
|
|
5
5
|
language: string;
|
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
import { getPersonaPrefix } from "../personas.js";
|
|
2
|
-
export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
|
|
2
|
+
export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
|
|
3
3
|
export function maintenanceTaskSuffix(enhanceType) {
|
|
4
4
|
return enhanceType === "maintenance" ? ` ${MAINTENANCE_SCOPE_NOTE}` : "";
|
|
5
5
|
}
|
|
6
6
|
const SHARED_RULES = [
|
|
7
|
+
{
|
|
8
|
+
title: "Assert the behaviour under test",
|
|
9
|
+
description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support; a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced.",
|
|
10
|
+
subPoints: [
|
|
11
|
+
"Name the behaviour, then ask what value in the response changes when it breaks. Assert that value exactly.",
|
|
12
|
+
'When the test claims a state change ("Updates X", "Increments Y", "rejects Z"), the assertion must prove the change, not merely that the call succeeded.',
|
|
13
|
+
"When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running service still returns the old value. The test is expected to fail until the code is fixed.",
|
|
14
|
+
],
|
|
15
|
+
examples: [
|
|
16
|
+
{
|
|
17
|
+
language: "javascript",
|
|
18
|
+
code: `// "PATCH /orders applies the coupon discount" — the discount is the behaviour.
|
|
19
|
+
expect(getResponseValue(patchResponse, "discount_amount"), 'discount applied').toBe(20);
|
|
20
|
+
expect(getResponseValue(patchResponse, "total_amount"), 'total after discount').toBe(80);`,
|
|
21
|
+
},
|
|
22
|
+
],
|
|
23
|
+
},
|
|
7
24
|
{
|
|
8
25
|
title: "Echo-back the request fields",
|
|
9
26
|
description: "For every field returned unchanged from the request body, assert the exact sent value.",
|
|
10
27
|
subPoints: [
|
|
11
28
|
"`is not None` / `not.toBeNull()` is only acceptable when the value is genuinely unknown — for server-generated timestamps or opaque IDs. This rule does not apply to computed fields.",
|
|
12
29
|
"When a response field's value equals the value sent in the request body, path, or query, assert that exact value rather than a null-check — the sent value is known and reproducible.",
|
|
13
|
-
"Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted.
|
|
30
|
+
"Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted. When the behaviour is that a resource was created with the fields sent, the id alone does not prove it — assert those fields.",
|
|
14
31
|
"Range matchers like `toBeGreaterThanOrEqual(0)` and type-only checks like `typeof X === 'number'` are not acceptable for fields whose exact values are known.",
|
|
15
32
|
"Assert the exact status code from the recorded trace or `expected_response_body` — for example, `201` for resource-creation endpoints. Action-style endpoints with verb path segments like `/run`, `/trigger`, `/execute`, `/deploy`, `/submit`, `/start`, `/stop`, or `/cancel` often use `200`, but assert the recorded or expected code. Permissive status matchers are never acceptable.",
|
|
16
33
|
],
|
|
@@ -34,7 +51,7 @@ expect(response.statusCode).toBe(201);`,
|
|
|
34
51
|
"Also apply the array-validation rule to any errors array.",
|
|
35
52
|
"Asserting only the status code is never sufficient when a body is present.",
|
|
36
53
|
"For no-body responses such as a successful DELETE (204): assert the status code only.",
|
|
37
|
-
|
|
54
|
+
'For a 2xx error body, assert the exact status code, the error field\'s exact value, and that the resource fields carry no value (`toBeNull()` / `is None` — the SDK helper reads a missing path and an explicit null the same way, so this asserts "no payload", not "key absent"; use `checkSchema` when the two must be told apart). Never assert the success shape (a non-empty collection, `error` absent) against a response the recorded trace or `expected_response_body` shows returning an error or no-match body — classify the response by what this request returns, not by the outcome the test was meant to check for. The one exception is a test the plan declared `expected.outcome: fail` (typically a `bug_caught` or `requirement_conflict` candidate; the category is only the fallback when nothing was declared): it asserts the intended behaviour established by the diff, the PR statement, or the code\'s evident intent — keep that assertion and let the test fail.',
|
|
38
55
|
],
|
|
39
56
|
examples: [
|
|
40
57
|
{
|
|
@@ -60,16 +77,6 @@ assert skyramp.get_response_value(record_post_response, "insertedId") is None`,
|
|
|
60
77
|
},
|
|
61
78
|
],
|
|
62
79
|
},
|
|
63
|
-
{
|
|
64
|
-
title: "Value ranges",
|
|
65
|
-
description: "For numeric fields where a realistic range is inferable from the field name, domain, or OpenAPI schema (`minimum` / `maximum`), assert the value falls within the expected range.",
|
|
66
|
-
examples: [
|
|
67
|
-
{
|
|
68
|
-
language: "javascript",
|
|
69
|
-
code: `expect(getResponseValue(productsPostResponse, "price")).toBeGreaterThanOrEqual(0);`,
|
|
70
|
-
},
|
|
71
|
-
],
|
|
72
|
-
},
|
|
73
80
|
{
|
|
74
81
|
title: "Specific known values",
|
|
75
82
|
description: "For enum or status fields where only one outcome is valid for this flow, assert the exact expected value.",
|
|
@@ -117,8 +124,8 @@ expect(getValue(response, "data.0")).toBeNull();`,
|
|
|
117
124
|
],
|
|
118
125
|
},
|
|
119
126
|
{
|
|
120
|
-
title: "
|
|
121
|
-
description: "For
|
|
127
|
+
title: "Fields whose value varies across runs",
|
|
128
|
+
description: "For a field whose exact value varies across runs — a UUID, an auto-incremented ID, an ISO timestamp, an IP address, an email, a URL, or a number with no knowable exact value — assert its format or its range instead of its value, and only when the behaviour under test depends on that field. A generated id the test chains onward needs a format check; the same id on a response nothing reads again needs no assertion at all.",
|
|
122
129
|
subPoints: [
|
|
123
130
|
"UUID matches `/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i`.",
|
|
124
131
|
"Auto-incremented ID is greater than 0.",
|
|
@@ -128,6 +135,7 @@ expect(getValue(response, "data.0")).toBeNull();`,
|
|
|
128
135
|
"URL starts with `http://` or `https://`.",
|
|
129
136
|
"Use `not.toBeNull()` / `is not None` only for truly opaque random tokens where no format is recognizable.",
|
|
130
137
|
"Use format info from the OpenAPI schema (`format: uuid`, `format: date-time`) to identify these fields.",
|
|
138
|
+
"A numeric field with no knowable exact value takes a range from the field name, the domain, or the OpenAPI `minimum` / `maximum` — a range is the weakest assertion in this rule set, so use it only where no exact value exists and the behaviour depends on the field.",
|
|
131
139
|
],
|
|
132
140
|
examples: [
|
|
133
141
|
{
|
|
@@ -139,9 +147,8 @@ expect(getResponseValue(productsPostResponse, "created_at")).toMatch(/^\\d{4}-\\
|
|
|
139
147
|
},
|
|
140
148
|
{
|
|
141
149
|
title: "Use exact recorded JSON paths",
|
|
142
|
-
description:
|
|
150
|
+
description: 'Read every field at the path the recorded body actually uses. A wrapper prefix the response does not have makes the assertion read null and the test fail for a reason that is not the behaviour — if the body is flat (`{"id": "acc_...", "tax_exempt": true}`), the path is `"tax_exempt"`, never `"account.tax_exempt"`.',
|
|
143
151
|
subPoints: [
|
|
144
|
-
`If the body is flat (\`{"id": "acc_...", "tax_exempt": true}\`), use \`"tax_exempt"\`, not \`"account.tax_exempt"\`.`,
|
|
145
152
|
"Only use dot notation when the recorded response genuinely nests the field under that key.",
|
|
146
153
|
],
|
|
147
154
|
examples: [
|
|
@@ -216,9 +223,18 @@ export function getAssertionsPrompt(specificRules, scope, testFile, enhanceType)
|
|
|
216
223
|
.join(",\n");
|
|
217
224
|
return `${getPersonaPrefix()}Your task is to enhance response body assertions in the given test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
|
|
218
225
|
|
|
226
|
+
### What to assert
|
|
227
|
+
Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour, and the values it depends on, exactly. Leave the rest of the response alone.
|
|
228
|
+
|
|
229
|
+
A field belongs in the test when a wrong value in it would mean the behaviour is broken. A field does not belong when its value can change for reasons that have nothing to do with the behaviour — a generated id, a timestamp, an unrelated sibling field. Asserting those does not make the test stronger. It makes the test fail for reasons the test is not about, and it buries the one assertion a reader needs to understand what the test is for.
|
|
230
|
+
|
|
231
|
+
So the rules below say how to assert a value, not how many values to assert. Apply a rule to the fields the behaviour depends on. \`[]\` is the correct answer for a rule whose fields this test does not depend on.
|
|
232
|
+
|
|
233
|
+
If nothing in the response distinguishes the behaviour working from the behaviour broken, the test is exercising the wrong request or reading the wrong response. Say so; do not pad it with assertions on whatever else the body carries.
|
|
234
|
+
|
|
219
235
|
### Pre-Edit Assertion Analysis
|
|
220
|
-
Before editing the given file, you must output a \`<thinking>\` block.
|
|
221
|
-
1.
|
|
236
|
+
Before editing the given file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the responses that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion rule title must appear as a key in the \`rule_checklist\`, with \`[]\` where the rule does not apply.
|
|
237
|
+
1. For each test function in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from. Generated test functions are usually named \`test_integration\` or \`testUi\`, which claim nothing — use the name only when it is specific enough to add something, and never treat a generic one as evidence the test has no single behaviour. Then scan the given test file and the expected responses for the code change tested.
|
|
222
238
|
2. Classify each response first by its response status type and then assign the applicable assertion rules to the response.
|
|
223
239
|
1. Success with body (2xx with a response body): all assertion rules below may apply — echo-back of request fields, computed response fields, array / items validation, and chained values across steps.
|
|
224
240
|
2. Success with no body (200/202/204 with an empty body — e.g. logout/cancel/submit action endpoints): assert the status code only. Also apply chained-values rules if a follow-up step uses this response's ID.
|
|
@@ -227,12 +243,14 @@ Before editing the given file, you must output a \`<thinking>\` block. The aim o
|
|
|
227
243
|
3. For each in-scope response, output one JSON object using the template below. The output is an array — one object per in-scope response.
|
|
228
244
|
- \`step\`: the HTTP method, path, and response variable name for this request (e.g. \`POST /products → products_POST_response\`).
|
|
229
245
|
- \`response_status\`: one of \`success\`, \`no_body\`, or \`error\` based on the classification in step 2.
|
|
230
|
-
- \`
|
|
246
|
+
- \`behaviour_under_test\`: the one-line behaviour from step 1 that this response is evidence for. When a response is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
|
|
247
|
+
- \`rule_checklist\`: an object that MUST contain every rule title below as a key. For each rule, the value is the assertion lines you will add for this response under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's fields are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
|
|
231
248
|
|
|
232
249
|
\`\`\`json
|
|
233
250
|
[{
|
|
234
251
|
"step": "<METHOD> <path> → <responseVar>",
|
|
235
252
|
"response_status": "success | no_body | error",
|
|
253
|
+
"behaviour_under_test": "<the one-line behaviour this response is evidence for, or none>",
|
|
236
254
|
"rule_checklist": {
|
|
237
255
|
${ruleChecklistKeys}
|
|
238
256
|
}
|
|
@@ -251,7 +269,8 @@ ${scope}
|
|
|
251
269
|
- Do not access response fields via dict syntax (\`response["field"]\`) or attribute access (\`response.field\`) — always use the SDK helper.
|
|
252
270
|
- Do not assert \`not.toBeNull()\` / \`is not None\` on a field whose exact value is in the request body or a prior response.
|
|
253
271
|
- Do not assert an exact value you learned only from a request you sent yourself while writing the test. Take the value from the diff, from what the request determines, or from a response this test received earlier.
|
|
254
|
-
- Do not
|
|
272
|
+
- Do not assert a field only because the body carries it. If a wrong value there would not mean the behaviour under test is broken, leave it alone.
|
|
273
|
+
- Do not leave a test whose assertions would all still pass with the behaviour under test broken.
|
|
255
274
|
- Do not use permissive status matchers (\`.toMatch(/^2/)\`, \`.toBeGreaterThanOrEqual(200)\`, \`checkStatusCode(response, '20x')\`).
|
|
256
275
|
- Do not use shape-only or type-only assertions as a substitute for exact value validation. Forbidden patterns: \`Array.isArray(...)\`, \`typeof X === '...'\`, \`X instanceof Array\`, \`Object.keys(X).length > 0\`. When a field's value comes from one of those sources, assert it exactly.
|
|
257
276
|
- Do not use shape-only, containment-only, range-only, or weak-length as the sole assertion on a populated array.
|
|
@@ -261,5 +280,5 @@ ${scope}
|
|
|
261
280
|
- Do not remove existing assertions.
|
|
262
281
|
|
|
263
282
|
### Verification of Assertions
|
|
264
|
-
After adding
|
|
283
|
+
After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.`;
|
|
265
284
|
}
|
|
@@ -51,6 +51,8 @@ expect(trigger).to_be_focused()`,
|
|
|
51
51
|
"Anchor on the removed attribute+value or role+name, never on text or position — a text-based absence check breaks when unrelated siblings change.",
|
|
52
52
|
"Do not add a guard for an element whose whole page or route was removed — that is maintenance of the tests that covered it, not a new assertion; a spec that opens a deleted route must be anchored on a surviving page first (a retained element asserted on a live page in the same spec) — a spec whose only visited page is the dead URL is not a valid guard.",
|
|
53
53
|
"A renamed test id or accessible name (a removed `aria-label` included) is not a removed element: assert the element under its new identifier and never assert the retired identifier `toBeHidden()` — it passes today and turns into a false failure the day the name is reused.",
|
|
54
|
+
"An element that exists only inside a control the page has to open — an `option` of a `select`, an item of a closed menu or dropdown — takes `toHaveCount(0)`, not `toBeHidden()`. A closed control's contents have no bounding box, so `toBeHidden()` passes whether the element was removed or not, and the guard stays green the day it comes back. If the spec opens the control before asserting, the element would have had a box had it survived, so assert both.",
|
|
55
|
+
"The positive companion for that guard is counted or read, never `toBeVisible`: the same missing bounding box that makes `toBeHidden()` a false green on the removed entry makes `toBeVisible()` a false red on the one that stayed. Assert a retained sibling with `toHaveCount(1)` or `toHaveText`, or assert the control itself, which does have a box.",
|
|
54
56
|
],
|
|
55
57
|
examples: [
|
|
56
58
|
{
|
|
@@ -83,7 +85,7 @@ expect(page.get_by_test_id('navbar-products')).to_have_text('Products')`,
|
|
|
83
85
|
},
|
|
84
86
|
{
|
|
85
87
|
title: "Tests target the changed behavior introduced by the pull request",
|
|
86
|
-
description: "
|
|
88
|
+
description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support: a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced. Target it on populated or updated state, not only on empty or zero state, and assert it on the page where the behaviour shows. When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running app still renders the old value; the test is expected to fail until the code is fixed.",
|
|
87
89
|
examples: [],
|
|
88
90
|
},
|
|
89
91
|
{
|
|
@@ -222,6 +224,9 @@ export function getUIAssertionsPrompt(testFile, enhanceType) {
|
|
|
222
224
|
const categoryTemplate = renderAssertionCategoriesTemplate(UI_ASSERTION_CATEGORIES);
|
|
223
225
|
return `${getPersonaPrefix()}Your task is to enhance assertions for the given UI test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
|
|
224
226
|
|
|
227
|
+
### What to assert
|
|
228
|
+
Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour on the page where it shows, and the rendered values it depends on, exactly. A page the test only passes through, and an element the behaviour does not touch, need no assertion: asserting them makes the test fail for reasons the test is not about and buries what the test is for. The rules below say how to assert a value, not how many values to assert.
|
|
229
|
+
|
|
225
230
|
### First Check
|
|
226
231
|
If the generated test file has no \`expect()\` assertions, you MUST manually add them before anything else. Use \`import { expect } from '@skyramp/skyramp';\` — never from \`@playwright/test\`. If an existing import pulls \`expect\` from \`@playwright/test\`, move it to \`@skyramp/skyramp\` (keep \`test\` on the playwright line).
|
|
227
232
|
|
|
@@ -328,25 +333,28 @@ if (await existingSecret.isVisible().catch(() => false)) {
|
|
|
328
333
|
\`\`\`
|
|
329
334
|
|
|
330
335
|
### Pre-Edit Assertion Analysis
|
|
331
|
-
Before editing the given test file, you must output a \`<thinking>\` block.
|
|
332
|
-
1.
|
|
333
|
-
2.
|
|
336
|
+
Before editing the given test file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the items that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion category and every rule title under it must appear as a key, with \`[]\` where the rule does not apply.
|
|
337
|
+
1. For each test in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from, and name the page where that behaviour shows. Generated specs are usually named \`testUi\`, which claims nothing — use the name only when it is specific enough to add something.
|
|
338
|
+
2. Selector inventory — list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use only selectors from this list. An assertion the recorder dropped because it failed during recording may use the selector of the element named in that recording's blueprint capture. Do not invent \`data-testid\` values, role names, or aria attributes. Also note captured network responses, repeated element patterns, exact rendered text/value/attribute from trace/source, and existing \`toBeVisible()\` assertions whose exact text is knowable.
|
|
339
|
+
3. Process — Replay → Identify → Fix or Add — walk through these three steps explicitly.
|
|
334
340
|
a. Replay the scenario mentally. At each state-changing action (form submit, item add/edit/delete), ask: "What is the EXPECTED outcome based on the action performed?"
|
|
335
341
|
b. Identify expectation mismatches. If the recorded trace shows a result that contradicts the action (e.g. removing 1 of 2 items but the page shows 3, submitting a form but getting a blank page, editing a field but the old value persists), that is an app bug the test should catch. List every mismatch you find.
|
|
336
342
|
c. Fix or add assertions for each mismatch.
|
|
337
343
|
- If an existing assertion uses the wrong (buggy) value, edit it to assert the correct expected value.
|
|
338
344
|
- If no assertion exists for the buggy behavior, add one immediately after the action that triggers it.
|
|
339
|
-
|
|
345
|
+
4. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
|
|
340
346
|
- Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
|
|
341
347
|
- Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
|
|
342
348
|
- Post-edit State applies whenever EITHER (a) the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), OR (b) the PR changes interactive behavior — in case (b), add the triggering action and assert the before/after transition (see "Exercise the changed interaction"). Mark NOT APPLICABLE only when the PR changes no interactive behavior and the test has no state-changing action.
|
|
343
|
-
|
|
349
|
+
5. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
|
|
344
350
|
- \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
|
|
345
|
-
- \`
|
|
351
|
+
- \`behaviour_under_test\`: the one-line behaviour from step 1 that this item is evidence for. When an item is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
|
|
352
|
+
- \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is the assertion lines you will add for this item under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's elements are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
|
|
346
353
|
|
|
347
354
|
\`\`\`json
|
|
348
355
|
[{
|
|
349
356
|
"action_or_selector_or_response": "<selector | action | response>",
|
|
357
|
+
"behaviour_under_test": "<the one-line behaviour this item is evidence for, or none>",
|
|
350
358
|
"assertion_categories": {
|
|
351
359
|
${categoryTemplate}
|
|
352
360
|
}
|
|
@@ -354,7 +362,7 @@ ${categoryTemplate}
|
|
|
354
362
|
\`\`\`
|
|
355
363
|
|
|
356
364
|
### Assertion Rules with Examples
|
|
357
|
-
Most-violated patterns —
|
|
365
|
+
Most-violated patterns — whenever the behaviour under test involves one of these, this is the shape it takes. (1) repeated elements → \`toHaveCount(N)\` + per-item \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` (Collection / repeated UI elements). (2) post-action state → \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\`/\`toBeChecked\`, not \`toBeVisible\` when exact value is knowable (Post-action visible state). (3) routing/URL-only tests → at least one rendered-element exact text (Positive-path companion).
|
|
358
366
|
Strength order. \`toHaveCount\`/\`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` > \`toContainText\` (only when string is genuinely dynamic) > \`toBeVisible\` (only when presence is the actual test).
|
|
359
367
|
|
|
360
368
|
${renderCategories(UI_ASSERTION_CATEGORIES)}
|
|
@@ -373,7 +381,7 @@ ${renderCategories(UI_ASSERTION_CATEGORIES)}
|
|
|
373
381
|
- Do not assert buggy/error text as expected (unless the intended UX is an error state)
|
|
374
382
|
|
|
375
383
|
### Verification of Assertions
|
|
376
|
-
After adding
|
|
384
|
+
After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.
|
|
377
385
|
|
|
378
386
|
The goal is tests that FAIL when the app has bugs, not tests that simply replay what happened.
|
|
379
387
|
`;
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
|
|
2
2
|
import { sectionBody } from "../promptAssets.js";
|
|
3
3
|
import { readPromptAsset } from "../promptAssets.js";
|
|
4
|
-
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
5
4
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
6
5
|
import { PromptPlan } from "./promptPlan.js";
|
|
7
6
|
import { buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, buildFinishChecks, } from "./recommendationSections.js";
|
|
8
7
|
const EXECUTION_PLAN_MD = readPromptAsset("generate-tests/execution-plan.md");
|
|
9
8
|
function _execEnrichBody(ctx) {
|
|
10
9
|
return fillPlaceholders(sectionBody(EXECUTION_PLAN_MD, "## Parameter grounding"), {
|
|
11
|
-
authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
|
|
12
10
|
enrichSurface: ctx.isDiffScope ? "the diff" : "the analysed surface",
|
|
13
11
|
testPatternGuidelines: buildTestPatternGuidelines(),
|
|
14
12
|
});
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { AnalysisScope, isDiff, } from "../../types/RepositoryAnalysis.js";
|
|
2
|
-
import { WorkspaceAuthType, getDefaultAuthHeader } from "../../utils/workspaceAuth.js";
|
|
2
|
+
import { WorkspaceAuthType, getDefaultAuthHeader, AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
4
|
import { buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildToolWorkflows, buildVerificationChecklist, } from "./recommendationSections.js";
|
|
5
5
|
import { buildExecutionPlan, EXEC_STEP_ENRICH } from "./diffExecutionPlan.js";
|
|
6
6
|
import { readPromptAsset } from "../promptAssets.js";
|
|
7
|
+
import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
|
|
7
8
|
import { TASK_GENERATE, taskRef, } from "./recommendationShared.js";
|
|
8
9
|
// Re-export for backward compatibility (tests and external callers import this from this module)
|
|
9
10
|
function formatTestLocations(locs) {
|
|
@@ -25,7 +26,12 @@ const SKYRAMP_TEST_FILE_PATTERN = /(?:_test|_smoke|_contract|_fuzz|_integration|
|
|
|
25
26
|
* one text and nothing to drift from. `isDiffScope` is unused here — the
|
|
26
27
|
* procedure reads the same for a diff and for a repository. */
|
|
27
28
|
function planningInstruction(_isDiffScope) {
|
|
28
|
-
|
|
29
|
+
// The procedure names the auth patterns. BOTH renderers of this asset fill the
|
|
30
|
+
// placeholder — this lane's and the testbot lane's — or one of them ships the
|
|
31
|
+
// literal `{authMiddlewarePatterns}` to the model.
|
|
32
|
+
return fillPlaceholders(readPromptAsset("plan-tests.md").trim(), {
|
|
33
|
+
authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
|
|
34
|
+
});
|
|
29
35
|
}
|
|
30
36
|
export function buildRecommendationPrompt(analysis, analysisScope = AnalysisScope.FullRepo, prContext, workspaceAuthHeader, workspaceAuthType, workspaceAuthScheme, sessionId) {
|
|
31
37
|
const isDiffScope = isDiff(analysisScope);
|
|
@@ -5,6 +5,7 @@ import { buildPathParamGuidance, } from "../test-recommendation/recommendationSe
|
|
|
5
5
|
import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
6
6
|
import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
|
|
7
7
|
import { recordPullRequestText } from "../../recommendation/pullRequestText.js";
|
|
8
|
+
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
8
9
|
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, TESTBOT_TASK1_MAINTAIN_LABELS, TESTBOT_TASK1_MULTIREPO_LABELS, TESTBOT_TASK1_STEP_LABELS, orderedLabels, stepSubRef, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
9
10
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
10
11
|
import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, isSkillsLoaded, } from "../../utils/featureFlags.js";
|
|
@@ -59,24 +60,28 @@ const PLAN_TESTS_MD = readPromptAsset("plan-tests.md").trim();
|
|
|
59
60
|
// rendered neither it nor a copy, so the agent planned without the claims step and
|
|
60
61
|
// without the backward surface map. Rendered here, never retyped.
|
|
61
62
|
//
|
|
62
|
-
// It sits OUTSIDE Task 1: the procedure is a
|
|
63
|
+
// It sits OUTSIDE Task 1: the procedure is a numbered list, Task 1's
|
|
63
64
|
// own steps are numbered too, and nested inside Task 1 the two share one ordinal
|
|
64
65
|
// space (`referenceIntegrity.test.ts` reads every `N.` there as a Task 1 step).
|
|
65
66
|
//
|
|
66
|
-
// Two steps run elsewhere in this lane —
|
|
67
|
-
//
|
|
68
|
-
//
|
|
67
|
+
// Two steps run elsewhere in this lane — the maintenance update in **Group the
|
|
68
|
+
// changes into behaviours and decide each** is Task 1's tool step, and **Register the
|
|
69
|
+
// whole plan in one call to `skyramp_register_test_plan`** happens in Task 2 after it.
|
|
70
|
+
// The note states that order; the markdown stays the source.
|
|
69
71
|
//
|
|
70
72
|
// The count sentence sits HERE, beside the register instruction, because the agent
|
|
71
73
|
// reads the URI first and plans straight after: on run 34005367281 it read
|
|
72
74
|
// `maxGenerate=3` as this run's budget and left its fourth planned test unwritten.
|
|
73
|
-
|
|
75
|
+
// The procedure names the auth patterns. BOTH renderers of this asset fill the
|
|
76
|
+
// placeholder — this lane's and the recommendation lane's — or one of them ships
|
|
77
|
+
// the literal `{authMiddlewarePatterns}` to the model.
|
|
78
|
+
const planProcedureBlock = fillPlaceholders(`${section(PLAN_TESTS_MD, "## Terms")}
|
|
74
79
|
|
|
75
80
|
${section(PLAN_TESTS_MD, "## The procedure")}
|
|
76
81
|
|
|
77
82
|
**In this run:** ${taskRef(TASK_ANALYZE_MAINTAIN)}'s maintenance step performs the maintenance update of an existing test, with the maintenance tools — a test those tools update is not a plan planned test. Register the plan in ${taskRef(TASK_GENERATE)}, after that step. Nothing caps how many planned tests the plan holds: \`maxGenerate\` and \`maxRecommendations\` arrive in this run's URI, and the server accepts both and applies neither — no check reads either one, so a count you see there is not a budget.
|
|
78
83
|
|
|
79
|
-
${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}
|
|
84
|
+
${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}`, { authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR });
|
|
80
85
|
// The checks on the test file itself. One home — `generation.md` — rendered by
|
|
81
86
|
// this lane at the end of generation and by the IDE lane after its generation
|
|
82
87
|
// rules, so neither carries a retyped copy.
|
|
@@ -460,7 +465,7 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
460
465
|
- \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
|
|
461
466
|
- \`skyramp_ui_test_generation\`: set \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript Playwright only — the default; leave \`codeReuse\` unset for other languages)
|
|
462
467
|
- **\`browser_assert\`**: the \`browser_navigate\` result states the rule this tool has to satisfy on every page you reach. Call multiple assertions in the same tool call batch when checking independent elements.
|
|
463
|
-
- **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
|
|
468
|
+
- **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. When the removed element lived inside a control the page has to open — an \`option\` of a \`select\`, an item of a closed menu — open that control before asserting: a closed control's contents are never visible, so a hidden assertion recorded against a shut one passes whether the element was removed or not. \`browser_assert_hidden\` stays the tool either way, because \`browser_assert\` needs a snapshot \`ref\` and a removed element has none; where the control cannot be opened, say so in \`reasoning\` so the assertion step turns that guard into a count. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
|
|
464
469
|
- **Renames are not removals — on every PR, whatever \`uiContext.removedElements\` holds**: when a test id, \`aria-label\` or other accessible name changed, assert the element by its new identifier and leave the retired identifier alone (no \`browser_assert_hidden\` on it). A rename PR removes nothing, so this rule applies when the removed list is empty.
|
|
465
470
|
- **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
|
|
466
471
|
- **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Objection, ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlannedTest, VerifyContext } from "./types.js";
|
|
1
|
+
import { Objection, ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlanMaintenance, PlannedTest, VerifyContext } from "./types.js";
|
|
2
2
|
/** The plan as stored for the run. Every registration produces one; the newest
|
|
3
3
|
* replaces the last. */
|
|
4
4
|
export interface Plan {
|
|
@@ -10,6 +10,10 @@ export interface Plan {
|
|
|
10
10
|
* test that proves it. */
|
|
11
11
|
defects: PlanDefect[];
|
|
12
12
|
plannedTests: PlannedTest[];
|
|
13
|
+
/** Existing tests this run edits instead of planning new ones. Stored because
|
|
14
|
+
* `coverage` counts the changes they cite, so a reader of the plan has to be
|
|
15
|
+
* able to see which coverage came from maintenance rather than a new test. */
|
|
16
|
+
maintains?: PlanMaintenance[];
|
|
13
17
|
/** Objections the agent answered, with the answers. */
|
|
14
18
|
answeredObjections: Array<{
|
|
15
19
|
objection: Objection;
|
|
@@ -110,6 +110,11 @@ export function registerPlan(registration, ctx, previous) {
|
|
|
110
110
|
changes,
|
|
111
111
|
defects,
|
|
112
112
|
plannedTests,
|
|
113
|
+
// Only when declared: the field is optional, and writing `[]` on every plan
|
|
114
|
+
// would change the stored shape of every run that maintains nothing.
|
|
115
|
+
...(Array.isArray(registration.maintains) && registration.maintains.length > 0
|
|
116
|
+
? { maintains: registration.maintains }
|
|
117
|
+
: {}),
|
|
113
118
|
answeredObjections,
|
|
114
119
|
unverifiedCloses,
|
|
115
120
|
openObjections,
|
|
@@ -41,9 +41,15 @@ export interface PlanChange {
|
|
|
41
41
|
* test asserts a value the plan decided, not the value the running app
|
|
42
42
|
* happened to return — the one source that always agrees with a defect. */
|
|
43
43
|
expectedValue?: string | number | boolean | null;
|
|
44
|
-
/** Where the agent read `expectedValue
|
|
45
|
-
* `convention:<file:line>`, or
|
|
44
|
+
/** Where the agent read `expectedValue` OR `derived` — the check reads it for
|
|
45
|
+
* either one: `pr-description`, `spec:<path>`, `convention:<file:line>`, or
|
|
46
|
+
* `code`. */
|
|
46
47
|
expectedFrom?: string;
|
|
48
|
+
/** The rule the value follows from, in the source's words, when the source
|
|
49
|
+
* states a rule and not a value. Stands in for `expectedValue`: the test
|
|
50
|
+
* computes the number from what it sent, so nothing pins one the source
|
|
51
|
+
* never wrote. */
|
|
52
|
+
derived?: string;
|
|
47
53
|
}>;
|
|
48
54
|
}
|
|
49
55
|
/** One defect the code review found, as the agent read it. Nothing on the server
|
|
@@ -124,6 +130,20 @@ export interface ObjectionAnswer {
|
|
|
124
130
|
* An objection about an untested change closes only with one of these. */
|
|
125
131
|
blocker?: string;
|
|
126
132
|
}
|
|
133
|
+
/** An existing test this run maintains instead of planning a new one, and the
|
|
134
|
+
* declared changes that maintenance covers. A removal already covered by a spec
|
|
135
|
+
* is maintained, not duplicated, so the plan needs a way to say "this change is
|
|
136
|
+
* tested by a file I am editing" — without it, `coverage:change:` is unanswerable
|
|
137
|
+
* for a correct maintenance decision, and it closes only with a `blocker`. */
|
|
138
|
+
export interface PlanMaintenance {
|
|
139
|
+
/** Repository-relative path of the existing test. Only its presence in the
|
|
140
|
+
* checkout is checked, the same rule `declarations.existingTests` follows. */
|
|
141
|
+
file: string;
|
|
142
|
+
/** The edit this run makes to it, in one sentence. */
|
|
143
|
+
differsBy: string;
|
|
144
|
+
/** The ids of the declared changes this maintenance covers. */
|
|
145
|
+
changes: string[];
|
|
146
|
+
}
|
|
127
147
|
export interface PlanInput {
|
|
128
148
|
/** The changes the diff makes. Each planned test cites the ones it tests. */
|
|
129
149
|
changes: PlanChange[];
|
|
@@ -132,6 +152,9 @@ export interface PlanInput {
|
|
|
132
152
|
* objection the agent answers. */
|
|
133
153
|
defects: PlanDefect[];
|
|
134
154
|
plannedTests: PlannedTest[];
|
|
155
|
+
/** Existing tests this run edits rather than replacing with new ones. Coverage
|
|
156
|
+
* counts the changes these cite, so a maintenance-only plan is a complete plan. */
|
|
157
|
+
maintains?: PlanMaintenance[];
|
|
135
158
|
answers: ObjectionAnswer[];
|
|
136
159
|
/** The tool owns this and overwrites whatever the caller sends. */
|
|
137
160
|
registrationNumber: number;
|
|
@@ -106,8 +106,16 @@ export declare const EXISTING_COVERAGE_CONTRACT: {
|
|
|
106
106
|
readonly message: "This planned test names an existing test file that is not in the checkout.";
|
|
107
107
|
readonly suggestion: "Name the file repository-relative, spelled as the analysis lists it. Drop the entry if it names a test you did not read — naming none is an answer too.";
|
|
108
108
|
};
|
|
109
|
+
readonly maintainedTestMissing: {
|
|
110
|
+
readonly message: "`maintains` names an existing test file that is not in the checkout.";
|
|
111
|
+
readonly suggestion: "Name the file repository-relative, spelled as the analysis lists it. A maintenance entry is what makes its changes count as covered, so it has to name a file this run can edit.";
|
|
112
|
+
};
|
|
113
|
+
readonly maintainedUnknownChange: {
|
|
114
|
+
readonly message: "A `maintains` entry cites a change id that this plan does not declare.";
|
|
115
|
+
readonly suggestion: "Spell the change's `id` exactly as your own `changes` list gives it — the match is exact apart from surrounding space. Declare the change if it is missing.";
|
|
116
|
+
};
|
|
109
117
|
};
|
|
110
|
-
readonly suggestion: "Which existing tests did you read on this change,
|
|
118
|
+
readonly suggestion: "Which existing tests did you read on this change, how does this planned test differ from them, and which changes does the maintenance you are doing cover?";
|
|
111
119
|
};
|
|
112
120
|
export declare const EXPECTED_OUTCOME_CONTRACT: {
|
|
113
121
|
readonly id: "expectedOutcome";
|
|
@@ -150,7 +158,7 @@ export declare const COVERAGE_CONTRACT: {
|
|
|
150
158
|
readonly objections: {
|
|
151
159
|
readonly change: {
|
|
152
160
|
readonly message: "This declared change has no test in the plan.";
|
|
153
|
-
readonly suggestion: "Plan a test that exercises it. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
|
|
161
|
+
readonly suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
|
|
154
162
|
};
|
|
155
163
|
readonly noChanges: {
|
|
156
164
|
readonly message: "The plan declares no changes, so nothing says what the pull request must make different.";
|
|
@@ -218,11 +226,15 @@ export declare const EXPECTED_VALUE_SOURCED_CONTRACT: {
|
|
|
218
226
|
};
|
|
219
227
|
readonly quoteMissing: {
|
|
220
228
|
readonly message: "This case reads its value from the pull request, and its change does not quote the sentence that states the rule.";
|
|
221
|
-
readonly suggestion: "Put the sentence from the title or description on the change as `quote`, as written there.
|
|
229
|
+
readonly suggestion: "Put the sentence from the title or description on the change as `quote`, as written there. If that sentence states the rule and not the number, put the rule in `derived` and leave `expectedValue` out.";
|
|
230
|
+
};
|
|
231
|
+
readonly valueNotQuoted: {
|
|
232
|
+
readonly message: "This case reads its value from the pull request, and the pull request does not state this value.";
|
|
233
|
+
readonly suggestion: "Quote the sentence that states it, or, if the pull request states a rule, put the rule in `derived` and leave `expectedValue` out; the test computes the value from what it sent.";
|
|
222
234
|
};
|
|
223
235
|
readonly failWithoutValue: {
|
|
224
236
|
readonly message: "This test expects to fail, and no case on the changes it cites states the value it must assert.";
|
|
225
|
-
readonly suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If
|
|
237
|
+
readonly suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If the source states a rule rather than a number, put the rule in `derived` instead. If no source states either, answer with what the test asserts instead and where that comes from.";
|
|
226
238
|
};
|
|
227
239
|
};
|
|
228
240
|
readonly suggestion: "Where does the correct value for this case come from?";
|