@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -8,7 +8,7 @@ const specificRules = [
|
|
|
8
8
|
"When an ID was sent in the request body, path, or query, assert its exact value in the response. Since the sent value is already known, asserting only that it is non-null is not sufficient.",
|
|
9
9
|
"Reserve `is not None` / `not.toBeNull()` for server-generated IDs only.",
|
|
10
10
|
"Apply the same rule to all other fields and to array items: use inline request body values as the expected values, not setup helper data.",
|
|
11
|
-
"
|
|
11
|
+
"The behaviour this test claims must be proven from `expected_response_body`, the inline request body, or path/query literals — do not rely on the integration test to cover it.",
|
|
12
12
|
],
|
|
13
13
|
examples: [
|
|
14
14
|
{
|
|
@@ -21,6 +21,7 @@ const specificRules = [
|
|
|
21
21
|
const SCOPE = `### Scope
|
|
22
22
|
- Only modify test functions — do not touch \`beforeAll\`, \`afterAll\`, or any setup or teardown helper.
|
|
23
23
|
- Only add assertions clearly supported by \`expected_response_body\`, inline request / path / query literals, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
|
|
24
|
+
- A response no behaviour in this file depends on needs its status code and nothing more.
|
|
24
25
|
- Add new assertions immediately after the existing status-code assertion — do not move or remove anything.
|
|
25
26
|
- Do not reference \`beforeAll\` / \`afterAll\` provisioning data in any assertion — every assertion value must come from the inline request body, path, query, prior response, or \`expected_response_body\`.`;
|
|
26
27
|
export function getContractProviderAssertionsPrompt(testFile, enhanceType) {
|
|
@@ -9,7 +9,7 @@ const specificRules = [
|
|
|
9
9
|
"After any POST, PATCH, DELETE, sort, reorder, or bulk operation, re-assert the chained, stable, and computed values on the follow-up read step: a follow-up GET after DELETE should return 404 or show the item absent; a follow-up GET after PATCH should assert the new value; a follow-up GET after sort/reorder should confirm the new ordering with chained IDs.",
|
|
10
10
|
"After a POST creates a resource and a GET retrieves the collection, assert that the created item appears in the list by its chained ID with its exact stable fields — do not stop at a null-check on the array or the id alone.",
|
|
11
11
|
"When one response field describes a count or summary of a collection returned by a related endpoint (for example, `active_session_count` from `/users/me` and the array length from `/users/me/sessions`), assert that relationship by comparing both extracted values — never hardcode the same number in two places.",
|
|
12
|
-
|
|
12
|
+
'A state change the test is for ("Updates X", "Increments Y" — from the recommendation it came from, or from the test name when that name says more than `test_integration`) must be proven across the calls that carry it — do not rely on the contract test to cover it.',
|
|
13
13
|
],
|
|
14
14
|
examples: [
|
|
15
15
|
{
|
|
@@ -29,6 +29,7 @@ await skyramp.sendRequest(\`/orders/\${orderId}\`);`,
|
|
|
29
29
|
const SCOPE = `### Scope
|
|
30
30
|
- Apply to every \`send_request\` / \`sendRequest\` call that returns a body.
|
|
31
31
|
- Only add assertions clearly supported by the request body, prior response values, field names, codebase evidence, or the test generation recommendations received for this test. Do not invent constraints.
|
|
32
|
+
- A response no behaviour in this file depends on needs its status code and nothing more.
|
|
32
33
|
- Add new assertions immediately after the existing status-code assertion — do not move or remove anything.`;
|
|
33
34
|
export function getIntegrationAssertionsPrompt(testFile, enhanceType) {
|
|
34
35
|
return getAssertionsPrompt(specificRules, SCOPE, testFile, enhanceType);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type EnhanceType = "generation" | "maintenance";
|
|
2
|
-
export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
|
|
2
|
+
export declare const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
|
|
3
3
|
export declare function maintenanceTaskSuffix(enhanceType: EnhanceType): string;
|
|
4
4
|
export interface AssertionEnrichmentExample {
|
|
5
5
|
language: string;
|
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
import { getPersonaPrefix } from "../personas.js";
|
|
2
|
-
export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff.";
|
|
2
|
+
export const MAINTENANCE_SCOPE_NOTE = "Apply only to new test functions you are adding and existing test functions affected by changes in the diff. Do NOT modify test functions unrelated to the diff. A test function you do touch is held to the same standard as a new one: the behaviour the diff changed must be asserted by its exact value there too, so an existing `is not None` on a field the diff gives a knowable value becomes that value.";
|
|
3
3
|
export function maintenanceTaskSuffix(enhanceType) {
|
|
4
4
|
return enhanceType === "maintenance" ? ` ${MAINTENANCE_SCOPE_NOTE}` : "";
|
|
5
5
|
}
|
|
6
6
|
const SHARED_RULES = [
|
|
7
|
+
{
|
|
8
|
+
title: "Assert the behaviour under test",
|
|
9
|
+
description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support; a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced.",
|
|
10
|
+
subPoints: [
|
|
11
|
+
"Name the behaviour, then ask what value in the response changes when it breaks. Assert that value exactly.",
|
|
12
|
+
'When the test claims a state change ("Updates X", "Increments Y", "rejects Z"), the assertion must prove the change, not merely that the call succeeded.',
|
|
13
|
+
"When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running service still returns the old value. The test is expected to fail until the code is fixed.",
|
|
14
|
+
],
|
|
15
|
+
examples: [
|
|
16
|
+
{
|
|
17
|
+
language: "javascript",
|
|
18
|
+
code: `// "PATCH /orders applies the coupon discount" — the discount is the behaviour.
|
|
19
|
+
expect(getResponseValue(patchResponse, "discount_amount"), 'discount applied').toBe(20);
|
|
20
|
+
expect(getResponseValue(patchResponse, "total_amount"), 'total after discount').toBe(80);`,
|
|
21
|
+
},
|
|
22
|
+
],
|
|
23
|
+
},
|
|
7
24
|
{
|
|
8
25
|
title: "Echo-back the request fields",
|
|
9
26
|
description: "For every field returned unchanged from the request body, assert the exact sent value.",
|
|
10
27
|
subPoints: [
|
|
11
28
|
"`is not None` / `not.toBeNull()` is only acceptable when the value is genuinely unknown — for server-generated timestamps or opaque IDs. This rule does not apply to computed fields.",
|
|
12
29
|
"When a response field's value equals the value sent in the request body, path, or query, assert that exact value rather than a null-check — the sent value is known and reproducible.",
|
|
13
|
-
"Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted.
|
|
30
|
+
"Assert the exact value of a response-only field when the request this test sends determines it (`filename_download`, `content_type`, `size`, an enum status after creation). A value the app generates gets a format check when the test case depends on that value; leave the rest unasserted. When the behaviour is that a resource was created with the fields sent, the id alone does not prove it — assert those fields.",
|
|
14
31
|
"Range matchers like `toBeGreaterThanOrEqual(0)` and type-only checks like `typeof X === 'number'` are not acceptable for fields whose exact values are known.",
|
|
15
32
|
"Assert the exact status code from the recorded trace or `expected_response_body` — for example, `201` for resource-creation endpoints. Action-style endpoints with verb path segments like `/run`, `/trigger`, `/execute`, `/deploy`, `/submit`, `/start`, `/stop`, or `/cancel` often use `200`, but assert the recorded or expected code. Permissive status matchers are never acceptable.",
|
|
16
33
|
],
|
|
@@ -28,12 +45,13 @@ expect(response.statusCode).toBe(201);`,
|
|
|
28
45
|
],
|
|
29
46
|
},
|
|
30
47
|
{
|
|
31
|
-
title: "Error path (HTTP 4xx/5xx
|
|
32
|
-
description: "For every
|
|
48
|
+
title: "Error path (HTTP 4xx/5xx, or a 2xx carrying only an error body)",
|
|
49
|
+
description: "For every error response that includes a body, assert every error body field with its exact value — including `error.code`, `error.message`, `detail`, `errors[0].message`, and `errors[0].extensions.code`. An error response is any 4xx/5xx with a body, and any 2xx whose body is an error object (`error`, `message`, `errors[]`) with no resource payload.",
|
|
33
50
|
subPoints: [
|
|
34
51
|
"Also apply the array-validation rule to any errors array.",
|
|
35
52
|
"Asserting only the status code is never sufficient when a body is present.",
|
|
36
53
|
"For no-body responses such as a successful DELETE (204): assert the status code only.",
|
|
54
|
+
'For a 2xx error body, assert the exact status code, the error field\'s exact value, and that the resource fields carry no value (`toBeNull()` / `is None` — the SDK helper reads a missing path and an explicit null the same way, so this asserts "no payload", not "key absent"; use `checkSchema` when the two must be told apart). Never assert the success shape (a non-empty collection, `error` absent) against a response the recorded trace or `expected_response_body` shows returning an error or no-match body — classify the response by what this request returns, not by the outcome the test was meant to check for. The one exception is a test the plan declared `expected.outcome: fail` (typically a `bug_caught` or `requirement_conflict` candidate; the category is only the fallback when nothing was declared): it asserts the intended behaviour established by the diff, the PR statement, or the code\'s evident intent — keep that assertion and let the test fail.',
|
|
37
55
|
],
|
|
38
56
|
examples: [
|
|
39
57
|
{
|
|
@@ -43,15 +61,19 @@ expect(getValue(response, "errors.0.message")).toBe("Item not found");
|
|
|
43
61
|
expect(getValue(response, "errors.0.extensions.code")).toBe("RECORD_NOT_FOUND");
|
|
44
62
|
expect(getValue(response, "errors.1")).toBeNull();`,
|
|
45
63
|
},
|
|
46
|
-
],
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
title: "Value ranges",
|
|
50
|
-
description: "For numeric fields where a realistic range is inferable from the field name, domain, or OpenAPI schema (`minimum` / `maximum`), assert the value falls within the expected range.",
|
|
51
|
-
examples: [
|
|
52
64
|
{
|
|
53
65
|
language: "javascript",
|
|
54
|
-
code:
|
|
66
|
+
code: `// Recorded response is a 200 carrying an error object and no payload
|
|
67
|
+
expect(searchPostResponse.statusCode, 'status code').toBe(200);
|
|
68
|
+
expect(getResponseValue(searchPostResponse, "error"), 'error').toBe("No items matched the criteria.");
|
|
69
|
+
expect(getResponseValue(searchPostResponse, "results"), 'results carries no value').toBeNull();`,
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
language: "python",
|
|
73
|
+
code: `# Recorded response: 400 {"error": "level must be one of Intern, Junior, Senior"}
|
|
74
|
+
assert record_post_response.status_code == 400
|
|
75
|
+
assert skyramp.get_response_value(record_post_response, "error") == "level must be one of Intern, Junior, Senior"
|
|
76
|
+
assert skyramp.get_response_value(record_post_response, "insertedId") is None`,
|
|
55
77
|
},
|
|
56
78
|
],
|
|
57
79
|
},
|
|
@@ -77,7 +99,7 @@ expect(getValue(response, "errors.1")).toBeNull();`,
|
|
|
77
99
|
"When the response is a non-empty array: assert the exact length, key fields on each item, and that the index after the last item is absent.",
|
|
78
100
|
"Shape-only checks such as `Array.isArray` or `typeof` are not sufficient when the response contains actual values.",
|
|
79
101
|
"If the response is sorted or ordered: assert the ordering direction across the first two items.",
|
|
80
|
-
"When the response is an empty or minimal body (an empty object, empty array, null, or only a few keys): assert the empty or minimal shape and the absence of error fields — do not stop at the status code.",
|
|
102
|
+
"When the response is an empty or minimal body (an empty object, empty array, null, or only a few keys): assert the empty or minimal shape and the absence of error fields — do not stop at the status code. A minimal body that is itself an error object falls under the Error path rule instead: assert its error field exactly.",
|
|
81
103
|
"When the request includes pagination or filter parameters, assert the response reflects them.",
|
|
82
104
|
],
|
|
83
105
|
examples: [
|
|
@@ -102,8 +124,8 @@ expect(getValue(response, "data.0")).toBeNull();`,
|
|
|
102
124
|
],
|
|
103
125
|
},
|
|
104
126
|
{
|
|
105
|
-
title: "
|
|
106
|
-
description: "For
|
|
127
|
+
title: "Fields whose value varies across runs",
|
|
128
|
+
description: "For a field whose exact value varies across runs — a UUID, an auto-incremented ID, an ISO timestamp, an IP address, an email, a URL, or a number with no knowable exact value — assert its format or its range instead of its value, and only when the behaviour under test depends on that field. A generated id the test chains onward needs a format check; the same id on a response nothing reads again needs no assertion at all.",
|
|
107
129
|
subPoints: [
|
|
108
130
|
"UUID matches `/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i`.",
|
|
109
131
|
"Auto-incremented ID is greater than 0.",
|
|
@@ -113,6 +135,7 @@ expect(getValue(response, "data.0")).toBeNull();`,
|
|
|
113
135
|
"URL starts with `http://` or `https://`.",
|
|
114
136
|
"Use `not.toBeNull()` / `is not None` only for truly opaque random tokens where no format is recognizable.",
|
|
115
137
|
"Use format info from the OpenAPI schema (`format: uuid`, `format: date-time`) to identify these fields.",
|
|
138
|
+
"A numeric field with no knowable exact value takes a range from the field name, the domain, or the OpenAPI `minimum` / `maximum` — a range is the weakest assertion in this rule set, so use it only where no exact value exists and the behaviour depends on the field.",
|
|
116
139
|
],
|
|
117
140
|
examples: [
|
|
118
141
|
{
|
|
@@ -124,9 +147,8 @@ expect(getResponseValue(productsPostResponse, "created_at")).toMatch(/^\\d{4}-\\
|
|
|
124
147
|
},
|
|
125
148
|
{
|
|
126
149
|
title: "Use exact recorded JSON paths",
|
|
127
|
-
description:
|
|
150
|
+
description: 'Read every field at the path the recorded body actually uses. A wrapper prefix the response does not have makes the assertion read null and the test fail for a reason that is not the behaviour — if the body is flat (`{"id": "acc_...", "tax_exempt": true}`), the path is `"tax_exempt"`, never `"account.tax_exempt"`.',
|
|
128
151
|
subPoints: [
|
|
129
|
-
`If the body is flat (\`{"id": "acc_...", "tax_exempt": true}\`), use \`"tax_exempt"\`, not \`"account.tax_exempt"\`.`,
|
|
130
152
|
"Only use dot notation when the recorded response genuinely nests the field under that key.",
|
|
131
153
|
],
|
|
132
154
|
examples: [
|
|
@@ -201,22 +223,34 @@ export function getAssertionsPrompt(specificRules, scope, testFile, enhanceType)
|
|
|
201
223
|
.join(",\n");
|
|
202
224
|
return `${getPersonaPrefix()}Your task is to enhance response body assertions in the given test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
|
|
203
225
|
|
|
226
|
+
### What to assert
|
|
227
|
+
Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour, and the values it depends on, exactly. Leave the rest of the response alone.
|
|
228
|
+
|
|
229
|
+
A field belongs in the test when a wrong value in it would mean the behaviour is broken. A field does not belong when its value can change for reasons that have nothing to do with the behaviour — a generated id, a timestamp, an unrelated sibling field. Asserting those does not make the test stronger. It makes the test fail for reasons the test is not about, and it buries the one assertion a reader needs to understand what the test is for.
|
|
230
|
+
|
|
231
|
+
So the rules below say how to assert a value, not how many values to assert. Apply a rule to the fields the behaviour depends on. \`[]\` is the correct answer for a rule whose fields this test does not depend on.
|
|
232
|
+
|
|
233
|
+
If nothing in the response distinguishes the behaviour working from the behaviour broken, the test is exercising the wrong request or reading the wrong response. Say so; do not pad it with assertions on whatever else the body carries.
|
|
234
|
+
|
|
204
235
|
### Pre-Edit Assertion Analysis
|
|
205
|
-
Before editing the given file, you must output a \`<thinking>\` block.
|
|
206
|
-
1.
|
|
236
|
+
Before editing the given file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the responses that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion rule title must appear as a key in the \`rule_checklist\`, with \`[]\` where the rule does not apply.
|
|
237
|
+
1. For each test function in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from. Generated test functions are usually named \`test_integration\` or \`testUi\`, which claim nothing — use the name only when it is specific enough to add something, and never treat a generic one as evidence the test has no single behaviour. Then scan the given test file and the expected responses for the code change tested.
|
|
207
238
|
2. Classify each response first by its response status type and then assign the applicable assertion rules to the response.
|
|
208
239
|
1. Success with body (2xx with a response body): all assertion rules below may apply — echo-back of request fields, computed response fields, array / items validation, and chained values across steps.
|
|
209
240
|
2. Success with no body (200/202/204 with an empty body — e.g. logout/cancel/submit action endpoints): assert the status code only. Also apply chained-values rules if a follow-up step uses this response's ID.
|
|
210
|
-
3. Error response (4xx/5xx with a body): assert every error body field with its exact value plus array / items validation on the \`errors[]\` array (exact length + per-item fields + next index null). Status code alone is never sufficient when a body is present — for example, also assert \`errors.0.extensions.code == 'INVALID_PAYLOAD'\` and that \`errors.1\` is null.
|
|
241
|
+
3. Error response (4xx/5xx with a body, or any status whose body is an error object such as \`{"error": "..."}\` with no resource payload): assert every error body field with its exact value plus array / items validation on the \`errors[]\` array (exact length + per-item fields + next index null). Status code alone is never sufficient when a body is present — for example, also assert \`errors.0.extensions.code == 'INVALID_PAYLOAD'\` and that \`errors.1\` is null.
|
|
242
|
+
Classify by the response the recorded trace or \`expected_response_body\` shows this request returning. The one exception is a test the plan declared \`expected.outcome: fail\` (typically a \`bug_caught\` or \`requirement_conflict\` candidate; read the declaration first and fall back to the category only when none was made): there the expected response is the intended behaviour established by the diff, the PR statement, or the code's evident intent, and the test is expected to fail until the code is fixed.
|
|
211
243
|
3. For each in-scope response, output one JSON object using the template below. The output is an array — one object per in-scope response.
|
|
212
244
|
- \`step\`: the HTTP method, path, and response variable name for this request (e.g. \`POST /products → products_POST_response\`).
|
|
213
245
|
- \`response_status\`: one of \`success\`, \`no_body\`, or \`error\` based on the classification in step 2.
|
|
214
|
-
- \`
|
|
246
|
+
- \`behaviour_under_test\`: the one-line behaviour from step 1 that this response is evidence for. When a response is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
|
|
247
|
+
- \`rule_checklist\`: an object that MUST contain every rule title below as a key. For each rule, the value is the assertion lines you will add for this response under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's fields are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
|
|
215
248
|
|
|
216
249
|
\`\`\`json
|
|
217
250
|
[{
|
|
218
251
|
"step": "<METHOD> <path> → <responseVar>",
|
|
219
252
|
"response_status": "success | no_body | error",
|
|
253
|
+
"behaviour_under_test": "<the one-line behaviour this response is evidence for, or none>",
|
|
220
254
|
"rule_checklist": {
|
|
221
255
|
${ruleChecklistKeys}
|
|
222
256
|
}
|
|
@@ -235,7 +269,8 @@ ${scope}
|
|
|
235
269
|
- Do not access response fields via dict syntax (\`response["field"]\`) or attribute access (\`response.field\`) — always use the SDK helper.
|
|
236
270
|
- Do not assert \`not.toBeNull()\` / \`is not None\` on a field whose exact value is in the request body or a prior response.
|
|
237
271
|
- Do not assert an exact value you learned only from a request you sent yourself while writing the test. Take the value from the diff, from what the request determines, or from a response this test received earlier.
|
|
238
|
-
- Do not
|
|
272
|
+
- Do not assert a field only because the body carries it. If a wrong value there would not mean the behaviour under test is broken, leave it alone.
|
|
273
|
+
- Do not leave a test whose assertions would all still pass with the behaviour under test broken.
|
|
239
274
|
- Do not use permissive status matchers (\`.toMatch(/^2/)\`, \`.toBeGreaterThanOrEqual(200)\`, \`checkStatusCode(response, '20x')\`).
|
|
240
275
|
- Do not use shape-only or type-only assertions as a substitute for exact value validation. Forbidden patterns: \`Array.isArray(...)\`, \`typeof X === '...'\`, \`X instanceof Array\`, \`Object.keys(X).length > 0\`. When a field's value comes from one of those sources, assert it exactly.
|
|
241
276
|
- Do not use shape-only, containment-only, range-only, or weak-length as the sole assertion on a populated array.
|
|
@@ -245,5 +280,5 @@ ${scope}
|
|
|
245
280
|
- Do not remove existing assertions.
|
|
246
281
|
|
|
247
282
|
### Verification of Assertions
|
|
248
|
-
After adding
|
|
283
|
+
After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.`;
|
|
249
284
|
}
|
|
@@ -51,6 +51,8 @@ expect(trigger).to_be_focused()`,
|
|
|
51
51
|
"Anchor on the removed attribute+value or role+name, never on text or position — a text-based absence check breaks when unrelated siblings change.",
|
|
52
52
|
"Do not add a guard for an element whose whole page or route was removed — that is maintenance of the tests that covered it, not a new assertion; a spec that opens a deleted route must be anchored on a surviving page first (a retained element asserted on a live page in the same spec) — a spec whose only visited page is the dead URL is not a valid guard.",
|
|
53
53
|
"A renamed test id or accessible name (a removed `aria-label` included) is not a removed element: assert the element under its new identifier and never assert the retired identifier `toBeHidden()` — it passes today and turns into a false failure the day the name is reused.",
|
|
54
|
+
"An element that exists only inside a control the page has to open — an `option` of a `select`, an item of a closed menu or dropdown — takes `toHaveCount(0)`, not `toBeHidden()`. A closed control's contents have no bounding box, so `toBeHidden()` passes whether the element was removed or not, and the guard stays green the day it comes back. If the spec opens the control before asserting, the element would have had a box had it survived, so assert both.",
|
|
55
|
+
"The positive companion for that guard is counted or read, never `toBeVisible`: the same missing bounding box that makes `toBeHidden()` a false green on the removed entry makes `toBeVisible()` a false red on the one that stayed. Assert a retained sibling with `toHaveCount(1)` or `toHaveText`, or assert the control itself, which does have a box.",
|
|
54
56
|
],
|
|
55
57
|
examples: [
|
|
56
58
|
{
|
|
@@ -83,7 +85,7 @@ expect(page.get_by_test_id('navbar-products')).to_have_text('Products')`,
|
|
|
83
85
|
},
|
|
84
86
|
{
|
|
85
87
|
title: "Tests target the changed behavior introduced by the pull request",
|
|
86
|
-
description: "
|
|
88
|
+
description: "Every test needs at least one assertion that fails when the behaviour it claims is broken. This is the rule the others support: a test that satisfies every rule below and still does not distinguish the behaviour working from the behaviour broken has not been enhanced. Target it on populated or updated state, not only on empty or zero state, and assert it on the page where the behaviour shows. When the PR changed the behaviour, the expected value is the one the diff, the PR description, or the referenced spec establishes — even when the running app still renders the old value; the test is expected to fail until the code is fixed.",
|
|
87
89
|
examples: [],
|
|
88
90
|
},
|
|
89
91
|
{
|
|
@@ -222,6 +224,9 @@ export function getUIAssertionsPrompt(testFile, enhanceType) {
|
|
|
222
224
|
const categoryTemplate = renderAssertionCategoriesTemplate(UI_ASSERTION_CATEGORIES);
|
|
223
225
|
return `${getPersonaPrefix()}Your task is to enhance assertions for the given UI test file: \`${testFile}\`.${maintenanceTaskSuffix(enhanceType)}
|
|
224
226
|
|
|
227
|
+
### What to assert
|
|
228
|
+
Each test evaluates ONE behaviour — the one the recommendation it came from asked for. Assert that behaviour on the page where it shows, and the rendered values it depends on, exactly. A page the test only passes through, and an element the behaviour does not touch, need no assertion: asserting them makes the test fail for reasons the test is not about and buries what the test is for. The rules below say how to assert a value, not how many values to assert.
|
|
229
|
+
|
|
225
230
|
### First Check
|
|
226
231
|
If the generated test file has no \`expect()\` assertions, you MUST manually add them before anything else. Use \`import { expect } from '@skyramp/skyramp';\` — never from \`@playwright/test\`. If an existing import pulls \`expect\` from \`@playwright/test\`, move it to \`@skyramp/skyramp\` (keep \`test\` on the playwright line).
|
|
227
232
|
|
|
@@ -328,25 +333,28 @@ if (await existingSecret.isVisible().catch(() => false)) {
|
|
|
328
333
|
\`\`\`
|
|
329
334
|
|
|
330
335
|
### Pre-Edit Assertion Analysis
|
|
331
|
-
Before editing the given test file, you must output a \`<thinking>\` block.
|
|
332
|
-
1.
|
|
333
|
-
2.
|
|
336
|
+
Before editing the given test file, you must output a \`<thinking>\` block. It states the behaviour each test evaluates, then works through the rules for the items that behaviour depends on, so no rule that applies is overlooked. The JSON array should match the template below — every assertion category and every rule title under it must appear as a key, with \`[]\` where the rule does not apply.
|
|
337
|
+
1. For each test in scope, name the behaviour it evaluates in one line, taken from the recommendation it came from, and name the page where that behaviour shows. Generated specs are usually named \`testUi\`, which claims nothing — use the name only when it is specific enough to add something.
|
|
338
|
+
2. Selector inventory — list every selector already present in the generated test file (\`data-testid\`, role + name, text, label, etc.). New assertions may use only selectors from this list. An assertion the recorder dropped because it failed during recording may use the selector of the element named in that recording's blueprint capture. Do not invent \`data-testid\` values, role names, or aria attributes. Also note captured network responses, repeated element patterns, exact rendered text/value/attribute from trace/source, and existing \`toBeVisible()\` assertions whose exact text is knowable.
|
|
339
|
+
3. Process — Replay → Identify → Fix or Add — walk through these three steps explicitly.
|
|
334
340
|
a. Replay the scenario mentally. At each state-changing action (form submit, item add/edit/delete), ask: "What is the EXPECTED outcome based on the action performed?"
|
|
335
341
|
b. Identify expectation mismatches. If the recorded trace shows a result that contradicts the action (e.g. removing 1 of 2 items but the page shows 3, submitting a form but getting a blank page, editing a field but the old value persists), that is an app bug the test should catch. List every mismatch you find.
|
|
336
342
|
c. Fix or add assertions for each mismatch.
|
|
337
343
|
- If an existing assertion uses the wrong (buggy) value, edit it to assert the correct expected value.
|
|
338
344
|
- If no assertion exists for the buggy behavior, add one immediately after the action that triggers it.
|
|
339
|
-
|
|
345
|
+
4. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
|
|
340
346
|
- Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
|
|
341
347
|
- Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
|
|
342
348
|
- Post-edit State applies whenever EITHER (a) the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), OR (b) the PR changes interactive behavior — in case (b), add the triggering action and assert the before/after transition (see "Exercise the changed interaction"). Mark NOT APPLICABLE only when the PR changes no interactive behavior and the test has no state-changing action.
|
|
343
|
-
|
|
349
|
+
5. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
|
|
344
350
|
- \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
|
|
345
|
-
- \`
|
|
351
|
+
- \`behaviour_under_test\`: the one-line behaviour from step 1 that this item is evidence for. When an item is evidence for no behaviour any test in scope claims, write \`none\` and add nothing for it.
|
|
352
|
+
- \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is the assertion lines you will add for this item under that rule. \`[]\` means the rule does not apply here, which is a normal and expected answer: it is the right one whenever the rule's elements are not part of \`behaviour_under_test\`. Listing every key keeps a rule that DOES apply from being skipped; it is not a quota to fill.
|
|
346
353
|
|
|
347
354
|
\`\`\`json
|
|
348
355
|
[{
|
|
349
356
|
"action_or_selector_or_response": "<selector | action | response>",
|
|
357
|
+
"behaviour_under_test": "<the one-line behaviour this item is evidence for, or none>",
|
|
350
358
|
"assertion_categories": {
|
|
351
359
|
${categoryTemplate}
|
|
352
360
|
}
|
|
@@ -354,7 +362,7 @@ ${categoryTemplate}
|
|
|
354
362
|
\`\`\`
|
|
355
363
|
|
|
356
364
|
### Assertion Rules with Examples
|
|
357
|
-
Most-violated patterns —
|
|
365
|
+
Most-violated patterns — whenever the behaviour under test involves one of these, this is the shape it takes. (1) repeated elements → \`toHaveCount(N)\` + per-item \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` (Collection / repeated UI elements). (2) post-action state → \`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\`/\`toBeChecked\`, not \`toBeVisible\` when exact value is knowable (Post-action visible state). (3) routing/URL-only tests → at least one rendered-element exact text (Positive-path companion).
|
|
358
366
|
Strength order. \`toHaveCount\`/\`toHaveText\`/\`toHaveValue\`/\`toHaveAttribute\` > \`toContainText\` (only when string is genuinely dynamic) > \`toBeVisible\` (only when presence is the actual test).
|
|
359
367
|
|
|
360
368
|
${renderCategories(UI_ASSERTION_CATEGORIES)}
|
|
@@ -373,7 +381,7 @@ ${renderCategories(UI_ASSERTION_CATEGORIES)}
|
|
|
373
381
|
- Do not assert buggy/error text as expected (unless the intended UX is an error state)
|
|
374
382
|
|
|
375
383
|
### Verification of Assertions
|
|
376
|
-
After adding
|
|
384
|
+
After adding the assertion lines in the given test file, re-read each test and check two things: every rule that applies to the behaviour under test has been applied correctly, and the test now fails if that behaviour breaks. Fix anything missing or weakly applied before completing. Remove any assertion you added that the behaviour does not depend on.
|
|
377
385
|
|
|
378
386
|
The goal is tests that FAIL when the app has bugs, not tests that simply replay what happened.
|
|
379
387
|
`;
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
|
|
2
2
|
import { sectionBody } from "../promptAssets.js";
|
|
3
3
|
import { readPromptAsset } from "../promptAssets.js";
|
|
4
|
-
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
5
4
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
6
5
|
import { PromptPlan } from "./promptPlan.js";
|
|
7
6
|
import { buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, buildFinishChecks, } from "./recommendationSections.js";
|
|
8
7
|
const EXECUTION_PLAN_MD = readPromptAsset("generate-tests/execution-plan.md");
|
|
9
8
|
function _execEnrichBody(ctx) {
|
|
10
9
|
return fillPlaceholders(sectionBody(EXECUTION_PLAN_MD, "## Parameter grounding"), {
|
|
11
|
-
authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
|
|
12
10
|
enrichSurface: ctx.isDiffScope ? "the diff" : "the analysed surface",
|
|
13
11
|
testPatternGuidelines: buildTestPatternGuidelines(),
|
|
14
12
|
});
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { AnalysisScope, isDiff, } from "../../types/RepositoryAnalysis.js";
|
|
2
|
-
import { WorkspaceAuthType, getDefaultAuthHeader } from "../../utils/workspaceAuth.js";
|
|
2
|
+
import { WorkspaceAuthType, getDefaultAuthHeader, AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
4
|
import { buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildToolWorkflows, buildVerificationChecklist, } from "./recommendationSections.js";
|
|
5
5
|
import { buildExecutionPlan, EXEC_STEP_ENRICH } from "./diffExecutionPlan.js";
|
|
6
6
|
import { readPromptAsset } from "../promptAssets.js";
|
|
7
|
-
import {
|
|
7
|
+
import { fillPlaceholders } from "../../recommendation/verifierContracts.js";
|
|
8
8
|
import { TASK_GENERATE, taskRef, } from "./recommendationShared.js";
|
|
9
9
|
// Re-export for backward compatibility (tests and external callers import this from this module)
|
|
10
10
|
function formatTestLocations(locs) {
|
|
@@ -22,11 +22,16 @@ function formatTestLocations(locs) {
|
|
|
22
22
|
// on subsequent testbot runs on the same PR.
|
|
23
23
|
const SKYRAMP_TEST_FILE_PATTERN = /(?:_test|_smoke|_contract|_fuzz|_integration|_load|_e2e|_ui)\.[^/]+$|scenario_[^/]+\.json$/;
|
|
24
24
|
/** How this prompt tells the agent to plan: the whole procedure, every time.
|
|
25
|
-
* Rendered from `plugin/prompts/plan-tests.md`
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
* Rendered from `plugin/prompts/plan-tests.md` rather than retyped, so there is
|
|
26
|
+
* one text and nothing to drift from. `isDiffScope` is unused here — the
|
|
27
|
+
* procedure reads the same for a diff and for a repository. */
|
|
28
28
|
function planningInstruction(_isDiffScope) {
|
|
29
|
-
|
|
29
|
+
// The procedure names the auth patterns. BOTH renderers of this asset fill the
|
|
30
|
+
// placeholder — this lane's and the testbot lane's — or one of them ships the
|
|
31
|
+
// literal `{authMiddlewarePatterns}` to the model.
|
|
32
|
+
return fillPlaceholders(readPromptAsset("plan-tests.md").trim(), {
|
|
33
|
+
authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR,
|
|
34
|
+
});
|
|
30
35
|
}
|
|
31
36
|
export function buildRecommendationPrompt(analysis, analysisScope = AnalysisScope.FullRepo, prContext, workspaceAuthHeader, workspaceAuthType, workspaceAuthScheme, sessionId) {
|
|
32
37
|
const isDiffScope = isDiff(analysisScope);
|
|
@@ -4,10 +4,11 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
|
4
4
|
import { buildPathParamGuidance, } from "../test-recommendation/recommendationSections.js";
|
|
5
5
|
import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
6
6
|
import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
|
|
7
|
+
import { recordPullRequestText } from "../../recommendation/pullRequestText.js";
|
|
8
|
+
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
7
9
|
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, TESTBOT_TASK1_MAINTAIN_LABELS, TESTBOT_TASK1_MULTIREPO_LABELS, TESTBOT_TASK1_STEP_LABELS, orderedLabels, stepSubRef, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
8
10
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
9
11
|
import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, isSkillsLoaded, } from "../../utils/featureFlags.js";
|
|
10
|
-
import { renderPlanDeclarationGuidance } from "./planDeclarations.js";
|
|
11
12
|
import { fixErrorsInstruction } from "../../skills/fixTestImportErrorsSkill.js";
|
|
12
13
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
13
14
|
import { section, sectionBody } from "../promptAssets.js";
|
|
@@ -59,24 +60,28 @@ const PLAN_TESTS_MD = readPromptAsset("plan-tests.md").trim();
|
|
|
59
60
|
// rendered neither it nor a copy, so the agent planned without the claims step and
|
|
60
61
|
// without the backward surface map. Rendered here, never retyped.
|
|
61
62
|
//
|
|
62
|
-
// It sits OUTSIDE Task 1: the procedure is
|
|
63
|
+
// It sits OUTSIDE Task 1: the procedure is a numbered list, Task 1's
|
|
63
64
|
// own steps are numbered too, and nested inside Task 1 the two share one ordinal
|
|
64
65
|
// space (`referenceIntegrity.test.ts` reads every `N.` there as a Task 1 step).
|
|
65
66
|
//
|
|
66
|
-
// Two steps run elsewhere in this lane —
|
|
67
|
-
//
|
|
68
|
-
//
|
|
67
|
+
// Two steps run elsewhere in this lane — the maintenance update in **Group the
|
|
68
|
+
// changes into behaviours and decide each** is Task 1's tool step, and **Register the
|
|
69
|
+
// whole plan in one call to `skyramp_register_test_plan`** happens in Task 2 after it.
|
|
70
|
+
// The note states that order; the markdown stays the source.
|
|
69
71
|
//
|
|
70
72
|
// The count sentence sits HERE, beside the register instruction, because the agent
|
|
71
73
|
// reads the URI first and plans straight after: on run 34005367281 it read
|
|
72
74
|
// `maxGenerate=3` as this run's budget and left its fourth planned test unwritten.
|
|
73
|
-
|
|
75
|
+
// The procedure names the auth patterns. BOTH renderers of this asset fill the
|
|
76
|
+
// placeholder — this lane's and the recommendation lane's — or one of them ships
|
|
77
|
+
// the literal `{authMiddlewarePatterns}` to the model.
|
|
78
|
+
const planProcedureBlock = fillPlaceholders(`${section(PLAN_TESTS_MD, "## Terms")}
|
|
74
79
|
|
|
75
80
|
${section(PLAN_TESTS_MD, "## The procedure")}
|
|
76
81
|
|
|
77
82
|
**In this run:** ${taskRef(TASK_ANALYZE_MAINTAIN)}'s maintenance step performs the maintenance update of an existing test, with the maintenance tools — a test those tools update is not a plan planned test. Register the plan in ${taskRef(TASK_GENERATE)}, after that step. Nothing caps how many planned tests the plan holds: \`maxGenerate\` and \`maxRecommendations\` arrive in this run's URI, and the server accepts both and applies neither — no check reads either one, so a count you see there is not a budget.
|
|
78
83
|
|
|
79
|
-
${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}
|
|
84
|
+
${section(PLAN_TESTS_MD, "## Decisions the checks leave to you")}`, { authMiddlewarePatterns: AUTH_MIDDLEWARE_PATTERNS_STR });
|
|
80
85
|
// The checks on the test file itself. One home — `generation.md` — rendered by
|
|
81
86
|
// this lane at the end of generation and by the IDE lane after its generation
|
|
82
87
|
// rules, so neither carries a retyped copy.
|
|
@@ -173,6 +178,8 @@ export function parseRelatedRepositories(raw) {
|
|
|
173
178
|
}
|
|
174
179
|
export function getTestbotPrompt(opts) {
|
|
175
180
|
const { prTitle, prDescription, repositoryPath, baseBranch, prNumber, userPrompt, services, uiCredentials, testsRepoDir, relatedRepositories, primaryRepo, planOnly = false, language, } = opts;
|
|
181
|
+
// Kept in the process for the plan tool, which is registered on this server.
|
|
182
|
+
recordPullRequestText(prTitle, prDescription);
|
|
176
183
|
// The legacy count arguments are DECLARED NOWHERE, so zod strips them and their
|
|
177
184
|
// names never reach the agent — not in a schema it lists, not in rendered text.
|
|
178
185
|
// A number in front of the agent reads as a ceiling whatever the words around it
|
|
@@ -357,7 +364,7 @@ ${uiGroundingBlock}
|
|
|
357
364
|
This is a plan-only evaluation run: the application under test is NOT running, and this run evaluates test SELECTION only. Nothing is generated or executed in this task.
|
|
358
365
|
|
|
359
366
|
- Draft your complete plannedTest list — every test you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
|
|
360
|
-
- Register that list through \`skyramp_register_test_plan\` as
|
|
367
|
+
- Register that list through \`skyramp_register_test_plan\` as the planning procedure above describes. What you register is the plan — nothing re-ranks or trims it.
|
|
361
368
|
- Include UI and E2E planned tests on the same footing as the API types. The app is not running, so you cannot capture a blueprint — plan them ungrounded rather than drop them. For every UI plannedTest, set \`elements.items\` to null and set \`elements.pageUrl\` to the route path the test would visit, read out of the source: \`/orders/1\`, not a full URL, because no host is serving the app. Set \`screenEvidence.file\` to the changed frontend file that renders that route — the app not running does not excuse this one, because you read the file out of the diff. Describe the page or feature the test would exercise; do not name an element you have not seen. This lane is the one case where you fall back without attempting a capture first, so do NOT log the fallback in \`issuesFound\` — capture was never applicable here, and nothing failed.
|
|
362
369
|
- Take no other actions in this task: no test generation tools, no browser traces or blueprint captures, no test files written, no test executions. Proceed directly to ${taskRef(TASK_SUBMIT)}.`;
|
|
363
370
|
}
|
|
@@ -458,7 +465,7 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
458
465
|
- \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
|
|
459
466
|
- \`skyramp_ui_test_generation\`: set \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript Playwright only — the default; leave \`codeReuse\` unset for other languages)
|
|
460
467
|
- **\`browser_assert\`**: the \`browser_navigate\` result states the rule this tool has to satisfy on every page you reach. Call multiple assertions in the same tool call batch when checking independent elements.
|
|
461
|
-
- **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
|
|
468
|
+
- **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server lists which elements this PR removed — each identified by a \`data-*\` attribute and value, with the file it came from (renames, moves and deleted files are excluded). Read that file and the route or page files that render it to decide which page each appeared on. Per PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the removal check requires it) — ground that one in the diff and prefix its \`reasoning\` with \`[removed-element diff-grounded]\`; prefix \`reasoning\` with \`[removed-element server-listed]\` when the element came from \`uiContext.removedElements\`. When the removed element lived inside a control the page has to open — an \`option\` of a \`select\`, an item of a closed menu — open that control before asserting: a closed control's contents are never visible, so a hidden assertion recorded against a shut one passes whether the element was removed or not. \`browser_assert_hidden\` stays the tool either way, because \`browser_assert\` needs a snapshot \`ref\` and a removed element has none; where the control cannot be opened, say so in \`reasoning\` so the assertion step turns that guard into a count. **Deleted routes or pages are deleted features, not removed elements**: test the removal through the link or entry point that survives. A trace may still open the retired URL — to assert the app's fallback there, or that the old page's elements are gone — after it has recorded a positive \`browser_assert\` on a surviving page; a trace whose only visited page is the dead URL has no positive companion and is not a guard spec.
|
|
462
469
|
- **Renames are not removals — on every PR, whatever \`uiContext.removedElements\` holds**: when a test id, \`aria-label\` or other accessible name changed, assert the element by its new identifier and leave the retired identifier alone (no \`browser_assert_hidden\` on it). A rename PR removes nothing, so this rule applies when the removed list is empty.
|
|
463
470
|
- **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
|
|
464
471
|
- **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
|
|
@@ -573,7 +580,7 @@ ${FINISH_CHECKS_BLOCK}
|
|
|
573
580
|
// to name the changed file it is about, and cite the file it sits
|
|
574
581
|
// in, and cite `routes` from the checkout, and all of those happen while
|
|
575
582
|
// it analyzes in Task 1.
|
|
576
|
-
const planningBlock = `\n${planProcedureBlock}\n
|
|
583
|
+
const planningBlock = `\n${planProcedureBlock}\n`;
|
|
577
584
|
return `<TITLE>${prTitle}</TITLE>
|
|
578
585
|
<DESCRIPTION>${prDescription}</DESCRIPTION>
|
|
579
586
|
${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
|
|
@@ -603,9 +610,9 @@ In these cases:
|
|
|
603
610
|
- \`issuesFound\` must be \`[]\` — do NOT add a "No testable behavioral surface" entry; the business case already explains the abstention
|
|
604
611
|
- \`businessCaseAnalysis\` must be a one-sentence summary of what the PR actually does (do NOT leave it blank)
|
|
605
612
|
|
|
606
|
-
**This zero-test path does NOT apply when the requirement check
|
|
613
|
+
**This zero-test path does NOT apply when the requirement check found a \`requirement_conflict\`.** A requirement the PR states and the diff did not implement is testable surface, whatever the diff otherwise contains. Report the conflict in \`issuesFound\` at severity \`high\` or above, and carry the failing \`requirement_conflict\` test in \`newTestsCreated\`. Where the requirement is observable nowhere you can reach, say that in \`businessCaseAnalysis\`. Abstaining there hands the author back their own code as though it were the requirement.
|
|
607
614
|
|
|
608
|
-
${task3CountRule ? `${task3CountRule}\n\n` : ""}${reportLanguageBlock}Call \`skyramp_submit_report\`. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
615
|
+
${task3CountRule ? `${task3CountRule}\n\n` : ""}${reportLanguageBlock}Call \`skyramp_submit_report\`. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly. A report check objects to an \`issuesFound\` entry with \`category: bug\` that no delivered test proves. The \`plannedTestId\` and \`defectId\` descriptions in the schema say what closes it. Answer an objection in one line.
|
|
609
616
|
|
|
610
617
|
${hasRelatedRepos
|
|
611
618
|
? `
|
|
@@ -19,13 +19,17 @@ export declare function unknownAnswerObjections(objections: Objection[], acknowl
|
|
|
19
19
|
* leave the bug. `verifierContracts.ts` carries the evidence. */
|
|
20
20
|
export declare const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
21
21
|
export declare function isNonAnswerable(objectionId: unknown): boolean;
|
|
22
|
-
/** The
|
|
23
|
-
* needs the test,
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
|
|
22
|
+
/** The objections a sentence alone does not close either. A change with no test
|
|
23
|
+
* needs the test, and a defect with no test needs the test — a defect the code
|
|
24
|
+
* already had before this pull request included. Either one closes only when the
|
|
25
|
+
* answer also names what stopped the test: a service that is not running, a
|
|
26
|
+
* branch that no longer exists, the one credential the run holds. Measured on run
|
|
27
|
+
* 34176038240: across eight fixtures 45 objections were raised and 45 were
|
|
28
|
+
* closed, and the one class that a sentence cannot close was never raised at all.
|
|
29
|
+
* Prose closed every uncovered change, including changes nothing prevented a test
|
|
30
|
+
* from reaching. Runs 34423214322 and 34423217796 then lost three defects the
|
|
31
|
+
* baseline reported: each was listed and closed by a sentence. */
|
|
32
|
+
export declare const BLOCKER_ONLY_PREFIXES: readonly ["coverage:change:", "defects:untested:"];
|
|
29
33
|
export declare function needsBlocker(objectionId: unknown): boolean;
|
|
30
34
|
/** The objection raised below carries this prefix and the id it objects to. */
|
|
31
35
|
export declare const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
@@ -49,15 +49,19 @@ export const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
|
49
49
|
export function isNonAnswerable(objectionId) {
|
|
50
50
|
return typeof objectionId === "string" && objectionId.startsWith(NON_ANSWERABLE_PREFIX);
|
|
51
51
|
}
|
|
52
|
-
/** The
|
|
53
|
-
* needs the test,
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
|
|
52
|
+
/** The objections a sentence alone does not close either. A change with no test
|
|
53
|
+
* needs the test, and a defect with no test needs the test — a defect the code
|
|
54
|
+
* already had before this pull request included. Either one closes only when the
|
|
55
|
+
* answer also names what stopped the test: a service that is not running, a
|
|
56
|
+
* branch that no longer exists, the one credential the run holds. Measured on run
|
|
57
|
+
* 34176038240: across eight fixtures 45 objections were raised and 45 were
|
|
58
|
+
* closed, and the one class that a sentence cannot close was never raised at all.
|
|
59
|
+
* Prose closed every uncovered change, including changes nothing prevented a test
|
|
60
|
+
* from reaching. Runs 34423214322 and 34423217796 then lost three defects the
|
|
61
|
+
* baseline reported: each was listed and closed by a sentence. */
|
|
62
|
+
export const BLOCKER_ONLY_PREFIXES = ["coverage:change:", "defects:untested:"];
|
|
59
63
|
export function needsBlocker(objectionId) {
|
|
60
|
-
return typeof objectionId === "string" && objectionId.startsWith(
|
|
64
|
+
return typeof objectionId === "string" && BLOCKER_ONLY_PREFIXES.some((prefix) => objectionId.startsWith(prefix));
|
|
61
65
|
}
|
|
62
66
|
/** The objection raised below carries this prefix and the id it objects to. */
|
|
63
67
|
export const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
@@ -84,11 +88,11 @@ export function refusedAnswerObjections(objections, acknowledged) {
|
|
|
84
88
|
objectionId: `${REFUSED_ANSWER_PREFIX}${id}`,
|
|
85
89
|
verifier: "answers",
|
|
86
90
|
message: needsBlocker(id)
|
|
87
|
-
? "The answer says why
|
|
91
|
+
? "The answer says why there is no test, and it does not say what stopped this run from writing one, so the objection stays open."
|
|
88
92
|
: "This objection is closed only by a planned test, so the answer closed nothing and the objection stays open.",
|
|
89
93
|
evidence: `answered "${id}"${needsBlocker(id) ? " with no `blocker`" : ", which no answer closes"}`,
|
|
90
94
|
suggestion: needsBlocker(id)
|
|
91
|
-
? "Plan
|
|
95
|
+
? "Plan the test and register the plan again. If this run cannot write one, send the same answer with `blocker` naming what stopped it — a service that is not running, a paired branch that no longer exists, the one credential the run holds. A reason it is not worth testing is not a blocker."
|
|
92
96
|
: "Add a planned test that names the record's state as `startState` and runs this route's mutation on it, then register the plan again. Every other objection takes an answer; this one takes a test.",
|
|
93
97
|
});
|
|
94
98
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Called by every entry point that renders the testbot prompt, with the values it
|
|
2
|
+
* renders. */
|
|
3
|
+
export declare function recordPullRequestText(title: unknown, description: unknown): void;
|
|
4
|
+
/** UNDEFINED when this process never rendered the prompt — a caller that drives the
|
|
5
|
+
* tools directly, or a test. A check that reads the pull request stays silent then;
|
|
6
|
+
* it cannot tell an empty pull request from one it never saw. */
|
|
7
|
+
export declare function pullRequestText(): {
|
|
8
|
+
title: string;
|
|
9
|
+
description: string;
|
|
10
|
+
} | undefined;
|
|
11
|
+
export declare function clearPullRequestText(): void;
|
|
12
|
+
/** The pull request as one searchable string. Whitespace runs fold to one space, so
|
|
13
|
+
* a quote wrapped across lines still matches the sentence it was taken from. */
|
|
14
|
+
export declare const searchable: (pr: {
|
|
15
|
+
title: string;
|
|
16
|
+
description: string;
|
|
17
|
+
}) => string;
|
|
18
|
+
export declare const appearsIn: (haystack: string, needle: string) => boolean;
|