@skyramp/mcp 0.4.4-rc.1 → 0.4.4-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +239 -210
- package/build/playwright/PlaywrightTraceService.d.ts +1 -1
- package/build/playwright/blueprintDigestStore.d.ts +1 -1
- package/build/playwright/registerPlaywrightTools.d.ts +1 -1
- package/build/playwright/traceRecordingPrompt.d.ts +1 -1
- package/build/playwright/traceRecordingPrompt.js +2 -2
- package/build/prompts/local-dev/local-dev-prompts.d.ts +1 -1
- package/build/prompts/local-dev/local-dev-prompts.js +2 -2
- package/build/prompts/startTraceCollectionPrompts.d.ts +1 -1
- package/build/prompts/startTraceCollectionPrompts.js +2 -3
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +1 -1
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +2 -2
- package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
- package/build/prompts/testbot/testbot-prompts.js +4 -7
- package/build/recommendation/answers.d.ts +17 -52
- package/build/recommendation/answers.js +29 -99
- package/build/recommendation/pullRequestText.d.ts +0 -2
- package/build/recommendation/pullRequestText.js +0 -16
- package/build/recommendation/registerPlan.d.ts +4 -7
- package/build/recommendation/registerPlan.js +11 -37
- package/build/recommendation/types.d.ts +4 -21
- package/build/recommendation/verifierContracts.d.ts +4 -4
- package/build/recommendation/verifierContracts.js +4 -4
- package/build/recommendation/verifiers/citedPath.d.ts +2 -2
- package/build/recommendation/verifiers/citedPath.js +2 -2
- package/build/recommendation/verifiers/issueTraceability.d.ts +5 -1
- package/build/recommendation/verifiers/issueTraceability.js +35 -4
- package/build/resources/analysisResources.d.ts +1 -1
- package/build/resources/analysisResources.js +1 -1
- package/build/resources/progressResource.d.ts +1 -1
- package/build/resources/sutSetupResource.d.ts +1 -1
- package/build/resources/sutSetupResource.js +1 -1
- package/build/resources/testbotResource.d.ts +1 -1
- package/build/resources/testbotResource.js +1 -1
- package/build/services/AnalyticsService.d.ts +1 -1
- package/build/services/FleetExecutionService.d.ts +130 -0
- package/build/services/FleetExecutionService.js +391 -0
- package/build/services/ModularizationService.d.ts +1 -1
- package/build/services/TestExecutionService.d.ts +7 -0
- package/build/services/TestExecutionService.js +21 -0
- package/build/services/TestGenerationService.d.ts +1 -1
- package/build/tool-phases.js +2 -0
- package/build/tools/auth/loginTool.d.ts +1 -1
- package/build/tools/auth/loginTool.js +2 -2
- package/build/tools/auth/logoutTool.d.ts +1 -1
- package/build/tools/auth/logoutTool.js +2 -2
- package/build/tools/code-refactor/alignAssertionsTool.d.ts +1 -1
- package/build/tools/code-refactor/alignAssertionsTool.js +1 -1
- package/build/tools/code-refactor/assertionSkillTools.d.ts +1 -1
- package/build/tools/code-refactor/assertionSkillTools.js +2 -2
- package/build/tools/code-refactor/codeReuseTool.d.ts +1 -1
- package/build/tools/code-refactor/codeReuseTool.js +2 -2
- package/build/tools/code-refactor/enhanceAssertionsTool.d.ts +1 -1
- package/build/tools/code-refactor/enhanceAssertionsTool.js +1 -1
- package/build/tools/code-refactor/modularizationTool.d.ts +1 -1
- package/build/tools/code-refactor/modularizationTool.js +2 -2
- package/build/tools/enrichTestWithMocksTool.d.ts +1 -1
- package/build/tools/enrichTestWithMocksTool.js +2 -2
- package/build/tools/executeFleetTool.d.ts +28 -0
- package/build/tools/executeFleetTool.js +107 -0
- package/build/tools/executeSkyrampTestTool.d.ts +8 -5
- package/build/tools/executeSkyrampTestTool.js +5 -5
- package/build/tools/fixErrorTool.d.ts +1 -1
- package/build/tools/fixErrorTool.js +1 -1
- package/build/tools/generate-tests/batchMockGenerationTool.d.ts +6 -57
- package/build/tools/generate-tests/batchMockGenerationTool.js +3 -3
- package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +9 -45
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +29 -11
- package/build/tools/generate-tests/generateContractRestTool.d.ts +1 -2
- package/build/tools/generate-tests/generateContractRestTool.js +1 -1
- package/build/tools/generate-tests/generateE2ERestTool.d.ts +1 -2
- package/build/tools/generate-tests/generateE2ERestTool.js +1 -1
- package/build/tools/generate-tests/generateFuzzRestTool.d.ts +1 -1
- package/build/tools/generate-tests/generateFuzzRestTool.js +1 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.d.ts +1 -2
- package/build/tools/generate-tests/generateIntegrationRestTool.js +1 -1
- package/build/tools/generate-tests/generateLoadRestTool.d.ts +1 -2
- package/build/tools/generate-tests/generateLoadRestTool.js +2 -1
- package/build/tools/generate-tests/generateMockRestTool.d.ts +1 -2
- package/build/tools/generate-tests/generateMockRestTool.js +3 -3
- package/build/tools/generate-tests/generateSmokeRestTool.d.ts +1 -1
- package/build/tools/generate-tests/generateSmokeRestTool.js +1 -1
- package/build/tools/generate-tests/generateUILoadRestTool.d.ts +64 -0
- package/build/tools/generate-tests/generateUILoadRestTool.js +512 -0
- package/build/tools/generate-tests/generateUIRestTool.d.ts +6 -7
- package/build/tools/generate-tests/generateUIRestTool.js +1 -1
- package/build/tools/generate-tests/loadTestSchema.d.ts +15 -15
- package/build/tools/generate-tests/trace-reuse-guard.d.ts +1 -1
- package/build/tools/generateEnrichedIntegrationTestTool.d.ts +1 -2
- package/build/tools/generateEnrichedIntegrationTestTool.js +1 -1
- package/build/tools/localDevWorkerComposeTool.d.ts +1 -1
- package/build/tools/localDevWorkerComposeTool.js +2 -2
- package/build/tools/one-click/oneClickTool.d.ts +1 -1
- package/build/tools/one-click/oneClickTool.js +1 -1
- package/build/tools/preflightMockCheckTool.d.ts +1 -1
- package/build/tools/preflightMockCheckTool.js +3 -3
- package/build/tools/queryProxyMocksTool.d.ts +1 -1
- package/build/tools/queryProxyMocksTool.js +3 -3
- package/build/tools/submitReportTool.d.ts +111 -414
- package/build/tools/submitReportTool.js +32 -134
- package/build/tools/test-management/actionsTool.d.ts +1 -1
- package/build/tools/test-management/actionsTool.js +7 -7
- package/build/tools/test-management/analyzeChangesTool.d.ts +22 -118
- package/build/tools/test-management/analyzeChangesTool.js +4 -4
- package/build/tools/test-management/analyzeTestHealthTool.d.ts +1 -1
- package/build/tools/test-management/analyzeTestHealthTool.js +4 -4
- package/build/tools/test-management/registerTestPlanTool.d.ts +111 -743
- package/build/tools/test-management/registerTestPlanTool.js +20 -36
- package/build/tools/test-management/resolveScreenTool.d.ts +1 -1
- package/build/tools/test-management/resolveScreenTool.js +6 -2
- package/build/tools/trace/startTraceCollectionTool.d.ts +1 -1
- package/build/tools/trace/startTraceCollectionTool.js +4 -4
- package/build/tools/trace/stopTraceCollectionTool.d.ts +1 -1
- package/build/tools/trace/stopTraceCollectionTool.js +2 -2
- package/build/tools/verifyTestDependenciesTool.d.ts +1 -1
- package/build/tools/verifyTestDependenciesTool.js +1 -1
- package/build/tools/workspace/initScanWorkspaceTool.d.ts +1 -1
- package/build/tools/workspace/initScanWorkspaceTool.js +1 -1
- package/build/tools/workspace/initializeWorkspaceTool.d.ts +1 -1
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/types/RepositoryAnalysis.d.ts +164 -1170
- package/build/types/RepositoryAnalysis.js +12 -10
- package/build/types/StepMethod.d.ts +24 -1
- package/build/types/StepMethod.js +1 -1
- package/build/types/TestTypes.d.ts +34 -133
- package/build/types/TestTypes.js +3 -3
- package/build/utils/docker.d.ts +29 -7
- package/build/utils/docker.js +83 -27
- package/build/utils/initAgent.d.ts +3 -3
- package/build/utils/initAgent.js +58 -26
- package/build/utils/progress.d.ts +3 -4
- package/build/utils/progress.js +4 -4
- package/build/utils/utils.d.ts +1 -1
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/workspace/workspace.d.ts +117 -538
- package/build/workspace/workspace.js +2 -2
- package/package.json +6 -5
- package/plugin/skills/repair-staged-files/SKILL.md +17 -0
- package/build/recommendation/blockerProbes.d.ts +0 -13
- package/build/recommendation/blockerProbes.js +0 -91
- package/build/recommendation/blockers.d.ts +0 -65
- package/build/recommendation/blockers.js +0 -135
|
@@ -12,11 +12,11 @@ export function registerRecommendTestsPrompt(server) {
|
|
|
12
12
|
"Parameters for generation tools must derive from the repository analysis, enrichedScenarios, " +
|
|
13
13
|
"or source code you read — no invented field names, no guessed URLs. Provide a stateFile path " +
|
|
14
14
|
"from skyramp_analyze_changes.",
|
|
15
|
-
argsSchema: {
|
|
15
|
+
argsSchema: z.object({
|
|
16
16
|
stateFile: z
|
|
17
17
|
.string()
|
|
18
18
|
.describe("State file path returned by skyramp_analyze_changes"),
|
|
19
|
-
},
|
|
19
|
+
}),
|
|
20
20
|
}, async (args) => {
|
|
21
21
|
const stateFile = args.stateFile;
|
|
22
22
|
if (!stateFile) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { McpServer } from "@modelcontextprotocol/
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/server";
|
|
2
2
|
import type { RelatedRepository, TestbotPromptOptions } from "../../types/TestbotPromptOptions.js";
|
|
3
3
|
/** "After generation, complete these steps" list at the end of Task 2. Exported
|
|
4
4
|
* for `referenceIntegrity.test.ts`, which compares it against the ordinals the
|
|
@@ -6,7 +6,7 @@ import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
|
6
6
|
import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
|
|
7
7
|
import { parseMaxFixAttemptsArg, resolveMaxFixAttempts, setMaxFixAttempts, } from "../../utils/fixAttempts.js";
|
|
8
8
|
import { setPrimaryRepository } from "../../utils/AnalysisStateManager.js";
|
|
9
|
-
import { recordPullRequestText
|
|
9
|
+
import { recordPullRequestText } from "../../recommendation/pullRequestText.js";
|
|
10
10
|
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
11
11
|
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, TESTBOT_TASK1_MAINTAIN_LABELS, TESTBOT_TASK1_MULTIREPO_LABELS, TESTBOT_TASK1_STEP_LABELS, orderedLabels, stepSubRef, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
12
12
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
@@ -239,9 +239,6 @@ export function getTestbotPrompt(opts) {
|
|
|
239
239
|
setPrimaryRepository({ repositoryPath, repo: primaryRepo });
|
|
240
240
|
// Kept in the process for the plan tool, which is registered on this server.
|
|
241
241
|
recordPullRequestText(prTitle, prDescription);
|
|
242
|
-
// Beside it and for the same reason: a `credential-missing` blocker is checked
|
|
243
|
-
// against whether this run was given any.
|
|
244
|
-
recordUiCredentials(uiCredentials);
|
|
245
242
|
// The legacy count arguments are DECLARED NOWHERE, so zod strips them and their
|
|
246
243
|
// names never reach the agent — not in a schema it lists, not in rendered text.
|
|
247
244
|
// A number in front of the agent reads as a ceiling whatever the words around it
|
|
@@ -480,7 +477,7 @@ ${userPrompt ? "Follow-up requests never reach this step — Task 1's guardrails
|
|
|
480
477
|
**How to generate each type (for ADD):**
|
|
481
478
|
- **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file${INTEGRATION_CODE_REUSE_GEN_CLAUSE}.
|
|
482
479
|
**Reuse the steps you declared in your own plan** — pass them straight through. Read a source model only for a field your plan does not already carry.
|
|
483
|
-
|
|
480
|
+
Test files go in ${testDirInstruction} Pass that same directory as \`outputDir\` to \`skyramp_batch_scenario_test_generation\`. The tool writes the scenario JSON outside the repository, so it is not committed. Do NOT copy it into the test directory.
|
|
484
481
|
**Pipeline for speed**: Call ALL \`skyramp_batch_scenario_test_generation\` calls in one batch. When they return, call ALL \`skyramp_integration_test_generation\` calls in the next batch. Do NOT serialize per-scenario (batch→integration→batch→integration) — batch ALL scenarios first, then generate ALL integration tests.
|
|
485
482
|
- **Contract**: call \`skyramp_contract_test_generation\` with \`endpointURL\`, \`method\`, and \`requestData\` for POST/PUT/PATCH.
|
|
486
483
|
Pass \`apiSchema\` if an OpenAPI spec exists.
|
|
@@ -711,7 +708,7 @@ export function registerTestbotPrompt(server) {
|
|
|
711
708
|
logger.info("Registering testbot prompt");
|
|
712
709
|
server.registerPrompt("skyramp_testbot", {
|
|
713
710
|
description: "Run Skyramp Testbot to generate test recommendations and perform test maintenance for a pull request.",
|
|
714
|
-
argsSchema: {
|
|
711
|
+
argsSchema: z.object({
|
|
715
712
|
prTitle: z.string().describe("Pull request title"),
|
|
716
713
|
prDescription: z.string().describe("Pull request description/body"),
|
|
717
714
|
repositoryPath: z
|
|
@@ -771,7 +768,7 @@ export function registerTestbotPrompt(server) {
|
|
|
771
768
|
// like the resource does, instead of rejecting the call.
|
|
772
769
|
.preprocess((v) => parseMaxFixAttemptsArg(v), z.number().int().min(1).max(10).optional())
|
|
773
770
|
.describe("Per-TEST-FILE cap on skyramp_execute_test calls in the final phase: the first run plus the fix-and-rerun iterations the agent may spend on an infrastructure failure for that one file (SKYR-4460). 1..10; falls back to SKYRAMP_MAX_FIX_ATTEMPTS, then 3."),
|
|
774
|
-
},
|
|
771
|
+
}),
|
|
775
772
|
}, async (args) => {
|
|
776
773
|
const services = await readWorkspaceServices(args.repositoryPath);
|
|
777
774
|
let prompt = getTestbotPrompt({
|
|
@@ -1,25 +1,10 @@
|
|
|
1
|
-
import { ObjectionAnswer, Objection
|
|
2
|
-
import { BlockerVerdict } from "./blockers.js";
|
|
3
|
-
export { BLOCKER_DESCRIPTION } from "./blockers.js";
|
|
4
|
-
/** What the server made of one answer's blocker. Passed in, never computed here:
|
|
5
|
-
* the probes are network and git calls and the verifiers reach neither. UNDEFINED
|
|
6
|
-
* where the caller ran no probes, which reads as `unverifiable` — the position
|
|
7
|
-
* every blocker close stood in before the checks existed. */
|
|
8
|
-
export type BlockerJudge = (blocker: PlanBlocker) => BlockerVerdict;
|
|
1
|
+
import { ObjectionAnswer, Objection } from "./types.js";
|
|
9
2
|
/** The objection raised below carries this prefix and the id it objects to. */
|
|
10
3
|
export declare const UNKNOWN_ANSWER_PREFIX = "answers:unknown:";
|
|
11
|
-
/** The
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
|
|
15
|
-
*
|
|
16
|
-
* The ENTRY, so a caller that files the blocker files the one that arrived with
|
|
17
|
-
* THIS answer. Searching for the two separately took them from different entries:
|
|
18
|
-
* a blank-answer entry supplied the blocker, and a refuted claim was published
|
|
19
|
-
* beside an answer a later entry had closed. */
|
|
20
|
-
export declare function closingAnswerFor(objection: Objection, acknowledged: ObjectionAnswer[], judge?: BlockerJudge): ObjectionAnswer | undefined;
|
|
21
|
-
/** The closing answer as the sentence alone, for a caller that files no blocker. */
|
|
22
|
-
export declare function answerFor(objection: Objection, acknowledged: ObjectionAnswer[], judge?: BlockerJudge): string | undefined;
|
|
4
|
+
/** The answer to one objection, matched by `objectionId` and never by position — the
|
|
5
|
+
* agent reorders its list between registrations. A whitespace-only answer closes
|
|
6
|
+
* nothing, and the FIRST NON-BLANK entry wins when one id is answered twice. */
|
|
7
|
+
export declare function answerFor(objection: Objection, acknowledged: ObjectionAnswer[]): string | undefined;
|
|
23
8
|
/** One objection per answer naming an objection this call did not raise — a stale
|
|
24
9
|
* id, a typo and a pre-emptive answer all used to pass in silence. An id is
|
|
25
10
|
* recognised BY ITS PREFIX and never draws an objection of its own. `scope` and
|
|
@@ -28,49 +13,29 @@ export declare function unknownAnswerObjections(objections: Objection[], acknowl
|
|
|
28
13
|
scope: string;
|
|
29
14
|
suggestion: string;
|
|
30
15
|
}): Objection[];
|
|
31
|
-
/** The
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
|
|
36
|
-
*
|
|
37
|
-
* `defects:untested:` — a declared defect's test can always be written, even when
|
|
38
|
-
* the service it runs against is down: it is committed and runs once the service is
|
|
39
|
-
* back. Every blocker Wipro repo-1 sent for a defect (14 on one run, 2 on the next)
|
|
40
|
-
* named a limit of the browser tools, and none got a test.
|
|
41
|
-
*
|
|
42
|
-
* `defects:expectedPass:` — a test that passes today proves nothing about a defect,
|
|
43
|
-
* and a sentence here would let one cited passing test stand in for the red one. */
|
|
44
|
-
export declare const NON_ANSWERABLE_PREFIXES: readonly ["coverage:stateTest:", "defects:untested:", "defects:expectedPass:"];
|
|
16
|
+
/** The one objection an answer does not close. Every other check in v2 can be
|
|
17
|
+
* answered, and this one cannot: the missed bugs it exists for were reached by
|
|
18
|
+
* prose twice and skipped twice, so an answer would close the objection and
|
|
19
|
+
* leave the bug. `verifierContracts.ts` carries the evidence. */
|
|
20
|
+
export declare const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
45
21
|
export declare function isNonAnswerable(objectionId: unknown): boolean;
|
|
46
22
|
/** The objections a sentence alone does not close either. A change with no test
|
|
47
23
|
* needs the test, a value a page and API change accepts or refuses needs a UI step
|
|
48
24
|
* that names it (the XR02 evidence is at `surfaceCase` and `surfaceReject` in
|
|
49
|
-
* `verifierContracts.ts`)
|
|
50
|
-
* the
|
|
25
|
+
* `verifierContracts.ts`), and a defect with no test needs the test — a defect
|
|
26
|
+
* the code already had before this pull request included. Each closes only when the
|
|
27
|
+
* answer also names what stopped the test: a service that is not running, a
|
|
28
|
+
* branch that no longer exists, the one credential the run holds. Measured on run
|
|
51
29
|
* 34176038240: across eight fixtures 45 objections were raised and 45 were
|
|
52
30
|
* closed, and the one class that a sentence cannot close was never raised at all.
|
|
53
31
|
* Prose closed every uncovered change, including changes nothing prevented a test
|
|
54
32
|
* from reaching. Runs 34423214322 and 34423217796 then lost three defects the
|
|
55
|
-
* baseline reported: each was listed and closed by a sentence.
|
|
56
|
-
|
|
57
|
-
* A bug the report states is the same promise made after the tests run, so it
|
|
58
|
-
* closes the same way. Runs 34434835959 (repo15 RG21 and RG15) and 34434838150
|
|
59
|
-
* (repo10 RG03) answered three `issueTraceability:` objections with "outside this
|
|
60
|
-
* PR's surface" and "unobservable" and wrote no test — and RG21 and RG15 were the
|
|
61
|
-
* same 404 bootstrap bug, reported twice and tested neither time. */
|
|
62
|
-
export declare const BLOCKER_ONLY_PREFIXES: readonly ["coverage:change:", "coverage:surfaceCase:", "coverage:surfaceReject:", "issueTraceability:"];
|
|
33
|
+
* baseline reported: each was listed and closed by a sentence. */
|
|
34
|
+
export declare const BLOCKER_ONLY_PREFIXES: readonly ["coverage:change:", "coverage:surfaceCase:", "coverage:surfaceReject:", "defects:untested:"];
|
|
63
35
|
export declare function needsBlocker(objectionId: unknown): boolean;
|
|
64
36
|
/** The objection raised below carries this prefix and the id it objects to. */
|
|
65
37
|
export declare const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
66
38
|
/** One objection per answer to an objection only a planned test closes. The answered
|
|
67
39
|
* objection STAYS OPEN — this says so, rather than letting the answer pass in
|
|
68
40
|
* silence while the plan still owes the test. */
|
|
69
|
-
export declare function refusedAnswerObjections(objections: Objection[], acknowledged: ObjectionAnswer[]
|
|
70
|
-
/** Whether an EARLIER call already closed this objection and that close still
|
|
71
|
-
* stands. The report tool carries answers across calls, so an objection closed
|
|
72
|
-
* on call 1 is raised again on call 2 and answered again there. Without this,
|
|
73
|
-
* a weaker second answer raised `answers:refused:` beside a close the report
|
|
74
|
-
* publishes as answered — the report then said both. `registerPlan` carries
|
|
75
|
-
* the previous registration's closes the same way. */
|
|
76
|
-
closedEarlier?: (objectionId: string) => boolean): Objection[];
|
|
41
|
+
export declare function refusedAnswerObjections(objections: Objection[], acknowledged: ObjectionAnswer[]): Objection[];
|
|
@@ -1,34 +1,9 @@
|
|
|
1
|
-
import { hasBlocker, refutedEvidence } from "./blockers.js";
|
|
2
|
-
export { BLOCKER_DESCRIPTION } from "./blockers.js";
|
|
3
|
-
/** Whether this answer's blocker closes the objection: present, and not refuted by
|
|
4
|
-
* the server's own probe. */
|
|
5
|
-
function blockerCloses(blocker, judge) {
|
|
6
|
-
if (!hasBlocker(blocker))
|
|
7
|
-
return false;
|
|
8
|
-
return !judge || judge(blocker) !== "refuted";
|
|
9
|
-
}
|
|
10
|
-
/** Whether one entry closes an objection that needs a blocker: the SAME rule
|
|
11
|
-
* `closingAnswerFor` applies, so a sibling that suppresses a refusal is a sibling
|
|
12
|
-
* that really closed. Reading the blocker alone let an entry with a good blocker
|
|
13
|
-
* and an empty answer suppress the refusal while closing nothing. */
|
|
14
|
-
function closes(entry, judge) {
|
|
15
|
-
const answer = entry?.answer;
|
|
16
|
-
if (typeof answer !== "string" || answer.trim().length === 0)
|
|
17
|
-
return false;
|
|
18
|
-
return blockerCloses(entry?.blocker, judge);
|
|
19
|
-
}
|
|
20
1
|
/** The objection raised below carries this prefix and the id it objects to. */
|
|
21
2
|
export const UNKNOWN_ANSWER_PREFIX = "answers:unknown:";
|
|
22
|
-
/** The
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
|
|
26
|
-
*
|
|
27
|
-
* The ENTRY, so a caller that files the blocker files the one that arrived with
|
|
28
|
-
* THIS answer. Searching for the two separately took them from different entries:
|
|
29
|
-
* a blank-answer entry supplied the blocker, and a refuted claim was published
|
|
30
|
-
* beside an answer a later entry had closed. */
|
|
31
|
-
export function closingAnswerFor(objection, acknowledged, judge) {
|
|
3
|
+
/** The answer to one objection, matched by `objectionId` and never by position — the
|
|
4
|
+
* agent reorders its list between registrations. A whitespace-only answer closes
|
|
5
|
+
* nothing, and the FIRST NON-BLANK entry wins when one id is answered twice. */
|
|
6
|
+
export function answerFor(objection, acknowledged) {
|
|
32
7
|
const wantsBlocker = needsBlocker(objection.objectionId);
|
|
33
8
|
for (const entry of acknowledged) {
|
|
34
9
|
if (entry.objectionId !== objection.objectionId)
|
|
@@ -37,16 +12,12 @@ export function closingAnswerFor(objection, acknowledged, judge) {
|
|
|
37
12
|
const answer = typeof rawAnswer === "string" ? rawAnswer.trim() : "";
|
|
38
13
|
if (answer.length === 0)
|
|
39
14
|
continue;
|
|
40
|
-
if (wantsBlocker &&
|
|
15
|
+
if (wantsBlocker && String(entry.blocker ?? "").trim().length === 0)
|
|
41
16
|
continue;
|
|
42
|
-
return
|
|
17
|
+
return answer;
|
|
43
18
|
}
|
|
44
19
|
return undefined;
|
|
45
20
|
}
|
|
46
|
-
/** The closing answer as the sentence alone, for a caller that files no blocker. */
|
|
47
|
-
export function answerFor(objection, acknowledged, judge) {
|
|
48
|
-
return closingAnswerFor(objection, acknowledged, judge)?.answer;
|
|
49
|
-
}
|
|
50
21
|
/** One objection per answer naming an objection this call did not raise — a stale
|
|
51
22
|
* id, a typo and a pre-emptive answer all used to pass in silence. An id is
|
|
52
23
|
* recognised BY ITS PREFIX and never draws an objection of its own. `scope` and
|
|
@@ -72,50 +43,32 @@ export function unknownAnswerObjections(objections, acknowledged, { scope, sugge
|
|
|
72
43
|
}
|
|
73
44
|
return raisedHere;
|
|
74
45
|
}
|
|
75
|
-
/** The
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
|
|
80
|
-
*
|
|
81
|
-
* `defects:untested:` — a declared defect's test can always be written, even when
|
|
82
|
-
* the service it runs against is down: it is committed and runs once the service is
|
|
83
|
-
* back. Every blocker Wipro repo-1 sent for a defect (14 on one run, 2 on the next)
|
|
84
|
-
* named a limit of the browser tools, and none got a test.
|
|
85
|
-
*
|
|
86
|
-
* `defects:expectedPass:` — a test that passes today proves nothing about a defect,
|
|
87
|
-
* and a sentence here would let one cited passing test stand in for the red one. */
|
|
88
|
-
export const NON_ANSWERABLE_PREFIXES = [
|
|
89
|
-
"coverage:stateTest:",
|
|
90
|
-
"defects:untested:",
|
|
91
|
-
"defects:expectedPass:",
|
|
92
|
-
];
|
|
93
|
-
const DEFECT_PREFIXES = ["defects:untested:", "defects:expectedPass:"];
|
|
46
|
+
/** The one objection an answer does not close. Every other check in v2 can be
|
|
47
|
+
* answered, and this one cannot: the missed bugs it exists for were reached by
|
|
48
|
+
* prose twice and skipped twice, so an answer would close the objection and
|
|
49
|
+
* leave the bug. `verifierContracts.ts` carries the evidence. */
|
|
50
|
+
export const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
94
51
|
export function isNonAnswerable(objectionId) {
|
|
95
52
|
return (typeof objectionId === "string" &&
|
|
96
|
-
|
|
53
|
+
objectionId.startsWith(NON_ANSWERABLE_PREFIX));
|
|
97
54
|
}
|
|
98
55
|
/** The objections a sentence alone does not close either. A change with no test
|
|
99
56
|
* needs the test, a value a page and API change accepts or refuses needs a UI step
|
|
100
57
|
* that names it (the XR02 evidence is at `surfaceCase` and `surfaceReject` in
|
|
101
|
-
* `verifierContracts.ts`)
|
|
102
|
-
* the
|
|
58
|
+
* `verifierContracts.ts`), and a defect with no test needs the test — a defect
|
|
59
|
+
* the code already had before this pull request included. Each closes only when the
|
|
60
|
+
* answer also names what stopped the test: a service that is not running, a
|
|
61
|
+
* branch that no longer exists, the one credential the run holds. Measured on run
|
|
103
62
|
* 34176038240: across eight fixtures 45 objections were raised and 45 were
|
|
104
63
|
* closed, and the one class that a sentence cannot close was never raised at all.
|
|
105
64
|
* Prose closed every uncovered change, including changes nothing prevented a test
|
|
106
65
|
* from reaching. Runs 34423214322 and 34423217796 then lost three defects the
|
|
107
|
-
* baseline reported: each was listed and closed by a sentence.
|
|
108
|
-
*
|
|
109
|
-
* A bug the report states is the same promise made after the tests run, so it
|
|
110
|
-
* closes the same way. Runs 34434835959 (repo15 RG21 and RG15) and 34434838150
|
|
111
|
-
* (repo10 RG03) answered three `issueTraceability:` objections with "outside this
|
|
112
|
-
* PR's surface" and "unobservable" and wrote no test — and RG21 and RG15 were the
|
|
113
|
-
* same 404 bootstrap bug, reported twice and tested neither time. */
|
|
66
|
+
* baseline reported: each was listed and closed by a sentence. */
|
|
114
67
|
export const BLOCKER_ONLY_PREFIXES = [
|
|
115
68
|
"coverage:change:",
|
|
116
69
|
"coverage:surfaceCase:",
|
|
117
70
|
"coverage:surfaceReject:",
|
|
118
|
-
"
|
|
71
|
+
"defects:untested:",
|
|
119
72
|
];
|
|
120
73
|
export function needsBlocker(objectionId) {
|
|
121
74
|
return (typeof objectionId === "string" &&
|
|
@@ -126,14 +79,7 @@ export const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
|
126
79
|
/** One objection per answer to an objection only a planned test closes. The answered
|
|
127
80
|
* objection STAYS OPEN — this says so, rather than letting the answer pass in
|
|
128
81
|
* silence while the plan still owes the test. */
|
|
129
|
-
export function refusedAnswerObjections(objections, acknowledged
|
|
130
|
-
/** Whether an EARLIER call already closed this objection and that close still
|
|
131
|
-
* stands. The report tool carries answers across calls, so an objection closed
|
|
132
|
-
* on call 1 is raised again on call 2 and answered again there. Without this,
|
|
133
|
-
* a weaker second answer raised `answers:refused:` beside a close the report
|
|
134
|
-
* publishes as answered — the report then said both. `registerPlan` carries
|
|
135
|
-
* the previous registration's closes the same way. */
|
|
136
|
-
closedEarlier) {
|
|
82
|
+
export function refusedAnswerObjections(objections, acknowledged) {
|
|
137
83
|
const raised = new Set(objections
|
|
138
84
|
.map((objection) => objection.objectionId)
|
|
139
85
|
.filter((id) => isNonAnswerable(id) || needsBlocker(id)));
|
|
@@ -146,37 +92,21 @@ closedEarlier) {
|
|
|
146
92
|
const answer = entry.answer;
|
|
147
93
|
if (typeof answer !== "string" || answer.trim().length === 0)
|
|
148
94
|
continue;
|
|
149
|
-
// An answer
|
|
150
|
-
//
|
|
151
|
-
|
|
152
|
-
// two, and one of them closing means the objection is closed — refusing on the
|
|
153
|
-
// other raised a refusal beside a close for the same objection.
|
|
154
|
-
if (needsBlocker(id) &&
|
|
155
|
-
(acknowledged.some((other) => other.objectionId === id && closes(other, judge)) ||
|
|
156
|
-
closedEarlier?.(id))) {
|
|
95
|
+
// An answer that names a blocker DOES close an untested change, so it is not
|
|
96
|
+
// refused. Only the sentence on its own is.
|
|
97
|
+
if (needsBlocker(id) && String(entry.blocker ?? "").trim().length > 0)
|
|
157
98
|
continue;
|
|
158
|
-
}
|
|
159
99
|
seen.add(id);
|
|
160
|
-
const blocker = entry?.blocker;
|
|
161
|
-
const refutedBlocker = needsBlocker(id) && hasBlocker(blocker) ? blocker : undefined;
|
|
162
100
|
refused.push({
|
|
163
101
|
objectionId: `${REFUSED_ANSWER_PREFIX}${id}`,
|
|
164
102
|
verifier: "answers",
|
|
165
|
-
message:
|
|
166
|
-
? "The
|
|
167
|
-
:
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
: `answered "${id}"${needsBlocker(id) ? " with no `blocker`" : ", which no answer closes"}`,
|
|
173
|
-
suggestion: refutedBlocker
|
|
174
|
-
? "Write the test: what you named as the blocker is not in the way. If something else is, send the blocker that names it — the server checks each kind against the run it can see."
|
|
175
|
-
: needsBlocker(id)
|
|
176
|
-
? "Plan the test and register the plan again. If this run cannot write one, send the same answer with `blocker` — `service-unreachable` with the URL, or `credential-missing` with the URL of the page that asked for the login. A reason it is not worth testing is not a blocker."
|
|
177
|
-
: DEFECT_PREFIXES.some((prefix) => id.startsWith(prefix))
|
|
178
|
-
? "Plan a test that cites the defect in `defects` and sets `expected.outcome` to `fail`, then register the plan again. If a service the test needs is down, write the test anyway: it is committed and runs once the service is back."
|
|
179
|
-
: "Add a planned test that names the record's state as `startState` and runs this route's mutation on it, then register the plan again. Every other objection takes an answer; this one takes a test.",
|
|
103
|
+
message: needsBlocker(id)
|
|
104
|
+
? "The answer says why there is no test, and it does not say what stopped this run from writing one, so the objection stays open."
|
|
105
|
+
: "This objection is closed only by a planned test, so the answer closed nothing and the objection stays open.",
|
|
106
|
+
evidence: `answered "${id}"${needsBlocker(id) ? " with no `blocker`" : ", which no answer closes"}`,
|
|
107
|
+
suggestion: needsBlocker(id)
|
|
108
|
+
? "Plan the test and register the plan again. If this run cannot write one, send the same answer with `blocker` naming what stopped it — a service that is not running, a paired branch that no longer exists, the one credential the run holds. A reason it is not worth testing is not a blocker."
|
|
109
|
+
: "Add a planned test that names the record's state as `startState` and runs this route's mutation on it, then register the plan again. Every other objection takes an answer; this one takes a test.",
|
|
180
110
|
});
|
|
181
111
|
}
|
|
182
112
|
return refused;
|
|
@@ -9,8 +9,6 @@ export declare function pullRequestText(): {
|
|
|
9
9
|
description: string;
|
|
10
10
|
} | undefined;
|
|
11
11
|
export declare function clearPullRequestText(): void;
|
|
12
|
-
export declare function recordUiCredentials(value: unknown): void;
|
|
13
|
-
export declare function uiCredentialsSupplied(): boolean | undefined;
|
|
14
12
|
/** The pull request as one searchable string. Whitespace runs fold to one space, so
|
|
15
13
|
* a quote wrapped across lines still matches the sentence it was taken from. */
|
|
16
14
|
export declare const searchable: (pr: {
|
|
@@ -21,22 +21,6 @@ export function pullRequestText() {
|
|
|
21
21
|
}
|
|
22
22
|
export function clearPullRequestText() {
|
|
23
23
|
recorded = undefined;
|
|
24
|
-
credentialsSupplied = undefined;
|
|
25
|
-
}
|
|
26
|
-
/** Whether the run was given browser credentials, recorded from the same place and
|
|
27
|
-
* for the same reason as the text above: the prompt and the plan tool share a
|
|
28
|
-
* process, so the flag is already here. A blocker of kind `credential-missing`
|
|
29
|
-
* is checked against it.
|
|
30
|
-
*
|
|
31
|
-
* UNDEFINED when this process never rendered the prompt, which is not `false` —
|
|
32
|
-
* the server cannot tell a run with no credentials from a run it never saw, and
|
|
33
|
-
* the blocker is then unverifiable rather than true. */
|
|
34
|
-
let credentialsSupplied;
|
|
35
|
-
export function recordUiCredentials(value) {
|
|
36
|
-
credentialsSupplied = typeof value === "string" && value.trim().length > 0;
|
|
37
|
-
}
|
|
38
|
-
export function uiCredentialsSupplied() {
|
|
39
|
-
return credentialsSupplied;
|
|
40
24
|
}
|
|
41
25
|
/** The pull request as one searchable string. Whitespace runs fold to one space, so
|
|
42
26
|
* a quote wrapped across lines still matches the sentence it was taken from. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Objection, ObjectionAnswer,
|
|
1
|
+
import { Objection, ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlanMaintenance, PlannedTest, VerifyContext } from "./types.js";
|
|
2
2
|
/** The plan as stored for the run. Every registration produces one; the newest
|
|
3
3
|
* replaces the last. */
|
|
4
4
|
export interface Plan {
|
|
@@ -14,14 +14,11 @@ export interface Plan {
|
|
|
14
14
|
* `coverage` counts the changes they cite, so a reader of the plan has to be
|
|
15
15
|
* able to see which coverage came from maintenance rather than a new test. */
|
|
16
16
|
maintains?: PlanMaintenance[];
|
|
17
|
-
/** Objections the agent answered, with the answers.
|
|
18
|
-
* server PROBED and could not refute — an unreachable service that really did not
|
|
19
|
-
* answer, a branch that really is gone — so the close is a checked one. */
|
|
17
|
+
/** Objections the agent answered, with the answers. */
|
|
20
18
|
answeredObjections: Array<{
|
|
21
19
|
objection: Objection;
|
|
22
20
|
answer: string;
|
|
23
|
-
blocker?:
|
|
24
|
-
blockerVerified?: boolean;
|
|
21
|
+
blocker?: string;
|
|
25
22
|
}>;
|
|
26
23
|
/** Closed by a blocker nothing checked. Its own bucket rather than a flag on
|
|
27
24
|
* `answeredObjections`, so a run's count of them is a number the report can
|
|
@@ -32,7 +29,7 @@ export interface Plan {
|
|
|
32
29
|
unverifiedCloses: Array<{
|
|
33
30
|
objection: Objection;
|
|
34
31
|
answer: string;
|
|
35
|
-
blocker:
|
|
32
|
+
blocker: string;
|
|
36
33
|
}>;
|
|
37
34
|
/** Objections with no answer. Rendered in the report — this is the accountability. */
|
|
38
35
|
openObjections: Objection[];
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { answerFor, closingAnswerFor, isNonAnswerable, refusedAnswerObjections, unknownAnswerObjections, } from "./answers.js";
|
|
1
|
+
import { answerFor, isNonAnswerable, refusedAnswerObjections, unknownAnswerObjections, } from "./answers.js";
|
|
3
2
|
import { runPlanTimeVerifiers } from "./runVerifiers.js";
|
|
4
3
|
import { logger } from "../utils/logger.js";
|
|
5
4
|
function storedRegistrationNumber(registration) {
|
|
@@ -38,7 +37,7 @@ function carriedAnswerFor(objectionId, previous) {
|
|
|
38
37
|
return {
|
|
39
38
|
objectionId,
|
|
40
39
|
answer,
|
|
41
|
-
...(
|
|
40
|
+
...(entry.blocker ? { blocker: entry.blocker } : {}),
|
|
42
41
|
};
|
|
43
42
|
}
|
|
44
43
|
return undefined;
|
|
@@ -66,9 +65,6 @@ export function registerPlan(registration, ctx, previous) {
|
|
|
66
65
|
const changes = registration.changes;
|
|
67
66
|
const defects = registration.defects;
|
|
68
67
|
const registrationNumber = storedRegistrationNumber(registration);
|
|
69
|
-
// Awaited by the caller before this context was built: every probe is an HTTP
|
|
70
|
-
// request or a `git` call, and nothing from here down is asynchronous.
|
|
71
|
-
const judge = ctx?.blockerVerdict?.bind(ctx);
|
|
72
68
|
const verified = runPlanTimeVerifiers(registration, ctx);
|
|
73
69
|
const objections = [
|
|
74
70
|
...verified,
|
|
@@ -76,15 +72,7 @@ export function registerPlan(registration, ctx, previous) {
|
|
|
76
72
|
scope: "on this plan",
|
|
77
73
|
suggestion: "Answer only the ids in the Objections list this registration returns. An objection you have since fixed is gone and needs no answer, and an objection raised after the tests run cannot be answered before them.",
|
|
78
74
|
}),
|
|
79
|
-
...refusedAnswerObjections(verified, acknowledged,
|
|
80
|
-
// The previous registration's close is carried below when this answer does
|
|
81
|
-
// not close. If that carried close still stands, it is the close.
|
|
82
|
-
const objection = verified.find((o) => o.objectionId === id);
|
|
83
|
-
const carried = carriedAnswerFor(id, previous);
|
|
84
|
-
return (!!objection &&
|
|
85
|
-
!!carried &&
|
|
86
|
-
!!closingAnswerFor(objection, [carried], judge));
|
|
87
|
-
}),
|
|
75
|
+
...refusedAnswerObjections(verified, acknowledged),
|
|
88
76
|
];
|
|
89
77
|
const carried = carriedAnswers(objections, acknowledged, previous);
|
|
90
78
|
// The carried answers go last: `answerFor` takes the first non-blank entry, so
|
|
@@ -99,29 +87,15 @@ export function registerPlan(registration, ctx, previous) {
|
|
|
99
87
|
// a planned test too, unless the answer names what stopped this run from
|
|
100
88
|
// writing one. Either way an answer that does not close it leaves it open and
|
|
101
89
|
// draws the refusal objection raised above.
|
|
102
|
-
const
|
|
90
|
+
const answer = isNonAnswerable(objection.objectionId)
|
|
103
91
|
? undefined
|
|
104
|
-
:
|
|
105
|
-
if (
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
|
|
111
|
-
? closing.blocker
|
|
112
|
-
: undefined;
|
|
113
|
-
// A blocker the server PROBED and could not refute is a checked close and sits
|
|
114
|
-
// with the answers. One nothing could look at is counted apart: a run whose
|
|
115
|
-
// objections all close that way must not read like one that answered.
|
|
116
|
-
if (blocker && judge?.(blocker) === "verified") {
|
|
117
|
-
answeredObjections.push({
|
|
118
|
-
objection,
|
|
119
|
-
answer,
|
|
120
|
-
blocker,
|
|
121
|
-
blockerVerified: true,
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
else if (blocker)
|
|
92
|
+
: answerFor(objection, answers);
|
|
93
|
+
if (answer) {
|
|
94
|
+
const blocker = answers.find((entry) => entry.objectionId === objection.objectionId &&
|
|
95
|
+
String(entry.blocker ?? "").trim().length > 0)?.blocker;
|
|
96
|
+
// A blocker closes the objection and nothing here can check it, so it is
|
|
97
|
+
// counted apart from an answer the plan itself makes checkable.
|
|
98
|
+
if (blocker)
|
|
125
99
|
unverifiedCloses.push({ objection, answer, blocker });
|
|
126
100
|
else
|
|
127
101
|
answeredObjections.push({ objection, answer });
|
|
@@ -164,24 +164,13 @@ export interface PlannedTest {
|
|
|
164
164
|
* included — the name is what joins the test to the diff it is checked against. */
|
|
165
165
|
repository: string;
|
|
166
166
|
}
|
|
167
|
-
/** The kinds of thing that stop a run from writing a test AND that the server can
|
|
168
|
-
* check. Free text let an argument close an untested defect — run 34434835959
|
|
169
|
-
* closed one with "the PR description states no redirect or fallback" — so a
|
|
170
|
-
* blocker is now a claim of one of these, and the server probes it. */
|
|
171
|
-
export type PlanBlockerKind = "service-unreachable" | "credential-missing";
|
|
172
|
-
/** What stopped this run from writing the test, as a checkable claim. */
|
|
173
|
-
export interface PlanBlocker {
|
|
174
|
-
kind: PlanBlockerKind;
|
|
175
|
-
/** `service-unreachable`: the URL the run needed. `credential-missing`: the URL
|
|
176
|
-
* of the page that asked for the login. */
|
|
177
|
-
detail: string;
|
|
178
|
-
}
|
|
179
167
|
export interface ObjectionAnswer {
|
|
180
168
|
objectionId: string;
|
|
181
169
|
answer: string;
|
|
182
|
-
/** What stopped this run from writing the test
|
|
183
|
-
*
|
|
184
|
-
|
|
170
|
+
/** What stopped this run from writing the test: a service that is not running,
|
|
171
|
+
* a paired branch that no longer exists, the one credential the run holds.
|
|
172
|
+
* An objection about an untested change closes only with one of these. */
|
|
173
|
+
blocker?: string;
|
|
185
174
|
}
|
|
186
175
|
/** An existing test this run maintains instead of planning a new one, and the
|
|
187
176
|
* declared changes that maintenance covers. A removal already covered by a spec
|
|
@@ -266,12 +255,6 @@ export interface VerifyContext {
|
|
|
266
255
|
* the verifiers filesystem-free. Named a repository it asks that one, and a name
|
|
267
256
|
* this run does not know answers false. Omitted, it asks every one. */
|
|
268
257
|
citedFileExists(relativePath: string, repository?: string): boolean;
|
|
269
|
-
/** What the server made of one answer's blocker. Injected for the same reason as
|
|
270
|
-
* `citedFileExists`, and more so: the probes are HTTP requests and `git` calls,
|
|
271
|
-
* and they are awaited before this context is built because the verifiers are
|
|
272
|
-
* synchronous. UNDEFINED where the caller ran no probes — every blocker then
|
|
273
|
-
* reads as unverifiable, which is where they all stood before the checks. */
|
|
274
|
-
blockerVerdict?(blocker: PlanBlocker): "verified" | "refuted" | "unverifiable";
|
|
275
258
|
/** The text of a `spec:` file a plan change cites, from the first repository that
|
|
276
259
|
* holds it. UNDEFINED for a path no repository holds, a path outside every
|
|
277
260
|
* checkout, or a file too large to read. Injected for the same reason as
|
|
@@ -126,7 +126,7 @@ export declare const EXPECTED_OUTCOME_CONTRACT: {
|
|
|
126
126
|
};
|
|
127
127
|
readonly executed: {
|
|
128
128
|
readonly message: "This test failed, but it did not reach the behavior it was written for — it fell over earlier. A failure like this is not a caught bug.";
|
|
129
|
-
readonly suggestion: "If the blocker is the test's own setup — sign-in, token, a wrong path — fix it and run the test again. If the blocker is the defect this run reports (a route that is not mounted, a server error before the behaviour), answer this objection naming that issue: the failure then stands as evidence of that issue, not as a caught bug.
|
|
129
|
+
readonly suggestion: "If the blocker is the test's own setup — sign-in, token, a wrong path — fix it and run the test again. If the blocker is the defect this run reports (a route that is not mounted, a server error before the behaviour), answer this objection naming that issue: the failure then stands as evidence of that issue, not as a caught bug.";
|
|
130
130
|
};
|
|
131
131
|
};
|
|
132
132
|
readonly suggestion: "Should this test fail until the defect is fixed, or is the category wrong for a test that pins behaviour the app already has?";
|
|
@@ -158,7 +158,7 @@ export declare const COVERAGE_CONTRACT: {
|
|
|
158
158
|
readonly objections: {
|
|
159
159
|
readonly change: {
|
|
160
160
|
readonly message: "This declared change has no test in the plan.";
|
|
161
|
-
readonly suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
|
|
161
|
+
readonly suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.";
|
|
162
162
|
};
|
|
163
163
|
readonly noChanges: {
|
|
164
164
|
readonly message: "The plan declares no changes, so nothing says what the pull request must make different.";
|
|
@@ -302,11 +302,11 @@ export declare const DEFECTS_CONTRACT: {
|
|
|
302
302
|
};
|
|
303
303
|
readonly untested: {
|
|
304
304
|
readonly message: "This declared defect has no test in the plan.";
|
|
305
|
-
readonly suggestion: "Add a planned test that cites the defect in `defects` and sets `expected.outcome` to `fail`. A defect in a file this pull request touches gets the test whether or not this pull request introduced it. If an existing test already proves it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this defect's id — maintenance is proof, and a duplicate spec beside the one you edited is the mistake.
|
|
305
|
+
readonly suggestion: "Add a planned test that cites the defect in `defects` and sets `expected.outcome` to `fail`. A defect in a file this pull request touches gets the test whether or not this pull request introduced it. If an existing test already proves it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this defect's id — maintenance is proof, and a duplicate spec beside the one you edited is the mistake. If something stopped this run from writing the test, answer it and set `blocker` to what stopped it.";
|
|
306
306
|
};
|
|
307
307
|
readonly expectedPass: {
|
|
308
308
|
readonly message: "Every test that cites this defect expects to pass, and a test that passes today proves nothing about a defect.";
|
|
309
|
-
readonly suggestion: "Assert what the fixed code returns
|
|
309
|
+
readonly suggestion: "Assert what the fixed code returns, set `expected.outcome` to `fail` with `why`, or answer in one line why a passing test is the proof.";
|
|
310
310
|
};
|
|
311
311
|
readonly unknownId: {
|
|
312
312
|
readonly message: "A planned test cites a defect id that this plan does not declare.";
|
|
@@ -114,7 +114,7 @@ export const EXPECTED_OUTCOME_CONTRACT = {
|
|
|
114
114
|
// 33988304641, P11), which the run that wrote `true` never drew.
|
|
115
115
|
executed: {
|
|
116
116
|
message: "This test failed, but it did not reach the behavior it was written for — it fell over earlier. A failure like this is not a caught bug.",
|
|
117
|
-
suggestion: "If the blocker is the test's own setup — sign-in, token, a wrong path — fix it and run the test again. If the blocker is the defect this run reports (a route that is not mounted, a server error before the behaviour), answer this objection naming that issue: the failure then stands as evidence of that issue, not as a caught bug.
|
|
117
|
+
suggestion: "If the blocker is the test's own setup — sign-in, token, a wrong path — fix it and run the test again. If the blocker is the defect this run reports (a route that is not mounted, a server error before the behaviour), answer this objection naming that issue: the failure then stands as evidence of that issue, not as a caught bug.",
|
|
118
118
|
},
|
|
119
119
|
},
|
|
120
120
|
suggestion: "Should this test fail until the defect is fixed, or is the category wrong for a test that pins behaviour the app already has?",
|
|
@@ -149,7 +149,7 @@ export const COVERAGE_CONTRACT = {
|
|
|
149
149
|
objections: {
|
|
150
150
|
change: {
|
|
151
151
|
message: "This declared change has no test in the plan.",
|
|
152
|
-
suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.",
|
|
152
|
+
suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.",
|
|
153
153
|
},
|
|
154
154
|
noChanges: {
|
|
155
155
|
message: "The plan declares no changes, so nothing says what the pull request must make different.",
|
|
@@ -375,11 +375,11 @@ export const DEFECTS_CONTRACT = {
|
|
|
375
375
|
},
|
|
376
376
|
untested: {
|
|
377
377
|
message: "This declared defect has no test in the plan.",
|
|
378
|
-
suggestion: "Add a planned test that cites the defect in `defects` and sets `expected.outcome` to `fail`. A defect in a file this pull request touches gets the test whether or not this pull request introduced it. If an existing test already proves it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this defect's id — maintenance is proof, and a duplicate spec beside the one you edited is the mistake.
|
|
378
|
+
suggestion: "Add a planned test that cites the defect in `defects` and sets `expected.outcome` to `fail`. A defect in a file this pull request touches gets the test whether or not this pull request introduced it. If an existing test already proves it and this run edits that test, or runs it unchanged as a VERIFY, rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this defect's id — maintenance is proof, and a duplicate spec beside the one you edited is the mistake. If something stopped this run from writing the test, answer it and set `blocker` to what stopped it.",
|
|
379
379
|
},
|
|
380
380
|
expectedPass: {
|
|
381
381
|
message: "Every test that cites this defect expects to pass, and a test that passes today proves nothing about a defect.",
|
|
382
|
-
suggestion: "Assert what the fixed code returns
|
|
382
|
+
suggestion: "Assert what the fixed code returns, set `expected.outcome` to `fail` with `why`, or answer in one line why a passing test is the proof.",
|
|
383
383
|
},
|
|
384
384
|
unknownId: {
|
|
385
385
|
message: "A planned test cites a defect id that this plan does not declare.",
|