@skyramp/mcp 0.4.4-rc.2 → 0.4.4

package/build/tools/submitReportTool.js

@@ -9,7 +9,7 @@ import { ProgrammingLanguage, TESTBOT_SUPPORTED_TEST_TYPES, TestType, } from "..
 import { STEP_METHOD_DESCRIPTION, stepMethodSchema, } from "../types/StepMethod.js";
 import { DriftAction } from "../types/TestAnalysis.js";
 import { isResult, TestExecutionStatus } from "../types/TestExecution.js";
-import { IssueFoundCategory } from "../types/TestbotReport.js";
+import { EnvironmentFamily, FailureCauseKind, IssueFoundCategory, } from "../types/TestbotReport.js";
 import { StateManager, runArtifactDir, getTestsRepoDir, resolveOwnRunStatePath, } from "../utils/AnalysisStateManager.js";
 import { toolError, testFileMatches } from "../utils/utils.js";
 import { isTestbotEnabled, isUtilsReuseEnabled, } from "../utils/featureFlags.js";
@@ -115,17 +115,46 @@ const repositoryField = z
     .trim()
     .min(1)
     .describe("The 'owner/repo' this item pertains to (e.g. 'letsramp/api-insight'). Set it on every item — the primary's owner/repo for primary-repo objections, or a related repo's owner/repo for objections derived from that repository's diff.");
+// One home for the family meanings: a `.describe()` on the base enum is replaced
+// by the `.optional().describe()` at each use site, so the meanings go there.
+const ENVIRONMENT_FAMILY_MEANINGS = "auth_token = an API token is missing or expired. " +
+    "secret = another secret or credential is missing. " +
+    "data_connection = the database or data store is not reachable. " +
+    "app_not_started = the app under test did not start or is not reachable. " +
+    "login = the sign-in step failed. " +
+    "external_dependency = an external service is not configured. " +
+    "recording_replay = recording or replay of the trace failed.";
+const environmentFamilyField = z.enum(EnvironmentFamily);
+// SKYR-4555. Why a Fail or Skipped row did not pass; the row-level refinement on
+// testResultSchema decides when it is required and which shapes are consistent.
+const failureCauseSchema = z
+    .object({
+    kind: z
+        .enum(FailureCauseKind)
+        .describe("application = the code under test misbehaved, or a test written to prove a defect failed as intended. " +
+        "environment = the test could not exercise the behaviour because of the test environment: a missing token or secret, no data connection, the app did not start, sign-in failed, an external service is not configured, or recording or replay failed. Name the gap in `family`. " +
+        "test_defect = the generated test itself is wrong, not the app: an ambiguous locator, a strict-mode clash, a bad assertion setup."),
+    family: environmentFamilyField
+        .optional()
+        .describe("The test-environment gap. Required when kind is 'environment'. Not allowed for the other kinds. " +
+        ENVIRONMENT_FAMILY_MEANINGS),
+})
+    .describe("Why this test did not pass. REQUIRED when status is 'Fail' or 'Skipped'. Not allowed when status is 'Pass'. " +
+    "It must agree with reachedBehavior: 'application' needs reachedBehavior true or omitted; 'environment' needs reachedBehavior false or omitted.");
 // videoPath is deliberately absent from this input contract: it is attached server-side
 // from the run's execution records (see attachVideoPath), and zod strips any the model
 // supplies anyway. SKYR-4156 is what happens when the agent owns that field instead.
-const testResultSchema = z.object({
+const testResultSchema = z
+    .object({
     testType: z
         .enum(TestType)
         .describe("Type of test. Do not include priority or other metadata in this field."),
     endpoint: z
         .string()
         .describe("For contract and integration tests, the HTTP verb and path, e.g. 'GET /api/v1/products'. For ui and e2e tests, the page path the test opens, e.g. '/notifications'."),
-    status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
+    status: z
+        .enum(["Pass", "Fail", "Skipped"])
+        .describe("Test execution result"),
     details: z
         .string()
         .describe("One sentence — no embedded newlines, no markdown. e.g. '10.8s, products_contract_test.py' or 'failed: <one-line error summary>, products_contract_test.py'"),
@@ -145,7 +174,76 @@ const testResultSchema = z.object({
         .describe("For a failing test only. `true` when the request that carries the behaviour under test was sent and answered by the code under test, and the assertion that failed is about that behaviour. " +
         "`false` when the failure came earlier: a 404, 401, 403 or 5xx on a route the change adds; an expired token or a sign-in page; a control or setup step that failed; a generator or SDK error. " +
         "If your own `details` text says the request was not sent or an earlier step tripped, this is `false`. Omit only if you cannot tell."),
+    failureCause: failureCauseSchema.optional(),
     repository: repositoryField,
+})
+    .superRefine((row, ctx) => {
+    // SKYR-4555: the business-user report sorts failures by cause, so every row
+    // that did not pass must name one and a passing row must not. The kind must
+    // agree with reachedBehavior where the agent set it: the two answer the same
+    // question — did the test reach the behaviour under test — from two sides.
+    const cause = row.failureCause;
+    if (row.status === "Pass") {
+        if (cause === undefined)
+            return;
+        ctx.addIssue({
+            code: "custom",
+            path: ["failureCause"],
+            message: `failureCause is not allowed when status is 'Pass'. A passing test has no failure cause. ` +
+                `Remove failureCause from this row.`,
+        });
+        return;
+    }
+    if (cause === undefined) {
+        ctx.addIssue({
+            code: "custom",
+            path: ["failureCause"],
+            message: `failureCause is required when status is '${row.status}'. Set failureCause.kind to one of: ` +
+                `'application' (the code under test misbehaved, or a test written to prove a defect failed as intended); ` +
+                `'environment' (the test environment stopped the test before it reached the behaviour — also set failureCause.family); ` +
+                `'test_defect' (the generated test itself is wrong, not the app).`,
+        });
+        return;
+    }
+    if (cause.kind === FailureCauseKind.Environment) {
+        if (cause.family === undefined) {
+            ctx.addIssue({
+                code: "custom",
+                path: ["failureCause", "family"],
+                message: `failureCause.family is required when failureCause.kind is 'environment'. ` +
+                    `Set it to the gap that stopped the test: ${Object.values(EnvironmentFamily).join(", ")}.`,
+            });
+        }
+    }
+    else if (cause.family !== undefined) {
+        ctx.addIssue({
+            code: "custom",
+            path: ["failureCause", "family"],
+            message: `failureCause.family is allowed only when failureCause.kind is 'environment'. ` +
+                `Remove failureCause.family, or set kind to 'environment' if the test environment stopped the test.`,
+        });
+    }
+    if (cause.kind === FailureCauseKind.Application) {
+        if (row.reachedBehavior === false) {
+            ctx.addIssue({
+                code: "custom",
+                path: ["failureCause", "kind"],
+                message: `failureCause.kind 'application' contradicts reachedBehavior false. 'application' means the test reached the behaviour under test and the code misbehaved. ` +
+                    `If the failure came earlier, set kind to 'environment' or 'test_defect'. If the test did reach the behaviour, set reachedBehavior to true.`,
+            });
+        }
+    }
+    else if (cause.kind === FailureCauseKind.Environment &&
+        row.reachedBehavior === true) {
+        // test_defect is not checked: a wrong assertion can fail after the test
+        // reached the behaviour, or a bad setup step can fail before it.
+        ctx.addIssue({
+            code: "custom",
+            path: ["failureCause", "kind"],
+            message: `failureCause.kind 'environment' contradicts reachedBehavior true. reachedBehavior true means the test reached the behaviour under test, so the environment did not stop it. ` +
+                `Set kind to 'application' or 'test_defect', or set reachedBehavior to false if the failure came earlier.`,
+        });
+    }
 });
 // Structured grounding for UI recommendations. Lifted verbatim from a
 // `browser_blueprint` element captured during this run. Source of truth for
@@ -388,8 +486,25 @@ const issueFoundSchema = z
         .describe("1-based line number in sourceFile. Optional and advisory. " +
         "Give the line of the offending statement itself, not the line of the enclosing function's signature — review views place the objection on exactly this line. " +
         "The tool does not check that the line still holds the cited code, because line numbers drift."),
+    environmentFamily: environmentFamilyField
+        .optional()
+        .describe("Only for category 'config'. Not allowed for the other categories. " +
+        "When this config issue is a test-environment gap, name the gap. Omit it for a config issue that is not a test-environment gap. " +
+        ENVIRONMENT_FAMILY_MEANINGS),
 })
     .superRefine((issue, ctx) => {
+    // SKYR-4555: the family names a test-environment gap, which only a config
+    // entry reports. On a bug, lint or type entry it would put the issue in two
+    // places at once.
+    if (issue.environmentFamily !== undefined &&
+        issue.category !== IssueFoundCategory.Config) {
+        ctx.addIssue({
+            code: "custom",
+            path: ["environmentFamily"],
+            message: `environmentFamily is allowed only when category is 'config'. It names the test-environment gap behind a config issue. ` +
+                `Remove environmentFamily from this '${issue.category}' entry, or set category to 'config' if the issue is a test-environment gap.`,
+        });
+    }
     // A `bug` entry asserts that product code is wrong, so it has to name the code:
     // without a file the entry is prose a reviewer cannot act on, and nothing
     // downstream can place it in the diff. Only the file is required — a config file
@@ -2750,7 +2865,10 @@ export function registerSubmitReportTool(server) {
             // these objects and don't expect it).
             testResults: params.testResults.map((row) => {
                 const { testFilePath: _tfp, ...wire } = attachVideoPath(row, stateData.executionVideos);
-                return wire;
+                // SKYR-4555: the basename is the join key to newTestsCreated and
+                // testMaintenance (consumers compare it with the basename of their
+                // fileName, which may be a relative path); the path stays internal.
+                return { ...wire, fileName: path.basename(row.testFilePath) };
             }),
             issuesFound,
             executionCoverage,

package/build/types/TestbotReport.d.ts

@@ -13,6 +13,26 @@ export declare enum IssueFoundCategory {
     Type = "type",
     Config = "config"
 }
+/** Why a Fail or Skipped testResults row did not pass. Application is the code
+ *  under test (or a test written to prove a defect, failing as intended);
+ *  Environment is the test environment stopping the test before it reached the
+ *  behaviour; TestDefect is the generated test itself being wrong. */
+export declare enum FailureCauseKind {
+    Application = "application",
+    Environment = "environment",
+    TestDefect = "test_defect"
+}
+/** Which test-environment gap stopped a test: the `family` of an Environment
+ *  failureCause, and the `environmentFamily` of a config issuesFound entry. */
+export declare enum EnvironmentFamily {
+    AuthToken = "auth_token",
+    Secret = "secret",
+    DataConnection = "data_connection",
+    AppNotStarted = "app_not_started",
+    Login = "login",
+    ExternalDependency = "external_dependency",
+    RecordingReplay = "recording_replay"
+}
 /**
  * Shape of the JSON report written by skyramp_submit_report and read by testbot
  * for rendering as Markdown. All fields mirror the corresponding Zod schemas in
@@ -104,6 +124,19 @@ export interface TestbotReport {
         endpoint: string;
         status: "Pass" | "Fail" | "Skipped";
         details: string;
+        /** Server-derived: the basename of the `testFilePath` the agent submitted,
+         *  which never reaches the report. The join key to `newTestsCreated` and
+         *  `testMaintenance`: match it against the basename of their `fileName`
+         *  (a `newTestsCreated` row may carry a repository-relative path), plus
+         *  `repository`. */
+        fileName: string;
+        /** Why a Fail or Skipped row did not pass. The submit schema requires it on
+         *  those rows and refuses it on a Pass row; `family` is set exactly when
+         *  `kind` is Environment. Absent in reports from MCP versions predating it. */
+        failureCause?: {
+            kind: FailureCauseKind;
+            family?: EnvironmentFamily;
+        };
         videoPath?: string;
         /** See newTestsCreated[].repository. */
         repository: string;
@@ -158,6 +191,10 @@ export interface TestbotReport {
         sourceFile?: string;
         sourceSymbol?: string;
         sourceLine?: number;
+        /** On a `config` entry that is a test-environment gap, which gap. The submit
+         *  schema refuses it on any other category. Optional: a config entry need
+         *  not be an environment gap, and older MCP versions never set it. */
+        environmentFamily?: EnvironmentFamily;
         /** See newTestsCreated[].repository. */
         repository: string;
     }[];

package/build/types/TestbotReport.js

@@ -8,3 +8,25 @@ export var IssueFoundCategory;
     IssueFoundCategory["Type"] = "type";
     IssueFoundCategory["Config"] = "config";
 })(IssueFoundCategory || (IssueFoundCategory = {}));
+/** Why a Fail or Skipped testResults row did not pass. Application is the code
+ *  under test (or a test written to prove a defect, failing as intended);
+ *  Environment is the test environment stopping the test before it reached the
+ *  behaviour; TestDefect is the generated test itself being wrong. */
+export var FailureCauseKind;
+(function (FailureCauseKind) {
+    FailureCauseKind["Application"] = "application";
+    FailureCauseKind["Environment"] = "environment";
+    FailureCauseKind["TestDefect"] = "test_defect";
+})(FailureCauseKind || (FailureCauseKind = {}));
+/** Which test-environment gap stopped a test: the `family` of an Environment
+ *  failureCause, and the `environmentFamily` of a config issuesFound entry. */
+export var EnvironmentFamily;
+(function (EnvironmentFamily) {
+    EnvironmentFamily["AuthToken"] = "auth_token";
+    EnvironmentFamily["Secret"] = "secret";
+    EnvironmentFamily["DataConnection"] = "data_connection";
+    EnvironmentFamily["AppNotStarted"] = "app_not_started";
+    EnvironmentFamily["Login"] = "login";
+    EnvironmentFamily["ExternalDependency"] = "external_dependency";
+    EnvironmentFamily["RecordingReplay"] = "recording_replay";
+})(EnvironmentFamily || (EnvironmentFamily = {}));

package/build/types/index.d.ts

@@ -2,7 +2,7 @@ export { TestExecutionStatus } from "./TestExecution.js";
 export { DriftAction } from "./TestAnalysis.js";
 export { TestType, HttpMethod } from "./TestTypes.js";
 export type { TestbotReport } from "./TestbotReport.js";
-export { IssueFoundCategory } from "./TestbotReport.js";
+export { IssueFoundCategory, FailureCauseKind, EnvironmentFamily, } from "./TestbotReport.js";
 export { ReuseDeclinedBy, ReuseVerificationOutcome, HelperVerificationOutcome, } from "./ReuseOutcome.js";
 export type { ReuseOutcome, ReuseSkippedEntry, ReuseMissedEntry, HelperReuseOutcome, RetrofitOutcome, ChangedHelperEntry, ChangedHelperCaller, OrphanedHelper, } from "./ReuseOutcome.js";
 export type { RelatedRepository, TestbotPromptOptions, } from "./TestbotPromptOptions.js";

package/build/types/index.js

@@ -1,5 +1,5 @@
 export { TestExecutionStatus } from "./TestExecution.js";
 export { DriftAction } from "./TestAnalysis.js";
 export { TestType, HttpMethod } from "./TestTypes.js";
-export { IssueFoundCategory } from "./TestbotReport.js";
+export { IssueFoundCategory, FailureCauseKind, EnvironmentFamily, } from "./TestbotReport.js";
 export { ReuseDeclinedBy, ReuseVerificationOutcome, HelperVerificationOutcome, } from "./ReuseOutcome.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.4.4-rc.2",
+  "version": "0.4.4",
   "main": "build/index.js",
   "exports": {
     ".": "./build/index.js",

package/plugin/prompts/testbot-task1.md

@@ -65,7 +65,7 @@ Log in once via the credentials in your <ui-credentials> context before you navi
    **One state file, per-repo sections.** There is exactly ONE state file for the whole run (created automatically and reused across calls). Each `skyramp_analyze_changes` call writes its repo's section into it, keyed by `repository`. The run declares its primary when it starts, so the root section is the primary's whichever call runs first. When you maintain or act on a related repo's tests, pass that repo's `repository` value on the call, exactly as step {analyzeStep} says to pass `{primaryRepo}` for the primary repo's tests.
 
    **Cross-repo test generation ({generateTask} spans all repos).** Draft your planned tests from the analysis of the primary AND every related repo together, and register them as ONE plan. When you generate a test for a related repo's endpoint:
-   - **Execute it only if that repo's service is already running and reachable.** The workflow's setup may have started multiple services; before generating an API test for a related repo, confirm its `base_url` (from that repo's workspace/Execution Plan) responds. If the service is unreachable, still GENERATE the test but mark its `testResults` status as `Skipped` with details "service not running in this run" — do NOT count an unreachable service as a failure.
+   - **Execute it only if that repo's service is already running and reachable.** The workflow's setup may have started multiple services; before generating an API test for a related repo, confirm its `base_url` (from that repo's workspace/Execution Plan) responds. If the service is unreachable, still GENERATE the test but mark its `testResults` status as `Skipped` with details "service not running in this run" and `failureCause` `{ "kind": "environment", "family": "app_not_started" }` — do NOT count an unreachable service as a failure.
    - **Write the test file into that service's own `testDirectory`** — the one declared for the service in the unified workspace.yml (the related repo's services were registered there in {registerServicesRef}, each with its `repository`). The `testDirectory` is interpreted relative to the **single delivery root** (the configured test repo if set, otherwise the primary repo), so all generated tests are delivered together by the existing single-target delivery. Do NOT invent a per-source-repo subdirectory, and do NOT write into the related repo's own checkout — it is read-only context. (If two repos happen to declare the same `testDirectory`, their files coexist there; the `repository` field on each report item — below — is what attributes ownership, not the path.)
    - **Set the `repository` field** (`owner/repo`) on every such `newTestsCreated` / `testResults` item so the report attributes it to the originating repo (see Report Guidelines).