@skyramp/mcp 0.4.1 → 0.4.2-rc.2

package/build/types/TestTypes.js

@@ -9,6 +9,12 @@ export var ProgrammingLanguage;
     ProgrammingLanguage["JAVASCRIPT"] = "javascript";
     ProgrammingLanguage["JAVA"] = "java";
 })(ProgrammingLanguage || (ProgrammingLanguage = {}));
+/** The languages skyramp_execute_test accepts, as prose for a prompt. Rendered from
+ *  the enum so the text and the gate that reads it cannot disagree. */
+export function runnableLanguagesText() {
+    const names = Object.values(ProgrammingLanguage);
+    return `${names.slice(0, -1).join(", ")} or ${names[names.length - 1]}`;
+}
 export var TestType;
 (function (TestType) {
     TestType["SMOKE"] = "smoke";
@@ -21,6 +27,16 @@ export var TestType;
     TestType["UNIT"] = "unit";
     TestType["UNKNOWN"] = "unknown";
 })(TestType || (TestType = {}));
+/** The four types the testbot can plan and deliver. `TestType`'s others are
+ *  maintenance outcomes or report-only labels, so a planned test can never carry
+ *  one. The plan schema, the stored plan's type and the report's join all read
+ *  THIS list. */
+export const TESTBOT_SUPPORTED_TEST_TYPES = [
+    TestType.CONTRACT,
+    TestType.INTEGRATION,
+    TestType.E2E,
+    TestType.UI,
+];
 /**
  * Mock is not a test type — mock files are deployed via apply_mock(),
  * not executed as tests. Kept as a standalone constant for mock generation.
@@ -123,9 +139,10 @@ export const baseSchema = z.object({
     }, {
         message: "Output file must have one of these extensions: .py, .ts, .js, .java",
     })
-        .describe("Name of the output test file. For Playwright framework, the file must match " +
-        "*.spec.ts so Playwright can discover it " +
-        "(e.g. contract_math_add.spec.ts, fuzz_orders.spec.ts). " +
+        .describe("Name of the output test file. For Playwright, name the file the way the " +
+        "repository names its tests: read the test files already in outputDir and " +
+        "match their spelling. Use *.test.ts where they use it, *.spec.ts otherwise " +
+        "(e.g. contract_math_add.spec.ts, fuzz_orders.test.ts). " +
         "For other frameworks (pytest, junit), standard extensions are fine (.py, .java). " +
         "Optional — when omitted, Skyramp generates a default name."),
     outputDir: z
@@ -274,6 +291,26 @@ export const codeRefactoringSchema = z.object({
         .default(false)
         .describe("Whether to modularize the code"),
 });
+/** The `testName` field for a recording-driven generation tool.
+ *
+ *  Single-sourced because the description is prompt text: it is the only thing
+ *  steering the agent's choice of title, it gets re-tuned against eval results,
+ *  and two copies would let the UI and E2E tools drift apart silently. Each
+ *  tool passes the title its own generator falls back to (SKYR-4468). */
+export const testNameField = (fallbackTitle) => z
+    .string()
+    // .trim() normalises incidental whitespace and, combined with .min(1),
+    // rejects a whitespace-only title, which would otherwise produce a
+    // meaningless test title and identifier (SKYR-4468).
+    .trim()
+    .min(1)
+    .optional()
+    .describe("Title for the generated test. Name the BEHAVIOUR this recording verifies, " +
+    "taken from the recommendation the recording came from — not the page it happens on. " +
+    'Example: "Add to cart from search results", not "Search page". ' +
+    "Skyramp emits it verbatim as the Playwright test title for TypeScript/JavaScript, " +
+    "and as a snake_case `def test_<name>` identifier for Python. " +
+    `Omit it and the test stays titled \`${fallbackTitle}\`, which claims nothing.`);
 /** Test types skyramp_enhance_assertions supports — single-sourced so the
  *  tool's schema enum and generation-time baseline seeding cannot drift
  *  (contract additionally requires provider mode at the seeding site). */

package/build/types/TestbotPromptOptions.d.ts

@@ -24,7 +24,15 @@ export interface TestbotPromptOptions {
     uiCredentials?: string;
     testsRepoDir?: string;
     relatedRepositories?: RelatedRepository[];
-    primaryRepo?: string;
+    /** `owner/repo` of the PRIMARY checkout, resolved by the action off the same
+     *  origin remote it reads for every sibling in `relatedRepositories`. Passed
+     *  rather than re-read here: the action already has it. */
+    primaryRepo: string;
     planOnly?: boolean;
     language?: string;
+    /** Per-file cap on `skyramp_execute_test` calls in the final phase — the first
+     *  run plus the fix-and-rerun iterations (SKYR-4460). Unset, out of range or
+     *  not a whole number falls back to SKYRAMP_MAX_FIX_ATTEMPTS, then the default
+     *  in utils/fixAttempts.ts. */
+    maxFixAttempts?: number;
 }

package/build/types/TestbotReport.d.ts

@@ -16,9 +16,14 @@ export declare enum IssueFoundCategory {
 /**
  * Shape of the JSON report written by skyramp_submit_report and read by testbot
  * for rendering as Markdown. All fields mirror the corresponding Zod schemas in
- * submitReportTool.ts — keep the two in sync. This type describes what a READER
- * may encounter across MCP versions; the Zod schemas define what the current
- * producer must submit, so a field can be required on submit but optional here.
+ * submitReportTool.ts — keep the two in sync.
+ *
+ * Required here means required on submit. Testbot's renderer extends this type,
+ * and a pinned older mcp could hand it a report missing a field this one calls
+ * required — but the common skew runs the other way (the version input defaults
+ * to `latest`, so a NEW mcp meets an older renderer, which simply ignores what
+ * it does not read). An optional field would have to be a real one the producer
+ * may omit, not a hedge against that pin.
  */
 export interface TestbotReport {
     businessCaseAnalysis: string;
@@ -34,9 +39,9 @@ export interface TestbotReport {
         fileName: string;
         reasoning: string;
         description: string;
-        /** `owner/repo` attribution in multi-repo runs (SKYR-3786). Absent = the
-         *  primary repo, or a single-repo run. */
-        repository?: string;
+        /** `owner/repo` this item pertains to. Set on every item, in single-repo
+         *  runs too. */
+        repository: string;
         scenarioFile?: string;
         traceFile?: string;
         frontendTrace?: string;
@@ -80,10 +85,19 @@ export interface TestbotReport {
         beforeDetails: string;
         afterStatus: TestExecutionStatus;
         afterDetails: string;
+        /** The plan defect ids this test's own run proves, as the agent recorded them
+         *  after executing it. Empty on a row whose run proved none. A defect named
+         *  here may also be proved by a delivered test: the two are recorded
+         *  independently, and naming one never stands in for the other. */
+        defects: string[];
         /** See newTestsCreated[].assertions — maintenance rows are matched by
          *  exact canonical path, so this carries the maintenance honesty labels
          *  (nothing-to-verify vs verified) into the report. */
         assertions?: AssertionOutcome;
+        /** See newTestsCreated[].reuse — an updated or regenerated test goes
+         *  through the same modularize → reuse chain, so its row carries the same
+         *  server-derived outcome. Absent in reports from MCP versions predating it. */
+        reuse?: ReuseOutcome;
     }[];
     testResults: {
         testType: TestType;
@@ -92,7 +106,7 @@ export interface TestbotReport {
         details: string;
         videoPath?: string;
         /** See newTestsCreated[].repository. */
-        repository?: string;
+        repository: string;
     }[];
     additionalRecommendations?: {
         testId: string;
@@ -119,7 +133,7 @@ export interface TestbotReport {
         backendTrace?: string;
         frontendTrace?: string;
         /** See newTestsCreated[].repository. */
-        repository?: string;
+        repository: string;
     }[];
     issuesFound: {
         description: string;
@@ -145,7 +159,7 @@ export interface TestbotReport {
         sourceSymbol?: string;
         sourceLine?: number;
         /** See newTestsCreated[].repository. */
-        repository?: string;
+        repository: string;
     }[];
     nextSteps: string[];
     /** Recommendation v2 only. Every change the plan declared, with the tests
@@ -196,5 +210,27 @@ export interface TestbotReport {
          *  objection still raised, answered or not — the two are different outcomes. */
         resolvedBy?: string;
     }[];
+    /** How many test files this run wrote, and how many of them this server ran.
+     *  `generated + updated` equals `executed` when every file was run; a
+     *  shortfall is the same shortfall the `testRanOnThisServer` objections name
+     *  one file at a time.
+     *
+     *  Every number is per FILE and comes from the server's own records, not from
+     *  the arrays beside it: the generators record what they wrote and
+     *  skyramp_execute_test records what it ran, both keyed by canonical path. So
+     *  these do not equal the array lengths — `testMaintenance` also carries rows
+     *  that were never edited (VERIFY, IGNORE) or whose file is gone (DELETE), and
+     *  `testResults` is the agent's own claim and can hold several rows per file. */
+    executionCoverage: {
+        /** Files a generation tool wrote. */
+        generated: number;
+        /** Files an UPDATE or REGENERATE verdict edited, minus any already counted as
+         *  generated — one file owes one run however it was written. */
+        updated: number;
+        /** How many of those `generated + updated` files were passed to
+         *  skyramp_execute_test. The file reached the runner; that is not proof a test
+         *  ran, because a runner can exit 0 having executed nothing. */
+        executed: number;
+    };
     commitMessage: string;
 }

package/build/utils/AnalysisStateManager.d.ts

@@ -1,11 +1,11 @@
 import type { ReuseRecord } from "../tools/code-refactor/reuse-outcome.js";
 import type { AssertionRecord } from "../types/AssertionOutcome.js";
+import type { TestType } from "../types/TestTypes.js";
 import { TestAnalysisResult, MaintenanceActionCore } from "../types/TestAnalysis.js";
 import { RepositoryAnalysis, AnalysisScope } from "../types/RepositoryAnalysis.js";
 import { PRTestContext } from "./pr-comment-parser.js";
 import type { RemovedUiElement } from "./removedUiElements.js";
-import type { ExternalTestRunRecord } from "../types/ExternalTestExecution.js";
-import type { VideoRecord } from "../types/TestExecution.js";
+import type { TestExecutionRecord, VideoRecord } from "../types/TestExecution.js";
 import type { RepoCheckout } from "./reportVerification.js";
 import type { Plan } from "../recommendation/registerPlan.js";
 import type { Objection } from "../recommendation/types.js";
@@ -21,6 +21,24 @@ export declare function setTestsRepoDir(dir: string | undefined): void;
  */
 export declare function rememberTestsRepoDir(dir: string | undefined): void;
 export declare function getTestsRepoDir(): string | undefined;
+export declare function rememberPrimaryRepository(repositoryPath: string, repo: string): void;
+/**
+ * Test seam, and the declaration the prompt render makes at the start of a run.
+ *
+ * A render is a run boundary, so it drops the in-memory state-file anchor as well:
+ * a server that renders a second prompt is serving a second run, and the first
+ * run's file must not take the second run's analysis. The pointer file stays, since
+ * it lives in the job's own `RUNNER_TEMP` — a later STAGE of the same run renders
+ * again in a fresh process and must still find that run's file.
+ */
+export declare function setPrimaryRepository(primary: {
+    repositoryPath: string;
+    repo: string;
+} | undefined): void;
+export declare function getPrimaryRepository(): {
+    repositoryPath: string;
+    repo: string;
+} | undefined;
 /** Filename of the run-scoped analysis state file under `runArtifactDir()`.
  *  Single-sourced so the constructor that WRITES there and `resolveRunStatePath`
  *  which LOOKS there cannot drift apart. */
@@ -53,18 +71,37 @@ export declare function clearActiveRunStatePath(): void;
  * The run's state-file path, for tools that need to read or amend it without the
  * agent naming it.
  *
- * `setActiveRunStatePath` is only called by skyramp_analyze_changes on multi-repo
- * runs (the ones carrying a `repository` param). It is read back two ways: from
- * memory in this process, and from the pointer file a previous stage of the same
- * run wrote — a staged run starts a fresh server per stage, so memory alone would
- * send later stages to the wrong file.
+ * `setActiveRunStatePath` is called by skyramp_analyze_changes on every run. It is
+ * read back two ways: from memory in this process, and from the pointer file a
+ * previous stage of the same run wrote — a staged run starts a fresh server per
+ * stage, so memory alone would send later stages to the wrong file.
  *
- * A single-repo Testbot run's path is still deterministic in CI: the StateManager
- * constructor above roots "analysis" state under `runArtifactDir()`. Falling back
- * to that covers single-repo runs. Standalone/IDE usage has no RUNNER_TEMP and
- * resolves to undefined — correct, since there is no run state.
+ * The path is deterministic in CI either way: the StateManager constructor above
+ * roots "analysis" state under `runArtifactDir()`, so a run whose anchor was cleared
+ * at a stage boundary resolves to the same file. Standalone/IDE usage has no
+ * RUNNER_TEMP and resolves to undefined — correct, since there is no run state.
  */
 export declare function resolveRunStatePath(explicitPath?: string): string | undefined;
+/**
+ * The state file a tool may act on, or a refusal saying why not.
+ *
+ * A run's state file holds THIS run's analysis and THIS run's plan. A path from
+ * another run carries another run's, and every check would then judge this diff
+ * against it — silently, because both files are valid state files. So a named path
+ * is accepted only when it names the file this run created.
+ *
+ * Gated on `currentRunStateFile`, never on `resolveRunStatePath`: that one builds a
+ * path out of `RUNNER_TEMP`, which GitHub Actions sets on every job including this
+ * repo's own CI, so it answers "where would the file be" and not "is there a run".
+ * Refusing on it would refuse a named path wherever that variable merely exists.
+ *
+ * With no run of its own — standalone use, an IDE, a test — the named path is the
+ * only one there is, and is taken as given.
+ */
+export declare function resolveOwnRunStatePath(explicitPath?: string): {
+    path?: string;
+    refusal?: string;
+};
 /**
  * The run's state file if this process is actually serving a run, else undefined.
  *
@@ -144,7 +181,13 @@ export interface ReuseHandOff {
     /** The framework the hand-off named, so a refusal can hand back a call the
      *  reuse tool's schema accepts (`framework` is required there). */
     framework: string;
+    /** Who recorded it. `generation` names a file this run wrote; `maintenance`
+     *  names a pre-existing test skyramp_actions told the agent to update or
+     *  regenerate, which is at HEAD by definition. Optional only so a reader can
+     *  state its own default: every recorder sets it. */
+    source?: ReuseHandOffSource;
 }
+export type ReuseHandOffSource = "generation" | "maintenance";
 /**
  * Unified state data combining test discovery + endpoint scanning
  * Used by the unified test-management workflow
@@ -165,12 +208,6 @@ export interface UnifiedAnalysisState {
     /** Maintenance actions applied via skyramp_actions — skyramp_submit_report derives
      *  testMaintenance from this. */
     maintenanceVerdicts?: MaintenanceActionCore[];
-    /**
-     * External (repo-owned) test runs recorded by skyramp_run_existing_tests, one
-     * record per confirm/verify invocation. Drift analysis and the report fold in
-     * CONFIRMED failures from here instead of guessing. Absent until the tool runs.
-     */
-    externalTestResults?: ExternalTestRunRecord[];
     /** POM code-reuse outcome per generated UI spec, keyed by test-file BASENAME
      *  (what `newTestsCreated[].fileName` carries). Written by skyramp_reuse_code,
      *  which computes every value in-process; skyramp_submit_report merges it into
@@ -190,6 +227,20 @@ export interface UnifiedAnalysisState {
      *  the report is accepted with the fault recorded in the row. Written by
      *  skyramp_submit_report only; never supplied by the LLM. */
     reuseRefusals?: Record<string, number>;
+    /** Declines skyramp_submit_report removed from a delivered file, keyed by the file's
+     *  ABSOLUTE PATH. The customer must not receive a comment addressed to one of our
+     *  gates, but the run still has to know what it declined: without this, the next
+     *  re-derivation refuses a fault the run already declined. The record is READ by a
+     *  later call, never written back into the file — editing the customer's file a
+     *  second time is what made a reworded or reformatted decline defeat the match. It
+     *  does not outlive the run, which is the point: a later run judges the file it is
+     *  given. Written by skyramp_submit_report only; never supplied by the LLM. */
+    strippedDeclines?: Record<string, {
+        kind?: string;
+        helper?: string;
+        keptInline?: string;
+        reason?: string;
+    }[]>;
     /**
      * SKYR-4156. Recorded video per executed browser test, keyed by test-file
      * BASENAME (the same key `reuseOutcomes` uses, so matching needs no path
@@ -204,6 +255,17 @@ export interface UnifiedAnalysisState {
      * the earlier attempt's now-stale directory.
      */
     executionVideos?: Record<string, VideoRecord>;
+    /** The latest skyramp_execute_test run of each file, keyed by canonical test
+     *  path — every file, not only maintained ones. Read by the post-execution
+     *  assertion-alignment step; a re-run replaces the file's record. */
+    testExecutions?: Record<string, TestExecutionRecord>;
+    /** The fix-and-rerun attempt cap this run enforces (SKYR-4460), pinned by the
+     *  first skyramp_execute_test call from the value the testbot prompt recorded.
+     *  Run-wide, so it lives at the ROOT whichever repository's section that call
+     *  writes, and is carried through later root rewrites. Once written it wins
+     *  over any later prompt render or environment change, so the tool enforces
+     *  the number the run's prompt stated for every file in the run. */
+    maxFixAttempts?: number;
     /** The registered plan with its answered and open objections. Run-wide, so it
      *  lives at the ROOT whichever repository the registration names, and the newest
      *  registration REPLACES it. No fallback: no registration means no plan.
@@ -246,6 +308,24 @@ export interface UnifiedAnalysisState {
      *  skyramp_execute_test verifies against it server-side before executing.
      *  Never supplied by the LLM. */
     assertionOutcomes?: Record<string, AssertionRecord>;
+    /** Every test file a generation tool wrote in this run, keyed by CANONICAL
+     *  ABSOLUTE PATH — the same key `testExecutions` uses, so asking whether a
+     *  generated file was run is a lookup, not a path match. Written in-process by
+     *  the generators, never supplied by the LLM.
+     *
+     *  The report otherwise has only the agent's `fileName`, which is a basename,
+     *  so two specs of one name in different directories answer for each other.
+     *  Separate from `assertionOutcomes`, which is also per-file but is written
+     *  only when assertion enhancement applies to the test type. */
+    generatedTests?: Record<string, GeneratedTestRecord>;
+}
+/** One file a generation tool wrote. The path is the key; this is what is known
+ *  about it at generation time. */
+export interface GeneratedTestRecord {
+    testType: TestType;
+    /** Epoch milliseconds, as `Date.now()` returns them. A number so the report
+     *  can compare it with a run's time directly; format it only to print it. */
+    generatedAt: number;
 }
 /**
  * State file metadata
@@ -255,9 +335,8 @@ interface StateMetadata {
     stateType: StateType;
     repositoryPath?: string;
     /** owner/repo of the PRIMARY repo, recorded on the root so related-repo writes
-     *  can be distinguished from the primary. Absent only for legacy single-repo
-     *  state files written before explicit tagging. */
-    repository?: string;
+     *  can be distinguished from the primary. Every caller supplies it. */
+    repository: string;
     createdAt: string;
     updatedAt: string;
     step?: string;
@@ -267,7 +346,7 @@ interface StateMetadata {
  * The primary repo's analysis stays at the root for backward compatibility; each
  * related repo's analysis is stored here keyed by its `owner/repo`.
  */
-interface RepoStateSection<T> {
+export interface RepoStateSection<T> {
     /** Analysis payload — same shape as the root (e.g. UnifiedAnalysisState). */
     data: T;
     /** Absolute checkout path of this repo (used to re-absolutize test paths). */
@@ -319,19 +398,27 @@ export declare class StateManager<T = any> {
      * Write data to state file
      * Data fields are spread at root level alongside metadata
      */
-    writeData(data: T, options?: {
+    writeData(data: T, options: {
         repositoryPath?: string;
         step?: string;
         /** owner/repo recorded on the root metadata — identifies the PRIMARY repo
-         *  so later section routing (isPrimaryRepo) can compare against it.
-         *  Preserved when omitted. */
-        repository?: string;
+         *  so later section routing (isPrimaryRepo) can compare against it. */
+        repository: string;
+    }): Promise<void>;
+    /**
+     * Append to the root payload of a state file the run has already created.
+     * Takes no repository: the name the first write recorded stays, and the
+     * metadata it wrote is what these callers read back before patching it.
+     */
+    appendData(data: T, options?: {
+        repositoryPath?: string;
+        step?: string;
     }): Promise<void>;
-    /** Is `repo` the primary section? By NAME when the caller declares one: the run
-     *  says which repo is primary, so whichever call lands first cannot decide it.
-     *  Otherwise the legacy rule — falsy `repo` (single-repo callers), the first
-     *  write of the run, or a `repo` matching the root metadata (SKYR-4333: a
-     *  docs-only primary diff let the related repo write first and take the root). */
+    private persist;
+    /** Is `repo` the primary section? By NAME, against the name the run declared
+     *  for its primary checkout — so whichever call lands first cannot decide it
+     *  (SKYR-4333: a docs-only primary diff let the related repo write first and
+     *  take the root). */
     private isPrimaryRepo;
     /**
      * Write one repo's analysis into the run-scoped state file. The primary repo
@@ -346,13 +433,29 @@ export declare class StateManager<T = any> {
      *  did so deleted the plan the agent had registered. NAMED rather than "keep
      *  everything omitted", which would resurrect fields a re-analysis clears. */
     writeRepoData(data: T, options: {
-        repo?: string;
+        repo: string;
+        repositoryPath?: string;
+        step?: string;
+        /** The run's declared primary, resolved by the caller from the checkout
+         *  the run named at its entry point. Passed on EVERY call, so a related
+         *  repo arriving first writes a section and never the root. */
+        primary: {
+            repositoryPath: string;
+            repo: string;
+        };
+    }): Promise<void>;
+    /**
+     * Update one repo's section of a state file the run has already created.
+     * Takes no primary: the root records the primary's name, and these callers
+     * (execute, run-existing-tests, actions) read the section back before they
+     * patch it, so the run is already under way.
+     */
+    updateRepoData(data: T, options: {
+        repo: string;
         repositoryPath?: string;
         step?: string;
-        /** The run's declared primary, passed on EVERY call of a multi-repo run.
-         *  Given it, write order decides nothing. */
-        primaryRepo?: string;
     }): Promise<void>;
+    private route;
     /**
      * Read one repo's analysis payload from the run-scoped state file. `repo`
      * undefined → the primary (root) section (identical to readData()).