@gfdelarue/testkit 0.3.7 → 0.4.1

package/README.md

@@ -49,6 +49,131 @@ Testkit writes the immutable, verifiable run bundle under
 `.testkit/results/runs/<run-id>/`. Upload `.testkit/results` with `if: always()`
 when CI evidence should survive a failed job.
 
+## Report a run to GitHub or Gitea
+
+`testkit ci report` reads the last run in the checkout and posts it in three
+places: one commit status, the job's step summary, and one pull request comment
+that it updates in place.
+
+```yaml
+permissions:
+  contents: read
+  issues: read
+  pull-requests: write
+  statuses: write
+
+steps:
+  - run: npm ci
+  - run: npm run testkit
+  - run: npx testkit ci report
+    if: always()
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
+```
+
+- The `testkit run` step decides whether the job fails. `ci report` exits zero
+  when it posts the report, and that includes the report of a failed run.
+- GitHub Actions passes the job's token to a `run` step only when the step sets
+  it, as the `env` lines above do. Gitea Actions accepts the same lines.
+- Testkit reads the token from `GITHUB_TOKEN` or `GH_TOKEN` for GitHub, and
+  from `GITEA_TOKEN` for Gitea. In a Gitea Actions job it also reads
+  `GITHUB_TOKEN`. With no token set, it asks `git credential fill`. The token
+  that `actions/checkout` keeps in the checkout does not answer that request.
+- Testkit reaches GitHub Enterprise Server only from a GitHub Actions job on
+  that server, so `testkit bug file` works there only in a job.
+- GitHub gives a pull request from a fork a read-only token. `ci report` then
+  writes the step summary and fails when it posts the status. Add
+  `continue-on-error: true` to the step when the repository takes pull requests
+  from forks.
+- When the checkout holds no results for its commit, `ci report` posts a
+  `failure` status with the description
+  `run failed: Testkit found no results for this commit`.
+- A job that runs Testkit more than once passes `--context <name>` to each
+  report, so each run keeps its own status and its own comment.
+
+## File a bug as an issue
+
+Testkit files a confirmed bug as an issue in the repository's GitHub or Gitea.
+The issue names the regression test that fails because of the bug, the commit
+that introduced the faulty lines, and the command that reproduces the failure.
+[Bug records](docs/bug-records.md) gives the issue format.
+
+1. Write a Testkit test that fails because of the bug.
+2. Write the bug record as a JSON file. `testkit bug guide` prints the fields
+   and the cause, impact, and escape lists.
+3. Run `testkit bug record bug.json`. It checks the record, stores a draft under
+   `.testkit/bugs/`, and prints the draft id.
+4. Commit the test and push the commit.
+5. Run `testkit bug file <id>`.
+
+`testkit bug file` runs the test at the checked-out commit and files the issue
+only when the test fails. It stops when tracked files have uncommitted changes,
+when no remote branch contains the commit, or when an open issue already covers
+the test. The token needs permission to create issues and labels.
+
+### Name the product's areas
+
+Add `areas` to `testkit.config.ts` so that each issue names the part of the
+product that holds the fault:
+
+```ts
+export default defineConfig({
+  areas: {
+    checkout: { name: "Checkout", paths: ["src/checkout/**"] },
+    billing: { name: "Billing", paths: ["src/billing/**", "src/invoices/**"] },
+  },
+});
+```
+
+`testkit bug file` puts an `area:<id>` label on the issue. If no area claims
+the fault's file, it stops and names the file. Without `areas`, Testkit files
+bugs with no area. See [`areas`](testkit-config-schema.md#areas).
+
+### Pass a run whose failures are all filed
+
+Every failing test fails the run by default. To keep CI passing while a filed
+bug waits for its fix, set `bugs.knownFailures`:
+
+```ts
+export default defineConfig({
+  bugs: { knownFailures: "allow" },
+});
+```
+
+A run then passes when an open issue with the `testkit` label covers each
+failure. A new failure still fails the run, and closing an issue makes its
+test count again. When Testkit cannot read the issues, every failure is new.
+[Known failures](docs/bug-records.md#known-failures) gives the exact rule.
+
+With `"allow"`, the run reads the open issues. In CI, pass the job's token to
+the `testkit run` step as well:
+
+```yaml
+steps:
+  - run: npm run testkit
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
+```
+
+The job also needs the `issues: read` permission.
+
+### Run a campaign
+
+A campaign gives a coding agent a target and the instructions to file what it
+finds.
+
+```bash
+npx testkit campaign start --target "Refunds and partial refunds"
+npx testkit campaign status
+npx testkit campaign finish
+```
+
+`campaign start` prints the agent's brief: the target, the files that no area
+claims, how to work, and the bug record format. Give that output to the agent.
+`campaign status` lists the bugs recorded so far. `campaign finish` writes the
+summary for the pull request that adds the tests to
+`.testkit/campaigns/<id>/summary.md` and prints it.
+
 ## Execution engines
 
 Testkit has three explicit built-in engines rather than one implicit runtime:

package/dist/app/bugs/areas.d.ts

@@ -0,0 +1,38 @@
+import type { AreaDefinition } from "../../config/loader/bugs.js";
+import type { CommitFile } from "../../repo/tree-listing.js";
+import type { BugFactsArea } from "./facts.js";
+export interface AreaCoverage {
+    readonly bytes: number;
+    readonly files: number;
+    readonly id: string;
+    readonly name: string;
+    readonly share: number;
+}
+export interface DoubleClaim {
+    readonly areas: readonly string[];
+    readonly path: string;
+}
+export interface AreaMapCoverage {
+    readonly areas: readonly AreaCoverage[];
+    readonly doubleClaimedShare: number;
+    readonly doubleClaims: readonly DoubleClaim[];
+    readonly sourceBytes: number;
+    readonly unclaimed: readonly CommitFile[];
+    readonly unclaimedShare: number;
+}
+export type FaultAreaResolution = {
+    readonly area: BugFactsArea;
+    readonly status: "resolved";
+} | {
+    readonly areas: readonly string[];
+    readonly status: "double-claimed";
+} | {
+    readonly status: "no-map";
+} | {
+    readonly status: "unclaimed";
+};
+export type AreaMatcher = (path: string) => readonly AreaDefinition[];
+export declare function isSourceFile(path: string): boolean;
+export declare function createAreaMatcher(areas: readonly AreaDefinition[]): AreaMatcher;
+export declare function measureAreaMap(areas: readonly AreaDefinition[], files: readonly CommitFile[]): AreaMapCoverage;
+export declare function resolveFaultArea(areas: readonly AreaDefinition[], files: readonly CommitFile[], faultPath: string): FaultAreaResolution;

package/dist/app/bugs/areas.js

@@ -0,0 +1 @@
+import{Minimatch as p}from"minimatch";const h=new Set("c cc cjs cpp cs css cts dart ex exs go h hpp html java js jsx kt kts mjs mts php py rb rs scala scss sh sql svelte swift ts tsx vue".split(" ")),f=new Set(["__testkit__","__tests__","build","dist","migrations","node_modules","test","tests","vendor"]),_=[/^\./u,/\.(?:test|spec|testkit|min)\./u,/\.d\.[cm]?ts$/u,/_test\.(?:go|py)$/u,/^test_.*\.py$/u],g=/\.config\.[a-z]+$/u,d=1e4;export function isSourceFile(a){const e=a.split("/"),i=e[e.length-1],t=/\.([a-z0-9]+)$/u.exec(i)?.[1];return t===void 0||!h.has(t)||_.some(s=>s.test(i))||e.length===1&&g.test(i)?!1:!e.slice(0,-1).some(s=>s.startsWith(".")||f.has(s))}export function createAreaMatcher(a){const e=a.map(i=>({area:i,globs:i.paths.map(t=>new p(t,{dot:!0}))}));return i=>e.filter(t=>t.globs.some(s=>s.match(i))).map(t=>t.area)}export function measureAreaMap(a,e){const i=createAreaMatcher(a),t=new Map,s=[],c=[];let r=0,m=0;for(const n of e){if(!isSourceFile(n.path))continue;r+=n.size;const o=i(n.path);if(o.length===0)c.push(n);else if(o.length>1)s.push({areas:o.map(l=>l.id),path:n.path}),m+=n.size;else{const l=t.get(o[0].id)??{bytes:0,files:0};t.set(o[0].id,{bytes:l.bytes+n.size,files:l.files+1})}}return{areas:a.map(n=>{const o=t.get(n.id)??{bytes:0,files:0};return{...o,id:n.id,name:n.name,share:u(o.bytes,r)}}),doubleClaimedShare:u(m,r),doubleClaims:s,sourceBytes:r,unclaimed:c,unclaimedShare:u(c.reduce((n,o)=>n+o.size,0),r)}}export function resolveFaultArea(a,e,i){if(a.length===0)return{status:"no-map"};const t=createAreaMatcher(a)(i);if(t.length===0)return{status:"unclaimed"};if(t.length>1)return{areas:t.map(r=>r.id),status:"double-claimed"};const[s]=t,c=measureAreaMap(a,e).areas.find(r=>r.id===s.id);return{area:{id:s.id,name:s.name,share:c?.share??0},status:"resolved"}}function u(a,e){return e===0?0:Math.round(a/e*d)/d}

package/dist/app/bugs/assessment.d.ts

@@ -0,0 +1,9 @@
+import type { Tracker } from "../../tracker/types.js";
+import { type RunOutcome, type UnreadableIssue } from "./known-failures.js";
+import type { RunReading } from "./run-reading.js";
+export interface RunAssessment {
+    readonly outcome: RunOutcome;
+    readonly trackerProblem: string | null;
+    readonly unreadableIssues: readonly UnreadableIssue[];
+}
+export declare function assessRun(reading: RunReading, resolveIssueTracker: () => Promise<Tracker>): Promise<RunAssessment>;

package/dist/app/bugs/assessment.js

@@ -0,0 +1 @@
+import{TESTKIT_LABEL as l}from"./issue.js";import{classifyFailures as r,readKnownTests as o}from"./known-failures.js";export async function assessRun(e,t){if(e.kind!=="tests-failed")return{outcome:e,trackerProblem:null,unreadableIssues:[]};try{const s=await t(),{tests:a,unreadable:n}=o(await s.listOpenIssues(l));return{outcome:{classification:r(e.failures,a),kind:"tests-failed"},trackerProblem:null,unreadableIssues:n}}catch(s){return{outcome:{classification:r(e.failures,null),kind:"tests-failed"},trackerProblem:s instanceof Error?s.message:String(s),unreadableIssues:[]}}}

package/dist/app/bugs/categories.d.ts

@@ -0,0 +1,41 @@
+export declare const BUG_FACTS_VERSION = 1;
+export interface BugCategory<Id extends string = string> {
+    readonly boundaryRule: string;
+    readonly definition: string;
+    readonly id: Id;
+}
+export interface BugCategoryList<Id extends string = string> {
+    readonly assignmentRule: string;
+    readonly categories: readonly BugCategory<Id>[];
+    readonly field: string;
+    readonly question: string;
+}
+export declare const BUG_CAUSES: {
+    readonly field: "cause";
+    readonly question: "What kind of mistake is in the code?";
+    readonly assignmentRule: "A bug gets exactly one cause. When a bug holds several mistakes, classify the earliest one in the causal chain. Use `other` when none fits.";
+    readonly categories: readonly [BugCategory<"missing-validation">, BugCategory<"mishandled-error">, BugCategory<"race-condition">, BugCategory<"unbounded-or-inefficient-work">, BugCategory<"incorrect-query-or-selection">, BugCategory<"missing-access-control">, BugCategory<"null-or-default-mishandling">, BugCategory<"stale-state">, BugCategory<"interface-mismatch">, BugCategory<"omitted-step">, BugCategory<"incorrect-logic">, BugCategory<"other">];
+};
+export declare const BUG_IMPACTS: {
+    readonly field: "impact";
+    readonly question: "What is at stake for the business?";
+    readonly assignmentRule: "A bug gets exactly one impact. The list is in precedence order, and the first type that applies wins. Classify the impact the test demonstrated, not a downstream one you expect. Use `other` when none fits.";
+    readonly categories: readonly [BugCategory<"unauthorized-access">, BugCategory<"financial-error">, BugCategory<"audit-or-compliance-gap">, BugCategory<"data-loss-or-corruption">, BugCategory<"workflow-breakdown">, BugCategory<"availability-or-performance">, BugCategory<"blocked-operation">, BugCategory<"wrong-information-shown">, BugCategory<"other">];
+};
+export declare const BUG_ESCAPES: {
+    readonly field: "escape";
+    readonly question: "Why did the tests that existed before yours not catch the bug?";
+    readonly assignmentRule: "A bug gets exactly one reason. Judge the tests that were there before yours. Use `other` when none fits.";
+    readonly categories: readonly [BugCategory<"no-test">, BugCategory<"untested-input">, BugCategory<"untested-interleaving">, BugCategory<"wrong-assertion">, BugCategory<"test-double">, BugCategory<"other">];
+};
+export type BugCause = (typeof BUG_CAUSES.categories)[number]["id"];
+export type BugImpact = (typeof BUG_IMPACTS.categories)[number]["id"];
+export type BugEscape = (typeof BUG_ESCAPES.categories)[number]["id"];
+export declare const BUG_SEVERITIES: readonly ["critical", "high", "medium", "low"];
+export type BugSeverity = (typeof BUG_SEVERITIES)[number];
+export declare const BUG_CONFIDENCES: readonly ["high", "low"];
+export type BugConfidence = (typeof BUG_CONFIDENCES)[number];
+export declare const BUG_CATCHERS: readonly ["campaign", "manual"];
+export type BugCatcher = (typeof BUG_CATCHERS)[number];
+export declare function categoryIds<Id extends string>(list: BugCategoryList<Id>): readonly Id[];
+export declare function isOneOf<Value extends string>(values: readonly Value[], value: unknown): value is Value;

package/dist/app/bugs/categories.js

@@ -0,0 +1 @@
+export const BUG_FACTS_VERSION=1;function e(i,t,o=""){return{boundaryRule:o,definition:t,id:i}}export const BUG_CAUSES={field:"cause",question:"What kind of mistake is in the code?",assignmentRule:"A bug gets exactly one cause. When a bug holds several mistakes, classify the earliest one in the causal chain. Use `other` when none fits.",categories:[e("missing-validation","The code never checks an input or a state precondition that the domain rules require: range, format, existence, uniqueness, cross-field consistency, or lifecycle state. Includes a validator that exists and accepts invalid input.","Earliest mistake wins. If application code never tested the condition, it is `missing-validation` even when a database constraint or the runtime later throws and the caller sees a server error. If the unchecked value is null, absent, empty, zero, or a default, use `null-or-default-mishandling`."),e("mishandled-error","A failure was detected and signalled, and the handling is wrong: not caught, translated to the wrong error, swallowed, ignored by the UI, or followed by a wrong retry or recovery.","The failure must already be signalled by something that checked. If nothing in the application checked and a lower layer failed, it is `missing-validation`."),e("race-condition","Correctness depends on how overlapping operations interleave: check-then-act without a lock or constraint, a lost update, or a late asynchronous completion that overwrites newer state.","Needs two overlapping operations to reproduce. If one sequential flow reproduces it, it is `stale-state` or `missing-validation`."),e("unbounded-or-inefficient-work","Results are correct and the cost is unbounded or needlessly repeated: unpaginated reads, N+1 queries, a missing or redundant index, serial waits, or loading everything and then slicing.","Correct results at excessive cost. If the result is wrong, it is `incorrect-query-or-selection`."),e("incorrect-query-or-selection","The code identifies the wrong set of records: a missing or wrong predicate, unstable ordering, a truncated page treated as complete, duplicate rows from a join, or a lookup key that omits a distinguishing part.","A missing predicate is `missing-access-control` only when it separates actors or tenants with different rights. The right records with wrong arithmetic on them is `incorrect-logic`."),e("missing-access-control","A permission, tenant, ownership, or separation-of-duties check is absent or wrong.","Use it when the missing check is about who the actor is. A missing check about what the data or state is belongs in `missing-validation`."),e("null-or-default-mishandling","Null, absent, empty, zero, or default values are conflated or not handled: null treated as omitted, a zero or an empty collection treated as missing, a null dereference, a wrong fallback default, or a caller that omits an argument and silently gets the default.","Wins over `missing-validation` and `incorrect-logic` whenever the triggering value is null, absent, empty, zero, or a default."),e("stale-state","Held state, such as a cache, local UI state, a derived or denormalised value, or leftover fields, is not refreshed, reset, persisted, or tied to the right identity when the thing it mirrors changes.","If the omission leaves old values visible where current ones should be, it is `stale-state`. If the missing action leaves nothing stale, it is `omitted-step`. If it reproduces only with overlapping operations, it is `race-condition`."),e("interface-mismatch","Data is lost or misread where it crosses between representations or components. One component's mapping drops, overwrites, hardcodes, or mis-assigns fields, or two components that must agree disagree on shape, type, field names, allowed values, semantics, or which copy of the data is authoritative.","Use it when the values were right before the crossing. Wrong computation on values that arrived intact is `incorrect-logic`. A held copy that goes out of date after its source changes is `stale-state`. An omitted argument that falls back to a default is `null-or-default-mishandling`."),e("omitted-step","A required follow-on action is not performed on some path: an audit record, a notification, a dispatch, a finalisation, or an alternate route that bypasses the shared routine.","If the missing thing is a check, it is `missing-validation`. If its absence leaves old values on display, it is `stale-state`."),e("incorrect-logic","The algorithm, arithmetic, date or time handling, rounding, boundary, comparison, boolean condition, parsing, or order of operations is wrong.","The default for wrong logic on valid, present values. If the trigger is a null, empty, zero, or default value, use `null-or-default-mishandling`. If the fault is which records were chosen, use `incorrect-query-or-selection`."),e("other","No category above describes the mistake.")]},BUG_IMPACTS={field:"impact",question:"What is at stake for the business?",assignmentRule:"A bug gets exactly one impact. The list is in precedence order, and the first type that applies wins. Classify the impact the test demonstrated, not a downstream one you expect. Use `other` when none fits.",categories:[e("unauthorized-access","Someone can see or change something they are not entitled to, including getting past a separation-of-duties rule.","The actor lacks the right. If an entitled actor skips a mandated control or leaves no evidence, it is `audit-or-compliance-gap`."),e("financial-error","A monetary amount, rate, currency, limit, payment or settlement state, or a money-committing decision is wrong where it is stored, charged, paid, or decided.","A money figure that is only displayed wrongly, with the stored value and the decision intact, is `wrong-information-shown`."),e("audit-or-compliance-gap","Required audit evidence or attribution is missing, duplicated, or false, or a mandated control did not run.","Use it when the harm is to evidence or to a regulatory or policy control. A skipped ordinary process step is `workflow-breakdown`."),e("data-loss-or-corruption","Stored data, or work the user entered, is destroyed, overwritten, duplicated, or left invalid or inconsistent.","If the action silently did nothing and prior stored state is intact, it is `blocked-operation`."),e("workflow-breakdown","A business process, such as a workflow, task, schedule, notification, or gate, is skipped, stalled, duplicated, or advanced wrongly.","Use it when the process goes wrong after the triggering request returns. A single user action that fails in front of the user is `blocked-operation`."),e("availability-or-performance","The service is slow, unavailable, or exhausts resources in a way that affects more than the triggering request or grows with data volume.","One request that fails for one user is `blocked-operation`."),e("blocked-operation","A legitimate action fails, returns an unusable or misleading error, or silently does not apply, with prior stored state intact.","The user was trying to do something and could not. If they were only reading and what they read was wrong, it is `wrong-information-shown`."),e("wrong-information-shown","Displayed or returned information is wrong, out of date, incomplete, or duplicated, with nothing stored wrongly.","Last in precedence. If the wrong value is also persisted, it is `data-loss-or-corruption`."),e("other","No type above describes what is at stake.")]},BUG_ESCAPES={field:"escape",question:"Why did the tests that existed before yours not catch the bug?",assignmentRule:"A bug gets exactly one reason. Judge the tests that were there before yours. Use `other` when none fits.",categories:[e("no-test","No existing test runs the faulty behaviour.","If any existing test reaches the faulty code path, use another reason."),e("untested-input","A test runs the behaviour and never with the input, state, or data volume that triggers the fault.","One sequential flow with the right input reproduces the bug. If it needs overlapping operations, it is `untested-interleaving`."),e("untested-interleaving","A test runs the behaviour in sequence, and the fault needs overlapping operations or a particular timing.","The triggering input is already covered, and only the overlap is missing."),e("wrong-assertion","A test runs the triggering case against the real code and passes, because its assertion is missing, too weak, or asserts the broken behaviour.","The faulty code ran in the test. If a stand-in ran in its place, it is `test-double`."),e("test-double","A test runs the triggering case against a test double or a fixture that hides the fault in the real component.","The real component would have failed the test."),e("other","No reason above applies.")]},BUG_SEVERITIES=["critical","high","medium","low"],BUG_CONFIDENCES=["high","low"],BUG_CATCHERS=["campaign","manual"];export function categoryIds(i){return i.categories.map(t=>t.id)}export function isOneOf(i,t){return typeof t=="string"&&i.some(o=>o===t)}

package/dist/app/bugs/ci-report.d.ts

@@ -0,0 +1,21 @@
+import type { TrackerCommentReference, TrackerCommitState } from "../../tracker/types.js";
+import { type FailedTest, type KnownFailure, type UnreadableIssue } from "./known-failures.js";
+import { type CheckoutOptions } from "./store.js";
+export interface CiReportOptions extends CheckoutOptions {
+    readonly context?: string | undefined;
+    readonly env: Readonly<NodeJS.ProcessEnv>;
+}
+export interface CiReport {
+    readonly comment: TrackerCommentReference | null;
+    readonly commit: string;
+    readonly context: string;
+    readonly description: string;
+    readonly known: readonly KnownFailure[];
+    readonly new: readonly FailedTest[];
+    readonly pullRequest: number | null;
+    readonly state: TrackerCommitState;
+    readonly stepSummary: string | null;
+    readonly trackerProblem: string | null;
+    readonly unreadableIssues: readonly UnreadableIssue[];
+}
+export declare function reportCiRun(options: CiReportOptions): Promise<CiReport>;

package/dist/app/bugs/ci-report.js

@@ -0,0 +1,2 @@
+import k from"node:fs";import y from"node:path";import{readBugPolicy as g}from"../../config/loader/bugs.js";import{loadTestkitConfig as v}from"../../config/loader/index.js";import{resolveCiProvenance as T}from"../../evidence/ci-provenance.js";import{loadLatestRunArtifact as h}from"../../evidence/reading.js";import{isUnknownRecord as q}from"../../kernel/unknown-values.js";import{resolveTracker as w}from"../tracker/index.js";import{assessRun as P}from"./assessment.js";import{ciReportMarker as U,readCiContext as b,renderCiSummary as x}from"./ci-summary.js";import{describeRunOutcome as D}from"./known-failures.js";import{readRun as F}from"./run-reading.js";import{resolveBugCheckout as A}from"./store.js";const R="Testkit found no results for this commit";export async function reportCiRun(e){const r=b(e.context),t=await A(e),s=g((await v(t.productDir)).config),a={env:e.env,productDir:t.productDir};let u=null;const c=()=>(u??=w(a),u),{repository:l}=s,o=await P(B(t),l===null?c:()=>w({...a,repository:l})),n=D(o.outcome,s.knownFailures==="allow"),i=t.pullRequestHeadCommit??t.headCommit,m=T(e.env).runUrl,d=x({...o,commit:i,context:r,knownFailuresAllowed:s.knownFailures==="allow",runUrl:m,status:n}),C=I(e.env,d),p=await c();await p.setCommitStatus({...n,context:r,sha:i,targetUrl:m});const S=t.pullRequest===null?null:await p.upsertPullRequestComment({body:d,marker:U(r),pullRequest:t.pullRequest}),f=o.outcome.kind==="tests-failed"?o.outcome.classification:{known:[],new:[]};return{comment:S,commit:i,context:r,description:n.description,known:f.known,new:f.new,pullRequest:t.pullRequest,state:n.state,stepSummary:C,trackerProblem:o.trackerProblem,unreadableIssues:o.unreadableIssues}}function B(e){if(!k.existsSync(y.join(e.productDir,".testkit","results","latest.json")))return{kind:"run-failed",reason:R};const r=h(e.productDir),t=r.git;return!q(t)||t.commitSha!==e.headCommit?{kind:"run-failed",reason:R}:F(r,e.productPath)}function I(e,r){const t=e.GITHUB_STEP_SUMMARY?.trim()??"";return t.length===0?null:(k.appendFileSync(t,`${r}
+`),t)}

package/dist/app/bugs/ci-summary.d.ts

@@ -0,0 +1,13 @@
+import type { RunAssessment } from "./assessment.js";
+import { type RunStatus } from "./known-failures.js";
+export declare const DEFAULT_CI_CONTEXT = "testkit";
+export interface CiSummaryInput extends RunAssessment {
+    readonly commit: string;
+    readonly context: string;
+    readonly knownFailuresAllowed: boolean;
+    readonly runUrl: string | null;
+    readonly status: RunStatus;
+}
+export declare function readCiContext(value: string | undefined): string;
+export declare function ciReportMarker(context: string): string;
+export declare function renderCiSummary(input: CiSummaryInput): string;

package/dist/app/bugs/ci-summary.js

@@ -0,0 +1,5 @@
+import{describeUnnamedFailure as T}from"./known-failures.js";import{redactBugText as I,redactRunResultText as k}from"../../tracker/redaction.js";export const DEFAULT_CI_CONTEXT="testkit";const p=/^[A-Za-z0-9][A-Za-z0-9 ._/:-]{0,59}$/u,b=12,l=50,E=10,d=160,_=/^https?:\/\/[^\s()<>]+$/u;export function readCiContext(e){if(e===void 0)return DEFAULT_CI_CONTEXT;if(!p.test(e)||e!==e.trim())throw new Error(`--context must be 1 to 60 characters, start with an ASCII letter or a digit, not end with a space, and hold only ASCII letters, digits, spaces, and the characters . _ / : and -. Received ${JSON.stringify(e)}.`);return e}export function ciReportMarker(e){return e===DEFAULT_CI_CONTEXT?"<!-- testkit-ci-report -->":`<!-- testkit-ci-report:${e} -->`}export function renderCiSummary(e){const{context:n,outcome:t}=e,r=[ciReportMarker(n),n===DEFAULT_CI_CONTEXT?"## Testkit":`## Testkit: ${n}`,w(e),`Commit \`${e.commit.slice(0,b)}\`.${h("CI run",e.runUrl," ",".")}`];if(t.kind==="tests-failed"){const{known:a,new:i}=t.classification;if(i.length>0&&r.push("### New failures",f(i.map(s=>`- ${c(s.file)}: ${$(s)}`))),a.length>0&&r.push("### Known failures",f(a.map(({failure:s,issue:o})=>`- ${c(s.file)}: ${$(s)}, ${g(o.issue,o.url)}`))),e.trackerProblem!==null&&r.push(`Testkit could not read the open bugs, so every failure counts as new. ${m(e.trackerProblem,400)}`),e.unreadableIssues.length>0){const s=e.unreadableIssues.slice(0,E).map(u=>g(u.number,u.url)),o=e.unreadableIssues.length-s.length;r.push(`Testkit cannot read the facts block of ${C(e.unreadableIssues.length,"open `testkit` issue")}, so ${e.unreadableIssues.length===1?"it matches":"they match"} no test: ${s.join(", ")}${o>0?`, and ${String(o)} more`:""}.`)}}return`${r.join(`
+
+`)}
+`}function w({knownFailuresAllowed:e,outcome:n,status:t}){return n.kind==="passed"?`${t.description}.`:n.kind==="run-failed"?`The run failed outside its tests. ${c(n.reason,400)}`:n.classification.new.length>0?`${t.description}.`:e?`${t.description}. Every failure is a known bug and \`bugs.knownFailures\` is \`"allow"\`, so the run passes.`:`${t.description}. Every failure is a known bug, and \`bugs.knownFailures\` is \`"fail"\`, so the run fails.`}function f(e){const n=e.length-l;return[...e.slice(0,l),...n>0?[`- and ${String(n)} more`]:[]].join(`
+`)}function h(e,n,t="",r=""){return n!==null&&_.test(n)?`${t}[${e}](${n})${r}`:""}function $(e){return e.title===null?T(e):c(e.title)}function g(e,n){const t=`#${String(e)}`;return h(t,n)||t}function C(e,n){return`${String(e)} ${n}${e===1?"":"s"}`}function c(e,n=d){return m(k(e),n)}function m(e,n=d){const t=Array.from(I(e.replace(/\s+/gu," ").trim())),r=t.length>n?`${t.slice(0,n-3).join("")}...`:t.join("");if(r.length===0)return"an empty name";const a=Math.max(0,...[...r.matchAll(/`+/gu)].map(s=>s[0].length)),i="`".repeat(a+1);return r.startsWith("`")||r.endsWith("`")?`${i} ${r} ${i}`:`${i}${r}${i}`}

package/dist/app/bugs/draft.d.ts

@@ -0,0 +1,70 @@
+import { type PublicTestType } from "../../domain/identifiers.js";
+import { type BugCatcher, type BugCause, type BugConfidence, type BugEscape, type BugImpact, type BugSeverity } from "./categories.js";
+export declare const BUG_DRAFT_SCHEMA_VERSION = 1;
+export declare const BUG_TITLE_MAX_LENGTH = 200;
+export declare const BUG_SECTION_MAX_LENGTH = 12000;
+export declare const CAMPAIGN_ID_MAX_LENGTH = 60;
+export interface BugReport {
+    readonly title: string;
+    readonly severity: BugSeverity;
+    readonly expected: string;
+    readonly authority: string;
+    readonly observed: string;
+    readonly rootCause: string;
+    readonly test: {
+        readonly file: string;
+        readonly title: string;
+    };
+    readonly fault: {
+        readonly lines: readonly [number, number];
+        readonly path: string;
+    };
+    readonly cause: BugCause;
+    readonly impact: BugImpact;
+    readonly confidence: {
+        readonly cause: BugConfidence;
+        readonly impact: BugConfidence;
+    };
+    readonly escape: BugEscape;
+}
+export interface BugDraftFiling {
+    readonly filedAt: string;
+    readonly foundAt: string;
+    readonly number: number;
+    readonly url: string;
+}
+export interface BugDraft extends BugReport {
+    readonly schemaVersion: typeof BUG_DRAFT_SCHEMA_VERSION;
+    readonly id: string;
+    readonly recordedAt: string;
+    readonly recordedCommit: string;
+    readonly productPath: string;
+    readonly caughtBy: BugCatcher;
+    readonly campaign?: string;
+    readonly testType: PublicTestType;
+    readonly filed?: BugDraftFiling;
+}
+export interface BugReportReading {
+    readonly redactedFields: readonly string[];
+    readonly report: BugReport;
+}
+export interface BugDraftOrigin {
+    readonly campaign: string | null;
+    readonly productPath: string;
+    readonly recordedAt: Date;
+    readonly recordedCommit: string;
+}
+export declare class BugRecordError extends Error {
+    readonly problems: readonly string[];
+    constructor(subject: string, problems: readonly string[]);
+}
+export declare function isCampaignId(value: unknown): value is string;
+export declare function testTypeOfFile(file: string): PublicTestType | null;
+export declare function toRepositoryPath(productPath: string, value: string): string | null;
+export declare function toProductPath(productPath: string, repositoryPath: string): string;
+export declare function readBugReport(value: unknown, productPath: string): BugReportReading;
+export declare function createBugDraft(report: BugReport, origin: BugDraftOrigin): BugDraft;
+export declare function readBugDraft(value: unknown): BugDraft;
+export declare function bugDraftId(report: Pick<BugReport, "test" | "title">): string;
+export declare function isBugDraftId(value: unknown): value is string;
+export declare function namesSameTest(left: Pick<BugReport, "test">, right: Pick<BugReport, "test">): boolean;

package/dist/app/bugs/draft.js

@@ -0,0 +1,4 @@
+import G from"node:crypto";import T from"node:path";import{isPublicTestType as P}from"../../domain/identifiers.js";import{isUnknownRecord as A}from"../../kernel/unknown-values.js";import{BUG_CATCHERS as O,BUG_CAUSES as R,BUG_CONFIDENCES as B,BUG_ESCAPES as U,BUG_IMPACTS as D,BUG_SEVERITIES as F,categoryIds as C,isOneOf as E}from"./categories.js";import{decodeFaultLines as z,isCommitId as $}from"./facts.js";import{redactBugText as S}from"../../tracker/redaction.js";export const BUG_DRAFT_SCHEMA_VERSION=1,BUG_TITLE_MAX_LENGTH=200,BUG_SECTION_MAX_LENGTH=12e3;const L=48,M=/^[a-z0-9]+(?:-[a-z0-9]+)*$/u;export const CAMPAIGN_ID_MAX_LENGTH=60;export class BugRecordError extends Error{problems;constructor(e,n){super(`${e}:
+${n.map(r=>`- ${r}`).join(`
+`)}`),this.name="BugRecordError",this.problems=n}}export function isCampaignId(t){return typeof t=="string"&&t.length<=CAMPAIGN_ID_MAX_LENGTH&&M.test(t)}export function testTypeOfFile(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return P(e)?e:null}export function toRepositoryPath(t,e){const n=e.replaceAll("\\","/");if(n.startsWith("/")||/^[a-z]:/iu.test(n))return null;const r=T.posix.normalize(T.posix.join(t,n));return r==="."||r===".."||r.startsWith("../")?null:r}export function toProductPath(t,e){return t===""?e:T.posix.relative(t,e)}export function readBugReport(t,e){if(!A(t))throw new BugRecordError("The bug record is invalid",["it must be a JSON object"]);const n=[],r=[],p=I(t,n,{path:i=>toRepositoryPath(e,i),text:(i,f)=>{const u=S(i);return u!==i&&r.push(f),u}});if(n.length>0)throw new BugRecordError("The bug record is invalid",n);return{redactedFields:r,report:p}}export function createBugDraft(t,e){const n=testTypeOfFile(t.test.file);if(n===null)throw new BugRecordError("The bug record is invalid",[w("test.file")]);return{schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:bugDraftId(t),recordedAt:e.recordedAt.toISOString(),recordedCommit:e.recordedCommit,productPath:e.productPath,caughtBy:e.campaign===null?"manual":"campaign",...e.campaign===null?{}:{campaign:e.campaign},testType:n,...t}}export function readBugDraft(t){const e="The bug draft is unreadable";if(!A(t))throw new BugRecordError(e,["it must be a JSON object"]);if(t.schemaVersion!==BUG_DRAFT_SCHEMA_VERSION)throw new BugRecordError(e,[`unknown schemaVersion ${JSON.stringify(t.schemaVersion)}`]);const n=[],r=(a,b,y,l)=>b(a)?a:(n.push(l),y),p=I(t,n,{path:a=>toRepositoryPath("",a),text:a=>a}),i=testTypeOfFile(p.test.file)??"int";t.testType!==i&&n.push("testType does not match test.file");const f=r(t.caughtBy,V,"manual","caughtBy must be campaign or manual"),u=r(t.campaign,H,void 0,"campaign must be a campaign id");f==="campaign"!=(u!==void 0)&&n.push("campaign must be present when caughtBy is campaign and absent otherwise");const x={schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:r(t.id,isBugDraftId,"","id is not a draft id"),recordedAt:r(t.recordedAt,N,"","recordedAt must be a timestamp"),recordedCommit:r(t.recordedCommit,$,"","recordedCommit must be a full commit id"),productPath:r(t.productPath,k,"","productPath must be a string"),caughtBy:f,...u===void 0?{}:{campaign:u},testType:i,...p,...t.filed===void 0?{}:{filed:j(t.filed,n)}};if(n.length>0)throw new BugRecordError(e,n);return x}export function bugDraftId(t){const e=t.title.normalize("NFKD").replace(/\p{M}+/gu,"").toLowerCase().replace(/[^a-z0-9]+/gu,"-").slice(0,L).replace(/^-+|-+$/gu,""),n=G.createHash("sha256").update(`${t.test.file}
+${t.test.title}`).digest("hex");return`${e||"bug"}-${n.slice(0,8)}`}export function isBugDraftId(t){return typeof t=="string"&&/^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/u.test(t)}export function namesSameTest(t,e){return t.test.file===e.test.file&&t.test.title===e.test.title}function I(t,e,n){const r=(s,o,c,d)=>{const m=s[o];if(typeof m!="string"||m.trim().length===0)return e.push(`${c} must be a non-empty string`),"";const h=m.trim(),g=h.length>d?h:n.text(h,c);return g.length>d&&e.push(`${c} must be at most ${String(d)} characters`),g},p=(s,o,c)=>{const d=s[o],m=typeof d=="string"?d.trim():"",h=m.length>0?n.path(m):null;return h===null?e.push(`${c} must be a path inside the repository`):[m,h].some(g=>S(g)!==g)&&e.push(`${c} is a path that holds text Testkit redacts as a secret`),h??""},i=(s,o,c)=>E(s,o)?o:(e.push(`${c} must be one of ${s.join(", ")}`),s[0]),f=s=>{const o=t[s];return A(o)?o:(e.push(`${s} must be an object`),{})},u=(s,o,c)=>{const d=r(s,o,c,BUG_TITLE_MAX_LENGTH);return/[\r\n]/u.test(d)&&e.push(`${c} must be one line`),d},x=u(t,"title","title"),a=f("test"),b=f("fault"),y=f("confidence"),l=p(a,"file","test.file");l!==""&&testTypeOfFile(l)===null&&e.push(w("test.file"));let _=[1,1];try{_=z(b.lines)}catch(s){e.push(s instanceof Error?s.message:"fault.lines is invalid")}return{title:x,severity:i(F,t.severity,"severity"),expected:r(t,"expected","expected",BUG_SECTION_MAX_LENGTH),authority:r(t,"authority","authority",BUG_SECTION_MAX_LENGTH),observed:r(t,"observed","observed",BUG_SECTION_MAX_LENGTH),rootCause:r(t,"rootCause","rootCause",BUG_SECTION_MAX_LENGTH),test:{file:l,title:u(a,"title","test.title")},fault:{lines:_,path:p(b,"path","fault.path")},cause:i(C(R),t.cause,"cause"),impact:i(C(D),t.impact,"impact"),confidence:{cause:i(B,y.cause,"confidence.cause"),impact:i(B,y.impact,"confidence.impact")},escape:i(C(U),t.escape,"escape")}}function j(t,e){return A(t)&&N(t.filedAt)&&$(t.foundAt)&&typeof t.number=="number"&&Number.isSafeInteger(t.number)&&t.number>=1&&typeof t.url=="string"?{filedAt:t.filedAt,foundAt:t.foundAt,number:t.number,url:t.url}:(e.push("filed must hold filedAt, foundAt, number, and url"),{filedAt:"",foundAt:"",number:0,url:""})}function H(t){return t===void 0||isCampaignId(t)}function V(t){return E(O,t)}function k(t){return typeof t=="string"}function N(t){return typeof t=="string"&&!Number.isNaN(Date.parse(t))}function w(t){return`${t} must be a Testkit test file, such as refund.contract.testkit.ts`}

package/dist/app/bugs/facts.d.ts

@@ -0,0 +1,48 @@
+import { type PublicTestType } from "../../domain/identifiers.js";
+import { BUG_FACTS_VERSION, type BugCatcher, type BugCause, type BugConfidence, type BugEscape, type BugImpact } from "./categories.js";
+export declare const FACTS_START_MARKER = "<!-- testkit-facts:start -->";
+export declare const FACTS_END_MARKER = "<!-- testkit-facts:end -->";
+export interface BugFactsArea {
+    readonly id: string;
+    readonly name: string;
+    readonly share: number;
+}
+export interface BugFactsFault {
+    readonly lines?: readonly [number, number];
+    readonly path: string;
+}
+export interface BugFacts {
+    readonly v: typeof BUG_FACTS_VERSION;
+    readonly cause: BugCause;
+    readonly impact: BugImpact;
+    readonly confidence: {
+        readonly cause: BugConfidence;
+        readonly impact: BugConfidence;
+    };
+    readonly escape?: BugEscape;
+    readonly caughtBy: BugCatcher;
+    readonly campaign?: string;
+    readonly testType: PublicTestType;
+    readonly test: {
+        readonly file: string;
+        readonly title?: string;
+    };
+    readonly foundAt: string;
+    readonly fault?: BugFactsFault;
+    readonly area?: BugFactsArea;
+    readonly introducedBy?: string;
+    readonly introducedMethod?: "blame";
+}
+export type BugFactsReading = {
+    readonly status: "invalid";
+    readonly reason: string;
+} | {
+    readonly status: "missing";
+} | {
+    readonly facts: BugFacts;
+    readonly status: "ok";
+};
+export declare function renderFactsBlock(facts: BugFacts): string;
+export declare function readFactsBlock(body: string): BugFactsReading;
+export declare function isCommitId(value: unknown): value is string;
+export declare function decodeFaultLines(value: unknown): readonly [number, number];

package/dist/app/bugs/facts.js

@@ -0,0 +1,2 @@
+import{isAreaId as g}from"../../config/loader/bugs.js";import{isPublicTestType as E}from"../../domain/identifiers.js";import{isUnknownRecord as m}from"../../kernel/unknown-values.js";import{BUG_CATCHERS as b,BUG_CAUSES as T,BUG_CONFIDENCES as p,BUG_ESCAPES as A,BUG_FACTS_VERSION as h,BUG_IMPACTS as B,categoryIds as a,isOneOf as S}from"./categories.js";export const FACTS_START_MARKER="<!-- testkit-facts:start -->",FACTS_END_MARKER="<!-- testkit-facts:end -->";const k=/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u;export function renderFactsBlock(e){return["<details>","<summary>Testkit facts</summary>","",FACTS_START_MARKER,"","```json",JSON.stringify(l(e),null,2),"```","",FACTS_END_MARKER,"","</details>"].join(`
+`)}export function readFactsBlock(e){const t=e.lastIndexOf(FACTS_START_MARKER);if(t===-1)return{status:"missing"};const n=t+FACTS_START_MARKER.length,r=e.indexOf(FACTS_END_MARKER,n);if(r===-1)return{status:"invalid",reason:"the facts block has no end marker"};const d=/```[^\n]*\r?\n([\s\S]*?)\r?\n[ \t]*```/u.exec(e.slice(n,r));if(d===null)return{status:"invalid",reason:"the facts block has no fenced code block"};let u;try{u=JSON.parse(d[1])}catch{return{status:"invalid",reason:"the facts block is not valid JSON"}}try{return{facts:O(u),status:"ok"}}catch(f){return{status:"invalid",reason:f instanceof Error?f.message:"the facts block is invalid"}}}export function isCommitId(e){return typeof e=="string"&&k.test(e)}function O(e){if(!m(e))throw new Error("the facts block is not a JSON object");if(e.v!==h)throw new Error(`unknown facts version ${JSON.stringify(e.v)}`);const t=c(e.confidence,"confidence"),n=c(e.test,"test");return l({v:h,cause:o(a(T),e.cause,"cause"),impact:o(a(B),e.impact,"impact"),confidence:{cause:o(p,t.cause,"confidence.cause"),impact:o(p,t.impact,"confidence.impact")},escape:i(e.escape,r=>o(a(A),r,"escape")),caughtBy:o(b,e.caughtBy,"caughtBy"),campaign:i(e.campaign,r=>s(r,"campaign")),testType:_(e.testType),test:{file:s(n.file,"test.file"),title:i(n.title,r=>s(r,"test.title"))},foundAt:y(e.foundAt,"foundAt"),fault:i(e.fault,C),area:i(e.area,N),introducedBy:i(e.introducedBy,r=>y(r,"introducedBy")),introducedMethod:i(e.introducedMethod,r=>o(["blame"],r,"introducedMethod"))})}function C(e){const t=c(e,"fault");return{lines:i(t.lines,decodeFaultLines),path:s(t.path,"fault.path")}}export function decodeFaultLines(e){if(!Array.isArray(e)||e.length!==2)throw new Error("fault.lines must be [start, end]");const t=e,[n,r]=t;if(!w(n)||!w(r)||n>r)throw new Error("fault.lines must be two line numbers counted from 1, with start no greater than end");return[n,r]}function N(e){const t=c(e,"area");if(!g(t.id))throw new Error("area.id is not an area id");const n=t.share;if(typeof n!="number"||!Number.isFinite(n)||n<0||n>1)throw new Error("area.share must be a number from 0 to 1");return{id:t.id,name:s(t.name,"area.name"),share:n}}function l(e){return{v:e.v,cause:e.cause,impact:e.impact,confidence:{cause:e.confidence.cause,impact:e.confidence.impact},...e.escape===void 0?{}:{escape:e.escape},caughtBy:e.caughtBy,...e.campaign===void 0?{}:{campaign:e.campaign},testType:e.testType,test:{file:e.test.file,...e.test.title===void 0?{}:{title:e.test.title}},foundAt:e.foundAt,...e.fault===void 0?{}:{fault:{path:e.fault.path,...e.fault.lines===void 0?{}:{lines:e.fault.lines}}},...e.area===void 0?{}:{area:{id:e.area.id,name:e.area.name,share:e.area.share}},...e.introducedBy===void 0?{}:{introducedBy:e.introducedBy},...e.introducedMethod===void 0?{}:{introducedMethod:e.introducedMethod}}}function i(e,t){return e===void 0?void 0:t(e)}function c(e,t){if(!m(e))throw new Error(`${t} must be an object`);return e}function s(e,t){if(typeof e!="string"||e.trim().length===0)throw new Error(`${t} must be a non-empty string`);return e}function o(e,t,n){if(!S(e,t))throw new Error(`${n} must be one of ${e.join(", ")}`);return t}function _(e){if(!E(e))throw new Error("testType is not a Testkit test type");return e}function y(e,t){if(!isCommitId(e))throw new Error(`${t} must be a full lowercase commit id`);return e}function w(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=1}

package/dist/app/bugs/file.d.ts

@@ -0,0 +1,45 @@
+import type { Tracker, TrackerIssueDraft } from "../../tracker/types.js";
+import { type BugDraft } from "./draft.js";
+import { type BugFactsArea } from "./facts.js";
+import { type BugCoverage, type BugFiling } from "./issue.js";
+import { type FailedTest } from "./known-failures.js";
+import type { RunReading } from "./run-reading.js";
+import { type BugCheckout, type CheckoutOptions } from "./store.js";
+export interface PrepareBugFilingOptions extends CheckoutOptions {
+    readonly env: Readonly<NodeJS.ProcessEnv>;
+    readonly id: string;
+}
+export interface BugIssueReference {
+    readonly number: number;
+    readonly url: string;
+}
+export interface ReadyBugFiling {
+    readonly checkout: BugCheckout;
+    readonly draft: BugDraft;
+    readonly draftFile: string;
+    readonly filing: BugFiling;
+    readonly status: "ready";
+    readonly testFile: string;
+    readonly tracker: Tracker;
+}
+export interface AlreadyFiledBug {
+    readonly draft: BugDraft;
+    readonly issue: BugIssueReference;
+    readonly status: "already-filed";
+}
+export type BugFilingPreparation = AlreadyFiledBug | ReadyBugFiling;
+export interface FiledBug {
+    readonly area: BugFactsArea | null;
+    readonly covers: BugCoverage;
+    readonly draft: BugDraft;
+    readonly introducedBy: string | null;
+    readonly issue: BugIssueReference;
+    readonly path: string;
+}
+export declare function prepareBugFiling(options: PrepareBugFilingOptions): Promise<BugFilingPreparation>;
+export declare function fileBug(ready: ReadyBugFiling, run: RunReading, options?: {
+    readonly at?: Date;
+}): Promise<FiledBug>;
+export declare function renderReadableBugIssue(draft: BugDraft, filing: BugFiling, covers: BugCoverage): TrackerIssueDraft;
+export declare function requireNoSecretInCommand(draft: Pick<BugDraft, "productPath" | "test">): void;
+export declare function requireRecordedFailure(draft: Pick<BugDraft, "productPath" | "test">, run: RunReading, commit: string): FailedTest;

package/dist/app/bugs/file.js

@@ -0,0 +1 @@
+import y from"node:path";import{readAreaMap as E,readBugPolicy as F}from"../../config/loader/bugs.js";import{loadTestkitConfig as R}from"../../config/loader/index.js";import{findIntroducingCommit as S,listRemoteRefsContaining as C,listUncommittedTrackedFiles as I}from"../../repo/history.js";import{listCommitFiles as A}from"../../repo/tree-listing.js";import{resolveTracker as x}from"../tracker/index.js";import{resolveFaultArea as B}from"./areas.js";import{toProductPath as c}from"./draft.js";import{FACTS_END_MARKER as N,FACTS_START_MARKER as D,readFactsBlock as O,renderFactsBlock as m}from"./facts.js";import{bugFacts as v,bugLabels as _,renderBugIssue as L,reproduceCommand as p,TESTKIT_LABEL as M}from"./issue.js";import{findKnownTest as j,readKnownTests as J}from"./known-failures.js";import{holdsRedaction as P,redactBugText as h}from"../../tracker/redaction.js";import{bugDraftFile as K,displayPath as g,readStoredDrafts as q,resolveBugCheckout as H,writeJsonFile as U}from"./store.js";const W=12,w=10;export async function prepareBugFiling(r){const t=await H(r),{draft:e,file:i}=G(t,r.id);if(e.filed!==void 0)return{draft:e,issue:{number:e.filed.number,url:e.filed.url},status:"already-filed"};requireNoSecretInCommand(e);const a=T(t.headCommit),n=await A(t.repoRoot,t.headCommit);for(const u of[e.test.file,e.fault.path])if(!n.some(k=>k.path===u))throw new Error(`${u} is not in commit ${a}. Commit it and push the commit, then run this again.`);const o=(await I(t.repoRoot)).filter(u=>!Q(t,u));if(o.length>0)throw new Error(`Tracked files have uncommitted changes: ${V(o)}. The issue says that the test fails at ${a}, so commit or discard the changes first.`);if((await C(t.repoRoot,t.headCommit)).length===0)throw new Error(`No remote branch contains commit ${a}. Push it, then run this again. The issue points readers to the test at that commit.`);const{config:d}=await R(t.productDir),l=z(B(E(d),n,e.fault.path),e.fault.path),f=await S(t.repoRoot,t.headCommit,e.fault.path,e.fault.lines),s=await x({env:r.env,productDir:t.productDir,repository:F(d).repository??void 0});return b(await $(s),e.test),{checkout:t,draft:e,draftFile:i,filing:{area:l,foundAt:t.headCommit,introducedBy:f.status==="found"?f.commit:null},status:"ready",testFile:y.join(t.productDir,c(t.productPath,e.test.file)),tracker:s}}export async function fileBug(r,t,e={}){const{checkout:i,draft:a,filing:n,tracker:o}=r,d=requireRecordedFailure(a,t,T(i.headCommit));b(await $(o),d);const l=d.script===!0?"file":"test",f=renderReadableBugIssue(a,n,l);await o.ensureLabels(_(a.severity,n.area));const s=await o.createIssue(f),u={...a,filed:{filedAt:(e.at??new Date).toISOString(),foundAt:n.foundAt,number:s.number,url:s.url}};return U(r.draftFile,u),{area:n.area,covers:l,draft:u,introducedBy:n.introducedBy,issue:{number:s.number,url:s.url},path:g(i,r.draftFile)}}export function renderReadableBugIssue(r,t,e){const i=L(r,t,e),a=O(i.body),n=m(v(r,t,e));if(a.status==="ok"&&m(a.facts)===n)return i;throw new Error(`Testkit cannot read back the facts of the issue it would file, so it files nothing. This happens when test.title, test.file, fault.path, or the area's name holds ${D} or ${N}. Rename the test, the file, or the area, then record the bug again.`)}export function requireNoSecretInCommand(r){const t=p(r);if(h(t)!==t)throw new Error("The command that reproduces this bug holds text that Testkit redacts as a secret, and an issue carries that command. The text is in the path of the product directory or in the test's path from that directory. Rename the directory that holds it, commit and push the change, then record the bug again.")}export function requireRecordedFailure(r,t,e){const{file:i,title:a}=r.test,n=`\`${p(r)}\``;if(t.kind==="passed")throw new Error(`${i} passed at ${e}, so Testkit has no failure to file. ${n} must fail.`);if(t.kind==="run-failed")throw new Error(`${n} failed outside its tests at ${e}: ${t.reason}. Testkit files a bug when the test itself fails.`);const o=t.failures.filter(s=>s.file===i);if(o.length===0)throw new Error(`${n} failed at ${e}, but ${i} is not among the files that failed.`);const d=o.flatMap(s=>s.title===null?[]:[s.title]),l=d.filter(s=>!/[\r\n]/u.test(s)&&!P(s)&&h(s)===s);if(l.includes(a))return{file:i,title:a};const f=o.some(s=>s.script===!0);if(d.length>0){const s=d.length-l.length;throw new Error([`${i} failed at ${e}, but not under the title ${JSON.stringify(a)}.`,...l.length===0?[]:[`Testkit reported ${l.map(u=>JSON.stringify(u)).join(", ")}. Set test.title to the title Testkit reports, then record the bug again.`],...s===0?[]:[`Testkit ${l.length===0?"":"also "}reported ${s===1?"1 title that spans lines or holds":`${String(s)} titles that span lines or hold`} text it redacts as a secret. It files no bug under such a title. Rename the test, or label the assertion, so that its title is one line and holds no such text.`],...f?["The script also failed as a whole. Testkit files a script failure only when the file reports no named failure."]:[]].join(" "))}if(f)return{file:i,script:!0,title:null};throw new Error(`${i} failed at ${e} without reaching a result, so Testkit cannot tell that ${JSON.stringify(a)} failed. A file fails this way when it crashes, times out, or does not load. Make the test fail on its assertion, then run this again.`)}async function $(r){return J(await r.listOpenIssues(M)).tests}function b(r,t){const e=j(r,t);if(e===null)return;const i=`Open issue #${String(e.issue)}`;throw new Error(e.title===null?`${i} already covers the whole file ${t.file}: ${e.url}. While it is open, Testkit files no other bug for that file.`:`${i} already covers this test: ${e.url}. Testkit keeps one open bug for each test.`)}function G(r,t){const e=q(r.productDir),i=e.drafts.find(o=>o.draft.id===t);if(i!==void 0)return i;const a=e.unreadable.find(o=>o.file===K(r.productDir,t));if(a!==void 0)throw new Error(`Testkit cannot read ${g(r,a.file)}: ${a.problem}. Record the bug again with \`testkit bug record\`.`);const n=e.drafts.filter(o=>o.draft.filed===void 0).map(o=>o.draft.id);throw new Error(n.length===0?`No bug draft has the id ${JSON.stringify(t)}, and this checkout has no unfiled draft. Record the bug with \`testkit bug record\` first.`:`No bug draft has the id ${JSON.stringify(t)}. Unfiled drafts: ${n.join(", ")}.`)}function z(r,t){if(r.status==="no-map")return null;const e="Fix `areas` in testkit.config.ts, commit and push the change, then run this again.";if(r.status==="resolved"){const{area:i}=r;if(h(i.name)===i.name)return i;throw new Error(`areas.${i.id}.name holds text that Testkit redacts as a secret, and an issue carries its area's name. ${e}`)}throw new Error(r.status==="unclaimed"?`No area claims ${t}. Every bug needs its fault in one area. ${e}`:`The areas ${r.areas.join(" and ")} both claim ${t}. A file belongs to one area. ${e}`)}function Q(r,t){return c(r.productPath,t).startsWith(".testkit/")}function V(r){const t=r.length-w;return`${r.slice(0,w).join(", ")}${t>0?`, and ${String(t)} more`:""}`}function T(r){return r.slice(0,W)}

package/dist/app/bugs/gate.d.ts

@@ -0,0 +1,22 @@
+import { type BugPolicy } from "../../config/loader/bugs.js";
+import type { FailedTest, KnownFailure } from "./known-failures.js";
+import { type RunDocumentReading } from "./run-reading.js";
+export interface KnownFailureGateInput {
+    readonly env: Readonly<NodeJS.ProcessEnv>;
+    readonly exitCode: number;
+    readonly failedOutsideTests: boolean;
+    readonly policy: BugPolicy;
+    readonly productDir: string;
+    readonly run: RunDocumentReading;
+}
+export interface KnownFailureReport {
+    readonly known: readonly KnownFailure[];
+    readonly new: readonly FailedTest[];
+    readonly trackerProblem: string | null;
+}
+export interface KnownFailureGate {
+    readonly exitCode: number;
+    readonly knownFailures: KnownFailureReport | null;
+}
+export declare function loadBugPolicy(productDir: string): Promise<BugPolicy>;
+export declare function applyKnownFailures(input: KnownFailureGateInput): Promise<KnownFailureGate>;

package/dist/app/bugs/gate.js

@@ -0,0 +1 @@
+import{readBugPolicy as f}from"../../config/loader/bugs.js";import{loadTestkitConfig as m}from"../../config/loader/index.js";import{resolveTracker as p}from"../tracker/index.js";import{assessRun as w}from"./assessment.js";import{readRun as k}from"./run-reading.js";import{resolveBugCheckout as g}from"./store.js";const h=1;export async function loadBugPolicy(o){return f((await m(o)).config)}export async function applyKnownFailures(o){const{env:e,exitCode:r,policy:n,productDir:t}=o,i={exitCode:r,knownFailures:null};if(n.knownFailures!=="allow"||r!==h||o.failedOutsideTests)return i;const l=await g({dir:t,env:e}).catch(()=>null),u=k(o.run,l?.productPath??""),{outcome:s,trackerProblem:d}=await w(u,()=>p({env:e,productDir:t,repository:n.repository??void 0}));if(s.kind!=="tests-failed")return i;const{known:a,new:c}=s.classification;return{exitCode:c.length===0&&a.length>0?0:r,knownFailures:{known:a,new:c,trackerProblem:d}}}

package/dist/app/bugs/guide.d.ts

@@ -0,0 +1,20 @@
+import { type BugCategoryList, type BugSeverity } from "./categories.js";
+import { type BugReport } from "./draft.js";
+export interface BugGuideField {
+    readonly meaning: string;
+    readonly name: string;
+}
+export interface BugGuideSeverity {
+    readonly definition: string;
+    readonly id: BugSeverity;
+}
+export interface BugGuide {
+    readonly categoryLists: readonly BugCategoryList[];
+    readonly example: BugReport;
+    readonly fields: readonly BugGuideField[];
+    readonly rules: readonly string[];
+    readonly severities: readonly BugGuideSeverity[];
+}
+export declare const BUG_RECORD_EXAMPLE: BugReport;
+export declare const BUG_GUIDE: BugGuide;
+export declare function renderBugGuide(guide: BugGuide): string;

package/dist/app/bugs/guide.js

@@ -0,0 +1,8 @@
+import{BUG_CAUSES as s,BUG_CONFIDENCES as c,BUG_ESCAPES as a,BUG_IMPACTS as n,BUG_SEVERITIES as o}from"./categories.js";import{BUG_TITLE_MAX_LENGTH as r}from"./draft.js";import{SEVERITY_LABELS as d}from"./issue.js";export const BUG_RECORD_EXAMPLE={title:"A refund pays out twice when two requests overlap",severity:"high",expected:"A refund pays out once.",authority:'`docs/refunds.md`, "Refunds are idempotent on the order id".',observed:"Two overlapping refund requests for one order both pay out.",rootCause:"`refund()` reads the order's refund state and writes the payout without a lock.",test:{file:"src/checkout/__testkit__/refund.contract.testkit.ts",title:"a refund pays out once"},fault:{path:"src/checkout/refunds.ts",lines:[41,58]},cause:"race-condition",impact:"financial-error",confidence:{cause:"high",impact:"high"},escape:"no-test"};const h=e=>e.map(t=>`\`${t}\``).join(", "),l=e=>`One of ${h(c)}. How sure you are of the ${e}.`,i=e=>({meaning:`One id from the ${e.field} list below.`,name:e.field});export const BUG_GUIDE={fields:[{name:"title",meaning:`One line of at most ${String(r)} characters that names the broken behaviour. It becomes the issue title.`},{name:"severity",meaning:`One of ${h(o)}. The severities are defined below.`},{name:"expected",meaning:"What the product should do."},{name:"authority",meaning:"The source that says so: documentation, a specification, an API contract, or the code's own stated intent. Name it and quote the part that applies."},{name:"observed",meaning:"What the product does instead."},{name:"rootCause",meaning:"Why the code behaves this way."},{name:"test.file",meaning:"The Testkit test file that fails because of the bug."},{name:"test.title",meaning:`The failure's title, or what the script tests when a k6 script fails as a whole. It is one line of at most ${String(r)} characters.`},{name:"fault.path",meaning:"The source file that holds the mistake."},{name:"fault.lines",meaning:"The first and last faulty line as `[start, end]`, counted from 1. One faulty line is `[41, 41]`."},i(s),i(n),{name:"confidence.cause",meaning:l("cause")},{name:"confidence.impact",meaning:l("impact")},i(a)],rules:["Write `test.file` and `fault.path` relative to the product directory, the directory that holds `testkit.config.ts`.","`test.title` must be the failure's title exactly as Testkit reports it. In a Vitest or Playwright file, that is the names of the `describe` blocks around the test and the test's own name, joined by single spaces. In a k6 file, that is the label of the failed assertion. A failed assertion on a response that has no label is named by its request and its default title, such as `GET /health > status is 201`. When `testkit bug file` rejects the title, its message lists the titles a bug record can use.","A k6 script that fails as a whole reports a script failure, which has no title. It fails that way when it calls `fail()`, throws while it runs, fails a k6 `check()`, or fails a threshold. When the script reports no named failure, write in `test.title` what the script tests. Testkit then files one issue that covers the whole file. While that issue is open, every named failure in the file and its script failure are known.","Keep credentials, tokens, and passwords out of the record. Testkit redacts the secrets it recognises before it stores the record and again before it files the issue. It files no bug under a failure's title that holds text it redacts, so rename such a test or label such an assertion first."],example:BUG_RECORD_EXAMPLE,severities:o.map(e=>({definition:d[e].description,id:e})),categoryLists:[s,n,a]};export function renderBugGuide(e){return["## Bug record","Write one JSON object for each confirmed bug. Every field is required.",e.fields.map(t=>`- \`${t.name}\`: ${t.meaning}`).join(`
+`),...e.rules,"Example:",["```json",JSON.stringify(e.example,null,2),"```"].join(`
+`),"## Severities",e.severities.map(t=>`- \`${t.id}\`: ${t.definition}`).join(`
+`),...e.categoryLists.flatMap(u)].join(`
+
+`)}function u(e){return[`## ${e.field.charAt(0).toUpperCase()}${e.field.slice(1)}`,`The \`${e.field}\` field answers one question. ${e.question}`,e.assignmentRule,e.categories.map(t=>[`- \`${t.id}\`: ${t.definition}`,...t.boundaryRule===""?[]:[`  Boundary rule: ${t.boundaryRule}`]].join(`
+`)).join(`
+`)]}

package/dist/app/bugs/issue.d.ts

@@ -0,0 +1,18 @@
+import type { TrackerIssueDraft, TrackerLabel } from "../../tracker/types.js";
+import type { BugSeverity } from "./categories.js";
+import { type BugDraft } from "./draft.js";
+import { type BugFacts, type BugFactsArea } from "./facts.js";
+export declare const TESTKIT_LABEL = "testkit";
+export declare const SEVERITY_LABEL_PREFIX = "severity:";
+export declare const AREA_LABEL_PREFIX = "area:";
+export declare const SEVERITY_LABELS: Readonly<Record<BugSeverity, Omit<TrackerLabel, "name">>>;
+export interface BugFiling {
+    readonly area: BugFactsArea | null;
+    readonly foundAt: string;
+    readonly introducedBy: string | null;
+}
+export declare function bugLabels(severity: BugSeverity, area: BugFactsArea | null): readonly TrackerLabel[];
+export type BugCoverage = "file" | "test";
+export declare function bugFacts(draft: BugDraft, filing: BugFiling, covers?: BugCoverage): BugFacts;
+export declare function renderBugIssue(recorded: BugDraft, filing: BugFiling, covers?: BugCoverage): TrackerIssueDraft;
+export declare function reproduceCommand(draft: Pick<BugDraft, "productPath" | "test">): string;