flakiness 0.213.0 → 0.214.0

package/lib/cli/cli.js

@@ -1023,12 +1023,12 @@ var WireTypes;
 // src/cli/cli.ts
 import { Command, Option } from "commander";
 import debug2 from "debug";
-import path6 from "path";
+import path7 from "path";
 
 // ../package.json
 var package_default = {
   name: "flakiness",
-  version: "0.213.0",
+  version: "0.214.0",
   type: "module",
   private: true,
   scripts: {
@@ -1233,8 +1233,8 @@ var TypedHTTP;
   }
   TypedHTTP2.Router = Router;
   function createClient(base, fetchCallback) {
-    function buildUrl(method, path7, input, options) {
-      const url = new URL(path7.join("/"), base);
+    function buildUrl(method, path8, input, options) {
+      const url = new URL(path8.join("/"), base);
       const signal = options?.signal;
       let body = void 0;
       if (method === "GET" && input)
@@ -1249,8 +1249,8 @@ var TypedHTTP;
         signal
       };
     }
-    function fetcher(method, path7, input, methodOptions) {
-      const options = buildUrl(method, path7, input, methodOptions);
+    function fetcher(method, path8, input, methodOptions) {
+      const options = buildUrl(method, path8, input, methodOptions);
       return fetchCallback(options.url, {
         method: options.method,
         body: options.body,
@@ -1273,17 +1273,17 @@ var TypedHTTP;
         }
       });
     }
-    function createProxy(path7 = []) {
+    function createProxy(path8 = []) {
       return new Proxy({}, {
         get(target, prop) {
           if (typeof prop === "symbol")
             return void 0;
           if (allHttpMethods.includes(prop)) {
-            const f = fetcher.bind(null, prop, path7);
-            f.prepare = buildUrl.bind(null, prop, path7);
+            const f = fetcher.bind(null, prop, path8);
+            f.prepare = buildUrl.bind(null, prop, path8);
             return f;
           }
-          const newPath = [...path7, prop];
+          const newPath = [...path8, prop];
           return createProxy(newPath);
         }
       });
@@ -1919,6 +1919,18 @@ function humanReadableMs(ms) {
   days %= 7;
   return `${weeks} weeks ${days} days ${hours} h`;
 }
+function durationTrendScale(currentMs, baseMs) {
+  const trend = baseMs > 0 ? currentMs / baseMs : 1;
+  if (trend > 1e3)
+    return ">x1000";
+  if (trend > 2)
+    return `x${trend.toFixed(2)}`;
+  if (trend > 1.01)
+    return `+${(trend - 1) * 100 | 0}%`;
+  if (trend > 0.99)
+    return "\xB10%";
+  return `-${(1 - trend) * 100 | 0}%`;
+}
 
 // src/cli/cmd-list-tests.ts
 async function cmdListTests(options) {
@@ -1930,9 +1942,15 @@ async function cmdListTests(options) {
     accessToken: options.accessToken,
     flakinessProject: options.flakinessProject
   });
+  let preset;
+  if (options.pr !== void 0)
+    preset = { type: "pr", number: options.pr };
+  else if (options.branch !== void 0)
+    preset = { type: "branch", name: options.branch };
   const result = await api.report.testStats.POST({
     orgSlug,
     projectSlug,
+    preset,
     pageOptions: {
       // CLI pages are 1-based while backend pages are 0-based.
       number: options.page - 1,
@@ -1945,8 +1963,13 @@ async function cmdListTests(options) {
     historyBuckets: 10,
     fql: options.fql
   });
+  let scope = "default branch";
+  if (options.pr !== void 0)
+    scope = `PR #${options.pr}`;
+  else if (options.branch !== void 0)
+    scope = `branch \`${options.branch}\``;
   const lines = [
-    `# Tests for \`${orgSlug}/${projectSlug}\``,
+    `# Tests for \`${orgSlug}/${projectSlug}\` (${scope})`,
     "",
     `Total tests: **${result.totalElements}**`,
     `Showing page: **${result.pageNumber + 1}/${Math.max(result.totalPages, 1)}** (page size ${result.pageSize})`,
@@ -1963,11 +1986,14 @@ async function cmdListTests(options) {
     const error = extractError(testStats);
     const status = outcomeToStatus(testStats.outcome);
     const duration = humanReadableMs(testStats.durationMs);
+    const trend = formatDurationTrend(testStats.durationChangeMs, testStats.durationMs);
     const flipRate = formatFlipRate(testStats.flipRate);
+    const env = formatEnv(testStats.env);
     lines.push(`### ${index + 1}. ${fullName}`);
     lines.push(`- Location: ${asInlineCode(location)}`);
+    lines.push(`- Env: ${asInlineCode(env)}`);
     lines.push(`- Status: ${asInlineCode(status)}`);
-    lines.push(`- Duration: ${asInlineCode(duration)}`);
+    lines.push(`- Duration: ${asInlineCode(duration)} (${asInlineCode(trend)})`);
     lines.push(`- Flip Rate: ${asInlineCode(flipRate)}`);
     lines.push(`- Error: ${error ? asInlineCode(error) : "None"}`);
     lines.push("");
@@ -1994,6 +2020,27 @@ function outcomeToStatus(outcome) {
       return "flaked";
   }
 }
+function formatEnv(env) {
+  const parts = [];
+  const { systemData } = env;
+  if (systemData.osName)
+    parts.push(`os=${systemData.osName}`);
+  if (systemData.osVersion)
+    parts.push(`osVersion=${systemData.osVersion}`);
+  if (systemData.osArch)
+    parts.push(`arch=${systemData.osArch}`);
+  if (env.userSuppliedData) {
+    for (const [key, value] of Object.entries(env.userSuppliedData))
+      parts.push(`${key}=${value}`);
+  }
+  return parts.join(", ") || env.name || "unknown";
+}
+function formatDurationTrend(changeMs, currentMs) {
+  const baseMs = currentMs - changeMs;
+  const scale = durationTrendScale(currentMs, baseMs);
+  const sign = changeMs >= 0 ? "+" : "-";
+  return `${sign}${humanReadableMs(Math.abs(changeMs))} (${scale})`;
+}
 function formatFlipRate(flipRate) {
   if (flipRate === void 0 || !Number.isFinite(flipRate))
     return "n/a";
@@ -2005,14 +2052,65 @@ function asInlineCode(text) {
   return `${ticks}${normalized}${ticks}`;
 }
 
+// src/cli/cmd-skills-install.ts
+import { execSync } from "child_process";
+import chalk3 from "chalk";
+import fs5 from "fs";
+import path5 from "path";
+
+// src/generated/bundledSkillsData.ts
+var BUNDLED_SKILLS = [
+  {
+    "name": "flakiness-investigation",
+    "description": "Use when querying Flakiness.io test data, writing FQL filters, finding flaky or failed tests, investigating regressions, analyzing test health, or fixing PR test failures with the flakiness CLI.",
+    "files": [
+      {
+        "path": "SKILL.md",
+        "content": "---\nname: flakiness-investigation\ndescription: Use when querying Flakiness.io test data, writing FQL filters, finding flaky or failed tests, investigating regressions, analyzing test health, or fixing PR test failures with the flakiness CLI.\n---\n\n# Flakiness Investigation\n\nUse the `flakiness` CLI to query and analyze test data from Flakiness.io.\n\n## Prerequisites\n\n- The `flakiness` CLI must be available in PATH (installed via `npm install -g flakiness` or `npx flakiness`).\n- A project must be specified: `--project <org/project>` or set `FLAKINESS_PROJECT` env var.\n- Authentication: run `flakiness auth login` first, or set `FLAKINESS_ACCESS_TOKEN`.\n\n## Core command\n\n```bash\nflakiness list tests --project <org/project> [--pr <number>] [--branch <name>] [--fql <query>] [--sort <axis>] [--sort-dir asc|desc] [--page <n>] [--page-size <n>]\n```\n\n### Scope options\n\n| Flag | Effect |\n|------|--------|\n| *(none)* | Default branch \u2014 tests from the day of its head commit (in the project's timezone) |\n| `--pr <number>` | Pull request \u2014 tests from the merge-commit of the PR branch into the target branch |\n| `--branch <name>` | Named branch \u2014 tests from the day of its head commit (in the project's timezone) |\n\n`--pr` and `--branch` are mutually exclusive.\n\n### Status semantics with `--pr`\n\nWhen querying a PR, the status values have specific meaning:\n\n| Status | Meaning |\n|--------|---------|\n| `regressed` | Test was passing on the target branch but fails in this PR \u2014 **caused by the PR** |\n| `failed` | Test also fails on the target branch \u2014 **pre-existing failure, not caused by the PR** |\n| `flaked` | Test failed but passed on retry \u2014 flaky, not a real failure |\n| `passed` | Test passes |\n| `skipped` | Test was skipped |\n\n### Sort axes\n\n- `outcome` \u2014 sort by test outcome severity (default)\n- `flip_rate` \u2014 sort by flip rate (how often a test flips between pass/fail)\n- `duration` \u2014 sort by test duration\n- `duration_trend` \u2014 sort by duration trend\n- `name` \u2014 sort alphabetically\n\n### Common queries\n\n| Goal | FQL + flags |\n|------|-------------|\n| Flaky tests | `--fql 'flip>0%' --sort flip_rate --sort-dir desc` |\n| Very flaky (>50%) | `--fql 'flip>50%' --sort flip_rate --sort-dir desc` |\n| Failed tests | `--fql 's:failed' --sort outcome --sort-dir desc` |\n| Regressions | `--fql 's:regressed'` |\n| All broken tests | `--fql 'status:(failed, regressed)'` |\n| Slow tests | `--fql 'd>5s' --sort duration --sort-dir desc` |\n| Tests matching error | `--fql '$timeout'` |\n| Tests in file | `--fql 'f:login.spec.ts'` |\n\n## FQL (Filter Query Language)\n\nFull reference: [references/fql.md](references/fql.md)\n\nKey rules:\n- Multiple tokens combine with AND: `s:failed f:e2e` means \"failed AND in e2e files\"\n- Prefix with `-` to exclude: `-#smoke` excludes smoke-tagged tests\n- Same filter type uses OR: `status:(failed, regressed)` means \"failed OR regressed\"\n- Quote values with spaces: `f:'tests/e2e checkout'`\n\n### Filter types\n\n| Filter | Syntax | Example |\n|--------|--------|---------|\n| Text search | `<text>` | `login` |\n| Status | `s:<status>` | `s:failed`, `status:(failed, regressed)` |\n| File | `f:<path>` | `f:login.spec.ts` |\n| Error | `$<text>` | `$timeout` |\n| Tag | `#<tag>` | `#smoke` |\n| Duration | `d><time>` | `d>2s`, `d<=500ms` |\n| Flip rate | `flip><pct>` | `flip>0%`, `fr>50%` |\n| Annotation | `@<type>` | `@skip` |\n\n### Status values\n\n`passed`, `failed`, `flaked`, `skipped`, `regressed`\n\n## Workflow: Fix My PR Tests\n\nWhen a user asks to fix failing tests in a PR, follow these steps:\n\n1. **Fetch regressions from the PR:**\n   ```bash\n   flakiness list tests --project <org/project> --pr <number> --fql 's:regressed' --page-size 50\n   ```\n   Tests with status `regressed` were passing on the target branch but fail in this PR \u2014 these are **caused by the PR** and must be fixed.\n\n2. **Analyze the output:** Look at the error messages, file paths, and test names to understand what the PR broke.\n\n3. **Fix the regressions** by reading the reported file paths and error messages, then making targeted code changes.\n\n4. **Optionally check pre-existing failures:** Tests with status `failed` also fail on the target branch and are not caused by the PR. You can list them with:\n   ```bash\n   flakiness list tests --project <org/project> --pr <number> --fql 's:failed' --page-size 50\n   ```\n   These are informational \u2014 fixing them is a bonus, not a requirement.\n\n5. **Ignore flakes:** Tests with status `flaked` failed but passed on retry \u2014 they are flaky and not actionable in the context of a PR fix.\n\n## Workflow tips\n\n1. Start broad: `flakiness list tests --project <org/project> --page-size 20`\n2. Filter down with FQL based on what you're investigating\n3. Use `--page-size 50` or higher to see more results at once\n4. Combine filters: `--fql 's:failed $timeout f:e2e -#smoke'`\n\nMore recipes: [references/recipes.md](references/recipes.md)\n"
+      },
+      {
+        "path": "references/fql.md",
+        "content": "# FQL Reference\n\nUse `--fql <query>` with `flakiness list tests` to filter tests before sorting and pagination.\n\n## Core rules\n\n- Multiple tokens are combined with `AND` by default.\n- Prefix a token with `-` to exclude matches.\n- Use quotes for spaces or special characters.\n- Repeating the same filter type behaves like `OR`, for example `status:(failed, regressed)`.\n\n## Filter types\n\n| Filter | Examples | Meaning |\n| --- | --- | --- |\n| Test text | `login` | Matches test titles, suite titles, and file path |\n| Status | `s:failed`, `status:(failed, regressed)` | Matches test status |\n| File | `f:login.spec.ts`, `file:e2e` | Matches file path |\n| Line | `:120`, `>200`, `<=50` | Matches line number |\n| Error | `$timeout`, `$'network error'` | Matches error text |\n| Annotation | `@skip`, `@'fails in ci'` | Matches annotation type or description |\n| Tag | `#smoke` | Matches test tags |\n| Duration | `d>2s`, `duration<=500ms` | Matches duration |\n| Flip rate | `flip>0%`, `fr<=50%` | Matches flip rate |\n\n## Status values\n\n- `passed`\n- `failed`\n- `flaked`\n- `skipped`\n- `regressed`\n\n## Examples\n\n```fql\ns:flaked\nstatus:(failed, regressed)\nf:e2e d>5s\nflip>0%\nfr>50%\n$timeout -#smoke\n@skip f:'tests/e2e checkout'\n```\n\n## Quoting\n\n- Single and double quotes both work.\n- Escape quotes and backslashes with JSON-style escaping.\n\n```fql\nf:'some path'\n$\"say \\\"hello\\\"\"\n'can\\'t touch this'\n```\n\n## Flip rate\n\nValues are percentages (0\u2013100). The `%` suffix is optional.\n\n```fql\nflip>0%\nflip>=50\nfr<=25%\n```\n\n## Duration units\n\n- `ns`\n- `ms` or no suffix\n- `s`\n- `m`\n- `h`\n\n```fql\nd>100\nd>1.5s\nduration<=2m\n```\n"
+      },
+      {
+        "path": "references/recipes.md",
+        "content": "# Investigation Recipes\n\nReplace `myorg/myproject` with the target project slug, or set `FLAKINESS_PROJECT`.\n\n## Query a pull request\n\nShows tests from the merge-commit of the PR branch into the target branch.\n\n```bash\nflakiness list tests --project myorg/myproject --pr 42\n```\n\n## Find PR regressions (caused by the PR)\n\nTests marked `regressed` were passing on the target branch but fail in this PR.\n\n```bash\nflakiness list tests --project myorg/myproject --pr 42 --fql 's:regressed'\n```\n\n## Find pre-existing failures in a PR\n\nTests marked `failed` also fail on the target branch \u2014 not caused by the PR.\n\n```bash\nflakiness list tests --project myorg/myproject --pr 42 --fql 's:failed'\n```\n\n## Find all broken tests in a PR\n\nIncludes both PR-caused regressions and pre-existing failures.\n\n```bash\nflakiness list tests --project myorg/myproject --pr 42 --fql 'status:(failed, regressed)' --page-size 50\n```\n\n## Query a specific branch\n\nShows tests from the day of the branch's head commit, in the project's timezone.\n\n```bash\nflakiness list tests --project myorg/myproject --branch feature/login\n```\n\n## Find all flaky tests\n\n```bash\nflakiness list tests --project myorg/myproject --fql 'flip>0%' --sort flip_rate --sort-dir desc\n```\n\n## Find the most flaky tests (>50% flip rate)\n\n```bash\nflakiness list tests --project myorg/myproject --fql 'flip>50%' --sort flip_rate --sort-dir desc\n```\n\n## Show failed tests\n\n```bash\nflakiness list tests --project myorg/myproject --fql 's:failed' --sort outcome --sort-dir desc\n```\n\n## Show regressions\n\n```bash\nflakiness list tests --project myorg/myproject --fql 's:regressed'\n```\n\n## Show all currently broken tests\n\n```bash\nflakiness list tests --project myorg/myproject --fql 'status:(failed, regressed)' --sort outcome --sort-dir desc\n```\n\n## Show flaked tests (passed on retry)\n\n```bash\nflakiness list tests --project myorg/myproject --fql 's:flaked' --sort flip_rate --sort-dir desc\n```\n\n## Find slow tests\n\n```bash\nflakiness list tests --project myorg/myproject --fql 'd>5s' --sort duration --sort-dir desc\n```\n\n## Find tests by error text\n\n```bash\nflakiness list tests --project myorg/myproject --fql '$timeout'\nflakiness list tests --project myorg/myproject --fql '$\"network error\"'\n```\n\n## Combine filters\n\nFailed tests in e2e files, excluding smoke-tagged tests:\n\n```bash\nflakiness list tests --project myorg/myproject --fql 's:failed f:e2e -#smoke'\n```\n\n## Narrow to a specific file\n\n```bash\nflakiness list tests --project myorg/myproject --fql 'f:login.spec.ts'\n```\n\n## Find tests with a specific annotation\n\n```bash\nflakiness list tests --project myorg/myproject --fql '@skip'\nflakiness list tests --project myorg/myproject --fql '@fixme'\n```\n"
+      }
+    ]
+  }
+];
+
+// src/cli/cmd-skills-install.ts
+var AGENTS = ["claude", "codex", "cursor"];
+function projectRoot() {
+  try {
+    return execSync("git rev-parse --show-toplevel", { encoding: "utf-8" }).trim();
+  } catch {
+    return process.cwd();
+  }
+}
+async function cmdSkillsInstall(options) {
+  for (const skill of BUNDLED_SKILLS) {
+    const dest = path5.join(projectRoot(), `.${options.agent}`, "skills", skill.name);
+    await fs5.promises.rm(dest, { recursive: true, force: true });
+    for (const file of skill.files) {
+      const filePath = path5.join(dest, file.path);
+      await fs5.promises.mkdir(path5.dirname(filePath), { recursive: true });
+      await fs5.promises.writeFile(filePath, file.content);
+    }
+    console.log(`${chalk3.green("\u2713")} Installed ${chalk3.bold(skill.name)} \u2192 ${chalk3.dim(dest)}`);
+  }
+  console.log("\nRestart your agent to pick up new skills.");
+}
+
 // src/cli/cmd-upload.ts
 import { readReport, uploadReport } from "@flakiness/sdk";
-import chalk3 from "chalk";
-import fs5 from "fs/promises";
+import chalk4 from "chalk";
+import fs6 from "fs/promises";
 import ora2 from "ora";
-import path5 from "path";
-var warn = (txt) => console.warn(chalk3.yellow(`[flakiness.io] WARN: ${txt}`));
-var err = (txt) => console.error(chalk3.red(`[flakiness.io] Error: ${txt}`));
+import path6 from "path";
+var warn = (txt) => console.warn(chalk4.yellow(`[flakiness.io] WARN: ${txt}`));
+var err = (txt) => console.error(chalk4.red(`[flakiness.io] Error: ${txt}`));
 async function cmdUpload(relativePaths, options) {
   const total = relativePaths.length;
   const spinner = total > 1 ? ora2("Uploading reports:").start() : void 0;
@@ -2029,14 +2127,14 @@ async function cmdUpload(relativePaths, options) {
     return sharedAuth;
   }
   for (const relativePath of relativePaths) {
-    const fullPath = path5.resolve(relativePath);
-    const stat = await fs5.stat(fullPath).catch(() => void 0);
+    const fullPath = path6.resolve(relativePath);
+    const stat = await fs6.stat(fullPath).catch(() => void 0);
     if (!stat) {
       spinner?.stop();
       err(`Path ${fullPath} is not accessible!`);
       process.exit(1);
     }
-    const reportDir = stat.isDirectory() ? fullPath : path5.dirname(fullPath);
+    const reportDir = stat.isDirectory() ? fullPath : path6.dirname(fullPath);
     const { report, attachments, missingAttachments } = await readReport(reportDir);
     if (missingAttachments.length)
       warn(`Missing ${missingAttachments.length} attachments`);
@@ -2099,13 +2197,22 @@ var optTestsPageSize = new Option("--page-size <page-size>", "Number of tests pe
   return parsed;
 }).default(20);
 var optTestsFQL = new Option("--fql <query>", "Filter Query Language (FQL) expression for tests. Learn about FQL: https://flakiness.io/docs/concepts/fql/");
-list.command("tests").description("query tests data").addOption(optTestsPage).addOption(optTestsPageSize).addOption(optTestsSort).addOption(optTestsSortDir).addOption(optTestsFQL).addOption(mustFlakinessProject).addOption(optEndpoint).addOption(optAccessToken).action(async (options) => runCommand(async () => {
+var optTestsPR = new Option("--pr <number>", "Show tests from a specific pull request").argParser((value) => {
+  const parsed = parseInt(value, 10);
+  if (isNaN(parsed) || parsed < 1)
+    throw new Error("PR number must be a positive integer");
+  return parsed;
+}).conflicts("branch");
+var optTestsBranch = new Option("--branch <name>", "Show tests from a specific branch").conflicts("pr");
+list.command("tests").description("Query tests data. Defaults to the default branch (day of head commit in project timezone). Use --pr for PR merge-commit tests, or --branch for a named branch.").addOption(optTestsPage).addOption(optTestsPageSize).addOption(optTestsSort).addOption(optTestsSortDir).addOption(optTestsFQL).addOption(optTestsPR).addOption(optTestsBranch).addOption(mustFlakinessProject).addOption(optEndpoint).addOption(optAccessToken).action(async (options) => runCommand(async () => {
   await cmdListTests({
     page: options.page,
     pageSize: options.pageSize,
     sort: options.sort,
     sortDir: options.sortDir,
     fql: options.fql,
+    pr: options.pr,
+    branch: options.branch,
     endpoint: options.endpoint,
     accessToken: options.accessToken,
     flakinessProject: options.project
@@ -2121,6 +2228,11 @@ auth.command("logout").description("Logout from current session").action(async (
 auth.command("whoami").description("Show current logged in user information").action(async () => runCommand(async () => {
   await cmdAuthWhoami();
 }));
+var skills = program.command("skills").description("Manage agent skills");
+var optAgent = new Option("--agent <agent>", "Target agent").choices(AGENTS).makeOptionMandatory();
+skills.command("install").description("Install bundled skills into the project").addOption(optAgent).action(async (options) => runCommand(async () => {
+  await cmdSkillsInstall({ agent: options.agent });
+}));
 program.command("access").description("Check access to a Flakiness.io project").addOption(mustFlakinessProject).addOption(optAccessToken).addOption(optEndpoint).option("--json", "Output result as JSON").option("-q, --quiet", "Suppress output, only set exit code").action(async (options) => runCommand(async () => {
   await cmdAccess({
     flakinessProject: options.project,
@@ -2172,7 +2284,7 @@ program.command("upload").description("Upload Flakiness report to the flakiness.
 });
 var optViewerUrl = new Option("-e, --viewer-url <url>", "A URL where report viewer is deployed").default("https://report.flakiness.io").env("FLAKINESS_ENDPOINT");
 program.command("show").description("Show flakiness report").argument("[relative-path]", "Path to the Flakiness report file or folder that contains `report.json`. (default: flakiness-report)").addOption(optViewerUrl).action(async (arg, options) => runCommand(async () => {
-  const dir = path6.resolve(arg ?? "flakiness-report");
+  const dir = path7.resolve(arg ?? "flakiness-report");
   await showReport(dir, {
     reportViewerUrl: options.viewerUrl
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flakiness",
-  "version": "0.213.0",
+  "version": "0.214.0",
   "private": false,
   "bin": {
     "flakiness": "./lib/cli/cli.js"
@@ -22,8 +22,8 @@
     "@playwright/test": "^1.57.0",
     "@types/debug": "^4.1.12",
     "@types/express": "^4.17.20",
-    "@flakiness/shared": "0.213.0",
-    "@flakiness/server": "0.213.0"
+    "@flakiness/shared": "0.214.0",
+    "@flakiness/server": "0.214.0"
   },
   "dependencies": {
     "@flakiness/flakiness-report": "^0.28.0",