executable-stories-e2e 0.0.0-stage → 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,76 @@
+import { Reporter, Report } from 'e2e';
+import { RawRun } from 'executable-stories-core/types/raw';
+
+/**
+ * Step markers for e2e tests.
+ *
+ * A marker records where and when the test called it. The reporter groups
+ * the e2e steps that ran after a marker under the marker's text:
+ *
+ * ```ts
+ * test('incrementing shows the new count', async ({ app, screen }) => {
+ *   story.given('the counter page is open');
+ *   await app.open('/');
+ *   story.when('the user presses Increment twice');
+ *   await screen.getByRole('button', { name: 'Increment' }).click();
+ *   await screen.getByRole('button', { name: 'Increment' }).click();
+ *   story.then('the count reads 2');
+ *   await expect(screen.getByRole('status')).toHaveText('2');
+ * });
+ * ```
+ */
+type MarkerKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';
+/** One line of a markers file. */
+interface Marker {
+    keyword: MarkerKeyword;
+    text: string;
+    /** Absolute path of the test file that called the marker. */
+    file: string;
+    line: number;
+    /** Orders a marker before a step on its own line; absent reads as the line's start. */
+    column?: number;
+    /** Epoch milliseconds. */
+    at: number;
+}
+/** Step markers. Call before the e2e steps the marker describes; they render under its text. */
+declare const story: {
+    given: (text: string) => void;
+    when: (text: string) => void;
+    then: (text: string) => void;
+    and: (text: string) => void;
+    but: (text: string) => void;
+};
+
+/** Step text for one non-agent step. */
+declare function phraseStep(api: string, label: string): string;
+
+/**
+ * executable-stories reporter for the `e2e` agentic test runner.
+ *
+ * The reporter reads the finished `report-1` document and writes the RawRun
+ * JSON every other adapter writes. `executable-stories format` takes it from
+ * there. Optional `story.given/when/then` markers in a test name the steps in
+ * the test's own words; without them, step text is phrased from e2e's record.
+ */
+
+interface StoryReporterOptions {
+    /** Where to write the RawRun JSON, relative to the project root. Default: `.executable-stories/raw-run.json`, where `executable-stories format` looks first. */
+    rawRunPath?: string;
+}
+/**
+ * Map an e2e `report-1` document to a RawRun. Pure: artifact paths resolve
+ * against `artifactsRoot` and nothing is read from disk.
+ */
+declare function toRawRun(args: {
+    report: Report;
+    projectRoot: string;
+    artifactsRoot: string;
+    /** Set when the run was `--last-failed`: it then covered only part of the suite. */
+    lastRun?: Report;
+    /** Step markers the run's tests recorded with `story.given/when/then`. */
+    markers?: readonly Marker[];
+}): RawRun;
+/** An e2e reporter that writes the run as RawRun JSON for the executable-stories CLI. */
+declare function storyReporter(options?: StoryReporterOptions): Reporter;
+
+export { type Marker, type MarkerKeyword, type StoryReporterOptions, phraseStep, story, storyReporter, toRawRun };

package/dist/index.d.ts

@@ -0,0 +1,76 @@
+import { Reporter, Report } from 'e2e';
+import { RawRun } from 'executable-stories-core/types/raw';
+
+/**
+ * Step markers for e2e tests.
+ *
+ * A marker records where and when the test called it. The reporter groups
+ * the e2e steps that ran after a marker under the marker's text:
+ *
+ * ```ts
+ * test('incrementing shows the new count', async ({ app, screen }) => {
+ *   story.given('the counter page is open');
+ *   await app.open('/');
+ *   story.when('the user presses Increment twice');
+ *   await screen.getByRole('button', { name: 'Increment' }).click();
+ *   await screen.getByRole('button', { name: 'Increment' }).click();
+ *   story.then('the count reads 2');
+ *   await expect(screen.getByRole('status')).toHaveText('2');
+ * });
+ * ```
+ */
+type MarkerKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';
+/** One line of a markers file. */
+interface Marker {
+    keyword: MarkerKeyword;
+    text: string;
+    /** Absolute path of the test file that called the marker. */
+    file: string;
+    line: number;
+    /** Orders a marker before a step on its own line; absent reads as the line's start. */
+    column?: number;
+    /** Epoch milliseconds. */
+    at: number;
+}
+/** Step markers. Call before the e2e steps the marker describes; they render under its text. */
+declare const story: {
+    given: (text: string) => void;
+    when: (text: string) => void;
+    then: (text: string) => void;
+    and: (text: string) => void;
+    but: (text: string) => void;
+};
+
+/** Step text for one non-agent step. */
+declare function phraseStep(api: string, label: string): string;
+
+/**
+ * executable-stories reporter for the `e2e` agentic test runner.
+ *
+ * The reporter reads the finished `report-1` document and writes the RawRun
+ * JSON every other adapter writes. `executable-stories format` takes it from
+ * there. Optional `story.given/when/then` markers in a test name the steps in
+ * the test's own words; without them, step text is phrased from e2e's record.
+ */
+
+interface StoryReporterOptions {
+    /** Where to write the RawRun JSON, relative to the project root. Default: `.executable-stories/raw-run.json`, where `executable-stories format` looks first. */
+    rawRunPath?: string;
+}
+/**
+ * Map an e2e `report-1` document to a RawRun. Pure: artifact paths resolve
+ * against `artifactsRoot` and nothing is read from disk.
+ */
+declare function toRawRun(args: {
+    report: Report;
+    projectRoot: string;
+    artifactsRoot: string;
+    /** Set when the run was `--last-failed`: it then covered only part of the suite. */
+    lastRun?: Report;
+    /** Step markers the run's tests recorded with `story.given/when/then`. */
+    markers?: readonly Marker[];
+}): RawRun;
+/** An e2e reporter that writes the run as RawRun JSON for the executable-stories CLI. */
+declare function storyReporter(options?: StoryReporterOptions): Reporter;
+
+export { type Marker, type MarkerKeyword, type StoryReporterOptions, phraseStep, story, storyReporter, toRawRun };

package/dist/index.js

@@ -0,0 +1,504 @@
+// src/index.ts
+import { rmSync } from "fs";
+import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
+import * as path2 from "path";
+
+// src/phrase.ts
+var ROLE_NOUNS = {
+  textbox: "field",
+  searchbox: "search field",
+  combobox: "dropdown",
+  spinbutton: "number field",
+  img: "image"
+};
+var SIMPLE_QUERY = /^getBy(\w+)\(("(?:[^"\\]|\\.)*")(?:, name: ("(?:[^"\\]|\\.)*"))?\)$/;
+function unquote(json) {
+  try {
+    return JSON.parse(json);
+  } catch {
+    return json;
+  }
+}
+function describeTarget(label) {
+  const match = SIMPLE_QUERY.exec(label);
+  if (!match) return void 0;
+  const [, kind, rawValue, rawName] = match;
+  const value = unquote(rawValue);
+  const name = rawName === void 0 ? void 0 : unquote(rawName);
+  switch (kind) {
+    case "Role": {
+      const noun = ROLE_NOUNS[value] ?? value;
+      return name === void 0 ? `the ${noun}` : `the "${name}" ${noun}`;
+    }
+    case "Text":
+      return `the text "${value}"`;
+    case "Label":
+    case "Placeholder":
+      return `the "${value}" field`;
+    case "AltText":
+      return `the "${value}" image`;
+    case "TestId":
+    case "Title":
+      return `the "${value}" element`;
+    default:
+      return void 0;
+  }
+}
+var ACTIONS = {
+  click: "click",
+  tap: "tap",
+  dblclick: "double-click",
+  hover: "hover over",
+  check: "check",
+  uncheck: "uncheck",
+  fill: "fill in",
+  clear: "clear",
+  press: "press a key on",
+  pressSequentially: "type into",
+  selectOption: "choose an option in",
+  longPress: "long-press",
+  swipe: "swipe",
+  focus: "focus"
+};
+var CLAIMS = {
+  toBeVisible: ["is visible", "is not visible"],
+  toBeHidden: ["is hidden", "is not hidden"],
+  toBeEnabled: ["is enabled", "is not enabled"],
+  toBeDisabled: ["is disabled", "is not disabled"],
+  toBeChecked: ["is checked", "is not checked"],
+  toBeFocused: ["is focused", "is not focused"],
+  toBeSelected: ["is selected", "is not selected"],
+  toBeExpanded: ["is expanded", "is not expanded"],
+  toBeAttached: ["is on the page", "is not on the page"],
+  toHaveText: ["shows the expected text", "does not show the given text"],
+  toContainText: [
+    "contains the expected text",
+    "does not contain the given text"
+  ],
+  toHaveValue: ["has the expected value", "does not have the given value"],
+  toHaveCount: [
+    "appears the expected number of times",
+    "does not appear the given number of times"
+  ],
+  toHaveAccessibleName: [
+    "has the expected accessible name",
+    "does not have the given accessible name"
+  ],
+  toHaveAttribute: [
+    "has the expected attribute",
+    "does not have the given attribute"
+  ]
+};
+var APP = {
+  "app.back": "go back",
+  "app.restart": "restart the app",
+  "app.clearState": "clear the app state",
+  "session.launch": "launch the app session",
+  "session.close": "close the app session"
+};
+function phraseStep(api, label) {
+  const raw = label ? `${api} ${label}` : api;
+  if (api === "app.open") return label ? `open ${label}` : "open the app";
+  if (api === "app.screenshot")
+    return label ? `take a screenshot: ${label}` : "take a screenshot";
+  if (api === "session.save")
+    return label ? `save the session as ${label}` : "save the session";
+  if (APP[api]) return APP[api];
+  const action = /^locator\.(\w+)$/.exec(api)?.[1];
+  if (action !== void 0) {
+    if (action === "waitFor") {
+      const [target, state] = label.split(" \u2192 ");
+      const noun2 = target === void 0 ? void 0 : describeTarget(target);
+      return noun2 && state ? `wait until ${noun2} is ${state}` : raw;
+    }
+    const verb = ACTIONS[action];
+    const noun = describeTarget(label);
+    return verb && noun ? `${verb} ${noun}` : raw;
+  }
+  const claim = /^expect\.(not\.)?(\w+)$/.exec(api);
+  if (claim) {
+    const words = CLAIMS[claim[2]];
+    const noun = describeTarget(label);
+    return words && noun ? `${noun} ${words[claim[1] ? 1 : 0]}` : raw;
+  }
+  return raw;
+}
+
+// src/story.ts
+import { appendFileSync, mkdirSync } from "fs";
+import * as path from "path";
+import { fileURLToPath } from "url";
+var MARKERS_DIR = ".executable-stories/e2e-markers";
+var FRAME = /\(?((?:file:\/\/)?[^\s()]+?):(\d+):(\d+)\)?$/;
+function callSite() {
+  const frames = (new Error().stack ?? "").split("\n").slice(1);
+  let own;
+  for (const frame of frames) {
+    const match = FRAME.exec(frame.trim());
+    if (!match) continue;
+    const file = match[1].startsWith("file://") ? fileURLToPath(match[1]) : match[1];
+    own ??= file;
+    if (file !== own)
+      return { file, line: Number(match[2]), column: Number(match[3]) };
+  }
+  return void 0;
+}
+function mark(keyword, text) {
+  const site = callSite();
+  if (site === void 0) return;
+  const marker = { keyword, text, ...site, at: Date.now() };
+  const dir = path.resolve(MARKERS_DIR);
+  mkdirSync(dir, { recursive: true });
+  appendFileSync(
+    path.join(dir, `${process.pid}.jsonl`),
+    `${JSON.stringify(marker)}
+`
+  );
+}
+var story = {
+  given: (text) => mark("Given", text),
+  when: (text) => mark("When", text),
+  then: (text) => mark("Then", text),
+  and: (text) => mark("And", text),
+  but: (text) => mark("But", text)
+};
+
+// src/index.ts
+function unknownValue(_value, fallback) {
+  return fallback;
+}
+function resultStatus(status) {
+  switch (status) {
+    case "passed":
+    case "flaky":
+      return "pass";
+    case "failed":
+      return "fail";
+    case "skipped":
+      return "skip";
+    case "timed-out":
+      return "timeout";
+    case "interrupted":
+      return "interrupted";
+    default:
+      return unknownValue(status, "unknown");
+  }
+}
+function stepStatus(status) {
+  switch (status) {
+    case "passed":
+      return "pass";
+    // Blocked and timed-out steps stopped the scenario, so they read as fail;
+    // the error message keeps e2e's code.
+    case "failed":
+    case "blocked":
+    case "timed-out":
+      return "fail";
+    // A cancelled step did not finish, so it reads as not run.
+    case "cancelled":
+      return "skip";
+    default:
+      return unknownValue(status, "unknown");
+  }
+}
+function primaryKeyword(step) {
+  switch (step.kind) {
+    case "app":
+    case "session":
+    case "resource":
+      return "Given";
+    case "locator":
+    case "screen":
+      return "When";
+    case "assertion":
+      return "Then";
+    case "agent":
+      return step.api === "agent.assert" ? "Then" : "When";
+    default:
+      return unknownValue(step.kind, "When");
+  }
+}
+function stepText(step) {
+  if (step.kind === "agent" && step.label) return step.label;
+  return phraseStep(step.api, step.label);
+}
+function errorMessage(error) {
+  return `${error.code}: ${error.message}`;
+}
+function toGroups(steps, markers, projectRoot) {
+  const groups = [];
+  let pending = 0;
+  let current;
+  const before = (m, step) => {
+    if (step === void 0) return true;
+    const at = Date.parse(step.startedAt);
+    if (m.at !== at) return m.at < at;
+    if (m.file !== path2.resolve(projectRoot, step.source.file)) return true;
+    return m.line < step.source.line || m.line === step.source.line && (m.column ?? 0) < step.source.column;
+  };
+  const takeMarkersBefore = (step) => {
+    while (pending < markers.length && before(markers[pending], step)) {
+      const m = markers[pending++];
+      current = {
+        keyword: m.keyword,
+        text: m.text,
+        marker: true,
+        children: []
+      };
+      groups.push(current);
+    }
+  };
+  for (const step of steps) {
+    takeMarkersBefore(step);
+    if (current) current.children.push(step);
+    else
+      groups.push({
+        keyword: primaryKeyword(step),
+        text: stepText(step),
+        marker: false,
+        children: [step]
+      });
+  }
+  takeMarkersBefore(void 0);
+  return groups;
+}
+function isObservedClaim(step) {
+  return primaryKeyword(step) === "Then" && (step.status === "passed" || step.status === "failed");
+}
+function groupStatus(group) {
+  const statuses = group.children.map((c) => stepStatus(c.status));
+  for (const worst of ["fail", "unknown", "skip"])
+    if (statuses.includes(worst)) return worst;
+  return "pass";
+}
+function agentExplanation(step) {
+  const why = step.explanation?.trim();
+  if (!why || step.error?.message.includes(why)) return [];
+  return [{ kind: "note", text: `Agent: ${why}`, phase: "runtime" }];
+}
+function toSteps(groups) {
+  const out = [];
+  groups.forEach((group, index) => {
+    const repeat = group.keyword !== "And" && group.keyword !== "But" && out.some((s) => s.keyword === group.keyword);
+    const keyword = repeat ? "And" : group.keyword;
+    const claims = group.children.filter(isObservedClaim).length;
+    const assertions = claims > 0 ? { assertions: claims } : {};
+    const docs = group.children.flatMap((c) => [
+      ...group.marker ? [
+        {
+          kind: "note",
+          text: stepText(c),
+          phase: "runtime"
+        }
+      ] : [],
+      ...agentExplanation(c)
+    ]);
+    out.push({
+      id: `step-${index}`,
+      keyword,
+      text: group.text,
+      durationMs: group.children.reduce((sum, c) => sum + c.durationMs, 0),
+      ...assertions,
+      ...docs.length > 0 ? { docs } : {}
+    });
+  });
+  return out;
+}
+function toMedia(attempt, groups, artifactsRoot) {
+  const attachments = [];
+  const docs = [];
+  for (const artifact of attempt.artifacts) {
+    if (artifact.path === void 0) {
+      if (artifact.kind === "video" && artifact.url !== void 0 && /^https?:\/\//.test(artifact.url)) {
+        docs.push({ kind: "video", path: artifact.url, phase: "runtime" });
+      }
+      continue;
+    }
+    const stepIndex = groups.findIndex(
+      (group) => group.children.some(
+        (step) => step.artifacts.includes(artifact.id) || artifact.producer?.kind === "step" && artifact.producer.stepId === step.id
+      )
+    );
+    attachments.push({
+      name: artifact.kind,
+      mediaType: artifact.mediaType,
+      path: path2.join(artifactsRoot, artifact.path),
+      fileName: path2.basename(artifact.path),
+      ...artifact.size !== void 0 ? { byteLength: artifact.size } : {},
+      ...stepIndex >= 0 ? { stepIndex, stepId: `step-${stepIndex}` } : {}
+    });
+  }
+  return { attachments, docs };
+}
+function finalAttempt(result, report) {
+  if (result.serialGroupId === void 0) {
+    return { attempt: result.attempts.at(-1), count: result.attempts.length };
+  }
+  const attempts = report.run.serialGroups.find((g) => g.id === result.serialGroupId)?.attempts ?? [];
+  const last = attempts.at(-1);
+  const member = last?.members.find((m) => m.testId === result.testId);
+  if (last === void 0) return { count: 0 };
+  return {
+    attempt: {
+      error: member?.error ?? last.error,
+      startedAt: member?.startedAt ?? last.startedAt,
+      durationMs: member?.durationMs ?? last.durationMs,
+      artifacts: last.artifacts,
+      steps: member?.steps ?? []
+    },
+    count: attempts.length
+  };
+}
+function markersFor(result, attempt, report, markers, projectRoot) {
+  if (attempt === void 0 || markers.length === 0) return [];
+  const file = path2.resolve(projectRoot, result.file);
+  const from = result.source.line;
+  const to = Math.min(
+    ...report.run.results.filter((r) => r.file === result.file && r.source.line > from).map((r) => r.source.line)
+  );
+  const start = Date.parse(attempt.startedAt);
+  const end = start + attempt.durationMs + 1e3;
+  return markers.filter(
+    (m) => m.file === file && m.line >= from && m.line < to && m.at >= start && m.at <= end
+  ).sort((a, b) => a.at - b.at);
+}
+function toTestCase(result, report, artifactsRoot, markers, projectRoot) {
+  const scenario = result.titlePath.at(-1) ?? result.testId;
+  const suitePath = result.titlePath.slice(0, -1);
+  const { attempt, count } = finalAttempt(result, report);
+  const steps = attempt?.steps ?? [];
+  const groups = toGroups(
+    steps,
+    markersFor(result, attempt, report, markers, projectRoot),
+    projectRoot
+  );
+  const { attachments, docs } = toMedia(
+    { artifacts: attempt?.artifacts ?? [], steps },
+    groups,
+    artifactsRoot
+  );
+  const error = attempt?.error;
+  const details = error?.details;
+  const stepEvents = groups.map((group, index) => {
+    const failed = group.children.find((c) => c.error !== void 0);
+    return {
+      index,
+      stepId: `step-${index}`,
+      title: group.text,
+      status: groupStatus(group),
+      durationMs: group.children.reduce((sum, c) => sum + c.durationMs, 0),
+      ...failed?.error ? { errorMessage: errorMessage(failed.error) } : {}
+    };
+  });
+  return {
+    externalId: result.id,
+    title: scenario,
+    titlePath: [...result.titlePath],
+    story: {
+      scenario,
+      steps: toSteps(groups),
+      ...result.tags.length > 0 ? { tags: [...result.tags] } : {},
+      ...suitePath.length > 0 ? { suitePath } : {},
+      ...docs.length > 0 ? { docs } : {}
+    },
+    sourceFile: result.file,
+    sourceLine: Math.max(1, result.source.line),
+    status: resultStatus(result.status),
+    durationMs: attempt?.durationMs ?? 0,
+    ...error ? {
+      error: {
+        message: errorMessage(error),
+        ...details?.expected !== void 0 ? { expected: details.expected } : {},
+        ...details?.observed !== void 0 ? { actual: details.observed } : {}
+      }
+    } : {},
+    ...stepEvents.length > 0 ? { stepEvents } : {},
+    ...attachments.length > 0 ? { attachments } : {},
+    retry: Math.max(0, count - 1),
+    projectName: result.targetId
+  };
+}
+function toRawRun(args) {
+  const { report, projectRoot, artifactsRoot } = args;
+  const results = report.run.results.filter((r) => r.selected !== false);
+  const filtered = args.lastRun !== void 0 || results.length < report.run.results.length;
+  return {
+    testCases: results.map(
+      (r) => toTestCase(r, report, artifactsRoot, args.markers ?? [], projectRoot)
+    ),
+    startedAtMs: Date.parse(report.run.startedAt),
+    finishedAtMs: Date.parse(report.run.finishedAt),
+    projectRoot,
+    // Only "filtered" is claimed: e2e can narrow a run in ways the report
+    // does not record, so "full" (which retires scenarios) is never guessed.
+    ...filtered ? { runScope: "filtered" } : {},
+    coveredSourceFiles: [...new Set(results.map((r) => r.file))].sort(),
+    ...report.run.vcs?.commit ? { gitSha: report.run.vcs.commit } : {}
+  };
+}
+async function readMarkers(dir) {
+  const files = await readdir(dir).catch(() => []);
+  const markers = [];
+  for (const file of files.filter((f) => f.endsWith(".jsonl"))) {
+    for (const line of (await readFile(path2.join(dir, file), "utf8")).split(
+      "\n"
+    )) {
+      if (!line) continue;
+      try {
+        markers.push(JSON.parse(line));
+      } catch {
+      }
+    }
+  }
+  return markers;
+}
+function storyReporter(options = {}) {
+  return {
+    name: "executable-stories",
+    onEvent(event) {
+      if (event.type === "run-started")
+        rmSync(path2.resolve(event.projectRoot, MARKERS_DIR), {
+          recursive: true,
+          force: true
+        });
+    },
+    async onRunFinished(run) {
+      const markersDir = path2.resolve(run.projectRoot, MARKERS_DIR);
+      if (run.report.run.explore !== void 0) {
+        await rm(markersDir, { recursive: true, force: true });
+        return [{ label: "Stories", text: "skipped (explore run)" }];
+      }
+      const rawRun = toRawRun({
+        report: run.report,
+        projectRoot: run.projectRoot,
+        artifactsRoot: run.artifactsRoot,
+        markers: await readMarkers(markersDir),
+        ...run.lastRun ? { lastRun: run.lastRun } : {}
+      });
+      await rm(markersDir, { recursive: true, force: true });
+      if (rawRun.testCases.length === 0) {
+        return [{ label: "Stories", text: "skipped (no tests ran)" }];
+      }
+      const target = path2.resolve(
+        run.projectRoot,
+        options.rawRunPath ?? ".executable-stories/raw-run.json"
+      );
+      await mkdir(path2.dirname(target), { recursive: true });
+      await writeFile(
+        target,
+        JSON.stringify({ schemaVersion: 1, ...rawRun }, null, 2),
+        "utf8"
+      );
+      return [
+        { label: "Stories", text: path2.relative(run.projectRoot, target) }
+      ];
+    }
+  };
+}
+export {
+  phraseStep,
+  story,
+  storyReporter,
+  toRawRun
+};
+//# sourceMappingURL=index.js.map