@roku-sdk/device-test-client 0.0.0-stage → 2.18.0

package/src/jest/device-test-environment.ts

@@ -0,0 +1,420 @@
+// Copyright (c) 2026 Roku, Inc.
+// SPDX-License-Identifier: MIT
+/**
+ * Custom Jest environment that hooks `handleTestEvent` to capture
+ * device-side artifacts on failure and writes them to
+ * `<testResultsDir>/failures/<sanitized>/`. Used over `afterEach`
+ * because circus emits structured `test_fn_failure`/`hook_failure`
+ * events with the actual error.
+ *
+ * Capture runs EAGERLY on each failure event (not at teardown) so
+ * screenshots/DOM/logs reflect the device state at the moment of
+ * failure rather than after the next test's `beforeAll` deploys
+ * over it. Disk writes are still deferred to `teardown` (off the
+ * hot path).
+ *
+ * Globals set by the consumer's `setup.ts`:
+ *   - `DEVICE_ENV_GLOBAL_KEY.CLIENT` — RkClient
+ *   - `DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR` — abs path
+ *   - `DEVICE_ENV_GLOBAL_KEY.CAPTURE_ENABLED` — opt-out flag
+ *
+ * All optional. Capture short-circuits when `CAPTURE_ENABLED ===
+ * false`, or when both `CAPTURE_ENABLED` and `TEST_RESULTS_DIR`
+ * are undefined (vanilla consumer that picked this env via
+ * misconfigured preset).
+ */
+
+import { join } from "node:path";
+
+import { DEVICE_ENV_GLOBAL_KEY, FAILURE_RECENT_LOG_LINES } from "../constants";
+import { TEST_RESULTS_DIR_NAME } from "../paths";
+import type { LogLine } from "../types";
+import { ensureFailuresDir, sanitizeTestPath, writeFailureBundle } from "./failure-artifacts";
+
+// Runtime `require()` of `jest-environment-node` keeps the
+// dependency out of our TS type graph and picks whichever Jest
+// version is hosting the worker (29 or 30).
+
+interface NodeEnvironmentConstructor {
+    new (config: unknown, context?: unknown): NodeEnvironmentLike;
+}
+
+/** Base class shape; `any` to avoid coupling to Jest 29 vs 30. */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type NodeEnvironmentLike = any;
+
+/* eslint-disable @typescript-eslint/no-require-imports */
+const NodeEnvModule = require("jest-environment-node");
+/* eslint-enable @typescript-eslint/no-require-imports */
+
+// Both Jest 29 and 30 export the class as `TestEnvironment`.
+const NodeEnvironment: NodeEnvironmentConstructor = NodeEnvModule.TestEnvironment;
+
+// Local circus-event shapes (stable across Jest 29/30) so we don't
+// need `@jest/types` as a dep.
+
+interface CircusNamedBlock {
+    name: string;
+    parent?: CircusNamedBlock;
+}
+
+interface CircusTestEntry extends CircusNamedBlock {
+    errors?: unknown[];
+}
+
+interface CircusHook extends CircusNamedBlock {
+    type: string;
+}
+
+interface TestFnFailureEvent {
+    name: "test_fn_failure";
+    test: CircusTestEntry;
+    error?: unknown;
+}
+
+interface HookFailureEvent {
+    name: "hook_failure";
+    hook: CircusHook;
+    test?: CircusTestEntry;
+    error?: unknown;
+}
+
+interface TeardownEvent {
+    name: "teardown";
+}
+
+type KnownEvent = TestFnFailureEvent | HookFailureEvent | TeardownEvent;
+
+// Structural client shape — keeps the environment decoupled from
+// the full client module graph so unit tests can inject fakes.
+export interface FailureCaptureClient {
+    captureScreenshot(): Promise<Buffer>;
+    queryAppUi(): Promise<unknown>;
+    recentLogLines(n: number): ReadonlyArray<{ seq: number; line: LogLine }>;
+    /** Optional; older clients/fakes may omit. Probed before calling. */
+    getLogStreamHealth?(): Promise<Record<string, unknown>>;
+}
+
+interface PendingFailure {
+    /** describe ancestors + it name (root excluded). */
+    testPath: string[];
+    error: unknown;
+    /** Device-side artifacts captured at failure event time (best-effort). */
+    screenshot: Buffer | null;
+    domTree: unknown;
+    recentLogs: ReadonlyArray<{ seq: number; line: LogLine }> | null;
+    streamHealth: Record<string, unknown> | null;
+    /** Capture-time warnings; merged with write-time on flush. */
+    captureWarnings: Array<{ artifact: string; cause: Error }>;
+}
+
+/**
+ * Subclass of `jest-environment-node`'s `TestEnvironment`. Implements
+ * `handleTestEvent` to buffer failures and flush them on `teardown`.
+ */
+export default class DeviceTestEnvironment extends NodeEnvironment {
+    /** Failure buffer — flushed once on the single `teardown` event. */
+    private pendingFailures: PendingFailure[] = [];
+
+    async handleTestEvent(event: unknown, _state: unknown): Promise<void> {
+        const typed = event as KnownEvent;
+        if (typed.name === "test_fn_failure") {
+            if (!this.shouldCapture()) return;
+            await this.recordFailure(buildTestPath(typed.test), pickError(typed.test.errors, typed.error));
+            return;
+        }
+        if (typed.name === "hook_failure") {
+            if (!this.shouldCapture()) return;
+            const testPath = typed.test ? buildTestPath(typed.test) : buildHookPath(typed.hook);
+            await this.recordFailure(testPath, typed.error ?? new Error(`${typed.hook.type} failed`));
+            return;
+        }
+        if (typed.name === "teardown") {
+            await this.flushPendingFailures();
+        }
+    }
+
+    /**
+     * Capture gate. `CAPTURE_ENABLED === false` → off; both
+     * `CAPTURE_ENABLED` and `TEST_RESULTS_DIR` undefined → off
+     * (vanilla consumer didn't opt in). Defensive overlap with the
+     * flush guard in case globals mutate between events.
+     */
+    private shouldCapture(): boolean {
+        const envGlobal = this.global as Record<string, unknown>;
+        const captureEnabled = envGlobal[DEVICE_ENV_GLOBAL_KEY.CAPTURE_ENABLED];
+        if (captureEnabled === false) return false;
+        if (captureEnabled === undefined && envGlobal[DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR] === undefined) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Capture device-side artifacts NOW (eager). Runs inside Jest's
+     * awaited `handleTestEvent`, blocking the next `beforeEach` until
+     * the snapshot is in hand. Each call wrapped in `captureArtifact`
+     * so one throw doesn't abort the others.
+     */
+    private async recordFailure(testPath: string[], error: unknown): Promise<void> {
+        const envGlobal = this.global as Record<string, unknown>;
+        const client = envGlobal[DEVICE_ENV_GLOBAL_KEY.CLIENT] as FailureCaptureClient | undefined;
+
+        const captureWarnings: Array<{ artifact: string; cause: Error }> = [];
+        let screenshot: Buffer | null = null;
+        let domTree: unknown = null;
+        let recentLogs: ReadonlyArray<{ seq: number; line: LogLine }> | null = null;
+        let streamHealth: Record<string, unknown> | null = null;
+
+        if (client) {
+            screenshot = await captureArtifact("screenshot", () => client.captureScreenshot(), captureWarnings);
+            domTree = await captureArtifact("dom-tree", () => client.queryAppUi(), captureWarnings);
+            recentLogs =
+                (await captureArtifact(
+                    "recent-logs",
+                    async () => client.recentLogLines(FAILURE_RECENT_LOG_LINES),
+                    captureWarnings
+                )) ?? [];
+            if (typeof client.getLogStreamHealth === "function") {
+                streamHealth = await captureArtifact(
+                    "stream-health",
+                    async () => {
+                        // Re-narrow locally; the closure doesn't see
+                        // the typeof guard above.
+                        const fn = client.getLogStreamHealth!;
+                        return await fn.call(client);
+                    },
+                    captureWarnings
+                );
+            }
+        }
+
+        // Surface the one-line stream-health summary so CI stdout
+        // shows gap context inline with the failure without
+        // requiring the reviewer to fetch the on-disk bundle.
+        // Routed through `console.warn` (not error.message/.stack
+        // mutation) because (a) Jest's reporter pipes our console
+        // calls right next to the failure, so visibility is
+        // preserved, and (b) mutating foreign Error objects
+        // throws on frozen errors and breaks custom matchers that
+        // pin on `expect(err.message).toMatch(...)`. The bundle
+        // already persists `streamHealth` separately for offline
+        // analysis, so this is purely an inline-visibility hook.
+        emitStreamHealthSummary(testPath, streamHealth);
+
+        this.pendingFailures.push({
+            testPath,
+            error,
+            screenshot,
+            domTree,
+            recentLogs,
+            streamHealth,
+            captureWarnings
+        });
+    }
+
+    /**
+     * Drain `pendingFailures` and write one bundle per failure. All
+     * work is best-effort and must not propagate out of `teardown`.
+     * Emits at most ONE warning per bundle covering every capture +
+     * write failure for that bundle.
+     */
+    private async flushPendingFailures(): Promise<void> {
+        if (this.pendingFailures.length === 0) return;
+        const envGlobal = this.global as Record<string, unknown>;
+
+        // Defensive re-check: a late `CAPTURE_ENABLED = false`
+        // write still suppresses on-disk output.
+        if (!this.shouldCapture()) {
+            this.pendingFailures = [];
+            return;
+        }
+
+        const resultsDir = resolveTestResultsDir(envGlobal);
+        let failuresDir: string;
+        try {
+            failuresDir = ensureFailuresDir(resultsDir);
+        } catch (err) {
+            console.warn(
+                `[device-test-client] could not create failures dir at "${resultsDir}": ` +
+                    `${(err as Error).message}. Failure diagnostics will be skipped.`,
+                err
+            );
+            this.pendingFailures = [];
+            return;
+        }
+
+        for (const failure of this.pendingFailures) {
+            const sanitized = sanitizeTestPath(failure.testPath);
+            const bundleDir = join(failuresDir, sanitized);
+
+            // Capture-time + write-time warnings → ONE warn per bundle.
+            const warnings: Array<{ artifact: string; cause: Error }> = [...failure.captureWarnings];
+
+            const result = writeFailureBundle(bundleDir, {
+                screenshot: failure.screenshot,
+                domTree: failure.domTree,
+                recentLogs: failure.recentLogs,
+                streamHealth: failure.streamHealth,
+                error: failure.error
+            });
+
+            for (const f of result.failed) {
+                warnings.push(f);
+            }
+
+            if (warnings.length > 0) {
+                emitBundleWarning(sanitized, warnings);
+            }
+
+            // One-line pointer so reviewers find the bundle without
+            // knowing the `test-results/failures/...` convention.
+            // Skipped when nothing landed (mkdir failure).
+            if (result.written.length > 0) {
+                console.log(
+                    `[device-test-client] failure bundle written: ${bundleDir} ` + `(${result.written.join(", ")})`
+                );
+            }
+        }
+
+        this.pendingFailures = [];
+    }
+}
+
+/**
+ * Await `fn`; on throw, push `{artifact, cause}` and return null so
+ * the bundle writer still gets a partial input. Caller emits one
+ * consolidated warning per bundle.
+ */
+async function captureArtifact<T>(
+    artifact: string,
+    fn: () => Promise<T>,
+    warnings: Array<{ artifact: string; cause: Error }>
+): Promise<T | null> {
+    try {
+        return await fn();
+    } catch (err) {
+        warnings.push({ artifact, cause: toError(err) });
+        return null;
+    }
+}
+
+/**
+ * One warn per bundle. Pass each underlying `Error` separately so
+ * Node's console printer renders the full stack.
+ */
+function emitBundleWarning(sanitized: string, warnings: ReadonlyArray<{ artifact: string; cause: Error }>): void {
+    const summary = warnings.map(w => `${w.artifact}=${w.cause.message}`).join("; ");
+    console.warn(`[device-test-client] bundle "${sanitized}" had warnings: ${summary}`, ...warnings.map(w => w.cause));
+}
+
+function toError(err: unknown): Error {
+    return err instanceof Error ? err : new Error(String(err));
+}
+
+/**
+ * Emit a one-line stream-health summary via `console.warn` so
+ * Jest's reporter renders it inline with the failure. Replaces an
+ * earlier in-place mutation of `error.message`/`error.stack`,
+ * which threw on frozen errors and broke custom matchers that
+ * pinned on `expect(err.message).toMatch(...)`.
+ *
+ * Skipped on trivial snapshots so healthy runs don't gain noise.
+ * `testPath` is included in the prefix so a multi-failure run
+ * still attributes each summary to the right test.
+ */
+function emitStreamHealthSummary(testPath: string[], health: Record<string, unknown> | null): void {
+    if (!health) return;
+    const summary = formatStreamHealthSummary(health);
+    if (!summary) return;
+    const label = testPath.length > 0 ? testPath.join(" › ") : "<unknown test>";
+    console.warn(`[device-test-client] [log-stream-health] ${label}: ${summary}`);
+}
+
+/**
+ * One-line `LogStreamHealth` summary; `""` for trivially-clean
+ * snapshots so healthy runs stay terse. Reads fields defensively
+ * to avoid an env→types import edge.
+ */
+function formatStreamHealthSummary(health: Record<string, unknown>): string {
+    const gapCount = numberFrom(health.gapCount, 0);
+    const totalGapSeqSpan = numberFrom(health.totalGapSeqSpan, 0);
+    const openGap = health.openGap === true;
+    const server = isObject(health.serverMetrics) ? health.serverMetrics : null;
+    const reconnectCount = server ? numberFrom(server.reconnectCount, 0) : 0;
+    const totalDowntime = server ? numberFrom(server.totalDowntime, 0) : 0;
+    const trivial = gapCount === 0 && reconnectCount === 0 && !openGap;
+    if (trivial) return "";
+    const parts: string[] = [];
+    parts.push(`client gaps=${gapCount} (span=${totalGapSeqSpan} seqs)`);
+    if (openGap) parts.push(`open-gap=true`);
+    if (server) {
+        parts.push(`server reconnects=${reconnectCount} downtime=${Math.round(totalDowntime)}ms`);
+    } else if (health.serverMetrics === null) {
+        parts.push(`server metrics unavailable (rk server unreachable)`);
+    }
+    return parts.join("; ");
+}
+
+function numberFrom(v: unknown, fallback: number): number {
+    return typeof v === "number" && Number.isFinite(v) ? v : fallback;
+}
+
+function isObject(v: unknown): v is Record<string, unknown> {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+/** Build `[...describeAncestors, testName]`; drops ROOT_DESCRIBE_BLOCK. */
+function buildTestPath(test: CircusTestEntry): string[] {
+    const parts: string[] = [];
+    const ancestors: string[] = [];
+    let cursor: CircusNamedBlock | undefined = test.parent;
+    while (cursor) {
+        ancestors.unshift(cursor.name);
+        cursor = cursor.parent;
+    }
+    for (const name of ancestors) {
+        if (name === "ROOT_DESCRIBE_BLOCK") continue;
+        parts.push(name);
+    }
+    parts.push(test.name);
+    return parts;
+}
+
+/** Path for a hook failure with no test; leaf = `hook-<type>`. */
+function buildHookPath(hook: CircusHook): string[] {
+    const parts: string[] = [];
+    const ancestors: string[] = [];
+    let cursor: CircusNamedBlock | undefined = hook.parent;
+    while (cursor) {
+        ancestors.unshift(cursor.name);
+        cursor = cursor.parent;
+    }
+    for (const name of ancestors) {
+        if (name === "ROOT_DESCRIBE_BLOCK") continue;
+        parts.push(name);
+    }
+    parts.push(`hook-${hook.type}`);
+    return parts;
+}
+
+/**
+ * Pick the most actionable error from a circus entry. Last entry
+ * wins (follow-on errors are less interesting than the original).
+ */
+function pickError(errors: unknown[] | undefined, fallback: unknown): unknown {
+    if (errors && errors.length > 0) {
+        const last = errors[errors.length - 1];
+        if (Array.isArray(last) && last.length > 0) return last[0];
+        return last;
+    }
+    return fallback ?? new Error("unknown test failure");
+}
+
+function resolveTestResultsDir(envGlobal: Record<string, unknown>): string {
+    const fromGlobal = envGlobal[DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR];
+    if (typeof fromGlobal === "string" && fromGlobal.length > 0) {
+        return fromGlobal;
+    }
+    return join(process.cwd(), TEST_RESULTS_DIR_NAME);
+}

package/src/jest/failure-artifacts.ts

@@ -0,0 +1,169 @@
+// Copyright (c) 2026 Roku, Inc.
+// SPDX-License-Identifier: MIT
+/**
+ * Pure-fs helpers for writing per-failure bundles to
+ * `<root>/failures/<sanitized>/`. Each sub-write is independent
+ * (best-effort): a failing `screenshot.png` write doesn't block
+ * the others. Leaf module — no `client.ts`/`assertions.ts` imports.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { FAILURE_ARTIFACT_FILE, FAILURES_DIR } from "../constants";
+import type { LogEntry } from "../logs/log-buffer";
+
+/**
+ * Join `[describe..., testName]` into a single filesystem-safe
+ * directory name. Each part: non-`[A-Za-z0-9._-]` → `-`, runs
+ * collapsed, edges trimmed. Parts joined with `--` so reviewers
+ * can recover describe boundaries by eye. Falls back to
+ * `"unnamed-test"` for empty input.
+ */
+export function sanitizeTestPath(parts: readonly string[]): string {
+    const cleaned = parts.map(sanitizePart).filter(p => p.length > 0);
+    if (cleaned.length === 0) return "unnamed-test";
+    return cleaned.join("--");
+}
+
+function sanitizePart(s: string): string {
+    return s
+        .replace(/[^A-Za-z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-+|-+$/g, "");
+}
+
+/** Create `<testResultsDir>/failures/` and return its absolute path. */
+export function ensureFailuresDir(testResultsDir: string): string {
+    const dir = join(testResultsDir, FAILURES_DIR);
+    mkdirSync(dir, { recursive: true });
+    return dir;
+}
+
+/**
+ * Inputs for a single failure bundle. Every field may be null —
+ * a client unavailable at teardown still gets an `error.txt`.
+ */
+export interface FailureBundleInput {
+    screenshot: Buffer | null;
+    domTree: unknown;
+    /** Empty array is valid and writes an empty `recent-logs.txt`. */
+    recentLogs: ReadonlyArray<Pick<LogEntry, "seq" | "line">> | null;
+    /** `LogStreamHealth` snapshot for transport-leg attribution. */
+    streamHealth?: Record<string, unknown> | null;
+    error: unknown;
+}
+
+/**
+ * Per-bundle write outcome. Caller surfaces a single warn instead
+ * of letting per-write errors propagate into Jest teardown.
+ */
+export interface FailureBundleWriteResult {
+    written: string[];
+    failed: Array<{ artifact: string; cause: Error }>;
+}
+
+/**
+ * Write every available artifact to `dir`. Each sub-write is
+ * independently try/caught (best-effort). `error.txt` is always
+ * attempted — a bundle with only `error.txt` is still useful.
+ */
+export function writeFailureBundle(dir: string, input: FailureBundleInput): FailureBundleWriteResult {
+    const written: string[] = [];
+    const failed: Array<{ artifact: string; cause: Error }> = [];
+
+    try {
+        mkdirSync(dir, { recursive: true });
+    } catch (err) {
+        // mkdir failure → all artifacts fail uniformly; one entry.
+        failed.push({ artifact: "mkdir", cause: toError(err) });
+        return { written, failed };
+    }
+
+    if (input.screenshot) {
+        const name = FAILURE_ARTIFACT_FILE.SCREENSHOT;
+        try {
+            writeFileSync(join(dir, name), input.screenshot);
+            written.push(name);
+        } catch (err) {
+            failed.push({ artifact: "screenshot", cause: toError(err) });
+        }
+    }
+
+    if (input.domTree !== null && input.domTree !== undefined) {
+        const name = FAILURE_ARTIFACT_FILE.DOM_TREE;
+        try {
+            const body = `${JSON.stringify(input.domTree, null, 2)}\n`;
+            writeFileSync(join(dir, name), body, "utf8");
+            written.push(name);
+        } catch (err) {
+            failed.push({ artifact: "dom-tree", cause: toError(err) });
+        }
+    }
+
+    if (input.recentLogs !== null) {
+        const name = FAILURE_ARTIFACT_FILE.RECENT_LOGS;
+        try {
+            const body = formatRecentLogs(input.recentLogs);
+            writeFileSync(join(dir, name), body, "utf8");
+            written.push(name);
+        } catch (err) {
+            failed.push({ artifact: "recent-logs", cause: toError(err) });
+        }
+    }
+
+    if (input.streamHealth !== null && input.streamHealth !== undefined) {
+        const name = FAILURE_ARTIFACT_FILE.STREAM_HEALTH;
+        try {
+            const body = `${JSON.stringify(input.streamHealth, null, 2)}\n`;
+            writeFileSync(join(dir, name), body, "utf8");
+            written.push(name);
+        } catch (err) {
+            failed.push({ artifact: "stream-health", cause: toError(err) });
+        }
+    }
+
+    const errorName = FAILURE_ARTIFACT_FILE.ERROR;
+    try {
+        writeFileSync(join(dir, errorName), `${stringifyError(input.error)}\n`, "utf8");
+        written.push(errorName);
+    } catch (err) {
+        failed.push({ artifact: "error", cause: toError(err) });
+    }
+
+    return { written, failed };
+}
+
+function formatRecentLogs(logs: ReadonlyArray<Pick<LogEntry, "seq" | "line">>): string {
+    if (logs.length === 0) return "";
+    return `${logs.map(e => `${e.seq}\t${e.line}`).join("\n")}\n`;
+}
+
+/**
+ * Render `err` for `error.txt`. Duck-types on `.stack`/`.message`
+ * because Jest VM-sandbox errors don't pass `instanceof Error`
+ * across the realm boundary; `JSON.stringify` would omit `.stack`.
+ * Precedence: stack > name: message > string > JSON > String().
+ */
+function stringifyError(err: unknown): string {
+    if (typeof err === "string") return err;
+    if (err !== null && typeof err === "object") {
+        const e = err as { stack?: unknown; message?: unknown; name?: unknown };
+        if (typeof e.stack === "string" && e.stack.length > 0) {
+            return e.stack;
+        }
+        if (typeof e.message === "string" && e.message.length > 0) {
+            const name = typeof e.name === "string" && e.name.length > 0 ? e.name : "Error";
+            return `${name}: ${e.message}`;
+        }
+    }
+    try {
+        return JSON.stringify(err, null, 2);
+    } catch {
+        return String(err);
+    }
+}
+
+function toError(err: unknown): Error {
+    return err instanceof Error ? err : new Error(String(err));
+}