@alchemy.run/test 0.0.0-stage → 2.0.0-beta.85

package/src/FileLog.ts

@@ -0,0 +1,158 @@
+import { closeSync, constants, openSync, writeSync } from "node:fs";
+/**
+ * Persistent run log: every test's pass/failure followed by its captured
+ * output, appended to `.alchemy/log/test.log` as the run progresses —
+ * color-free so agents and tooling can read it directly.
+ */
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+import type { LogEntry } from "./Model.ts";
+import { formatPlanPreview } from "./Plan.ts";
+import type { TestEvent } from "./Reporter.ts";
+
+const formatDuration = (ms: number): string =>
+  ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;
+
+const formatLogs = (logs: ReadonlyArray<LogEntry>): string =>
+  logs.map((log) => log.message).join("\n");
+
+const GLYPH = {
+  pass: "PASS",
+  fail: "FAIL",
+  skip: "SKIP",
+  todo: "TODO",
+} as const;
+
+/** Render an event as a log chunk; `undefined` = nothing to write. */
+export const formatEvent = (event: TestEvent): string | undefined => {
+  switch (event._tag) {
+    case "PlanPreview":
+      return `${formatPlanPreview(event.phases)}\n`;
+    case "PlanPhaseStart":
+      return `Plan phase ${event.phase}/${event.phases}: ${event.tests} tests\n`;
+    case "RunStart":
+      return `running ${event.tests.length} tests from ${event.files} files (${new Date().toISOString()})\n\n`;
+    case "TestEnd": {
+      const title = `${event.test.file} > ${event.test.titlePath.join(" > ")}`;
+      const retries = event.result.retries > 0 ? ` [retried x${event.result.retries}]` : "";
+      const lines = [
+        `${GLYPH[event.result.status]} ${title} (${formatDuration(event.result.durationMs)})${retries}`,
+      ];
+      if (event.result.error !== undefined) {
+        lines.push(event.result.error);
+      }
+      if (event.result.logs.length > 0) {
+        lines.push("--- captured output ---");
+        lines.push(formatLogs(event.result.logs));
+        lines.push("--- end output ---");
+      }
+      // One line per plain result; a blank separator only after multi-line
+      // entries (error/output blocks) so they don't run into the next entry.
+      return lines.length === 1 ? `${lines[0]}\n` : `${lines.join("\n")}\n\n`;
+    }
+    case "FileEnd": {
+      const lines: Array<string> = [];
+      if (event.error !== undefined) {
+        lines.push(`FAIL ${event.file} failed to run`, event.error);
+      }
+      if (event.logs.length > 0) {
+        lines.push(`--- file hook output (${event.file}) ---`);
+        lines.push(formatLogs(event.logs));
+        lines.push("--- end output ---");
+      }
+      return lines.length === 0 ? undefined : `${lines.join("\n")}\n\n`;
+    }
+    case "RunEnd": {
+      const s = event.summary;
+      return (
+        `Tests: ${s.failed} failed | ${s.passed} passed` +
+        (s.skipped > 0 ? ` | ${s.skipped} skipped` : "") +
+        (s.todo > 0 ? ` | ${s.todo} todo` : "") +
+        (s.plan
+          ? ` | ${s.plan.found} found | ${s.plan.selected} selected by plan | ${s.plan.excluded} excluded by plan`
+          : "") +
+        ` (${s.files} files, ${formatDuration(s.durationMs)})\n`
+      );
+    }
+    default:
+      return undefined;
+  }
+};
+
+export interface FileLog {
+  readonly append: (event: TestEvent) => Effect.Effect<void>;
+  /** Synchronous persistence applies backpressure to synchronous logger callbacks. */
+  readonly appendHookLine: (file: string, entry: LogEntry) => void;
+  readonly appendTestLine: (id: string, entry: LogEntry) => void;
+  readonly close: Effect.Effect<void>;
+}
+
+const LOG_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * Delete sibling run logs whose mtime (from stat — file names are never
+ * trusted) is older than a week, so the per-run log directory can't grow
+ * unboundedly. Best-effort: failures are ignored.
+ */
+const pruneOldLogs = Effect.fn(function* (dir: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const entries = yield* fs
+    .readDirectory(dir)
+    .pipe(Effect.orElseSucceed(() => [] as Array<string>));
+  const cutoff = Date.now() - LOG_RETENTION_MS;
+  for (const entry of entries) {
+    if (!entry.endsWith(".log")) continue;
+    const file = path.join(dir, entry);
+    const info = yield* fs.stat(file).pipe(Effect.orElseSucceed(() => undefined));
+    const mtime = info === undefined ? undefined : Option.getOrUndefined(info.mtime);
+    if (mtime !== undefined && mtime.getTime() < cutoff) {
+      yield* fs.remove(file).pipe(Effect.ignore);
+    }
+  }
+});
+
+/** Create (truncate) the log file and return an event appender. */
+export const makeFileLog = Effect.fn(function* (logFile: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  yield* fs.makeDirectory(path.dirname(logFile), { recursive: true }).pipe(Effect.ignore);
+  yield* pruneOldLogs(path.dirname(logFile));
+  const fd = yield* Effect.sync(() =>
+    openSync(
+      logFile,
+      constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_APPEND,
+    ),
+  );
+  let closed = false;
+  const write = (text: string): void => {
+    if (closed) return;
+    const bytes = Buffer.from(text);
+    let offset = 0;
+    while (offset < bytes.length) {
+      offset += writeSync(fd, bytes, offset, bytes.length - offset);
+    }
+  };
+  const append: FileLog["append"] = (event) =>
+    Effect.sync(() => {
+      const chunk = formatEvent(event);
+      if (chunk !== undefined) write(chunk);
+    });
+  const appendLine = (label: string, entry: LogEntry): void => {
+    write(`[${label}] ${entry.message}\n`);
+  };
+  const close = Effect.sync(() => {
+    if (closed) return;
+    closed = true;
+    closeSync(fd);
+  });
+  yield* Effect.addFinalizer(() => close);
+  return {
+    append,
+    appendHookLine: (file, entry) => appendLine(`hook ${file}`, entry),
+    appendTestLine: (id, entry) => appendLine(`test ${id}`, entry),
+    close,
+  } satisfies FileLog;
+});

package/src/Http.ts

@@ -0,0 +1,201 @@
+import * as Data from "effect/Data";
+import * as Effect from "effect/Effect";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as HttpClient from "effect/http/HttpClient";
+import { DecodeError, HttpClientError } from "effect/http/HttpClientError";
+import * as HttpClientRequest from "effect/http/HttpClientRequest";
+import type { HttpClientResponse } from "effect/http/HttpClientResponse";
+import * as Layer from "effect/Layer";
+import * as RpcClient from "effect/rpc/RpcClient";
+import * as RpcSerialization from "effect/rpc/RpcSerialization";
+import * as Schedule from "effect/Schedule";
+
+/**
+ * A freshly-deployed Cloudflare Worker is not instantly reachable over HTTP.
+ * Its `workers.dev` route, the script, and each binding (R2 / D1 / DO / Secrets
+ * Store) propagate to the edge independently and asynchronously, so the first
+ * requests to a new URL can transiently return:
+ *
+ *  - `404` while the `workers.dev` subdomain / route is still propagating
+ *    (Cloudflare serves its "There is nothing here yet" placeholder), or
+ *  - `5xx` while the script is up but a binding it depends on isn't ready yet.
+ *
+ * This is ordinary eventual consistency that belongs at the call site, not in
+ * the resource provider — the provider returning before every edge PoP has
+ * converged is correct. Consumers ride out the window by retrying the request.
+ */
+export class WorkerNotReady extends Data.TaggedError("WorkerNotReady")<{ status: number }> {}
+
+/**
+ * Status codes that indicate the edge hasn't finished converging on a fresh
+ * deploy. Deliberate client errors (`401`, `403`, `400`, …) are NOT in this
+ * set, so assertions on those statuses still observe them immediately rather
+ * than being retried away.
+ */
+const isColdStartStatus = (status: number): boolean => status === 404 || status >= 500;
+
+export interface WhenReadyOptions {
+  /** Max retry attempts before surfacing {@link WorkerNotReady}. Default `20`. */
+  times?: number;
+}
+
+/**
+ * Execute an arbitrary {@link HttpClientRequest.HttpClientRequest}, retrying
+ * through the Cloudflare cold-start window ({@link isColdStartStatus}) until the
+ * Worker serves a non-transient response. The returned response can carry any
+ * non-cold-start status (e.g. `200`, `202`, `401`) for the caller to assert on.
+ */
+export const executeWhenReady = (
+  request: HttpClientRequest.HttpClientRequest,
+  options?: WhenReadyOptions,
+): Effect.Effect<HttpClientResponse, unknown, HttpClient.HttpClient> =>
+  Effect.gen(function* () {
+    const client = yield* HttpClient.HttpClient;
+    return yield* client.execute(request).pipe(
+      Effect.flatMap((response) =>
+        isColdStartStatus(response.status)
+          ? Effect.fail(new WorkerNotReady({ status: response.status }))
+          : Effect.succeed(response),
+      ),
+      Effect.retry({
+        while: (error) => error instanceof WorkerNotReady,
+        schedule: Schedule.max([
+          Schedule.exponential("500 millis"),
+          Schedule.recurs(options?.times ?? 20),
+        ]),
+      }),
+    );
+  });
+
+/**
+ * Convenience wrapper over {@link executeWhenReady} for a plain `GET`.
+ */
+export const getWhenReady = (
+  url: string,
+  options?: WhenReadyOptions,
+): Effect.Effect<HttpClientResponse, unknown, HttpClient.HttpClient> =>
+  executeWhenReady(HttpClientRequest.get(url), options);
+
+/**
+ * Options for the edge-transient response guard applied by
+ * {@link guardContentType} / {@link rpcClientLayer}. Pass per suite via
+ * `Test.make({ http: { ... } })` or per call site.
+ */
+export interface EdgeGuardOptions {
+  /**
+   * Retry schedule for edge-transient responses (and transport errors).
+   * Default: exponential from 500ms capped at 3s.
+   */
+  schedule?: Schedule.Schedule<unknown, unknown>;
+  /** Max transport-level retry attempts. Default `5`. */
+  times?: number;
+}
+
+const defaultGuardSchedule = Schedule.min([
+  Schedule.exponential("500 millis"),
+  Schedule.spaced("3 seconds"),
+]);
+
+/**
+ * Guard an `HttpClient` against edge-generated bodies on a freshly deployed
+ * Worker URL.
+ *
+ * Protocol clients like effect RPC's `layerProtocolHttp` never inspect the
+ * response status or content-type — they pipe the raw body straight into the
+ * serialization parser, so every edge-generated HTML page (the workers.dev
+ * placeholder, which serves with HTTP **200**; 1101/1102 error pages;
+ * 429/1015 rate limits) surfaces as an opaque decode defect (e.g.
+ * `RpcClientDefect: Error decoding HTTP response`).
+ *
+ * This transform rejects any response whose `content-type` doesn't match the
+ * expected one as a typed `HttpClientError` that records the status + a body
+ * snippet — so a failure names its cause — and retries the request through a
+ * bounded schedule so callers ride out the cold-start window
+ * ({@link WorkerNotReady} documents why this belongs at the call site).
+ */
+export const guardContentType =
+  (contentType: string, options?: EdgeGuardOptions) =>
+  (client: HttpClient.HttpClient): HttpClient.HttpClient =>
+    client.pipe(
+      HttpClient.transform((effect, request) =>
+        Effect.flatMap(effect, (response) => {
+          const observed = response.headers["content-type"] ?? "";
+          if (observed.includes(contentType)) {
+            return Effect.succeed(response);
+          }
+          return response.text.pipe(
+            Effect.orElseSucceed(() => "<unreadable body>"),
+            Effect.flatMap((body) =>
+              Effect.fail(
+                new HttpClientError({
+                  reason: new DecodeError({
+                    request,
+                    response,
+                    description: `expected "${contentType}", got "${observed}" (status ${response.status}): ${body.slice(0, 200)}`,
+                  }),
+                }),
+              ),
+            ),
+          );
+        }),
+      ),
+      HttpClient.retry({
+        schedule: options?.schedule ?? defaultGuardSchedule,
+        times: options?.times ?? 5,
+      }),
+    );
+
+/**
+ * A fetch-backed `HttpClient` layer wrapped with {@link guardContentType}.
+ *
+ * Deliberately a standalone layer rather than a transform of the ambient
+ * test `HttpClient`: the ambient client also serves the engine's own cloud
+ * API calls during `deploy`/`destroy` and distilled SDK calls in
+ * `test.provider` bodies, which must NOT be subjected to this guard.
+ */
+export const guardedFetchLayer = (
+  contentType: string,
+  options?: EdgeGuardOptions,
+): Layer.Layer<HttpClient.HttpClient> =>
+  Layer.effect(HttpClient.HttpClient)(
+    Effect.gen(function* () {
+      const client = yield* HttpClient.HttpClient;
+      return guardContentType(contentType, options)(client);
+    }),
+  ).pipe(Layer.provide(FetchHttpClient.layer));
+
+/**
+ * Complete RPC-over-HTTP client transport for driving a deployed Worker:
+ * `RpcClient.layerProtocolHttp` + the given serialization (ndjson by
+ * default) + a {@link guardedFetchLayer} expecting that serialization's
+ * content-type.
+ *
+ * ```ts
+ * const client = yield* RpcClient.make(WorkerRpcs);
+ * // ...
+ * }).pipe(Effect.scoped, Effect.provide(Test.Http.rpcClientLayer(url)));
+ * ```
+ */
+export const rpcClientLayer = (
+  url: string,
+  options?: EdgeGuardOptions & {
+    /** RPC wire serialization. Default {@link RpcSerialization.ndjson}. */
+    serialization?: RpcSerialization.RpcSerialization["Service"];
+    /**
+     * Set `false` to skip the edge guard entirely and use a plain fetch
+     * transport — e.g. for a test that asserts on the raw edge behavior
+     * itself. Default `true`.
+     */
+    guard?: boolean;
+  },
+): Layer.Layer<RpcClient.Protocol> => {
+  const serialization = options?.serialization ?? RpcSerialization.ndjson;
+  return RpcClient.layerProtocolHttp({ url }).pipe(
+    Layer.provide(
+      options?.guard === false
+        ? FetchHttpClient.layer
+        : guardedFetchLayer(serialization.contentType, options),
+    ),
+    Layer.provide(Layer.succeed(RpcSerialization.RpcSerialization, serialization)),
+  );
+};

package/src/Model.ts

@@ -0,0 +1,160 @@
+/**
+ * Core data model for the alchemy-test harness.
+ *
+ * A test file is collected into a tree of {@link Suite}s and {@link TestCase}s
+ * by evaluating its module body (describe/test calls register nodes). The
+ * runner then walks the tree and executes every test as an Effect.
+ */
+import type * as Effect from "effect/Effect";
+import { mergeTags, type Tags } from "./Tags.ts";
+
+/** Execution mode attached to a suite or test at registration time. */
+export type Mode = "run" | "skip" | "only" | "todo";
+
+/** A single log line captured from a test's Effect Logger or Console. */
+export interface LogEntry {
+  readonly level: string;
+  readonly message: string;
+  readonly time: Date;
+}
+
+/**
+ * The body of a test. Always an Effect — plain-function tests are wrapped
+ * at registration time. The Effect must be self-contained (R = never);
+ * wrapping (scope, TestClock/TestConsole, shared layers) happens in the
+ * harness closures, NOT in the runner.
+ */
+export type TestBody = () => Effect.Effect<unknown, unknown, never>;
+
+/** A hook body. Same contract as {@link TestBody}. */
+export type HookBody = () => Effect.Effect<unknown, unknown, never>;
+
+/** `true` for the whole-process lock, or a label naming a narrower lock. */
+export type Exclusive = boolean | string;
+
+export interface Hook {
+  readonly body: HookBody;
+  readonly timeout?: number | undefined;
+  /**
+   * `true` takes the whole-process write lock for this hook. A label (e.g.
+   * `"hetzner"`) only serializes the hook with tests and hooks holding the
+   * same label — use it for deploys that consume a scarce cloud quota.
+   */
+  readonly exclusive?: Exclusive;
+}
+
+export interface TestCase {
+  /** Deduplicated labels, including tags inherited from suites. */
+  readonly tags: ReadonlyArray<string>;
+  /** Inherited tags requiring explicit selection. */
+  readonly optInTags: ReadonlyArray<string>;
+  readonly type: "test";
+  readonly name: string;
+  readonly mode: Mode;
+  /** Invert the result: the test passes only if the body fails. */
+  readonly fails?: boolean;
+  /**
+   * `true` runs this test with the whole-process write lock: no other test
+   * in the run executes concurrently with it. Required for tests that mutate
+   * process-global state (e.g. `process.env.PATH`) — the runner executes
+   * everything in ONE bun process, so such mutations are visible everywhere.
+   *
+   * A label (e.g. `"hetzner"`) only serializes the test with others holding
+   * the same label — use it for a shared cloud quota, not global state.
+   */
+  readonly exclusive?: Exclusive;
+  /** Override the run-wide retry budget for this test. */
+  readonly retry?: number | undefined;
+  readonly timeout?: number | undefined;
+  readonly body: TestBody | undefined;
+  readonly parent: Suite;
+}
+
+export interface Suite {
+  /** Deduplicated labels, including tags inherited from parent suites. */
+  readonly tags: ReadonlyArray<string>;
+  /** Inherited tags requiring explicit selection. */
+  readonly optInTags: ReadonlyArray<string>;
+  readonly type: "suite";
+  readonly name: string;
+  mode: Mode;
+  /** When true, children run one at a time (the default; describe.concurrent opts out). */
+  sequential: boolean;
+  readonly children: Array<Suite | TestCase>;
+  readonly beforeAll: Array<Hook>;
+  readonly afterAll: Array<Hook>;
+  readonly beforeEach: Array<Hook>;
+  readonly afterEach: Array<Hook>;
+  readonly parent: Suite | undefined;
+}
+
+/** The root suite of a single test file. */
+export interface FileSuite extends Suite {
+  /** Path relative to the run root, e.g. `test/Cloudflare/R2/Bucket.test.ts`. */
+  readonly file: string;
+  /** Framework resources, disposed even when user hooks are skipped. */
+  readonly cleanups: Array<Hook>;
+}
+
+export const makeSuite = (
+  name: string,
+  parent: Suite | undefined,
+  mode: Mode = "run",
+  tags?: Tags,
+  optInTags?: Tags,
+): Suite => ({
+  tags: mergeTags(parent?.tags ?? [], tags),
+  optInTags: mergeTags(parent?.optInTags ?? [], optInTags),
+  type: "suite",
+  name,
+  mode,
+  sequential: true,
+  children: [],
+  beforeAll: [],
+  afterAll: [],
+  beforeEach: [],
+  afterEach: [],
+  parent,
+});
+
+export const makeFileSuite = (file: string): FileSuite => ({
+  ...makeSuite(file, undefined),
+  file,
+  cleanups: [],
+});
+
+/** Full title path from the file root down to (and including) this node. */
+export const titlePath = (node: Suite | TestCase): ReadonlyArray<string> => {
+  const parts: Array<string> = [];
+  let current: Suite | TestCase | undefined = node;
+  while (current !== undefined) {
+    // Skip the file-root suite name (it's reported separately as `file`).
+    if (current.parent !== undefined || current.type === "test") {
+      parts.unshift(current.name);
+    } else if (current.parent === undefined && current.type === "suite") {
+      // file root — stop
+    }
+    current = current.parent;
+  }
+  return parts;
+};
+
+export const fullTitle = (node: Suite | TestCase): string => titlePath(node).join(" > ");
+
+/** Walk every test in a suite subtree (depth-first, registration order). */
+export const forEachTest = (suite: Suite, f: (test: TestCase) => void): void => {
+  for (const child of suite.children) {
+    if (child.type === "test") f(child);
+    else forEachTest(child, f);
+  }
+};
+
+/** True if the subtree contains a node marked `only`. */
+export const containsOnly = (suite: Suite): boolean => {
+  if (suite.mode === "only") return true;
+  for (const child of suite.children) {
+    if (child.type === "test" && child.mode === "only") return true;
+    if (child.type === "suite" && containsOnly(child)) return true;
+  }
+  return false;
+};