@specific.dev/spectest 0.88.1 → 0.88.3

package/README.md

@@ -0,0 +1,68 @@
+# Spectest SDK
+
+Spectest runs your application and its dependencies in isolated environments.
+Tests can fork a parent's live state, including memory and files, then drive
+user flows with recorded assertions and graphical replays.
+
+Install the SDK in your project's `spectest/` directory:
+
+```sh
+mkdir -p spectest
+npm install --prefix spectest --save-exact @specific.dev/spectest@0.88.3
+```
+
+Install the [Spectest CLI](https://github.com/specific-dev/spectest/releases/latest)
+and run `spectest docs /installation` for project setup and authentication.
+`spectest docs` contains the complete offline authoring guide.
+
+## Electron apps
+
+Use `electron()` to build and run a real Linux Electron app under Xvfb, with
+its main process, preload, IPC and renderer. Spectest supplies the container
+setup; no custom Dockerfile is needed.
+
+Create `spectest/index.ts`:
+
+```ts
+import { defineEnvironment } from "@specific.dev/spectest";
+import { electron } from "@specific.dev/spectest/components";
+
+export const env = defineEnvironment({
+  name: "desktop-app",
+  services: { app: electron() },
+});
+
+export default env.project();
+```
+
+Create `spectest/tests/notes.ts`, using your app's labels:
+
+```ts
+import { expect } from "@specific.dev/spectest";
+import { env } from "../index";
+
+env.test("Save a note", async (ctx) => {
+  const app = await ctx.desktop(ctx.svc.app);
+  await app.getByRole("textbox", { name: "New note" }).fill("Hello desktop");
+  await app.getByRole("button", { name: "Add note", exact: true }).click();
+  await expect(app.getByRole("listitem")).toHaveText(["Hello desktop"]);
+});
+```
+
+Run `spectest test .` from the project root. The dashboard records desktop
+interactions and replays the app with window chrome derived from its Electron
+configuration. Child tests using `dependsOn` inherit the live app state,
+including unsaved input, in their own isolated environment.
+
+The app must declare Electron in `package.json`. Defaults are `npm ci` with a
+package lock (otherwise `npm install`), `npm run build`, and the package's
+`main` entry. Use `appDir`, `installCommand`, `buildCommand`, `entry`, and
+`nodeVersion` to customize the build. `env` supplies runtime variables;
+`dependsOn` waits for backend services in the same environment.
+
+Applications need Linux-compatible dependencies and binaries. Native OS dialogs
+and macOS-specific APIs are outside this basic support. Replays capture the
+renderer DOM from acquisition onward, not native OS surfaces.
+
+Run `spectest docs /components/electron` for the full configuration reference,
+backend setup, session behavior, and replay scope (CLI 0.2.124 or later).

package/dist/aws-sigv4.js

@@ -4,8 +4,8 @@
 // egress forward. The app under test signs with throwaway dummy credentials
 // (any AWS SDK refuses to build a request without *some* credential); the
 // broker strips that dummy signature and re-signs the exact outbound request
-// with the real credentials brokered eval-scoped from the platform Secrets
-// store. Because signing happens at forward time over the real outbound
+// with real credentials supplied by the platform for the environment lifetime.
+// Because signing happens at forward time over the real outbound
 // canonical request, `x-amz-date` / `x-amz-content-sha256` / `authorization`
 // are always internally consistent — which is exactly what static header
 // injection cannot achieve for SigV4 (the signature is a keyed HMAC over the

package/dist/components/emulate/entry.mjs

@@ -0,0 +1,242 @@
+// The `socialAuth()` container's entry point.
+//
+// It runs one emulator per configured provider (the `@emulators/*` packages
+// from vercel-labs/emulate) and puts all of them behind ONE port, routed by
+// `Host` header. That is what lets the component answer at the providers'
+// real endpoints: the daemon owns DNS and TLS for those names and proxies
+// every one of them here.
+//
+// The engine is generic. Everything provider-specific — which hosts to
+// claim, which paths differ from the real ones, which discovery values to
+// correct — arrives as JSON in SPECTEST_AUTH_CONFIG, built by
+// `../social-auth.ts`. Add a provider there, not here.
+//
+// Four things happen to a response on the way out:
+//
+//   * discovery — the emulator advertises every endpoint on its own single
+//     origin. Real providers spread them over several hosts. We merge the
+//     real URLs in.
+//   * JWKS — we serve one RSA public key, for every provider.
+//   * id_token — we re-issue it: same claims, RS256, our key, and an
+//     explicit `iss` where the provider declares one. The Google emulator
+//     signs HS256 and serves an empty JWKS, so no client can verify its
+//     token. Providers whose issuer differs from their base URL can
+//     declare it explicitly. No signature check is needed on the way in: the
+//     token comes from an in-process function call, not from the network.
+//   * the account picker page — each button gets a `data-testid`, so a
+//     browser test has a stable locator.
+//
+// Every request is logged. Per-case service logs reach the dashboard, so
+// the whole OAuth exchange is readable on the case page with no helper API.
+
+import { createServer } from "@emulators/core";
+import { SignJWT, decodeJwt, exportJWK } from "jose";
+
+const CONFIG = JSON.parse(process.env.SPECTEST_AUTH_CONFIG ?? "{}");
+const PORT = Number(CONFIG.port ?? 4000);
+/** Key id published in the JWKS and stamped on every re-issued id_token. */
+const KID = "spectest-social-auth";
+
+// ──────────────────────────────────────────────────────────────────────────
+// Signing key
+//
+// Generated once, at boot — which is before the warm-template snapshot, so
+// every fork of this environment holds the same key and a token minted in a
+// parent stays valid in a child. Never generate it later: after a fork the
+// guest RNG is frozen, and sibling forks would produce identical keys.
+// ──────────────────────────────────────────────────────────────────────────
+
+const KEY_PAIR = await crypto.subtle.generateKey(
+  {
+    name: "RSASSA-PKCS1-v1_5",
+    modulusLength: 2048,
+    publicExponent: new Uint8Array([1, 0, 1]),
+    hash: "SHA-256",
+  },
+  true,
+  ["sign", "verify"],
+);
+const JWKS = {
+  keys: [{ ...(await exportJWK(KEY_PAIR.publicKey)), kid: KID, use: "sig", alg: "RS256" }],
+};
+
+// ──────────────────────────────────────────────────────────────────────────
+// Providers
+// ──────────────────────────────────────────────────────────────────────────
+
+/** host → runtime record. One emulator can claim several hosts. */
+const BY_HOST = new Map();
+
+for (const spec of CONFIG.providers ?? []) {
+  const mod = await import(spec.module);
+  const plugin = mod[spec.pluginExport];
+  if (!plugin) {
+    throw new Error(
+      `social-auth: ${spec.module} has no export ${spec.pluginExport}`,
+    );
+  }
+
+  // `port` only feeds the emulator's default base URL, which we always
+  // override — nothing listens on it. Each emulator is a fetch handler.
+  const { app, store, webhooks } = createServer(plugin, {
+    port: PORT,
+    baseUrl: spec.baseUrl,
+    fallbackUser: spec.fallbackUser,
+  });
+  plugin.seed?.(store, spec.baseUrl);
+  if (spec.seed && mod.seedFromConfig) {
+    mod.seedFromConfig(store, spec.baseUrl, spec.seed, webhooks);
+  }
+
+  const runtime = {
+    name: spec.name,
+    baseUrl: spec.baseUrl,
+    oidc: spec.oidc !== false,
+    issuer: spec.issuer,
+    discovery: spec.discovery ?? {},
+    jwksPath: spec.jwksPath ? new RegExp(spec.jwksPath) : null,
+    aliases: spec.aliases,
+    rewrites: (spec.rewrites ?? []).map((r) => ({ re: new RegExp(r.from), to: r.to })),
+    fetch: app.fetch,
+  };
+  for (const host of spec.hosts) BY_HOST.set(host.toLowerCase(), runtime);
+  console.log(`[social-auth] ${spec.name} → ${spec.hosts.join(", ")}`);
+}
+
+// ──────────────────────────────────────────────────────────────────────────
+// Response rewriting
+// ──────────────────────────────────────────────────────────────────────────
+
+const DISCOVERY_RE = /\/\.well-known\/openid-configuration$/;
+
+/** Correct the discovery document: real hosts per endpoint, and RS256,
+ *  which is what we actually sign with after `reissueIdToken`. */
+async function patchDiscovery(res, provider) {
+  const doc = await res.json();
+  const patched = {
+    ...doc,
+    ...provider.discovery,
+    id_token_signing_alg_values_supported: ["RS256"],
+  };
+  return json(patched, res.status);
+}
+
+/** Re-issue the id_token on a token response. Claims pass through
+ *  unchanged except `iss`, which only a provider that cannot derive it
+ *  from its base URL overrides. */
+async function patchTokenResponse(res, provider) {
+  const body = await res.json();
+  if (typeof body.id_token !== "string") return json(body, res.status);
+
+  const claims = decodeJwt(body.id_token);
+  if (provider.issuer) claims.iss = provider.issuer;
+  body.id_token = await new SignJWT(claims)
+    .setProtectedHeader({ alg: "RS256", kid: KID, typ: "JWT" })
+    .sign(KEY_PAIR.privateKey);
+  return json(body, res.status);
+}
+
+/** Give every account button on the picker page a stable test id, and mark
+ *  the page itself. The hidden field that identifies the account is named
+ *  differently per provider, and does not always hold the address — GitHub
+ *  identifies an account by its login — so `aliases` maps it back. The test
+ *  id is the account's email on every provider. */
+async function patchPickerPage(res, provider) {
+  let html = await res.text();
+  html = html.replace(/<form class="user-form"[\s\S]*?<\/form>/g, (form) => {
+    const id = /<input type="hidden" name="(?:email|login|sub|username|user_ref)" value="([^"]*)"/.exec(form);
+    if (!id) return form;
+    const account = provider.aliases?.[id[1]] ?? id[1];
+    return form.replace(
+      "<button type=\"submit\"",
+      `<button type="submit" data-testid="spectest-user-${account}"`,
+    );
+  });
+  html = html.replace("<body", '<body data-spectest-signin="1"');
+  return new Response(html, { status: res.status, headers: res.headers });
+}
+
+function json(value, status) {
+  return new Response(JSON.stringify(value), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+// ──────────────────────────────────────────────────────────────────────────
+// Server
+// ──────────────────────────────────────────────────────────────────────────
+
+Bun.serve({
+  port: PORT,
+  // The default 10s kills nothing here, but an authorize page fetched by a
+  // cold browser can be slower than it looks. Ingress uses the same margin.
+  idleTimeout: 60,
+  async fetch(req) {
+    const url = new URL(req.url);
+
+    // Answered on any Host, because the ready check probes the container's
+    // own IP and never sends a provider name.
+    if (url.pathname === "/healthz") return new Response("ok\n");
+
+    // The daemon reverse-proxies us and rewrites `Host` to the service-net
+    // name, so the provider the client asked for only survives in
+    // `X-Forwarded-Host`. Read that first; `Host` covers a direct call.
+    const host = (req.headers.get("x-forwarded-host") ?? req.headers.get("host") ?? "")
+      .toLowerCase()
+      .replace(/:\d+$/, "");
+    const provider = BY_HOST.get(host);
+    if (!provider) {
+      return new Response(
+        `spectest social-auth: no provider claims Host=${JSON.stringify(host)}\n` +
+          `claimed hosts: ${[...BY_HOST.keys()].join(", ")}\n`,
+        { status: 404, headers: { "content-type": "text/plain" } },
+      );
+    }
+
+    // Real path → the path this emulator serves it on. Identity by default,
+    // which is also what carries the emulator's own internal endpoints (the
+    // picker's POST target) through untouched.
+    let path = url.pathname;
+    for (const rule of provider.rewrites) {
+      if (rule.re.test(path)) {
+        path = path.replace(rule.re, rule.to);
+        break;
+      }
+    }
+
+    let res;
+    if (provider.oidc && provider.jwksPath?.test(path)) {
+      // Our key, never the emulator's — we re-sign every id_token.
+      res = json(JWKS, 200);
+    } else {
+      const body =
+        req.method === "GET" || req.method === "HEAD" ? undefined : await req.arrayBuffer();
+      // Restore the Host the client used, so an emulator that reads it sees
+      // the provider rather than our container.
+      const headers = new Headers(req.headers);
+      headers.set("host", host);
+      const inner = new Request(new URL(path + url.search, provider.baseUrl), {
+        method: req.method,
+        headers,
+        body,
+      });
+      res = await provider.fetch(inner);
+
+      const type = res.headers.get("content-type") ?? "";
+      if (type.includes("json")) {
+        if (DISCOVERY_RE.test(path)) res = await patchDiscovery(res, provider);
+        else if (provider.oidc) res = await patchTokenResponse(res, provider);
+      } else if (type.includes("text/html")) {
+        res = await patchPickerPage(res, provider);
+      }
+    }
+
+    console.log(
+      `[social-auth] ${provider.name} ${req.method} ${host}${url.pathname} → ${res.status}`,
+    );
+    return res;
+  },
+});
+
+console.log(`[social-auth] listening on :${PORT}`);

package/dist/components/replayFake.d.ts

@@ -38,9 +38,9 @@ export interface InjectMatch {
  *  are SET on the egress forward — overwriting whatever the app sent (so app
  *  code can't smuggle a different value past the broker). Header VALUES may
  *  embed `{{secret:REF}}` tokens; each `REF` is resolved server-side from the
- *  project's Secrets store and pushed eval-scoped — the real value never
- *  enters project code, the
- *  tarball, the warm-cache hash, or a cassette (it's redacted, fail-closed). */
+ *  project's Secrets store and supplied for the environment lifetime. Values
+ *  never enter project files, the warm-cache hash, or a cassette (redaction
+ *  fails closed). */
 export interface InjectRule {
     match?: InjectMatch;
     headers: Record<string, string>;
@@ -52,7 +52,7 @@ export interface InjectRule {
  *  signature is a keyed HMAC over the whole request, not a static token.
  *
  *  The three fields are **secret refs, resolved directly** from the project's
- *  Secrets store (NOT `{{secret:}}` templates) — same eval-scoped push and
+ *  Secrets store (NOT `{{secret:}}` templates), with the same lifetime and
  *  fail-closed redaction as `inject`. Region + service are inferred from the
  *  incoming request (its credential scope, else the host); the app never
  *  configures them. */

package/dist/components/replayFake.js

@@ -19,7 +19,7 @@
 //   - RECORD (MITM): the request is forwarded to the REAL host and the
 //     request/response pair is captured (decoded, redacted), and the real
 //     response is returned to the app so a manual session behaves like
-//     production. This runs under `spectest env eval` against a manual env.
+//     production. This runs for all interactions with a manual environment.
 //
 // Because the fake's hostname IS the real host, the in-VM resolver points
 // that name at the daemon — so a naive `fetch("https://api.stripe.com")`
@@ -33,26 +33,25 @@
 //
 // Mode is chosen by `isRecording()`: it is true only inside an active
 // recorder (a `spectest test` case), false in eval/manual. So `auto`
-// (the default) replays under test and records under eval — no new
+// (the default) replays under test and records during manual use — no new
 // control-plane mode flag. `mode: "record" | "replay"` overrides it.
 //
 // Credential brokering (per-fake): `inject` rules set
 // headers on the egress forward — overwriting whatever the app sent — so
 // app code never holds the credential. Header values embed `{{secret:REF}}`
 // tokens; each REF is resolved server-side from the platform Secrets store
-// and pushed eval-scoped from the control plane (via {@link getRecordSecret}).
-// The real value lives only on the outbound wire to the real upstream and
-// is redacted from the cassette (fail-closed) — it never enters project
-// code, the tarball, the warm-cache hash, or any snapshot a hermetic run
-// could fork.
+// and supplied for the environment lifetime (via {@link getRecordSecret}).
+// Values live in harness memory (including snapshots) and on the outbound
+// wire. They are redacted from cassettes (fail-closed) and never enter
+// project files or the warm-cache hash.
 //
 // AWS SigV4 (`sign: { type: "awsSigv4", ... }`): static header injection
 // can't broker AWS auth — the `Authorization` value is a keyed HMAC over the
 // entire request, not a static token. So the forward instead RE-SIGNS: the
 // app signs with throwaway dummy creds (any AWS SDK refuses to build a request
 // with none), we strip that signature and re-sign the exact outbound request
-// with the real credentials (also direct secret refs, same eval-scoped push +
-// redaction). Region/service are inferred from the request. See
+// with the real credentials (direct secret refs, supplied for the environment
+// lifetime and redacted). Region/service are inferred from the request. See
 // `../aws-sigv4.ts`.
 import { createSocket } from "node:dgram";
 import { createHash } from "node:crypto";
@@ -308,7 +307,7 @@ function ruleMatches(match, req) {
     return true;
 }
 /** Resolve a rule's header templates, substituting every `{{secret:REF}}`
- *  with the eval-scoped value. Collects resolved secrets (for redaction)
+ *  with the environment credential. Collects resolved secrets (for redaction)
  *  and any refs the control plane didn't supply (fail loud). */
 function brokerHeaders(rule) {
     const headers = {};
@@ -438,7 +437,7 @@ export function replayFake(opts) {
     const match = defaultMatch(opts.match);
     const injectRules = opts.inject ?? [];
     // Refs every header template references — the control plane resolves these
-    // server-side and pushes them eval-scoped (see the daemon's
+    // server-side and pushes them for the environment lifetime (see the daemon's
     // /record-secret-refs endpoint, which reads `def.secretRefs`).
     const secretRefs = collectSecretRefs(injectRules, opts.sign);
     const resolveMode = () => {
@@ -477,7 +476,7 @@ export function replayFake(opts) {
     // ── RECORD (MITM) ───────────────────────────────────────────────────
     async function record(req, url, reqLike, bodyBytes, state) {
         // Credential brokering (static header inject): first matching rule wins.
-        // Resolve its `{{secret:REF}}` tokens from the eval-scoped store.
+        // Resolve its `{{secret:REF}}` tokens from the environment secret store.
         const rule = injectRules.find((r) => ruleMatches(r.match, {
             method: reqLike.method,
             path: reqLike.path,
@@ -509,7 +508,7 @@ export function replayFake(opts) {
         if (missing.length > 0) {
             const refs = [...new Set(missing)];
             return new Response(`replayFake(${opts.name}): secret(s) ${JSON.stringify(refs)} were not supplied ` +
-                `(configure them on the project's Secrets page, and record via \`spectest env eval\`).\n`, { status: 599, headers: { "content-type": "text/plain" } });
+                `(configure them on the project's Secrets page, then start a new environment or fork).\n`, { status: 599, headers: { "content-type": "text/plain" } });
         }
         // Resolve the REAL host's IP via an external resolver so we don't loop
         // back into the daemon (the in-VM resolver answers our own gateway for

package/dist/daemon.js

@@ -57,7 +57,7 @@ import { startTlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
 import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
 import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
-import { conflict, notFound, requireString, } from "./harness/methods.js";
+import { badRequest, conflict, notFound, requireString, } from "./harness/methods.js";
 import { openTerminal } from "./terminal.js";
 // `ctx.mcp(url)`. One client per call, no registry: an authenticated
 // client is passed to descendants as a test's return value.
@@ -65,7 +65,7 @@ import { openMcp } from "./mcp.js";
 import { readAnnotation } from "./annotate.js";
 import { isRecording, recordEmail, recordEnv, recordExec, recordFake, recordStep, recordTerminal, recordWait, reserveEvent, recorderEventCount, recorderMarkChildren, recorderTruncate, startRecording, stopRecording, truncateUtf8, } from "./recorder.js";
 import { deepUnwrap, readRaw, wrap } from "./inspect.js";
-import { clearRecordSecrets, setRecordSecrets } from "./record-secrets.js";
+import { setRecordSecrets } from "./record-secrets.js";
 import { encodeReplayBundle, replayChunk } from "./replay-bundle.js";
 function namedServices(cfg) {
     return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def }));
@@ -5030,13 +5030,8 @@ function explainEvalExportError(code, message) {
         "    return out;\n" +
         "  })();\n");
 }
-async function evalCode(code, secrets) {
+async function evalCode(code) {
     const start = Date.now();
-    // Eval-scoped secret channel for record-mode fakes — set before the
-    // snippet runs, cleared in the `finally` below so a secret never
-    // persists into daemon memory (and thus into a forkable snapshot) past
-    // the eval that supplied it. See record-secrets.ts.
-    setRecordSecrets(secrets);
     const chunks = [];
     const origStdout = process.stdout.write.bind(process.stdout);
     const origStderr = process.stderr.write.bind(process.stderr);
@@ -5204,7 +5199,6 @@ async function evalCode(code, secrets) {
         };
     }
     finally {
-        clearRecordSecrets();
         fetchScope.active = false;
         restoreFetch();
         restoreConsole();
@@ -5515,6 +5509,14 @@ function loadedSummary(l) {
  * cutover.
  */
 export function harnessMethods(state) {
+    const configureSecrets = (params) => {
+        const secrets = params.secrets;
+        if (!secrets || typeof secrets !== "object" || Array.isArray(secrets) ||
+            Object.values(secrets).some(value => typeof value !== "string")) {
+            throw badRequest("secrets must be an object of string values");
+        }
+        setRecordSecrets(secrets);
+    };
     return {
         health: async () => ({ ok: true, capabilities: ["prepared-images-v1"] }),
         // Live bootstrap progress, polled by the control plane during
@@ -5606,14 +5608,18 @@ export function harnessMethods(state) {
         }),
         // Union of platform secret refs the loaded fakes declare (replayFake's
         // `secretRefs`). The control plane resolves these server-side and pushes
-        // the values on the eval path only. Empty if nothing's loaded.
+        // the values for the environment lifetime. Empty if nothing's loaded.
         recordSecretRefs: async () => {
             const refs = new Set();
             for (const fake of FAKES.values()) {
                 for (const ref of fake.def.secretRefs ?? [])
                     refs.add(ref);
             }
-            return { refs: [...refs] };
+            return { refs: [...refs], environmentScoped: true };
+        },
+        setRecordSecrets: async (params) => {
+            configureSecrets(params);
+            return { ok: true };
         },
         prepareImages: async (params) => exclusive(state, "inFlightBootstrap", "bootstrap already in progress", async () => {
             const restore = captureConsole(BOOT_LOG_SINK);
@@ -5631,11 +5637,13 @@ export function harnessMethods(state) {
         projectSetup: async () => exclusive(state, "inFlightProjectSetup", "project-setup already in progress", () => runProjectSetup()),
         eval: async (params) => {
             const code = requireString(params, "code");
-            // `secrets` are eval-scoped: the control plane resolves the loaded
-            // project's declared `replayFake` refs and pushes the values here on
-            // the eval path only. Never present on the test path.
-            const secrets = params.secrets;
-            return exclusive(state, "inFlightTest", "a test or eval is already running", () => evalCode(code, secrets));
+            return exclusive(state, "inFlightTest", "a test or eval is already running", () => {
+                // Compatibility with older control planes: adopt their supplied values,
+                // but keep them for subsequent browser/background requests as well.
+                if (params.secrets !== undefined)
+                    configureSecrets(params);
+                return evalCode(code);
+            });
         },
         /**
          * Snapshot each service's log output produced during env bring-up

package/dist/harness/protocol.d.ts

@@ -23,7 +23,7 @@
 /** Protocol version this build speaks. Must match `PROTOCOL_VERSION` in protocol.rs. */
 export declare const PROTOCOL_VERSION = 1;
 /** Methods either side can send. */
-export type Method = "hello" | "load" | "loadTests" | "fingerprint" | "bootstrap" | "run" | "eval" | "teardown" | "shutdown" | "createArtifact" | "completeArtifact" | "progress" | "step";
+export type Method = "hello" | "load" | "loadTests" | "fingerprint" | "bootstrap" | "run" | "eval" | "setRecordSecrets" | "teardown" | "shutdown" | "createArtifact" | "completeArtifact" | "progress" | "step";
 /** Methods this build knows how to dispatch. A method outside this set still
  *  *parses* — it is answered with an error, never dropped. */
 export declare const KNOWN_METHODS: ReadonlySet<string>;

package/dist/harness/protocol.js

@@ -32,6 +32,7 @@ export const KNOWN_METHODS = new Set([
     "bootstrap",
     "run",
     "eval",
+    "setRecordSecrets",
     "teardown",
     "shutdown",
     "createArtifact",

package/dist/index.d.ts

@@ -1469,8 +1469,8 @@ export interface FakeDefinition<S = any, H extends Record<string, unknown> = Rec
      * Internal. Platform secret references this fake needs at *record* time
      * (set by {@link replayFake}). The daemon reports the union of these to
      * the control plane, which resolves each via its `SecretResolver` and
-     * pushes the values on the eval path only — they never enter project
-     * files, the config hash, or a cassette. Not part of the authoring
+     * supplies them for the environment lifetime, including snapshots. Values
+     * never enter project files, the config hash, or a cassette. Not part of the authoring
      * surface; `JSON.stringify` ignores it (fakes never serialize to config).
      */
     secretRefs?: readonly string[];

package/dist/record-secrets.d.ts

@@ -1,9 +1,4 @@
-/** Replace the eval-scoped secret set. Called by the daemon at the start
- *  of each `/eval` with the values the control plane resolved. */
-export declare function setRecordSecrets(secrets: Record<string, string> | undefined): void;
-/** Drop all secrets. Called from the eval's `finally` so nothing survives
- *  past the eval that supplied them. */
-export declare function clearRecordSecrets(): void;
-/** Resolve a secret by its platform `ref`. `undefined` if the control
- *  plane didn't supply it (ref not configured, or not on the eval path). */
+/** Replace the complete environment secret set, including removing stale refs. */
+export declare function setRecordSecrets(secrets: Record<string, string>): void;
+/** Resolve a declared project secret for an outbound recording request. */
 export declare function getRecordSecret(ref: string): string | undefined;

package/dist/record-secrets.js

@@ -1,39 +1,18 @@
-// Eval-scoped secret channel for record-mode fakes (see
-// `components/replayFake.ts`).
-//
-// The control plane resolves a platform secret (e.g. `STRIPE_API_KEY`)
-// server-side and pushes it into the daemon on the `/eval` request body
-// only — never on the `run_tests` path, never into the project tarball,
-// the config hash, or any cassette. The daemon stashes the resolved
-// values here for exactly the duration of one eval (set before the
-// snippet runs, cleared in the eval's `finally`) and the record-mode fake
-// forwarder reads them via {@link getRecordSecret}.
-//
-// Eval-scoped is load-bearing: daemon memory survives snapshot/fork, so a
-// long-lived secrets map could ride a warm/post-test snapshot into a
-// forkable state a hermetic `spectest test` could observe. Clearing per
-// eval means a secret never persists into anything a replay run can reach.
-// This module is shared (one instance per daemon process) so the daemon
-// writes and the component reads the same map.
+// Environment-lifetime credentials for record-mode fakes. The control plane
+// resolves declared references from the owning project's Secrets store before
+// service startup and refreshes them on warm starts and interactive forks.
+// They persist across evals, background requests and memory snapshots, just
+// like other environment state. Test forks inherit their parent's credentials;
+// replay mode still uses cassettes and never forwards upstream.
+// Values stay out of project files, config hashes and recorded cassettes.
 const SECRETS = new Map();
-/** Replace the eval-scoped secret set. Called by the daemon at the start
- *  of each `/eval` with the values the control plane resolved. */
+/** Replace the complete environment secret set, including removing stale refs. */
 export function setRecordSecrets(secrets) {
     SECRETS.clear();
-    if (!secrets)
-        return;
-    for (const [ref, value] of Object.entries(secrets)) {
-        if (typeof value === "string")
-            SECRETS.set(ref, value);
-    }
+    for (const [ref, value] of Object.entries(secrets))
+        SECRETS.set(ref, value);
 }
-/** Drop all secrets. Called from the eval's `finally` so nothing survives
- *  past the eval that supplied them. */
-export function clearRecordSecrets() {
-    SECRETS.clear();
-}
-/** Resolve a secret by its platform `ref`. `undefined` if the control
- *  plane didn't supply it (ref not configured, or not on the eval path). */
+/** Resolve a declared project secret for an outbound recording request. */
 export function getRecordSecret(ref) {
     return SECRETS.get(ref);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.88.1",
+  "version": "0.88.3",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/aws-sigv4.ts

@@ -4,8 +4,8 @@
 // egress forward. The app under test signs with throwaway dummy credentials
 // (any AWS SDK refuses to build a request without *some* credential); the
 // broker strips that dummy signature and re-signs the exact outbound request
-// with the real credentials brokered eval-scoped from the platform Secrets
-// store. Because signing happens at forward time over the real outbound
+// with real credentials supplied by the platform for the environment lifetime.
+// Because signing happens at forward time over the real outbound
 // canonical request, `x-amz-date` / `x-amz-content-sha256` / `authorization`
 // are always internally consistent — which is exactly what static header
 // injection cannot achieve for SigV4 (the signature is a keyed HMAC over the

package/src/components/replayFake.ts

@@ -19,7 +19,7 @@
 //   - RECORD (MITM): the request is forwarded to the REAL host and the
 //     request/response pair is captured (decoded, redacted), and the real
 //     response is returned to the app so a manual session behaves like
-//     production. This runs under `spectest env eval` against a manual env.
+//     production. This runs for all interactions with a manual environment.
 //
 // Because the fake's hostname IS the real host, the in-VM resolver points
 // that name at the daemon — so a naive `fetch("https://api.stripe.com")`
@@ -33,26 +33,25 @@
 //
 // Mode is chosen by `isRecording()`: it is true only inside an active
 // recorder (a `spectest test` case), false in eval/manual. So `auto`
-// (the default) replays under test and records under eval — no new
+// (the default) replays under test and records during manual use — no new
 // control-plane mode flag. `mode: "record" | "replay"` overrides it.
 //
 // Credential brokering (per-fake): `inject` rules set
 // headers on the egress forward — overwriting whatever the app sent — so
 // app code never holds the credential. Header values embed `{{secret:REF}}`
 // tokens; each REF is resolved server-side from the platform Secrets store
-// and pushed eval-scoped from the control plane (via {@link getRecordSecret}).
-// The real value lives only on the outbound wire to the real upstream and
-// is redacted from the cassette (fail-closed) — it never enters project
-// code, the tarball, the warm-cache hash, or any snapshot a hermetic run
-// could fork.
+// and supplied for the environment lifetime (via {@link getRecordSecret}).
+// Values live in harness memory (including snapshots) and on the outbound
+// wire. They are redacted from cassettes (fail-closed) and never enter
+// project files or the warm-cache hash.
 //
 // AWS SigV4 (`sign: { type: "awsSigv4", ... }`): static header injection
 // can't broker AWS auth — the `Authorization` value is a keyed HMAC over the
 // entire request, not a static token. So the forward instead RE-SIGNS: the
 // app signs with throwaway dummy creds (any AWS SDK refuses to build a request
 // with none), we strip that signature and re-sign the exact outbound request
-// with the real credentials (also direct secret refs, same eval-scoped push +
-// redaction). Region/service are inferred from the request. See
+// with the real credentials (direct secret refs, supplied for the environment
+// lifetime and redacted). Region/service are inferred from the request. See
 // `../aws-sigv4.ts`.
 
 import { createSocket } from "node:dgram";
@@ -116,9 +115,9 @@ export interface InjectMatch {
  *  are SET on the egress forward — overwriting whatever the app sent (so app
  *  code can't smuggle a different value past the broker). Header VALUES may
  *  embed `{{secret:REF}}` tokens; each `REF` is resolved server-side from the
- *  project's Secrets store and pushed eval-scoped — the real value never
- *  enters project code, the
- *  tarball, the warm-cache hash, or a cassette (it's redacted, fail-closed). */
+ *  project's Secrets store and supplied for the environment lifetime. Values
+ *  never enter project files, the warm-cache hash, or a cassette (redaction
+ *  fails closed). */
 export interface InjectRule {
   match?: InjectMatch;
   headers: Record<string, string>;
@@ -131,7 +130,7 @@ export interface InjectRule {
  *  signature is a keyed HMAC over the whole request, not a static token.
  *
  *  The three fields are **secret refs, resolved directly** from the project's
- *  Secrets store (NOT `{{secret:}}` templates) — same eval-scoped push and
+ *  Secrets store (NOT `{{secret:}}` templates), with the same lifetime and
  *  fail-closed redaction as `inject`. Region + service are inferred from the
  *  incoming request (its credential scope, else the host); the app never
  *  configures them. */
@@ -531,7 +530,7 @@ interface BrokeredHeaders {
 }
 
 /** Resolve a rule's header templates, substituting every `{{secret:REF}}`
- *  with the eval-scoped value. Collects resolved secrets (for redaction)
+ *  with the environment credential. Collects resolved secrets (for redaction)
  *  and any refs the control plane didn't supply (fail loud). */
 function brokerHeaders(rule: InjectRule): BrokeredHeaders {
   const headers: Record<string, string> = {};
@@ -675,7 +674,7 @@ export function replayFake(
   const match = defaultMatch(opts.match);
   const injectRules = opts.inject ?? [];
   // Refs every header template references — the control plane resolves these
-  // server-side and pushes them eval-scoped (see the daemon's
+  // server-side and pushes them for the environment lifetime (see the daemon's
   // /record-secret-refs endpoint, which reads `def.secretRefs`).
   const secretRefs = collectSecretRefs(injectRules, opts.sign);
 
@@ -722,7 +721,7 @@ export function replayFake(
     state: CassetteState,
   ): Promise<Response> {
     // Credential brokering (static header inject): first matching rule wins.
-    // Resolve its `{{secret:REF}}` tokens from the eval-scoped store.
+    // Resolve its `{{secret:REF}}` tokens from the environment secret store.
     const rule = injectRules.find((r) =>
       ruleMatches(r.match, {
         method: reqLike.method,
@@ -756,7 +755,7 @@ export function replayFake(
       const refs = [...new Set(missing)];
       return new Response(
         `replayFake(${opts.name}): secret(s) ${JSON.stringify(refs)} were not supplied ` +
-          `(configure them on the project's Secrets page, and record via \`spectest env eval\`).\n`,
+          `(configure them on the project's Secrets page, then start a new environment or fork).\n`,
         { status: 599, headers: { "content-type": "text/plain" } },
       );
     }

package/src/daemon.ts

@@ -168,6 +168,7 @@ import {
   resolveChownIds,
 } from "./harness/file-mounts.js";
 import {
+  badRequest,
   codeOf,
   conflict,
   notFound,
@@ -201,7 +202,7 @@ import {
 } from "./recorder.js";
 import { deepUnwrap, readRaw, wrap } from "./inspect.js";
 import type { Wrapped } from "./inspect.js";
-import { clearRecordSecrets, setRecordSecrets } from "./record-secrets.js";
+import { setRecordSecrets } from "./record-secrets.js";
 import { generateId } from "./ids.js";
 import { encodeReplayBundle, replayChunk } from "./replay-bundle.js";
 
@@ -6029,16 +6030,8 @@ function explainEvalExportError(code: string, message: string): string {
   );
 }
 
-async function evalCode(
-  code: string,
-  secrets?: Record<string, string>,
-): Promise<EvalResult> {
+async function evalCode(code: string): Promise<EvalResult> {
   const start = Date.now();
-  // Eval-scoped secret channel for record-mode fakes — set before the
-  // snippet runs, cleared in the `finally` below so a secret never
-  // persists into daemon memory (and thus into a forkable snapshot) past
-  // the eval that supplied it. See record-secrets.ts.
-  setRecordSecrets(secrets);
   const chunks: string[] = [];
   const origStdout = process.stdout.write.bind(process.stdout);
   const origStderr = process.stderr.write.bind(process.stderr);
@@ -6233,7 +6226,6 @@ async function evalCode(
       error: { message: explainEvalExportError(code, message), stack: e.stack },
     };
   } finally {
-    clearRecordSecrets();
     fetchScope.active = false;
     restoreFetch();
     restoreConsole();
@@ -6609,6 +6601,14 @@ function loadedSummary(l: ReturnType<typeof requireLoaded>) {
  * cutover.
  */
 export function harnessMethods(state: RouteState): MethodTable {
+  const configureSecrets = (params: Record<string, unknown>) => {
+    const secrets = params.secrets;
+    if (!secrets || typeof secrets !== "object" || Array.isArray(secrets) ||
+        Object.values(secrets).some(value => typeof value !== "string")) {
+      throw badRequest("secrets must be an object of string values");
+    }
+    setRecordSecrets(secrets as Record<string, string>);
+  };
   return {
     health: async () => ({ ok: true, capabilities: ["prepared-images-v1"] }),
 
@@ -6710,13 +6710,18 @@ export function harnessMethods(state: RouteState): MethodTable {
 
     // Union of platform secret refs the loaded fakes declare (replayFake's
     // `secretRefs`). The control plane resolves these server-side and pushes
-    // the values on the eval path only. Empty if nothing's loaded.
+    // the values for the environment lifetime. Empty if nothing's loaded.
     recordSecretRefs: async () => {
       const refs = new Set<string>();
       for (const fake of FAKES.values()) {
         for (const ref of fake.def.secretRefs ?? []) refs.add(ref);
       }
-      return { refs: [...refs] };
+      return { refs: [...refs], environmentScoped: true };
+    },
+
+    setRecordSecrets: async (params) => {
+      configureSecrets(params);
+      return { ok: true };
     },
 
     prepareImages: async (params) =>
@@ -6744,13 +6749,12 @@ export function harnessMethods(state: RouteState): MethodTable {
 
     eval: async (params) => {
       const code = requireString(params, "code");
-      // `secrets` are eval-scoped: the control plane resolves the loaded
-      // project's declared `replayFake` refs and pushes the values here on
-      // the eval path only. Never present on the test path.
-      const secrets = params.secrets as Record<string, string> | undefined;
-      return exclusive(state, "inFlightTest", "a test or eval is already running", () =>
-        evalCode(code, secrets),
-      );
+      return exclusive(state, "inFlightTest", "a test or eval is already running", () => {
+        // Compatibility with older control planes: adopt their supplied values,
+        // but keep them for subsequent browser/background requests as well.
+        if (params.secrets !== undefined) configureSecrets(params);
+        return evalCode(code);
+      });
     },
 
     /**

package/src/harness/protocol.ts

@@ -34,6 +34,7 @@ export type Method =
   | "bootstrap"
   | "run"
   | "eval"
+  | "setRecordSecrets"
   | "teardown"
   | "shutdown"
   // up: harness → supervisor
@@ -52,6 +53,7 @@ export const KNOWN_METHODS: ReadonlySet<string> = new Set<Method>([
   "bootstrap",
   "run",
   "eval",
+  "setRecordSecrets",
   "teardown",
   "shutdown",
   "createArtifact",

package/src/index.ts

@@ -2120,8 +2120,8 @@ export interface FakeDefinition<
    * Internal. Platform secret references this fake needs at *record* time
    * (set by {@link replayFake}). The daemon reports the union of these to
    * the control plane, which resolves each via its `SecretResolver` and
-   * pushes the values on the eval path only — they never enter project
-   * files, the config hash, or a cassette. Not part of the authoring
+   * supplies them for the environment lifetime, including snapshots. Values
+   * never enter project files, the config hash, or a cassette. Not part of the authoring
    * surface; `JSON.stringify` ignores it (fakes never serialize to config).
    */
   secretRefs?: readonly string[];

package/src/record-secrets.test.ts

@@ -0,0 +1,54 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+test("environment secrets survive successful and failed evals and refresh as a complete set", async () => {
+  // Isolate daemon globals, eval instrumentation and app paths from other tests.
+  const root = await mkdtemp(join(tmpdir(), "spectest-secrets-"));
+  try {
+    const daemon = new URL("./daemon.ts", import.meta.url).pathname;
+    const secrets = new URL("./record-secrets.ts", import.meta.url).pathname;
+    const script = `
+      import { strict as assert } from "node:assert";
+      import { METHODS } from ${JSON.stringify(daemon)};
+      import { getRecordSecret } from ${JSON.stringify(secrets)};
+      const refs = await METHODS.recordSecretRefs({});
+      assert.equal(refs.environmentScoped, true);
+      await METHODS.setRecordSecrets({ secrets: { API_KEY: "first", REMOVED: "old" } });
+      // A request outside eval can use credentials immediately.
+      assert.equal(getRecordSecret("API_KEY"), "first");
+      const success = await METHODS.eval({ code: "export default 42;" });
+      assert.equal(success.ok, true, JSON.stringify(success));
+      assert.equal(getRecordSecret("API_KEY"), "first");
+      const failure = await METHODS.eval({ code: 'throw new Error("expected");' });
+      assert.equal(failure.ok, false);
+      assert.equal(getRecordSecret("API_KEY"), "first");
+      await assert.rejects(METHODS.setRecordSecrets({ secrets: { API_KEY: 123 } }));
+      assert.equal(getRecordSecret("API_KEY"), "first");
+      // Warm starts and manual forks replace inherited values, including deletions.
+      await METHODS.setRecordSecrets({ secrets: { API_KEY: "rotated" } });
+      assert.equal(getRecordSecret("API_KEY"), "rotated");
+      assert.equal(getRecordSecret("REMOVED"), undefined);
+      await METHODS.setRecordSecrets({ secrets: {} });
+      assert.equal(getRecordSecret("API_KEY"), undefined);
+      // A new SDK also works with a control plane that still sends eval secrets.
+      await METHODS.eval({ code: "export default 1;", secrets: { API_KEY: "legacy" } });
+      await METHODS.eval({ code: "export default 2;" });
+      assert.equal(getRecordSecret("API_KEY"), "legacy");
+    `;
+    const child = Bun.spawn([process.execPath, "--eval", script], {
+      env: { ...process.env, SPECTEST_APP_DIR: root },
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    const [code, stdout, stderr] = await Promise.all([
+      child.exited,
+      new Response(child.stdout).text(),
+      new Response(child.stderr).text(),
+    ]);
+    expect(code, stdout + stderr).toBe(0);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}, 20_000);

package/src/record-secrets.ts

@@ -1,41 +1,20 @@
-// Eval-scoped secret channel for record-mode fakes (see
-// `components/replayFake.ts`).
-//
-// The control plane resolves a platform secret (e.g. `STRIPE_API_KEY`)
-// server-side and pushes it into the daemon on the `/eval` request body
-// only — never on the `run_tests` path, never into the project tarball,
-// the config hash, or any cassette. The daemon stashes the resolved
-// values here for exactly the duration of one eval (set before the
-// snippet runs, cleared in the eval's `finally`) and the record-mode fake
-// forwarder reads them via {@link getRecordSecret}.
-//
-// Eval-scoped is load-bearing: daemon memory survives snapshot/fork, so a
-// long-lived secrets map could ride a warm/post-test snapshot into a
-// forkable state a hermetic `spectest test` could observe. Clearing per
-// eval means a secret never persists into anything a replay run can reach.
-// This module is shared (one instance per daemon process) so the daemon
-// writes and the component reads the same map.
+// Environment-lifetime credentials for record-mode fakes. The control plane
+// resolves declared references from the owning project's Secrets store before
+// service startup and refreshes them on warm starts and interactive forks.
+// They persist across evals, background requests and memory snapshots, just
+// like other environment state. Test forks inherit their parent's credentials;
+// replay mode still uses cassettes and never forwards upstream.
+// Values stay out of project files, config hashes and recorded cassettes.
 
 const SECRETS = new Map<string, string>();
 
-/** Replace the eval-scoped secret set. Called by the daemon at the start
- *  of each `/eval` with the values the control plane resolved. */
-export function setRecordSecrets(secrets: Record<string, string> | undefined): void {
+/** Replace the complete environment secret set, including removing stale refs. */
+export function setRecordSecrets(secrets: Record<string, string>): void {
   SECRETS.clear();
-  if (!secrets) return;
-  for (const [ref, value] of Object.entries(secrets)) {
-    if (typeof value === "string") SECRETS.set(ref, value);
-  }
+  for (const [ref, value] of Object.entries(secrets)) SECRETS.set(ref, value);
 }
 
-/** Drop all secrets. Called from the eval's `finally` so nothing survives
- *  past the eval that supplied them. */
-export function clearRecordSecrets(): void {
-  SECRETS.clear();
-}
-
-/** Resolve a secret by its platform `ref`. `undefined` if the control
- *  plane didn't supply it (ref not configured, or not on the eval path). */
+/** Resolve a declared project secret for an outbound recording request. */
 export function getRecordSecret(ref: string): string | undefined {
   return SECRETS.get(ref);
 }