@workos/quickstudy 0.0.1

package/src/hash.ts

@@ -0,0 +1,74 @@
+/**
+ * Content-hashing primitives for the v3 run manifest. Extracted from the
+ * legacy experiment-identity module (deleted with the v2 stack) as a
+ * byte-for-byte behavior match, so digests recorded by v2 runs stay
+ * comparable with v3 ones.
+ */
+
+import { createHash } from "node:crypto";
+import { lstatSync, readdirSync, readFileSync, readlinkSync } from "node:fs";
+import { basename, join, relative } from "node:path";
+
+/** A content digest plus the file accounting that produced it. */
+export interface ContentIdentity {
+  sha256: string;
+  files?: number;
+  excluded?: number;
+}
+
+/**
+ * Basenames that look like credentials never enter a content digest.
+ * (The legacy scenario loader's sidecar filename guard shared this exact
+ * regex before its deletion; it stays exported for tests and any future
+ * consumer with the same "invisible to the digest ⇒ carries no behavior"
+ * invariant.)
+ */
+export const SECRET_FILE = /(?:^|[._-])(?:env|credentials?|tokens?|secrets?|private[._-]?keys?|mcp[._-]?auth)(?:$|[._-])|\.(?:pem|key|p12|pfx)$/i;
+const SKIP_DIR = new Set([".git", "node_modules", "vendor", ".venv", "results", "dist"]);
+
+/** SHA-256 hex digest of a string or byte buffer. */
+export function hashString(value: string | Uint8Array): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+/**
+ * Hash a directory exactly by relative path, type, executable bit and bytes.
+ * Secret-looking files are rejected before read and represented only by an
+ * exclusion count; their names and contents never enter the digest.
+ */
+export function hashTree(root: string): ContentIdentity {
+  const hash = createHash("sha256");
+  let files = 0;
+  let excluded = 0;
+  const walk = (dir: string): void => {
+    for (const name of readdirSync(dir).sort()) {
+      const path = join(dir, name);
+      const rel = relative(root, path).split("\\").join("/");
+      const stat = lstatSync(path);
+      if (stat.isDirectory()) {
+        if (SKIP_DIR.has(name) || SECRET_FILE.test(name)) {
+          excluded += 1;
+          continue;
+        }
+        walk(path);
+        continue;
+      }
+      if (SECRET_FILE.test(basename(path))) {
+        excluded += 1;
+        continue;
+      }
+      if (stat.isSymbolicLink()) {
+        hash.update(`L\0${rel}\0${readlinkSync(path)}\0`);
+        files += 1;
+        continue;
+      }
+      if (!stat.isFile()) continue;
+      hash.update(`F\0${rel}\0${stat.mode & 0o111 ? "x" : "-"}\0`);
+      hash.update(readFileSync(path));
+      hash.update("\0");
+      files += 1;
+    }
+  };
+  walk(root);
+  return { sha256: hash.digest("hex"), files, excluded };
+}

package/src/identity-diff.ts

@@ -0,0 +1,30 @@
+/** Browser-safe canonicalization and structural identity diffing. */
+
+export function canonicalJson(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  if (value !== null && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .filter(([, item]) => item !== undefined)
+      .sort(([a], [b]) => a.localeCompare(b));
+    return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
+  }
+  return JSON.stringify(value);
+}
+
+export interface IdentityMismatch {
+  path: string;
+  a: unknown;
+  b: unknown;
+}
+
+export function diffIdentity(a: unknown, b: unknown, path = "identity"): IdentityMismatch[] {
+  if (canonicalJson(a) === canonicalJson(b)) return [];
+  if (a !== null && b !== null && typeof a === "object" && typeof b === "object" && !Array.isArray(a) && !Array.isArray(b)) {
+    const left = a as Record<string, unknown>;
+    const right = b as Record<string, unknown>;
+    return [...new Set([...Object.keys(left), ...Object.keys(right)])]
+      .sort()
+      .flatMap((key) => diffIdentity(left[key], right[key], `${path}.${key}`));
+  }
+  return [{ path, a: a ?? null, b: b ?? null }];
+}

package/src/ids.ts

@@ -0,0 +1,30 @@
+/**
+ * Minimal ULID generator (https://github.com/ulid/spec).
+ *
+ * The data model keys runs and attempts by ulid, but this phase's runtime
+ * dependencies are limited to `zod` and `yaml` — so we hand-roll the 26
+ * characters here rather than pull a package: 48-bit millisecond timestamp +
+ * 80 bits of randomness, Crockford base32. Lexicographic order is creation
+ * order, which keeps `results/<run>/<attempt>/` listings chronological.
+ */
+
+const ENCODING = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+
+/**
+ * `random` overrides the 16 entropy bytes — only fixture seeders pass it, so
+ * committed dev data regenerates byte-identically. Production callers omit it.
+ */
+export function ulid(now: number = Date.now(), random?: Uint8Array): string {
+  const chars: string[] = Array.from({ length: 26 }, () => "0");
+  let time = now;
+  for (let i = 9; i >= 0; i -= 1) {
+    chars[i] = ENCODING[time % 32] as string;
+    time = Math.floor(time / 32);
+  }
+  // 256 is an exact multiple of 32, so a modulo here is bias-free.
+  const bytes = random ?? crypto.getRandomValues(new Uint8Array(16));
+  for (let i = 0; i < 16; i += 1) {
+    chars[10 + i] = ENCODING[(bytes[i] as number) % 32] as string;
+  }
+  return chars.join("");
+}

package/src/index.ts

@@ -0,0 +1,58 @@
+/**
+ * The package's public entry point (`exports["."]` in package.json).
+ *
+ * The v3 authoring API: what an EVAL.ts scorer and an experiment module
+ * import from "quickstudy".
+ */
+
+export type {
+  CheckResult,
+  EvalContext,
+  EvalExecResult,
+  EvalMetadata,
+  EvalResult,
+  EvalScorer,
+  EvalSuite,
+  EvidenceReference,
+  LoadedEval,
+} from "./evals/types.ts";
+export { EVAL_SUITES } from "./evals/types.ts";
+export { defineExperiment } from "./experiments/types.ts";
+export type { AgentSpec, Experiment, LoadedExperiment, Runtime } from "./experiments/types.ts";
+export type {
+  ProvisionedEnvironment,
+  RuntimeCapabilities,
+  RuntimeProvisionContext,
+  ScorerCapabilities,
+  ScorerCapabilityContext,
+  WebPolicy,
+} from "./runtime/types.ts";
+export type { McpServerAuthConfig, McpServerConfig } from "./isolation/mcp.ts";
+export { compareGroups } from "./experiments/groups.ts";
+export type { ComparisonGroup } from "./experiments/groups.ts";
+
+// Probe scaffolding: app boot, readiness, Playwright invocation, and probe
+// utilities consumed by scorer libraries. Exported so consumer scorers import
+// the harness by package name, never by relative path into src/.
+export {
+  buildStartScript,
+  deepJsonEqual,
+  deliverSpecSource,
+  evaluateAssertions,
+  fetchViaExec,
+  readAppLogTail,
+  runPlaywrightSpec,
+  startAppInBackground,
+  stopBackgroundApp,
+  truncateOutput,
+  waitForAppReady,
+} from "./probe.ts";
+export type { JsonAssertion, ProbeExecFn } from "./probe.ts";
+
+export type { SurfaceUsageConfig, SurfaceUsage } from "./surface-usage.ts";
+
+export { defineSemanticScorer, SemanticScoringError } from "./semantic.ts";
+export type { SemanticRubric, SemanticJudge, SemanticJudgeRequest, SemanticJudgeResponse } from "./semantic.ts";
+export type { AgentOutput, JudgeRecord } from "./evals/types.ts";
+
+export { extractSurfaceUsage, surfaceUsageConfigFromRuntime, observationConfig } from "./surface-usage.ts";