@prismatic-io/lux 0.0.1

package/src/core/hash.ts

@@ -0,0 +1,237 @@
+import { createHash } from "node:crypto";
+
+type Traversal = {
+  active: Map<object, number>;
+  nextId: number;
+};
+
+const typedArrayConstructors = new Set<unknown>([
+  Int8Array,
+  Uint8Array,
+  Uint8ClampedArray,
+  Int16Array,
+  Uint16Array,
+  Int32Array,
+  Uint32Array,
+  Float32Array,
+  Float64Array,
+  BigInt64Array,
+  BigUint64Array,
+  Buffer,
+]);
+
+const ownStateKeys = (value: object, allowed: ReadonlySet<string>): string[] =>
+  Reflect.ownKeys(value).flatMap((key) => {
+    if (typeof key === "symbol") {
+      throw new TypeError("valueHash does not support symbol-keyed state");
+    }
+    return allowed.has(key) ? [] : [key];
+  });
+
+const assertNoAttachedState = (value: object, allowed: ReadonlySet<string>): void => {
+  const keys = ownStateKeys(value, allowed);
+  if (keys.length > 0) {
+    throw new TypeError(
+      `valueHash does not support attached state on ${value.constructor?.name ?? "value"}: ${keys.join(", ")}`,
+    );
+  }
+};
+
+const numberValue = (value: number): string | number => {
+  if (Number.isNaN(value)) return "NaN";
+  if (value === Number.POSITIVE_INFINITY) return "Infinity";
+  if (value === Number.NEGATIVE_INFINITY) return "-Infinity";
+  if (Object.is(value, -0)) return "-0";
+  return value;
+};
+
+const arrayIndex = (key: string, length: number): number | null => {
+  if (!/^(?:0|[1-9]\d*)$/.test(key)) return null;
+  const index = Number(key);
+  return Number.isSafeInteger(index) && index >= 0 && index < length ? index : null;
+};
+
+const canonicalizeObject = (value: object, traversal: Traversal): unknown => {
+  const reference = traversal.active.get(value);
+  if (reference !== undefined) return ["reference", reference];
+
+  const id = traversal.nextId++;
+  traversal.active.set(value, id);
+  try {
+    if (Array.isArray(value)) {
+      if (Object.getPrototypeOf(value) !== Array.prototype) {
+        throw new TypeError("valueHash does not support Array subclasses");
+      }
+      const ownKeys = ownStateKeys(value, new Set(["length"]));
+      for (const key of ownKeys) {
+        if (arrayIndex(key, value.length) === null) {
+          throw new TypeError(`valueHash does not support array property '${key}'`);
+        }
+        const descriptor = Object.getOwnPropertyDescriptor(value, key);
+        if (descriptor && !("value" in descriptor)) {
+          throw new TypeError(`valueHash does not support accessor property '${key}'`);
+        }
+      }
+      return [
+        "array",
+        Array.from({ length: value.length }, (_, index) =>
+          Object.hasOwn(value, index)
+            ? canonicalize(value[index], traversal)
+            : (["array-hole"] as const),
+        ),
+      ];
+    }
+
+    if (value instanceof Date) {
+      if (Object.getPrototypeOf(value) !== Date.prototype) {
+        throw new TypeError("valueHash does not support Date subclasses");
+      }
+      assertNoAttachedState(value, new Set());
+      return ["date", numberValue(value.getTime())];
+    }
+    if (value instanceof RegExp) {
+      if (Object.getPrototypeOf(value) !== RegExp.prototype) {
+        throw new TypeError("valueHash does not support RegExp subclasses");
+      }
+      assertNoAttachedState(value, new Set(["lastIndex"]));
+      return ["regexp", value.source, value.flags, value.lastIndex];
+    }
+    if (value instanceof Map) {
+      if (Object.getPrototypeOf(value) !== Map.prototype) {
+        throw new TypeError("valueHash does not support Map subclasses");
+      }
+      assertNoAttachedState(value, new Set());
+      return [
+        "map",
+        Array.from(value.entries(), ([key, item]) => [
+          canonicalize(key, traversal),
+          canonicalize(item, traversal),
+        ]),
+      ];
+    }
+    if (value instanceof Set) {
+      if (Object.getPrototypeOf(value) !== Set.prototype) {
+        throw new TypeError("valueHash does not support Set subclasses");
+      }
+      assertNoAttachedState(value, new Set());
+      return ["set", Array.from(value, (item) => canonicalize(item, traversal))];
+    }
+    if (value instanceof ArrayBuffer) {
+      if (Object.getPrototypeOf(value) !== ArrayBuffer.prototype) {
+        throw new TypeError("valueHash does not support ArrayBuffer subclasses");
+      }
+      assertNoAttachedState(value, new Set());
+      return ["array-buffer", Buffer.from(value).toString("base64")];
+    }
+    if (ArrayBuffer.isView(value)) {
+      if (typeof SharedArrayBuffer !== "undefined" && value.buffer instanceof SharedArrayBuffer) {
+        throw new TypeError("valueHash does not support shared mutable buffers");
+      }
+      if (value instanceof DataView) {
+        if (Object.getPrototypeOf(value) !== DataView.prototype) {
+          throw new TypeError("valueHash does not support DataView subclasses");
+        }
+        assertNoAttachedState(value, new Set());
+        return [
+          "data-view",
+          Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString("base64"),
+        ];
+      }
+      const viewConstructor = Object.getPrototypeOf(value)?.constructor as unknown;
+      if (typeof viewConstructor !== "function" || !typedArrayConstructors.has(viewConstructor)) {
+        throw new TypeError("valueHash does not support custom typed-array classes");
+      }
+      const indexedKeys = new Set(
+        Array.from({ length: (value as unknown as ArrayLike<unknown>).length }, (_, index) =>
+          String(index),
+        ),
+      );
+      assertNoAttachedState(value, indexedKeys);
+      return [
+        "typed-array",
+        viewConstructor.name,
+        Array.from(value as unknown as ArrayLike<unknown>, (item) => canonicalize(item, traversal)),
+      ];
+    }
+
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null) {
+      throw new TypeError(
+        `valueHash does not support class instance '${value.constructor?.name ?? "unknown"}'`,
+      );
+    }
+    const entries = ownStateKeys(value, new Set())
+      .sort((a, b) => a.localeCompare(b))
+      .map((key) => {
+        const descriptor = Object.getOwnPropertyDescriptor(value, key);
+        if (!descriptor?.enumerable) {
+          throw new TypeError(`valueHash does not support non-enumerable property '${key}'`);
+        }
+        if (!("value" in descriptor)) {
+          throw new TypeError(`valueHash does not support accessor property '${key}'`);
+        }
+        return [key, canonicalize(descriptor.value, traversal)];
+      });
+    return [prototype === null ? "null-object" : "object", entries];
+  } finally {
+    traversal.active.delete(value);
+  }
+};
+
+const canonicalize = (value: unknown, traversal: Traversal): unknown => {
+  if (value === null) return ["null"];
+  if (value === undefined) return ["undefined"];
+  if (typeof value === "string") return ["string", value];
+  if (typeof value === "boolean") return ["boolean", value];
+  if (typeof value === "number") return ["number", numberValue(value)];
+  if (typeof value === "bigint") return ["bigint", value.toString()];
+  if (typeof value === "symbol") {
+    throw new TypeError("valueHash does not support symbols");
+  }
+  if (typeof value === "function") {
+    const ownKeys = Reflect.ownKeys(value);
+    if (ownKeys.some((key) => typeof key === "symbol")) {
+      throw new TypeError("valueHash does not support symbol-keyed function state");
+    }
+    const standardKeys = new Set(["length", "name", "arguments", "caller", "prototype"]);
+    const attached = ownKeys.filter(
+      (key): key is string => typeof key === "string" && !standardKeys.has(key),
+    );
+    if (attached.length > 0) {
+      throw new TypeError(
+        `valueHash does not support attached function state: ${attached.join(", ")}`,
+      );
+    }
+    if (Object.hasOwn(value, "prototype")) {
+      const prototype = (value as { prototype?: unknown }).prototype;
+      if (
+        !prototype ||
+        typeof prototype !== "object" ||
+        Reflect.ownKeys(prototype).some((key) => key !== "constructor") ||
+        Object.getOwnPropertyDescriptor(prototype, "constructor")?.value !== value
+      ) {
+        throw new TypeError("valueHash does not support modified function prototypes");
+      }
+    }
+    const source = Function.prototype.toString.call(value);
+    if (source.includes("[native code]")) {
+      throw new TypeError("valueHash does not support native or bound functions");
+    }
+    // JavaScript exposes source text but not captured closure values. Evaluator
+    // state that affects behavior must therefore live in source or explicit config.
+    return ["function-source", source];
+  }
+  return canonicalizeObject(value, traversal);
+};
+
+/**
+ * Tagged structural encoding for ledger identity. Functions encode observable
+ * source text only; JavaScript provides no access to captured closure values.
+ */
+export const stableJson = (value: unknown): string =>
+  JSON.stringify(["lux-value", 2, canonicalize(value, { active: new Map(), nextId: 0 })]);
+
+export const contentHash = (value: string | Uint8Array): string =>
+  createHash("sha256").update(value).digest("hex");
+
+export const valueHash = (value: unknown): string => contentHash(stableJson(value));

package/src/core/index.ts

@@ -0,0 +1,28 @@
+export * from "./annotation.js";
+export * from "./answerer.js";
+export * from "./artifact-evidence.js";
+export * from "./artifact-snapshot.js";
+export * from "./assertion.js";
+export * from "./case.js";
+export * from "./conversation-turn.js";
+export * from "./driver.js";
+export * from "./emitter.js";
+export * from "./events.js";
+export * from "./exec.js";
+export * from "./experiment.js";
+export * from "./glob.js";
+export * from "./hash.js";
+export * from "./lifecycle-fixtures.js";
+export * from "./lifecycle-hooks.js";
+export * from "./mcp-events.js";
+export * from "./model-cli.js";
+export * from "./observer.js";
+export * from "./registry.js";
+export * from "./rubric.js";
+export * from "./run.js";
+export * from "./statistics.js";
+export * from "./subject.js";
+export * from "./tool-events.js";
+export * from "./usage.js";
+export * from "./wire.js";
+export * from "./zod.js";

package/src/core/lifecycle-fixtures.ts

@@ -0,0 +1,145 @@
+import { valueHash } from "./hash.js";
+import {
+  type CaseLifecycleHookContext,
+  defineLifecycleHooks,
+  type LifecycleHookCleanup,
+  type LifecycleHooksPlugin,
+  type SuiteLifecycleHookContext,
+} from "./lifecycle-hooks.js";
+
+export type LifecycleFixtureUse = () => PromiseLike<void>;
+
+export type LifecycleFixture<Context extends SuiteLifecycleHookContext> = (
+  context: Context,
+  use: LifecycleFixtureUse,
+) => void | Promise<void>;
+
+export type LifecycleFixturesDefinition = {
+  name: string;
+  pluginVersion: string;
+  /**
+   * Public, non-secret values captured by the fixture callbacks and relevant
+   * to execution identity.
+   */
+  identity?: unknown;
+  suite?: LifecycleFixture<SuiteLifecycleHookContext>;
+  case?: LifecycleFixture<CaseLifecycleHookContext>;
+};
+
+const errorMessage = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const activateFixture = async <Context extends SuiteLifecycleHookContext>(
+  fixture: LifecycleFixture<Context>,
+  context: Context,
+  label: string,
+): Promise<LifecycleHookCleanup> => {
+  let releaseUse: (() => void) | undefined;
+  const useLifetime = new Promise<void>((resolve) => {
+    releaseUse = resolve;
+  });
+  let acceptSetup: (() => void) | undefined;
+  let rejectSetup: ((error: unknown) => void) | undefined;
+  const setup = new Promise<void>((resolve, reject) => {
+    acceptSetup = resolve;
+    rejectSetup = reject;
+  });
+  let useCalls = 0;
+  let useObservations = 0;
+  let settled = false;
+  let completionError: unknown;
+  const completion = (async () => {
+    try {
+      await fixture(context, () => {
+        useCalls += 1;
+        if (useCalls > 1) {
+          throw new TypeError(`${label} called use() more than once`);
+        }
+        return {
+          // biome-ignore lint/suspicious/noThenProperty: a custom thenable lets us detect a discarded use() lifetime.
+          then: (onfulfilled, onrejected) => {
+            useObservations += 1;
+            if (useObservations > 1) {
+              return Promise.reject(
+                new TypeError(`${label} awaited one use() more than once`),
+              ).then(onfulfilled, onrejected);
+            }
+            acceptSetup?.();
+            return useLifetime.then(onfulfilled, onrejected);
+          },
+        };
+      });
+      if (useCalls === 0) {
+        throw new TypeError(`${label} must call and await use() exactly once`);
+      }
+      if (useObservations === 0) {
+        throw new TypeError(`${label} must await use()`);
+      }
+    } catch (error) {
+      completionError = error;
+      if (useObservations === 0) rejectSetup?.(error);
+    } finally {
+      settled = true;
+    }
+  })();
+
+  await setup;
+  await Promise.resolve();
+  if (settled) {
+    throw new TypeError(
+      completionError
+        ? `${label} failed before its lifetime began: ${errorMessage(completionError)}`
+        : `${label} must await use()`,
+      { cause: completionError },
+    );
+  }
+
+  let cleaned = false;
+  return async () => {
+    if (cleaned) return;
+    cleaned = true;
+    releaseUse?.();
+    await completion;
+    if (completionError) throw completionError;
+  };
+};
+
+/**
+ * Define suite and case lifetimes with Vitest-style `await use()` fixtures.
+ * Setup belongs before `use`; teardown belongs after it.
+ */
+export const defineFixtures = (definition: LifecycleFixturesDefinition): LifecycleHooksPlugin => {
+  if (!definition.suite && !definition.case) {
+    throw new TypeError("fixtures plugin must define suite or case");
+  }
+  if (definition.suite !== undefined && typeof definition.suite !== "function") {
+    throw new TypeError("fixtures plugin suite must be a function");
+  }
+  if (definition.case !== undefined && typeof definition.case !== "function") {
+    throw new TypeError("fixtures plugin case must be a function");
+  }
+  const suiteFixture = definition.suite;
+  const caseFixture = definition.case;
+  return defineLifecycleHooks({
+    name: definition.name,
+    pluginVersion: definition.pluginVersion,
+    ...(definition.identity === undefined ? {} : { identity: definition.identity }),
+    implementationHash: valueHash({
+      identity: definition.identity,
+      suite: definition.suite,
+      case: definition.case,
+    }),
+    ...(suiteFixture
+      ? {
+          setupSuite: (context) =>
+            activateFixture(suiteFixture, context, `suite fixture '${definition.name}'`),
+        }
+      : {}),
+    ...(caseFixture
+      ? {
+          setupCase: (context) =>
+            activateFixture(caseFixture, context, `case fixture '${definition.name}'`),
+        }
+      : {}),
+  });
+};

package/src/core/lifecycle-hooks.ts

@@ -0,0 +1,101 @@
+import { valueHash } from "./hash.js";
+
+export type LifecycleHookCleanup = () => void | Promise<void>;
+export type LifecycleHookSetupResult = LifecycleHookCleanup | undefined;
+
+export type SuiteLifecycleHookItem = {
+  id: string;
+  caseId: string;
+  path: string;
+};
+
+export type SuiteLifecycleHookContext = {
+  suiteId: string;
+  suiteDir: string;
+  runsRoot: string;
+  concurrency: number;
+  items: readonly SuiteLifecycleHookItem[];
+  abortSignal?: AbortSignal;
+};
+
+export type CaseLifecycleHookContext = SuiteLifecycleHookContext & {
+  index: number;
+  item: SuiteLifecycleHookItem;
+};
+
+export type LifecycleHooksPlugin = {
+  name: string;
+  /** Author-maintained identity for setup and teardown behavior. */
+  pluginVersion: string;
+  /**
+   * Public, non-secret values captured by hook closures and relevant to
+   * execution identity. Included in the implementation hash.
+   */
+  identity?: unknown;
+  /**
+   * Hash of the authored callbacks when an adapter wraps them in closures.
+   * `defineLifecycleHooks` and `defineFixtures` populate this automatically.
+   */
+  implementationHash?: string;
+  /**
+   * Prepare resources shared by the selected suite. A returned cleanup runs
+   * exactly once after all cases settle, including abort and failure paths.
+   */
+  setupSuite?(
+    context: SuiteLifecycleHookContext,
+  ): LifecycleHookSetupResult | Promise<LifecycleHookSetupResult>;
+  /**
+   * Prepare resources private to one case attempt. A returned cleanup runs
+   * exactly once after that attempt settles. Concurrent cases may invoke this
+   * hook concurrently.
+   */
+  setupCase?(
+    context: CaseLifecycleHookContext,
+  ): LifecycleHookSetupResult | Promise<LifecycleHookSetupResult>;
+};
+
+export const lifecycleHooksPluginIssue = (
+  definition: Partial<LifecycleHooksPlugin>,
+): string | undefined => {
+  if (typeof definition.name !== "string" || !definition.name.trim()) {
+    return "lifecycle hooks plugin name must not be empty";
+  }
+  if (typeof definition.pluginVersion !== "string" || !definition.pluginVersion.trim()) {
+    return "lifecycle hooks plugin pluginVersion must not be empty";
+  }
+  if (
+    definition.implementationHash !== undefined &&
+    !/^[0-9a-f]{64}$/.test(definition.implementationHash)
+  ) {
+    return "lifecycle hooks plugin implementationHash must be a SHA-256 hex digest";
+  }
+  if (definition.setupSuite !== undefined && typeof definition.setupSuite !== "function") {
+    return "lifecycle hooks plugin setupSuite must be a function";
+  }
+  if (definition.setupCase !== undefined && typeof definition.setupCase !== "function") {
+    return "lifecycle hooks plugin setupCase must be a function";
+  }
+  if (!definition.setupSuite && !definition.setupCase) {
+    return "lifecycle hooks plugin must define setupSuite or setupCase";
+  }
+  return undefined;
+};
+
+/**
+ * Define versioned suite/case resource hooks for registration in
+ * the `lifecycleFixtures` list in `lux.config.ts`.
+ */
+export const defineLifecycleHooks = (definition: LifecycleHooksPlugin): LifecycleHooksPlugin => {
+  const issue = lifecycleHooksPluginIssue(definition);
+  if (issue) throw new TypeError(issue);
+  return {
+    ...definition,
+    implementationHash:
+      definition.implementationHash ??
+      valueHash({
+        identity: definition.identity,
+        setupSuite: definition.setupSuite,
+        setupCase: definition.setupCase,
+      }),
+  };
+};

package/src/core/mcp-events.ts

@@ -0,0 +1,57 @@
+import type { RunEvent } from "./events.js";
+
+/**
+ * MCP-specific `progress` event kinds. Generic tool calls and results live in
+ * `tool-events`; MCP discovery metadata rides the same open progress stream.
+ */
+export const MCP_INIT_KIND = "mcp-init" as const;
+export const MCP_TOOLS_LIST_KIND = "mcp-tools-list" as const;
+
+/** A tool as reported by `tools/list`. */
+export type ListedTool = {
+  name: string;
+  title?: string;
+  description?: string;
+  inputSchema?: unknown;
+  outputSchema?: unknown;
+  annotations?: Record<string, unknown>;
+  execution?: Record<string, unknown>;
+  icons?: unknown[];
+  _meta?: Record<string, unknown>;
+};
+
+/** `initialize` result metadata, captured once when the server connects. */
+export type McpInitPayload = {
+  serverInfo?: { name?: string; version?: string };
+  capabilities?: Record<string, unknown>;
+  instructions?: string;
+};
+
+export type McpToolsListPayload = { tools: ListedTool[] };
+
+type ProgressEvent = Extract<RunEvent, { type: "progress" }>;
+
+const isProgressKind = (e: RunEvent, kind: string): e is ProgressEvent =>
+  e.type === "progress" && e.kind === kind;
+
+export const isMcpInitEvent = (e: RunEvent): e is ProgressEvent => isProgressKind(e, MCP_INIT_KIND);
+
+export const isMcpToolsListEvent = (e: RunEvent): e is ProgressEvent =>
+  isProgressKind(e, MCP_TOOLS_LIST_KIND);
+
+/** The `initialize` metadata (the last `mcp-init` event wins), or undefined. */
+export const mcpInit = (events: RunEvent[]): McpInitPayload | undefined => {
+  const hit = events.filter(isMcpInitEvent).at(-1);
+  return hit ? (hit.payload as McpInitPayload) : undefined;
+};
+
+/** Tools from the last `tools/list` capture in the run. */
+export const listedTools = (events: RunEvent[]): ListedTool[] => {
+  const hit = events.filter(isMcpToolsListEvent).at(-1);
+  const payload = hit?.payload as McpToolsListPayload | undefined;
+  return Array.isArray(payload?.tools) ? payload.tools : [];
+};
+
+export const hasMcpInit = (events: RunEvent[]): boolean => events.some(isMcpInitEvent);
+
+export const hasMcpToolsList = (events: RunEvent[]): boolean => events.some(isMcpToolsListEvent);