@prismatic-io/lux 0.0.1

package/src/assertions/core/tool-invocation.ts

@@ -0,0 +1,198 @@
+import { z } from "zod";
+import {
+  type AssertionImpl,
+  malformedToolEventCount,
+  toolInvocations,
+  toolResultValue,
+} from "../../core/index.js";
+import { deepEquals, resolvePointer, weightThresholdFields } from "./helpers.js";
+import { inspectSafeRegexes, type SafeRegexGroup, testSafeRegexGroups } from "./safe-regex.js";
+
+const ValueMatchSchema = z
+  .object({
+    jsonPointer: z.string().default(""),
+    equals: z.unknown().optional(),
+    contains: z.string().min(1).optional(),
+    matches: z.string().max(2_000).optional(),
+    flags: z.string().max(16).optional(),
+  })
+  .strict()
+  .refine(
+    (config) =>
+      [config.equals, config.contains, config.matches].filter((value) => value !== undefined)
+        .length === 1,
+    { message: "set exactly one of equals, contains, or matches" },
+  )
+  .superRefine(async (config, ctx) => {
+    if (config.matches === undefined) {
+      if (config.flags !== undefined) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["flags"],
+          message: "flags requires matches",
+        });
+      }
+      return;
+    }
+    const [inspection] = await inspectSafeRegexes([config.matches], config.flags ?? "");
+    if (inspection && "error" in inspection) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["matches"],
+        message: inspection.error,
+      });
+    }
+  });
+type ValueMatch = z.infer<typeof ValueMatchSchema>;
+const MAX_TOOL_MATCH_CHARS = 1024 * 1024;
+const MAX_TOOL_MATCH_TOTAL_CHARS = 8 * 1024 * 1024;
+
+export const ToolInvocationConfigSchema = z
+  .object({
+    type: z.literal("tool-invocation"),
+    name: z.string(),
+    /** Filter paired results before choosing first, last, or any. */
+    result: z.enum(["success", "error", "any"]).default("success"),
+    occurrence: z.enum(["first", "last", "any"]).default("any"),
+    input: ValueMatchSchema.optional(),
+    output: ValueMatchSchema.optional(),
+    ...weightThresholdFields,
+  })
+  .strict();
+export type ToolInvocationConfig = z.infer<typeof ToolInvocationConfigSchema>;
+
+const valueText = (value: unknown): string | null => {
+  let text: string;
+  if (typeof value === "string") {
+    text = value;
+  } else {
+    try {
+      text = JSON.stringify(value) ?? String(value);
+    } catch {
+      return null;
+    }
+  }
+  return text.length <= MAX_TOOL_MATCH_CHARS ? text : null;
+};
+
+type PreparedValueMatch = {
+  matched: boolean;
+  regexText?: string;
+};
+
+const prepareValueMatch = (root: unknown, match: ValueMatch): PreparedValueMatch => {
+  const value = resolvePointer(root, match.jsonPointer);
+  if (match.equals !== undefined) return { matched: deepEquals(value, match.equals) };
+  if (match.contains !== undefined) {
+    if (value === undefined) return { matched: false };
+    return { matched: valueText(value)?.includes(match.contains) ?? false };
+  }
+  if (match.matches !== undefined) {
+    if (value === undefined) return { matched: false };
+    const text = valueText(value);
+    return text === null ? { matched: false } : { matched: true, regexText: text };
+  }
+  return { matched: false };
+};
+
+const selectOccurrences = <T>(values: T[], occurrence: ToolInvocationConfig["occurrence"]): T[] => {
+  if (occurrence === "first") return values.slice(0, 1);
+  if (occurrence === "last") return values.slice(-1);
+  return values;
+};
+
+export const toolInvocationAssertion: AssertionImpl<ToolInvocationConfig> = {
+  type: "tool-invocation",
+  configSchema: ToolInvocationConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    const malformed = malformedToolEventCount(run.events, ["tool-call", "tool-result"]);
+    if (malformed > 0) {
+      const errorMessage = `${malformed} malformed standardized tool event${malformed === 1 ? "" : "s"} captured`;
+      return {
+        passed: false,
+        score: 0,
+        comment: errorMessage,
+        errorMessage,
+      };
+    }
+    const invocations = toolInvocations(run.events, config.name);
+    const selected = selectOccurrences(invocations, config.occurrence);
+    const paired = selected.filter(
+      (
+        invocation,
+      ): invocation is typeof invocation & {
+        result: NonNullable<typeof invocation.result>;
+      } => invocation.result !== undefined,
+    );
+    const relevant = paired.filter(({ result }) => {
+      if (config.result === "any") return true;
+      return result.isError === (config.result === "error");
+    });
+    const candidates: Array<{
+      inputRegexText?: string;
+      outputRegexText?: string;
+    }> = [];
+    let regexTextChars = 0;
+    for (const invocation of relevant) {
+      const input = config.input
+        ? prepareValueMatch(invocation.call.input, config.input)
+        : { matched: true };
+      if (!input.matched) continue;
+      const output = config.output
+        ? prepareValueMatch(toolResultValue(invocation.result), config.output)
+        : { matched: true };
+      if (!output.matched) continue;
+      regexTextChars += (input.regexText?.length ?? 0) + (output.regexText?.length ?? 0);
+      if (regexTextChars > MAX_TOOL_MATCH_TOTAL_CHARS) {
+        throw new RangeError(
+          `tool invocation regex evidence exceeds the ${MAX_TOOL_MATCH_TOTAL_CHARS}-character assertion limit`,
+        );
+      }
+      candidates.push({
+        ...(input.regexText !== undefined ? { inputRegexText: input.regexText } : {}),
+        ...(output.regexText !== undefined ? { outputRegexText: output.regexText } : {}),
+      });
+    }
+
+    const groups: SafeRegexGroup[] = [];
+    if (config.input?.matches !== undefined) {
+      groups.push({
+        pattern: config.input.matches,
+        flags: config.input.flags ?? "",
+        texts: candidates.map(({ inputRegexText }) => inputRegexText ?? ""),
+      });
+    }
+    if (config.output?.matches !== undefined) {
+      groups.push({
+        pattern: config.output.matches,
+        flags: config.output.flags ?? "",
+        texts: candidates.map(({ outputRegexText }) => outputRegexText ?? ""),
+      });
+    }
+    const regexResults =
+      groups.length > 0 ? await testSafeRegexGroups(groups, { abortSignal: ctx?.abortSignal }) : [];
+    let groupIndex = 0;
+    const inputResults =
+      config.input?.matches !== undefined ? regexResults[groupIndex++] : undefined;
+    const outputResults =
+      config.output?.matches !== undefined ? regexResults[groupIndex] : undefined;
+    const matches = candidates.filter(
+      (_candidate, index) => (inputResults?.[index] ?? true) && (outputResults?.[index] ?? true),
+    );
+    const passed = matches.length > 0;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `'${config.name}' has a matching ${config.result} invocation`
+        : `'${config.name}' has no matching ${config.result} invocation`,
+      detail: {
+        calls: invocations.length,
+        selected: selected.length,
+        paired: paired.length,
+        resultMatches: relevant.length,
+        matches: matches.length,
+      },
+    };
+  },
+};

package/src/assertions/core/tool-result-checks.ts

@@ -0,0 +1,211 @@
+import { z } from "zod";
+import {
+  type AssertionImpl,
+  malformedToolEventCount,
+  type ToolResultPayload,
+  toolResults,
+  toolResultValue,
+} from "../../core/index.js";
+import { deepEquals, resolvePointer, weightThresholdFields } from "./helpers.js";
+
+const OccurrenceSchema = z.enum(["first", "last", "any"]);
+type Occurrence = z.infer<typeof OccurrenceSchema>;
+
+const select = (results: ToolResultPayload[], occurrence: Occurrence): ToolResultPayload[] => {
+  if (occurrence === "first") return results.slice(0, 1);
+  if (occurrence === "last") return results.slice(-1);
+  return results;
+};
+
+const malformedResultEvidence = (run: Parameters<AssertionImpl["evaluate"]>[1]) => {
+  const malformed = malformedToolEventCount(run.events, ["tool-result"]);
+  if (malformed === 0) return null;
+  const errorMessage = `${malformed} malformed tool result event${malformed === 1 ? "" : "s"} captured`;
+  return {
+    passed: false as const,
+    score: 0 as const,
+    comment: errorMessage,
+    errorMessage,
+  };
+};
+
+export const ToolResultIsErrorConfigSchema = z
+  .object({
+    type: z.literal("tool-result-is-error"),
+    /** Tool whose result to inspect; omit to consider every result. */
+    name: z.string().optional(),
+    /** The `isError` value to require. Default true (assert the tool DID error). */
+    expected: z.boolean().default(true),
+    /** Which matching result(s) to require it of. Default "last". */
+    occurrence: OccurrenceSchema.default("last"),
+    ...weightThresholdFields,
+  })
+  .strict();
+export type ToolResultIsErrorConfig = z.infer<typeof ToolResultIsErrorConfigSchema>;
+
+export const toolResultIsErrorAssertion: AssertionImpl<ToolResultIsErrorConfig> = {
+  type: "tool-result-is-error",
+  configSchema: ToolResultIsErrorConfigSchema,
+  evaluate: async (config, run) => {
+    const malformed = malformedResultEvidence(run);
+    if (malformed) return malformed;
+    const results = toolResults(run.events, config.name);
+    const label = config.name ? `'${config.name}'` : "any tool";
+    if (results.length === 0) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `no result captured for ${label}`,
+      };
+    }
+    const relevant = config.occurrence === "any" ? results : select(results, config.occurrence);
+    const passed =
+      config.occurrence === "any"
+        ? relevant.some((result) => result.isError === config.expected)
+        : relevant.every((result) => result.isError === config.expected);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${label} isError expected ${config.expected} (${config.occurrence})`,
+      detail: { isError: relevant.map((result) => result.isError) },
+    };
+  },
+};
+
+export const ToolResultContainsConfigSchema = z
+  .object({
+    type: z.literal("tool-result-contains"),
+    name: z.string().optional(),
+    /** Substring the result's text (or stringified content) must contain. */
+    text: z.string().min(1),
+    occurrence: OccurrenceSchema.default("any"),
+    ...weightThresholdFields,
+  })
+  .strict();
+export type ToolResultContainsConfig = z.infer<typeof ToolResultContainsConfigSchema>;
+
+const resultText = (result: ToolResultPayload): string =>
+  result.text ||
+  (typeof result.content === "string" ? result.content : JSON.stringify(result.content ?? ""));
+
+export const toolResultContainsAssertion: AssertionImpl<ToolResultContainsConfig> = {
+  type: "tool-result-contains",
+  configSchema: ToolResultContainsConfigSchema,
+  evaluate: async (config, run) => {
+    const malformed = malformedResultEvidence(run);
+    if (malformed) return malformed;
+    const results = toolResults(run.events, config.name);
+    const label = config.name ? `'${config.name}'` : "any tool";
+    if (results.length === 0) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `no result captured for ${label}`,
+      };
+    }
+    const relevant = config.occurrence === "any" ? results : select(results, config.occurrence);
+    const hit = relevant.filter((result) => resultText(result).includes(config.text));
+    const passed = hit.length > 0;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${label} result ${passed ? "contains" : "does not contain"} ${JSON.stringify(config.text)}`,
+      detail: { matches: hit.length },
+    };
+  },
+};
+
+export const ToolResultJsonPointerEqualsConfigSchema = z
+  .object({
+    type: z.literal("tool-result-json-pointer-equals"),
+    name: z.string(),
+    /**
+     * RFC 6901 pointer into the result's `structuredContent`, falling back to
+     * the result text parsed as JSON. Defaults to "" (the whole value).
+     */
+    jsonPointer: z.string().default(""),
+    /** Structural (key-order-insensitive) equality against the value. */
+    value: z.unknown().optional(),
+    /** Substring containment against the stringified value. */
+    contains: z.string().min(1).optional(),
+    occurrence: OccurrenceSchema.default("last"),
+    ...weightThresholdFields,
+  })
+  .strict()
+  .refine((config) => (config.value === undefined) !== (config.contains === undefined), {
+    message: "set exactly one of value or contains",
+  });
+export type ToolResultJsonPointerEqualsConfig = z.infer<
+  typeof ToolResultJsonPointerEqualsConfigSchema
+>;
+
+export const toolResultJsonPointerEqualsAssertion: AssertionImpl<ToolResultJsonPointerEqualsConfig> =
+  {
+    type: "tool-result-json-pointer-equals",
+    configSchema: ToolResultJsonPointerEqualsConfigSchema,
+    evaluate: async (config, run) => {
+      const malformed = malformedResultEvidence(run);
+      if (malformed) return malformed;
+      const results = toolResults(run.events, config.name);
+      if (results.length === 0) {
+        return {
+          passed: false,
+          score: 0,
+          comment: `no result captured for '${config.name}'`,
+        };
+      }
+      const relevant = config.occurrence === "any" ? results : select(results, config.occurrence);
+      const matched = relevant.some((result) => {
+        const value = resolvePointer(toolResultValue(result), config.jsonPointer);
+        if (config.value !== undefined) return deepEquals(value, config.value);
+        if (config.contains !== undefined) {
+          if (value === undefined) return false;
+          const stringified = typeof value === "string" ? value : JSON.stringify(value);
+          return stringified.includes(config.contains);
+        }
+        return value !== undefined;
+      });
+      return {
+        passed: matched,
+        score: matched ? 1 : 0,
+        comment: `'${config.name}' result at '${config.jsonPointer || "/"}' ${matched ? "matched" : "did not match"}`,
+      };
+    },
+  };
+
+export const NoErrorResultsConfigSchema = z
+  .object({
+    type: z.literal("no-error-results"),
+    /** Permit a run with no captured calls/results. Defaults false to prevent vacuous passes. */
+    allowEmpty: z.boolean().default(false),
+    ...weightThresholdFields,
+  })
+  .strict();
+export type NoErrorResultsConfig = z.infer<typeof NoErrorResultsConfigSchema>;
+
+export const noErrorResultsAssertion: AssertionImpl<NoErrorResultsConfig> = {
+  type: "no-error-results",
+  configSchema: NoErrorResultsConfigSchema,
+  evaluate: async (config, run) => {
+    const malformed = malformedResultEvidence(run);
+    if (malformed) return malformed;
+    const results = toolResults(run.events);
+    if (results.length === 0 && !config.allowEmpty) {
+      return {
+        passed: false,
+        score: 0,
+        comment: "no tool results captured",
+      };
+    }
+    const offenders = results.filter((result) => result.isError);
+    const passed = offenders.length === 0;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? "no tool returned an error result"
+        : `error results from: ${offenders.map((result) => result.name).join(", ")}`,
+      detail: { offenders: offenders.map((result) => result.name) },
+    };
+  },
+};

package/src/assertions/mcp/discovery-checks.ts

@@ -0,0 +1,139 @@
+import { z } from "zod";
+import {
+  type AssertionImpl,
+  hasMcpInit,
+  hasMcpToolsList,
+  listedTools,
+  mcpInit,
+} from "../../core/index.js";
+import { inRange } from "../core/helpers.js";
+
+export const ServerInfoEqualsConfigSchema = z
+  .object({
+    type: z.literal("server-info-equals"),
+    /** Expected `serverInfo.name` from `initialize`. */
+    name: z.string().optional(),
+    /** Expected `serverInfo.version` from `initialize`. */
+    version: z.string().optional(),
+  })
+  .strict()
+  .refine((c) => c.name !== undefined || c.version !== undefined, {
+    message: "set at least one of name or version",
+  });
+export type ServerInfoEqualsConfig = z.infer<typeof ServerInfoEqualsConfigSchema>;
+
+export const serverInfoEqualsAssertion: AssertionImpl<ServerInfoEqualsConfig> = {
+  type: "server-info-equals",
+  configSchema: ServerInfoEqualsConfigSchema,
+  evaluate: async (config, run) => {
+    const info = mcpInit(run.events)?.serverInfo;
+    if (!info) {
+      return { passed: false, score: 0, comment: "no initialize metadata captured" };
+    }
+    const mismatches: string[] = [];
+    if (config.name !== undefined && info.name !== config.name) {
+      mismatches.push(`name '${info.name}' != '${config.name}'`);
+    }
+    if (config.version !== undefined && info.version !== config.version) {
+      mismatches.push(`version '${info.version}' != '${config.version}'`);
+    }
+    const passed = mismatches.length === 0;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed ? `serverInfo ${info.name}@${info.version} matched` : mismatches.join("; "),
+      detail: { serverInfo: info },
+    };
+  },
+};
+
+export const ToolListedConfigSchema = z
+  .object({
+    type: z.literal("tool-listed"),
+    /** Tool name whose presence is checked. */
+    name: z.string(),
+    /** Whether `tools/list` should include the tool. Default true. */
+    expected: z.boolean().default(true),
+  })
+  .strict();
+export type ToolListedConfig = z.infer<typeof ToolListedConfigSchema>;
+
+export const toolListedAssertion: AssertionImpl<ToolListedConfig> = {
+  type: "tool-listed",
+  configSchema: ToolListedConfigSchema,
+  evaluate: async (config, run) => {
+    if (!hasMcpToolsList(run.events)) {
+      return { passed: false, score: 0, comment: "no tools/list result captured" };
+    }
+    const tools = listedTools(run.events);
+    const listed = tools.some((t) => t.name === config.name);
+    const passed = listed === config.expected;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `tool '${config.name}' is ${listed ? "listed" : `not among ${tools.length} listed tools`}`,
+    };
+  },
+};
+
+export const ToolsListedCountConfigSchema = z
+  .object({
+    type: z.literal("tools-listed-count"),
+    min: z.number().int().nonnegative().optional(),
+    max: z.number().int().nonnegative().optional(),
+  })
+  .strict()
+  .refine((c) => c.min !== undefined || c.max !== undefined, {
+    message: "set at least one of min or max",
+  })
+  .refine((c) => c.min === undefined || c.max === undefined || c.min <= c.max, {
+    message: "min must be <= max",
+  });
+export type ToolsListedCountConfig = z.infer<typeof ToolsListedCountConfigSchema>;
+
+export const toolsListedCountAssertion: AssertionImpl<ToolsListedCountConfig> = {
+  type: "tools-listed-count",
+  configSchema: ToolsListedCountConfigSchema,
+  evaluate: async (config, run) => {
+    if (!hasMcpToolsList(run.events)) {
+      return { passed: false, score: 0, comment: "no tools/list result captured" };
+    }
+    const count = listedTools(run.events).length;
+    const { passed, comment } = inRange(count, config.min, config.max);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${count} tool${count === 1 ? "" : "s"} listed (${comment})`,
+      detail: { count },
+    };
+  },
+};
+
+export const CapabilityAbsentConfigSchema = z
+  .object({
+    type: z.literal("capability-absent"),
+    /** Server capability that must NOT be advertised. */
+    capability: z.enum(["resources", "prompts", "logging", "completions", "tools"]),
+  })
+  .strict();
+export type CapabilityAbsentConfig = z.infer<typeof CapabilityAbsentConfigSchema>;
+
+export const capabilityAbsentAssertion: AssertionImpl<CapabilityAbsentConfig> = {
+  type: "capability-absent",
+  configSchema: CapabilityAbsentConfigSchema,
+  evaluate: async (config, run) => {
+    if (!hasMcpInit(run.events)) {
+      return { passed: false, score: 0, comment: "no initialize metadata captured" };
+    }
+    const caps = mcpInit(run.events)?.capabilities ?? {};
+    const value = (caps as Record<string, unknown>)[config.capability];
+    const present = value !== undefined && value !== false && value !== null;
+    return {
+      passed: !present,
+      score: present ? 0 : 1,
+      comment: present
+        ? `server advertises '${config.capability}'`
+        : `server does not advertise '${config.capability}'`,
+    };
+  },
+};

package/src/assertions/mcp/index.ts

@@ -0,0 +1,37 @@
+import type { AssertionImpl, Registry } from "../../core/index.js";
+import {
+  noErrorResultsAssertion,
+  toolResultContainsAssertion,
+  toolResultIsErrorAssertion,
+  toolResultJsonPointerEqualsAssertion,
+} from "../core/tool-result-checks.js";
+import {
+  capabilityAbsentAssertion,
+  serverInfoEqualsAssertion,
+  toolListedAssertion,
+  toolsListedCountAssertion,
+} from "./discovery-checks.js";
+
+export * from "./discovery-checks.js";
+
+/**
+ * The result assertions remain in this compatibility collection for
+ * embedders that historically registered only the MCP assertion bundle.
+ * Their implementations are generic and also ship in `allCoreAssertions`.
+ */
+export const allMcpAssertions: AssertionImpl[] = [
+  toolResultIsErrorAssertion,
+  toolResultContainsAssertion,
+  toolResultJsonPointerEqualsAssertion,
+  noErrorResultsAssertion,
+  serverInfoEqualsAssertion,
+  toolListedAssertion,
+  toolsListedCountAssertion,
+  capabilityAbsentAssertion,
+];
+
+export const registerMcpAssertions = (registry: Registry): void => {
+  for (const a of allMcpAssertions) {
+    if (!registry.getAssertion(a.type)) registry.registerAssertion(a);
+  }
+};