@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/assertions/rubric/internal.ts

@@ -3,7 +3,7 @@ import {
   matchesAnyGlob,
   type Run,
   type RunEvent,
-  readRunArtifactText,
+  readRunArtifactExcerpt,
   tryParseJson,
 } from "../../core/index.js";
 import type { RubricConfig, RubricEvidence } from "./index.js";
@@ -44,7 +44,11 @@ const eventSelected = (event: RunEvent, evidence?: RubricEvidence): boolean => {
 };
 
 const interruptDetail = (event: Extract<RunEvent, { type: "interrupt" }>): string => {
-  if (event.kind === "ask") return event.question;
+  if (event.kind === "ask") {
+    return event.context === undefined
+      ? event.question
+      : `${event.question} context=${safeJson(event.context)}`;
+  }
   if (event.kind === "approve") return event.request;
   return "<custom>";
 };
@@ -213,6 +217,11 @@ const eventLine = (
   renderedToolText: ReadonlyMap<RunEvent, string>,
 ): string | null => {
   if (event.type === "interrupt") {
+    // Native question tools may put their actual choices and headers in context,
+    // rather than duplicating them in the human-readable question string.
+    if (event.kind === "ask") {
+      return `interrupt(ask): ${boundedText(interruptDetail(event), 4_000, "ask interrupt")}`;
+    }
     return `interrupt(${event.kind}): ${interruptDetail(event).slice(0, 160)}`;
   }
   if (event.type === "responded") {
@@ -232,19 +241,47 @@ export const summarizeEvents = (
   evidence?: RubricEvidence,
 ): string => {
   const lines: string[] = [];
-  const selected = events.filter((event) => eventSelected(event, evidence));
+  const groups: RunEvent[][] = [];
+  let previous: RunEvent | undefined;
+  for (const event of events) {
+    if (eventSelected(event, evidence)) {
+      const sameSpeaker =
+        event.type === "progress" &&
+        previous?.type === "progress" &&
+        event.kind === "agent-message" &&
+        previous.kind === "agent-message" &&
+        event.agent?.id === previous.agent?.id;
+      if (sameSpeaker && groups.length > 0) groups.at(-1)?.push(event);
+      else groups.push([event]);
+    }
+    previous = event;
+  }
   // Grading only the tail would silently hide the start of a long run from
   // the judge; make the omission explicit rather than invisible.
-  if (selected.length > cap) {
-    lines.push(`[... ${selected.length - cap} earlier events omitted ...]`);
+  if (groups.length > cap) {
+    lines.push(`[... ${groups.length - cap} earlier events omitted ...]`);
   }
-  const tail = cap === 0 ? [] : selected.slice(-cap);
+  const tailGroups = cap === 0 ? [] : groups.slice(-cap);
+  const tail = tailGroups.flat();
   const agentTextAllotments = allotAgentText(tail, evidence?.maxAgentChars ?? 30_000);
   const toolDetail = evidence?.toolDetail ?? "names";
   const toolText = allotToolText(tail, evidence?.maxToolChars ?? 20_000, toolDetail);
-  for (const event of tail) {
-    const line = eventLine(event, agentTextAllotments, toolText.allotments, toolText.rendered);
-    if (line !== null) lines.push(line);
+  for (const group of tailGroups) {
+    if (group.length > 1) {
+      const parts: string[] = [];
+      for (const event of group) {
+        if (event.type !== "progress") continue;
+        const text = (event.payload as { text?: unknown })?.text;
+        if (typeof text !== "string") continue;
+        const take = agentTextAllotments.get(event) ?? 0;
+        if (take > 0) parts.push(text.slice(0, take));
+        if (take < text.length && parts.at(-1) !== "… [truncated]") parts.push("… [truncated]");
+      }
+      lines.push(`agent: ${parts.join("")}`);
+    } else if (group[0]) {
+      const line = eventLine(group[0], agentTextAllotments, toolText.allotments, toolText.rendered);
+      if (line !== null) lines.push(line);
+    }
   }
   return lines.join("\n");
 };
@@ -267,12 +304,41 @@ const MAX_ARTIFACT_EXCERPT = 20_000;
 
 const boundedText = (value: string, limit: number, label: string): string =>
   value.length <= limit ? value : `${value.slice(0, limit)}\n[... ${label} truncated ...]`;
+const OPTIONAL_TRACE_TRUNCATION = "[... optional recent trace truncated from start ...]\n";
+const OPTIONAL_ARTIFACT_TRUNCATION = "\n[... optional artifact evidence truncated ...]";
+const OPTIONAL_EVIDENCE_OMISSION = "[... optional evidence omitted: prompt budget ...]";
 
-const artifactListing = (paths: readonly string[]): string[] => {
+// Complete invocation evidence is protected. Share the remaining prompt capacity
+// between contextual trace and artifacts, including the omission markers themselves.
+const fitOptionalEvidence = (
+  trace: string,
+  artifacts: string,
+  available: number,
+): [string, string] => {
+  if (trace.length + artifacts.length <= available) return [trace, artifacts];
+  const traceMarker = OPTIONAL_TRACE_TRUNCATION;
+  const artifactMarker = OPTIONAL_ARTIFACT_TRUNCATION;
+  const discretionary = Math.max(0, available - traceMarker.length - artifactMarker.length);
+  const traceBudget =
+    traceMarker.length +
+    Math.floor((discretionary * trace.length) / (trace.length + artifacts.length));
+  const artifactBudget = available - traceBudget;
+  const traceTake = Math.max(0, traceBudget - traceMarker.length);
+  // Recent evidence is chronological: preserve the final answer and newest context.
+  const fittedTrace =
+    trace.length <= traceBudget ? trace : `${traceMarker}${trace.slice(trace.length - traceTake)}`;
+  const fittedArtifacts =
+    artifacts.length <= artifactBudget
+      ? artifacts
+      : `${artifacts.slice(0, Math.max(0, artifactBudget - artifactMarker.length))}${artifactMarker}`;
+  return [fittedTrace, fittedArtifacts];
+};
+
+const artifactListing = (artifacts: Run["artifacts"]): string[] => {
   const listed: string[] = [];
   let listingChars = 0;
-  for (const artifactPath of paths.slice(0, ARTIFACT_LISTING_CAP)) {
-    const path = boundedText(artifactPath, 500, "artifact path");
+  for (const artifact of artifacts.slice(0, ARTIFACT_LISTING_CAP)) {
+    const path = `[origin: ${artifact.origin ?? "unknown"}] ${boundedText(artifact.path, 500, "artifact path")}`;
     if (listingChars + path.length > ARTIFACT_LISTING_CHARS) {
       listed.push("[... artifact listing character budget exhausted ...]");
       break;
@@ -280,8 +346,8 @@ const artifactListing = (paths: readonly string[]): string[] => {
     listed.push(path);
     listingChars += path.length;
   }
-  if (paths.length > listed.length) {
-    listed.push(`[... ${paths.length - listed.length} more selected files omitted ...]`);
+  if (artifacts.length > listed.length) {
+    listed.push(`[... ${artifacts.length - listed.length} more selected files omitted ...]`);
   }
   return listed;
 };
@@ -293,34 +359,142 @@ export const sampleArtifacts = async (
 ): Promise<string> => {
   if (run.artifacts.length === 0) return "(no artifacts captured)";
   const selected = evidence
-    ? run.artifacts.filter((artifact) => matchesAnyGlob(artifact.path, evidence.artifactPatterns))
-    : run.artifacts;
+    ? run.artifacts.filter(
+        (artifact) =>
+          matchesAnyGlob(artifact.path, evidence.artifactPatterns) &&
+          !matchesAnyGlob(artifact.path, evidence.artifactExcludes ?? []),
+      )
+    : [...run.artifacts];
   if (selected.length === 0) return "(no artifacts matched the rubric evidence selector)";
-  const listed = artifactListing(selected.map((artifact) => artifact.path));
+  // Keep the complete index visible, but spend excerpt budget on delivered changes first.
+  const priority = (origin: (typeof selected)[number]["origin"]) => {
+    if (origin === "fixture") return 2;
+    return origin === undefined ? 1 : 0;
+  };
+  selected.sort((left, right) => priority(left.origin) - priority(right.origin));
+  const listed = artifactListing(selected);
   const out: string[] = [
     `Files selected (${selected.length} of ${run.artifacts.length}):\n${listed.join("\n")}`,
+    "Selection counts refer to captured paths matching this criterion, not capture failures. Listed files are not necessarily shown in full below.",
+    "Origins compare captured paths and bytes with the supplied fixture snapshot: fixture = unchanged supplied file; modified = changed supplied file; generated = captured path absent from that snapshot; unknown = not established. Supplied content is baseline context, not proof of newly implemented work. Retained behavior can satisfy requirements; modified or generated files do not by themselves prove correctness or authorship.",
   ];
-  if (evidence?.includeArtifactContents === false) return out.join("\n\n");
+  if (evidence?.includeArtifactContents === false) {
+    out.push("[File contents excluded by the evidence selector.]");
+    return out.join("\n\n");
+  }
   let budget = Math.min(evidence?.maxArtifactChars ?? 60_000, MAX_ARTIFACT_CHARS);
-  for (const artifact of selected) {
-    if (budget <= 0) break;
+  // Empty and unreadable files still cost framing; do not let them evade the cap.
+  let framingBudget = 8_000;
+  for (const [index, artifact] of selected.entries()) {
+    const path = boundedText(artifact.path, 500, "artifact path");
+    if (budget <= 0 || framingBudget < path.length + 100) {
+      out.push(
+        `[Content omitted for ${selected.length - index} remaining selected file(s): artifact ${budget <= 0 ? "character" : "framing"} budget exhausted.]`,
+      );
+      break;
+    }
     const limit = Math.min(perFileLimit, budget);
-    const content = await readRunArtifactText(run, artifact.path, {
-      // Four UTF-8 bytes per requested character preserves the character cap.
-      maxBytes: Math.max(1, limit * 4),
-    })
-      .then((text) => text.slice(0, limit))
-      .catch(() => null);
-    if (content === null) continue;
-    budget -= content.length;
-    out.push(`--- ${artifact.path} ---\n${content}`);
+    const content = await readRunArtifactExcerpt(run, artifact.path, limit).catch(() => null);
+    if (content === null) {
+      const marker = `[Content unavailable: ${path}. The captured file could not be read safely.]`;
+      framingBudget -= marker.length + 2;
+      out.push(marker);
+      continue;
+    }
+    budget -= content.text.length;
+    const block = `Origin: ${artifact.origin ?? "unknown"}\n--- ${path} ---\n${content.text}${content.truncated ? "\n[File excerpt truncated; omitted content is unassessed.]" : ""}`;
+    framingBudget -= block.length - content.text.length + 2;
+    out.push(block);
   }
   return out.join("\n\n");
 };
 
-export const buildJudgePrompt = async (config: RubricConfig, run: Run): Promise<string> => {
+type InvocationCompleteness = {
+  complete: boolean;
+  total: number;
+  included: number;
+  bytes: number;
+};
+
+export class IncompleteInvocationEvidence extends Error {
+  readonly detail: InvocationCompleteness;
+  constructor(message: string, detail: InvocationCompleteness) {
+    super(message);
+    this.detail = detail;
+  }
+}
+
+const allToolInvocations = (
+  run: Run,
+  budget: number,
+): { text: string; detail: InvocationCompleteness } => {
+  const calls = run.events.filter(
+    (event) => event.type === "progress" && event.kind === "tool-call",
+  );
+  const detail: InvocationCompleteness = {
+    complete: false,
+    total: calls.length,
+    included: 0,
+    bytes: 0,
+  };
+  const lines: string[] = [];
+  try {
+    for (const event of calls) {
+      const payload = (event as Extract<RunEvent, { type: "progress" }>).payload as {
+        name?: unknown;
+        input?: unknown;
+      };
+      if (
+        typeof payload?.name !== "string" ||
+        !("input" in payload) ||
+        payload.input === undefined
+      ) {
+        throw new Error("tool invocation lacks a name or input");
+      }
+      lines.push(JSON.stringify({ name: payload.name, input: payload.input }));
+    }
+  } catch {
+    throw new IncompleteInvocationEvidence(
+      "Complete tool invocation evidence cannot be serialized.",
+      detail,
+    );
+  }
+  const inventory = lines.join("\n");
+  detail.bytes = Buffer.byteLength(inventory, "utf8");
+  if (inventory.length > budget) {
+    throw new IncompleteInvocationEvidence(
+      "Complete tool invocation evidence exceeds the explicit invocation-character budget.",
+      detail,
+    );
+  }
+  detail.complete = true;
+  detail.included = calls.length;
+  return {
+    text: `Complete chronological tool invocations: ${JSON.stringify(detail)}\nThis inventory includes every recorded tool invocation independently of omissions in the recent trace. It does not establish completeness of tool results or artifact contents.\n${inventory}`,
+    detail,
+  };
+};
+
+export const buildJudgeEvidence = async (
+  config: RubricConfig,
+  run: Run,
+): Promise<{
+  prompt: string;
+  toolInvocations?: InvocationCompleteness;
+}> => {
+  const invocations =
+    config.evidence.toolInvocations === "all"
+      ? allToolInvocations(run, config.evidence.maxInvocationChars)
+      : undefined;
   if (config.promptOverride) {
-    return boundedText(config.promptOverride, JUDGE_PROMPT_CHARS, "judge prompt override");
+    if (invocations)
+      throw new IncompleteInvocationEvidence(
+        "A prompt override cannot establish complete tool invocation evidence.",
+        { ...invocations.detail, complete: false, included: 0 },
+      );
+    return {
+      prompt: boundedText(config.promptOverride, JUDGE_PROMPT_CHARS, "judge prompt override"),
+    };
   }
   const evidence: RubricEvidence = {
     ...config.evidence,
@@ -329,35 +503,79 @@ export const buildJudgePrompt = async (config: RubricConfig, run: Run): Promise<
     maxToolChars: Math.min(config.evidence.maxToolChars, MAX_TOOL_CHARS),
     maxArtifactChars: Math.min(config.evidence.maxArtifactChars, MAX_ARTIFACT_CHARS),
   };
-  const prompt = [
+  if (invocations) {
+    evidence.eventTypes = evidence.eventTypes.filter((type) => type !== "tool-call");
+  }
+  const required = [
     `Agent task: ${boundedText(run.case.prompt, 20_000, "agent task")}`,
     "",
     `Criterion to judge: ${boundedText(config.criteria, 8_000, "criterion")}`,
     "",
-    "Recent run trace:",
-    summarizeEvents(run.events, evidence.maxEvents, evidence),
+    ...(config.guidance ? [`Interpretation guidance: ${config.guidance}`, ""] : []),
+    `Evidence selection: recent trace event types are ${evidence.eventTypes.join(", ") || "none"}; tool detail is ${evidence.toolDetail}. Unselected events and excluded artifact contents are not evidence of missing work.`,
     "",
-    "Captured artifacts (excerpted):",
-    await sampleArtifacts(run, Math.min(config.maxArtifactExcerpt, MAX_ARTIFACT_EXCERPT), evidence),
+    ...(invocations ? [invocations.text, ""] : []),
   ].join("\n");
-  return boundedText(prompt, JUDGE_PROMPT_CHARS, "judge prompt");
+  const assemble = (recent: string, files: string): string =>
+    `${required}\nRecent run trace:\n${recent}\n\nCaptured artifacts (excerpted):\n${files}`;
+  const available = JUDGE_PROMPT_CHARS - assemble("", "").length;
+  if (invocations) {
+    // Reserve the minimal explicit omission notice, not verbose optional section markers.
+    if (`${required}\n${OPTIONAL_EVIDENCE_OMISSION}`.length > JUDGE_PROMPT_CHARS) {
+      throw new IncompleteInvocationEvidence(
+        "Complete tool invocation evidence and required framing exceed the judge-prompt budget.",
+        { ...invocations.detail, complete: false, included: 0 },
+      );
+    }
+  }
+  let trace = summarizeEvents(run.events, evidence.maxEvents, evidence);
+  let artifacts = await sampleArtifacts(
+    run,
+    Math.min(config.maxArtifactExcerpt, MAX_ARTIFACT_EXCERPT),
+    evidence,
+  );
+  let prompt: string;
+  if (
+    invocations &&
+    trace.length + artifacts.length > available &&
+    available < OPTIONAL_TRACE_TRUNCATION.length + OPTIONAL_ARTIFACT_TRUNCATION.length
+  ) {
+    prompt = `${required}\n${OPTIONAL_EVIDENCE_OMISSION}`;
+  } else {
+    if (invocations) [trace, artifacts] = fitOptionalEvidence(trace, artifacts, available);
+    prompt = assemble(trace, artifacts);
+  }
+  return {
+    prompt: boundedText(prompt, JUDGE_PROMPT_CHARS, "judge prompt"),
+    ...(invocations ? { toolInvocations: invocations.detail } : {}),
+  };
 };
 
-const JudgeResponseSchema = z
-  .object({
-    score: z.number().min(0).max(1),
-    comment: z.string().min(1).max(2_000),
-  })
-  .strict();
+export const buildJudgePrompt = async (config: RubricConfig, run: Run): Promise<string> =>
+  (await buildJudgeEvidence(config, run)).prompt;
+
+const JudgeResponseSchema = z.union([
+  z.object({ score: z.number().min(0).max(1), comment: z.string().min(1).max(2_000) }).strict(),
+  z
+    .object({
+      abstention: z.literal("insufficient-evidence"),
+      comment: z.string().min(1).max(2_000),
+    })
+    .strict()
+    .transform((response) => ({ ...response, score: 0 })),
+]);
+type JudgeResponse = z.output<typeof JudgeResponseSchema>;
 
-export const tryParseJudgeResponse = (text: string): { score: number; comment: string } | null => {
+export const tryParseJudgeResponse = (text: string): JudgeResponse | null => {
   const trimmed = text.trim();
   const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i.exec(trimmed);
   const cleaned = fenced?.[1]?.trim() ?? trimmed;
-  return JudgeResponseSchema.safeParse(tryParseJson(cleaned)).data ?? null;
+  const response = JudgeResponseSchema.safeParse(tryParseJson(cleaned)).data;
+  if (!response) return null;
+  return response;
 };
 
-export const parseJudgeResponse = (text: string): { score: number; comment: string } => {
+export const parseJudgeResponse = (text: string): JudgeResponse => {
   const verdict = tryParseJudgeResponse(text);
   if (verdict) return verdict;
   return {