@tracemarketplace/shared 0.0.10 → 0.0.13

package/src/tool-normalization.ts

@@ -0,0 +1,670 @@
+import type { ContentBlock, NormalizedTrace } from "./types.js";
+import { toolNormalizationSpec } from "./tool-normalization.generated.js";
+
+const SED_IN_PLACE_FLAG_REGEX = /(?:^|\s)-i(?:$|\s|['"=])|--in-place(?:$|\s|=)/;
+const SORT_OUTPUT_FLAG_REGEX = /(?:^|\s)-o(?:$|\s|['"=])|--output(?:$|\s|=)/;
+const ANSI_ESCAPE_REGEX = /\u001b\[[0-9;]*m/g;
+const RM_RF_REGEX = /-\w*r\w*f|-\w*f\w*r|--recursive.*--force/;
+const SHELL_WRAPPER_VERBS = new Set(["bash", "sh", "zsh", "fish"]);
+const SHELL_COMMAND_FLAGS = new Set(["-c", "-lc", "-ic", "-cl", "-l", "-i"]);
+const SHELL_CHAIN_SEPARATORS = new Set(["&&", ";", "||", "|"]);
+const SHELL_SETUP_VERBS = new Set(["cd", "pushd", "popd", "dirs", "export", "source", ".", "set"]);
+
+type ToolDescriptorMap = typeof toolNormalizationSpec.normalized_tools;
+export type NormalizedToolId = keyof ToolDescriptorMap;
+export type NormalizedToolFamily = "shell" | "file";
+export type NormalizedToolDescriptor = {
+  family: NormalizedToolFamily;
+  kind: string;
+  aliases: readonly string[];
+  token_prefix: string;
+  mutates_file: boolean;
+};
+export type NormalizedToolIdentifier = NormalizedToolId | "tool.generic";
+export type FailureExchangeOutcome =
+  | "success"
+  | "failure"
+  | "success_after_retry"
+  | "canceled";
+export const TRACE_NORMALIZATION_VERSION = "evaluation-normalization-v1";
+
+export type ResolvedNormalizedTool = NormalizedToolDescriptor & {
+  normalizedToolId: NormalizedToolId;
+};
+
+export interface NormalizedToolUse {
+  toolCallId: string;
+  rawToolName: string;
+  normalizedToolId: NormalizedToolIdentifier;
+  family: NormalizedToolFamily | "generic";
+  kind: string;
+  token: string;
+  exitCode: number | null;
+}
+
+export interface FailureExchange {
+  toolTokens: string[];
+  hasError: boolean;
+  outcome: FailureExchangeOutcome;
+}
+
+export interface TraceNormalizationExchange {
+  exchangeIndex: number;
+  actions: NormalizedToolUse[];
+  toolTokens: string[];
+  hasError: boolean;
+  outcome: FailureExchangeOutcome;
+  prevOutcome: FailureExchangeOutcome | null;
+  tokenCount: number;
+}
+
+export interface TraceNormalization {
+  version: string;
+  exchanges: TraceNormalizationExchange[];
+}
+
+type ToolUseBlock = Extract<ContentBlock, { type: "tool_use" }>;
+
+const normalizedToolEntries = Object.entries(
+  toolNormalizationSpec.normalized_tools,
+) as Array<[NormalizedToolId, NormalizedToolDescriptor]>;
+const shellSpec = toolNormalizationSpec.shell;
+const fileSpec = toolNormalizationSpec.file;
+const textFilterWriteVerbs = new Set<string>(shellSpec.text_filter_write_verbs);
+const dangerousVerbs = new Set<string>(shellSpec.dangerous_verbs);
+
+export function normalizeToolName(toolName: string): string {
+  return toolName.trim().toLowerCase();
+}
+
+export function toolNameMatches(toolName: string, candidate: string): boolean {
+  const normalized = normalizeToolName(toolName);
+  return normalized === candidate || normalized.endsWith(`.${candidate}`);
+}
+
+export function getNormalizedToolHierarchy(
+  toolName: string,
+): ResolvedNormalizedTool | null {
+  for (const [normalizedToolId, descriptor] of normalizedToolEntries) {
+    if (descriptor.aliases.some((candidate) => toolNameMatches(toolName, candidate))) {
+      return { normalizedToolId, ...descriptor };
+    }
+  }
+
+  return null;
+}
+
+export function listNormalizedToolsByFamily(
+  family: NormalizedToolFamily,
+): ResolvedNormalizedTool[] {
+  return normalizedToolEntries
+    .filter(([, descriptor]) => descriptor.family === family)
+    .map(([normalizedToolId, descriptor]) => ({ normalizedToolId, ...descriptor }));
+}
+
+export function isShellToolName(toolName: string): boolean {
+  return getNormalizedToolHierarchy(toolName)?.family === "shell";
+}
+
+export function isWriteStdinToolName(toolName: string): boolean {
+  return getNormalizedToolHierarchy(toolName)?.normalizedToolId === "shell.stdin";
+}
+
+export function extractShellCommand(
+  toolInput: Record<string, unknown>,
+): string | null {
+  if (typeof toolInput["command"] === "string") {
+    return toolInput["command"];
+  }
+
+  if (typeof toolInput["cmd"] === "string") {
+    return toolInput["cmd"];
+  }
+
+  if (typeof toolInput["input"] === "string") {
+    return toolInput["input"];
+  }
+
+  if (Array.isArray(toolInput["input"])) {
+    return toolInput["input"].map((part) => String(part)).join(" ").trim();
+  }
+
+  return null;
+}
+
+function stripOuterQuotes(value: string): string {
+  const normalized = value.trim();
+  if (
+    normalized.length >= 2 &&
+    normalized[0] === normalized[normalized.length - 1] &&
+    (normalized[0] === "\"" || normalized[0] === "'")
+  ) {
+    return normalized.slice(1, -1).trim();
+  }
+  return normalized;
+}
+
+function splitShellParts(command: string): string[] {
+  const normalized = stripOuterQuotes(command.trim())
+    .replace(/\r/g, " ")
+    .replace(/\n/g, " ; ");
+  const matches = normalized.match(/"[^"]*"|'[^']*'|&&|\|\||;|\||\S+/g);
+  return matches ?? [];
+}
+
+function skipAssignmentTokens(parts: string[], start = 0): number {
+  let index = start;
+  while (index < parts.length && /^\w+=\S+$/.test(parts[index] ?? "")) {
+    index += 1;
+  }
+  return index;
+}
+
+function basenameLower(token: string): string {
+  const parts = token.split("/");
+  return (parts[parts.length - 1] ?? "").toLowerCase();
+}
+
+function remainderAfterSeparator(parts: string[], start: number): string {
+  for (let index = start; index < parts.length; index += 1) {
+    if (!SHELL_CHAIN_SEPARATORS.has(parts[index] ?? "")) {
+      continue;
+    }
+
+    const remainder = parts.slice(index + 1).join(" ").trim();
+    if (remainder) {
+      return remainder;
+    }
+  }
+
+  return "";
+}
+
+function extractShellVerb(command: string): string {
+  const normalized = stripOuterQuotes(command.trim().replace(/^\(+/, "").trim());
+  const parts = splitShellParts(normalized);
+  if (parts.length === 0) {
+    return "";
+  }
+
+  const index = skipAssignmentTokens(parts);
+  if (index >= parts.length) {
+    return "";
+  }
+
+  const verb = basenameLower(parts[index] ?? "");
+
+  if (SHELL_WRAPPER_VERBS.has(verb)) {
+    let nextIndex = index + 1;
+    while (nextIndex < parts.length && SHELL_COMMAND_FLAGS.has(parts[nextIndex] ?? "")) {
+      nextIndex += 1;
+    }
+
+    const remainder = parts.slice(nextIndex).join(" ").trim();
+    if (remainder) {
+      return extractShellVerb(remainder);
+    }
+  }
+
+  if (verb === "timeout") {
+    let nextIndex = index + 1;
+    while (
+      nextIndex < parts.length &&
+      (/^-/.test(parts[nextIndex] ?? "") || /^\d+(?:\.\d+)?[smhd]?$/.test(parts[nextIndex] ?? ""))
+    ) {
+      nextIndex += 1;
+    }
+
+    const remainder = parts.slice(nextIndex).join(" ").trim();
+    if (remainder) {
+      return extractShellVerb(remainder);
+    }
+  }
+
+  if (verb === "env") {
+    const remainder = parts.slice(index + 1).join(" ").trim();
+    if (remainder) {
+      return extractShellVerb(remainder);
+    }
+  }
+
+  if (SHELL_SETUP_VERBS.has(verb)) {
+    const remainder = remainderAfterSeparator(parts, index + 1);
+    if (remainder) {
+      return extractShellVerb(remainder);
+    }
+  }
+
+  return verb;
+}
+
+export function hasShellWriteRedirect(command: string): boolean {
+  let inSingleQuotes = false;
+  let inDoubleQuotes = false;
+  let escaped = false;
+
+  for (let index = 0; index < command.length; index += 1) {
+    const char = command[index];
+
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+
+    if (!inSingleQuotes && char === "\\") {
+      escaped = true;
+      continue;
+    }
+
+    if (!inDoubleQuotes && char === "'") {
+      inSingleQuotes = !inSingleQuotes;
+      continue;
+    }
+
+    if (!inSingleQuotes && char === "\"") {
+      inDoubleQuotes = !inDoubleQuotes;
+      continue;
+    }
+
+    if (inSingleQuotes || inDoubleQuotes || char !== ">") {
+      continue;
+    }
+
+    const nextChar = command[index + 1] ?? "";
+    if (nextChar === "&" || nextChar === "(") {
+      continue;
+    }
+
+    if (nextChar === ">") {
+      const afterAppend = command[index + 2] ?? "";
+      if (afterAppend === "&" || afterAppend === "(") {
+        continue;
+      }
+    }
+
+    return true;
+  }
+
+  return false;
+}
+
+export function isWriteShapedShellCommand(command: string): boolean {
+  if (/\bapply_patch\b|\bperl\s+-pi\b/.test(command)) {
+    return true;
+  }
+
+  const verb = extractShellVerb(command);
+  if (verb === "sed") {
+    return SED_IN_PLACE_FLAG_REGEX.test(command) || hasShellWriteRedirect(command);
+  }
+
+  if (verb === "sort" && SORT_OUTPUT_FLAG_REGEX.test(command)) {
+    return true;
+  }
+
+  return (
+    textFilterWriteVerbs.has(verb) && hasShellWriteRedirect(command)
+  );
+}
+
+export function isFileMutationTool(
+  toolName: string,
+  toolInput: Record<string, unknown>,
+): boolean {
+  const hierarchy = getNormalizedToolHierarchy(toolName);
+  if (hierarchy?.mutates_file) {
+    return true;
+  }
+
+  if (
+    fileSpec.extra_mutation_aliases.some((candidate) =>
+      toolNameMatches(toolName, candidate),
+    )
+  ) {
+    return true;
+  }
+
+  if (!isShellToolName(toolName)) return false;
+
+  const command = extractShellCommand(toolInput);
+  return command !== null && isWriteShapedShellCommand(command);
+}
+
+function suffixFromExitCode(exitCode: number | null | undefined): ":pass" | ":fail" {
+  return exitCode != null && exitCode !== 0 ? ":fail" : ":pass";
+}
+
+function isEmptyWriteStdinPoll(toolInput: Record<string, unknown>): boolean {
+  const chars = toolInput["chars"];
+  return chars == null || chars === "";
+}
+
+function isControlOnlyWriteStdin(chars: string): boolean {
+  return Array.from(chars).every((char) => {
+    const code = char.charCodeAt(0);
+    return code < 0x20 || code === 0x7f;
+  });
+}
+
+function isInterruptWriteStdin(chars: string): boolean {
+  return chars.includes("\u0003");
+}
+
+function normalizeToolResultOutput(output: string | null | undefined): string {
+  return (output ?? "")
+    .split(/\r?\n/)
+    .map((line) => line.replace(ANSI_ESCAPE_REGEX, "").trim())
+    .filter((line) => {
+      if (!line) {
+        return false;
+      }
+
+      return !toolNormalizationSpec.tool_result_metadata_prefixes.some((prefix) =>
+        line.startsWith(prefix),
+      );
+    })
+    .join("\n")
+    .trim();
+}
+
+function isCleanInterruptResult(
+  resultContent: string | null | undefined,
+  exitCode: number | null | undefined,
+): boolean {
+  return exitCode != null && exitCode !== 0 && normalizeToolResultOutput(resultContent) === "^C";
+}
+
+export function tokenizeWriteStdinChars(
+  chars: string,
+  exitCode: number | null | undefined,
+): string {
+  const suffix = suffixFromExitCode(exitCode);
+
+  if (isInterruptWriteStdin(chars)) {
+    return `Bash:pty_control:interrupt${suffix}`;
+  }
+
+  if (isControlOnlyWriteStdin(chars)) {
+    return `Bash:pty_control${suffix}`;
+  }
+
+  return `Bash:pty_input${suffix}`;
+}
+
+export function tokenizeBashCommand(
+  command: string,
+  exitCode: number | null | undefined,
+): string {
+  const verb = extractShellVerb(command);
+  const suffix = suffixFromExitCode(exitCode);
+
+  if (/^\$[0-9a-z]+$/.test(verb)) {
+    return `Bash:placeholder${suffix}`;
+  }
+
+  if (verb === "rm" && RM_RF_REGEX.test(command)) {
+    return "Bash:rm_rf";
+  }
+
+  if (verb === "sed") {
+    return `Bash:${isWriteShapedShellCommand(command) ? "fs_write" : "fs_read"}${suffix}`;
+  }
+
+  const textFilterToken =
+    shellSpec.text_filter_tokens[verb as keyof typeof shellSpec.text_filter_tokens];
+  if (textFilterToken) {
+    const category = isWriteShapedShellCommand(command)
+      ? `text_filter_write:${verb}`
+      : textFilterToken;
+    return `Bash:${category}${suffix}`;
+  }
+
+  if ((verb === "perl" || verb === "apply_patch") && isWriteShapedShellCommand(command)) {
+    return `Bash:fs_write${suffix}`;
+  }
+
+  if (dangerousVerbs.has(verb)) {
+    return `Bash:dangerous:${verb}`;
+  }
+
+  const category =
+    shellSpec.verb_categories[verb as keyof typeof shellSpec.verb_categories] ?? "other";
+  return `Bash:${category}${suffix}`;
+}
+
+export function tokenizeFileTool(toolName: string, rawPath: string): string {
+  const hierarchy = getNormalizedToolHierarchy(toolName);
+  const tokenPrefix = hierarchy?.token_prefix ?? toolName;
+  const path = rawPath.toLowerCase().replace(/\\/g, "/");
+
+  if (fileSpec.sensitive_path_signals.some((signal) => path.includes(signal))) {
+    return `${tokenPrefix}:sensitive`;
+  }
+  if (fileSpec.system_prefixes.some((prefix) => path.startsWith(prefix))) {
+    return `${tokenPrefix}:system`;
+  }
+  if (
+    fileSpec.config_exts.some((ext) => path.endsWith(ext)) ||
+    fileSpec.config_names.some((name) => path.includes(name))
+  ) {
+    return `${tokenPrefix}:config`;
+  }
+  return `${tokenPrefix}:project`;
+}
+
+function extractToolPath(toolInput: Record<string, unknown>): string {
+  if (typeof toolInput["file_path"] === "string") {
+    return toolInput["file_path"];
+  }
+
+  if (typeof toolInput["path"] === "string") {
+    return toolInput["path"];
+  }
+
+  return "";
+}
+
+export function normalizeToolUse(
+  block: ToolUseBlock,
+  exitCode: number | null | undefined,
+): NormalizedToolUse | null {
+  const hierarchy = getNormalizedToolHierarchy(block.tool_name);
+
+  if (hierarchy?.normalizedToolId === "shell.stdin") {
+    if (isEmptyWriteStdinPoll(block.tool_input)) {
+      return null;
+    }
+
+    const chars = block.tool_input["chars"];
+    if (typeof chars === "string") {
+      return {
+        toolCallId: block.tool_call_id,
+        rawToolName: block.tool_name,
+        normalizedToolId: hierarchy.normalizedToolId,
+        family: hierarchy.family,
+        kind: hierarchy.kind,
+        token: tokenizeWriteStdinChars(chars, exitCode),
+        exitCode: exitCode ?? null,
+      };
+    }
+  }
+
+  if (hierarchy?.normalizedToolId === "shell.command") {
+    const command = extractShellCommand(block.tool_input);
+    if (command !== null) {
+      return {
+        toolCallId: block.tool_call_id,
+        rawToolName: block.tool_name,
+        normalizedToolId: hierarchy.normalizedToolId,
+        family: hierarchy.family,
+        kind: hierarchy.kind,
+        token: tokenizeBashCommand(command, exitCode),
+        exitCode: exitCode ?? null,
+      };
+    }
+  }
+
+  if (hierarchy?.family === "file") {
+    return {
+      toolCallId: block.tool_call_id,
+      rawToolName: block.tool_name,
+      normalizedToolId: hierarchy.normalizedToolId,
+      family: hierarchy.family,
+      kind: hierarchy.kind,
+      token: tokenizeFileTool(block.tool_name, extractToolPath(block.tool_input)),
+      exitCode: exitCode ?? null,
+    };
+  }
+
+  return {
+    toolCallId: block.tool_call_id,
+    rawToolName: block.tool_name,
+    normalizedToolId: "tool.generic",
+    family: "generic",
+    kind: "tool",
+    token: `Tool:${block.tool_name}${suffixFromExitCode(exitCode)}`,
+    exitCode: exitCode ?? null,
+  };
+}
+
+function outcomeFromExchange(
+  toolTokens: string[],
+  hasError: boolean,
+): FailureExchangeOutcome {
+  if (!hasError) {
+    return "success";
+  }
+
+  for (let i = 0; i < toolTokens.length; i += 1) {
+    const token = toolTokens[i];
+    if (!token?.includes(":fail")) {
+      continue;
+    }
+
+    const base = token.replace(/:fail$/, "");
+    for (const laterToken of toolTokens.slice(i + 1)) {
+      if (laterToken.startsWith(base) && laterToken.includes(":pass")) {
+        return "success_after_retry";
+      }
+    }
+  }
+
+  return "failure";
+}
+
+export function extractFailureExchanges(trace: NormalizedTrace): FailureExchange[] {
+  return normalizeTraceForEvaluation(trace).exchanges.map((exchange) => ({
+    toolTokens: exchange.toolTokens,
+    hasError: exchange.hasError,
+    outcome: exchange.outcome,
+  }));
+}
+
+export function normalizeTraceForEvaluation(
+  trace: NormalizedTrace,
+): TraceNormalization {
+  const exchanges: TraceNormalizationExchange[] = [];
+  const turns = trace.turns;
+  let turnIndex = 0;
+  let prevOutcome: FailureExchangeOutcome | null = null;
+
+  while (turnIndex < turns.length) {
+    if (turns[turnIndex]?.role !== "user") {
+      turnIndex += 1;
+      continue;
+    }
+
+    let assistantEnd = turnIndex + 1;
+    while (assistantEnd < turns.length && turns[assistantEnd]?.role === "assistant") {
+      assistantEnd += 1;
+    }
+
+    const assistantTurns = turns.slice(turnIndex + 1, assistantEnd);
+    const actions: NormalizedToolUse[] = [];
+    let hasError = false;
+    let hasCancellation = false;
+    const exitCodes = new Map<string, number | null>();
+    const interruptToolCallIds = new Set<string>();
+    const tokenCount = assistantTurns.reduce(
+      (sum, turn) => sum + (turn.usage?.output_tokens ?? 0),
+      0,
+    );
+
+    for (const turn of assistantTurns) {
+      for (const block of turn.content) {
+        if (
+          block.type === "tool_use" &&
+          isWriteStdinToolName(block.tool_name) &&
+          typeof block.tool_input["chars"] === "string" &&
+          isInterruptWriteStdin(block.tool_input["chars"])
+        ) {
+          interruptToolCallIds.add(block.tool_call_id);
+        }
+      }
+    }
+
+    for (const turn of assistantTurns) {
+      for (const block of turn.content) {
+        if (block.type !== "tool_result") {
+          continue;
+        }
+
+        exitCodes.set(block.tool_call_id, block.exit_code);
+        if (
+          block.is_error &&
+          interruptToolCallIds.has(block.tool_call_id) &&
+          isCleanInterruptResult(block.result_content, block.exit_code)
+        ) {
+          hasCancellation = true;
+          continue;
+        }
+
+        if (block.is_error) {
+          hasError = true;
+        }
+      }
+    }
+
+    for (const turn of assistantTurns) {
+      for (const block of turn.content) {
+        if (block.type !== "tool_use") {
+          continue;
+        }
+
+        const normalizedTool = normalizeToolUse(
+          block,
+          exitCodes.get(block.tool_call_id),
+        );
+        if (normalizedTool !== null) {
+          actions.push(normalizedTool);
+        }
+      }
+    }
+
+    const toolTokens = actions.map((action) => action.token);
+
+    if (toolTokens.length > 0 || hasError || hasCancellation) {
+      const outcome =
+        hasCancellation && !hasError
+          ? "canceled"
+          : outcomeFromExchange(toolTokens, hasError);
+      exchanges.push({
+        exchangeIndex: exchanges.length,
+        actions,
+        toolTokens,
+        hasError,
+        outcome,
+        prevOutcome,
+        tokenCount,
+      });
+      prevOutcome = outcome;
+    }
+
+    turnIndex = assistantEnd;
+  }
+
+  return {
+    version: TRACE_NORMALIZATION_VERSION,
+    exchanges,
+  };
+}