@tracemarketplace/shared 0.0.10 → 0.0.11

package/src/tool-normalization.generated.ts

@@ -0,0 +1,262 @@
+// This file is generated from tool-normalization.spec.json. Do not edit directly.
+export const toolNormalizationSpec = {
+  "schema_version": 1,
+  "normalized_tools": {
+    "shell.command": {
+      "family": "shell",
+      "kind": "command",
+      "aliases": [
+        "exec_command",
+        "bash",
+        "bash_command",
+        "run_shell_command",
+        "shell"
+      ],
+      "token_prefix": "Bash",
+      "mutates_file": false
+    },
+    "shell.stdin": {
+      "family": "shell",
+      "kind": "stdin",
+      "aliases": [
+        "write_stdin"
+      ],
+      "token_prefix": "Bash",
+      "mutates_file": false
+    },
+    "file.read": {
+      "family": "file",
+      "kind": "read",
+      "aliases": [
+        "read",
+        "read_file",
+        "grep",
+        "list_directory"
+      ],
+      "token_prefix": "Read",
+      "mutates_file": false
+    },
+    "file.write": {
+      "family": "file",
+      "kind": "write",
+      "aliases": [
+        "write",
+        "write_file",
+        "create_file"
+      ],
+      "token_prefix": "Write",
+      "mutates_file": true
+    },
+    "file.edit": {
+      "family": "file",
+      "kind": "edit",
+      "aliases": [
+        "edit",
+        "edit_file",
+        "replace"
+      ],
+      "token_prefix": "Edit",
+      "mutates_file": true
+    },
+    "file.multi_edit": {
+      "family": "file",
+      "kind": "multi_edit",
+      "aliases": [
+        "multiedit"
+      ],
+      "token_prefix": "MultiEdit",
+      "mutates_file": true
+    }
+  },
+  "shell": {
+    "dangerous_verbs": [
+      "sudo",
+      "su",
+      "doas",
+      "chmod",
+      "chown",
+      "dd",
+      "shred"
+    ],
+    "text_filter_tokens": {
+      "awk": "text_filter:awk",
+      "cut": "text_filter:cut",
+      "sort": "text_filter:sort"
+    },
+    "text_filter_write_verbs": [
+      "awk",
+      "cut",
+      "sort"
+    ],
+    "verb_categories": {
+      "npm": "build",
+      "pnpm": "build",
+      "yarn": "build",
+      "bun": "build",
+      "npx": "build",
+      "cargo": "build",
+      "rustup": "build",
+      "pip": "build",
+      "pip3": "build",
+      "uv": "build",
+      "poetry": "build",
+      "pipenv": "build",
+      "conda": "build",
+      "make": "build",
+      "cmake": "build",
+      "ninja": "build",
+      "go": "build",
+      "gradle": "build",
+      "mvn": "build",
+      "bazel": "build",
+      "tsc": "build",
+      "vite": "build",
+      "webpack": "build",
+      "pytest": "test",
+      "jest": "test",
+      "vitest": "test",
+      "mocha": "test",
+      "rspec": "test",
+      "phpunit": "test",
+      "cargo-test": "test",
+      "git": "vcs",
+      "gh": "vcs",
+      "svn": "vcs",
+      "curl": "network",
+      "wget": "network",
+      "ssh": "network",
+      "scp": "network",
+      "rsync": "network",
+      "nc": "network",
+      "netcat": "network",
+      "ncat": "network",
+      "sftp": "network",
+      "ftp": "network",
+      "cat": "fs_read",
+      "head": "fs_read",
+      "tail": "fs_read",
+      "less": "fs_read",
+      "ls": "fs_read",
+      "find": "fs_read",
+      "locate": "fs_read",
+      "grep": "fs_read",
+      "rg": "fs_read",
+      "ag": "fs_read",
+      "ack": "fs_read",
+      "wc": "fs_read",
+      "stat": "fs_read",
+      "file": "fs_read",
+      "du": "fs_read",
+      "diff": "fs_read",
+      "od": "fs_read",
+      "nl": "fs_read",
+      "which": "fs_read",
+      "xxd": "fs_read",
+      "rm": "fs_write",
+      "cp": "fs_write",
+      "mv": "fs_write",
+      "mkdir": "fs_write",
+      "touch": "fs_write",
+      "ln": "fs_write",
+      "tar": "fs_write",
+      "zip": "fs_write",
+      "unzip": "fs_write",
+      "python": "run",
+      "python3": "run",
+      "node": "run",
+      "ruby": "run",
+      "php": "run",
+      "perl": "run",
+      "bash": "run",
+      "sh": "run",
+      "zsh": "run",
+      "fish": "run",
+      "deno": "run",
+      "r": "run",
+      "rscript": "run",
+      "kill": "process",
+      "pkill": "process",
+      "killall": "process",
+      "ps": "process",
+      "top": "process",
+      "htop": "process",
+      "lsof": "process",
+      "sleep": "process",
+      "export": "env",
+      "unset": "env",
+      "env": "env",
+      "source": "env",
+      "printenv": "env",
+      "set": "env",
+      "apt": "sysadmin",
+      "apt-get": "sysadmin",
+      "brew": "sysadmin",
+      "yum": "sysadmin",
+      "dnf": "sysadmin",
+      "pacman": "sysadmin",
+      "snap": "sysadmin",
+      "eval": "shell_meta",
+      "exec": "shell_meta",
+      "xargs": "shell_meta",
+      "cd": "shell_meta",
+      "pwd": "shell_meta",
+      "echo": "shell_meta",
+      "if": "shell_meta"
+    }
+  },
+  "file": {
+    "extra_mutation_aliases": [
+      "apply_patch",
+      "write_file",
+      "create_file",
+      "delete_file",
+      "rename_file",
+      "move_file",
+      "file_change"
+    ],
+    "sensitive_path_signals": [
+      ".ssh",
+      ".aws",
+      "id_rsa",
+      "credentials",
+      ".env",
+      "secrets",
+      "token",
+      "password",
+      "private_key"
+    ],
+    "system_prefixes": [
+      "/etc/",
+      "/usr/",
+      "/var/",
+      "/sys/",
+      "/proc/",
+      "/boot/",
+      "/root/"
+    ],
+    "config_exts": [
+      ".env",
+      ".yaml",
+      ".yml",
+      ".toml",
+      ".ini",
+      ".cfg"
+    ],
+    "config_names": [
+      "config",
+      "settings",
+      "configuration"
+    ]
+  },
+  "tool_result_metadata_prefixes": [
+    "Command:",
+    "Chunk ID:",
+    "Wall time:",
+    "Process exited with code",
+    "Process running with session ID",
+    "Original token count:",
+    "Output:"
+  ]
+} as const;
+
+export type ToolNormalizationSpec = typeof toolNormalizationSpec;

package/src/tool-normalization.spec.json

@@ -0,0 +1,205 @@
+{
+  "schema_version": 1,
+  "normalized_tools": {
+    "shell.command": {
+      "family": "shell",
+      "kind": "command",
+      "aliases": ["exec_command", "bash", "bash_command", "run_shell_command", "shell"],
+      "token_prefix": "Bash",
+      "mutates_file": false
+    },
+    "shell.stdin": {
+      "family": "shell",
+      "kind": "stdin",
+      "aliases": ["write_stdin"],
+      "token_prefix": "Bash",
+      "mutates_file": false
+    },
+    "file.read": {
+      "family": "file",
+      "kind": "read",
+      "aliases": ["read", "read_file", "grep", "list_directory"],
+      "token_prefix": "Read",
+      "mutates_file": false
+    },
+    "file.write": {
+      "family": "file",
+      "kind": "write",
+      "aliases": ["write", "write_file", "create_file"],
+      "token_prefix": "Write",
+      "mutates_file": true
+    },
+    "file.edit": {
+      "family": "file",
+      "kind": "edit",
+      "aliases": ["edit", "edit_file", "replace"],
+      "token_prefix": "Edit",
+      "mutates_file": true
+    },
+    "file.multi_edit": {
+      "family": "file",
+      "kind": "multi_edit",
+      "aliases": ["multiedit"],
+      "token_prefix": "MultiEdit",
+      "mutates_file": true
+    }
+  },
+  "shell": {
+    "dangerous_verbs": ["sudo", "su", "doas", "chmod", "chown", "dd", "shred"],
+    "text_filter_tokens": {
+      "awk": "text_filter:awk",
+      "cut": "text_filter:cut",
+      "sort": "text_filter:sort"
+    },
+    "text_filter_write_verbs": ["awk", "cut", "sort"],
+    "verb_categories": {
+      "npm": "build",
+      "pnpm": "build",
+      "yarn": "build",
+      "bun": "build",
+      "npx": "build",
+      "cargo": "build",
+      "rustup": "build",
+      "pip": "build",
+      "pip3": "build",
+      "uv": "build",
+      "poetry": "build",
+      "pipenv": "build",
+      "conda": "build",
+      "make": "build",
+      "cmake": "build",
+      "ninja": "build",
+      "go": "build",
+      "gradle": "build",
+      "mvn": "build",
+      "bazel": "build",
+      "tsc": "build",
+      "vite": "build",
+      "webpack": "build",
+      "pytest": "test",
+      "jest": "test",
+      "vitest": "test",
+      "mocha": "test",
+      "rspec": "test",
+      "phpunit": "test",
+      "cargo-test": "test",
+      "git": "vcs",
+      "gh": "vcs",
+      "svn": "vcs",
+      "curl": "network",
+      "wget": "network",
+      "ssh": "network",
+      "scp": "network",
+      "rsync": "network",
+      "nc": "network",
+      "netcat": "network",
+      "ncat": "network",
+      "sftp": "network",
+      "ftp": "network",
+      "cat": "fs_read",
+      "head": "fs_read",
+      "tail": "fs_read",
+      "less": "fs_read",
+      "ls": "fs_read",
+      "find": "fs_read",
+      "locate": "fs_read",
+      "grep": "fs_read",
+      "rg": "fs_read",
+      "ag": "fs_read",
+      "ack": "fs_read",
+      "wc": "fs_read",
+      "stat": "fs_read",
+      "file": "fs_read",
+      "du": "fs_read",
+      "diff": "fs_read",
+      "od": "fs_read",
+      "nl": "fs_read",
+      "which": "fs_read",
+      "xxd": "fs_read",
+      "rm": "fs_write",
+      "cp": "fs_write",
+      "mv": "fs_write",
+      "mkdir": "fs_write",
+      "touch": "fs_write",
+      "ln": "fs_write",
+      "tar": "fs_write",
+      "zip": "fs_write",
+      "unzip": "fs_write",
+      "python": "run",
+      "python3": "run",
+      "node": "run",
+      "ruby": "run",
+      "php": "run",
+      "perl": "run",
+      "bash": "run",
+      "sh": "run",
+      "zsh": "run",
+      "fish": "run",
+      "deno": "run",
+      "r": "run",
+      "rscript": "run",
+      "kill": "process",
+      "pkill": "process",
+      "killall": "process",
+      "ps": "process",
+      "top": "process",
+      "htop": "process",
+      "lsof": "process",
+      "sleep": "process",
+      "export": "env",
+      "unset": "env",
+      "env": "env",
+      "source": "env",
+      "printenv": "env",
+      "set": "env",
+      "apt": "sysadmin",
+      "apt-get": "sysadmin",
+      "brew": "sysadmin",
+      "yum": "sysadmin",
+      "dnf": "sysadmin",
+      "pacman": "sysadmin",
+      "snap": "sysadmin",
+      "eval": "shell_meta",
+      "exec": "shell_meta",
+      "xargs": "shell_meta",
+      "cd": "shell_meta",
+      "pwd": "shell_meta",
+      "echo": "shell_meta",
+      "if": "shell_meta"
+    }
+  },
+  "file": {
+    "extra_mutation_aliases": [
+      "apply_patch",
+      "write_file",
+      "create_file",
+      "delete_file",
+      "rename_file",
+      "move_file",
+      "file_change"
+    ],
+    "sensitive_path_signals": [
+      ".ssh",
+      ".aws",
+      "id_rsa",
+      "credentials",
+      ".env",
+      "secrets",
+      "token",
+      "password",
+      "private_key"
+    ],
+    "system_prefixes": ["/etc/", "/usr/", "/var/", "/sys/", "/proc/", "/boot/", "/root/"],
+    "config_exts": [".env", ".yaml", ".yml", ".toml", ".ini", ".cfg"],
+    "config_names": ["config", "settings", "configuration"]
+  },
+  "tool_result_metadata_prefixes": [
+    "Command:",
+    "Chunk ID:",
+    "Wall time:",
+    "Process exited with code",
+    "Process running with session ID",
+    "Original token count:",
+    "Output:"
+  ]
+}

package/src/tool-normalization.test.ts

@@ -0,0 +1,221 @@
+import { describe, expect, it } from "vitest";
+import type { ContentBlock, NormalizedTrace, Turn } from "./types.js";
+import {
+  extractFailureExchanges,
+  getNormalizedToolHierarchy,
+  isFileMutationTool,
+  listNormalizedToolsByFamily,
+  normalizeTraceForEvaluation,
+  normalizeToolUse,
+  TRACE_NORMALIZATION_VERSION,
+  tokenizeBashCommand,
+} from "./tool-normalization.js";
+
+function makeTrace(turns: Turn[]): NormalizedTrace {
+  return {
+    trace_id: "test-id",
+    schema_version: "1.0",
+    source_tool: "codex_cli",
+    source_session_id: "session-abc",
+    source_version: null,
+    submitted_by: "user1",
+    submitted_at: "2024-01-01T00:00:00Z",
+    extracted_at: "2024-01-01T00:00:00Z",
+    git_branch: null,
+    cwd_hash: null,
+    working_language: null,
+    started_at: "2024-01-01T00:00:00Z",
+    ended_at: "2024-01-01T00:01:00Z",
+    turns,
+    turn_count: turns.length,
+    tool_call_count: turns.flatMap((turn) => turn.content).filter((block) => block.type === "tool_use")
+      .length,
+    has_tool_calls: true,
+    has_thinking_blocks: false,
+    has_file_changes: false,
+    has_shell_commands: false,
+    total_input_tokens: null,
+    total_output_tokens: null,
+    total_cache_read_tokens: null,
+    content_fidelity: "full",
+    env_state: null,
+    score: null,
+    raw_r2_key: "",
+    normalized_r2_key: "",
+  };
+}
+
+function makeTurn(role: "user" | "assistant", content: ContentBlock[]): Turn {
+  return {
+    turn_id: Math.random().toString(36).slice(2),
+    parent_turn_id: null,
+    role,
+    timestamp: null,
+    content,
+    model: null,
+    usage: null,
+    source_metadata: {},
+  };
+}
+
+describe("tool hierarchy", () => {
+  it("normalizes namespaced shell tools into a single hierarchy", () => {
+    const normalized = getNormalizedToolHierarchy("functions.exec_command");
+
+    expect(normalized).toMatchObject({
+      normalizedToolId: "shell.command",
+      family: "shell",
+      kind: "command",
+      token_prefix: "Bash",
+    });
+  });
+
+  it("lists canonical file tools from the shared hierarchy", () => {
+    expect(listNormalizedToolsByFamily("file").map((tool) => tool.normalizedToolId)).toEqual([
+      "file.read",
+      "file.write",
+      "file.edit",
+      "file.multi_edit",
+    ]);
+  });
+});
+
+describe("tokenizeBashCommand", () => {
+  it("treats read-shaped and write-shaped shell commands differently", () => {
+    expect(tokenizeBashCommand("sed -n '1,40p' src/server.ts", 0)).toBe(
+      "Bash:fs_read:pass",
+    );
+    expect(tokenizeBashCommand("sed -i '' 's/old/new/' src/server.ts", 0)).toBe(
+      "Bash:fs_write:pass",
+    );
+    expect(tokenizeBashCommand("sort -o tmp/out.txt src/server.ts", 0)).toBe(
+      "Bash:text_filter_write:sort:pass",
+    );
+  });
+});
+
+describe("normalizeToolUse", () => {
+  it("drops empty write_stdin polls", () => {
+    const normalized = normalizeToolUse(
+      {
+        type: "tool_use",
+        tool_call_id: "stdin-1",
+        tool_name: "functions.write_stdin",
+        tool_input: { session_id: 1234, chars: "", yield_time_ms: 1000 },
+      },
+      null,
+    );
+
+    expect(normalized).toBeNull();
+  });
+
+  it("classifies file mutation tools through the shared hierarchy", () => {
+    expect(isFileMutationTool("Edit", { path: "src/server.ts" })).toBe(true);
+    expect(
+      isFileMutationTool("functions.exec_command", {
+        cmd: "awk 'NF' src/server.ts > tmp/server.filtered.ts",
+      }),
+    ).toBe(true);
+  });
+});
+
+describe("extractFailureExchanges", () => {
+  it("treats clean ctrl-c write_stdin flows as canceled exchanges", () => {
+    const turns = [
+      makeTurn("user", [{ type: "text", text: "Run the watch command, then stop it." }]),
+      makeTurn("assistant", [
+        {
+          type: "tool_use",
+          tool_call_id: "exec-1",
+          tool_name: "functions.exec_command",
+          tool_input: { cmd: "vitest --watch", tty: true },
+        },
+        {
+          type: "tool_result",
+          tool_call_id: "exec-1",
+          is_error: false,
+          result_content: "Process running with session ID 4242",
+          exit_code: null,
+        },
+        {
+          type: "tool_use",
+          tool_call_id: "stdin-1",
+          tool_name: "functions.write_stdin",
+          tool_input: { session_id: 4242, chars: "\u0003", yield_time_ms: 1000 },
+        },
+        {
+          type: "tool_result",
+          tool_call_id: "stdin-1",
+          is_error: true,
+          result_content: "Process exited with code 130\n^C",
+          exit_code: 130,
+        },
+      ]),
+    ];
+
+    expect(extractFailureExchanges(makeTrace(turns))).toEqual([
+      {
+        toolTokens: ["Bash:test:pass", "Bash:pty_control:interrupt:fail"],
+        hasError: false,
+        outcome: "canceled",
+      },
+    ]);
+  });
+});
+
+describe("normalizeTraceForEvaluation", () => {
+  it("builds a persisted exchange artifact with actions and versioning", () => {
+    const turns = [
+      makeTurn("user", [{ type: "text", text: "Run the tests." }]),
+      {
+        ...makeTurn("assistant", [
+          {
+            type: "tool_use",
+            tool_call_id: "exec-1",
+            tool_name: "functions.exec_command",
+            tool_input: { cmd: "pnpm test" },
+          },
+          {
+            type: "tool_result",
+            tool_call_id: "exec-1",
+            is_error: false,
+            result_content: "ok",
+            exit_code: 0,
+          },
+        ]),
+        usage: {
+          input_tokens: 10,
+          output_tokens: 42,
+          cache_read_input_tokens: null,
+          cache_creation_input_tokens: null,
+          reasoning_tokens: null,
+        },
+      },
+    ];
+
+    expect(normalizeTraceForEvaluation(makeTrace(turns))).toEqual({
+      version: TRACE_NORMALIZATION_VERSION,
+      exchanges: [
+        {
+          exchangeIndex: 0,
+          actions: [
+            {
+              toolCallId: "exec-1",
+              rawToolName: "functions.exec_command",
+              normalizedToolId: "shell.command",
+              family: "shell",
+              kind: "command",
+              token: "Bash:build:pass",
+              exitCode: 0,
+            },
+          ],
+          toolTokens: ["Bash:build:pass"],
+          hasError: false,
+          outcome: "success",
+          prevOutcome: null,
+          tokenCount: 42,
+        },
+      ],
+    });
+  });
+});