muonroi-cli 1.4.0 → 1.4.1

package/src/orchestrator/text-tool-call-detector.ts

@@ -1,200 +0,0 @@
-/**
- * src/orchestrator/text-tool-call-detector.ts
- *
- * Detect when a model emitted a TOOL CALL as plain assistant TEXT instead of
- * using the real tool-calling interface. Some models (esp. cheap ones trained
- * on other agent harnesses — Cline / Roo / Continue dialects) write tool calls
- * as XML blocks like:
- *
- *     <read_file>
- *       <path>src/app/foo.ts</path>
- *     </read_file>
- *
- * The OpenAI-compatible layer never recognizes these as tool calls, so the SDK
- * reports finishReason="stop" with this XML as the final answer — the agentic
- * loop ends, the intended action never runs, and (live: storyflow_ui A/B,
- * deepseek-v4-flash session 905d564dbde4) the turn is silently wasted, often
- * leaving a half-finished edit behind. This detector lets the orchestrator
- * surface the failure instead of returning the broken text as a "final answer".
- *
- * Distinct from tool-args-repair.ts, which repairs the `arguments` JSON of a
- * tool call the SDK ALREADY recognized. Here there is NO recognized tool call.
- *
- * Precision is the priority — a false positive would wrongly flag a legitimate
- * final answer (e.g. documentation that quotes a tool block). The detector only
- * fires when a KNOWN tool-name tag appears as a structural invocation: an
- * opening tag immediately followed (whitespace/newlines allowed) by either a
- * known nested parameter tag or its own closing tag. A bare inline mention
- * ("use the <read_file> tool") never matches.
- */
-
-// Tool-name tags from the common text-dialect agent harnesses (Cline, Roo,
-// Continue, aider-ish) plus the generic wrappers. These are the OPENING tags a
-// model emits when it tries to invoke a tool as text.
-const TOOL_TAGS = [
-  "read_file",
-  "write_to_file",
-  "write_file",
-  "edit_file",
-  "apply_diff",
-  "replace_in_file",
-  "search_and_replace",
-  "insert_content",
-  "search_files",
-  "list_files",
-  "list_code_definition_names",
-  "execute_command",
-  "run_command",
-  "browser_action",
-  "use_mcp_tool",
-  "access_mcp_resource",
-  "ask_followup_question",
-  "attempt_completion",
-  // muonroi builtin tool names the NON_ANTHROPIC_TOOL_PREAMBLE explicitly
-  // forbids as text. Live: deepseek emitted `<bash>find …</bash>` as text at
-  // turn end (session ab/ds-final) — `bash` was missing here so it slipped the
-  // detector → no re-steer. Match only with a close tag or nested param,
-  // never a bare mention.
-  "bash",
-  "grep",
-  "glob",
-  "read_multiple_files",
-  "delegate",
-  "task",
-] as const;
-
-// Nested parameter tags that legitimately appear INSIDE a text-dialect tool
-// block. An opening tool tag followed shortly by one of these is a strong
-// signal of an actual (mis-formatted) invocation rather than a prose mention.
-const PARAM_TAGS = [
-  "path",
-  "content",
-  "command",
-  "diff",
-  "args",
-  "query",
-  "regex",
-  "file_path",
-  "search",
-  "replace",
-  "line",
-  "operations",
-  "question",
-  "result",
-  "recursive",
-  "uri",
-  "server_name",
-  "tool_name",
-  "arguments",
-];
-
-const PARAM_ALTERNATION = PARAM_TAGS.join("|");
-
-// Generic tool-call wrappers used by other native formats. `<invoke name="...">`
-// is the Anthropic XML style; `<tool_call>` / `<function_calls>` are Qwen/other.
-// These are matched directly (the wrapper itself is the signal).
-const GENERIC_WRAPPER_RE =
-  /<\/?(?:tool_call|function_calls|tool_use)\b|<invoke\b[^>]*\bname\s*=|<function\b[^>]*\bname\s*=/i;
-
-// DeepSeek native tool-call markup leaking into text content. Signature is the
-// fullwidth-bar sentinel `｜｜DSML｜｜` (U+FF5C, NOT ASCII pipes) wrapping
-// invoke/tool_calls/parameter tokens, e.g. `<｜｜DSML｜｜invoke name="read_file">`.
-// Live: storyflow_ui explore-A/B, deepseek T3 (session 799f0508e830) emitted a
-// full `<｜｜DSML｜｜invoke name="read_file">…` block as text and made no real tool
-// call → empty, silent turn. The generic `<invoke` matcher misses it because the
-// `<` is followed by the sentinel, not `invoke`. (BUG-A telemetry already flags
-// the `｜｜DSML｜｜` substring; this wires it into detect → re-steer/surface.)
-const DSML_WRAPPER_RE = /｜｜DSML｜｜\s*(?:invoke|tool_calls?|parameter)\b/i;
-const DSML_INVOKE_NAME_RE = /｜｜DSML｜｜\s*invoke\s+name\s*=\s*"([^"]+)"/i;
-
-/** Build a per-tool detector: `<tool>` then (within a small gap) a `<param>` or `</tool>`. */
-function buildToolRegexes(): RegExp[] {
-  return TOOL_TAGS.map(
-    (tag) =>
-      // <tag ...> [up to ~400 chars of whitespace/attrs/text] then <param> or </tag>
-      new RegExp(`<${tag}\\b[^>]*>[\\s\\S]{0,400}?(?:<(?:${PARAM_ALTERNATION})\\b[^>]*>|</${tag}>)`, "i"),
-  );
-}
-
-const TOOL_REGEXES = buildToolRegexes();
-
-export interface TextToolCallDetection {
-  detected: boolean;
-  /** The first matched tool/wrapper name, for telemetry. Null when not detected. */
-  tool: string | null;
-}
-
-/**
- * Detect a tool call emitted as plain text. Returns the first matching tool
- * name (or generic wrapper label) for telemetry. High precision: requires a
- * structural invocation shape, not a bare tag mention.
- */
-export function detectTextEmittedToolCall(text: string): TextToolCallDetection {
-  if (!text || text.length === 0) return { detected: false, tool: null };
-  // Cap the scan — pathological inputs shouldn't cost more than a glance.
-  const scan = text.length > 200_000 ? text.slice(0, 200_000) : text;
-
-  if (DSML_WRAPPER_RE.test(scan)) {
-    const nameMatch = scan.match(DSML_INVOKE_NAME_RE);
-    return { detected: true, tool: nameMatch ? nameMatch[1]! : "dsml" };
-  }
-
-  if (GENERIC_WRAPPER_RE.test(scan)) {
-    const m = scan.match(GENERIC_WRAPPER_RE);
-    return { detected: true, tool: m ? normalizeWrapperName(m[0]) : "tool_call" };
-  }
-
-  for (let i = 0; i < TOOL_REGEXES.length; i++) {
-    if (TOOL_REGEXES[i]!.test(scan)) {
-      return { detected: true, tool: TOOL_TAGS[i]! };
-    }
-  }
-  return { detected: false, tool: null };
-}
-
-function normalizeWrapperName(raw: string): string {
-  const m = raw.match(/tool_call|function_calls|tool_use|invoke|function/i);
-  return m ? m[0].toLowerCase() : "tool_call";
-}
-
-/**
- * Parse the DeepSeek-native DSML tool-call markup into a structured list so the
- * re-steer can restate the model's EXACT intent (much more effective than a
- * generic "use the tool interface" nudge). Pure — no execution. Recognizes:
- *   <｜｜DSML｜｜invoke name="read_file">
- *     <｜｜DSML｜｜parameter name="file_path" string="true">src/app/foo.ts</｜｜DSML｜｜parameter>
- *   </｜｜DSML｜｜invoke>
- * Returns one entry per invoke block; args preserve insertion order. Tolerant of
- * missing close tags (cheap models truncate) — captures whatever parameters are
- * present. Returns [] when no parseable invoke block exists.
- */
-export interface ParsedDsmlCall {
-  name: string;
-  args: Record<string, string>;
-}
-
-const DSML_INVOKE_BLOCK_RE = /｜｜DSML｜｜\s*invoke\s+name\s*=\s*"([^"]+)"([\s\S]*?)(?=｜｜DSML｜｜\s*invoke\s|$)/gi;
-const DSML_PARAM_RE = /｜｜DSML｜｜\s*parameter\s+name\s*=\s*"([^"]+)"[^>]*>([\s\S]*?)<\/?｜｜DSML｜｜\s*parameter/gi;
-
-export function parseDsmlToolCalls(text: string): ParsedDsmlCall[] {
-  if (!text || !text.includes("｜｜DSML｜｜")) return [];
-  const calls: ParsedDsmlCall[] = [];
-  DSML_INVOKE_BLOCK_RE.lastIndex = 0;
-  let block: RegExpExecArray | null;
-  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex.exec loop
-  while ((block = DSML_INVOKE_BLOCK_RE.exec(text)) !== null) {
-    const name = block[1]!;
-    const body = block[2] ?? "";
-    const args: Record<string, string> = {};
-    DSML_PARAM_RE.lastIndex = 0;
-    let p: RegExpExecArray | null;
-    // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex.exec loop
-    while ((p = DSML_PARAM_RE.exec(body)) !== null) {
-      args[p[1]!] = (p[2] ?? "").trim();
-    }
-    calls.push({ name, args });
-  }
-  return calls;
-}
-
-export const _internals = { TOOL_TAGS, PARAM_TAGS, GENERIC_WRAPPER_RE };

package/src/orchestrator/token-counter.test.ts

@@ -1,69 +0,0 @@
-import { afterEach, describe, expect, it } from "vitest";
-import { __forceFallbackForTests, countTokens, isTokenizerReady, setProviderHint } from "./token-counter";
-
-describe("token-counter", () => {
-  afterEach(() => {
-    __forceFallbackForTests(false);
-    setProviderHint(null);
-  });
-
-  it("returns 0 for empty input", () => {
-    expect(countTokens("")).toBe(0);
-  });
-
-  it("uses BPE tokenizer when available — code is denser than chars/4", () => {
-    __forceFallbackForTests(false);
-    expect(isTokenizerReady()).toBe(true);
-    // A short code-like string of 20 chars: chars/4 = 5, BPE typically 6–10.
-    const code = "const x = foo(bar);";
-    const bpe = countTokens(code);
-    const heuristic = Math.ceil(code.length / 4);
-    // BPE for code is generally >= heuristic (code uses more, denser tokens).
-    expect(bpe).toBeGreaterThanOrEqual(heuristic - 1);
-    // And it should NOT be a wild over-count.
-    expect(bpe).toBeLessThan(heuristic * 3);
-  });
-
-  it("falls back to chars/4 when forced", () => {
-    __forceFallbackForTests(true);
-    expect(countTokens("hello world")).toBe(Math.ceil("hello world".length / 4));
-    expect(isTokenizerReady()).toBe(false);
-  });
-
-  it("BPE is closer to ground truth on prose vs chars/4", () => {
-    // English prose: chars/4 typically overcounts. BPE gives a smaller number.
-    __forceFallbackForTests(false);
-    const prose = "The quick brown fox jumps over the lazy dog. Repeated several times for a longer sample.";
-    const bpe = countTokens(prose);
-    const heuristic = Math.ceil(prose.length / 4);
-    expect(bpe).toBeLessThan(heuristic);
-  });
-
-  it("applies the deepseek correction multiplier (~+5%) on top of cl100k_base", () => {
-    __forceFallbackForTests(false);
-    const text = "function add(a, b) { return a + b; }".repeat(10);
-
-    setProviderHint(null);
-    const baseline = countTokens(text);
-
-    setProviderHint("deepseek");
-    const adjusted = countTokens(text);
-
-    // Multiplier of 1.05 must round up — adjusted is strictly greater than baseline.
-    expect(adjusted).toBeGreaterThan(baseline);
-    expect(adjusted).toBe(Math.ceil(baseline * 1.05));
-  });
-
-  it("unknown providers get no correction (multiplier = 1)", () => {
-    __forceFallbackForTests(false);
-    const text = "hello world this is some sample text for tokenization";
-
-    setProviderHint(null);
-    const baseline = countTokens(text);
-
-    setProviderHint("some-unknown-provider");
-    const adjusted = countTokens(text);
-
-    expect(adjusted).toBe(baseline);
-  });
-});

package/src/orchestrator/token-counter.ts

@@ -1,81 +0,0 @@
-/**
- * Token counting backed by gpt-tokenizer (o200k_base / cl100k_base BPE) with a
- * chars/4 fallback.
- *
- * The tokenizer is a proxy: not every provider uses these encodings, but for
- * compaction-trigger decisions it is much closer to ground truth than chars/4
- * (which underestimates code by ~25–35% and overestimates prose by ~15–20%).
- *
- * If the import fails for any reason (bundler quirk, runtime issue), we
- * silently fall back so compaction never crashes on a token estimate.
- */
-
-import { encode as encodeCl100k } from "gpt-tokenizer";
-
-let useFallback = false;
-let providerHint: string | null = null;
-
-/**
- * Per-provider correction factors applied on top of cl100k_base BPE counts.
- * cl100k_base is 0% for OpenAI GPT-3.5/4 and a close proxy for tiktoken-family
- * tokenizers. For providers with mildly different vocabularies, a small
- * multiplier compensates for systematic under/over-count.
- *
- * Empirical baselines (English + code; CJK varies more):
- *   deepseek: ~+5%  (similar tiktoken family, slightly larger vocab)
- *
- * Keep this map small and conservative — multipliers > 1.15 indicate the
- * provider really needs its own tokenizer, not a fudge factor.
- */
-const PROVIDER_MULTIPLIER: Record<string, number> = {
-  deepseek: 1.05,
-};
-
-function multiplierFor(provider: string | null): number {
-  if (!provider) return 1;
-  return PROVIDER_MULTIPLIER[provider] ?? 1;
-}
-
-/**
- * Tell the tokenizer which provider is in use. Set once at orchestrator init
- * (or whenever the active provider changes). Defaults to no correction if
- * the provider is unknown.
- */
-export function setProviderHint(provider: string | null | undefined): void {
-  providerHint = provider ?? null;
-}
-
-export function getProviderHint(): string | null {
-  return providerHint;
-}
-
-function fallbackCharCount(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-/**
- * Best-effort token count for a single string.
- * Returns 0 for empty input. Falls back to chars/4 if the tokenizer throws.
- */
-export function countTokens(text: string): number {
-  if (!text) return 0;
-  if (useFallback) return fallbackCharCount(text);
-  try {
-    const base = encodeCl100k(text).length;
-    const mult = multiplierFor(providerHint);
-    return mult === 1 ? base : Math.ceil(base * mult);
-  } catch {
-    useFallback = true;
-    return fallbackCharCount(text);
-  }
-}
-
-/** For tests / diagnostics. */
-export function isTokenizerReady(): boolean {
-  return !useFallback;
-}
-
-/** For tests — force the fallback path. */
-export function __forceFallbackForTests(value: boolean): void {
-  useFallback = value;
-}

package/src/orchestrator/tool-args-hash.test.ts

@@ -1,121 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { canonicalizeBashCommand, hashToolArgs } from "./tool-args-hash.js";
-
-describe("canonicalizeBashCommand", () => {
-  it("strips trailing pipe + tail/head/grep", () => {
-    expect(canonicalizeBashCommand("bunx vitest run | tail -20")).toBe("bunx vitest run");
-    expect(canonicalizeBashCommand("bunx vitest run | head -10")).toBe("bunx vitest run");
-    expect(canonicalizeBashCommand("bunx vitest run | grep FAIL")).toBe("bunx vitest run");
-  });
-
-  it("strips stderr merge + redirect", () => {
-    expect(canonicalizeBashCommand("bunx vitest run 2>&1 | tail -5")).toBe("bunx vitest run");
-    expect(canonicalizeBashCommand("bunx vitest run > /tmp/out.log")).toBe("bunx vitest run");
-    expect(canonicalizeBashCommand("bunx vitest run >> /tmp/out.log")).toBe("bunx vitest run");
-  });
-
-  it("strips leading cd && prefix", () => {
-    expect(canonicalizeBashCommand("cd /d/Personal/Core/muonroi-cli && bunx vitest run | tail -5")).toBe(
-      "bunx vitest run",
-    );
-    expect(canonicalizeBashCommand('cd "D:/Personal/Core/muonroi-cli" && bunx vitest')).toBe("bunx vitest");
-  });
-
-  it("collapses whitespace", () => {
-    expect(canonicalizeBashCommand("bunx    vitest   run")).toBe("bunx vitest run");
-  });
-
-  it("does not strip when no redirection present", () => {
-    expect(canonicalizeBashCommand("git status")).toBe("git status");
-  });
-});
-
-describe("hashToolArgs", () => {
-  // Phase 5 BUG-F+G: verification commands now return `verify:` sentinel
-  // hashes with a per-call nonce so they NEVER collide with each other (each
-  // re-run of typecheck is a fresh progress marker, not a loop signal).
-  // We pick a non-verification cosmetic-variant set (npm-style install) to
-  // assert the original collapse behaviour still works.
-  it("collapses cosmetic bash variations to one hash (non-verification)", () => {
-    const variants = [
-      { command: "bun add zod | tail -20" },
-      { command: "bun add zod 2>&1" },
-      { command: "bun add zod > /tmp/out.log" },
-      { command: "cd /d/Personal/Core/muonroi-cli && bun add zod" },
-    ];
-    const hashes = new Set(variants.map((v) => hashToolArgs("bash", v)));
-    expect(hashes.size).toBe(1);
-  });
-
-  // Verification commands NEVER collide — every call must return a unique hash
-  // so the pattern detector skips them (edit→typecheck→fix is normal work).
-  it("verification commands produce unique hashes per call", () => {
-    const a = hashToolArgs("bash", { command: "bunx vitest run | tail -20" });
-    const b = hashToolArgs("bash", { command: "bunx vitest run | head -10" });
-    expect(a).not.toBe(b);
-    expect(a.startsWith("verify:")).toBe(true);
-    expect(b.startsWith("verify:")).toBe(true);
-  });
-
-  it("distinguishes substantively different non-verify bash commands", () => {
-    const a = hashToolArgs("bash", { command: "git status" });
-    const b = hashToolArgs("bash", { command: "git log -5" });
-    expect(a).not.toBe(b);
-  });
-
-  it("hashes non-bash tools by stable JSON of args", () => {
-    const a = hashToolArgs("read_file", { file_path: "a.ts", start_line: 1 });
-    const b = hashToolArgs("read_file", { start_line: 1, file_path: "a.ts" });
-    expect(a).toBe(b);
-  });
-
-  it("different args produce different hashes for same tool", () => {
-    const a = hashToolArgs("read_file", { file_path: "a.ts" });
-    const b = hashToolArgs("read_file", { file_path: "b.ts" });
-    expect(a).not.toBe(b);
-  });
-
-  it("hash is namespaced by tool name", () => {
-    const a = hashToolArgs("bash", { command: "x" });
-    const b = hashToolArgs("read_file", { command: "x" });
-    expect(a.startsWith("bash:")).toBe(true);
-    expect(b.startsWith("read_file:")).toBe(true);
-    expect(a).not.toBe(b);
-  });
-
-  it("collapses edit_file calls on same file regardless of old/new string", () => {
-    // Real session 39884b072b5f scenario — agent edited the same file 7 times
-    // with different old_string each attempt. Generic JSON hash kept them
-    // distinct; the dedicated file_path hash now collapses them.
-    const variants = [
-      { file_path: "src/ee/export.ts", old_string: "@ts-expect-error", new_string: "@ts-ignore" },
-      { file_path: "src/ee/export.ts", old_string: "  @ts-expect-error", new_string: "// @ts-ignore" },
-      { file_path: "src/ee/export.ts", old_string: "    const mod", new_string: "// @ts-ignore\n    const mod" },
-    ];
-    const hashes = new Set(variants.map((v) => hashToolArgs("edit_file", v)));
-    expect(hashes.size).toBe(1);
-  });
-
-  it("distinguishes edit_file on different files", () => {
-    const a = hashToolArgs("edit_file", { file_path: "src/a.ts", old_string: "x", new_string: "y" });
-    const b = hashToolArgs("edit_file", { file_path: "src/b.ts", old_string: "x", new_string: "y" });
-    expect(a).not.toBe(b);
-  });
-
-  it("normalizes path separators (D:\\foo vs D:/foo collapse)", () => {
-    const win = hashToolArgs("edit_file", { file_path: "D:\\Personal\\Core\\file.ts", old_string: "x" });
-    const posix = hashToolArgs("edit_file", { file_path: "D:/Personal/Core/file.ts", old_string: "x" });
-    expect(win).toBe(posix);
-  });
-
-  it("write_file uses same file-path hash as edit_file (cross-tool overwrite loops collapse separately)", () => {
-    const e = hashToolArgs("edit_file", { file_path: "a.ts" });
-    const w = hashToolArgs("write_file", { file_path: "a.ts" });
-    // Same file, but namespaced by tool — overwrite-loop and edit-loop are
-    // tracked separately so a legitimate "edit then verify-via-write" doesn't
-    // trip the guard, but 5 repeated write_file does.
-    expect(e).not.toBe(w);
-    expect(e.startsWith("edit_file:")).toBe(true);
-    expect(w.startsWith("write_file:")).toBe(true);
-  });
-});