@jsonstudio/llms 0.6.1892 → 0.6.2172

package/dist/conversion/shared/text-markup-normalizer/extractors-shared.js

@@ -0,0 +1,177 @@
+// Shared types/constants/helpers for text markup tool-call harvest.
+// Strict allowlist for tool names and their argument keys to avoid picking up
+// stray markup or free-form text as JSON keys (reduces false positives).
+export const KNOWN_TOOLS = new Set([
+    'shell',
+    'exec_command',
+    'write_stdin',
+    'apply_patch',
+    'update_plan',
+    'view_image',
+    'list_mcp_resources',
+    'read_mcp_resource',
+    'list_mcp_resource_templates',
+    // 文件/目录类工具（CLI 侧已有约定；此处只做文本→tool_calls 收割）
+    'list_directory'
+]);
+const ALLOWED_KEYS = {
+    shell: new Set(['command', 'justification', 'timeout_ms', 'with_escalated_permissions', 'workdir']),
+    exec_command: new Set([
+        'cmd',
+        'command',
+        'workdir',
+        'justification',
+        'login',
+        'tty',
+        'shell',
+        'yield_time_ms',
+        'max_output_tokens',
+        'sandbox_permissions',
+        'timeout_ms'
+    ]),
+    write_stdin: new Set(['session_id', 'chars', 'text', 'yield_time_ms', 'max_output_tokens']),
+    // apply_patch supports both freeform patch text (patch/input/instructions/text) and structured payloads (file/changes)
+    apply_patch: new Set(['patch', 'input', 'instructions', 'text', 'file', 'changes']),
+    update_plan: new Set(['explanation', 'plan']),
+    view_image: new Set(['path']),
+    list_mcp_resources: new Set(['server', 'cursor', 'filter', 'root']),
+    read_mcp_resource: new Set(['server', 'uri', 'cursor']),
+    list_mcp_resource_templates: new Set(['server', 'cursor']),
+    list_directory: new Set(['path', 'recursive'])
+};
+export function genToolCallId() {
+    return `call_${Math.random().toString(36).slice(2, 10)}`;
+}
+export function normalizeKey(raw) {
+    try {
+        const t = String(raw || '').trim();
+        if (!t)
+            return '';
+        // Pick the last alphanumeric/underscore token (e.g., '空的\n<tool_call>command' -> 'command')
+        const m = t.match(/([A-Za-z_][A-Za-z0-9_]*)$/);
+        return m ? m[1] : t;
+    }
+    catch {
+        return String(raw || '');
+    }
+}
+export function readDefaultWorkdirFromEnv() {
+    try {
+        const env = String(process?.env?.ROUTECODEX_WORKDIR || '').trim() ||
+            String(process?.env?.RCC_WORKDIR || '').trim() ||
+            String(process?.env?.CLAUDE_WORKDIR || '').trim();
+        return env ? env : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function filterArgsForTool(name, args) {
+    try {
+        const nm = String(name || '').toLowerCase();
+        const allow = ALLOWED_KEYS[nm];
+        if (!allow) {
+            // Unknown tool; be conservative: keep only simple safe keys if present
+            const safe = new Set([
+                'cmd',
+                'command',
+                'workdir',
+                'patch',
+                'input',
+                'instructions',
+                'text',
+                'plan',
+                'explanation',
+                'path',
+                'server',
+                'uri',
+                'cursor',
+                'filter',
+                'root'
+            ]);
+            const out = {};
+            for (const [k, v] of Object.entries(args || {})) {
+                const key = normalizeKey(k);
+                if (safe.has(key))
+                    out[key] = v;
+            }
+            return out;
+        }
+        const out = {};
+        for (const [k, v] of Object.entries(args || {})) {
+            const key = normalizeKey(k);
+            if (allow.has(key))
+                out[key] = v;
+        }
+        return out;
+    }
+    catch {
+        return args;
+    }
+}
+export function tryParseJsonValue(text) {
+    try {
+        const trimmed = String(text || '').trim();
+        if (!trimmed)
+            return null;
+        if (!(trimmed.startsWith('{') || trimmed.startsWith('[') || trimmed.startsWith('"'))) {
+            return null;
+        }
+        return JSON.parse(trimmed);
+    }
+    catch {
+        return null;
+    }
+}
+export function tryParsePrimitiveValue(text) {
+    try {
+        const trimmed = String(text ?? '').trim();
+        if (trimmed.length === 0)
+            return '';
+        const parsed = tryParseJsonValue(trimmed);
+        if (parsed !== null)
+            return parsed;
+        if (trimmed === 'true' || trimmed === 'false')
+            return trimmed === 'true';
+        if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) {
+            const n = Number(trimmed);
+            if (Number.isFinite(n))
+                return n;
+        }
+        return trimmed;
+    }
+    catch {
+        return String(text ?? '');
+    }
+}
+export function coerceCommandValueToString(value) {
+    try {
+        if (value === null || value === undefined)
+            return '';
+        if (typeof value === 'string')
+            return value.trim();
+        if (Array.isArray(value)) {
+            return value.map((v) => String(v)).join(' ').trim();
+        }
+        if (typeof value === 'object') {
+            const rec = value;
+            const direct = typeof rec.cmd === 'string'
+                ? String(rec.cmd)
+                : typeof rec.command === 'string'
+                    ? String(rec.command)
+                    : '';
+            if (direct.trim().length)
+                return direct.trim();
+            try {
+                return JSON.stringify(value);
+            }
+            catch {
+                return String(value);
+            }
+        }
+        return String(value).trim();
+    }
+    catch {
+        return '';
+    }
+}

package/dist/conversion/shared/text-markup-normalizer/extractors-transcript.d.ts

@@ -0,0 +1,5 @@
+import { type ToolCallLite } from './extractors-shared.js';
+export declare function extractBareExecCommandFromText(text: string): ToolCallLite[] | null;
+export declare function extractExecuteBlocksFromText(text: string): ToolCallLite[] | null;
+export declare function extractExploredListDirectoryCallsFromText(text: string): ToolCallLite[] | null;
+export declare function extractQwenToolCallTokensFromText(text: string): ToolCallLite[] | null;

package/dist/conversion/shared/text-markup-normalizer/extractors-transcript.js

@@ -0,0 +1,385 @@
+import { filterArgsForTool, genToolCallId, readDefaultWorkdirFromEnv } from './extractors-shared.js';
+import { salvageToolArgsFromRawText, tryParseJsonWithModelRepairs } from './extractors-json.js';
+function tryParseJsonArrayOnLine(line) {
+    try {
+        const trimmed = String(line || '').trim();
+        if (!trimmed.startsWith('[') || !trimmed.endsWith(']'))
+            return null;
+        const parsed = JSON.parse(trimmed);
+        if (!Array.isArray(parsed))
+            return null;
+        const out = [];
+        for (const entry of parsed) {
+            if (entry == null)
+                continue;
+            out.push(String(entry));
+        }
+        return out.length ? out : null;
+    }
+    catch {
+        return null;
+    }
+}
+function stripTrailingXmlGarbage(line) {
+    try {
+        const idx = line.indexOf('<');
+        if (idx >= 0) {
+            return line.slice(0, idx).trim();
+        }
+        return line.trim();
+    }
+    catch {
+        return String(line || '').trim();
+    }
+}
+function normalizePossibleCommandLine(raw) {
+    try {
+        let line = String(raw || '');
+        line = line.replace(/^\s*(?:[•*+-]\s*)?/, '');
+        line = line.replace(/^\s*\$\s*/, '');
+        return stripTrailingXmlGarbage(line);
+    }
+    catch {
+        return '';
+    }
+}
+function looksLikeShellCommand(line) {
+    try {
+        const t = String(line || '').trim();
+        if (!t)
+            return false;
+        if (t.startsWith('```') || t.startsWith('>'))
+            return false;
+        if (t.startsWith('zsh:') || t.startsWith('bash:'))
+            return false;
+        // Conservative allowlist: commands commonly emitted by Codex/Claude in tool-mode.
+        return /^(?:rg|ls|cat|node|npm|pnpm|yarn|git|find|sed|head|tail|wc|mkdir|rm|cp|mv|pwd)\b/.test(t);
+    }
+    catch {
+        return false;
+    }
+}
+function looksLikeBrokenToolMarkup(text) {
+    try {
+        const t = String(text || '');
+        if (!t)
+            return false;
+        const ranCommandMatch = t.match(/^\s*(?:[•*+-]\s*)?Ran\s+(.+)$/im);
+        const hasRanShellCommand = !!ranCommandMatch &&
+            (() => {
+                const payload = String(ranCommandMatch[1] || '').trim();
+                if (!payload)
+                    return false;
+                const arr = tryParseJsonArrayOnLine(payload);
+                if (arr && arr.length) {
+                    return looksLikeShellCommand(arr.join(' '));
+                }
+                return looksLikeShellCommand(payload);
+            })();
+        return (t.includes('<exec_command') ||
+            t.includes('<invoke') ||
+            t.includes('<function_calls') ||
+            t.includes('<parameter') ||
+            t.includes('<tool_call') ||
+            t.includes('</tool_call') ||
+            t.includes('<arg_value') ||
+            t.includes('</arg_value') ||
+            t.includes('</func_call') ||
+            /\bRan\b\s+\[/.test(t) ||
+            hasRanShellCommand);
+    }
+    catch {
+        return false;
+    }
+}
+export function extractBareExecCommandFromText(text) {
+    try {
+        if (typeof text !== 'string' || !text)
+            return null;
+        // Only attempt this when we see clear evidence of a broken tool markup (avoid turning examples into tool calls).
+        if (!looksLikeBrokenToolMarkup(text))
+            return null;
+        const candidates = [];
+        const lines = text.split(/\r?\n/);
+        for (let i = 0; i < lines.length; i += 1) {
+            const raw = String(lines[i] || '');
+            const line = normalizePossibleCommandLine(raw);
+            if (!line)
+                continue;
+            // Pattern A: "Ran [\"ls\", \"-l\", ...]" or "Ran git status"
+            const ran = line.match(/^Ran\s+(.+)$/i);
+            if (ran && ran[1]) {
+                const payload = String(ran[1] || '').trim();
+                const arr = tryParseJsonArrayOnLine(payload);
+                if (arr && arr.length) {
+                    const cmd = arr.join(' ').trim();
+                    if (looksLikeShellCommand(cmd)) {
+                        candidates.push({ cmd });
+                        continue;
+                    }
+                }
+                if (looksLikeShellCommand(payload)) {
+                    candidates.push({ cmd: payload });
+                    continue;
+                }
+            }
+            // Pattern B: a single-line shell command (rg/ls/node/...)
+            if (looksLikeShellCommand(line)) {
+                candidates.push({ cmd: line.trim() });
+                continue;
+            }
+        }
+        if (!candidates.length)
+            return null;
+        const picked = candidates[candidates.length - 1];
+        const argsObj = { cmd: picked.cmd };
+        const envWorkdir = readDefaultWorkdirFromEnv();
+        if (envWorkdir) {
+            argsObj.workdir = envWorkdir;
+        }
+        const filtered = filterArgsForTool('exec_command', argsObj);
+        let argsStr = '{}';
+        try {
+            argsStr = JSON.stringify(filtered);
+        }
+        catch {
+            argsStr = '{}';
+        }
+        return [
+            {
+                id: genToolCallId(),
+                name: 'exec_command',
+                args: argsStr
+            }
+        ];
+    }
+    catch {
+        return null;
+    }
+}
+export function extractExecuteBlocksFromText(text) {
+    try {
+        if (typeof text !== 'string' || !text)
+            return null;
+        const re = /<function=execute>\s*<parameter=command>([\s\S]*?)<\/parameter>\s*<\/function>/gi;
+        const out = [];
+        let m;
+        while ((m = re.exec(text)) !== null) {
+            const commandRaw = (m[1] || '').trim();
+            if (!commandRaw)
+                continue;
+            let args = '{}';
+            try {
+                args = JSON.stringify({ command: commandRaw });
+            }
+            catch {
+                args = '{"command":""}';
+            }
+            out.push({ id: genToolCallId(), name: 'shell', args });
+        }
+        return out.length ? out : null;
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeExploredListPath(raw) {
+    try {
+        let path = String(raw || '').trim();
+        // Drop common tree/marker prefixes from terminal transcript lines.
+        path = path.replace(/^[\s│]+/, '');
+        path = path.replace(/^[└├─]+\s*/, '');
+        // Strip wrapping quotes/backticks.
+        path = path.replace(/^([`"'])([\s\S]*?)\1$/, '$2').trim();
+        if (!path)
+            return '';
+        // Keep only first line to avoid swallowing multiline status output.
+        const firstLine = path.split(/\r?\n/)[0] || '';
+        path = firstLine.trim();
+        if (!path)
+            return '';
+        // Guard obvious non-path prose fragments.
+        if (/^(?:and|with|for|to)\b/i.test(path))
+            return '';
+        if (/[<>|;&]/.test(path))
+            return '';
+        return path;
+    }
+    catch {
+        return '';
+    }
+}
+// Harvest Codex-style transcript snippets:
+//   • Explored
+//     └ List xiaohongshu
+//       List app
+// into list_directory tool calls.
+export function extractExploredListDirectoryCallsFromText(text) {
+    try {
+        if (typeof text !== 'string' || !text)
+            return null;
+        const hasExploredMarker = /(?:^|\n)\s*(?:[•*+-]\s*)?Explored\b/i.test(text);
+        const hasTreeListMarker = /(?:^|\n)\s*[│\s]*[└├]\s*List\s+/im.test(text);
+        if (!hasExploredMarker && !hasTreeListMarker) {
+            return null;
+        }
+        const out = [];
+        const seen = new Set();
+        const lines = text.split(/\r?\n/);
+        for (const rawLine of lines) {
+            let line = String(rawLine || '').trim();
+            if (!line)
+                continue;
+            // Remove transcript bullets/tree prefixes.
+            line = line.replace(/^[•*+-]\s*/, '');
+            line = line.replace(/^[\s│]+/, '');
+            line = line.replace(/^[└├─]+\s*/, '');
+            const listMatch = /^List\s+([\s\S]+)$/i.exec(line);
+            if (!listMatch?.[1]) {
+                continue;
+            }
+            const path = normalizeExploredListPath(listMatch[1]);
+            if (!path) {
+                continue;
+            }
+            const dedupeKey = path.toLowerCase();
+            if (seen.has(dedupeKey)) {
+                continue;
+            }
+            seen.add(dedupeKey);
+            const filtered = filterArgsForTool('list_directory', { path, recursive: false });
+            let argsStr = '{}';
+            try {
+                argsStr = JSON.stringify(filtered);
+            }
+            catch {
+                argsStr = '{}';
+            }
+            out.push({
+                id: genToolCallId(),
+                name: 'list_directory',
+                args: argsStr
+            });
+        }
+        return out.length ? out : null;
+    }
+    catch {
+        return null;
+    }
+}
+// Extract Qwen-style tool call tokens that appear in plain text.
+export function extractQwenToolCallTokensFromText(text) {
+    try {
+        if (typeof text !== 'string' || !text)
+            return null;
+        // Upstreams (notably Qwen-family models behind OpenAI-compatible shells) may insert
+        // whitespace/newlines inside token markers, e.g. "<|  tool_call_begin|>".
+        if (!/<\|\s*tool_call_begin\s*\|>/i.test(text) && !/<\|\s*tool_call_argument_begin\s*\|>/i.test(text)) {
+            return null;
+        }
+        const out = [];
+        // tolerate whitespace/newlines and missing argument_end token
+        const re = /<\|\s*tool_call_begin\s*\|>\s*([A-Za-z0-9_.-]+)(?::(\d+))?\s*(?:<\|\s*tool_call_argument_begin\s*\|>\s*([\s\S]*?)\s*)?<\|\s*tool_call_end\s*\|>/gi;
+        let m;
+        while ((m = re.exec(text)) !== null) {
+            const rawName = String(m[1] || '').trim();
+            const numericId = typeof m[2] === 'string' && m[2].trim().length ? m[2].trim() : undefined;
+            const rawArgs = typeof m[3] === 'string' ? String(m[3]).trim() : '';
+            if (!rawName)
+                continue;
+            let name = rawName;
+            if (name.startsWith('functions.')) {
+                name = name.slice('functions.'.length);
+            }
+            const lname = name.toLowerCase();
+            if (!lname)
+                continue;
+            let parsedArgs = null;
+            if (!rawArgs) {
+                parsedArgs = {};
+            }
+            else {
+                parsedArgs = tryParseJsonWithModelRepairs(rawArgs);
+                if (parsedArgs === null) {
+                    parsedArgs = salvageToolArgsFromRawText(lname, rawArgs) ?? { _raw: rawArgs };
+                }
+            }
+            const argsObj = parsedArgs && typeof parsedArgs === 'object' && !Array.isArray(parsedArgs)
+                ? parsedArgs
+                : { value: parsedArgs };
+            // Alias normalization (models often use "command" instead of "cmd", etc.)
+            if (lname === 'exec_command') {
+                const cmd = typeof argsObj.cmd === 'string' ? String(argsObj.cmd) : '';
+                const command = typeof argsObj.command === 'string' ? String(argsObj.command) : '';
+                if (!cmd && command) {
+                    argsObj.cmd = command;
+                }
+                const cwd = typeof argsObj.cwd === 'string' ? String(argsObj.cwd) : '';
+                const workdir = typeof argsObj.workdir === 'string' ? String(argsObj.workdir) : '';
+                if (!workdir && cwd) {
+                    argsObj.workdir = cwd;
+                }
+            }
+            if (lname === 'write_stdin') {
+                const chars = typeof argsObj.chars === 'string' ? String(argsObj.chars) : '';
+                const textVal = typeof argsObj.text === 'string' ? String(argsObj.text) : '';
+                const input = typeof argsObj.input === 'string' ? String(argsObj.input) : '';
+                if (!chars && (textVal || input)) {
+                    argsObj.chars = textVal || input;
+                }
+                const sid = argsObj.session_id;
+                const sidAlt = argsObj.sessionId;
+                if ((sid === undefined || sid === null || String(sid).trim() === '') && sidAlt !== undefined && sidAlt !== null) {
+                    argsObj.session_id = sidAlt;
+                }
+            }
+            if (lname === 'apply_patch') {
+                const patch = typeof argsObj.patch === 'string' ? String(argsObj.patch) : '';
+                const textPatch = typeof argsObj.text === 'string' ? String(argsObj.text) : '';
+                const inputPatch = typeof argsObj.input === 'string' ? String(argsObj.input) : '';
+                const instructions = typeof argsObj.instructions === 'string' ? String(argsObj.instructions) : '';
+                if (!patch && (textPatch || inputPatch || instructions)) {
+                    argsObj.patch = textPatch || inputPatch || instructions;
+                }
+            }
+            const filtered = filterArgsForTool(lname, argsObj);
+            // Basic guards for known tools
+            if (lname === 'exec_command') {
+                const cmd = typeof filtered?.cmd === 'string' ? String(filtered.cmd).trim() : '';
+                if (!cmd)
+                    continue;
+            }
+            if (lname === 'write_stdin') {
+                // Allow empty args: our servertool schema may inject session_id automatically when a session is active.
+            }
+            if (lname === 'apply_patch') {
+                const patchText = typeof filtered?.patch === 'string' ? String(filtered.patch).trim() : '';
+                if (!patchText)
+                    continue;
+            }
+            let argsStr = '{}';
+            try {
+                argsStr = JSON.stringify(filtered);
+            }
+            catch {
+                argsStr = '{}';
+            }
+            const stableId = (() => {
+                if (numericId && /^[0-9]{1,10}$/.test(numericId)) {
+                    const namePart = lname.replace(/[^A-Za-z0-9_-]/g, '_');
+                    const id = `call_${namePart}_${numericId}`
+                        .replace(/_{2,}/g, '_')
+                        .replace(/^_+/, '')
+                        .replace(/_+$/, '');
+                    return id || undefined;
+                }
+                return undefined;
+            })();
+            out.push({ id: stableId ?? genToolCallId(), name: lname, args: argsStr });
+        }
+        return out.length ? out : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/conversion/shared/text-markup-normalizer/extractors-xml.d.ts

@@ -0,0 +1,10 @@
+import { type ToolCallLite } from './extractors-shared.js';
+export declare function extractToolNamespaceXmlBlocksFromText(text: string): ToolCallLite[] | null;
+export declare function extractParameterXmlToolsFromText(text: string): ToolCallLite[] | null;
+export declare function extractInvokeToolsFromText(text: string): ToolCallLite[] | null;
+export declare function extractXMLToolCallsFromText(text: string): ToolCallLite[] | null;
+/**
+ * 提取简单 XML 形式的工具调用块，例如：
+ *   <list_directory><path>/path</path><recursive>false</recursive></list_directory>
+ */
+export declare function extractSimpleXmlToolsFromText(text: string): ToolCallLite[] | null;