@jsonstudio/llms 0.6.2172 → 0.6.3214

package/dist/conversion/shared/text-markup-normalizer/extractors-shared.js

@@ -1,177 +0,0 @@
-// Shared types/constants/helpers for text markup tool-call harvest.
-// Strict allowlist for tool names and their argument keys to avoid picking up
-// stray markup or free-form text as JSON keys (reduces false positives).
-export const KNOWN_TOOLS = new Set([
-    'shell',
-    'exec_command',
-    'write_stdin',
-    'apply_patch',
-    'update_plan',
-    'view_image',
-    'list_mcp_resources',
-    'read_mcp_resource',
-    'list_mcp_resource_templates',
-    // 文件/目录类工具（CLI 侧已有约定；此处只做文本→tool_calls 收割）
-    'list_directory'
-]);
-const ALLOWED_KEYS = {
-    shell: new Set(['command', 'justification', 'timeout_ms', 'with_escalated_permissions', 'workdir']),
-    exec_command: new Set([
-        'cmd',
-        'command',
-        'workdir',
-        'justification',
-        'login',
-        'tty',
-        'shell',
-        'yield_time_ms',
-        'max_output_tokens',
-        'sandbox_permissions',
-        'timeout_ms'
-    ]),
-    write_stdin: new Set(['session_id', 'chars', 'text', 'yield_time_ms', 'max_output_tokens']),
-    // apply_patch supports both freeform patch text (patch/input/instructions/text) and structured payloads (file/changes)
-    apply_patch: new Set(['patch', 'input', 'instructions', 'text', 'file', 'changes']),
-    update_plan: new Set(['explanation', 'plan']),
-    view_image: new Set(['path']),
-    list_mcp_resources: new Set(['server', 'cursor', 'filter', 'root']),
-    read_mcp_resource: new Set(['server', 'uri', 'cursor']),
-    list_mcp_resource_templates: new Set(['server', 'cursor']),
-    list_directory: new Set(['path', 'recursive'])
-};
-export function genToolCallId() {
-    return `call_${Math.random().toString(36).slice(2, 10)}`;
-}
-export function normalizeKey(raw) {
-    try {
-        const t = String(raw || '').trim();
-        if (!t)
-            return '';
-        // Pick the last alphanumeric/underscore token (e.g., '空的\n<tool_call>command' -> 'command')
-        const m = t.match(/([A-Za-z_][A-Za-z0-9_]*)$/);
-        return m ? m[1] : t;
-    }
-    catch {
-        return String(raw || '');
-    }
-}
-export function readDefaultWorkdirFromEnv() {
-    try {
-        const env = String(process?.env?.ROUTECODEX_WORKDIR || '').trim() ||
-            String(process?.env?.RCC_WORKDIR || '').trim() ||
-            String(process?.env?.CLAUDE_WORKDIR || '').trim();
-        return env ? env : undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
-export function filterArgsForTool(name, args) {
-    try {
-        const nm = String(name || '').toLowerCase();
-        const allow = ALLOWED_KEYS[nm];
-        if (!allow) {
-            // Unknown tool; be conservative: keep only simple safe keys if present
-            const safe = new Set([
-                'cmd',
-                'command',
-                'workdir',
-                'patch',
-                'input',
-                'instructions',
-                'text',
-                'plan',
-                'explanation',
-                'path',
-                'server',
-                'uri',
-                'cursor',
-                'filter',
-                'root'
-            ]);
-            const out = {};
-            for (const [k, v] of Object.entries(args || {})) {
-                const key = normalizeKey(k);
-                if (safe.has(key))
-                    out[key] = v;
-            }
-            return out;
-        }
-        const out = {};
-        for (const [k, v] of Object.entries(args || {})) {
-            const key = normalizeKey(k);
-            if (allow.has(key))
-                out[key] = v;
-        }
-        return out;
-    }
-    catch {
-        return args;
-    }
-}
-export function tryParseJsonValue(text) {
-    try {
-        const trimmed = String(text || '').trim();
-        if (!trimmed)
-            return null;
-        if (!(trimmed.startsWith('{') || trimmed.startsWith('[') || trimmed.startsWith('"'))) {
-            return null;
-        }
-        return JSON.parse(trimmed);
-    }
-    catch {
-        return null;
-    }
-}
-export function tryParsePrimitiveValue(text) {
-    try {
-        const trimmed = String(text ?? '').trim();
-        if (trimmed.length === 0)
-            return '';
-        const parsed = tryParseJsonValue(trimmed);
-        if (parsed !== null)
-            return parsed;
-        if (trimmed === 'true' || trimmed === 'false')
-            return trimmed === 'true';
-        if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) {
-            const n = Number(trimmed);
-            if (Number.isFinite(n))
-                return n;
-        }
-        return trimmed;
-    }
-    catch {
-        return String(text ?? '');
-    }
-}
-export function coerceCommandValueToString(value) {
-    try {
-        if (value === null || value === undefined)
-            return '';
-        if (typeof value === 'string')
-            return value.trim();
-        if (Array.isArray(value)) {
-            return value.map((v) => String(v)).join(' ').trim();
-        }
-        if (typeof value === 'object') {
-            const rec = value;
-            const direct = typeof rec.cmd === 'string'
-                ? String(rec.cmd)
-                : typeof rec.command === 'string'
-                    ? String(rec.command)
-                    : '';
-            if (direct.trim().length)
-                return direct.trim();
-            try {
-                return JSON.stringify(value);
-            }
-            catch {
-                return String(value);
-            }
-        }
-        return String(value).trim();
-    }
-    catch {
-        return '';
-    }
-}

package/dist/conversion/shared/text-markup-normalizer/extractors-transcript.d.ts

@@ -1,5 +0,0 @@
-import { type ToolCallLite } from './extractors-shared.js';
-export declare function extractBareExecCommandFromText(text: string): ToolCallLite[] | null;
-export declare function extractExecuteBlocksFromText(text: string): ToolCallLite[] | null;
-export declare function extractExploredListDirectoryCallsFromText(text: string): ToolCallLite[] | null;
-export declare function extractQwenToolCallTokensFromText(text: string): ToolCallLite[] | null;

package/dist/conversion/shared/text-markup-normalizer/extractors-transcript.js

@@ -1,385 +0,0 @@
-import { filterArgsForTool, genToolCallId, readDefaultWorkdirFromEnv } from './extractors-shared.js';
-import { salvageToolArgsFromRawText, tryParseJsonWithModelRepairs } from './extractors-json.js';
-function tryParseJsonArrayOnLine(line) {
-    try {
-        const trimmed = String(line || '').trim();
-        if (!trimmed.startsWith('[') || !trimmed.endsWith(']'))
-            return null;
-        const parsed = JSON.parse(trimmed);
-        if (!Array.isArray(parsed))
-            return null;
-        const out = [];
-        for (const entry of parsed) {
-            if (entry == null)
-                continue;
-            out.push(String(entry));
-        }
-        return out.length ? out : null;
-    }
-    catch {
-        return null;
-    }
-}
-function stripTrailingXmlGarbage(line) {
-    try {
-        const idx = line.indexOf('<');
-        if (idx >= 0) {
-            return line.slice(0, idx).trim();
-        }
-        return line.trim();
-    }
-    catch {
-        return String(line || '').trim();
-    }
-}
-function normalizePossibleCommandLine(raw) {
-    try {
-        let line = String(raw || '');
-        line = line.replace(/^\s*(?:[•*+-]\s*)?/, '');
-        line = line.replace(/^\s*\$\s*/, '');
-        return stripTrailingXmlGarbage(line);
-    }
-    catch {
-        return '';
-    }
-}
-function looksLikeShellCommand(line) {
-    try {
-        const t = String(line || '').trim();
-        if (!t)
-            return false;
-        if (t.startsWith('```') || t.startsWith('>'))
-            return false;
-        if (t.startsWith('zsh:') || t.startsWith('bash:'))
-            return false;
-        // Conservative allowlist: commands commonly emitted by Codex/Claude in tool-mode.
-        return /^(?:rg|ls|cat|node|npm|pnpm|yarn|git|find|sed|head|tail|wc|mkdir|rm|cp|mv|pwd)\b/.test(t);
-    }
-    catch {
-        return false;
-    }
-}
-function looksLikeBrokenToolMarkup(text) {
-    try {
-        const t = String(text || '');
-        if (!t)
-            return false;
-        const ranCommandMatch = t.match(/^\s*(?:[•*+-]\s*)?Ran\s+(.+)$/im);
-        const hasRanShellCommand = !!ranCommandMatch &&
-            (() => {
-                const payload = String(ranCommandMatch[1] || '').trim();
-                if (!payload)
-                    return false;
-                const arr = tryParseJsonArrayOnLine(payload);
-                if (arr && arr.length) {
-                    return looksLikeShellCommand(arr.join(' '));
-                }
-                return looksLikeShellCommand(payload);
-            })();
-        return (t.includes('<exec_command') ||
-            t.includes('<invoke') ||
-            t.includes('<function_calls') ||
-            t.includes('<parameter') ||
-            t.includes('<tool_call') ||
-            t.includes('</tool_call') ||
-            t.includes('<arg_value') ||
-            t.includes('</arg_value') ||
-            t.includes('</func_call') ||
-            /\bRan\b\s+\[/.test(t) ||
-            hasRanShellCommand);
-    }
-    catch {
-        return false;
-    }
-}
-export function extractBareExecCommandFromText(text) {
-    try {
-        if (typeof text !== 'string' || !text)
-            return null;
-        // Only attempt this when we see clear evidence of a broken tool markup (avoid turning examples into tool calls).
-        if (!looksLikeBrokenToolMarkup(text))
-            return null;
-        const candidates = [];
-        const lines = text.split(/\r?\n/);
-        for (let i = 0; i < lines.length; i += 1) {
-            const raw = String(lines[i] || '');
-            const line = normalizePossibleCommandLine(raw);
-            if (!line)
-                continue;
-            // Pattern A: "Ran [\"ls\", \"-l\", ...]" or "Ran git status"
-            const ran = line.match(/^Ran\s+(.+)$/i);
-            if (ran && ran[1]) {
-                const payload = String(ran[1] || '').trim();
-                const arr = tryParseJsonArrayOnLine(payload);
-                if (arr && arr.length) {
-                    const cmd = arr.join(' ').trim();
-                    if (looksLikeShellCommand(cmd)) {
-                        candidates.push({ cmd });
-                        continue;
-                    }
-                }
-                if (looksLikeShellCommand(payload)) {
-                    candidates.push({ cmd: payload });
-                    continue;
-                }
-            }
-            // Pattern B: a single-line shell command (rg/ls/node/...)
-            if (looksLikeShellCommand(line)) {
-                candidates.push({ cmd: line.trim() });
-                continue;
-            }
-        }
-        if (!candidates.length)
-            return null;
-        const picked = candidates[candidates.length - 1];
-        const argsObj = { cmd: picked.cmd };
-        const envWorkdir = readDefaultWorkdirFromEnv();
-        if (envWorkdir) {
-            argsObj.workdir = envWorkdir;
-        }
-        const filtered = filterArgsForTool('exec_command', argsObj);
-        let argsStr = '{}';
-        try {
-            argsStr = JSON.stringify(filtered);
-        }
-        catch {
-            argsStr = '{}';
-        }
-        return [
-            {
-                id: genToolCallId(),
-                name: 'exec_command',
-                args: argsStr
-            }
-        ];
-    }
-    catch {
-        return null;
-    }
-}
-export function extractExecuteBlocksFromText(text) {
-    try {
-        if (typeof text !== 'string' || !text)
-            return null;
-        const re = /<function=execute>\s*<parameter=command>([\s\S]*?)<\/parameter>\s*<\/function>/gi;
-        const out = [];
-        let m;
-        while ((m = re.exec(text)) !== null) {
-            const commandRaw = (m[1] || '').trim();
-            if (!commandRaw)
-                continue;
-            let args = '{}';
-            try {
-                args = JSON.stringify({ command: commandRaw });
-            }
-            catch {
-                args = '{"command":""}';
-            }
-            out.push({ id: genToolCallId(), name: 'shell', args });
-        }
-        return out.length ? out : null;
-    }
-    catch {
-        return null;
-    }
-}
-function normalizeExploredListPath(raw) {
-    try {
-        let path = String(raw || '').trim();
-        // Drop common tree/marker prefixes from terminal transcript lines.
-        path = path.replace(/^[\s│]+/, '');
-        path = path.replace(/^[└├─]+\s*/, '');
-        // Strip wrapping quotes/backticks.
-        path = path.replace(/^([`"'])([\s\S]*?)\1$/, '$2').trim();
-        if (!path)
-            return '';
-        // Keep only first line to avoid swallowing multiline status output.
-        const firstLine = path.split(/\r?\n/)[0] || '';
-        path = firstLine.trim();
-        if (!path)
-            return '';
-        // Guard obvious non-path prose fragments.
-        if (/^(?:and|with|for|to)\b/i.test(path))
-            return '';
-        if (/[<>|;&]/.test(path))
-            return '';
-        return path;
-    }
-    catch {
-        return '';
-    }
-}
-// Harvest Codex-style transcript snippets:
-//   • Explored
-//     └ List xiaohongshu
-//       List app
-// into list_directory tool calls.
-export function extractExploredListDirectoryCallsFromText(text) {
-    try {
-        if (typeof text !== 'string' || !text)
-            return null;
-        const hasExploredMarker = /(?:^|\n)\s*(?:[•*+-]\s*)?Explored\b/i.test(text);
-        const hasTreeListMarker = /(?:^|\n)\s*[│\s]*[└├]\s*List\s+/im.test(text);
-        if (!hasExploredMarker && !hasTreeListMarker) {
-            return null;
-        }
-        const out = [];
-        const seen = new Set();
-        const lines = text.split(/\r?\n/);
-        for (const rawLine of lines) {
-            let line = String(rawLine || '').trim();
-            if (!line)
-                continue;
-            // Remove transcript bullets/tree prefixes.
-            line = line.replace(/^[•*+-]\s*/, '');
-            line = line.replace(/^[\s│]+/, '');
-            line = line.replace(/^[└├─]+\s*/, '');
-            const listMatch = /^List\s+([\s\S]+)$/i.exec(line);
-            if (!listMatch?.[1]) {
-                continue;
-            }
-            const path = normalizeExploredListPath(listMatch[1]);
-            if (!path) {
-                continue;
-            }
-            const dedupeKey = path.toLowerCase();
-            if (seen.has(dedupeKey)) {
-                continue;
-            }
-            seen.add(dedupeKey);
-            const filtered = filterArgsForTool('list_directory', { path, recursive: false });
-            let argsStr = '{}';
-            try {
-                argsStr = JSON.stringify(filtered);
-            }
-            catch {
-                argsStr = '{}';
-            }
-            out.push({
-                id: genToolCallId(),
-                name: 'list_directory',
-                args: argsStr
-            });
-        }
-        return out.length ? out : null;
-    }
-    catch {
-        return null;
-    }
-}
-// Extract Qwen-style tool call tokens that appear in plain text.
-export function extractQwenToolCallTokensFromText(text) {
-    try {
-        if (typeof text !== 'string' || !text)
-            return null;
-        // Upstreams (notably Qwen-family models behind OpenAI-compatible shells) may insert
-        // whitespace/newlines inside token markers, e.g. "<|  tool_call_begin|>".
-        if (!/<\|\s*tool_call_begin\s*\|>/i.test(text) && !/<\|\s*tool_call_argument_begin\s*\|>/i.test(text)) {
-            return null;
-        }
-        const out = [];
-        // tolerate whitespace/newlines and missing argument_end token
-        const re = /<\|\s*tool_call_begin\s*\|>\s*([A-Za-z0-9_.-]+)(?::(\d+))?\s*(?:<\|\s*tool_call_argument_begin\s*\|>\s*([\s\S]*?)\s*)?<\|\s*tool_call_end\s*\|>/gi;
-        let m;
-        while ((m = re.exec(text)) !== null) {
-            const rawName = String(m[1] || '').trim();
-            const numericId = typeof m[2] === 'string' && m[2].trim().length ? m[2].trim() : undefined;
-            const rawArgs = typeof m[3] === 'string' ? String(m[3]).trim() : '';
-            if (!rawName)
-                continue;
-            let name = rawName;
-            if (name.startsWith('functions.')) {
-                name = name.slice('functions.'.length);
-            }
-            const lname = name.toLowerCase();
-            if (!lname)
-                continue;
-            let parsedArgs = null;
-            if (!rawArgs) {
-                parsedArgs = {};
-            }
-            else {
-                parsedArgs = tryParseJsonWithModelRepairs(rawArgs);
-                if (parsedArgs === null) {
-                    parsedArgs = salvageToolArgsFromRawText(lname, rawArgs) ?? { _raw: rawArgs };
-                }
-            }
-            const argsObj = parsedArgs && typeof parsedArgs === 'object' && !Array.isArray(parsedArgs)
-                ? parsedArgs
-                : { value: parsedArgs };
-            // Alias normalization (models often use "command" instead of "cmd", etc.)
-            if (lname === 'exec_command') {
-                const cmd = typeof argsObj.cmd === 'string' ? String(argsObj.cmd) : '';
-                const command = typeof argsObj.command === 'string' ? String(argsObj.command) : '';
-                if (!cmd && command) {
-                    argsObj.cmd = command;
-                }
-                const cwd = typeof argsObj.cwd === 'string' ? String(argsObj.cwd) : '';
-                const workdir = typeof argsObj.workdir === 'string' ? String(argsObj.workdir) : '';
-                if (!workdir && cwd) {
-                    argsObj.workdir = cwd;
-                }
-            }
-            if (lname === 'write_stdin') {
-                const chars = typeof argsObj.chars === 'string' ? String(argsObj.chars) : '';
-                const textVal = typeof argsObj.text === 'string' ? String(argsObj.text) : '';
-                const input = typeof argsObj.input === 'string' ? String(argsObj.input) : '';
-                if (!chars && (textVal || input)) {
-                    argsObj.chars = textVal || input;
-                }
-                const sid = argsObj.session_id;
-                const sidAlt = argsObj.sessionId;
-                if ((sid === undefined || sid === null || String(sid).trim() === '') && sidAlt !== undefined && sidAlt !== null) {
-                    argsObj.session_id = sidAlt;
-                }
-            }
-            if (lname === 'apply_patch') {
-                const patch = typeof argsObj.patch === 'string' ? String(argsObj.patch) : '';
-                const textPatch = typeof argsObj.text === 'string' ? String(argsObj.text) : '';
-                const inputPatch = typeof argsObj.input === 'string' ? String(argsObj.input) : '';
-                const instructions = typeof argsObj.instructions === 'string' ? String(argsObj.instructions) : '';
-                if (!patch && (textPatch || inputPatch || instructions)) {
-                    argsObj.patch = textPatch || inputPatch || instructions;
-                }
-            }
-            const filtered = filterArgsForTool(lname, argsObj);
-            // Basic guards for known tools
-            if (lname === 'exec_command') {
-                const cmd = typeof filtered?.cmd === 'string' ? String(filtered.cmd).trim() : '';
-                if (!cmd)
-                    continue;
-            }
-            if (lname === 'write_stdin') {
-                // Allow empty args: our servertool schema may inject session_id automatically when a session is active.
-            }
-            if (lname === 'apply_patch') {
-                const patchText = typeof filtered?.patch === 'string' ? String(filtered.patch).trim() : '';
-                if (!patchText)
-                    continue;
-            }
-            let argsStr = '{}';
-            try {
-                argsStr = JSON.stringify(filtered);
-            }
-            catch {
-                argsStr = '{}';
-            }
-            const stableId = (() => {
-                if (numericId && /^[0-9]{1,10}$/.test(numericId)) {
-                    const namePart = lname.replace(/[^A-Za-z0-9_-]/g, '_');
-                    const id = `call_${namePart}_${numericId}`
-                        .replace(/_{2,}/g, '_')
-                        .replace(/^_+/, '')
-                        .replace(/_+$/, '');
-                    return id || undefined;
-                }
-                return undefined;
-            })();
-            out.push({ id: stableId ?? genToolCallId(), name: lname, args: argsStr });
-        }
-        return out.length ? out : null;
-    }
-    catch {
-        return null;
-    }
-}

package/dist/conversion/shared/text-markup-normalizer/extractors-xml.d.ts

@@ -1,10 +0,0 @@
-import { type ToolCallLite } from './extractors-shared.js';
-export declare function extractToolNamespaceXmlBlocksFromText(text: string): ToolCallLite[] | null;
-export declare function extractParameterXmlToolsFromText(text: string): ToolCallLite[] | null;
-export declare function extractInvokeToolsFromText(text: string): ToolCallLite[] | null;
-export declare function extractXMLToolCallsFromText(text: string): ToolCallLite[] | null;
-/**
- * 提取简单 XML 形式的工具调用块，例如：
- *   <list_directory><path>/path</path><recursive>false</recursive></list_directory>
- */
-export declare function extractSimpleXmlToolsFromText(text: string): ToolCallLite[] | null;