@jsonstudio/llms 0.6.1892 → 0.6.2172

package/dist/conversion/shared/text-markup-normalizer/extractors-json.js

@@ -0,0 +1,637 @@
+import { KNOWN_TOOLS, coerceCommandValueToString, filterArgsForTool, genToolCallId } from './extractors-shared.js';
+const DEFAULT_JSON_TOOL_NAME_ALIASES = {
+    shell_command: 'exec_command',
+    shell: 'exec_command',
+    bash: 'exec_command',
+    terminal: 'exec_command'
+};
+const DEFAULT_JSON_TOOL_ARGUMENT_ALIASES = {
+    exec_command: {
+        cmd: ['cmd', 'command', 'script', 'toon', 'input.command', 'input.cmd', 'input.script', 'input.toon'],
+        command: ['cmd', 'command', 'script', 'toon', 'input.command', 'input.cmd', 'input.script', 'input.toon'],
+        workdir: ['workdir', 'cwd', 'input.workdir', 'input.cwd']
+    },
+    write_stdin: {
+        chars: ['chars', 'text', 'input', 'data', 'input.chars', 'input.text', 'input.input', 'input.data'],
+        session_id: ['session_id', 'sessionId', 'input.session_id', 'input.sessionId']
+    },
+    apply_patch: {
+        patch: ['patch', 'text', 'input', 'instructions', 'input.patch', 'input.text', 'input.input', 'input.instructions']
+    }
+};
+function escapeControlCharsInsideJsonStrings(raw) {
+    try {
+        if (typeof raw !== 'string' || !raw)
+            return null;
+        let out = '';
+        let changed = false;
+        let inString = false;
+        let escaped = false;
+        for (let i = 0; i < raw.length; i++) {
+            const ch = raw[i];
+            if (inString) {
+                if (!escaped && ch === '"') {
+                    inString = false;
+                    out += ch;
+                    continue;
+                }
+                if (!escaped && ch === '\\') {
+                    escaped = true;
+                    out += ch;
+                    continue;
+                }
+                // JSON disallows unescaped control chars inside strings; models sometimes emit raw newlines/tabs.
+                if (!escaped) {
+                    if (ch === '\n') {
+                        out += '\\n';
+                        changed = true;
+                        continue;
+                    }
+                    if (ch === '\r') {
+                        if (raw[i + 1] === '\n') {
+                            i++;
+                        }
+                        out += '\\n';
+                        changed = true;
+                        continue;
+                    }
+                    if (ch === '\t') {
+                        out += '\\t';
+                        changed = true;
+                        continue;
+                    }
+                    const code = ch.charCodeAt(0);
+                    if (code >= 0 && code < 0x20) {
+                        out += `\\u${code.toString(16).padStart(4, '0')}`;
+                        changed = true;
+                        continue;
+                    }
+                }
+                out += ch;
+                escaped = false;
+                continue;
+            }
+            if (ch === '"') {
+                inString = true;
+                out += ch;
+                continue;
+            }
+            out += ch;
+        }
+        return changed ? out : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function tryParseJsonWithModelRepairs(raw) {
+    try {
+        if (typeof raw !== 'string')
+            return null;
+        const text = raw.trim();
+        if (!text)
+            return null;
+        try {
+            return JSON.parse(text);
+        }
+        catch {
+            const repaired = escapeControlCharsInsideJsonStrings(text);
+            if (!repaired)
+                return null;
+            try {
+                return JSON.parse(repaired);
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    catch {
+        return null;
+    }
+}
+export function salvageToolArgsFromRawText(toolName, rawArgs) {
+    try {
+        const lname = String(toolName || '').toLowerCase();
+        const text = String(rawArgs || '');
+        const out = {};
+        const pickString = (re) => {
+            const m = text.match(re);
+            if (!m)
+                return undefined;
+            return String(m[1] ?? '');
+        };
+        const pickNumber = (re) => {
+            const m = text.match(re);
+            if (!m)
+                return undefined;
+            const n = Number.parseInt(String(m[1] ?? ''), 10);
+            return Number.isFinite(n) ? n : undefined;
+        };
+        if (lname === 'exec_command') {
+            const cmd = pickString(/"(?:cmd|command)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
+            const workdir = pickString(/"(?:workdir|cwd)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
+            const timeout = pickNumber(/"(?:timeout_ms|timeout)"\s*:\s*(\d+)\s*(?:,|})/i);
+            if (cmd !== undefined)
+                out.cmd = cmd;
+            if (cmd !== undefined)
+                out.command = cmd;
+            if (workdir !== undefined)
+                out.workdir = workdir;
+            if (timeout !== undefined)
+                out.timeout_ms = timeout;
+            return Object.keys(out).length ? out : null;
+        }
+        if (lname === 'write_stdin') {
+            const sessionId = pickNumber(/"(?:session_id|sessionId)"\s*:\s*(\d+)\s*(?:,|})/i);
+            const chars = pickString(/"(?:chars|text|input|data)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
+            if (sessionId !== undefined)
+                out.session_id = sessionId;
+            if (chars !== undefined)
+                out.chars = chars;
+            return Object.keys(out).length ? out : null;
+        }
+        if (lname === 'apply_patch') {
+            const patch = pickString(/"(?:patch|text|input|instructions)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
+            if (patch !== undefined)
+                out.patch = patch;
+            return Object.keys(out).length ? out : null;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+function extractToolCallEntriesFromUnknown(value) {
+    if (Array.isArray(value)) {
+        return value;
+    }
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return [];
+    }
+    const rec = value;
+    if (Array.isArray(rec.tool_calls)) {
+        return rec.tool_calls;
+    }
+    if (typeof rec.tool_calls === 'string') {
+        const nested = tryParseJsonWithModelRepairs(rec.tool_calls);
+        return extractToolCallEntriesFromUnknown(nested);
+    }
+    return [];
+}
+function resolveJsonToolNameAliases(options) {
+    const merged = { ...DEFAULT_JSON_TOOL_NAME_ALIASES };
+    const raw = options?.jsonToolRepair?.toolNameAliases;
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        return merged;
+    }
+    for (const [key, value] of Object.entries(raw)) {
+        const src = String(key || '').trim().toLowerCase();
+        const dst = String(value || '').trim().toLowerCase();
+        if (!src || !dst)
+            continue;
+        merged[src] = dst;
+    }
+    return merged;
+}
+function resolveJsonToolArgAliases(toolName, options) {
+    const lname = String(toolName || '').trim().toLowerCase();
+    const merged = {};
+    const base = DEFAULT_JSON_TOOL_ARGUMENT_ALIASES[lname];
+    if (base && typeof base === 'object' && !Array.isArray(base)) {
+        for (const [target, paths] of Object.entries(base)) {
+            merged[target] = Array.isArray(paths) ? [...paths] : [];
+        }
+    }
+    const customRoot = options?.jsonToolRepair?.argumentAliases;
+    if (!customRoot || typeof customRoot !== 'object' || Array.isArray(customRoot)) {
+        return merged;
+    }
+    const custom = customRoot[lname];
+    if (!custom || typeof custom !== 'object' || Array.isArray(custom)) {
+        return merged;
+    }
+    for (const [target, paths] of Object.entries(custom)) {
+        if (!Array.isArray(paths))
+            continue;
+        merged[target] = paths.map((item) => String(item || '').trim()).filter((item) => item.length > 0);
+    }
+    return merged;
+}
+function hasNonEmptyAliasValue(value) {
+    if (value === undefined || value === null) {
+        return false;
+    }
+    if (typeof value === 'string') {
+        return value.trim().length > 0;
+    }
+    return true;
+}
+function readAliasPathValue(source, path) {
+    const trimmed = String(path || '').trim();
+    if (!trimmed) {
+        return undefined;
+    }
+    if (!trimmed.includes('.')) {
+        return source[trimmed];
+    }
+    const parts = trimmed.split('.').map((part) => part.trim()).filter((part) => part.length > 0);
+    if (!parts.length) {
+        return undefined;
+    }
+    let cursor = source;
+    for (const part of parts) {
+        if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) {
+            return undefined;
+        }
+        cursor = cursor[part];
+    }
+    return cursor;
+}
+function applyJsonToolArgumentAliases(toolName, argsObj, options) {
+    const mapping = resolveJsonToolArgAliases(toolName, options);
+    for (const [target, candidates] of Object.entries(mapping)) {
+        if (!Array.isArray(candidates) || candidates.length === 0) {
+            continue;
+        }
+        const current = argsObj[target];
+        if (hasNonEmptyAliasValue(current)) {
+            continue;
+        }
+        for (const candidate of candidates) {
+            const picked = readAliasPathValue(argsObj, candidate);
+            if (!hasNonEmptyAliasValue(picked)) {
+                continue;
+            }
+            argsObj[target] = picked;
+            break;
+        }
+    }
+}
+function canonicalizeJsonToolName(value, options) {
+    if (typeof value !== 'string') {
+        return null;
+    }
+    let name = value.trim();
+    if (!name) {
+        return null;
+    }
+    if (name.startsWith('functions.')) {
+        name = name.slice('functions.'.length);
+    }
+    const lowered = name.toLowerCase();
+    const aliasMap = resolveJsonToolNameAliases(options);
+    const canonical = aliasMap[lowered] ?? lowered;
+    return KNOWN_TOOLS.has(canonical) ? canonical : null;
+}
+function coerceToolCallArgsObject(raw) {
+    if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
+        return { ...raw };
+    }
+    if (typeof raw === 'string') {
+        const parsed = tryParseJsonWithModelRepairs(raw);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            return { ...parsed };
+        }
+        return {};
+    }
+    if (Array.isArray(raw)) {
+        const command = raw.map((entry) => String(entry ?? '')).join(' ').trim();
+        return command ? { command } : {};
+    }
+    return {};
+}
+function normalizeJsonToolArgs(toolName, rawArgs, options) {
+    const lname = String(toolName || '').toLowerCase();
+    const argsObj = coerceToolCallArgsObject(rawArgs);
+    applyJsonToolArgumentAliases(lname, argsObj, options);
+    if (lname === 'exec_command') {
+        const cmd = coerceCommandValueToString(argsObj.cmd);
+        if (!cmd) {
+            return null;
+        }
+        argsObj.cmd = cmd;
+        argsObj.command = cmd;
+        if (typeof argsObj.workdir === 'string') {
+            argsObj.workdir = String(argsObj.workdir).trim();
+        }
+    }
+    if (lname === 'write_stdin') {
+        if (argsObj.chars !== undefined && argsObj.chars !== null) {
+            argsObj.chars = String(argsObj.chars);
+        }
+        if (argsObj.session_id !== undefined && argsObj.session_id !== null) {
+            const sid = Number.parseInt(String(argsObj.session_id), 10);
+            if (Number.isFinite(sid)) {
+                argsObj.session_id = sid;
+            }
+        }
+    }
+    if (lname === 'apply_patch') {
+        if (typeof argsObj.patch === 'string') {
+            argsObj.patch = String(argsObj.patch);
+        }
+    }
+    const filtered = filterArgsForTool(lname, argsObj);
+    if (lname === 'exec_command') {
+        const cmd = typeof filtered?.cmd === 'string' ? String(filtered.cmd).trim() : '';
+        if (!cmd) {
+            return null;
+        }
+    }
+    if (lname === 'write_stdin') {
+        const sid = filtered?.session_id;
+        if (sid === undefined || sid === null || String(sid).trim().length === 0) {
+            return null;
+        }
+    }
+    if (lname === 'apply_patch') {
+        const patch = typeof filtered?.patch === 'string' ? String(filtered.patch).trim() : '';
+        if (!patch) {
+            return null;
+        }
+    }
+    return filtered;
+}
+function normalizeJsonToolCallEntry(entry, options) {
+    if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
+        return null;
+    }
+    const rec = entry;
+    const fn = rec.function && typeof rec.function === 'object' && !Array.isArray(rec.function)
+        ? rec.function
+        : undefined;
+    const name = canonicalizeJsonToolName(rec.name ?? fn?.name, options);
+    if (!name) {
+        return null;
+    }
+    const argsSource = rec.input !== undefined
+        ? rec.input
+        : rec.arguments !== undefined
+            ? rec.arguments
+            : fn?.arguments !== undefined
+                ? fn.arguments
+                : fn?.input;
+    const argsObj = normalizeJsonToolArgs(name, argsSource, options);
+    if (!argsObj) {
+        return null;
+    }
+    let args = '{}';
+    try {
+        args = JSON.stringify(argsObj);
+    }
+    catch {
+        args = '{}';
+    }
+    const id = typeof rec.call_id === 'string' && rec.call_id.trim().length
+        ? rec.call_id.trim()
+        : typeof rec.id === 'string' && rec.id.trim().length
+            ? rec.id.trim()
+            : genToolCallId();
+    return { id, name, args };
+}
+function extractBalancedJsonObjectAt(text, startIndex) {
+    if (startIndex < 0 || startIndex >= text.length || text[startIndex] !== '{') {
+        return null;
+    }
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = startIndex; i < text.length; i += 1) {
+        const ch = text[i] ?? '';
+        if (inString) {
+            if (escaped) {
+                escaped = false;
+                continue;
+            }
+            if (ch === '\\') {
+                escaped = true;
+                continue;
+            }
+            if (ch === '"') {
+                inString = false;
+            }
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+            continue;
+        }
+        if (ch === '{') {
+            depth += 1;
+            continue;
+        }
+        if (ch === '}') {
+            depth -= 1;
+            if (depth === 0) {
+                return text.slice(startIndex, i + 1);
+            }
+        }
+    }
+    return null;
+}
+function collectEmbeddedToolCallObjects(text) {
+    const out = [];
+    const seen = new Set();
+    const quoteRanges = [];
+    {
+        const lower = text.toLowerCase();
+        let cursor = 0;
+        while (cursor < lower.length) {
+            const open = lower.indexOf('<quote>', cursor);
+            if (open < 0)
+                break;
+            const close = lower.indexOf('</quote>', open + '<quote>'.length);
+            if (close < 0)
+                break;
+            quoteRanges.push({ start: open, end: close + '</quote>'.length });
+            cursor = close + '</quote>'.length;
+        }
+    }
+    const isInsideQuoteRange = (index) => quoteRanges.some((range) => index >= range.start && index < range.end);
+    for (let i = 0; i < text.length; i += 1) {
+        if (text[i] !== '{') {
+            continue;
+        }
+        if (isInsideQuoteRange(i)) {
+            continue;
+        }
+        const objectText = extractBalancedJsonObjectAt(text, i);
+        if (!objectText) {
+            continue;
+        }
+        const trimmed = objectText.trim();
+        if (trimmed.length > 0 &&
+            /["']tool_calls["']\s*:/.test(trimmed) &&
+            !seen.has(trimmed)) {
+            seen.add(trimmed);
+            out.push(trimmed);
+        }
+    }
+    return out;
+}
+function decodeJsonEscapes(raw) {
+    return String(raw || '')
+        .replace(/\\r\\n/g, '\n')
+        .replace(/\\n/g, '\n')
+        .replace(/\\r/g, '\n')
+        .replace(/\\t/g, '\t')
+        .replace(/\\"/g, '"');
+}
+function extractPossiblyBrokenQuotedValue(text, keys) {
+    for (const key of keys) {
+        const re = new RegExp(`"${key}"\\s*:\\s*"`, 'ig');
+        let match;
+        while ((match = re.exec(text)) !== null) {
+            const start = match.index + match[0].length;
+            let escaped = false;
+            for (let i = start; i < text.length; i += 1) {
+                const ch = text[i] ?? '';
+                if (escaped) {
+                    escaped = false;
+                    continue;
+                }
+                if (ch === '\\') {
+                    escaped = true;
+                    continue;
+                }
+                if (ch === '"') {
+                    return decodeJsonEscapes(text.slice(start, i));
+                }
+            }
+            return decodeJsonEscapes(text.slice(start));
+        }
+    }
+    return null;
+}
+function salvageToolCallFromBrokenJsonText(candidate, options) {
+    const rawName = candidate.match(/"name"\s*:\s*"([^"]+)"/i)?.[1];
+    const canonicalName = canonicalizeJsonToolName(rawName, options);
+    if (!canonicalName) {
+        return null;
+    }
+    let args = null;
+    if (canonicalName === 'exec_command') {
+        const cmd = extractPossiblyBrokenQuotedValue(candidate, ['cmd', 'command', 'script', 'toon']);
+        if (cmd && cmd.trim().length) {
+            args = { cmd, command: cmd };
+        }
+    }
+    if (!args) {
+        args = salvageToolArgsFromRawText(canonicalName, candidate);
+    }
+    if (!args) {
+        return null;
+    }
+    const normalizedArgs = normalizeJsonToolArgs(canonicalName, args, options);
+    if (!normalizedArgs) {
+        return null;
+    }
+    return {
+        id: genToolCallId(),
+        name: canonicalName,
+        args: JSON.stringify(normalizedArgs)
+    };
+}
+export function extractJsonToolCallsFromText(text, options) {
+    try {
+        if (typeof text !== 'string' || !text.trim()) {
+            return null;
+        }
+        const candidates = [];
+        const seenCandidates = new Set();
+        const addCandidate = (raw) => {
+            const trimmed = String(raw || '').trim();
+            if (!trimmed || seenCandidates.has(trimmed)) {
+                return;
+            }
+            seenCandidates.add(trimmed);
+            candidates.push(trimmed);
+            const decoded = decodeJsonEscapes(trimmed).trim();
+            if (!decoded || decoded === trimmed || seenCandidates.has(decoded)) {
+                return;
+            }
+            seenCandidates.add(decoded);
+            candidates.push(decoded);
+        };
+        addCandidate(text);
+        const fenceRe = /```(?:json|tool_call|tool_calls|function_call)?\s*([\s\S]*?)\s*```/gi;
+        let fm;
+        while ((fm = fenceRe.exec(text)) !== null) {
+            const body = String(fm[1] || '').trim();
+            if (body) {
+                addCandidate(body);
+            }
+        }
+        for (const embedded of collectEmbeddedToolCallObjects(text)) {
+            addCandidate(embedded);
+        }
+        const decodedText = decodeJsonEscapes(text);
+        if (decodedText && decodedText !== text) {
+            for (const embedded of collectEmbeddedToolCallObjects(decodedText)) {
+                addCandidate(embedded);
+            }
+        }
+        const out = [];
+        const seen = new Set();
+        const pushEntry = (rawEntry) => {
+            const normalized = normalizeJsonToolCallEntry(rawEntry, options);
+            if (!normalized) {
+                return;
+            }
+            const key = `${normalized.name}:${normalized.args}`;
+            if (seen.has(key)) {
+                return;
+            }
+            seen.add(key);
+            out.push(normalized);
+        };
+        for (const candidate of candidates) {
+            const parsed = tryParseJsonWithModelRepairs(candidate);
+            if (parsed !== null) {
+                for (const entry of extractToolCallEntriesFromUnknown(parsed)) {
+                    pushEntry(entry);
+                }
+            }
+            if (out.length > 0) {
+                continue;
+            }
+            const standaloneWrapper = candidate.match(/^\s*\{\s*["']tool_calls["']\s*:\s*(\[[\s\S]*\])\s*\}\s*$/);
+            if (standaloneWrapper?.[1]) {
+                const wrapped = `{"tool_calls":${standaloneWrapper[1]}}`;
+                const parsedWrapped = tryParseJsonWithModelRepairs(wrapped);
+                if (parsedWrapped !== null) {
+                    for (const entry of extractToolCallEntriesFromUnknown(parsedWrapped)) {
+                        pushEntry(entry);
+                    }
+                }
+            }
+        }
+        // Last-resort salvage should run only when normal JSON extraction found nothing.
+        if (out.length === 0) {
+            for (const candidate of candidates) {
+                const lower = candidate.toLowerCase();
+                if (lower.includes('<quote>') && lower.includes('</quote>')) {
+                    continue;
+                }
+                const salvaged = salvageToolCallFromBrokenJsonText(candidate, options);
+                if (!salvaged) {
+                    continue;
+                }
+                pushEntry({
+                    id: salvaged.id,
+                    name: salvaged.name,
+                    function: {
+                        name: salvaged.name,
+                        arguments: salvaged.args
+                    }
+                });
+            }
+        }
+        return out.length ? out : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/conversion/shared/text-markup-normalizer/extractors-shared.d.ts

@@ -0,0 +1,21 @@
+export declare const KNOWN_TOOLS: Set<string>;
+export type JsonToolArgumentAliasMap = Record<string, string[]>;
+export type JsonToolRepairConfig = {
+    toolNameAliases?: Record<string, string>;
+    argumentAliases?: Record<string, JsonToolArgumentAliasMap>;
+};
+export type TextMarkupNormalizeOptions = {
+    jsonToolRepair?: JsonToolRepairConfig;
+};
+export type ToolCallLite = {
+    id?: string;
+    name: string;
+    args: string;
+};
+export declare function genToolCallId(): string;
+export declare function normalizeKey(raw: string): string;
+export declare function readDefaultWorkdirFromEnv(): string | undefined;
+export declare function filterArgsForTool(name: string, args: Record<string, unknown>): Record<string, unknown>;
+export declare function tryParseJsonValue(text: string): unknown | null;
+export declare function tryParsePrimitiveValue(text: string): unknown;
+export declare function coerceCommandValueToString(value: unknown): string;