@jsonstudio/llms 0.6.2172 → 0.6.3214

package/dist/conversion/shared/text-markup-normalizer/extractors-json.js

@@ -1,637 +0,0 @@
-import { KNOWN_TOOLS, coerceCommandValueToString, filterArgsForTool, genToolCallId } from './extractors-shared.js';
-const DEFAULT_JSON_TOOL_NAME_ALIASES = {
-    shell_command: 'exec_command',
-    shell: 'exec_command',
-    bash: 'exec_command',
-    terminal: 'exec_command'
-};
-const DEFAULT_JSON_TOOL_ARGUMENT_ALIASES = {
-    exec_command: {
-        cmd: ['cmd', 'command', 'script', 'toon', 'input.command', 'input.cmd', 'input.script', 'input.toon'],
-        command: ['cmd', 'command', 'script', 'toon', 'input.command', 'input.cmd', 'input.script', 'input.toon'],
-        workdir: ['workdir', 'cwd', 'input.workdir', 'input.cwd']
-    },
-    write_stdin: {
-        chars: ['chars', 'text', 'input', 'data', 'input.chars', 'input.text', 'input.input', 'input.data'],
-        session_id: ['session_id', 'sessionId', 'input.session_id', 'input.sessionId']
-    },
-    apply_patch: {
-        patch: ['patch', 'text', 'input', 'instructions', 'input.patch', 'input.text', 'input.input', 'input.instructions']
-    }
-};
-function escapeControlCharsInsideJsonStrings(raw) {
-    try {
-        if (typeof raw !== 'string' || !raw)
-            return null;
-        let out = '';
-        let changed = false;
-        let inString = false;
-        let escaped = false;
-        for (let i = 0; i < raw.length; i++) {
-            const ch = raw[i];
-            if (inString) {
-                if (!escaped && ch === '"') {
-                    inString = false;
-                    out += ch;
-                    continue;
-                }
-                if (!escaped && ch === '\\') {
-                    escaped = true;
-                    out += ch;
-                    continue;
-                }
-                // JSON disallows unescaped control chars inside strings; models sometimes emit raw newlines/tabs.
-                if (!escaped) {
-                    if (ch === '\n') {
-                        out += '\\n';
-                        changed = true;
-                        continue;
-                    }
-                    if (ch === '\r') {
-                        if (raw[i + 1] === '\n') {
-                            i++;
-                        }
-                        out += '\\n';
-                        changed = true;
-                        continue;
-                    }
-                    if (ch === '\t') {
-                        out += '\\t';
-                        changed = true;
-                        continue;
-                    }
-                    const code = ch.charCodeAt(0);
-                    if (code >= 0 && code < 0x20) {
-                        out += `\\u${code.toString(16).padStart(4, '0')}`;
-                        changed = true;
-                        continue;
-                    }
-                }
-                out += ch;
-                escaped = false;
-                continue;
-            }
-            if (ch === '"') {
-                inString = true;
-                out += ch;
-                continue;
-            }
-            out += ch;
-        }
-        return changed ? out : null;
-    }
-    catch {
-        return null;
-    }
-}
-export function tryParseJsonWithModelRepairs(raw) {
-    try {
-        if (typeof raw !== 'string')
-            return null;
-        const text = raw.trim();
-        if (!text)
-            return null;
-        try {
-            return JSON.parse(text);
-        }
-        catch {
-            const repaired = escapeControlCharsInsideJsonStrings(text);
-            if (!repaired)
-                return null;
-            try {
-                return JSON.parse(repaired);
-            }
-            catch {
-                return null;
-            }
-        }
-    }
-    catch {
-        return null;
-    }
-}
-export function salvageToolArgsFromRawText(toolName, rawArgs) {
-    try {
-        const lname = String(toolName || '').toLowerCase();
-        const text = String(rawArgs || '');
-        const out = {};
-        const pickString = (re) => {
-            const m = text.match(re);
-            if (!m)
-                return undefined;
-            return String(m[1] ?? '');
-        };
-        const pickNumber = (re) => {
-            const m = text.match(re);
-            if (!m)
-                return undefined;
-            const n = Number.parseInt(String(m[1] ?? ''), 10);
-            return Number.isFinite(n) ? n : undefined;
-        };
-        if (lname === 'exec_command') {
-            const cmd = pickString(/"(?:cmd|command)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
-            const workdir = pickString(/"(?:workdir|cwd)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
-            const timeout = pickNumber(/"(?:timeout_ms|timeout)"\s*:\s*(\d+)\s*(?:,|})/i);
-            if (cmd !== undefined)
-                out.cmd = cmd;
-            if (cmd !== undefined)
-                out.command = cmd;
-            if (workdir !== undefined)
-                out.workdir = workdir;
-            if (timeout !== undefined)
-                out.timeout_ms = timeout;
-            return Object.keys(out).length ? out : null;
-        }
-        if (lname === 'write_stdin') {
-            const sessionId = pickNumber(/"(?:session_id|sessionId)"\s*:\s*(\d+)\s*(?:,|})/i);
-            const chars = pickString(/"(?:chars|text|input|data)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
-            if (sessionId !== undefined)
-                out.session_id = sessionId;
-            if (chars !== undefined)
-                out.chars = chars;
-            return Object.keys(out).length ? out : null;
-        }
-        if (lname === 'apply_patch') {
-            const patch = pickString(/"(?:patch|text|input|instructions)"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i);
-            if (patch !== undefined)
-                out.patch = patch;
-            return Object.keys(out).length ? out : null;
-        }
-        return null;
-    }
-    catch {
-        return null;
-    }
-}
-function extractToolCallEntriesFromUnknown(value) {
-    if (Array.isArray(value)) {
-        return value;
-    }
-    if (!value || typeof value !== 'object' || Array.isArray(value)) {
-        return [];
-    }
-    const rec = value;
-    if (Array.isArray(rec.tool_calls)) {
-        return rec.tool_calls;
-    }
-    if (typeof rec.tool_calls === 'string') {
-        const nested = tryParseJsonWithModelRepairs(rec.tool_calls);
-        return extractToolCallEntriesFromUnknown(nested);
-    }
-    return [];
-}
-function resolveJsonToolNameAliases(options) {
-    const merged = { ...DEFAULT_JSON_TOOL_NAME_ALIASES };
-    const raw = options?.jsonToolRepair?.toolNameAliases;
-    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
-        return merged;
-    }
-    for (const [key, value] of Object.entries(raw)) {
-        const src = String(key || '').trim().toLowerCase();
-        const dst = String(value || '').trim().toLowerCase();
-        if (!src || !dst)
-            continue;
-        merged[src] = dst;
-    }
-    return merged;
-}
-function resolveJsonToolArgAliases(toolName, options) {
-    const lname = String(toolName || '').trim().toLowerCase();
-    const merged = {};
-    const base = DEFAULT_JSON_TOOL_ARGUMENT_ALIASES[lname];
-    if (base && typeof base === 'object' && !Array.isArray(base)) {
-        for (const [target, paths] of Object.entries(base)) {
-            merged[target] = Array.isArray(paths) ? [...paths] : [];
-        }
-    }
-    const customRoot = options?.jsonToolRepair?.argumentAliases;
-    if (!customRoot || typeof customRoot !== 'object' || Array.isArray(customRoot)) {
-        return merged;
-    }
-    const custom = customRoot[lname];
-    if (!custom || typeof custom !== 'object' || Array.isArray(custom)) {
-        return merged;
-    }
-    for (const [target, paths] of Object.entries(custom)) {
-        if (!Array.isArray(paths))
-            continue;
-        merged[target] = paths.map((item) => String(item || '').trim()).filter((item) => item.length > 0);
-    }
-    return merged;
-}
-function hasNonEmptyAliasValue(value) {
-    if (value === undefined || value === null) {
-        return false;
-    }
-    if (typeof value === 'string') {
-        return value.trim().length > 0;
-    }
-    return true;
-}
-function readAliasPathValue(source, path) {
-    const trimmed = String(path || '').trim();
-    if (!trimmed) {
-        return undefined;
-    }
-    if (!trimmed.includes('.')) {
-        return source[trimmed];
-    }
-    const parts = trimmed.split('.').map((part) => part.trim()).filter((part) => part.length > 0);
-    if (!parts.length) {
-        return undefined;
-    }
-    let cursor = source;
-    for (const part of parts) {
-        if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) {
-            return undefined;
-        }
-        cursor = cursor[part];
-    }
-    return cursor;
-}
-function applyJsonToolArgumentAliases(toolName, argsObj, options) {
-    const mapping = resolveJsonToolArgAliases(toolName, options);
-    for (const [target, candidates] of Object.entries(mapping)) {
-        if (!Array.isArray(candidates) || candidates.length === 0) {
-            continue;
-        }
-        const current = argsObj[target];
-        if (hasNonEmptyAliasValue(current)) {
-            continue;
-        }
-        for (const candidate of candidates) {
-            const picked = readAliasPathValue(argsObj, candidate);
-            if (!hasNonEmptyAliasValue(picked)) {
-                continue;
-            }
-            argsObj[target] = picked;
-            break;
-        }
-    }
-}
-function canonicalizeJsonToolName(value, options) {
-    if (typeof value !== 'string') {
-        return null;
-    }
-    let name = value.trim();
-    if (!name) {
-        return null;
-    }
-    if (name.startsWith('functions.')) {
-        name = name.slice('functions.'.length);
-    }
-    const lowered = name.toLowerCase();
-    const aliasMap = resolveJsonToolNameAliases(options);
-    const canonical = aliasMap[lowered] ?? lowered;
-    return KNOWN_TOOLS.has(canonical) ? canonical : null;
-}
-function coerceToolCallArgsObject(raw) {
-    if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
-        return { ...raw };
-    }
-    if (typeof raw === 'string') {
-        const parsed = tryParseJsonWithModelRepairs(raw);
-        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
-            return { ...parsed };
-        }
-        return {};
-    }
-    if (Array.isArray(raw)) {
-        const command = raw.map((entry) => String(entry ?? '')).join(' ').trim();
-        return command ? { command } : {};
-    }
-    return {};
-}
-function normalizeJsonToolArgs(toolName, rawArgs, options) {
-    const lname = String(toolName || '').toLowerCase();
-    const argsObj = coerceToolCallArgsObject(rawArgs);
-    applyJsonToolArgumentAliases(lname, argsObj, options);
-    if (lname === 'exec_command') {
-        const cmd = coerceCommandValueToString(argsObj.cmd);
-        if (!cmd) {
-            return null;
-        }
-        argsObj.cmd = cmd;
-        argsObj.command = cmd;
-        if (typeof argsObj.workdir === 'string') {
-            argsObj.workdir = String(argsObj.workdir).trim();
-        }
-    }
-    if (lname === 'write_stdin') {
-        if (argsObj.chars !== undefined && argsObj.chars !== null) {
-            argsObj.chars = String(argsObj.chars);
-        }
-        if (argsObj.session_id !== undefined && argsObj.session_id !== null) {
-            const sid = Number.parseInt(String(argsObj.session_id), 10);
-            if (Number.isFinite(sid)) {
-                argsObj.session_id = sid;
-            }
-        }
-    }
-    if (lname === 'apply_patch') {
-        if (typeof argsObj.patch === 'string') {
-            argsObj.patch = String(argsObj.patch);
-        }
-    }
-    const filtered = filterArgsForTool(lname, argsObj);
-    if (lname === 'exec_command') {
-        const cmd = typeof filtered?.cmd === 'string' ? String(filtered.cmd).trim() : '';
-        if (!cmd) {
-            return null;
-        }
-    }
-    if (lname === 'write_stdin') {
-        const sid = filtered?.session_id;
-        if (sid === undefined || sid === null || String(sid).trim().length === 0) {
-            return null;
-        }
-    }
-    if (lname === 'apply_patch') {
-        const patch = typeof filtered?.patch === 'string' ? String(filtered.patch).trim() : '';
-        if (!patch) {
-            return null;
-        }
-    }
-    return filtered;
-}
-function normalizeJsonToolCallEntry(entry, options) {
-    if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
-        return null;
-    }
-    const rec = entry;
-    const fn = rec.function && typeof rec.function === 'object' && !Array.isArray(rec.function)
-        ? rec.function
-        : undefined;
-    const name = canonicalizeJsonToolName(rec.name ?? fn?.name, options);
-    if (!name) {
-        return null;
-    }
-    const argsSource = rec.input !== undefined
-        ? rec.input
-        : rec.arguments !== undefined
-            ? rec.arguments
-            : fn?.arguments !== undefined
-                ? fn.arguments
-                : fn?.input;
-    const argsObj = normalizeJsonToolArgs(name, argsSource, options);
-    if (!argsObj) {
-        return null;
-    }
-    let args = '{}';
-    try {
-        args = JSON.stringify(argsObj);
-    }
-    catch {
-        args = '{}';
-    }
-    const id = typeof rec.call_id === 'string' && rec.call_id.trim().length
-        ? rec.call_id.trim()
-        : typeof rec.id === 'string' && rec.id.trim().length
-            ? rec.id.trim()
-            : genToolCallId();
-    return { id, name, args };
-}
-function extractBalancedJsonObjectAt(text, startIndex) {
-    if (startIndex < 0 || startIndex >= text.length || text[startIndex] !== '{') {
-        return null;
-    }
-    let depth = 0;
-    let inString = false;
-    let escaped = false;
-    for (let i = startIndex; i < text.length; i += 1) {
-        const ch = text[i] ?? '';
-        if (inString) {
-            if (escaped) {
-                escaped = false;
-                continue;
-            }
-            if (ch === '\\') {
-                escaped = true;
-                continue;
-            }
-            if (ch === '"') {
-                inString = false;
-            }
-            continue;
-        }
-        if (ch === '"') {
-            inString = true;
-            continue;
-        }
-        if (ch === '{') {
-            depth += 1;
-            continue;
-        }
-        if (ch === '}') {
-            depth -= 1;
-            if (depth === 0) {
-                return text.slice(startIndex, i + 1);
-            }
-        }
-    }
-    return null;
-}
-function collectEmbeddedToolCallObjects(text) {
-    const out = [];
-    const seen = new Set();
-    const quoteRanges = [];
-    {
-        const lower = text.toLowerCase();
-        let cursor = 0;
-        while (cursor < lower.length) {
-            const open = lower.indexOf('<quote>', cursor);
-            if (open < 0)
-                break;
-            const close = lower.indexOf('</quote>', open + '<quote>'.length);
-            if (close < 0)
-                break;
-            quoteRanges.push({ start: open, end: close + '</quote>'.length });
-            cursor = close + '</quote>'.length;
-        }
-    }
-    const isInsideQuoteRange = (index) => quoteRanges.some((range) => index >= range.start && index < range.end);
-    for (let i = 0; i < text.length; i += 1) {
-        if (text[i] !== '{') {
-            continue;
-        }
-        if (isInsideQuoteRange(i)) {
-            continue;
-        }
-        const objectText = extractBalancedJsonObjectAt(text, i);
-        if (!objectText) {
-            continue;
-        }
-        const trimmed = objectText.trim();
-        if (trimmed.length > 0 &&
-            /["']tool_calls["']\s*:/.test(trimmed) &&
-            !seen.has(trimmed)) {
-            seen.add(trimmed);
-            out.push(trimmed);
-        }
-    }
-    return out;
-}
-function decodeJsonEscapes(raw) {
-    return String(raw || '')
-        .replace(/\\r\\n/g, '\n')
-        .replace(/\\n/g, '\n')
-        .replace(/\\r/g, '\n')
-        .replace(/\\t/g, '\t')
-        .replace(/\\"/g, '"');
-}
-function extractPossiblyBrokenQuotedValue(text, keys) {
-    for (const key of keys) {
-        const re = new RegExp(`"${key}"\\s*:\\s*"`, 'ig');
-        let match;
-        while ((match = re.exec(text)) !== null) {
-            const start = match.index + match[0].length;
-            let escaped = false;
-            for (let i = start; i < text.length; i += 1) {
-                const ch = text[i] ?? '';
-                if (escaped) {
-                    escaped = false;
-                    continue;
-                }
-                if (ch === '\\') {
-                    escaped = true;
-                    continue;
-                }
-                if (ch === '"') {
-                    return decodeJsonEscapes(text.slice(start, i));
-                }
-            }
-            return decodeJsonEscapes(text.slice(start));
-        }
-    }
-    return null;
-}
-function salvageToolCallFromBrokenJsonText(candidate, options) {
-    const rawName = candidate.match(/"name"\s*:\s*"([^"]+)"/i)?.[1];
-    const canonicalName = canonicalizeJsonToolName(rawName, options);
-    if (!canonicalName) {
-        return null;
-    }
-    let args = null;
-    if (canonicalName === 'exec_command') {
-        const cmd = extractPossiblyBrokenQuotedValue(candidate, ['cmd', 'command', 'script', 'toon']);
-        if (cmd && cmd.trim().length) {
-            args = { cmd, command: cmd };
-        }
-    }
-    if (!args) {
-        args = salvageToolArgsFromRawText(canonicalName, candidate);
-    }
-    if (!args) {
-        return null;
-    }
-    const normalizedArgs = normalizeJsonToolArgs(canonicalName, args, options);
-    if (!normalizedArgs) {
-        return null;
-    }
-    return {
-        id: genToolCallId(),
-        name: canonicalName,
-        args: JSON.stringify(normalizedArgs)
-    };
-}
-export function extractJsonToolCallsFromText(text, options) {
-    try {
-        if (typeof text !== 'string' || !text.trim()) {
-            return null;
-        }
-        const candidates = [];
-        const seenCandidates = new Set();
-        const addCandidate = (raw) => {
-            const trimmed = String(raw || '').trim();
-            if (!trimmed || seenCandidates.has(trimmed)) {
-                return;
-            }
-            seenCandidates.add(trimmed);
-            candidates.push(trimmed);
-            const decoded = decodeJsonEscapes(trimmed).trim();
-            if (!decoded || decoded === trimmed || seenCandidates.has(decoded)) {
-                return;
-            }
-            seenCandidates.add(decoded);
-            candidates.push(decoded);
-        };
-        addCandidate(text);
-        const fenceRe = /```(?:json|tool_call|tool_calls|function_call)?\s*([\s\S]*?)\s*```/gi;
-        let fm;
-        while ((fm = fenceRe.exec(text)) !== null) {
-            const body = String(fm[1] || '').trim();
-            if (body) {
-                addCandidate(body);
-            }
-        }
-        for (const embedded of collectEmbeddedToolCallObjects(text)) {
-            addCandidate(embedded);
-        }
-        const decodedText = decodeJsonEscapes(text);
-        if (decodedText && decodedText !== text) {
-            for (const embedded of collectEmbeddedToolCallObjects(decodedText)) {
-                addCandidate(embedded);
-            }
-        }
-        const out = [];
-        const seen = new Set();
-        const pushEntry = (rawEntry) => {
-            const normalized = normalizeJsonToolCallEntry(rawEntry, options);
-            if (!normalized) {
-                return;
-            }
-            const key = `${normalized.name}:${normalized.args}`;
-            if (seen.has(key)) {
-                return;
-            }
-            seen.add(key);
-            out.push(normalized);
-        };
-        for (const candidate of candidates) {
-            const parsed = tryParseJsonWithModelRepairs(candidate);
-            if (parsed !== null) {
-                for (const entry of extractToolCallEntriesFromUnknown(parsed)) {
-                    pushEntry(entry);
-                }
-            }
-            if (out.length > 0) {
-                continue;
-            }
-            const standaloneWrapper = candidate.match(/^\s*\{\s*["']tool_calls["']\s*:\s*(\[[\s\S]*\])\s*\}\s*$/);
-            if (standaloneWrapper?.[1]) {
-                const wrapped = `{"tool_calls":${standaloneWrapper[1]}}`;
-                const parsedWrapped = tryParseJsonWithModelRepairs(wrapped);
-                if (parsedWrapped !== null) {
-                    for (const entry of extractToolCallEntriesFromUnknown(parsedWrapped)) {
-                        pushEntry(entry);
-                    }
-                }
-            }
-        }
-        // Last-resort salvage should run only when normal JSON extraction found nothing.
-        if (out.length === 0) {
-            for (const candidate of candidates) {
-                const lower = candidate.toLowerCase();
-                if (lower.includes('<quote>') && lower.includes('</quote>')) {
-                    continue;
-                }
-                const salvaged = salvageToolCallFromBrokenJsonText(candidate, options);
-                if (!salvaged) {
-                    continue;
-                }
-                pushEntry({
-                    id: salvaged.id,
-                    name: salvaged.name,
-                    function: {
-                        name: salvaged.name,
-                        arguments: salvaged.args
-                    }
-                });
-            }
-        }
-        return out.length ? out : null;
-    }
-    catch {
-        return null;
-    }
-}

package/dist/conversion/shared/text-markup-normalizer/extractors-shared.d.ts

@@ -1,21 +0,0 @@
-export declare const KNOWN_TOOLS: Set<string>;
-export type JsonToolArgumentAliasMap = Record<string, string[]>;
-export type JsonToolRepairConfig = {
-    toolNameAliases?: Record<string, string>;
-    argumentAliases?: Record<string, JsonToolArgumentAliasMap>;
-};
-export type TextMarkupNormalizeOptions = {
-    jsonToolRepair?: JsonToolRepairConfig;
-};
-export type ToolCallLite = {
-    id?: string;
-    name: string;
-    args: string;
-};
-export declare function genToolCallId(): string;
-export declare function normalizeKey(raw: string): string;
-export declare function readDefaultWorkdirFromEnv(): string | undefined;
-export declare function filterArgsForTool(name: string, args: Record<string, unknown>): Record<string, unknown>;
-export declare function tryParseJsonValue(text: string): unknown | null;
-export declare function tryParsePrimitiveValue(text: string): unknown;
-export declare function coerceCommandValueToString(value: unknown): string;