@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/core/files/rag/extract/extract.js

@@ -0,0 +1,81 @@
+import { extractCsvText } from "./csv-text.js";
+import { classifyRagDocument, RAG_DOCUMENT_KIND } from "./document-types.js";
+import { extractHtmlText } from "./html-text.js";
+import { extractPdfText } from "./pdf-text.js";
+import { extractRtfText } from "./rtf-text.js";
+import { decodeTextBytes } from "./text-decoding.js";
+import { RAG_EXTRACT_ERROR_CODE, RagExtractError } from "./types.js";
+export async function extractRagDocumentFromBytes(input, opts = {}) {
+    const fileName = input.fileName ?? input.title ?? input.sourcePath ?? 'document';
+    const kind = classifyRagDocument({
+        fileName,
+        bytes: input.bytes,
+        ...(input.mimeType ? { mimeType: input.mimeType } : {})
+    });
+    if (kind === RAG_DOCUMENT_KIND.unsupported) {
+        throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, `Unsupported document type: ${fileName}`);
+    }
+    try {
+        const content = await extractText(kind, input.bytes, fileName, opts);
+        return buildDocumentInput(input, fileName, content);
+    }
+    catch (e) {
+        if (e instanceof RagExtractError)
+            throw e;
+        throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.extractionFailed, errorMessage(e));
+    }
+}
+function buildDocumentInput(input, fileName, content) {
+    return {
+        title: input.title ?? fileNameFromPath(fileName),
+        content,
+        ...(input.id !== undefined ? { id: input.id } : {}),
+        ...(input.blobId !== undefined ? { blobId: input.blobId } : {}),
+        ...(input.sourcePath !== undefined ? { sourcePath: input.sourcePath } : {}),
+        ...(input.sourceUrl !== undefined ? { sourceUrl: input.sourceUrl } : {}),
+        ...(input.mimeType !== undefined ? { mimeType: input.mimeType } : {}),
+        ...(input.artifactId !== undefined ? { artifactId: input.artifactId } : {}),
+        ...(input.attachmentId !== undefined ? { attachmentId: input.attachmentId } : {}),
+        ...(input.messageId !== undefined ? { messageId: input.messageId } : {}),
+        ...(input.chatId !== undefined ? { chatId: input.chatId } : {})
+    };
+}
+async function extractText(kind, bytes, fileName, opts) {
+    if (kind === RAG_DOCUMENT_KIND.pdf) {
+        const extractPdf = opts.extractors?.pdf ?? extractPdfText;
+        try {
+            return await extractPdf(bytes);
+        }
+        catch (e) {
+            throw new Error(`PDF extraction failed: ${errorMessage(e)}`);
+        }
+    }
+    if (kind === RAG_DOCUMENT_KIND.office) {
+        const extractOffice = opts.extractors?.office;
+        if (!extractOffice) {
+            throw new RagExtractError(RAG_EXTRACT_ERROR_CODE.unsupportedFile, 'Office document extraction is not available on this device');
+        }
+        try {
+            return await extractOffice(bytes, fileName, { signal: opts.signal });
+        }
+        catch (e) {
+            throw new Error(`Office document extraction failed: ${errorMessage(e)}`);
+        }
+    }
+    const text = decodeTextBytes(bytes);
+    if (kind === RAG_DOCUMENT_KIND.rtf)
+        return extractRtfText(text);
+    if (kind === RAG_DOCUMENT_KIND.html)
+        return extractHtmlText(text);
+    if (kind === RAG_DOCUMENT_KIND.csv)
+        return extractCsvText(text);
+    if (kind === RAG_DOCUMENT_KIND.tsv)
+        return extractCsvText(text, '\t');
+    return text;
+}
+function fileNameFromPath(path) {
+    return path.split(/[\\/]/).filter(Boolean).pop() ?? path;
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts

@@ -0,0 +1 @@
+export declare function formatHtmlTables(source: string): string;

package/dist-lib/lib/core/files/rag/extract/html-table-text.js

@@ -0,0 +1,125 @@
+const TABLE_BLOCK = /<table\b[^>]*>[\s\S]*?<\/table\s*>/gi;
+const CAPTION_BLOCK = /<caption\b[^>]*>([\s\S]*?)<\/caption\s*>/i;
+const TABLE_TAG = /<\/?(tr|th|td)\b[^>]*>/gi;
+const DROP_BLOCKS = /<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
+const COMMENT = /<!--[\s\S]*?-->/g;
+const BREAK_TAG = /<br\s*\/?>/gi;
+const TAG = /<[^>]*>/g;
+export function formatHtmlTables(source) {
+    return source.replace(TABLE_BLOCK, (table) => `\n\n${formatTable(table)}\n\n`);
+}
+function formatTable(table) {
+    const caption = extractCaption(table);
+    const rowsText = formatTableRows(table.replace(CAPTION_BLOCK, ' '));
+    return caption ? joinTableParts(caption, rowsText) : rowsText;
+}
+function extractCaption(table) {
+    const caption = normalizeCell(CAPTION_BLOCK.exec(table)?.[1] ?? '');
+    return caption.length > 0 ? caption : null;
+}
+function joinTableParts(caption, rowsText) {
+    return rowsText.length > 0 ? `${caption}\n\n${rowsText}` : caption;
+}
+function formatTableRows(table) {
+    const rows = parseRows(table).filter((row) => row.some((cell) => cell.text.length > 0));
+    const first = rows[0];
+    if (!first)
+        return normalizeCell(table);
+    if (!isColumnHeaderRow(first, rows.length))
+        return formatFlatRows(rows);
+    const headers = first.map((cell, index) => cell.text || `column_${index + 1}`);
+    const dataRows = rows
+        .slice(1)
+        .map((row) => formatDataRow(headers, row))
+        .filter((line) => line.length > 0);
+    return dataRows.length > 0
+        ? [`Columns: ${headers.join(', ')}`, ...dataRows].join('\n\n')
+        : formatFlatRows([first]);
+}
+function parseRows(table) {
+    const rows = [];
+    let row = null;
+    let cell = null;
+    let cursor = 0;
+    for (const match of table.matchAll(TABLE_TAG)) {
+        if (cell) {
+            cell.html += table.slice(cursor, match.index);
+        }
+        ;
+        ({ row, cell } = applyTableTag(rows, row, cell, match[0] ?? ''));
+        cursor = match.index + match[0].length;
+    }
+    if (cell) {
+        cell.html += table.slice(cursor);
+    }
+    finishRow(rows, row, cell);
+    return rows;
+}
+function applyTableTag(rows, row, cell, source) {
+    const tag = parseTableTag(source);
+    if (tag?.name === 'tr') {
+        return tag.closing ? finishRow(rows, row, cell) : startRow(rows, row, cell);
+    }
+    if (tag?.name === 'th' || tag?.name === 'td') {
+        return tag.closing ? finishCell(row, cell) : startCell(row, cell, tag.name === 'th');
+    }
+    return { row, cell };
+}
+function startRow(rows, row, cell) {
+    finishRow(rows, row, cell);
+    return { row: [], cell: null };
+}
+function startCell(row, cell, header) {
+    const state = finishCell(row, cell);
+    return { row: state.row ?? [], cell: { html: '', header } };
+}
+function finishRow(rows, row, cell) {
+    const state = finishCell(row, cell);
+    if (state.row && state.row.length > 0) {
+        rows.push(state.row);
+    }
+    return { row: null, cell: null };
+}
+function finishCell(row, cell) {
+    if (!cell)
+        return { row, cell: null };
+    const nextRow = row ?? [];
+    nextRow.push({ text: normalizeCell(cell.html), header: cell.header });
+    return { row: nextRow, cell: null };
+}
+function parseTableTag(tag) {
+    const match = /^<\s*(\/)?\s*(tr|th|td)\b/i.exec(tag);
+    return match ? { name: match[2] ?? '', closing: Boolean(match[1]) } : null;
+}
+function isColumnHeaderRow(row, rowCount) {
+    return rowCount > 1 && row.length >= 2 && row.every((cell) => cell.header);
+}
+function formatDataRow(headers, row) {
+    const parts = [];
+    const columnCount = Math.max(headers.length, row.length);
+    for (let index = 0; index < columnCount; index++) {
+        const value = row[index]?.text ?? '';
+        if (value.length === 0)
+            continue;
+        parts.push(`${headers[index] ?? `column_${index + 1}`}: ${value}`);
+    }
+    return parts.join(' | ');
+}
+function formatFlatRows(rows) {
+    return rows
+        .map((row) => row
+        .map((cell) => cell.text)
+        .filter((cell) => cell.length > 0)
+        .join(' | '))
+        .filter((line) => line.length > 0)
+        .join('\n');
+}
+function normalizeCell(html) {
+    return html
+        .replace(DROP_BLOCKS, ' ')
+        .replace(COMMENT, ' ')
+        .replace(BREAK_TAG, ' ')
+        .replace(TAG, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}

package/dist-lib/lib/core/files/rag/extract/html-text.d.ts

@@ -0,0 +1 @@
+export declare function extractHtmlText(source: string): string;

package/dist-lib/lib/core/files/rag/extract/html-text.js

@@ -0,0 +1,132 @@
+import { formatHtmlTables } from "./html-table-text.js";
+const DROP_BLOCKS = /<(script|style|noscript|template|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
+const COMMENT = /<!--[\s\S]*?-->/g;
+const HEADING_OPEN = /<h([1-6])\b[^>]*>/gi;
+const HEADING_CLOSE = /<\/h[1-6]\s*>/gi;
+const LIST_ITEM_OPEN = /<li\b[^>]*>/gi;
+const LIST_ITEM_CLOSE = /<\/li\s*>/gi;
+const TABLE_CELL_OPEN = /<t[hd]\b[^>]*>/gi;
+const TABLE_CELL_CLOSE = /<\/t[hd]\s*>/gi;
+const TABLE_ROW_OPEN = /<tr\b[^>]*>/gi;
+const TABLE_ROW_CLOSE = /<\/tr\s*>/gi;
+const BREAK_TAG = /<br\s*\/?>/gi;
+const BLOCK_CONTAINER_TAGS = 'p|div|section|article|header|footer|main|aside|nav|blockquote|pre|figure|figcaption|ul|ol|table|thead|tbody|tfoot|address|form|fieldset';
+const BLOCK_CLOSE = new RegExp(`</(${BLOCK_CONTAINER_TAGS})\\s*>`, 'gi');
+const BLOCK_OPEN = new RegExp(`<(${BLOCK_CONTAINER_TAGS}|hr)\\b[^>]*>`, 'gi');
+const STRAY_CLOSE_BRACKET = />/g;
+const NAMED_ENTITIES = new Map([
+    ['amp', '&'],
+    ['lt', '<'],
+    ['gt', '>'],
+    ['quot', '"'],
+    ['apos', "'"],
+    ['nbsp', ' '],
+    ['copy', '©'],
+    ['reg', '®'],
+    ['trade', '™'],
+    ['hellip', '…'],
+    ['mdash', '—'],
+    ['ndash', '–'],
+    ['lsquo', '‘'],
+    ['rsquo', '’'],
+    ['ldquo', '“'],
+    ['rdquo', '”'],
+    ['middot', '·'],
+    ['bull', '•'],
+    ['deg', '°'],
+    ['times', '×'],
+    ['divide', '÷'],
+    ['plusmn', '±'],
+    ['laquo', '«'],
+    ['raquo', '»'],
+    ['euro', '€'],
+    ['pound', '£'],
+    ['yen', '¥'],
+    ['cent', '¢'],
+    ['sect', '§'],
+    ['para', '¶'],
+    ['iexcl', '¡'],
+    ['iquest', '¿'],
+    ['dagger', '†'],
+    ['Dagger', '‡'],
+    ['sbquo', '‚'],
+    ['bdquo', '„'],
+    ['prime', '′'],
+    ['Prime', '″'],
+    ['frac12', '½'],
+    ['frac14', '¼'],
+    ['frac34', '¾']
+]);
+const NUMERIC_ENTITY = /&#(x[0-9a-fA-F]+|[0-9]+);/g;
+const NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]+);/g;
+export function extractHtmlText(source) {
+    const structured = markStructure(formatHtmlTables(dropNonContent(source)));
+    const text = stripTagsFully(structured);
+    return normalizeWhitespace(decodeEntities(text));
+}
+function dropNonContent(source) {
+    let previous = source;
+    let current = source.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
+    while (current !== previous) {
+        previous = current;
+        current = current.replace(DROP_BLOCKS, ' ').replace(COMMENT, ' ');
+    }
+    return current;
+}
+function stripTagsFully(source) {
+    let output = '';
+    let cursor = 0;
+    while (cursor < source.length) {
+        const open = source.indexOf('<', cursor);
+        if (open === -1) {
+            return output + stripStrayCloseBrackets(source.slice(cursor));
+        }
+        output += stripStrayCloseBrackets(source.slice(cursor, open));
+        const close = source.indexOf('>', open + 1);
+        if (close === -1) {
+            return `${output} `;
+        }
+        cursor = close + 1;
+    }
+    return output;
+}
+function stripStrayCloseBrackets(text) {
+    return text.replace(STRAY_CLOSE_BRACKET, ' ');
+}
+function markStructure(html) {
+    return html
+        .replace(HEADING_OPEN, (_match, level) => `\n\n${'#'.repeat(Number(level))} `)
+        .replace(HEADING_CLOSE, '\n\n')
+        .replace(TABLE_CELL_OPEN, ' | ')
+        .replace(TABLE_CELL_CLOSE, '')
+        .replace(TABLE_ROW_OPEN, '\n')
+        .replace(TABLE_ROW_CLOSE, '')
+        .replace(LIST_ITEM_OPEN, '\n- ')
+        .replace(LIST_ITEM_CLOSE, '')
+        .replace(BREAK_TAG, '\n')
+        .replace(BLOCK_OPEN, '\n\n')
+        .replace(BLOCK_CLOSE, '\n\n');
+}
+function decodeEntities(text) {
+    return text
+        .replace(NUMERIC_ENTITY, (_match, code) => {
+        const codePoint = code.startsWith('x') || code.startsWith('X')
+            ? Number.parseInt(code.slice(1), 16)
+            : Number.parseInt(code, 10);
+        return Number.isFinite(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff
+            ? String.fromCodePoint(codePoint)
+            : '';
+    })
+        .replace(NAMED_ENTITY, (match, name) => NAMED_ENTITIES.get(name) ?? match);
+}
+function normalizeWhitespace(text) {
+    return text
+        .replace(/\r\n?/g, '\n')
+        .replace(/[ \t\f\v]+/g, ' ')
+        .replace(/[ \t]+\n/g, '\n')
+        .replace(/\n[ \t]+/g, '\n')
+        .replace(/^\| ?/gm, '')
+        .replace(/ \|\s*$/gm, '')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}

package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts

@@ -0,0 +1,2 @@
+import type { RagFileExtractor } from './extract.ts';
+export declare const extractOfficeText: RagFileExtractor;

package/dist-lib/lib/core/files/rag/extract/office-collabora.js

@@ -0,0 +1,8 @@
+import { convertToText } from "../../../../collabora/convert.js";
+import { MAX_OFFICE_BYTES } from "../limits.js";
+export const extractOfficeText = (bytes, fileName, opts) => {
+    if (bytes.byteLength > MAX_OFFICE_BYTES) {
+        throw new Error(`document is too large to convert: ${bytes.byteLength} bytes, limit ${MAX_OFFICE_BYTES}`);
+    }
+    return convertToText({ bytes, fileName, to: 'md', signal: opts?.signal });
+};

package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts

@@ -0,0 +1,17 @@
+export declare class BareTextDecoder {
+    readonly encoding: string;
+    readonly fatal: boolean;
+    readonly ignoreBOM: boolean;
+    constructor(label?: string, options?: {
+        fatal?: boolean;
+        ignoreBOM?: boolean;
+    });
+    decode(input?: ArrayBufferView | ArrayBuffer): string;
+}
+export declare class BareTextEncoder {
+    encode(input?: string): Uint8Array;
+}
+export declare function decodeBytes(bytes: Uint8Array, label: string, options?: {
+    fatal?: boolean;
+    ignoreBOM?: boolean;
+}): string;

package/dist-lib/lib/core/files/rag/extract/pdf-platform.js

@@ -0,0 +1,141 @@
+import b4a from 'b4a';
+import { AbortController, AbortSignal } from 'bare-abort-controller';
+import { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.mjs';
+import { decodeLatin1, decodeUtf16 } from "./utf.js";
+const g = globalThis;
+if (!g.ReadableStream)
+    g.ReadableStream = ReadableStream;
+if (!g.DOMException) {
+    g.DOMException = class DOMException extends Error {
+        constructor(message, name = 'Error') {
+            super(message);
+            this.name = name;
+        }
+    };
+}
+export class BareTextDecoder {
+    encoding;
+    fatal;
+    ignoreBOM;
+    constructor(label = 'utf-8', options = {}) {
+        this.encoding = parseEncoding(label);
+        this.fatal = options.fatal === true;
+        this.ignoreBOM = options.ignoreBOM === true;
+    }
+    decode(input) {
+        if (input === undefined)
+            return '';
+        const bytes = ArrayBuffer.isView(input)
+            ? b4a.from(input.buffer, input.byteOffset, input.byteLength)
+            : b4a.from(input);
+        return decodeBytes(bytes, this.encoding, { fatal: this.fatal, ignoreBOM: this.ignoreBOM });
+    }
+}
+export class BareTextEncoder {
+    encode(input = '') {
+        return b4a.from(String(input), 'utf8');
+    }
+}
+if (!g.TextDecoder)
+    g.TextDecoder = BareTextDecoder;
+if (!g.TextEncoder)
+    g.TextEncoder = BareTextEncoder;
+export function decodeBytes(bytes, label, options = {}) {
+    const fatal = options.fatal === true;
+    const encoding = parseEncoding(label);
+    const text = encoding === 'utf-16le'
+        ? utf16(bytes, true, fatal)
+        : encoding === 'utf-16be'
+            ? utf16(bytes, false, fatal)
+            : encoding === 'latin1'
+                ? decodeLatin1(bytes)
+                : decodeUtf8(bytes, fatal);
+    return options.ignoreBOM || text.charCodeAt(0) !== 0xfeff ? text : text.slice(1);
+}
+function parseEncoding(label) {
+    const l = String(label).trim().toLowerCase();
+    if (l.includes('utf-16') || l.includes('utf16') || l === 'ucs-2' || l === 'ucs2') {
+        return l.includes('be') ? 'utf-16be' : 'utf-16le';
+    }
+    if (l.includes('latin1') || l.includes('8859'))
+        return 'latin1';
+    return 'utf-8';
+}
+function utf16(bytes, littleEndian, fatal) {
+    if (fatal && bytes.length % 2 !== 0)
+        throw new TypeError('truncated utf-16 sequence');
+    const text = decodeUtf16(bytes, littleEndian);
+    if (fatal)
+        assertPairedSurrogates(text);
+    return text;
+}
+function assertPairedSurrogates(text) {
+    for (let i = 0; i < text.length; i++) {
+        const unit = text.charCodeAt(i);
+        if (unit >= 0xd800 && unit <= 0xdbff) {
+            const low = text.charCodeAt(i + 1);
+            if (Number.isNaN(low) || low < 0xdc00 || low > 0xdfff) {
+                throw new TypeError('lone high surrogate');
+            }
+            i++;
+        }
+        else if (unit >= 0xdc00 && unit <= 0xdfff) {
+            throw new TypeError('lone low surrogate');
+        }
+    }
+}
+function decodeUtf8(bytes, fatal) {
+    if (fatal && !isValidUtf8(bytes))
+        throw new TypeError('invalid utf-8 sequence');
+    return b4a.toString(bytes, 'utf8');
+}
+function isValidUtf8(bytes) {
+    for (let i = 0; i < bytes.length;) {
+        const b0 = bytes[i];
+        if (b0 < 0x80) {
+            i++;
+            continue;
+        }
+        let size;
+        let min;
+        let cp;
+        if ((b0 & 0xe0) === 0xc0) {
+            size = 2;
+            min = 0x80;
+            cp = b0 & 0x1f;
+        }
+        else if ((b0 & 0xf0) === 0xe0) {
+            size = 3;
+            min = 0x800;
+            cp = b0 & 0x0f;
+        }
+        else if ((b0 & 0xf8) === 0xf0) {
+            size = 4;
+            min = 0x10000;
+            cp = b0 & 0x07;
+        }
+        else {
+            return false;
+        }
+        if (i + size > bytes.length)
+            return false;
+        for (let j = 1; j < size; j++) {
+            const bj = bytes[i + j];
+            if ((bj & 0xc0) !== 0x80)
+                return false;
+            cp = (cp << 6) | (bj & 0x3f);
+        }
+        if (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
+            return false;
+        i += size;
+    }
+    return true;
+}
+if (!g.AbortController) {
+    g.AbortController = AbortController;
+    g.AbortSignal = AbortSignal;
+}
+const clone = globalThis.structuredClone;
+if (clone) {
+    g.structuredClone = (value, options) => clone(value, (options ?? undefined));
+}

package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts

@@ -0,0 +1,12 @@
+import './pdf-platform.ts';
+export declare function extractPdfText(bytes: Uint8Array, options?: {
+    maxChars?: number;
+}): Promise<string>;
+export interface PdfHead {
+    readonly text: string;
+    readonly pagesRead: number;
+    readonly totalPages: number;
+}
+export declare function extractPdfHead(bytes: Uint8Array, { maxChars }?: {
+    maxChars?: number;
+}): Promise<PdfHead>;

package/dist-lib/lib/core/files/rag/extract/pdf-text.js

@@ -0,0 +1,43 @@
+import "./pdf-platform.js";
+import { getDocument } from 'unpdf/pdfjs';
+export function extractPdfText(bytes, options = {}) {
+    return extractPdfHead(bytes, options).then((head) => head.text);
+}
+export async function extractPdfHead(bytes, { maxChars = Infinity } = {}) {
+    const pdf = await getDocument({
+        data: new Uint8Array(bytes),
+        disableFontFace: true,
+        isEvalSupported: false,
+        useSystemFonts: true,
+        verbosity: 0
+    }).promise;
+    try {
+        const pages = [];
+        let total = 0;
+        for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) {
+            const page = await pdf.getPage(pageNumber);
+            const textContent = await page.getTextContent();
+            const pageText = textContent.items.map(textItem).filter(Boolean).join(' ');
+            pages.push(pageText);
+            total += pageText.length;
+            if (total > maxChars)
+                break;
+        }
+        const text = pages
+            .join('\n\n')
+            .replace(/[ \t]+\n/g, '\n')
+            .trim();
+        return { text, pagesRead: pages.length, totalPages: pdf.numPages };
+    }
+    finally {
+        await pdf.destroy();
+    }
+}
+function textItem(item) {
+    if (!isRecord(item) || typeof item.str !== 'string')
+        return '';
+    return item.str.trim();
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}

package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts

@@ -0,0 +1 @@
+export declare function extractRtfText(source: string): string;

package/dist-lib/lib/core/files/rag/extract/rtf-text.js

@@ -0,0 +1,15 @@
+const HEX_ESCAPE = /\\'([0-9a-fA-F]{2})/g;
+const CONTROL_WORD = /\\[a-zA-Z]+-?\d* ?/g;
+const CONTROL_SYMBOL = /\\[^a-zA-Z0-9]/g;
+export function extractRtfText(source) {
+    return source
+        .replaceAll('\\par', '\n')
+        .replaceAll('\\line', '\n')
+        .replace(HEX_ESCAPE, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
+        .replace(CONTROL_WORD, '')
+        .replace(CONTROL_SYMBOL, '')
+        .replace(/[{}]/g, '')
+        .replace(/[ \t]+\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}

package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts

@@ -0,0 +1 @@
+export declare function decodeTextBytes(bytes: Uint8Array): string;