@qvac/core 0.0.0 → 0.1.1

package/dist-lib/lib/core/files/rag/chunk/chunking.js

@@ -0,0 +1,117 @@
+import { buildRagChunkMetadata } from "./metadata.js";
+import { tokenizeForRag } from "./tokenizer.js";
+const DEFAULT_CHUNK_SIZE = 256;
+const DEFAULT_CHUNK_OVERLAP = 50;
+export function chunkRagDocuments(documents, options) {
+    return documents.flatMap((document) => chunkDocument(document, normalizeChunkOptions(options)));
+}
+function chunkDocument(document, options) {
+    const tokens = tokenizeForRag(document.content);
+    if (tokens.length === 0) {
+        return [];
+    }
+    const chunks = [];
+    const paragraphEnds = paragraphEndIndexes(tokens);
+    for (let start = skipBreakTokens(tokens, 0); start < tokens.length;) {
+        const end = chunkEnd(start, tokens.length, paragraphEnds, options.chunkSize);
+        const first = tokens[start];
+        const last = tokens[end - 1];
+        if (!first || !last) {
+            break;
+        }
+        const content = document.content.slice(first.start, last.end).trim();
+        chunks.push(buildChunk(document, chunks.length, content, end - start));
+        if (end === tokens.length) {
+            break;
+        }
+        start = skipBreakTokens(tokens, nextStart(start, end, options.chunkOverlap));
+    }
+    return chunks;
+}
+function paragraphEndIndexes(tokens) {
+    const ends = [];
+    let paragraphStart = 0;
+    let breakStart = null;
+    let newlineCount = 0;
+    tokens.forEach((token, index) => {
+        const newlines = countNewlines(token.text);
+        if (newlines === 0 && token.text.trim().length > 0) {
+            breakStart = null;
+            newlineCount = 0;
+            return;
+        }
+        if (newlines === 0) {
+            return;
+        }
+        breakStart ??= index;
+        newlineCount += newlines;
+        if (newlineCount < 2) {
+            return;
+        }
+        if (paragraphStart < breakStart) {
+            ends.push(breakStart);
+        }
+        paragraphStart = index + 1;
+        breakStart = null;
+        newlineCount = 0;
+    });
+    return [...ends, tokens.length];
+}
+function chunkEnd(start, tokenCount, paragraphEnds, chunkSize) {
+    const maxEnd = Math.min(start + chunkSize, tokenCount);
+    return paragraphEnds.filter((end) => end > start && end <= maxEnd).at(-1) ?? maxEnd;
+}
+function nextStart(start, end, overlap) {
+    const overlapped = end - overlap;
+    return overlapped > start ? overlapped : end;
+}
+function skipBreakTokens(tokens, start) {
+    let next = start;
+    let token = tokens[next];
+    while (token && isParagraphSeparatorToken(token.text)) {
+        next++;
+        token = tokens[next];
+    }
+    return next;
+}
+function isParagraphSeparatorToken(text) {
+    return text.trim().length === 0 && text.includes('\n');
+}
+function countNewlines(text) {
+    return [...text].filter((char) => char === '\n').length;
+}
+function buildChunk(document, chunkIndex, content, tokenCount) {
+    const documentId = document.id ?? stableDocumentId(document);
+    return {
+        id: chunkId(documentId, document.chatId, chunkIndex),
+        content,
+        tokenCount,
+        metadataJson: JSON.stringify(buildRagChunkMetadata({ document, documentId, chunkIndex, tokenCount }))
+    };
+}
+function chunkId(documentId, chatId, chunkIndex) {
+    return chatId ? `${documentId}:${chatId}:${chunkIndex}` : `${documentId}:${chunkIndex}`;
+}
+function normalizeChunkOptions(options) {
+    const chunkSize = positiveInteger(options.chunkSize, DEFAULT_CHUNK_SIZE);
+    const requestedOverlap = nonNegativeInteger(options.chunkOverlap, DEFAULT_CHUNK_OVERLAP);
+    const chunkOverlap = Math.min(requestedOverlap, Math.max(0, chunkSize - 1));
+    return { chunkSize, chunkOverlap };
+}
+function positiveInteger(value, fallback) {
+    return Number.isInteger(value) && value !== undefined && value > 0 ? value : fallback;
+}
+function nonNegativeInteger(value, fallback) {
+    return Number.isInteger(value) && value !== undefined && value >= 0 ? value : fallback;
+}
+function stableDocumentId(document) {
+    return `doc-${fnv1a(`${document.title}\n${document.sourcePath ?? ''}\n${document.content}`)}`;
+}
+function fnv1a(value) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < value.length; i++) {
+        hash ^= value.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193);
+    }
+    return (hash >>> 0).toString(16).padStart(8, '0');
+}

package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts

@@ -0,0 +1,30 @@
+import type { RagDocumentInput } from '../extract/types.ts';
+export type RagChunkMetadata = {
+    readonly documentId: string;
+    readonly title: string;
+    readonly sourcePath: string | null;
+    readonly sourceUrl: string | null;
+    readonly mimeType: string | null;
+    readonly artifactId: string | null;
+    readonly attachmentId: string | null;
+    readonly messageId: string | null;
+    readonly chatId: string | null;
+    readonly chunkIndex: number;
+    readonly tokenCount: number;
+    readonly indexedAt: string;
+};
+export declare function attachmentDocumentId(attachmentId: string): string;
+export declare function buildRagChunkMetadata(input: {
+    readonly document: RagDocumentInput;
+    readonly documentId: string;
+    readonly chunkIndex: number;
+    readonly tokenCount: number;
+}): RagChunkMetadata;
+export declare function parseRagMetadata(metadataJson: string | null): RagChunkMetadata | null;
+export declare function hitMatchesFilters(hit: {
+    readonly metadataJson: string | null;
+}, filters: {
+    readonly chatId?: string;
+    readonly documentIds?: ReadonlyArray<string>;
+    readonly attachmentIds?: ReadonlyArray<string>;
+}, liveSourceIds?: ReadonlySet<string>): boolean;

package/dist-lib/lib/core/files/rag/chunk/metadata.js

@@ -0,0 +1,89 @@
+const OPTIONAL_STRING_KEYS = [
+    'sourcePath',
+    'sourceUrl',
+    'mimeType',
+    'artifactId',
+    'attachmentId',
+    'messageId',
+    'chatId'
+];
+export function attachmentDocumentId(attachmentId) {
+    return `attachment-${attachmentId}`;
+}
+export function buildRagChunkMetadata(input) {
+    return {
+        documentId: input.documentId,
+        title: input.document.title,
+        sourcePath: input.document.sourcePath ?? null,
+        sourceUrl: input.document.sourceUrl ?? null,
+        mimeType: input.document.mimeType ?? null,
+        artifactId: input.document.artifactId ?? null,
+        attachmentId: input.document.attachmentId ?? null,
+        messageId: input.document.messageId ?? null,
+        chatId: input.document.chatId ?? null,
+        chunkIndex: input.chunkIndex,
+        tokenCount: input.tokenCount,
+        indexedAt: new Date().toISOString()
+    };
+}
+export function parseRagMetadata(metadataJson) {
+    if (!metadataJson)
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(metadataJson);
+    }
+    catch {
+        return null;
+    }
+    if (typeof parsed !== 'object' || parsed === null)
+        return null;
+    const value = parsed;
+    const { documentId, title, indexedAt, chunkIndex, tokenCount } = value;
+    if (typeof documentId !== 'string' || typeof title !== 'string')
+        return null;
+    if (typeof indexedAt !== 'string')
+        return null;
+    if (!isNumber(chunkIndex) || !isNumber(tokenCount))
+        return null;
+    for (const key of OPTIONAL_STRING_KEYS) {
+        const optional = value[key];
+        if (optional !== undefined && optional !== null && typeof optional !== 'string')
+            return null;
+    }
+    return {
+        documentId,
+        title,
+        sourcePath: stringOrNull(value.sourcePath),
+        sourceUrl: stringOrNull(value.sourceUrl),
+        mimeType: stringOrNull(value.mimeType),
+        artifactId: stringOrNull(value.artifactId),
+        attachmentId: stringOrNull(value.attachmentId),
+        messageId: stringOrNull(value.messageId),
+        chatId: stringOrNull(value.chatId),
+        chunkIndex,
+        tokenCount,
+        indexedAt
+    };
+}
+export function hitMatchesFilters(hit, filters, liveSourceIds) {
+    const metadata = parseRagMetadata(hit.metadataJson);
+    if (!metadata)
+        return false;
+    if (filters.chatId && metadata.chatId !== filters.chatId)
+        return false;
+    if (filters.documentIds?.length && !filters.documentIds.includes(metadata.documentId)) {
+        return false;
+    }
+    if (filters.attachmentIds?.length &&
+        (!metadata.attachmentId || !filters.attachmentIds.includes(metadata.attachmentId))) {
+        return false;
+    }
+    return !(liveSourceIds && metadata.attachmentId && !liveSourceIds.has(metadata.attachmentId));
+}
+function isNumber(value) {
+    return typeof value === 'number' && !Number.isNaN(value);
+}
+function stringOrNull(value) {
+    return typeof value === 'string' ? value : null;
+}

package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts

@@ -0,0 +1,7 @@
+export type RagToken = {
+    readonly text: string;
+    readonly start: number;
+    readonly end: number;
+};
+export declare function tokenizeForRag(text: string): ReadonlyArray<RagToken>;
+export declare function estimateRagTokens(text: string): number;

package/dist-lib/lib/core/files/rag/chunk/tokenizer.js

@@ -0,0 +1,18 @@
+const TOKEN_PATTERN = /'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+export function tokenizeForRag(text) {
+    const tokens = [];
+    TOKEN_PATTERN.lastIndex = 0;
+    let match = TOKEN_PATTERN.exec(text);
+    while (match) {
+        tokens.push({
+            text: match[0],
+            start: match.index,
+            end: match.index + match[0].length
+        });
+        match = TOKEN_PATTERN.exec(text);
+    }
+    return tokens;
+}
+export function estimateRagTokens(text) {
+    return tokenizeForRag(text).length;
+}

package/dist-lib/lib/core/files/rag/document-types.d.ts

@@ -0,0 +1,11 @@
+export declare const RAG_ACCEPT_EXTENSIONS: readonly [".pdf", ".txt", ".md", ".markdown", ".rtf", ".html", ".htm", ".xhtml", ".log", ".json", ".jsonl", ".csv", ".tsv", ".xml", ".yaml", ".yml"];
+export declare const EXEC_ATTACHABLE_EXTENSIONS: readonly [".docx", ".xlsx", ".pptx"];
+export declare const EXEC_ATTACHABLE_MIME_TYPES: readonly ["application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.presentationml.presentation"];
+export declare function isExecAttachableDocument(input: {
+    readonly fileName?: string;
+    readonly mimeType?: string;
+}): boolean;
+export declare function isKnownUnsupportedDocument(input: {
+    readonly fileName?: string;
+    readonly mimeType?: string;
+}): boolean;

package/dist-lib/lib/core/files/rag/document-types.js

@@ -0,0 +1,94 @@
+function extensionForName(fileName) {
+    if (!fileName) {
+        return null;
+    }
+    const index = fileName.lastIndexOf('.');
+    return index >= 0 ? fileName.slice(index).toLowerCase() : null;
+}
+function normalizeMimeType(mimeType) {
+    return mimeType?.split(';')[0]?.trim().toLowerCase() ?? '';
+}
+export const RAG_ACCEPT_EXTENSIONS = [
+    '.pdf',
+    '.txt',
+    '.md',
+    '.markdown',
+    '.rtf',
+    '.html',
+    '.htm',
+    '.xhtml',
+    '.log',
+    '.json',
+    '.jsonl',
+    '.csv',
+    '.tsv',
+    '.xml',
+    '.yaml',
+    '.yml'
+];
+const KNOWN_UNSUPPORTED_EXTENSIONS = new Set([
+    '.7z',
+    '.apk',
+    '.avi',
+    '.bin',
+    '.bz2',
+    '.dmg',
+    '.doc',
+    '.epub',
+    '.exe',
+    '.gz',
+    '.iso',
+    '.key',
+    '.mkv',
+    '.mov',
+    '.numbers',
+    '.odp',
+    '.ods',
+    '.odt',
+    '.pages',
+    '.ppt',
+    '.rar',
+    '.tar',
+    '.tgz',
+    '.xls',
+    '.zip'
+]);
+const KNOWN_UNSUPPORTED_MIME_PREFIXES = [
+    'video/',
+    'application/zip',
+    'application/x-tar',
+    'application/gzip',
+    'application/x-7z',
+    'application/x-rar',
+    'application/vnd.openxmlformats-officedocument',
+    'application/vnd.ms-',
+    'application/vnd.oasis.opendocument',
+    'application/msword',
+    'application/epub'
+];
+export const EXEC_ATTACHABLE_EXTENSIONS = ['.docx', '.xlsx', '.pptx'];
+export const EXEC_ATTACHABLE_MIME_TYPES = [
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+];
+const EXEC_ATTACHABLE_EXTENSION_SET = new Set(EXEC_ATTACHABLE_EXTENSIONS);
+const EXEC_ATTACHABLE_MIME_SET = new Set(EXEC_ATTACHABLE_MIME_TYPES);
+export function isExecAttachableDocument(input) {
+    const extension = extensionForName(input.fileName);
+    if (extension && EXEC_ATTACHABLE_EXTENSION_SET.has(extension)) {
+        return true;
+    }
+    return EXEC_ATTACHABLE_MIME_SET.has(normalizeMimeType(input.mimeType));
+}
+export function isKnownUnsupportedDocument(input) {
+    if (isExecAttachableDocument(input)) {
+        return false;
+    }
+    const extension = extensionForName(input.fileName);
+    if (extension && KNOWN_UNSUPPORTED_EXTENSIONS.has(extension)) {
+        return true;
+    }
+    const mimeType = normalizeMimeType(input.mimeType);
+    return KNOWN_UNSUPPORTED_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix));
+}

package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts

@@ -0,0 +1 @@
+export declare function extractCsvText(source: string, delimiter?: string): string;

package/dist-lib/lib/core/files/rag/extract/csv-text.js

@@ -0,0 +1,117 @@
+const FIELD_SEPARATOR = ' | ';
+const KEY_VALUE_SEPARATOR = ': ';
+const DEFAULT_DELIMITER = ',';
+export function extractCsvText(source, delimiter = DEFAULT_DELIMITER) {
+    const rows = parseCsv(source, delimiter);
+    if (rows.length === 0)
+        return '';
+    const headerRow = rows[0];
+    if (!headerRow || headerRow.length === 0)
+        return '';
+    const headers = headerRow
+        .map(normalizeCell)
+        .map((value, index) => (value.length > 0 ? value : `column_${index + 1}`));
+    const dataRows = rows.slice(1);
+    if (dataRows.length === 0 || headers.length < 2) {
+        return formatFlat(rows);
+    }
+    const headerLine = `Columns: ${headers.join(', ')}`;
+    const formatted = dataRows
+        .filter((row) => row.some((cell) => cell.trim().length > 0))
+        .map((row) => formatRow(headers, row));
+    return [headerLine, ...formatted].join('\n').trim();
+}
+function formatFlat(rows) {
+    return rows
+        .map((row) => row
+        .map(normalizeCell)
+        .filter((cell) => cell.length > 0)
+        .join(', '))
+        .filter((line) => line.length > 0)
+        .join('\n')
+        .trim();
+}
+function formatRow(headers, row) {
+    const parts = [];
+    for (let index = 0; index < headers.length; index++) {
+        const value = normalizeCell(row[index] ?? '');
+        if (value.length === 0)
+            continue;
+        const header = headers[index] ?? `column_${index + 1}`;
+        parts.push(`${header}${KEY_VALUE_SEPARATOR}${value}`);
+    }
+    return parts.join(FIELD_SEPARATOR);
+}
+function normalizeCell(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}
+function parseCsv(source, delimiter) {
+    const state = {
+        rows: [],
+        current: [],
+        field: '',
+        inQuotes: false,
+        index: 0,
+        delimiter
+    };
+    while (state.index < source.length) {
+        if (state.inQuotes) {
+            consumeQuotedChar(state, source);
+        }
+        else {
+            consumeUnquotedChar(state, source);
+        }
+    }
+    flushFinalField(state);
+    return state.rows.filter((row) => row.length > 1 || (row[0] ?? '').length > 0);
+}
+function consumeQuotedChar(state, source) {
+    const char = source[state.index];
+    if (char !== '"') {
+        state.field += char;
+        state.index += 1;
+        return;
+    }
+    if (source[state.index + 1] === '"') {
+        state.field += '"';
+        state.index += 2;
+        return;
+    }
+    state.inQuotes = false;
+    state.index += 1;
+}
+function consumeUnquotedChar(state, source) {
+    const char = source[state.index];
+    if (char === '"' && state.field.length === 0) {
+        state.inQuotes = true;
+        state.index += 1;
+        return;
+    }
+    if (char === state.delimiter) {
+        finishField(state);
+        state.index += 1;
+        return;
+    }
+    if (char === '\n' || char === '\r') {
+        finishRow(state);
+        state.index += char === '\r' && source[state.index + 1] === '\n' ? 2 : 1;
+        return;
+    }
+    state.field += char;
+    state.index += 1;
+}
+function finishField(state) {
+    state.current.push(state.field);
+    state.field = '';
+}
+function finishRow(state) {
+    finishField(state);
+    state.rows.push(state.current);
+    state.current = [];
+}
+function flushFinalField(state) {
+    if (state.field.length === 0 && state.current.length === 0)
+        return;
+    state.current.push(state.field);
+    state.rows.push(state.current);
+}

package/dist-lib/lib/core/files/rag/extract/document-types.d.ts

@@ -0,0 +1,22 @@
+export declare const RAG_DOCUMENT_KIND: {
+    readonly pdf: "pdf";
+    readonly office: "office";
+    readonly rtf: "rtf";
+    readonly html: "html";
+    readonly csv: "csv";
+    readonly tsv: "tsv";
+    readonly text: "text";
+    readonly unsupported: "unsupported";
+};
+export type RagDocumentKind = (typeof RAG_DOCUMENT_KIND)[keyof typeof RAG_DOCUMENT_KIND];
+export declare const RAG_ACCEPT_EXTENSIONS: readonly [".pdf", ".docx", ".doc", ".docm", ".dot", ".dotx", ".odt", ".fodt", ".ott", ".txt", ".md", ".markdown", ".rtf", ".html", ".htm", ".xhtml", ".log", ".json", ".jsonl", ".csv", ".tsv", ".xml", ".yaml", ".yml"];
+export declare function classifyRagDocument(input: {
+    readonly fileName?: string;
+    readonly mimeType?: string;
+    readonly bytes?: Uint8Array;
+}): RagDocumentKind;
+export declare function isSupportedRagDocument(input: {
+    readonly fileName?: string;
+    readonly mimeType?: string;
+    readonly bytes?: Uint8Array;
+}): boolean;

package/dist-lib/lib/core/files/rag/extract/document-types.js

@@ -0,0 +1,152 @@
+export const RAG_DOCUMENT_KIND = {
+    pdf: 'pdf',
+    office: 'office',
+    rtf: 'rtf',
+    html: 'html',
+    csv: 'csv',
+    tsv: 'tsv',
+    text: 'text',
+    unsupported: 'unsupported'
+};
+const PDF_EXTENSIONS = new Set(['.pdf']);
+const OFFICE_EXTENSIONS = new Set([
+    '.docx',
+    '.doc',
+    '.docm',
+    '.dot',
+    '.dotx',
+    '.odt',
+    '.fodt',
+    '.ott'
+]);
+const OFFICE_MIME_TYPES = new Set([
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
+    'application/msword',
+    'application/vnd.oasis.opendocument.text',
+    'application/vnd.oasis.opendocument.text-template'
+]);
+const RTF_EXTENSIONS = new Set(['.rtf']);
+const HTML_EXTENSIONS = new Set(['.html', '.htm', '.xhtml']);
+const CSV_EXTENSIONS = new Set(['.csv']);
+const TSV_EXTENSIONS = new Set(['.tsv']);
+const TEXT_EXTENSIONS = new Set([
+    '.json',
+    '.jsonl',
+    '.log',
+    '.markdown',
+    '.md',
+    '.mdown',
+    '.mkd',
+    '.text',
+    '.txt',
+    '.xml',
+    '.yaml',
+    '.yml'
+]);
+const TEXT_MIME_TYPES = new Set([
+    'application/json',
+    'application/ld+json',
+    'application/rtf',
+    'application/x-ndjson',
+    'application/xml',
+    'application/yaml',
+    'text/rtf'
+]);
+export const RAG_ACCEPT_EXTENSIONS = [
+    '.pdf',
+    '.docx',
+    '.doc',
+    '.docm',
+    '.dot',
+    '.dotx',
+    '.odt',
+    '.fodt',
+    '.ott',
+    '.txt',
+    '.md',
+    '.markdown',
+    '.rtf',
+    '.html',
+    '.htm',
+    '.xhtml',
+    '.log',
+    '.json',
+    '.jsonl',
+    '.csv',
+    '.tsv',
+    '.xml',
+    '.yaml',
+    '.yml'
+];
+export function classifyRagDocument(input) {
+    const extension = extensionForName(input.fileName);
+    const mimeType = normalizeMimeType(input.mimeType);
+    const explicit = kindFromExtension(extension) ?? kindFromMimeType(mimeType);
+    if (explicit)
+        return explicit;
+    return input.bytes && looksLikeText(input.bytes)
+        ? RAG_DOCUMENT_KIND.text
+        : RAG_DOCUMENT_KIND.unsupported;
+}
+export function isSupportedRagDocument(input) {
+    return classifyRagDocument(input) !== RAG_DOCUMENT_KIND.unsupported;
+}
+function extensionForName(fileName) {
+    if (!fileName)
+        return null;
+    const index = fileName.lastIndexOf('.');
+    return index >= 0 ? fileName.slice(index).toLowerCase() : null;
+}
+function normalizeMimeType(mimeType) {
+    return mimeType?.split(';')[0]?.trim().toLowerCase() ?? '';
+}
+function kindFromExtension(extension) {
+    if (!extension)
+        return null;
+    if (PDF_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.pdf;
+    if (OFFICE_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.office;
+    if (RTF_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.rtf;
+    if (HTML_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.html;
+    if (CSV_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.csv;
+    if (TSV_EXTENSIONS.has(extension))
+        return RAG_DOCUMENT_KIND.tsv;
+    return TEXT_EXTENSIONS.has(extension) ? RAG_DOCUMENT_KIND.text : null;
+}
+function kindFromMimeType(mimeType) {
+    if (mimeType === 'application/pdf')
+        return RAG_DOCUMENT_KIND.pdf;
+    if (OFFICE_MIME_TYPES.has(mimeType))
+        return RAG_DOCUMENT_KIND.office;
+    if (mimeType === 'application/rtf' || mimeType === 'text/rtf')
+        return RAG_DOCUMENT_KIND.rtf;
+    if (mimeType === 'text/html' || mimeType === 'application/xhtml+xml') {
+        return RAG_DOCUMENT_KIND.html;
+    }
+    if (mimeType === 'text/csv' || mimeType === 'application/csv')
+        return RAG_DOCUMENT_KIND.csv;
+    if (mimeType === 'text/tab-separated-values')
+        return RAG_DOCUMENT_KIND.tsv;
+    return mimeType.startsWith('text/') || TEXT_MIME_TYPES.has(mimeType)
+        ? RAG_DOCUMENT_KIND.text
+        : null;
+}
+function looksLikeText(bytes) {
+    const length = Math.min(bytes.byteLength, 4096);
+    if (length === 0)
+        return true;
+    let suspicious = 0;
+    for (let index = 0; index < length; index++) {
+        const byte = bytes[index] ?? 0;
+        if (byte === 0)
+            return false;
+        if (byte < 32 && byte !== 9 && byte !== 10 && byte !== 12 && byte !== 13)
+            suspicious++;
+    }
+    return suspicious / length < 0.02;
+}

package/dist-lib/lib/core/files/rag/extract/extract.d.ts

@@ -0,0 +1,17 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import { type RagDocumentInput, type RagFileInput } from './types.ts';
+export type RagTextExtractor = (bytes: Uint8Array) => Promise<string>;
+export type RagFileExtractor = (bytes: Uint8Array, fileName: string, opts?: {
+    readonly signal?: AbortSignal;
+}) => Promise<string>;
+export type RagExtractors = {
+    readonly pdf?: RagTextExtractor;
+    readonly office?: RagFileExtractor;
+};
+export type RagExtractOptions = {
+    readonly extractors?: RagExtractors;
+    readonly signal?: AbortSignal;
+};
+export declare function extractRagDocumentFromBytes(input: RagFileInput & {
+    readonly bytes: Uint8Array;
+}, opts?: RagExtractOptions): Promise<RagDocumentInput>;