@fllf/agent-sdk 0.1.0

package/dist/rag/ingestion/loaders.js

@@ -0,0 +1,653 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AutoDocumentLoader = exports.PdfToMarkdownLoader = exports.DocxToMarkdownLoader = exports.HtmlToMarkdownLoader = exports.MarkdownLoader = exports.PlainTextLoader = void 0;
+exports.createDefaultAutoDocumentLoaderRules = createDefaultAutoDocumentLoaderRules;
+const metadata_1 = require("./metadata");
+class PlainTextLoader {
+    metadataResolver;
+    defaultSource;
+    constructor(options = {}) {
+        this.metadataResolver = options.metadataResolver ?? new metadata_1.DefaultMetadataResolver();
+        this.defaultSource = options.defaultSource ?? 'inline';
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const content = requireInlineContent(input, 'PlainTextLoader');
+        const metadataInput = createMetadataInput({
+            content,
+            source: input.source ?? this.defaultSource,
+            metadata: withDefaultMimeType(input.metadata, 'text/plain'),
+            tenantId: input.tenantId,
+            knowledgeBaseId: input.knowledgeBaseId,
+            acl: input.acl,
+        });
+        const { id, metadata } = this.metadataResolver.resolve(metadataInput);
+        const document = {
+            id,
+            content,
+            metadata,
+            blocks: parsePlainTextBlocks(content),
+        };
+        return [document];
+    }
+}
+exports.PlainTextLoader = PlainTextLoader;
+class MarkdownLoader {
+    metadataResolver;
+    defaultSource;
+    constructor(options = {}) {
+        this.metadataResolver = options.metadataResolver ?? new metadata_1.DefaultMetadataResolver();
+        this.defaultSource = options.defaultSource ?? 'inline';
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const content = requireInlineContent(input, 'MarkdownLoader');
+        const blocks = parseMarkdownBlocks(content);
+        const metadataInput = createMetadataInput({
+            content,
+            source: input.source ?? this.defaultSource,
+            metadata: withDefaultMimeType(input.metadata, 'text/markdown'),
+            title: inferTitleFromBlocks(blocks),
+            tenantId: input.tenantId,
+            knowledgeBaseId: input.knowledgeBaseId,
+            acl: input.acl,
+        });
+        const { id, metadata } = this.metadataResolver.resolve(metadataInput);
+        const document = {
+            id,
+            content,
+            metadata,
+            blocks,
+        };
+        return [document];
+    }
+}
+exports.MarkdownLoader = MarkdownLoader;
+class HtmlToMarkdownLoader {
+    markdownLoader;
+    defaultSource;
+    convertToMarkdown;
+    constructor(options = {}) {
+        this.markdownLoader = new MarkdownLoader(options);
+        this.defaultSource = options.defaultSource ?? 'inline';
+        this.convertToMarkdown = options.convertToMarkdown ?? htmlToMarkdown;
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const html = requireInlineContent(input, 'HtmlToMarkdownLoader');
+        const markdown = normalizeConvertedMarkdown(await this.convertToMarkdown(html, input));
+        return await this.markdownLoader.load(createConvertedMarkdownInput({
+            input,
+            markdown,
+            source: input.source ?? this.defaultSource,
+            mimeType: 'text/html',
+        }));
+    }
+}
+exports.HtmlToMarkdownLoader = HtmlToMarkdownLoader;
+class DocxToMarkdownLoader {
+    markdownLoader;
+    defaultSource;
+    convertToMarkdown;
+    constructor(options = {}) {
+        this.markdownLoader = new MarkdownLoader(options);
+        this.defaultSource = options.defaultSource ?? 'inline';
+        this.convertToMarkdown = options.convertToMarkdown ?? docxToMarkdown;
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const data = requireBinaryData(input, 'DocxToMarkdownLoader');
+        const markdown = normalizeConvertedMarkdown(await this.convertToMarkdown(data, input));
+        return await this.markdownLoader.load(createConvertedMarkdownInput({
+            input,
+            markdown,
+            source: input.source ?? this.defaultSource,
+            mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        }));
+    }
+}
+exports.DocxToMarkdownLoader = DocxToMarkdownLoader;
+class PdfToMarkdownLoader {
+    markdownLoader;
+    defaultSource;
+    convertToMarkdown;
+    constructor(options = {}) {
+        this.markdownLoader = new MarkdownLoader(options);
+        this.defaultSource = options.defaultSource ?? 'inline';
+        this.convertToMarkdown = options.convertToMarkdown ?? pdfToMarkdown;
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const data = requireBinaryData(input, 'PdfToMarkdownLoader');
+        const markdown = normalizeConvertedMarkdown(await this.convertToMarkdown(data, input));
+        return await this.markdownLoader.load(createConvertedMarkdownInput({
+            input,
+            markdown,
+            source: input.source ?? this.defaultSource,
+            mimeType: 'application/pdf',
+        }));
+    }
+}
+exports.PdfToMarkdownLoader = PdfToMarkdownLoader;
+class AutoDocumentLoader {
+    rules;
+    fallbackLoader;
+    constructor(options = {}) {
+        this.rules = (options.rules ?? createDefaultAutoDocumentLoaderRules())
+            .map(normalizeAutoDocumentLoaderRule);
+        this.fallbackLoader = options.fallbackLoader ?? new PlainTextLoader();
+    }
+    async load(input) {
+        if (input.documents !== undefined) {
+            return input.documents.map(cloneRawDocument);
+        }
+        const loader = this.resolveLoader(input);
+        return await loader.load(createRoutedLoaderInput(input));
+    }
+    resolveLoader(input) {
+        const mimeType = getInputMimeType(input);
+        const source = getInputSource(input);
+        const extension = getSourceExtension(source);
+        if (mimeType !== undefined && !isGenericMimeType(mimeType)) {
+            const loader = this.findLoaderByMimeType(mimeType);
+            if (loader !== undefined) {
+                return loader;
+            }
+        }
+        if (extension !== undefined) {
+            const loader = this.findLoaderByExtension(extension);
+            if (loader !== undefined) {
+                return loader;
+            }
+        }
+        if (mimeType !== undefined) {
+            const loader = this.findLoaderByMimeType(mimeType);
+            if (loader !== undefined) {
+                return loader;
+            }
+        }
+        return this.fallbackLoader;
+    }
+    findLoaderByMimeType(mimeType) {
+        return this.rules.find((rule) => rule.mimeTypes.has(mimeType))?.loader;
+    }
+    findLoaderByExtension(extension) {
+        return this.rules.find((rule) => rule.extensions.has(extension))?.loader;
+    }
+}
+exports.AutoDocumentLoader = AutoDocumentLoader;
+function createDefaultAutoDocumentLoaderRules() {
+    return [
+        {
+            name: 'markdown',
+            loader: new MarkdownLoader(),
+            mimeTypes: ['text/markdown', 'text/x-markdown'],
+            extensions: ['.md', '.markdown', '.mdown', '.mkd'],
+        },
+        {
+            name: 'html',
+            loader: new HtmlToMarkdownLoader(),
+            mimeTypes: ['text/html', 'application/xhtml+xml'],
+            extensions: ['.html', '.htm', '.xhtml'],
+        },
+        {
+            name: 'docx',
+            loader: new DocxToMarkdownLoader(),
+            mimeTypes: [
+                'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            ],
+            extensions: ['.docx'],
+        },
+        {
+            name: 'pdf',
+            loader: new PdfToMarkdownLoader(),
+            mimeTypes: ['application/pdf'],
+            extensions: ['.pdf'],
+        },
+        {
+            name: 'text',
+            loader: new PlainTextLoader(),
+            mimeTypes: ['text/plain', 'text/csv'],
+            extensions: ['.txt', '.text', '.log', '.csv'],
+        },
+    ];
+}
+function requireInlineContent(input, loaderName) {
+    if (input.content === undefined) {
+        throw new Error(`${loaderName} requires input.content or input.documents.`);
+    }
+    return input.content;
+}
+function requireBinaryData(input, loaderName) {
+    if (input.data === undefined) {
+        throw new Error(`${loaderName} requires input.data or input.documents.`);
+    }
+    return toUint8Array(input.data);
+}
+function normalizeAutoDocumentLoaderRule(rule) {
+    return {
+        name: rule.name,
+        loader: rule.loader,
+        mimeTypes: normalizeMimeTypes(rule.mimeTypes),
+        extensions: normalizeExtensions(rule.extensions),
+    };
+}
+function normalizeMimeTypes(mimeTypes) {
+    const normalized = new Set();
+    for (const mimeType of mimeTypes ?? []) {
+        const value = normalizeMimeType(mimeType);
+        if (value !== undefined) {
+            normalized.add(value);
+        }
+    }
+    return normalized;
+}
+function normalizeExtensions(extensions) {
+    const normalized = new Set();
+    for (const extension of extensions ?? []) {
+        const value = normalizeExtension(extension);
+        if (value !== undefined) {
+            normalized.add(value);
+        }
+    }
+    return normalized;
+}
+function getInputMimeType(input) {
+    return normalizeMimeType(input.metadata?.mimeType ?? input.mimeType);
+}
+function getInputSource(input) {
+    return input.source ?? input.metadata?.source;
+}
+function createRoutedLoaderInput(input) {
+    const mimeType = normalizeMimeType(input.mimeType);
+    if (input.metadata?.mimeType !== undefined
+        || mimeType === undefined
+        || isGenericMimeType(mimeType)) {
+        return input;
+    }
+    return {
+        ...input,
+        metadata: {
+            ...(input.metadata ?? {}),
+            mimeType,
+        },
+    };
+}
+function getSourceExtension(source) {
+    if (source === undefined) {
+        return undefined;
+    }
+    const withoutQuery = source.split(/[?#]/, 1)[0] ?? source;
+    const normalizedPath = withoutQuery.replace(/\\/g, '/');
+    const filename = normalizedPath.split('/').filter(Boolean).at(-1);
+    if (filename === undefined) {
+        return undefined;
+    }
+    const dotIndex = filename.lastIndexOf('.');
+    if (dotIndex <= 0 || dotIndex === filename.length - 1) {
+        return undefined;
+    }
+    return filename.slice(dotIndex).toLowerCase();
+}
+function normalizeMimeType(mimeType) {
+    const normalized = (mimeType?.split(';', 1)[0] ?? '').trim().toLowerCase();
+    return normalized.length > 0 ? normalized : undefined;
+}
+function normalizeExtension(extension) {
+    const normalized = extension.trim().toLowerCase();
+    if (normalized.length === 0) {
+        return undefined;
+    }
+    return normalized.startsWith('.') ? normalized : `.${normalized}`;
+}
+function isGenericMimeType(mimeType) {
+    return mimeType === 'text/plain' || mimeType === 'application/octet-stream';
+}
+function withDefaultMimeType(metadata, mimeType) {
+    return {
+        ...metadata,
+        mimeType: metadata?.mimeType ?? mimeType,
+    };
+}
+function createConvertedMarkdownInput(draft) {
+    const metadata = {
+        ...(draft.input.metadata ?? {}),
+        mimeType: draft.input.metadata?.mimeType ?? draft.mimeType,
+    };
+    metadata.extra = {
+        ...(draft.input.metadata?.extra ?? {}),
+        convertedToMimeType: 'text/markdown',
+        originalMimeType: draft.mimeType,
+    };
+    const input = {
+        content: draft.markdown,
+        source: draft.source,
+        metadata,
+    };
+    assignIfDefined(input, 'tenantId', draft.input.tenantId);
+    assignIfDefined(input, 'knowledgeBaseId', draft.input.knowledgeBaseId);
+    assignIfDefined(input, 'acl', draft.input.acl);
+    assignIfDefined(input, 'replaceExisting', draft.input.replaceExisting);
+    return input;
+}
+async function htmlToMarkdown(html) {
+    const service = await createTurndownService();
+    const title = extractHtmlTitle(html);
+    const markdown = service.turndown(stripHtmlNoise(html));
+    if (title !== undefined && !hasMarkdownHeading(markdown)) {
+        return `# ${escapeMarkdownHeading(title)}\n\n${markdown}`;
+    }
+    return markdown;
+}
+async function docxToMarkdown(data) {
+    const mammoth = await importRuntimeDependency('mammoth', 'DocxToMarkdownLoader');
+    const converter = readProperty(mammoth, 'convertToHtml');
+    if (typeof converter !== 'function') {
+        throw new Error('DocxToMarkdownLoader dependency "mammoth" does not expose convertToHtml().');
+    }
+    const result = await converter({ buffer: toBuffer(data) });
+    const html = typeof result.value === 'string' ? result.value : '';
+    return await htmlToMarkdown(html);
+}
+async function pdfToMarkdown(data) {
+    const pdfParseModule = await importRuntimeDependency('pdf-parse', 'PdfToMarkdownLoader');
+    const parser = readDefaultExport(pdfParseModule);
+    if (typeof parser !== 'function') {
+        throw new Error('PdfToMarkdownLoader dependency "pdf-parse" does not expose a parser function.');
+    }
+    const result = await parser(toBuffer(data));
+    const text = typeof result.text === 'string' ? result.text : '';
+    return plainTextToMarkdown(text, typeof result.numpages === 'number' ? result.numpages : undefined);
+}
+async function createTurndownService() {
+    const turndown = await importRuntimeDependency('turndown', 'HtmlToMarkdownLoader');
+    const plugin = await importRuntimeDependency('turndown-plugin-gfm', 'HtmlToMarkdownLoader');
+    const Service = readDefaultExport(turndown);
+    if (typeof Service !== 'function') {
+        throw new Error('HtmlToMarkdownLoader dependency "turndown" does not expose a constructor.');
+    }
+    const TurndownService = Service;
+    const service = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        bulletListMarker: '-',
+    });
+    const gfm = readProperty(plugin, 'gfm');
+    if (typeof gfm === 'function') {
+        service.use(gfm);
+    }
+    return service;
+}
+async function importRuntimeDependency(name, loaderName) {
+    try {
+        return await import(name);
+    }
+    catch (error) {
+        throw new Error(`${loaderName} requires dependency "${name}". Install it with: npm install ${name}`, { cause: error });
+    }
+}
+function readDefaultExport(module) {
+    const defaultExport = readProperty(module, 'default');
+    return defaultExport ?? module;
+}
+function readProperty(value, key) {
+    return isRecord(value) ? value[key] : undefined;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function stripHtmlNoise(html) {
+    return html
+        .replace(/<!--[\s\S]*?-->/g, '')
+        .replace(/<(script|style|noscript)\b[\s\S]*?<\/\1>/gi, '');
+}
+function extractHtmlTitle(html) {
+    const match = html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i);
+    const title = match?.[1]
+        ?.replace(/<[^>]+>/g, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+    return title && title.length > 0 ? title : undefined;
+}
+function hasMarkdownHeading(markdown) {
+    return /^\s{0,3}#{1,6}\s+\S/m.test(markdown);
+}
+function escapeMarkdownHeading(text) {
+    return text
+        .replace(/^#+\s*/, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function plainTextToMarkdown(text, pageCount) {
+    const normalized = normalizeConvertedMarkdown(text);
+    const pages = normalized
+        .split(/\f/g)
+        .map((page) => page.trim())
+        .filter((page) => page.length > 0);
+    if (pages.length > 1) {
+        return pages
+            .map((page, index) => `## Page ${index + 1}\n\n${page}`)
+            .join('\n\n');
+    }
+    if (pageCount !== undefined && pageCount > 1 && normalized.length > 0) {
+        return `## PDF Text\n\n${normalized}`;
+    }
+    return normalized;
+}
+function normalizeConvertedMarkdown(markdown) {
+    return normalizeLineEndings(markdown)
+        .replace(/[ \t]+\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+function toUint8Array(data) {
+    if (data instanceof Uint8Array) {
+        return data;
+    }
+    return new Uint8Array(data);
+}
+function toBuffer(data) {
+    return Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+}
+function parsePlainTextBlocks(content) {
+    return normalizeLineEndings(content)
+        .split(/\n{2,}/)
+        .map((paragraph) => paragraph.trim())
+        .filter((paragraph) => paragraph.length > 0)
+        .map((text) => ({ type: 'paragraph', text }));
+}
+/**
+ * MarkdownLoader 只做轻量结构识别，不尝试成为完整 Markdown 解析器。
+ * 它保留标题、表格、代码块和段落边界，后续 chunker 可以基于 blocks 继续切割。
+ */
+function parseMarkdownBlocks(content) {
+    const lines = normalizeLineEndings(content).split('\n');
+    const blocks = [];
+    let index = 0;
+    while (index < lines.length) {
+        const line = lines[index] ?? '';
+        if (line.trim().length === 0) {
+            index += 1;
+            continue;
+        }
+        const fence = parseFenceStart(line);
+        if (fence !== undefined) {
+            const codeLines = [];
+            index += 1;
+            while (index < lines.length) {
+                const current = lines[index] ?? '';
+                if (current.trim().startsWith(fence.marker)) {
+                    index += 1;
+                    break;
+                }
+                codeLines.push(current);
+                index += 1;
+            }
+            const codeBlock = {
+                type: 'code',
+                code: codeLines.join('\n'),
+            };
+            if (fence.language !== undefined) {
+                codeBlock.language = fence.language;
+            }
+            blocks.push(codeBlock);
+            continue;
+        }
+        const heading = line.match(/^(#{1,6})\s+(.+?)\s*#*$/);
+        if (heading !== null) {
+            blocks.push({
+                type: 'heading',
+                level: heading[1]?.length ?? 1,
+                text: heading[2]?.trim() ?? '',
+            });
+            index += 1;
+            continue;
+        }
+        if (isTableStart(lines, index)) {
+            const tableLines = [];
+            while (index < lines.length && isPipeLine(lines[index] ?? '')) {
+                tableLines.push((lines[index] ?? '').trim());
+                index += 1;
+            }
+            blocks.push({
+                type: 'table',
+                markdown: tableLines.join('\n'),
+            });
+            continue;
+        }
+        const paragraphLines = [];
+        while (index < lines.length) {
+            const current = lines[index] ?? '';
+            if (current.trim().length === 0
+                || parseFenceStart(current) !== undefined
+                || current.match(/^(#{1,6})\s+(.+?)\s*#*$/) !== null
+                || isTableStart(lines, index)) {
+                break;
+            }
+            paragraphLines.push(current.trim());
+            index += 1;
+        }
+        if (paragraphLines.length > 0) {
+            blocks.push({
+                type: 'paragraph',
+                text: paragraphLines.join('\n'),
+            });
+        }
+    }
+    return blocks;
+}
+function normalizeLineEndings(content) {
+    return content.replace(/\r\n?/g, '\n');
+}
+function parseFenceStart(line) {
+    const match = line.trim().match(/^(```|~~~)\s*([A-Za-z0-9_+.-]+)?\s*$/);
+    if (match === null) {
+        return undefined;
+    }
+    const result = {
+        marker: match[1] ?? '```',
+    };
+    const language = match[2]?.trim();
+    if (language !== undefined && language.length > 0) {
+        result.language = language;
+    }
+    return result;
+}
+function isTableStart(lines, index) {
+    const current = lines[index] ?? '';
+    const next = lines[index + 1] ?? '';
+    return isPipeLine(current) && isTableSeparator(next);
+}
+function isPipeLine(line) {
+    return line.includes('|') && line.trim().length > 0;
+}
+function isTableSeparator(line) {
+    const cells = line.trim().split('|').map((cell) => cell.trim()).filter(Boolean);
+    return cells.length > 0 && cells.every((cell) => /^:?-{3,}:?$/.test(cell));
+}
+function inferTitleFromBlocks(blocks) {
+    const firstHeading = blocks.find((block) => block.type === 'heading');
+    return firstHeading?.type === 'heading' ? firstHeading.text : undefined;
+}
+function createMetadataInput(draft) {
+    const input = {
+        content: draft.content,
+    };
+    assignIfDefined(input, 'source', draft.source);
+    assignIfDefined(input, 'metadata', draft.metadata);
+    assignIfDefined(input, 'title', draft.title);
+    assignIfDefined(input, 'mimeType', draft.mimeType);
+    assignIfDefined(input, 'tenantId', draft.tenantId);
+    assignIfDefined(input, 'knowledgeBaseId', draft.knowledgeBaseId);
+    assignIfDefined(input, 'acl', draft.acl);
+    return input;
+}
+function cloneRawDocument(document) {
+    const clone = {
+        id: document.id,
+        content: document.content,
+        metadata: cloneMetadata(document.metadata),
+    };
+    if (document.blocks !== undefined) {
+        clone.blocks = document.blocks.map(cloneBlock);
+    }
+    return clone;
+}
+function cloneMetadata(metadata) {
+    const clone = {
+        source: metadata.source,
+    };
+    assignIfDefined(clone, 'title', metadata.title);
+    assignIfDefined(clone, 'mimeType', metadata.mimeType);
+    assignIfDefined(clone, 'tenantId', metadata.tenantId);
+    assignIfDefined(clone, 'knowledgeBaseId', metadata.knowledgeBaseId);
+    assignIfDefined(clone, 'version', metadata.version);
+    assignIfDefined(clone, 'checksum', metadata.checksum);
+    assignIfDefined(clone, 'createdAt', metadata.createdAt);
+    assignIfDefined(clone, 'updatedAt', metadata.updatedAt);
+    if (metadata.acl !== undefined) {
+        clone.acl = [...metadata.acl];
+    }
+    if (metadata.extra !== undefined) {
+        clone.extra = { ...metadata.extra };
+    }
+    return clone;
+}
+function cloneBlock(block) {
+    switch (block.type) {
+        case 'heading':
+            return { ...block };
+        case 'paragraph':
+            return { ...block };
+        case 'table': {
+            const clone = {
+                type: 'table',
+                markdown: block.markdown,
+            };
+            if (block.rows !== undefined) {
+                clone.rows = [...block.rows];
+            }
+            return clone;
+        }
+        case 'code':
+            return { ...block };
+        case 'image':
+            return { ...block };
+    }
+}
+function assignIfDefined(target, key, value) {
+    if (value !== undefined) {
+        target[key] = value;
+    }
+}
+//# sourceMappingURL=loaders.js.map