@helloxiaohu/plugin-mineru 0.0.11

package/dist/lib/mineru.tool.js

@@ -0,0 +1,132 @@
+import { tool } from '@langchain/core/tools';
+import { getCurrentTaskInput } from '@langchain/langgraph';
+import { getErrorMessage } from '@xpert-ai/plugin-sdk';
+import { z } from 'zod';
+import { MinerUClient } from './mineru.client.js';
+/**
+ * Build MinerU PDF parser tool
+ * This tool converts PDF files to markdown format using MinerU service
+ */
+export function buildMinerUTool(configService, resultParser, integration, fileSystem) {
+    return tool(async (input) => {
+        try {
+            const { fileUrl, filePath, fileName, isOcr, enableFormula, enableTable, language, modelVersion } = input;
+            if (!fileUrl && !filePath) {
+                throw new Error('Either fileUrl or filePath must be provided');
+            }
+            // Get workspace context from current task
+            const currentState = getCurrentTaskInput();
+            const workspacePath = currentState?.['sys']?.['volume'] ?? '/tmp/xpert';
+            const baseUrl = currentState?.['sys']?.['workspace_url'];
+            // Create MinerU client with integration and file system permissions
+            const mineruClient = new MinerUClient(configService, {
+                fileSystem,
+                integration,
+            });
+            // Determine file name if not provided
+            let finalFileName = fileName;
+            if (!finalFileName) {
+                if (fileUrl) {
+                    try {
+                        const url = new URL(fileUrl);
+                        finalFileName = url.pathname.split('/').pop() || 'document.pdf';
+                    }
+                    catch {
+                        finalFileName = 'document.pdf';
+                    }
+                }
+                else if (filePath) {
+                    const pathParts = filePath.split(/[/\\]/);
+                    finalFileName = pathParts[pathParts.length - 1] || 'document.pdf';
+                }
+                else {
+                    finalFileName = 'document.pdf';
+                }
+            }
+            // Create MinerU task
+            const { taskId } = await mineruClient.createTask({
+                url: fileUrl,
+                filePath: filePath,
+                fileName: finalFileName,
+                isOcr: isOcr ?? true,
+                enableFormula: enableFormula ?? true,
+                enableTable: enableTable ?? true,
+                language: language ?? 'ch',
+                modelVersion: modelVersion ?? 'pipeline',
+            });
+            let parsedResult;
+            if (mineruClient.serverType === 'self-hosted') {
+                // Self-hosted: get result immediately
+                const taskResult = mineruClient.getSelfHostedTask(taskId);
+                if (!taskResult) {
+                    throw new Error('Failed to get MinerU task result');
+                }
+                parsedResult = await resultParser.parseLocalTask(taskResult, taskId, {
+                    fileUrl,
+                    filePath,
+                    name: finalFileName,
+                    folder: workspacePath,
+                }, fileSystem);
+            }
+            else {
+                // Official API: wait for completion
+                const result = await mineruClient.waitForTask(taskId, 5 * 60 * 1000, 5000);
+                parsedResult = await resultParser.parseFromUrl(result.full_zip_url, taskId, {
+                    fileUrl,
+                    filePath,
+                    name: finalFileName,
+                    folder: workspacePath,
+                }, fileSystem);
+            }
+            // Build file artifacts from parsed result
+            const fileArtifacts = [];
+            if (parsedResult.metadata?.assets) {
+                for (const asset of parsedResult.metadata.assets) {
+                    if (asset.type === 'file' || asset.type === 'image') {
+                        const fileName = asset.filePath?.split(/[/\\]/).pop() || asset.url?.split('/').pop() || 'file';
+                        const extension = fileName.split('.').pop()?.toLowerCase() || 'md';
+                        const mimeType = asset.type === 'image'
+                            ? (extension === 'png' ? 'image/png' : 'image/jpeg')
+                            : (extension === 'md' ? 'text/markdown' : 'application/json');
+                        fileArtifacts.push({
+                            fileName: fileName,
+                            filePath: asset.filePath,
+                            fileUrl: asset.url,
+                            mimeType: mimeType,
+                            extension: extension,
+                        });
+                    }
+                }
+            }
+            // Extract markdown content from chunks
+            const markdownContent = parsedResult.chunks
+                ?.map((chunk) => chunk.pageContent)
+                .join('\n\n') || '';
+            return [
+                `PDF processed successfully by MinerU.\n\nTask ID: ${taskId}\n\nMarkdown Content:\n${markdownContent.substring(0, 1000)}${markdownContent.length > 1000 ? '...' : ''}`,
+                {
+                    files: fileArtifacts,
+                    taskId,
+                    metadata: parsedResult.metadata,
+                },
+            ];
+        }
+        catch (error) {
+            throw new Error(`MinerU processing failed: ${getErrorMessage(error)}`);
+        }
+    }, {
+        name: 'mineru_pdf_parser',
+        description: 'Convert PDF files to markdown format using MinerU. Supports OCR, formula recognition, and table extraction. Returns markdown content and extracted files (images, JSON, etc.).',
+        schema: z.object({
+            fileUrl: z.string().optional().nullable().describe('URL of the PDF file to process'),
+            filePath: z.string().optional().nullable().describe('Local file path of the PDF file'),
+            fileName: z.string().optional().nullable().describe('Name of the PDF file'),
+            isOcr: z.boolean().optional().nullable().describe('Enable OCR for image-based PDFs (default: true)'),
+            enableFormula: z.boolean().optional().nullable().describe('Enable formula recognition (default: true)'),
+            enableTable: z.boolean().optional().nullable().describe('Enable table recognition (default: true)'),
+            language: z.enum(['en', 'ch']).optional().nullable().describe('Document language: "en" for English, "ch" for Chinese (default: "ch")'),
+            modelVersion: z.enum(['pipeline', 'vlm']).optional().nullable().describe('Model version: "pipeline" or "vlm" (default: "pipeline")'),
+        }),
+        responseFormat: 'content_and_artifact',
+    });
+}

package/dist/lib/mineru.toolset.d.ts

@@ -0,0 +1,40 @@
+import { StructuredToolInterface, ToolSchemaBase } from '@langchain/core/tools';
+import { BuiltinToolset, XpFileSystem } from '@xpert-ai/plugin-sdk';
+import { ConfigService } from '@nestjs/config';
+import { MinerUResultParserService } from './result-parser.service.js';
+import { IIntegration } from '@metad/contracts';
+import { MinerUIntegrationOptions } from './types.js';
+/**
+ * Configuration for MinerU Toolset
+ */
+export interface MinerUToolsetConfig {
+    integration?: Partial<IIntegration<MinerUIntegrationOptions>>;
+    fileSystem?: XpFileSystem;
+    configService?: ConfigService;
+    resultParser?: MinerUResultParserService;
+}
+/**
+ * MinerU Toolset implementation
+ * Provides PDF to markdown conversion tool using MinerU service
+ */
+export declare class MinerUToolset extends BuiltinToolset<StructuredToolInterface, MinerUToolsetConfig> {
+    private readonly config;
+    /**
+     * Constructor for MinerU Toolset
+     * Accepts config which contains credentials and dependencies
+     * Note: Using 'as any' for params because TBuiltinToolsetParams requires system-provided
+     * properties (tenantId, env) that are added at runtime
+     */
+    constructor(config: MinerUToolsetConfig);
+    /**
+     * Validate credentials for MinerU toolset
+     * Requires integration configuration to be provided
+     */
+    _validateCredentials(credentials: MinerUToolsetConfig): Promise<void>;
+    /**
+     * Initialize tools for MinerU toolset
+     * Creates the PDF parser tool with necessary dependencies
+     */
+    initTools(): Promise<StructuredToolInterface<ToolSchemaBase, any, any>[]>;
+}
+//# sourceMappingURL=mineru.toolset.d.ts.map

package/dist/lib/mineru.toolset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mineru.toolset.d.ts","sourceRoot":"","sources":["../../src/lib/mineru.toolset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAC9D,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,yBAAyB,CAAC;CAC1C;AAED;;;GAGG;AACH,qBAAa,aAAc,SAAQ,cAAc,CAAC,uBAAuB,EAAE,mBAAmB,CAAC;IAC7F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C;;;;;OAKG;gBACS,MAAM,EAAE,mBAAmB;IAKvC;;;OAGG;IACY,oBAAoB,CAAC,WAAW,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpF;;;OAGG;IACY,SAAS,IAAI,OAAO,CAAC,uBAAuB,CAAC,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;CAezF"}

package/dist/lib/mineru.toolset.js

@@ -0,0 +1,47 @@
+import { BuiltinToolset } from '@xpert-ai/plugin-sdk';
+import { buildMinerUTool } from './mineru.tool.js';
+/**
+ * MinerU Toolset implementation
+ * Provides PDF to markdown conversion tool using MinerU service
+ */
+export class MinerUToolset extends BuiltinToolset {
+    /**
+     * Constructor for MinerU Toolset
+     * Accepts config which contains credentials and dependencies
+     * Note: Using 'as any' for params because TBuiltinToolsetParams requires system-provided
+     * properties (tenantId, env) that are added at runtime
+     */
+    constructor(config) {
+        super('mineru', undefined, config);
+        this.config = config;
+    }
+    /**
+     * Validate credentials for MinerU toolset
+     * Requires integration configuration to be provided
+     */
+    async _validateCredentials(credentials) {
+        if (!credentials.integration) {
+            throw new Error('MinerU integration is required');
+        }
+        if (!credentials.configService) {
+            throw new Error('ConfigService is required');
+        }
+        if (!credentials.resultParser) {
+            throw new Error('MinerUResultParserService is required');
+        }
+    }
+    /**
+     * Initialize tools for MinerU toolset
+     * Creates the PDF parser tool with necessary dependencies
+     */
+    async initTools() {
+        const { configService, resultParser, integration, fileSystem } = this.config;
+        if (!configService || !resultParser) {
+            throw new Error('ConfigService and MinerUResultParserService are required');
+        }
+        this.tools = [
+            buildMinerUTool(configService, resultParser, integration, fileSystem),
+        ];
+        return this.tools;
+    }
+}

package/dist/lib/path-meta.d.ts

@@ -0,0 +1,5 @@
+export declare function getModuleMeta(meta?: ImportMeta): {
+    __filename: string;
+    __dirname: string;
+};
+//# sourceMappingURL=path-meta.d.ts.map

package/dist/lib/path-meta.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"path-meta.d.ts","sourceRoot":"","sources":["../../src/lib/path-meta.ts"],"names":[],"mappings":"AAGA,wBAAgB,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU;;;EAK9C"}

package/dist/lib/path-meta.js

@@ -0,0 +1,8 @@
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+export function getModuleMeta(meta) {
+    const isESM = typeof require === 'undefined';
+    const filename = isESM ? fileURLToPath(meta.url) : __filename;
+    const dir = isESM ? dirname(filename) : __dirname;
+    return { __filename: filename, __dirname: dir };
+}

package/dist/lib/result-parser.service.d.ts

@@ -0,0 +1,18 @@
+import { Document } from '@langchain/core/documents';
+import { IKnowledgeDocument } from '@metad/contracts';
+import { ChunkMetadata, XpFileSystem } from '@xpert-ai/plugin-sdk';
+import { MinerUDocumentMetadata, MineruSelfHostedTaskResult } from './types.js';
+export declare class MinerUResultParserService {
+    private readonly logger;
+    parseFromUrl(fullZipUrl: string, taskId: string, document: Partial<IKnowledgeDocument>, fileSystem: XpFileSystem): Promise<{
+        id?: string;
+        chunks: Document<ChunkMetadata>[];
+        metadata: MinerUDocumentMetadata;
+    }>;
+    parseLocalTask(result: MineruSelfHostedTaskResult, taskId: string, document: Partial<IKnowledgeDocument>, fileSystem: XpFileSystem): Promise<{
+        id?: string;
+        chunks: Document<ChunkMetadata>[];
+        metadata: MinerUDocumentMetadata;
+    }>;
+}
+//# sourceMappingURL=result-parser.service.d.ts.map

package/dist/lib/result-parser.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"result-parser.service.d.ts","sourceRoot":"","sources":["../../src/lib/result-parser.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EACL,aAAa,EAEb,YAAY,EACb,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAEL,sBAAsB,EACtB,0BAA0B,EAC3B,MAAM,YAAY,CAAC;AAEpB,qBACa,yBAAyB;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8C;IAE/D,YAAY,CAChB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,UAAU,EAAE,YAAY,GACvB,OAAO,CAAC;QACT,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,QAAQ,EAAE,sBAAsB,CAAC;KAClC,CAAC;IAqFI,cAAc,CAClB,MAAM,EAAE,0BAA0B,EAClC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,UAAU,EAAE,YAAY,GACvB,OAAO,CAAC;QACT,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,QAAQ,EAAE,sBAAsB,CAAC;KAClC,CAAC;CAkDH"}

package/dist/lib/result-parser.service.js

@@ -0,0 +1,142 @@
+var MinerUResultParserService_1;
+import { __decorate } from "tslib";
+import { Document } from '@langchain/core/documents';
+import { Injectable, Logger } from '@nestjs/common';
+import axios from 'axios';
+import { join } from 'path';
+import unzipper from 'unzipper';
+import { v4 as uuidv4 } from 'uuid';
+import { MinerU, } from './types.js';
+let MinerUResultParserService = MinerUResultParserService_1 = class MinerUResultParserService {
+    constructor() {
+        this.logger = new Logger(MinerUResultParserService_1.name);
+    }
+    async parseFromUrl(fullZipUrl, taskId, document, fileSystem) {
+        this.logger.debug(`Downloading MinerU result from: ${fullZipUrl}`);
+        // 1. Download the zip file to memory
+        const response = await axios.get(fullZipUrl, {
+            responseType: 'arraybuffer',
+        });
+        const zipBuffer = Buffer.from(response.data);
+        const metadata = {
+            parser: MinerU,
+            taskId,
+        };
+        // 2. Unzip the file
+        const zipEntries = [];
+        const assets = [];
+        const directory = await unzipper.Open.buffer(zipBuffer);
+        const pathMap = new Map();
+        let fullMd = '';
+        let layoutJson = null;
+        for (const entry of directory.files) {
+            if (!entry.type || entry.type !== 'File')
+                continue;
+            const data = await entry.buffer();
+            zipEntries.push({ entryName: entry.path, data });
+            const fileName = entry.path;
+            const filePath = join(document.folder || '', entry.path);
+            const url = await fileSystem.writeFile(filePath, data);
+            pathMap.set(fileName, url);
+            // Write images to local file system
+            if (fileName.startsWith('images/')) {
+                assets.push({
+                    type: 'image',
+                    url: url,
+                    filePath: filePath,
+                });
+            }
+            else if (fileName.endsWith('layout.json')) {
+                layoutJson = JSON.parse(data.toString('utf-8'));
+                metadata.mineruBackend = layoutJson?._backend;
+                metadata.mineruVersion = layoutJson?._version_name;
+                assets.push({
+                    type: 'file',
+                    url,
+                    filePath: filePath,
+                });
+            }
+            else if (fileName.endsWith('content_list.json')) {
+                assets.push({
+                    type: 'file',
+                    url,
+                    filePath: filePath,
+                });
+            }
+            else if (fileName.endsWith('full.md')) {
+                fullMd = data.toString('utf-8');
+                assets.push({
+                    type: 'file',
+                    url,
+                    filePath: filePath,
+                });
+            }
+            else if (fileName.endsWith('origin.pdf')) {
+                metadata.originPdfUrl = fileName;
+            }
+        }
+        metadata.assets = assets;
+        // 3. Replace image relative path in full.md with file url
+        fullMd = fullMd.replace(/!\[(.*)\]\((images\/.+?)\)/g, (match, p1, p2) => {
+            const localPath = pathMap.get(p2);
+            return localPath ? `![${p1}](${localPath})` : match;
+        });
+        const chunks = [
+            new Document({
+                pageContent: fullMd,
+                metadata: { parser: MinerU, taskId, chunkId: uuidv4() },
+            }),
+        ];
+        return {
+            chunks,
+            metadata,
+        };
+    }
+    async parseLocalTask(result, taskId, document, fileSystem) {
+        const metadata = {
+            parser: MinerU,
+            taskId,
+        };
+        const assets = [];
+        const pathMap = new Map();
+        for (const image of result.images) {
+            const filePath = join(document.folder || '', 'images', image.name);
+            const url = await fileSystem.writeFile(filePath, Buffer.from(image.dataUrl.split(',')[1], 'base64'));
+            pathMap.set(`images/${image.name}`, url);
+            assets.push({
+                type: 'image',
+                url: url,
+                filePath: filePath,
+            });
+        }
+        if (result.sourceUrl) {
+            assets.push({
+                type: 'file',
+                url: result.sourceUrl,
+                filePath: join(document.folder || '', result.fileName || 'source.pdf'),
+            });
+            metadata.originPdfUrl = result.sourceUrl;
+        }
+        metadata.assets = assets;
+        let fullMd = result.mdContent;
+        // 3. Replace image relative path in full.md with file url
+        fullMd = fullMd.replace(/!\[(.*)\]\((images\/.+?)\)/g, (match, p1, p2) => {
+            const localPath = pathMap.get(p2);
+            return localPath ? `![${p1}](${localPath})` : match;
+        });
+        const chunks = [
+            new Document({
+                pageContent: fullMd,
+                metadata: { parser: MinerU, taskId, chunkId: uuidv4() },
+            }),
+        ];
+        return {
+            chunks,
+            metadata,
+        };
+    }
+};
+MinerUResultParserService = MinerUResultParserService_1 = __decorate([
+    Injectable()
+], MinerUResultParserService);
+export { MinerUResultParserService };

package/dist/lib/transformer-mineru.strategy.d.ts

@@ -0,0 +1,95 @@
+import { IconType, IKnowledgeDocument } from '@metad/contracts';
+import { ChunkMetadata, FileSystemPermission, IDocumentTransformerStrategy, IntegrationPermission } from '@xpert-ai/plugin-sdk';
+import { TMinerUTransformerConfig } from './types.js';
+export declare class MinerUTransformerStrategy implements IDocumentTransformerStrategy<TMinerUTransformerConfig> {
+    private readonly resultParser;
+    private readonly configService;
+    readonly permissions: (IntegrationPermission | FileSystemPermission)[];
+    readonly meta: {
+        name: string;
+        label: {
+            en_US: string;
+            zh_Hans: string;
+        };
+        description: {
+            en_US: string;
+            zh_Hans: string;
+        };
+        icon: {
+            type: IconType;
+            value: string;
+            color: string;
+        };
+        helpUrl: string;
+        configSchema: {
+            type: string;
+            properties: {
+                isOcr: {
+                    type: string;
+                    title: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    description: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    default: boolean;
+                };
+                enableFormula: {
+                    type: string;
+                    title: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    description: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    default: boolean;
+                };
+                enableTable: {
+                    type: string;
+                    title: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    description: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    default: boolean;
+                };
+                language: {
+                    type: string;
+                    title: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    description: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    default: string;
+                };
+                modelVersion: {
+                    type: string;
+                    title: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    description: {
+                        en_US: string;
+                        zh_Hans: string;
+                    };
+                    enum: string[];
+                    default: string;
+                };
+            };
+            required: any[];
+        };
+    };
+    validateConfig(config: any): Promise<void>;
+    transformDocuments(documents: Partial<IKnowledgeDocument>[], config: TMinerUTransformerConfig): Promise<Partial<IKnowledgeDocument<ChunkMetadata>>[]>;
+}
+//# sourceMappingURL=transformer-mineru.strategy.d.ts.map

package/dist/lib/transformer-mineru.strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transformer-mineru.strategy.d.ts","sourceRoot":"","sources":["../../src/lib/transformer-mineru.strategy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAG/D,OAAO,EACL,aAAa,EAEb,oBAAoB,EACpB,4BAA4B,EAC5B,qBAAqB,EACtB,MAAM,sBAAsB,CAAA;AAI7B,OAAO,EAAgB,wBAAwB,EAAE,MAAM,YAAY,CAAA;AAEnE,qBAEa,yBAA0B,YAAW,4BAA4B,CAAC,wBAAwB,CAAC;IAEtG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA2B;IAGxD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;IAE7C,QAAQ,CAAC,WAAW,mDAWnB;IAED,QAAQ,CAAC,IAAI;;;;;;;;;;;kBAWM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAwE1B;IAED,cAAc,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpC,kBAAkB,CACtB,SAAS,EAAE,OAAO,CAAC,kBAAkB,CAAC,EAAE,EACxC,MAAM,EAAE,wBAAwB,GAC/B,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;CAsDzD"}

package/dist/lib/transformer-mineru.strategy.js

@@ -0,0 +1,163 @@
+import { __decorate, __metadata } from "tslib";
+import { forwardRef, Inject, Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { DocumentTransformerStrategy, } from '@xpert-ai/plugin-sdk';
+import { isNil, omitBy, pick } from 'lodash-es';
+import { MinerUClient } from './mineru.client.js';
+import { MinerUResultParserService } from './result-parser.service.js';
+import { icon, MinerU } from './types.js';
+let MinerUTransformerStrategy = class MinerUTransformerStrategy {
+    constructor() {
+        this.permissions = [
+            {
+                type: 'integration',
+                service: MinerU,
+                description: 'Access to MinerU system integrations'
+            },
+            {
+                type: 'filesystem',
+                operations: ['read', 'write', 'list'],
+                scope: []
+            }
+        ];
+        this.meta = {
+            name: MinerU,
+            label: {
+                en_US: 'MinerU',
+                zh_Hans: 'MinerU'
+            },
+            description: {
+                en_US: 'A high-quality tool for convert PDF to Markdown and JSON.',
+                zh_Hans: '一站式开源高质量数据提取工具，将PDF转换成Markdown和JSON格式。'
+            },
+            icon: {
+                type: 'svg',
+                value: icon,
+                color: '#14b8a6'
+            },
+            helpUrl: 'https://mineru.net/apiManage/docs',
+            configSchema: {
+                type: 'object',
+                properties: {
+                    isOcr: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable OCR',
+                            zh_Hans: '启用 OCR'
+                        },
+                        description: {
+                            en_US: 'Enable OCR for image-based PDFs.',
+                            zh_Hans: '对基于图像的 PDF 启用 OCR。'
+                        },
+                        default: true
+                    },
+                    enableFormula: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable Formula Recognition',
+                            zh_Hans: '启用公式识别'
+                        },
+                        description: {
+                            en_US: 'Enable recognition of mathematical formulas in documents.',
+                            zh_Hans: '启用对文档中数学公式的识别。'
+                        },
+                        default: true
+                    },
+                    enableTable: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable Table Recognition',
+                            zh_Hans: '启用表格识别'
+                        },
+                        description: {
+                            en_US: 'Enable recognition of tables in documents.',
+                            zh_Hans: '启用对文档中表格的识别。'
+                        },
+                        default: true
+                    },
+                    language: {
+                        type: 'string',
+                        title: {
+                            en_US: 'Document Language',
+                            zh_Hans: '文档语言'
+                        },
+                        description: {
+                            en_US: 'The primary language of the document (e.g., "en" for English, "ch" for Chinese).',
+                            zh_Hans: '文档的主要语言（例如，英文为 "en"，中文为 "ch"）。'
+                        },
+                        default: 'ch'
+                    },
+                    modelVersion: {
+                        type: 'string',
+                        title: {
+                            en_US: 'Model Version',
+                            zh_Hans: '模型版本'
+                        },
+                        description: {
+                            en_US: 'The model version to use for extraction (e.g., "vlm" or "pipeline").',
+                            zh_Hans: '用于提取的模型版本（例如，“vlm”或“pipeline”）。'
+                        },
+                        enum: ['pipeline', 'vlm'],
+                        default: 'pipeline'
+                    }
+                },
+                required: []
+            }
+        };
+    }
+    validateConfig(config) {
+        throw new Error('Method not implemented.');
+    }
+    async transformDocuments(documents, config) {
+        const mineru = new MinerUClient(this.configService, config.permissions);
+        const parsedResults = [];
+        for await (const document of documents) {
+            if (mineru.serverType === 'self-hosted') {
+                const { taskId } = await mineru.createTask({
+                    url: document.fileUrl,
+                    filePath: document.filePath,
+                    fileName: document.name,
+                    isOcr: true,
+                    enableFormula: true,
+                    enableTable: true,
+                    // language: 'ch',
+                    // modelVersion: 'vlm'
+                });
+                const result = mineru.getSelfHostedTask(taskId);
+                const parsedResult = await this.resultParser.parseLocalTask(result, taskId, document, config.permissions.fileSystem);
+                parsedResult.id = document.id;
+                parsedResults.push(parsedResult);
+            }
+            else {
+                const { taskId } = await mineru.createTask({
+                    url: document.fileUrl,
+                    isOcr: true,
+                    enableFormula: true,
+                    enableTable: true,
+                    language: 'ch',
+                    modelVersion: 'vlm',
+                    ...omitBy(pick(config, ['isOcr', 'enableFormula', 'enableTable', 'language', 'modelVersion']), isNil)
+                });
+                // Waiting for completion
+                const result = await mineru.waitForTask(taskId, 5 * 60 * 1000, 5000);
+                const parsedResult = await this.resultParser.parseFromUrl(result.full_zip_url, taskId, document, config.permissions.fileSystem);
+                parsedResult.id = document.id;
+                parsedResults.push(parsedResult);
+            }
+        }
+        return parsedResults;
+    }
+};
+__decorate([
+    Inject(MinerUResultParserService),
+    __metadata("design:type", MinerUResultParserService)
+], MinerUTransformerStrategy.prototype, "resultParser", void 0);
+__decorate([
+    Inject(forwardRef(() => ConfigService)),
+    __metadata("design:type", ConfigService)
+], MinerUTransformerStrategy.prototype, "configService", void 0);
+MinerUTransformerStrategy = __decorate([
+    Injectable(),
+    DocumentTransformerStrategy(MinerU)
+], MinerUTransformerStrategy);
+export { MinerUTransformerStrategy };