@helloxiaohu/plugin-mineru6 0.0.2

package/dist/lib/transformer-mineru.strategy.js

@@ -0,0 +1,163 @@
+import { __decorate, __metadata } from "tslib";
+import { forwardRef, Inject, Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { DocumentTransformerStrategy, } from '@xpert-ai/plugin-sdk';
+import { isNil, omitBy, pick } from 'lodash-es';
+import { MinerUClient } from './mineru.client.js';
+import { MinerUResultParserService } from './result-parser.service.js';
+import { icon, MinerUIntegration, MinerUTransformer } from './types.js';
+let MinerUTransformerStrategy = class MinerUTransformerStrategy {
+    constructor() {
+        this.permissions = [
+            {
+                type: 'integration',
+                service: MinerUIntegration,
+                description: 'Access to MinerU system integrations'
+            },
+            {
+                type: 'filesystem',
+                operations: ['read', 'write', 'list'],
+                scope: []
+            }
+        ];
+        this.meta = {
+            name: MinerUTransformer,
+            label: {
+                en_US: 'MinerU',
+                zh_Hans: 'MinerU'
+            },
+            description: {
+                en_US: 'A high-quality tool for convert PDF to Markdown and JSON.',
+                zh_Hans: '一站式开源高质量数据提取工具，将PDF转换成Markdown和JSON格式。'
+            },
+            icon: {
+                type: 'svg',
+                value: icon,
+                color: '#14b8a6'
+            },
+            helpUrl: 'https://mineru.net/apiManage/docs',
+            configSchema: {
+                type: 'object',
+                properties: {
+                    isOcr: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable OCR',
+                            zh_Hans: '启用 OCR'
+                        },
+                        description: {
+                            en_US: 'Enable OCR for image-based PDFs.',
+                            zh_Hans: '对基于图像的 PDF 启用 OCR。'
+                        },
+                        default: true
+                    },
+                    enableFormula: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable Formula Recognition',
+                            zh_Hans: '启用公式识别'
+                        },
+                        description: {
+                            en_US: 'Enable recognition of mathematical formulas in documents.',
+                            zh_Hans: '启用对文档中数学公式的识别。'
+                        },
+                        default: true
+                    },
+                    enableTable: {
+                        type: 'boolean',
+                        title: {
+                            en_US: 'Enable Table Recognition',
+                            zh_Hans: '启用表格识别'
+                        },
+                        description: {
+                            en_US: 'Enable recognition of tables in documents.',
+                            zh_Hans: '启用对文档中表格的识别。'
+                        },
+                        default: true
+                    },
+                    language: {
+                        type: 'string',
+                        title: {
+                            en_US: 'Document Language',
+                            zh_Hans: '文档语言'
+                        },
+                        description: {
+                            en_US: 'The primary language of the document (e.g., "en" for English, "ch" for Chinese).',
+                            zh_Hans: '文档的主要语言（例如，英文为 "en"，中文为 "ch"）。'
+                        },
+                        default: 'ch'
+                    },
+                    modelVersion: {
+                        type: 'string',
+                        title: {
+                            en_US: 'Model Version',
+                            zh_Hans: '模型版本'
+                        },
+                        description: {
+                            en_US: 'The model version to use for extraction (e.g., "vlm" or "pipeline").',
+                            zh_Hans: '用于提取的模型版本（例如，“vlm”或“pipeline”）。'
+                        },
+                        enum: ['pipeline', 'vlm'],
+                        default: 'pipeline'
+                    }
+                },
+                required: []
+            }
+        };
+    }
+    validateConfig(config) {
+        throw new Error('Method not implemented.');
+    }
+    async transformDocuments(documents, config) {
+        const mineru = new MinerUClient(this.configService, config.permissions);
+        const parsedResults = [];
+        for await (const document of documents) {
+            if (mineru.serverType === 'self-hosted') {
+                const { taskId } = await mineru.createTask({
+                    url: document.fileUrl,
+                    filePath: document.filePath,
+                    fileName: document.name,
+                    isOcr: true,
+                    enableFormula: true,
+                    enableTable: true,
+                    // language: 'ch',
+                    // modelVersion: 'vlm'
+                });
+                const result = mineru.getSelfHostedTask(taskId);
+                const parsedResult = await this.resultParser.parseLocalTask(result, taskId, document, config.permissions.fileSystem);
+                parsedResult.id = document.id;
+                parsedResults.push(parsedResult);
+            }
+            else {
+                const { taskId } = await mineru.createTask({
+                    url: document.fileUrl,
+                    isOcr: true,
+                    enableFormula: true,
+                    enableTable: true,
+                    language: 'ch',
+                    modelVersion: 'vlm',
+                    ...omitBy(pick(config, ['isOcr', 'enableFormula', 'enableTable', 'language', 'modelVersion']), isNil)
+                });
+                // Waiting for completion
+                const result = await mineru.waitForTask(taskId, 5 * 60 * 1000, 5000);
+                const parsedResult = await this.resultParser.parseFromUrl(result.full_zip_url, taskId, document, config.permissions.fileSystem);
+                parsedResult.id = document.id;
+                parsedResults.push(parsedResult);
+            }
+        }
+        return parsedResults;
+    }
+};
+__decorate([
+    Inject(MinerUResultParserService),
+    __metadata("design:type", MinerUResultParserService)
+], MinerUTransformerStrategy.prototype, "resultParser", void 0);
+__decorate([
+    Inject(forwardRef(() => ConfigService)),
+    __metadata("design:type", ConfigService)
+], MinerUTransformerStrategy.prototype, "configService", void 0);
+MinerUTransformerStrategy = __decorate([
+    Injectable(),
+    DocumentTransformerStrategy(MinerUTransformer)
+], MinerUTransformerStrategy);
+export { MinerUTransformerStrategy };

package/dist/lib/types.d.ts

@@ -0,0 +1,53 @@
+import { TDocumentAsset, TDocumentTransformerConfig } from "@xpert-ai/plugin-sdk";
+export declare const MinerU = "mineru";
+/**
+ * Integration provider key for MinerU API credentials.
+ *
+ * IMPORTANT:
+ * Keep this different from the builtin toolset provider key (`mineru`) to avoid the platform
+ * treating builtin toolset authorization as an "integration-backed" flow (which may read
+ * `credentials.integration` and crash when credentials is null).
+ */
+export declare const MinerUIntegration = "mineru_api";
+/**
+ * Document transformer provider key (distinct from toolset key).
+ */
+export declare const MinerUTransformer = "mineru_transformer";
+export declare const ENV_MINERU_API_BASE_URL = "MINERU_API_BASE_URL";
+export declare const ENV_MINERU_API_TOKEN = "MINERU_API_TOKEN";
+export declare const ENV_MINERU_SERVER_TYPE = "MINERU_SERVER_TYPE";
+export declare const icon = "<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\n<path d=\"M19.7238 3.86898C19.7238 4.57597 19.1502 5.1491 18.4427 5.1491C17.7352 5.1491 17.1616 4.57597 17.1616 3.86898C17.1616 3.16199 17.7352 2.58887 18.4427 2.58887C19.1502 2.58887 19.7238 3.16199 19.7238 3.86898Z\" fill=\"url(#paint0_linear_8609_1645)\"/>\n<path d=\"M19.7238 3.86898C19.7238 4.57597 19.1502 5.1491 18.4427 5.1491C17.7352 5.1491 17.1616 4.57597 17.1616 3.86898C17.1616 3.16199 17.7352 2.58887 18.4427 2.58887C19.1502 2.58887 19.7238 3.16199 19.7238 3.86898Z\" fill=\"#010101\"/>\n<path d=\"M15.3681 5.1491C15.3681 5.85609 14.7945 6.42921 14.087 6.42921C13.3794 6.42921 12.8059 5.85609 12.8059 5.1491C12.8059 4.44211 13.3794 3.86898 14.087 3.86898C14.7945 3.86898 15.3681 4.44211 15.3681 5.1491Z\" fill=\"url(#paint1_linear_8609_1645)\"/>\n<path d=\"M15.3681 5.1491C15.3681 5.85609 14.7945 6.42921 14.087 6.42921C13.3794 6.42921 12.8059 5.85609 12.8059 5.1491C12.8059 4.44211 13.3794 3.86898 14.087 3.86898C14.7945 3.86898 15.3681 4.44211 15.3681 5.1491Z\" fill=\"#010101\"/>\n<path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M8.05175 11.2368C8.05175 13.4605 9.14375 15.4293 10.8211 16.6371C11.8241 15.7389 12.4551 14.4345 12.4551 12.9828V9.39673C12.4551 8.85661 12.8197 8.38448 13.3426 8.24757L19.8924 6.53265C20.6459 6.33534 21.3826 6.90341 21.3826 7.6818L21.3826 12.0452C21.3826 17.2179 17.1861 21.4111 12.0095 21.4111L11.9942 21.4111C6.81758 21.4111 2.62109 17.2179 2.62109 12.0452V9.03388C2.62109 8.49175 2.9884 8.01839 3.51385 7.88336L6.56677 7.09882C7.31904 6.9055 8.05175 7.47318 8.05175 8.24934V11.2368ZM3.9798 12.0452C3.9798 13.8476 4.57565 15.5108 5.58124 16.849C6.04996 17.4728 6.7655 17.8884 7.54573 17.8884V17.8884C8.28848 17.8884 8.9927 17.7236 9.62376 17.4286C7.83439 15.9596 6.69304 13.7314 6.69304 11.2368V8.46821L3.9798 9.16546V12.0452Z\" fill=\"url(#paint2_linear_8609_1645)\"/>\n<path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M8.05175 11.2368C8.05175 13.4605 9.14375 15.4293 10.8211 16.6371C11.8241 15.7389 12.4551 14.4345 12.4551 12.9828V9.39673C12.4551 8.85661 12.8197 8.38448 13.3426 8.24757L19.8924 6.53265C20.6459 6.33534 21.3826 6.90341 21.3826 7.6818L21.3826 12.0452C21.3826 17.2179 17.1861 21.4111 12.0095 21.4111L11.9942 21.4111C6.81758 21.4111 2.62109 17.2179 2.62109 12.0452V9.03388C2.62109 8.49175 2.9884 8.01839 3.51385 7.88336L6.56677 7.09882C7.31904 6.9055 8.05175 7.47318 8.05175 8.24934V11.2368ZM3.9798 12.0452C3.9798 13.8476 4.57565 15.5108 5.58124 16.849C6.04996 17.4728 6.7655 17.8884 7.54573 17.8884V17.8884C8.28848 17.8884 8.9927 17.7236 9.62376 17.4286C7.83439 15.9596 6.69304 13.7314 6.69304 11.2368V8.46821L3.9798 9.16546V12.0452Z\" fill=\"#010101\"/>\n<defs>\n<linearGradient id=\"paint0_linear_8609_1645\" x1=\"14.3898\" y1=\"8.36821\" x2=\"13.1876\" y2=\"19.4461\" gradientUnits=\"userSpaceOnUse\">\n<stop stop-color=\"white\"/>\n<stop offset=\"1\" stop-color=\"#2E2E2E\"/>\n</linearGradient>\n<linearGradient id=\"paint1_linear_8609_1645\" x1=\"14.3898\" y1=\"8.36821\" x2=\"13.1876\" y2=\"19.4461\" gradientUnits=\"userSpaceOnUse\">\n<stop stop-color=\"white\"/>\n<stop offset=\"1\" stop-color=\"#2E2E2E\"/>\n</linearGradient>\n<linearGradient id=\"paint2_linear_8609_1645\" x1=\"14.3898\" y1=\"8.36821\" x2=\"13.1876\" y2=\"19.4461\" gradientUnits=\"userSpaceOnUse\">\n<stop stop-color=\"white\"/>\n<stop offset=\"1\" stop-color=\"#2E2E2E\"/>\n</linearGradient>\n</defs>\n</svg>\n";
+export type TMinerUTransformerConfig = TDocumentTransformerConfig & {
+    isOcr?: boolean;
+    enableFormula?: boolean;
+    enableTable?: boolean;
+    language?: 'en' | 'ch';
+    modelVersion?: 'vlm' | 'pipeline';
+};
+export type MinerUDocumentMetadata = {
+    parser: 'mineru';
+    assets?: TDocumentAsset[];
+    taskId: string;
+    originPdfUrl?: string;
+    mineruBackend?: string;
+    mineruVersion?: string;
+};
+export type MinerUServerType = 'official' | 'self-hosted';
+export type MinerUIntegrationOptions = {
+    apiUrl?: string;
+    apiKey?: string;
+    serverType?: MinerUServerType;
+};
+export interface MineruSelfHostedImage {
+    name: string;
+    dataUrl: string;
+}
+export interface MineruSelfHostedTaskResult {
+    mdContent: string;
+    contentList?: any;
+    images: MineruSelfHostedImage[];
+    raw: any;
+    fileName?: string;
+    sourceUrl?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAA;AAEjF,eAAO,MAAM,MAAM,WAAW,CAAA;AAC9B;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,eAAe,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,iBAAiB,uBAAuB,CAAA;AAErD,eAAO,MAAM,uBAAuB,wBAAwB,CAAA;AAC5D,eAAO,MAAM,oBAAoB,qBAAqB,CAAA;AACtD,eAAO,MAAM,sBAAsB,uBAAuB,CAAA;AAE1D,eAAO,MAAM,IAAI,m5GAsBhB,CAAA;AAED,MAAM,MAAM,wBAAwB,GAAG,0BAA0B,GAAG;IAClE,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAA;IACtB,YAAY,CAAC,EAAE,KAAK,GAAG,UAAU,CAAA;CAClC,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,aAAa,CAAA;AAEzD,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAC9B,CAAA;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,0BAA0B;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAChC,GAAG,EAAE,GAAG,CAAC;IACT,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/lib/types.js

@@ -0,0 +1,40 @@
+export const MinerU = 'mineru';
+/**
+ * Integration provider key for MinerU API credentials.
+ *
+ * IMPORTANT:
+ * Keep this different from the builtin toolset provider key (`mineru`) to avoid the platform
+ * treating builtin toolset authorization as an "integration-backed" flow (which may read
+ * `credentials.integration` and crash when credentials is null).
+ */
+export const MinerUIntegration = 'mineru_api';
+/**
+ * Document transformer provider key (distinct from toolset key).
+ */
+export const MinerUTransformer = 'mineru_transformer';
+export const ENV_MINERU_API_BASE_URL = 'MINERU_API_BASE_URL';
+export const ENV_MINERU_API_TOKEN = 'MINERU_API_TOKEN';
+export const ENV_MINERU_SERVER_TYPE = 'MINERU_SERVER_TYPE';
+export const icon = `<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M19.7238 3.86898C19.7238 4.57597 19.1502 5.1491 18.4427 5.1491C17.7352 5.1491 17.1616 4.57597 17.1616 3.86898C17.1616 3.16199 17.7352 2.58887 18.4427 2.58887C19.1502 2.58887 19.7238 3.16199 19.7238 3.86898Z" fill="url(#paint0_linear_8609_1645)"/>
+<path d="M19.7238 3.86898C19.7238 4.57597 19.1502 5.1491 18.4427 5.1491C17.7352 5.1491 17.1616 4.57597 17.1616 3.86898C17.1616 3.16199 17.7352 2.58887 18.4427 2.58887C19.1502 2.58887 19.7238 3.16199 19.7238 3.86898Z" fill="#010101"/>
+<path d="M15.3681 5.1491C15.3681 5.85609 14.7945 6.42921 14.087 6.42921C13.3794 6.42921 12.8059 5.85609 12.8059 5.1491C12.8059 4.44211 13.3794 3.86898 14.087 3.86898C14.7945 3.86898 15.3681 4.44211 15.3681 5.1491Z" fill="url(#paint1_linear_8609_1645)"/>
+<path d="M15.3681 5.1491C15.3681 5.85609 14.7945 6.42921 14.087 6.42921C13.3794 6.42921 12.8059 5.85609 12.8059 5.1491C12.8059 4.44211 13.3794 3.86898 14.087 3.86898C14.7945 3.86898 15.3681 4.44211 15.3681 5.1491Z" fill="#010101"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.05175 11.2368C8.05175 13.4605 9.14375 15.4293 10.8211 16.6371C11.8241 15.7389 12.4551 14.4345 12.4551 12.9828V9.39673C12.4551 8.85661 12.8197 8.38448 13.3426 8.24757L19.8924 6.53265C20.6459 6.33534 21.3826 6.90341 21.3826 7.6818L21.3826 12.0452C21.3826 17.2179 17.1861 21.4111 12.0095 21.4111L11.9942 21.4111C6.81758 21.4111 2.62109 17.2179 2.62109 12.0452V9.03388C2.62109 8.49175 2.9884 8.01839 3.51385 7.88336L6.56677 7.09882C7.31904 6.9055 8.05175 7.47318 8.05175 8.24934V11.2368ZM3.9798 12.0452C3.9798 13.8476 4.57565 15.5108 5.58124 16.849C6.04996 17.4728 6.7655 17.8884 7.54573 17.8884V17.8884C8.28848 17.8884 8.9927 17.7236 9.62376 17.4286C7.83439 15.9596 6.69304 13.7314 6.69304 11.2368V8.46821L3.9798 9.16546V12.0452Z" fill="url(#paint2_linear_8609_1645)"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.05175 11.2368C8.05175 13.4605 9.14375 15.4293 10.8211 16.6371C11.8241 15.7389 12.4551 14.4345 12.4551 12.9828V9.39673C12.4551 8.85661 12.8197 8.38448 13.3426 8.24757L19.8924 6.53265C20.6459 6.33534 21.3826 6.90341 21.3826 7.6818L21.3826 12.0452C21.3826 17.2179 17.1861 21.4111 12.0095 21.4111L11.9942 21.4111C6.81758 21.4111 2.62109 17.2179 2.62109 12.0452V9.03388C2.62109 8.49175 2.9884 8.01839 3.51385 7.88336L6.56677 7.09882C7.31904 6.9055 8.05175 7.47318 8.05175 8.24934V11.2368ZM3.9798 12.0452C3.9798 13.8476 4.57565 15.5108 5.58124 16.849C6.04996 17.4728 6.7655 17.8884 7.54573 17.8884V17.8884C8.28848 17.8884 8.9927 17.7236 9.62376 17.4286C7.83439 15.9596 6.69304 13.7314 6.69304 11.2368V8.46821L3.9798 9.16546V12.0452Z" fill="#010101"/>
+<defs>
+<linearGradient id="paint0_linear_8609_1645" x1="14.3898" y1="8.36821" x2="13.1876" y2="19.4461" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="#2E2E2E"/>
+</linearGradient>
+<linearGradient id="paint1_linear_8609_1645" x1="14.3898" y1="8.36821" x2="13.1876" y2="19.4461" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="#2E2E2E"/>
+</linearGradient>
+<linearGradient id="paint2_linear_8609_1645" x1="14.3898" y1="8.36821" x2="13.1876" y2="19.4461" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="#2E2E2E"/>
+</linearGradient>
+</defs>
+</svg>
+`;

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@helloxiaohu/plugin-mineru6",
+  "version": "0.0.2",
+  "license": "AGPL-3.0",
+  "description": "MinerU PDF to Markdown converter plugin for XpertAI platform with toolset support",
+  "keywords": [
+    "xpert-ai",
+    "plugin",
+    "mineru",
+    "pdf",
+    "markdown",
+    "ocr",
+    "document-conversion"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/xpert-ai/xpert-plugins.git"
+  },
+  "bugs": {
+    "url": "https://github.com/xpert-ai/xpert-plugins/issues"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "@xpert-plugins-starter/source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "package.json",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "form-data": "^4.0.0",
+    "tslib": "^2.3.0",
+    "unzipper": "0.12.3"
+  },
+  "peerDependencies": {
+    "@langchain/core": "0.3.72",
+    "@langchain/langgraph": "0.4.7",
+    "@nestjs/common": "^11.1.6",
+    "@nestjs/config": "^4.0.2",
+    "@xpert-ai/plugin-sdk": "^3.6.2",
+    "@metad/contracts": "^3.6.2",
+    "axios": "1.12.2",
+    "nestjs-i18n": "10.5.1",
+    "chalk": "4.1.2",
+    "lodash-es": "4.17.21",
+    "uuid": "8.3.2",
+    "zod": "3.25.67"
+  },
+  "devDependencies": {
+    "@types/unzipper": "^0.10.11"
+  }
+}