@xpert-ai/plugin-rag-vlm 0.0.1

package/README.md

@@ -0,0 +1,11 @@
+# rag-vlm
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build rag-vlm` to build the library.
+
+## Running unit tests
+
+Run `nx test rag-vlm` to execute the unit tests via [Jest](https://jestjs.io).

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+import type { XpertPlugin } from '@xpert-ai/plugin-sdk';
+import { z } from 'zod';
+declare const ConfigSchema: z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>;
+declare const plugin: XpertPlugin<z.infer<typeof ConfigSchema>>;
+export default plugin;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,QAAA,MAAM,YAAY,gDAChB,CAAC;AAEH,QAAA,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CA2BrD,CAAC;AAEF,eAAe,MAAM,CAAC"}

package/dist/index.js

@@ -0,0 +1,39 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { z } from 'zod';
+import { VlmDefaultPlugin } from './lib/vlm.module.js';
+import { SvgIcon } from './lib/types.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf8'));
+const ConfigSchema = z.object({});
+const plugin = {
+    meta: {
+        name: packageJson.name,
+        version: packageJson.version,
+        category: 'vlm',
+        icon: {
+            type: 'svg',
+            value: SvgIcon
+        },
+        displayName: 'VLM Default',
+        description: 'Provide VLM functionality using default settings',
+        keywords: ['pdf', 'markdown', 'json', 'vlm', 'default'],
+        author: 'XpertAI Team',
+    },
+    config: {
+        schema: ConfigSchema,
+    },
+    register(ctx) {
+        ctx.logger.log('register VLM default plugin');
+        return { module: VlmDefaultPlugin, global: true };
+    },
+    async onStart(ctx) {
+        ctx.logger.log('vlm default plugin started');
+    },
+    async onStop(ctx) {
+        ctx.logger.log('vlm default plugin stopped');
+    },
+};
+export default plugin;

package/dist/lib/types.d.ts

@@ -0,0 +1,3 @@
+export declare const VlmDefault = "vlm-default";
+export declare const SvgIcon = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<svg version=\"1.1\" id=\"Layer_1\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" x=\"0px\" y=\"0px\"\n\t viewBox=\"0 0 960 960\" style=\"enable-background:new 0 0 960 960;\" xml:space=\"preserve\">\n<style type=\"text/css\">\n\t.st0{fill:#135468;}\n\t.st1{fill:#FFFFFF;}\n\t.st2{fill:#6DBCDB;}\n\t.st3{fill:#333333;}\n\t.st4{fill:none;stroke:#FFFFFF;stroke-width:22.1618;stroke-linecap:round;stroke-miterlimit:10;}\n</style>\n<g>\n\t<circle class=\"st0\" cx=\"480\" cy=\"480\" r=\"300.9\"/>\n\t<path class=\"st1\" d=\"M750.4,480c0,0-121.1,130-270.4,130S209.6,480,209.6,480S330.7,350,480,350S750.4,480,750.4,480z\"/>\n\t<path class=\"st1\" d=\"M750.4,480c0,0-121.1,130-270.4,130S209.6,480,209.6,480S330.7,350,480,350S750.4,480,750.4,480z\"/>\n\t<circle class=\"st2\" cx=\"480\" cy=\"478.4\" r=\"102.8\"/>\n\t<circle class=\"st3\" cx=\"480\" cy=\"478.4\" r=\"58.6\"/>\n\t<circle class=\"st1\" cx=\"441.2\" cy=\"444.1\" r=\"24.3\"/>\n\t<line class=\"st4\" x1=\"363.1\" y1=\"323.7\" x2=\"328.1\" y2=\"286.8\"/>\n\t<line class=\"st4\" x1=\"631.9\" y1=\"286.8\" x2=\"596.9\" y2=\"323.7\"/>\n\t<line class=\"st4\" x1=\"482.7\" y1=\"300.9\" x2=\"482.7\" y2=\"256.8\"/>\n</g>\n</svg>\n";
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,gBAAgB,CAAA;AAEvC,eAAO,MAAM,OAAO,8uCAsBnB,CAAA"}

package/dist/lib/types.js

@@ -0,0 +1,24 @@
+export const VlmDefault = 'vlm-default';
+export const SvgIcon = `<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 960 960" style="enable-background:new 0 0 960 960;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#135468;}
+	.st1{fill:#FFFFFF;}
+	.st2{fill:#6DBCDB;}
+	.st3{fill:#333333;}
+	.st4{fill:none;stroke:#FFFFFF;stroke-width:22.1618;stroke-linecap:round;stroke-miterlimit:10;}
+</style>
+<g>
+	<circle class="st0" cx="480" cy="480" r="300.9"/>
+	<path class="st1" d="M750.4,480c0,0-121.1,130-270.4,130S209.6,480,209.6,480S330.7,350,480,350S750.4,480,750.4,480z"/>
+	<path class="st1" d="M750.4,480c0,0-121.1,130-270.4,130S209.6,480,209.6,480S330.7,350,480,350S750.4,480,750.4,480z"/>
+	<circle class="st2" cx="480" cy="478.4" r="102.8"/>
+	<circle class="st3" cx="480" cy="478.4" r="58.6"/>
+	<circle class="st1" cx="441.2" cy="444.1" r="24.3"/>
+	<line class="st4" x1="363.1" y1="323.7" x2="328.1" y2="286.8"/>
+	<line class="st4" x1="631.9" y1="286.8" x2="596.9" y2="323.7"/>
+	<line class="st4" x1="482.7" y1="300.9" x2="482.7" y2="256.8"/>
+</g>
+</svg>
+`;

package/dist/lib/vlm.module.d.ts

@@ -0,0 +1,13 @@
+import { IOnPluginBootstrap, IOnPluginDestroy } from '@xpert-ai/plugin-sdk';
+export declare class VlmDefaultPlugin implements IOnPluginBootstrap, IOnPluginDestroy {
+    private logEnabled;
+    /**
+     * Called when the plugin is being initialized.
+     */
+    onPluginBootstrap(): void | Promise<void>;
+    /**
+     * Called when the plugin is being destroyed.
+     */
+    onPluginDestroy(): void | Promise<void>;
+}
+//# sourceMappingURL=vlm.module.d.ts.map

package/dist/lib/vlm.module.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vlm.module.d.ts","sourceRoot":"","sources":["../../src/lib/vlm.module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAK/F,qBAea,gBAAiB,YAAW,kBAAkB,EAAE,gBAAgB;IAE5E,OAAO,CAAC,UAAU,CAAQ;IAE1B;;OAEG;IACH,iBAAiB,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC;;OAEG;IACH,eAAe,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAKvC"}

package/dist/lib/vlm.module.js

@@ -0,0 +1,45 @@
+var VlmDefaultPlugin_1;
+import { __decorate } from "tslib";
+import { XpertServerPlugin } from '@xpert-ai/plugin-sdk';
+import { ConfigModule } from '@nestjs/config';
+import chalk from 'chalk';
+import { VlmDefaultStrategy } from './vlm.strategy.js';
+let VlmDefaultPlugin = VlmDefaultPlugin_1 = class VlmDefaultPlugin {
+    constructor() {
+        // We disable by default additional logging for each event to avoid cluttering the logs
+        this.logEnabled = true;
+    }
+    /**
+     * Called when the plugin is being initialized.
+     */
+    onPluginBootstrap() {
+        if (this.logEnabled) {
+            console.log(chalk.green(`${VlmDefaultPlugin_1.name} is being bootstrapped...`));
+        }
+    }
+    /**
+     * Called when the plugin is being destroyed.
+     */
+    onPluginDestroy() {
+        if (this.logEnabled) {
+            console.log(chalk.green(`${VlmDefaultPlugin_1.name} is being destroyed...`));
+        }
+    }
+};
+VlmDefaultPlugin = VlmDefaultPlugin_1 = __decorate([
+    XpertServerPlugin({
+        /**
+         * An array of modules that will be imported and registered with the plugin.
+         */
+        imports: [ConfigModule],
+        /**
+         * An array of Entity classes. The plugin (or ORM) will
+         * register these entities for use within the application.
+         */
+        entities: [],
+        providers: [
+            VlmDefaultStrategy,
+        ]
+    })
+], VlmDefaultPlugin);
+export { VlmDefaultPlugin };

package/dist/lib/vlm.strategy.d.ts

@@ -0,0 +1,29 @@
+import { ChunkMetadata, FileSystemPermission, IImageUnderstandingStrategy, LLMPermission, TImageUnderstandingConfig, TImageUnderstandingResult } from '@xpert-ai/plugin-sdk';
+import { IconType, IKnowledgeDocument } from '@metad/contracts';
+export declare class VlmDefaultStrategy implements IImageUnderstandingStrategy {
+    readonly permissions: (FileSystemPermission | LLMPermission)[];
+    readonly meta: {
+        name: string;
+        label: {
+            en_US: string;
+            zh_Hans: string;
+        };
+        description: {
+            en_US: string;
+            zh_Hans: string;
+        };
+        configSchema: {
+            type: string;
+            properties: {};
+        };
+        icon: {
+            type: IconType;
+            value: string;
+            color: string;
+        };
+    };
+    validateConfig(config: TImageUnderstandingConfig): Promise<void>;
+    understandImages(doc: IKnowledgeDocument<ChunkMetadata>, config: TImageUnderstandingConfig): Promise<TImageUnderstandingResult>;
+    private runV;
+}
+//# sourceMappingURL=vlm.strategy.d.ts.map

package/dist/lib/vlm.strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vlm.strategy.d.ts","sourceRoot":"","sources":["../../src/lib/vlm.strategy.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAE3B,aAAa,EACb,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAqC,QAAQ,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AASlG,qBAEa,kBAAmB,YAAW,2BAA2B;IACpE,QAAQ,CAAC,WAAW,2CAUnB;IAED,QAAQ,CAAC,IAAI;;;;;;;;;;;;;;;kBAYM,QAAQ;;;;MAI1B;IAEK,cAAc,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhE,gBAAgB,CACpB,GAAG,EAAE,kBAAkB,CAAC,aAAa,CAAC,EACtC,MAAM,EAAE,yBAAyB,GAChC,OAAO,CAAC,yBAAyB,CAAC;YAoDvB,IAAI;CA4BnB"}

package/dist/lib/vlm.strategy.js

@@ -0,0 +1,121 @@
+import { __decorate } from "tslib";
+import { Injectable } from '@nestjs/common';
+import { ImageUnderstandingStrategy } from '@xpert-ai/plugin-sdk';
+import { buildChunkTree, collectTreeLeaves } from '@metad/contracts';
+import { Document } from '@langchain/core/documents';
+import sharp from 'sharp';
+import { v4 as uuid } from 'uuid';
+import { SvgIcon, VlmDefault } from './types.js';
+// Regex for markdown image tag: ![](image.png) or ![alt](image.png)
+const IMAGE_REGEX = /!\[[^\]]*\]\s*\(((?:https?:\/\/[^)]+|[^)\s]+))(\s*"[^"]*")?\)/g;
+let VlmDefaultStrategy = class VlmDefaultStrategy {
+    constructor() {
+        this.permissions = [
+            {
+                type: 'filesystem',
+                operations: ['read'],
+                scope: []
+            },
+            {
+                type: 'llm',
+                capability: 'vision',
+            }
+        ];
+        this.meta = {
+            name: VlmDefault,
+            label: { en_US: 'VLM', zh_Hans: '视觉语言模型' },
+            description: {
+                en_US: 'Use V(ision)LM to understand images, the knowledge base needs to be configured with a visual model.',
+                zh_Hans: '使用视觉大模型来理解图片，知识库需要配置视觉模型。'
+            },
+            configSchema: {
+                type: 'object',
+                properties: {}
+            },
+            icon: {
+                type: 'svg',
+                value: SvgIcon,
+                color: '#2d8cf0'
+            }
+        };
+    }
+    async validateConfig(config) {
+        if (!config?.visionModel) {
+            throw new Error('Vision Model is required');
+        }
+    }
+    async understandImages(doc, config) {
+        await this.validateConfig(config);
+        const client = config.visionModel; // ✅ Injected by the core system
+        const params = {
+            files: doc.metadata?.assets?.filter((asset) => asset.type === 'image'),
+            chunks: doc.chunks
+        };
+        const tree = buildChunkTree(doc.chunks);
+        const leaves = collectTreeLeaves(tree);
+        const chunks = [];
+        // const pages : Document<Partial<ChunkMetadata>>[] = []
+        for await (const chunk of leaves) {
+            const assets = [];
+            chunk.metadata['chunkId'] ??= uuid();
+            // Source Document Block
+            chunks.push(chunk);
+            // Find image tags inside the chunk
+            const matches = Array.from(chunk.pageContent.matchAll(IMAGE_REGEX));
+            for (const match of matches) {
+                const url = match[1]; // image-url.png
+                const asset = params.files.find((a) => a.url === url);
+                if (asset && !assets.some((_) => _ === asset.url)) {
+                    const description = await this.runV(client, chunk.pageContent, asset.filePath, config);
+                    assets.push(asset.url);
+                    chunks.push(new Document({
+                        pageContent: description,
+                        metadata: {
+                            mediaType: 'image',
+                            chunkId: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+                            parentId: chunk.metadata['chunkId'],
+                            imagePath: asset.filePath,
+                            // source: asset.filename,
+                            parser: 'vlm'
+                        }
+                    }));
+                }
+            }
+        }
+        return {
+            chunks,
+            metadata: {}
+        };
+    }
+    async runV(client, context, imagePath, config) {
+        const imageStr = await config.permissions.fileSystem.readFile(imagePath);
+        const sharped = sharp(imageStr);
+        const imageData = await sharped.resize(1024).toBuffer();
+        const fileInfo = await sharped.metadata();
+        const mimetype = fileInfo.format ? `image/${fileInfo.format}` : 'image/png';
+        const response = await client.invoke([
+            {
+                role: 'system',
+                content: 'You are a professional assistant, helping people understand images in context. Please provide a narrative description of the image.'
+            },
+            {
+                role: 'user',
+                content: [
+                    // { type: 'text', text: context },
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: `data:${mimetype};base64,${imageData.toString('base64')}`
+                        }
+                    }
+                ]
+            }
+        ]);
+        return response.content;
+    }
+};
+VlmDefaultStrategy = __decorate([
+    Injectable(),
+    ImageUnderstandingStrategy(VlmDefault)
+], VlmDefaultStrategy);
+export { VlmDefaultStrategy };

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@xpert-ai/plugin-rag-vlm",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "@xpert-plugins-starter/source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "sharp": "0.33.5",
+    "tslib": "^2.3.0"
+  },
+  "peerDependencies": {
+    "@xpert-ai/plugin-sdk": "^3.6.0",
+    "zod": "3.25.67",
+    "@nestjs/config": "^4.0.2",
+    "chalk": "4.1.2",
+    "@langchain/core": "0.3.72",
+    "@nestjs/common": "^11.1.6",
+    "@metad/contracts": "^3.6.0",
+    "uuid": "8.3.2"
+  }
+}