@xpert-ai/plugin-rag-vlm 0.0.1 → 0.1.1

package/dist/lib/vlm.strategy.d.ts

@@ -1,6 +1,9 @@
-import { ChunkMetadata, FileSystemPermission, IImageUnderstandingStrategy, LLMPermission, TImageUnderstandingConfig, TImageUnderstandingResult } from '@xpert-ai/plugin-sdk';
-import { IconType, IKnowledgeDocument } from '@metad/contracts';
-export declare class VlmDefaultStrategy implements IImageUnderstandingStrategy {
+import { FileSystemPermission, IImageUnderstandingStrategy, LLMPermission, TImageUnderstandingConfig, TImageUnderstandingResult } from '@xpert-ai/plugin-sdk';
+import { IconType } from '@xpert-ai/contracts';
+type VlmConfig = TImageUnderstandingConfig & {
+    promptTemplate?: string;
+};
+export declare class VlmDefaultStrategy implements IImageUnderstandingStrategy<VlmConfig> {
     readonly permissions: (FileSystemPermission | LLMPermission)[];
     readonly meta: {
         name: string;
@@ -23,7 +26,8 @@ export declare class VlmDefaultStrategy implements IImageUnderstandingStrategy {
         };
     };
     validateConfig(config: TImageUnderstandingConfig): Promise<void>;
-    understandImages(doc: IKnowledgeDocument<ChunkMetadata>, config: TImageUnderstandingConfig): Promise<TImageUnderstandingResult>;
+    understandImages(doc: Parameters<IImageUnderstandingStrategy['understandImages']>[0], config: VlmConfig): Promise<TImageUnderstandingResult>;
     private runV;
 }
+export {};
 //# sourceMappingURL=vlm.strategy.d.ts.map

package/dist/lib/vlm.strategy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vlm.strategy.d.ts","sourceRoot":"","sources":["../../src/lib/vlm.strategy.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAE3B,aAAa,EACb,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAqC,QAAQ,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AASlG,qBAEa,kBAAmB,YAAW,2BAA2B;IACpE,QAAQ,CAAC,WAAW,2CAUnB;IAED,QAAQ,CAAC,IAAI;;;;;;;;;;;;;;;kBAYM,QAAQ;;;;MAI1B;IAEK,cAAc,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhE,gBAAgB,CACpB,GAAG,EAAE,kBAAkB,CAAC,aAAa,CAAC,EACtC,MAAM,EAAE,yBAAyB,GAChC,OAAO,CAAC,yBAAyB,CAAC;YAoDvB,IAAI;CA4BnB"}
+{"version":3,"file":"vlm.strategy.d.ts","sourceRoot":"","sources":["../../src/lib/vlm.strategy.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,oBAAoB,EACpB,2BAA2B,EAE3B,aAAa,EACb,yBAAyB,EAEzB,yBAAyB,EAC1B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAqC,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AASjF,KAAK,SAAS,GAAG,yBAAyB,GAAG;IAAE,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAExE,qBAEa,kBAAmB,YAAW,2BAA2B,CAAC,SAAS,CAAC;IAC/E,QAAQ,CAAC,WAAW,2CAUnB;IAED,QAAQ,CAAC,IAAI;;;;;;;;;;;;;;;kBAYM,QAAQ;;;;MAI1B;IAEK,cAAc,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhE,gBAAgB,CACpB,GAAG,EAAE,UAAU,CAAC,2BAA2B,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,MAAM,EAAE,SAAS,GAChB,OAAO,CAAC,yBAAyB,CAAC;YA0DvB,IAAI;CAqDnB"}

package/dist/lib/vlm.strategy.js

@@ -1,7 +1,7 @@
 import { __decorate } from "tslib";
 import { Injectable } from '@nestjs/common';
 import { ImageUnderstandingStrategy } from '@xpert-ai/plugin-sdk';
-import { buildChunkTree, collectTreeLeaves } from '@metad/contracts';
+import { buildChunkTree, collectTreeLeaves } from '@xpert-ai/contracts';
 import { Document } from '@langchain/core/documents';
 import sharp from 'sharp';
 import { v4 as uuid } from 'uuid';
@@ -18,7 +18,7 @@ let VlmDefaultStrategy = class VlmDefaultStrategy {
             },
             {
                 type: 'llm',
-                capability: 'vision',
+                capability: 'vision'
             }
         ];
         this.meta = {
@@ -47,71 +47,101 @@ let VlmDefaultStrategy = class VlmDefaultStrategy {
     async understandImages(doc, config) {
         await this.validateConfig(config);
         const client = config.visionModel; // ✅ Injected by the core system
-        const params = {
-            files: doc.metadata?.assets?.filter((asset) => asset.type === 'image'),
-            chunks: doc.chunks
-        };
-        const tree = buildChunkTree(doc.chunks);
-        const leaves = collectTreeLeaves(tree);
+        const files = doc.metadata?.assets?.filter((asset) => asset.type === 'image') ?? [];
+        const leaves = collectTreeLeaves(buildChunkTree(doc.chunks));
+        // Keep context parents as well as retrieval leaves, and avoid repeated OCR for overlapping chunks.
         const chunks = [];
-        // const pages : Document<Partial<ChunkMetadata>>[] = []
-        for await (const chunk of leaves) {
-            const assets = [];
-            chunk.metadata['chunkId'] ??= uuid();
-            // Source Document Block
+        const leafIds = new Set(leaves.map((chunk) => chunk.metadata.chunkId));
+        chunks.push(...doc.chunks.filter((chunk) => !leafIds.has(chunk.metadata.chunkId)));
+        const processed = new Set();
+        const warnings = [];
+        for (const chunk of leaves) {
             chunks.push(chunk);
-            // Find image tags inside the chunk
-            const matches = Array.from(chunk.pageContent.matchAll(IMAGE_REGEX));
-            for (const match of matches) {
-                const url = match[1]; // image-url.png
-                const asset = params.files.find((a) => a.url === url);
-                if (asset && !assets.some((_) => _ === asset.url)) {
-                    const description = await this.runV(client, chunk.pageContent, asset.filePath, config);
-                    assets.push(asset.url);
+            for (const match of chunk.pageContent.matchAll(IMAGE_REGEX)) {
+                const asset = files.find((item) => item.url === match[1]);
+                if (!asset || processed.has(asset.filePath))
+                    continue;
+                processed.add(asset.filePath);
+                try {
+                    const result = await this.runV(client, asset, config, chunk.pageContent);
+                    if (result.type === 'skipped') {
+                        warnings.push({ type: 'image_understanding_skipped', message: result.reason, imagePath: asset.filePath });
+                        continue;
+                    }
+                    const description = result.text;
+                    if (!description.trim())
+                        throw new Error('The vision model returned no text.');
                     chunks.push(new Document({
                         pageContent: description,
                         metadata: {
                             mediaType: 'image',
-                            chunkId: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
-                            parentId: chunk.metadata['chunkId'],
+                            chunkId: uuid(),
+                            // Page transcriptions are independent source text and will be split by the host.
+                            ...(asset.sourceType === 'pdf_page'
+                                ? { page: asset.page, sourceType: 'pdf_page', contentFormat: 'markdown' }
+                                : { parentId: chunk.metadata.chunkId }),
                             imagePath: asset.filePath,
-                            // source: asset.filename,
+                            imageUrl: asset.url,
                             parser: 'vlm'
                         }
                     }));
                 }
+                catch (error) {
+                    warnings.push({
+                        type: 'image_understanding_failed',
+                        message: error instanceof Error ? error.message : 'Image recognition failed.',
+                        imagePath: asset.filePath
+                    });
+                }
             }
         }
-        return {
-            chunks,
-            metadata: {}
-        };
+        return { chunks, metadata: { warnings } };
     }
-    async runV(client, context, imagePath, config) {
-        const imageStr = await config.permissions.fileSystem.readFile(imagePath);
-        const sharped = sharp(imageStr);
-        const imageData = await sharped.resize(1024).toBuffer();
-        const fileInfo = await sharped.metadata();
-        const mimetype = fileInfo.format ? `image/${fileInfo.format}` : 'image/png';
+    async runV(client, asset, config, context) {
+        const imageStr = await config.permissions.fileSystem.readFile(asset.filePath);
+        const page = asset.sourceType === 'pdf_page';
+        const image = sharp(imageStr);
+        const { width, height } = await image.metadata();
+        // Embedded tracking/spacer pixels contain no readable content. Never silently skip a PDF page.
+        if (width === 1 || height === 1) {
+            if (page)
+                throw new Error(`PDF page image is too small to recognize (${width}×${height}).`);
+            return { type: 'skipped', reason: `Skipped a ${width}×${height} placeholder image.` };
+        }
+        const imageData = await image
+            .resize({ width: page ? 2200 : 1024, height: page ? 2200 : 1024, fit: 'inside', withoutEnlargement: true })
+            .png()
+            .toBuffer();
         const response = await client.invoke([
             {
                 role: 'system',
-                content: 'You are a professional assistant, helping people understand images in context. Please provide a narrative description of the image.'
+                content: page
+                    ? 'Transcribe all visible content of this document page into Markdown in reading order. Preserve headings, lists, table rows and merged-cell relationships. Copy numbers, leading zeroes, codes and punctuation exactly. Do not summarize, describe the page, invent text, or add a preface. Mark unreadable text as [unreadable].'
+                    : 'You are a professional assistant, helping people understand images in context. Please provide a narrative description of the image.'
             },
             {
                 role: 'user',
                 content: [
-                    // { type: 'text', text: context },
+                    ...(config.promptTemplate?.trim()
+                        ? [{ type: 'text', text: config.promptTemplate.replaceAll('{{context}}', () => context) }]
+                        : []),
                     {
                         type: 'image_url',
                         image_url: {
-                            url: `data:${mimetype};base64,${imageData.toString('base64')}`
+                            url: `data:image/png;base64,${imageData.toString('base64')}`
                         }
                     }
                 ]
             }
         ]);
-        return response.content;
+        return {
+            type: 'recognized',
+            text: typeof response.content === 'string'
+                ? response.content
+                : response.content
+                    .flatMap((part) => (part.type === 'text' && typeof part.text === 'string' ? [part.text] : []))
+                    .join('\n')
+        };
     }
 };
 VlmDefaultStrategy = __decorate([

package/package.json

@@ -1,6 +1,10 @@
 {
   "name": "@xpert-ai/plugin-rag-vlm",
-  "version": "0.0.1",
+  "version": "0.1.1",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/xpert-ai/xpert-plugins"
+  },
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
@@ -23,13 +27,13 @@
     "tslib": "^2.3.0"
   },
   "peerDependencies": {
-    "@xpert-ai/plugin-sdk": "^3.6.0",
+    "@xpert-ai/plugin-sdk": "^3.15.18",
     "zod": "3.25.67",
     "@nestjs/config": "^4.0.2",
     "chalk": "4.1.2",
     "@langchain/core": "0.3.72",
     "@nestjs/common": "^11.1.6",
-    "@metad/contracts": "^3.6.0",
+    "@xpert-ai/contracts": "^3.6.0",
     "uuid": "8.3.2"
   }
 }