@robthepcguy/rag-vault 1.3.0 → 1.5.0

package/dist/embedder/index.d.ts

@@ -51,5 +51,13 @@ export declare class Embedder {
      * @returns Array of embedding vectors (dimension depends on model)
      */
     embedBatch(texts: string[], signal?: AbortSignal): Promise<number[][]>;
+    /**
+     * Detect known cache-corruption signatures from ONNX/protobuf loaders.
+     */
+    private isRecoverableCacheError;
+    /**
+     * Build a model-specific fallback cache path to avoid reusing corrupted artifacts.
+     */
+    private getRecoveryCacheDir;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/embedder/index.js

@@ -1,7 +1,12 @@
 "use strict";
 // Embedder implementation with Transformers.js
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Embedder = exports.EmbeddingError = void 0;
+const promises_1 = require("node:fs/promises");
+const node_path_1 = __importDefault(require("node:path"));
 const transformers_1 = require("@huggingface/transformers");
 const index_js_1 = require("../errors/index.js");
 // Re-export error class for backwards compatibility
@@ -49,6 +54,21 @@ class Embedder {
             console.error('Embedder: Model loaded successfully');
         }
         catch (error) {
+            // Some ONNX caches fail with "Protobuf parsing failed". Retry once with isolated cache path.
+            if (this.isRecoverableCacheError(error)) {
+                const recoveryCacheDir = this.getRecoveryCacheDir();
+                console.error(`Embedder: Detected corrupted model cache. Retrying with isolated cache: "${recoveryCacheDir}"`);
+                try {
+                    await (0, promises_1.mkdir)(recoveryCacheDir, { recursive: true });
+                    transformers_1.env.cacheDir = recoveryCacheDir;
+                    this.model = await (0, transformers_1.pipeline)('feature-extraction', this.config.modelPath);
+                    console.error('Embedder: Model loaded successfully via recovery cache');
+                    return;
+                }
+                catch (recoveryError) {
+                    throw new index_js_1.EmbeddingError(`Failed to initialize Embedder after cache recovery attempt: ${recoveryError.message}`, recoveryError);
+                }
+            }
             throw new index_js_1.EmbeddingError(`Failed to initialize Embedder: ${error.message}`, error);
         }
     }
@@ -141,6 +161,24 @@ class Embedder {
             throw new index_js_1.EmbeddingError(`Failed to generate batch embeddings: ${message}`, error instanceof Error ? error : undefined);
         }
     }
+    /**
+     * Detect known cache-corruption signatures from ONNX/protobuf loaders.
+     */
+    isRecoverableCacheError(error) {
+        if (!(error instanceof Error)) {
+            return false;
+        }
+        const message = error.message.toLowerCase();
+        return (message.includes('protobuf parsing failed') ||
+            (message.includes('protobuf') && message.includes('failed to parse')));
+    }
+    /**
+     * Build a model-specific fallback cache path to avoid reusing corrupted artifacts.
+     */
+    getRecoveryCacheDir() {
+        const safeModelName = this.config.modelPath.replace(/[^a-z0-9_./-]/gi, '_').replace(/\//g, '__');
+        return node_path_1.default.join(this.config.cacheDir, '.recovery-cache', safeModelName);
+    }
 }
 exports.Embedder = Embedder;
 //# sourceMappingURL=index.js.map

package/dist/parser/index.js

@@ -113,10 +113,27 @@ class DocumentParser {
         if (!(0, node_path_1.isAbsolute)(filePath)) {
             throw new index_js_1.ParserValidationError(`File path must be absolute path (received: ${filePath}). Please provide an absolute path within BASE_DIR.`);
         }
-        // Check if path is within BASE_DIR
-        const baseDir = (0, node_path_1.resolve)(this.config.baseDir);
-        const normalizedPath = (0, node_path_1.resolve)(filePath);
-        if (!normalizedPath.startsWith(baseDir)) {
+        // Resolve symlinks for both base and target to prevent symlink escape attacks.
+        // Fall back to resolve() only when path does not exist yet.
+        const resolveCanonicalPath = (targetPath) => {
+            try {
+                return (0, node_fs_1.realpathSync)(targetPath);
+            }
+            catch (error) {
+                const nodeError = error;
+                if (nodeError.code === 'ENOENT') {
+                    return (0, node_path_1.resolve)(targetPath);
+                }
+                throw new index_js_1.ParserValidationError(`Failed to resolve path for security validation: ${targetPath}`);
+            }
+        };
+        // Check if path is within BASE_DIR using relative-path boundary check.
+        // This avoids prefix bypasses such as /base matching /base2.
+        const baseDir = resolveCanonicalPath(this.config.baseDir);
+        const normalizedPath = resolveCanonicalPath(filePath);
+        const relativePath = (0, node_path_1.relative)(baseDir, normalizedPath);
+        const isOutsideBaseDir = relativePath.startsWith('..') || relativePath === '..' || (0, node_path_1.isAbsolute)(relativePath);
+        if (isOutsideBaseDir) {
             throw new index_js_1.ParserValidationError(`File path must be within BASE_DIR (${baseDir}). Received path outside BASE_DIR: ${filePath}`);
         }
     }

package/dist/server/index.js

@@ -145,7 +145,10 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             sourceQuery: zod_1.z.string().describe('The query that returned this result'),
             targetFilePath: zod_1.z.string().describe('File path of the result to pin'),
             targetChunkIndex: zod_1.z.number().describe('Chunk index of the result to pin'),
-            targetFingerprint: zod_1.z.string().optional().describe('Optional fingerprint for resilient matching'),
+            targetFingerprint: zod_1.z
+                .string()
+                .optional()
+                .describe('Optional fingerprint for resilient matching'),
         }, async (args) => {
             try {
                 const result = this.executeFeedbackPin(args);
@@ -155,7 +158,12 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             }
             catch (error) {
                 return {
-                    content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
+                    content: [
+                        {
+                            type: 'text',
+                            text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }),
+                        },
+                    ],
                     isError: true,
                 };
             }
@@ -165,7 +173,10 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             sourceQuery: zod_1.z.string().describe('The query that returned this result'),
             targetFilePath: zod_1.z.string().describe('File path of the result to dismiss'),
             targetChunkIndex: zod_1.z.number().describe('Chunk index of the result to dismiss'),
-            targetFingerprint: zod_1.z.string().optional().describe('Optional fingerprint for resilient matching'),
+            targetFingerprint: zod_1.z
+                .string()
+                .optional()
+                .describe('Optional fingerprint for resilient matching'),
         }, async (args) => {
             try {
                 const result = this.executeFeedbackDismiss(args);
@@ -175,7 +186,12 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             }
             catch (error) {
                 return {
-                    content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
+                    content: [
+                        {
+                            type: 'text',
+                            text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }),
+                        },
+                    ],
                     isError: true,
                 };
             }
@@ -190,7 +206,12 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             }
             catch (error) {
                 return {
-                    content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
+                    content: [
+                        {
+                            type: 'text',
+                            text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }),
+                        },
+                    ],
                     isError: true,
                 };
             }
@@ -288,7 +309,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
                 score: result.score,
             };
             // Restore source for raw-data files (ingested via ingest_data)
-            if ((0, raw_data_utils_js_1.isRawDataPath)(result.filePath)) {
+            if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, result.filePath)) {
                 const source = (0, raw_data_utils_js_1.extractSourceFromPath)(result.filePath);
                 if (source) {
                     queryResult.source = source;
@@ -336,7 +357,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
         // since the path is internally generated and content is already processed
         const isPdf = args.filePath.toLowerCase().endsWith('.pdf');
         let text;
-        if ((0, raw_data_utils_js_1.isRawDataPath)(args.filePath)) {
+        if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, args.filePath)) {
             // Raw-data files: skip validation, read directly
             text = await (0, promises_1.readFile)(args.filePath, 'utf-8');
             console.error(`Read raw-data file: ${args.filePath} (${text.length} characters)`);
@@ -505,7 +526,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
         const files = await this.vectorStore.listFiles();
         // Enrich raw-data files with source information
         return files.map((file) => {
-            if ((0, raw_data_utils_js_1.isRawDataPath)(file.filePath)) {
+            if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, file.filePath)) {
                 const source = (0, raw_data_utils_js_1.extractSourceFromPath)(file.filePath);
                 if (source) {
                     return { ...file, source };
@@ -648,7 +669,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
         // Delete chunks from vector database
         await this.vectorStore.deleteChunks(targetPath);
         // Also delete physical raw-data file if applicable
-        if ((0, raw_data_utils_js_1.isRawDataPath)(targetPath)) {
+        if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, targetPath)) {
             try {
                 await (0, promises_1.unlink)(targetPath);
                 console.error(`Deleted raw-data file: ${targetPath}`);
@@ -693,7 +714,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             const chunks = await this.vectorStore.getDocumentChunks(filePath);
             // Enrich with source information for raw-data files
             const enrichedChunks = chunks.map((chunk) => {
-                if ((0, raw_data_utils_js_1.isRawDataPath)(chunk.filePath)) {
+                if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, chunk.filePath)) {
                     const source = (0, raw_data_utils_js_1.extractSourceFromPath)(chunk.filePath);
                     if (source) {
                         return { ...chunk, source };
@@ -724,7 +745,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
             const relatedChunks = await this.vectorStore.findRelatedChunks(filePath, chunkIndex, limit ?? 5, excludeSameDocument ?? true);
             // Enrich with source information for raw-data files
             const enrichedChunks = relatedChunks.map((chunk) => {
-                if ((0, raw_data_utils_js_1.isRawDataPath)(chunk.filePath)) {
+                if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, chunk.filePath)) {
                     const source = (0, raw_data_utils_js_1.extractSourceFromPath)(chunk.filePath);
                     if (source) {
                         return { ...chunk, source };
@@ -760,7 +781,7 @@ Results include score (0 = most relevant, higher = less relevant). Set explain=t
                 );
                 // Enrich with source information
                 results[key] = relatedChunks.map((related) => {
-                    if ((0, raw_data_utils_js_1.isRawDataPath)(related.filePath)) {
+                    if ((0, raw_data_utils_js_1.isManagedRawDataPath)(this.dbPath, related.filePath)) {
                         const source = (0, raw_data_utils_js_1.extractSourceFromPath)(related.filePath);
                         if (source) {
                             return { ...related, source };

package/dist/server/raw-data-utils.d.ts

@@ -70,6 +70,15 @@ export declare function saveRawData(dbPath: string, source: string, content: str
  * @returns True if path is in raw-data directory
  */
 export declare function isRawDataPath(filePath: string): boolean;
+/**
+ * Check whether a path belongs to this database's managed raw-data directory.
+ * This is stricter than isRawDataPath() and should be used for trust decisions.
+ *
+ * @param dbPath - LanceDB database path for the current server
+ * @param filePath - File path to validate
+ * @returns True only if filePath is inside <dbPath>/raw-data
+ */
+export declare function isManagedRawDataPath(dbPath: string, filePath: string): boolean;
 /**
  * Extract original source from raw-data file path
  * Returns null if not a raw-data path

package/dist/server/raw-data-utils.js

@@ -10,6 +10,7 @@ exports.getRawDataDir = getRawDataDir;
 exports.generateRawDataPath = generateRawDataPath;
 exports.saveRawData = saveRawData;
 exports.isRawDataPath = isRawDataPath;
+exports.isManagedRawDataPath = isManagedRawDataPath;
 exports.extractSourceFromPath = extractSourceFromPath;
 const promises_1 = require("node:fs/promises");
 const node_path_1 = require("node:path");
@@ -167,6 +168,20 @@ const RAW_DATA_NATIVE = `${node_path_1.sep}raw-data${node_path_1.sep}`;
 function isRawDataPath(filePath) {
     return filePath.includes(RAW_DATA_NATIVE) || filePath.includes(RAW_DATA_POSIX);
 }
+/**
+ * Check whether a path belongs to this database's managed raw-data directory.
+ * This is stricter than isRawDataPath() and should be used for trust decisions.
+ *
+ * @param dbPath - LanceDB database path for the current server
+ * @param filePath - File path to validate
+ * @returns True only if filePath is inside <dbPath>/raw-data
+ */
+function isManagedRawDataPath(dbPath, filePath) {
+    const rawDataDir = (0, node_path_1.resolve)(getRawDataDir(dbPath));
+    const resolvedPath = (0, node_path_1.resolve)(filePath);
+    const rel = (0, node_path_1.relative)(rawDataDir, resolvedPath);
+    return rel === '' || (!rel.startsWith('..') && !(0, node_path_1.isAbsolute)(rel));
+}
 /**
  * Extract original source from raw-data file path
  * Returns null if not a raw-data path

package/dist/vectordb/index.js

@@ -143,7 +143,8 @@ function isDocumentMetadata(value) {
         return false;
     }
     // Optional custom field must be an object if present
-    if (obj['custom'] !== undefined && (typeof obj['custom'] !== 'object' || obj['custom'] === null)) {
+    if (obj['custom'] !== undefined &&
+        (typeof obj['custom'] !== 'object' || obj['custom'] === null)) {
         return false;
     }
     return true;

package/dist/web/database-manager.d.ts

@@ -246,5 +246,9 @@ export declare class DatabaseManager {
      * Ensure config directory exists
      */
     private ensureConfigDir;
+    /**
+     * Enforce allowed-roots policy for database mutation operations.
+     */
+    private assertPathAllowedForMutation;
 }
 //# sourceMappingURL=database-manager.d.ts.map

package/dist/web/database-manager.js

@@ -226,6 +226,7 @@ class DatabaseManager {
         }
         // Expand tilde to home directory
         const resolvedPath = expandTilde(newDbPath);
+        await this.assertPathAllowedForMutation('Switch path', resolvedPath);
         // Validate the new path exists and is a valid database
         if (!(0, node_fs_1.existsSync)(resolvedPath)) {
             throw new Error(`Database path does not exist: ${resolvedPath}`);
@@ -290,6 +291,7 @@ class DatabaseManager {
     async createDatabase(options) {
         // Expand tilde to home directory
         const resolvedPath = expandTilde(options.dbPath);
+        await this.assertPathAllowedForMutation('Create path', resolvedPath);
         // Check if path already exists
         if ((0, node_fs_1.existsSync)(resolvedPath)) {
             const lanceDbPath = node_path_1.default.join(resolvedPath, LANCEDB_DIR_NAME);
@@ -662,6 +664,7 @@ class DatabaseManager {
      */
     async deleteDatabase(dbPath, deleteFiles = false) {
         const resolvedPath = expandTilde(dbPath);
+        await this.assertPathAllowedForMutation('Delete path', resolvedPath);
         // Cannot delete the currently active database
         if (this.currentConfig && this.currentConfig.dbPath === resolvedPath) {
             throw new Error('Cannot delete the currently active database. Switch to another database first.');
@@ -699,6 +702,18 @@ class DatabaseManager {
             await (0, promises_1.mkdir)(CONFIG_DIR, { recursive: true });
         }
     }
+    /**
+     * Enforce allowed-roots policy for database mutation operations.
+     */
+    async assertPathAllowedForMutation(action, targetPath) {
+        if (await this.isPathAllowed(targetPath)) {
+            return;
+        }
+        const allowedRoots = await this.getEffectiveAllowedRoots();
+        throw new Error(`${action} "${targetPath}" is outside allowed roots. ` +
+            `Allowed: ${allowedRoots.join(', ')}. ` +
+            `Add this path to allowed roots or set ALLOWED_SCAN_ROOTS environment variable.`);
+    }
 }
 exports.DatabaseManager = DatabaseManager;
 //# sourceMappingURL=database-manager.js.map

package/dist/web/http-server.js

@@ -96,6 +96,9 @@ const ALLOWED_MIME_TYPES = [
     'text/markdown',
     'text/html',
     'application/json',
+    'application/x-ndjson',
+    'application/ndjson',
+    'application/jsonl',
 ];
 /**
  * Create and configure Express app with DatabaseManager
@@ -252,7 +255,16 @@ async function createHttpServerInternal(serverAccessor, config, configRouter) {
         },
         fileFilter: (_req, file, cb) => {
             // Allow common document types by MIME type or extension
-            const allowedExtensions = ['.pdf', '.docx', '.txt', '.md', '.html', '.json'];
+            const allowedExtensions = [
+                '.pdf',
+                '.docx',
+                '.txt',
+                '.md',
+                '.html',
+                '.json',
+                '.jsonl',
+                '.ndjson',
+            ];
             const ext = node_path_1.default.extname(file.originalname).toLowerCase();
             if (ALLOWED_MIME_TYPES.includes(file.mimetype) || allowedExtensions.includes(ext)) {
                 cb(null, true);