@robthepcguy/rag-vault 1.5.0 → 1.5.2

package/README.md

@@ -397,6 +397,7 @@ Copy the `DB_PATH` directory (default: `./lancedb/`).
 | File too large | Default limit is 100MB. Set `MAX_FILE_SIZE` higher or split the file. |
 | Path outside BASE_DIR | All file paths must be under `BASE_DIR`. Use absolute paths. |
 | MCP tools not showing | Verify config syntax, restart your AI tool completely (Cmd+Q on Mac). |
+| `mcp-publisher login github` fails with `slow_down` | Use token login instead: `mcp-publisher login github --token "$(gh auth token)"` (or pass a PAT). |
 | 401 Unauthorized | API key required. Set `RAG_API_KEY` or use correct header format. |
 | 429 Too Many Requests | Rate limited. Wait for reset or increase `RATE_LIMIT_MAX_REQUESTS`. |
 | CORS errors | Add your origin to `CORS_ORIGINS` environment variable. |
@@ -429,6 +430,12 @@ pnpm dev
 
 # Run web server locally
 pnpm web:dev
+
+# Release to npm (local, guarded)
+pnpm release:patch
+pnpm release:minor
+pnpm release:major
+pnpm release:dry
 ```
 
 
@@ -439,11 +446,13 @@ pnpm web:dev
 
 Use `RUN_EMBEDDING_INTEGRATION=1` to explicitly opt into network/model-dependent suites.
 
-### CI Strategy
+### Release Strategy
 
-- `quality.yml` runs on PRs and pushes and enforces the root quality gate (`pnpm check:all`), which includes backend checks and web-ui type/lint/format checks plus unit tests.
-- A nightly scheduled job runs the integration/E2E suite so model-dependent workflows stay healthy without blocking every PR.
-- `publish-npm.yml` publishes to npm on GitHub Releases, validates tag/version alignment, blocks duplicate npm versions, and supports a manual dry-run, while a real publish requires `NPM_TOKEN`.
+- Releases are local and scripted via `scripts/release-npm.sh`.
+- Supported bumps: `patch`, `minor`, `major`.
+- The script runs dependency installs, `pnpm check:all`, and `pnpm ui:build` before touching version files.
+- `package.json` and `server.json` versions are updated only after checks pass, and auto-restored if any later step fails.
+- `pnpm release:dry` performs the full gate plus npm dry-run publish and always restores version files.
 
 ### Project Structure
 

package/dist/vectordb/index.d.ts

@@ -149,6 +149,30 @@ export declare class VectorStore {
      * Record FTS success (resets circuit breaker)
      */
     private recordFtsSuccess;
+    /**
+     * Extract unsupported custom metadata field from LanceDB schema mismatch errors.
+     *
+     * Returns:
+     * - specific key (e.g., "character") for metadata.custom.character mismatch
+     * - CUSTOM_METADATA_ALL_FIELDS when metadata.custom itself is unsupported
+     * - null when error is unrelated
+     */
+    private extractUnsupportedCustomMetadataField;
+    /**
+     * Remove unsupported custom metadata field from chunks for schema compatibility.
+     *
+     * @param chunks - Source chunks
+     * @param field - Unsupported field name, or CUSTOM_METADATA_ALL_FIELDS to drop all custom metadata
+     * @returns Sanitized chunks and whether any changes were applied
+     */
+    private stripUnsupportedCustomMetadata;
+    /**
+     * Add chunks to existing table with automatic fallback for custom metadata schema mismatches.
+     *
+     * LanceDB infers struct fields from early inserts, so later custom metadata keys may fail.
+     * This method retries by stripping only unsupported custom fields when needed.
+     */
+    private addChunksWithSchemaFallback;
     /**
      * Initialize LanceDB and create table
      */

package/dist/vectordb/index.js

@@ -74,6 +74,13 @@ const DELETE_IGNORABLE_PATTERNS = [
     'no rows',
     'empty result',
 ];
+/**
+ * LanceDB schema mismatch pattern for custom metadata fields.
+ * Example: "Found field not in schema: metadata.custom.character at row 0"
+ */
+const CUSTOM_METADATA_SCHEMA_MISMATCH_REGEX = /Found field not in schema:\s*metadata\.custom(?:\.([A-Za-z0-9_-]+))?/i;
+/** Sentinel for "metadata.custom itself is not in schema" */
+const CUSTOM_METADATA_ALL_FIELDS = '__all__';
 /**
  * Regex for validating file paths before use in queries.
  * Allows alphanumeric characters, slashes, dots, underscores, hyphens, colons (Windows), and spaces.
@@ -311,6 +318,108 @@ class VectorStore {
             this.ftsLastFailure = null;
         }
     }
+    /**
+     * Extract unsupported custom metadata field from LanceDB schema mismatch errors.
+     *
+     * Returns:
+     * - specific key (e.g., "character") for metadata.custom.character mismatch
+     * - CUSTOM_METADATA_ALL_FIELDS when metadata.custom itself is unsupported
+     * - null when error is unrelated
+     */
+    extractUnsupportedCustomMetadataField(error) {
+        const message = error instanceof Error ? error.message : typeof error === 'string' ? error : String(error);
+        const match = CUSTOM_METADATA_SCHEMA_MISMATCH_REGEX.exec(message);
+        if (!match)
+            return null;
+        return match[1] || CUSTOM_METADATA_ALL_FIELDS;
+    }
+    /**
+     * Remove unsupported custom metadata field from chunks for schema compatibility.
+     *
+     * @param chunks - Source chunks
+     * @param field - Unsupported field name, or CUSTOM_METADATA_ALL_FIELDS to drop all custom metadata
+     * @returns Sanitized chunks and whether any changes were applied
+     */
+    stripUnsupportedCustomMetadata(chunks, field) {
+        let changed = false;
+        const sanitizedChunks = chunks.map((chunk) => {
+            const custom = chunk.metadata.custom;
+            if (!custom)
+                return chunk;
+            // Entire custom object is unsupported by table schema
+            if (field === CUSTOM_METADATA_ALL_FIELDS) {
+                changed = true;
+                const metadataWithoutCustom = { ...chunk.metadata };
+                delete metadataWithoutCustom.custom;
+                return {
+                    ...chunk,
+                    metadata: metadataWithoutCustom,
+                };
+            }
+            // Specific custom key is unsupported by table schema
+            if (!(field in custom)) {
+                return chunk;
+            }
+            changed = true;
+            const filteredCustom = Object.fromEntries(Object.entries(custom).filter(([key]) => key !== field));
+            if (Object.keys(filteredCustom).length === 0) {
+                const metadataWithoutCustom = { ...chunk.metadata };
+                delete metadataWithoutCustom.custom;
+                return {
+                    ...chunk,
+                    metadata: metadataWithoutCustom,
+                };
+            }
+            return {
+                ...chunk,
+                metadata: {
+                    ...chunk.metadata,
+                    custom: filteredCustom,
+                },
+            };
+        });
+        return { chunks: sanitizedChunks, changed };
+    }
+    /**
+     * Add chunks to existing table with automatic fallback for custom metadata schema mismatches.
+     *
+     * LanceDB infers struct fields from early inserts, so later custom metadata keys may fail.
+     * This method retries by stripping only unsupported custom fields when needed.
+     */
+    async addChunksWithSchemaFallback(chunks) {
+        if (!this.table) {
+            throw new index_js_1.DatabaseError('VectorStore is not initialized. Call initialize() first.');
+        }
+        let chunksToInsert = chunks;
+        const removedFields = new Set();
+        // Multiple unknown keys can surface one-by-one; allow bounded retries.
+        for (let attempt = 0; attempt < 10; attempt++) {
+            try {
+                const records = chunksToInsert.map(toDbRecord);
+                await this.table.add(records);
+                if (removedFields.size > 0) {
+                    const removed = Array.from(removedFields)
+                        .map((field) => (field === CUSTOM_METADATA_ALL_FIELDS ? 'metadata.custom' : field))
+                        .join(', ');
+                    console.warn(`VectorStore: Removed unsupported custom metadata field(s) for schema compatibility: ${removed}`);
+                }
+                return;
+            }
+            catch (error) {
+                const unsupportedField = this.extractUnsupportedCustomMetadataField(error);
+                if (!unsupportedField) {
+                    throw error;
+                }
+                const { chunks: strippedChunks, changed } = this.stripUnsupportedCustomMetadata(chunksToInsert, unsupportedField);
+                if (!changed) {
+                    throw error;
+                }
+                removedFields.add(unsupportedField);
+                chunksToInsert = strippedChunks;
+            }
+        }
+        throw new index_js_1.DatabaseError('Failed to insert chunks after schema fallback retries');
+    }
     /**
      * Initialize LanceDB and create table
      */
@@ -323,15 +432,15 @@ class VectorStore {
             if (tableNames.includes(this.config.tableName)) {
                 // Open existing table
                 this.table = await this.db.openTable(this.config.tableName);
-                console.log(`VectorStore: Opened existing table "${this.config.tableName}"`);
+                console.error(`VectorStore: Opened existing table "${this.config.tableName}"`);
                 // Ensure FTS index exists (migration for existing databases)
                 await this.ensureFtsIndex();
             }
             else {
                 // Create new table (schema auto-defined on first data insertion)
-                console.log(`VectorStore: Table "${this.config.tableName}" will be created on first data insertion`);
+                console.error(`VectorStore: Table "${this.config.tableName}" will be created on first data insertion`);
             }
-            console.log(`VectorStore initialized: ${this.config.dbPath}`);
+            console.error(`VectorStore initialized: ${this.config.dbPath}`);
         }
         catch (error) {
             // Clean up partially initialized resources on failure
@@ -365,7 +474,7 @@ class VectorStore {
     async deleteChunks(filePath) {
         if (!this.table) {
             // If table doesn't exist, no deletion targets, return normally
-            console.log('VectorStore: Skipping deletion as table does not exist');
+            console.error('VectorStore: Skipping deletion as table does not exist');
             return;
         }
         // Validate file path before use in query to prevent SQL injection
@@ -381,7 +490,7 @@ class VectorStore {
             // so call delete directly
             // Note: Field names are case-sensitive, use backticks for camelCase fields
             await this.table.delete(`\`filePath\` = '${escapedFilePath}'`);
-            console.log(`VectorStore: Deleted chunks for file "${filePath}"`);
+            console.error(`VectorStore: Deleted chunks for file "${filePath}"`);
             // Rebuild FTS index after deleting data
             await this.rebuildFtsIndex();
         }
@@ -435,7 +544,7 @@ class VectorStore {
                         // Convert to LanceDB record format using explicit field mapping
                         const records = chunksWithFingerprints.map(toDbRecord);
                         this.table = await this.db.createTable(this.config.tableName, records);
-                        console.log(`VectorStore: Created table "${this.config.tableName}"`);
+                        console.error(`VectorStore: Created table "${this.config.tableName}"`);
                         // Create FTS index for hybrid search
                         await this.ensureFtsIndex();
                     })();
@@ -445,16 +554,15 @@ class VectorStore {
                     finally {
                         this.tableCreationPromise = null;
                     }
-                    console.log(`VectorStore: Inserted ${chunks.length} chunks`);
+                    console.error(`VectorStore: Inserted ${chunks.length} chunks`);
                     return;
                 }
             }
             // Add data to existing table
-            const records = chunksWithFingerprints.map(toDbRecord);
-            await this.table.add(records);
+            await this.addChunksWithSchemaFallback(chunksWithFingerprints);
             // Rebuild FTS index after adding new data
             await this.rebuildFtsIndex();
-            console.log(`VectorStore: Inserted ${chunks.length} chunks`);
+            console.error(`VectorStore: Inserted ${chunks.length} chunks`);
         }
         catch (error) {
             throw new index_js_1.DatabaseError('Failed to insert chunks', error);
@@ -492,12 +600,12 @@ class VectorStore {
             name: FTS_INDEX_NAME,
         });
         this.ftsEnabled = true;
-        console.log(`VectorStore: FTS index "${FTS_INDEX_NAME}" created successfully`);
+        console.error(`VectorStore: FTS index "${FTS_INDEX_NAME}" created successfully`);
         // Drop old FTS indices
         for (const idx of existingFtsIndices) {
             if (idx.name !== FTS_INDEX_NAME) {
                 await this.table.dropIndex(idx.name);
-                console.log(`VectorStore: Dropped old FTS index "${idx.name}"`);
+                console.error(`VectorStore: Dropped old FTS index "${idx.name}"`);
             }
         }
     }
@@ -579,7 +687,7 @@ class VectorStore {
      */
     async search(queryVector, queryText, limit = 10) {
         if (!this.table) {
-            console.log('VectorStore: Returning empty results as table does not exist');
+            console.error('VectorStore: Returning empty results as table does not exist');
             return [];
         }
         if (limit < 1 || limit > 20) {
@@ -779,7 +887,7 @@ class VectorStore {
         this.ftsEnabled = false;
         this.ftsFailureCount = 0;
         this.ftsLastFailure = null;
-        console.log('VectorStore: Connection closed');
+        console.error('VectorStore: Connection closed');
         // Propagate errors to caller after cleanup is complete
         if (errors.length > 0) {
             throw new index_js_1.DatabaseError(`Errors during close: ${errors.map((e) => e.message).join('; ')}`, errors[0]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@robthepcguy/rag-vault",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Local RAG MCP Server - Easy-to-setup document search with minimal configuration",
   "main": "dist/index.js",
   "bin": {
@@ -41,42 +41,6 @@
     "type": "git",
     "url": "git+https://github.com/RobThePCGuy/rag-vault.git"
   },
-  "scripts": {
-    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
-    "check": "pnpm type-check && pnpm lint && pnpm format:check",
-    "check:all": "pnpm check && pnpm check:web-ui && pnpm check:unused && pnpm check:deps && pnpm build && pnpm test:unit",
-    "check:deps": "madge --circular --extensions ts src",
-    "check:deps:graph": "madge --extensions ts --image graph.svg src",
-    "check:web-ui": "pnpm --prefix web-ui check",
-    "check:unused": "node scripts/check-unused-exports.js",
-    "check:unused:all": "knip",
-    "cleanup:processes": "bash ./scripts/cleanup-test-processes.sh",
-    "clean:dev": "rm -rf ./node_modules ./tmp ./uploads ./models ./lancedb ./dist ./package-lock.json && cd web-ui && rm -rf ./dist ./node_modules ./package-lock.json",
-    "dev": "tsx src/index.ts",
-    "format": "biome format --write src",
-    "format:check": "biome format src",
-    "lint": "biome lint src",
-    "lint:fix": "biome lint --write src",
-    "start": "node dist/index.js",
-    "test": "vitest run",
-    "test:coverage": "vitest run --coverage",
-    "test:safe": "pnpm test && pnpm cleanup:processes",
-    "test:watch": "vitest",
-    "type-check": "tsc --noEmit",
-    "audit": "pnpm audit --audit-level=moderate",
-    "audit:fix": "pnpm audit --fix",
-    "setup:web": "pnpm install && pnpm web:build && pnpm --prefix web-ui install && pnpm ui:build && pnpm web:start",
-    "ui:build": "pnpm --prefix web-ui build",
-    "ui:dev": "cd web-ui && pnpm dev",
-    "web:build": "pnpm build",
-    "web:dev": "concurrently -n api,ui -c blue,magenta \"pnpm web:watch\" \"pnpm --prefix web-ui dev\"",
-    "web:start": "node dist/web/index.js",
-    "web:watch": "tsx watch src/web/index.ts",
-    "web": "tsx src/web/index.ts",
-    "test:unit": "vitest run --project backend-unit --project web-ui",
-    "test:integration": "RUN_EMBEDDING_INTEGRATION=1 vitest run --project backend-integration",
-    "hooks:install": "git config core.hooksPath .githooks"
-  },
   "dependencies": {
     "@huggingface/transformers": "^3.7.6",
     "@lancedb/lancedb": "^0.23.0",
@@ -118,17 +82,45 @@
     "node": ">=20"
   },
   "mcpName": "io.github.RobThePCGuy/rag-vault",
-  "pnpm": {
-    "overrides": {
-      "tar": ">=7.5.7",
-      "diff": ">=4.0.4"
-    },
-    "onlyBuiltDependencies": [
-      "esbuild",
-      "onnxruntime-node",
-      "protobufjs",
-      "@robthepcguy/rag-vault",
-      "sharp"
-    ]
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
+    "check": "pnpm type-check && pnpm lint && pnpm format:check",
+    "check:all": "pnpm check && pnpm check:web-ui && pnpm check:unused && pnpm check:deps && pnpm build && pnpm test:unit",
+    "check:deps": "madge --circular --extensions ts src",
+    "check:deps:graph": "madge --extensions ts --image graph.svg src",
+    "check:web-ui": "pnpm --prefix web-ui check",
+    "check:unused": "node scripts/check-unused-exports.js",
+    "check:unused:all": "knip",
+    "cleanup:processes": "bash ./scripts/cleanup-test-processes.sh",
+    "clean:dev": "rm -rf ./node_modules ./tmp ./uploads ./models ./lancedb ./dist ./package-lock.json && cd web-ui && rm -rf ./dist ./node_modules ./package-lock.json",
+    "dev": "tsx src/index.ts",
+    "format": "biome format --write src",
+    "format:check": "biome format src",
+    "lint": "biome lint src",
+    "lint:fix": "biome lint --write src",
+    "start": "node dist/index.js",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "test:safe": "pnpm test && pnpm cleanup:processes",
+    "test:watch": "vitest",
+    "type-check": "tsc --noEmit",
+    "audit": "pnpm audit --audit-level=moderate",
+    "audit:fix": "pnpm audit --fix",
+    "setup:web": "pnpm install && pnpm web:build && pnpm --prefix web-ui install && pnpm ui:build && pnpm web:start",
+    "ui:build": "pnpm --prefix web-ui build",
+    "ui:dev": "cd web-ui && pnpm dev",
+    "web:build": "pnpm build",
+    "web:dev": "concurrently -n api,ui -c blue,magenta \"pnpm web:watch\" \"pnpm --prefix web-ui dev\"",
+    "web:start": "node dist/web/index.js",
+    "web:watch": "tsx watch src/web/index.ts",
+    "web": "tsx src/web/index.ts",
+    "test:unit": "vitest run --project backend-unit --project web-ui",
+    "test:integration": "RUN_EMBEDDING_INTEGRATION=1 vitest run --project backend-integration",
+    "hooks:install": "git config core.hooksPath .githooks",
+    "release:patch": "bash ./scripts/release-npm.sh --bump patch",
+    "release:minor": "bash ./scripts/release-npm.sh --bump minor",
+    "release:major": "bash ./scripts/release-npm.sh --bump major",
+    "release:dry": "bash ./scripts/release-npm.sh --bump patch --dry-run",
+    "release:publish": "bash ./scripts/release-npm.sh --bump patch"
   }
-}
+}