@robthepcguy/rag-vault 1.5.1 → 1.5.2

package/README.md

@@ -430,6 +430,12 @@ pnpm dev
 
 # Run web server locally
 pnpm web:dev
+
+# Release to npm (local, guarded)
+pnpm release:patch
+pnpm release:minor
+pnpm release:major
+pnpm release:dry
 ```
 
 
@@ -440,11 +446,13 @@ pnpm web:dev
 
 Use `RUN_EMBEDDING_INTEGRATION=1` to explicitly opt into network/model-dependent suites.
 
-### CI Strategy
+### Release Strategy
 
-- `quality.yml` runs on PRs and pushes and enforces the root quality gate (`pnpm check:all`), which includes backend checks and web-ui type/lint/format checks plus unit tests.
-- A nightly scheduled job runs the integration/E2E suite so model-dependent workflows stay healthy without blocking every PR.
-- `publish-npm.yml` publishes to npm on GitHub Releases, validates tag/version alignment, blocks duplicate npm versions, and supports a manual dry-run, while a real publish requires `NPM_TOKEN`.
+- Releases are local and scripted via `scripts/release-npm.sh`.
+- Supported bumps: `patch`, `minor`, `major`.
+- The script runs dependency installs, `pnpm check:all`, and `pnpm ui:build` before touching version files.
+- `package.json` and `server.json` versions are updated only after checks pass, and auto-restored if any later step fails.
+- `pnpm release:dry` performs the full gate plus npm dry-run publish and always restores version files.
 
 ### Project Structure
 

package/dist/vectordb/index.d.ts

@@ -149,6 +149,30 @@ export declare class VectorStore {
      * Record FTS success (resets circuit breaker)
      */
     private recordFtsSuccess;
+    /**
+     * Extract unsupported custom metadata field from LanceDB schema mismatch errors.
+     *
+     * Returns:
+     * - specific key (e.g., "character") for metadata.custom.character mismatch
+     * - CUSTOM_METADATA_ALL_FIELDS when metadata.custom itself is unsupported
+     * - null when error is unrelated
+     */
+    private extractUnsupportedCustomMetadataField;
+    /**
+     * Remove unsupported custom metadata field from chunks for schema compatibility.
+     *
+     * @param chunks - Source chunks
+     * @param field - Unsupported field name, or CUSTOM_METADATA_ALL_FIELDS to drop all custom metadata
+     * @returns Sanitized chunks and whether any changes were applied
+     */
+    private stripUnsupportedCustomMetadata;
+    /**
+     * Add chunks to existing table with automatic fallback for custom metadata schema mismatches.
+     *
+     * LanceDB infers struct fields from early inserts, so later custom metadata keys may fail.
+     * This method retries by stripping only unsupported custom fields when needed.
+     */
+    private addChunksWithSchemaFallback;
     /**
      * Initialize LanceDB and create table
      */

package/dist/vectordb/index.js

@@ -74,6 +74,13 @@ const DELETE_IGNORABLE_PATTERNS = [
     'no rows',
     'empty result',
 ];
+/**
+ * LanceDB schema mismatch pattern for custom metadata fields.
+ * Example: "Found field not in schema: metadata.custom.character at row 0"
+ */
+const CUSTOM_METADATA_SCHEMA_MISMATCH_REGEX = /Found field not in schema:\s*metadata\.custom(?:\.([A-Za-z0-9_-]+))?/i;
+/** Sentinel for "metadata.custom itself is not in schema" */
+const CUSTOM_METADATA_ALL_FIELDS = '__all__';
 /**
  * Regex for validating file paths before use in queries.
  * Allows alphanumeric characters, slashes, dots, underscores, hyphens, colons (Windows), and spaces.
@@ -311,6 +318,108 @@ class VectorStore {
             this.ftsLastFailure = null;
         }
     }
+    /**
+     * Extract unsupported custom metadata field from LanceDB schema mismatch errors.
+     *
+     * Returns:
+     * - specific key (e.g., "character") for metadata.custom.character mismatch
+     * - CUSTOM_METADATA_ALL_FIELDS when metadata.custom itself is unsupported
+     * - null when error is unrelated
+     */
+    extractUnsupportedCustomMetadataField(error) {
+        const message = error instanceof Error ? error.message : typeof error === 'string' ? error : String(error);
+        const match = CUSTOM_METADATA_SCHEMA_MISMATCH_REGEX.exec(message);
+        if (!match)
+            return null;
+        return match[1] || CUSTOM_METADATA_ALL_FIELDS;
+    }
+    /**
+     * Remove unsupported custom metadata field from chunks for schema compatibility.
+     *
+     * @param chunks - Source chunks
+     * @param field - Unsupported field name, or CUSTOM_METADATA_ALL_FIELDS to drop all custom metadata
+     * @returns Sanitized chunks and whether any changes were applied
+     */
+    stripUnsupportedCustomMetadata(chunks, field) {
+        let changed = false;
+        const sanitizedChunks = chunks.map((chunk) => {
+            const custom = chunk.metadata.custom;
+            if (!custom)
+                return chunk;
+            // Entire custom object is unsupported by table schema
+            if (field === CUSTOM_METADATA_ALL_FIELDS) {
+                changed = true;
+                const metadataWithoutCustom = { ...chunk.metadata };
+                delete metadataWithoutCustom.custom;
+                return {
+                    ...chunk,
+                    metadata: metadataWithoutCustom,
+                };
+            }
+            // Specific custom key is unsupported by table schema
+            if (!(field in custom)) {
+                return chunk;
+            }
+            changed = true;
+            const filteredCustom = Object.fromEntries(Object.entries(custom).filter(([key]) => key !== field));
+            if (Object.keys(filteredCustom).length === 0) {
+                const metadataWithoutCustom = { ...chunk.metadata };
+                delete metadataWithoutCustom.custom;
+                return {
+                    ...chunk,
+                    metadata: metadataWithoutCustom,
+                };
+            }
+            return {
+                ...chunk,
+                metadata: {
+                    ...chunk.metadata,
+                    custom: filteredCustom,
+                },
+            };
+        });
+        return { chunks: sanitizedChunks, changed };
+    }
+    /**
+     * Add chunks to existing table with automatic fallback for custom metadata schema mismatches.
+     *
+     * LanceDB infers struct fields from early inserts, so later custom metadata keys may fail.
+     * This method retries by stripping only unsupported custom fields when needed.
+     */
+    async addChunksWithSchemaFallback(chunks) {
+        if (!this.table) {
+            throw new index_js_1.DatabaseError('VectorStore is not initialized. Call initialize() first.');
+        }
+        let chunksToInsert = chunks;
+        const removedFields = new Set();
+        // Multiple unknown keys can surface one-by-one; allow bounded retries.
+        for (let attempt = 0; attempt < 10; attempt++) {
+            try {
+                const records = chunksToInsert.map(toDbRecord);
+                await this.table.add(records);
+                if (removedFields.size > 0) {
+                    const removed = Array.from(removedFields)
+                        .map((field) => (field === CUSTOM_METADATA_ALL_FIELDS ? 'metadata.custom' : field))
+                        .join(', ');
+                    console.warn(`VectorStore: Removed unsupported custom metadata field(s) for schema compatibility: ${removed}`);
+                }
+                return;
+            }
+            catch (error) {
+                const unsupportedField = this.extractUnsupportedCustomMetadataField(error);
+                if (!unsupportedField) {
+                    throw error;
+                }
+                const { chunks: strippedChunks, changed } = this.stripUnsupportedCustomMetadata(chunksToInsert, unsupportedField);
+                if (!changed) {
+                    throw error;
+                }
+                removedFields.add(unsupportedField);
+                chunksToInsert = strippedChunks;
+            }
+        }
+        throw new index_js_1.DatabaseError('Failed to insert chunks after schema fallback retries');
+    }
     /**
      * Initialize LanceDB and create table
      */
@@ -450,8 +559,7 @@ class VectorStore {
                 }
             }
             // Add data to existing table
-            const records = chunksWithFingerprints.map(toDbRecord);
-            await this.table.add(records);
+            await this.addChunksWithSchemaFallback(chunksWithFingerprints);
             // Rebuild FTS index after adding new data
             await this.rebuildFtsIndex();
             console.error(`VectorStore: Inserted ${chunks.length} chunks`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@robthepcguy/rag-vault",
-  "version": "1.5.1",
+  "version": "1.5.2",
   "description": "Local RAG MCP Server - Easy-to-setup document search with minimal configuration",
   "main": "dist/index.js",
   "bin": {
@@ -116,6 +116,11 @@
     "web": "tsx src/web/index.ts",
     "test:unit": "vitest run --project backend-unit --project web-ui",
     "test:integration": "RUN_EMBEDDING_INTEGRATION=1 vitest run --project backend-integration",
-    "hooks:install": "git config core.hooksPath .githooks"
+    "hooks:install": "git config core.hooksPath .githooks",
+    "release:patch": "bash ./scripts/release-npm.sh --bump patch",
+    "release:minor": "bash ./scripts/release-npm.sh --bump minor",
+    "release:major": "bash ./scripts/release-npm.sh --bump major",
+    "release:dry": "bash ./scripts/release-npm.sh --bump patch --dry-run",
+    "release:publish": "bash ./scripts/release-npm.sh --bump patch"
   }
 }