@mcp-monorepo/notion-query 1.2.0 → 1.3.0

package/dist/local-rag/vectordb/manager.js

@@ -0,0 +1,91 @@
+import { buildWhereClause } from './retriever.js'; // Assuming you export this helper
+import { DataMapper } from './schema.js';
+/**
+ * Handles administrative and management tasks for the vector store, like listing and cleaning documents.
+ */
+export class StoreManager {
+    table;
+    constructor(table) {
+        this.table = table;
+    }
+    /**
+     * Lists all unique documents in the store, with optional filtering and pagination.
+     * This method first filters in the DB, then aggregates unique files in memory.
+     * @param options - Optional filters and pagination settings.
+     * @returns A promise that resolves to an array of list items.
+     */
+    async listFiles(options = {}) {
+        const { limit = 20, offset = 0, filters } = options;
+        const query = this.table.query().select(['filePath', 'timestamp', 'metadata']);
+        const whereClause = buildWhereClause(filters);
+        if (whereClause) {
+            query.where(whereClause);
+        }
+        // This still loads all *filtered* records into memory, but it's far more
+        // efficient than loading the entire table, especially with filters applied.
+        const filteredRecords = await query.toArray();
+        const fileMap = new Map();
+        for (const record of filteredRecords) {
+            const filePath = String(record.filePath);
+            const timestamp = String(record.timestamp);
+            const metadata = DataMapper.toDocumentMetadata(record.metadata);
+            const existing = fileMap.get(filePath);
+            if (existing) {
+                existing.chunkCount++;
+                if (timestamp > existing.latestTimestamp) {
+                    existing.latestTimestamp = timestamp;
+                    existing.metadata = metadata;
+                }
+            }
+            else {
+                fileMap.set(filePath, { chunkCount: 1, latestTimestamp: timestamp, metadata });
+            }
+        }
+        const sortedResults = Array.from(fileMap.entries())
+            .map(([filePath, data]) => ({
+            filePath,
+            chunkCount: data.chunkCount,
+            timestamp: data.latestTimestamp,
+            metadata: data.metadata,
+        }))
+            .sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+        // Apply pagination to the final, aggregated list
+        return sortedResults.slice(offset, offset + limit);
+    }
+    /**
+     * Removes all documents (and their chunks) that have expired.
+     * @returns A promise that resolves to the number of documents deleted.
+     */
+    async cleanupExpired() {
+        // LanceDB's WHERE clause is limited for string-based date comparisons.
+        // Fetch candidates and filter locally for safety.
+        const candidates = await this.table.query().where('metadata.expiresAt IS NOT NULL').toArray();
+        const now = new Date().toISOString();
+        const expiredFilePaths = new Set();
+        for (const record of candidates) {
+            const metadata = record.metadata;
+            if (metadata.expiresAt && metadata.expiresAt < now) {
+                expiredFilePaths.add(record.filePath);
+            }
+        }
+        if (expiredFilePaths.size > 0) {
+            const filePathsArray = Array.from(expiredFilePaths);
+            const deletePromises = filePathsArray.map((filePath) => this.table.delete(`\`filePath\` = '${filePath.replace(/'/g, "''")}'`));
+            await Promise.all(deletePromises);
+        }
+        return expiredFilePaths.size;
+    }
+    /**
+     * Retrieves all chunks associated with a specific file path.
+     * @param filePath - The full path of the document (e.g., '/path/to/file.pdf' or 'memory://label').
+     * @returns A promise that resolves to an array of vector chunks.
+     */
+    async getChunksByPath(filePath) {
+        const records = await this.table
+            .query()
+            .where(`\`filePath\` = '${filePath.replace(/'/g, "''")}'`)
+            .toArray();
+        return records.map(DataMapper.toVectorChunk);
+    }
+}
+//# sourceMappingURL=manager.js.map

package/dist/local-rag/vectordb/manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../../src/local-rag/vectordb/manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA,CAAC,kCAAkC;AACpF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAKxC;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,KAAK,CAAO;IAE7B,YAAY,KAAY;QACtB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACpB,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,SAAS,CAAC,UAAuB,EAAE;QAC9C,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,MAAM,GAAG,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAA;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAA;QAE9E,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;QAC7C,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QAC1B,CAAC;QAED,yEAAyE;QACzE,4EAA4E;QAC5E,MAAM,eAAe,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;QAE7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAGpB,CAAA;QAEH,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YACxC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YAC1C,MAAM,QAAQ,GAAG,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAE/D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACtC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,UAAU,EAAE,CAAA;gBACrB,IAAI,SAAS,GAAG,QAAQ,CAAC,eAAe,EAAE,CAAC;oBACzC,QAAQ,CAAC,eAAe,GAAG,SAAS,CAAA;oBACpC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAA;gBAC9B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;YAChF,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;aAChD,GAAG,CACF,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAY,EAAE,CAAC,CAAC;YAC/B,QAAQ;YACR,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,eAAe;YAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC,CACH;aACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;QAEzD,iDAAiD;QACjD,OAAO,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAA;IACpD,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,cAAc;QACzB,uEAAuE;QACvE,kDAAkD;QAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC,OAAO,EAAE,CAAA;QAE7F,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QACpC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;QAC1C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAkC,CAAA;YAC1D,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;gBACnD,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAkB,CAAC,CAAA;YACjD,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YACnD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACrD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CACtE,CAAA;YACD,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QACnC,CAAC;QAED,OAAO,gBAAgB,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,eAAe,CAAC,QAAgB;QAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC7B,KAAK,EAAE;aACP,KAAK,CAAC,mBAAmB,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;aACzD,OAAO,EAAE,CAAA;QACZ,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAA;IAC9C,CAAC;CACF"}

package/dist/local-rag/vectordb/migration.d.ts

@@ -0,0 +1,27 @@
+import type { Connection, Table } from '@lancedb/lancedb';
+/**
+ * Manages the detection and execution of database schema migrations.
+ */
+export declare class SchemaMigrator {
+    private readonly db;
+    private readonly tableName;
+    constructor(db: Connection, tableName: string);
+    /**
+     * Checks if the table exists, and if so, whether it needs migration.
+     * @returns A Promise that resolves to the migrated or validated table, or undefined if the table does not exist.
+     */
+    run(): Promise<Table | undefined>;
+    /**
+     * Checks if the table's schema is outdated.
+     * @param table - The table to inspect.
+     * @returns A Promise that resolves to true if migration is needed.
+     */
+    private _needsMigration;
+    /**
+     * Performs the migration of data from the old table to a new one with the correct schema.
+     * @param oldTable - The outdated table instance.
+     * @returns A Promise that resolves to the new, migrated table instance.
+     */
+    private _migrate;
+}
+//# sourceMappingURL=migration.d.ts.map

package/dist/local-rag/vectordb/migration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"migration.d.ts","sourceRoot":"","sources":["../../../src/local-rag/vectordb/migration.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAEzD;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;gBAEtB,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;IAK7C;;;OAGG;IACU,GAAG,IAAI,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC;IAmB9C;;;;OAIG;YACW,eAAe;IAoB7B;;;;OAIG;YACW,QAAQ;CAsEvB"}

package/dist/local-rag/vectordb/migration.js

@@ -0,0 +1,121 @@
+import { logger } from '@mcp-monorepo/shared';
+import * as arrow from 'apache-arrow';
+import { DatabaseError } from '../errors.js';
+import { getDocumentsSchema } from './schema.js';
+/**
+ * Manages the detection and execution of database schema migrations.
+ */
+export class SchemaMigrator {
+    db;
+    tableName;
+    constructor(db, tableName) {
+        this.db = db;
+        this.tableName = tableName;
+    }
+    /**
+     * Checks if the table exists, and if so, whether it needs migration.
+     * @returns A Promise that resolves to the migrated or validated table, or undefined if the table does not exist.
+     */
+    async run() {
+        const tableNames = await this.db.tableNames();
+        if (!tableNames.includes(this.tableName)) {
+            logger.info(`VectorStore: Table "${this.tableName}" will be created on first data insertion.`);
+            return undefined;
+        }
+        const table = await this.db.openTable(this.tableName);
+        logger.info(`VectorStore: Opened existing table "${this.tableName}".`);
+        if (await this._needsMigration(table)) {
+            logger.warn('VectorStore: Schema migration required. Starting migration process...');
+            return this._migrate(table);
+        }
+        logger.info('VectorStore: Table schema is up-to-date.');
+        return table;
+    }
+    /**
+     * Checks if the table's schema is outdated.
+     * @param table - The table to inspect.
+     * @returns A Promise that resolves to true if migration is needed.
+     */
+    async _needsMigration(table) {
+        try {
+            const schema = await table.schema();
+            const metadataField = schema.fields.find((f) => f.name === 'metadata');
+            if (!metadataField || metadataField.type.typeId !== arrow.Type.Struct) {
+                return true; // Missing metadata struct entirely
+            }
+            const metadataChildren = metadataField.type.children;
+            const fieldNames = new Set(metadataChildren.map((f) => f.name));
+            // Check for key fields introduced in newer schemas.
+            return !fieldNames.has('createdAt') || !fieldNames.has('updatedAt') || !fieldNames.has('tags');
+        }
+        catch (error) {
+            logger.error('VectorStore: Error checking schema, assuming migration is needed.', error);
+            return true;
+        }
+    }
+    /**
+     * Performs the migration of data from the old table to a new one with the correct schema.
+     * @param oldTable - The outdated table instance.
+     * @returns A Promise that resolves to the new, migrated table instance.
+     */
+    async _migrate(oldTable) {
+        try {
+            const allRecords = await oldTable.query().toArray();
+            logger.info(`VectorStore: Read ${allRecords.length} records for migration.`);
+            if (allRecords.length === 0) {
+                await this.db.dropTable(this.tableName);
+                logger.info('VectorStore: Dropped empty table. It will be recreated on the next insert.');
+                // This will result in `run()` returning `undefined`, which is correct.
+                throw new Error('Migration resulted in an empty table, which was dropped.');
+            }
+            const now = new Date().toISOString();
+            const migratedRecords = allRecords.map((record) => {
+                const rawMetadata = (record.metadata ?? {});
+                // Normalize tags from various possible formats
+                let tags = [];
+                if (Array.isArray(rawMetadata.tags)) {
+                    tags = rawMetadata.tags;
+                }
+                else if (rawMetadata.tags &&
+                    typeof rawMetadata.tags.toArray === 'function') {
+                    tags = rawMetadata.tags.toArray();
+                }
+                const migratedMetadata = {
+                    fileName: String(rawMetadata.fileName ?? 'unknown'),
+                    fileSize: Number(rawMetadata.fileSize ?? 0),
+                    fileType: String(rawMetadata.fileType ?? 'unknown'),
+                    language: rawMetadata.language || undefined,
+                    memoryType: rawMetadata.memoryType || undefined,
+                    tags,
+                    project: rawMetadata.project || undefined,
+                    expiresAt: rawMetadata.expiresAt || undefined,
+                    createdAt: String(rawMetadata.createdAt ?? record.timestamp ?? now),
+                    updatedAt: String(rawMetadata.updatedAt ?? record.timestamp ?? now),
+                    sourceUrl: rawMetadata.sourceUrl || undefined,
+                };
+                return {
+                    id: String(record.id),
+                    filePath: String(record.filePath),
+                    chunkIndex: Number(record.chunkIndex),
+                    text: String(record.text),
+                    vector: Array.from(record.vector ?? []),
+                    metadata: migratedMetadata,
+                    timestamp: String(record.timestamp ?? now),
+                };
+            });
+            await this.db.dropTable(this.tableName);
+            logger.info('VectorStore: Dropped old table.');
+            const newTable = await this.db.createTable(this.tableName, migratedRecords, { schema: getDocumentsSchema() });
+            logger.info(`VectorStore: Created new table with ${migratedRecords.length} migrated records. Migration complete.`);
+            return newTable;
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Migration resulted in an empty table')) {
+                // This is an expected outcome, not a true failure. Re-throw to be handled by the caller.
+                throw error;
+            }
+            throw new DatabaseError('Failed to migrate table schema.', error);
+        }
+    }
+}
+//# sourceMappingURL=migration.js.map

package/dist/local-rag/vectordb/migration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migration.js","sourceRoot":"","sources":["../../../src/local-rag/vectordb/migration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAC7C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAErC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAKhD;;GAEG;AACH,MAAM,OAAO,cAAc;IACR,EAAE,CAAY;IACd,SAAS,CAAQ;IAElC,YAAY,EAAc,EAAE,SAAiB;QAC3C,IAAI,CAAC,EAAE,GAAG,EAAE,CAAA;QACZ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,GAAG;QACd,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAA;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,4CAA4C,CAAC,CAAA;YAC9F,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACrD,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAA;QAEtE,IAAI,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAA;YACpF,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC7B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAA;QACvD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,eAAe,CAAC,KAAY;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;YACnC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;YAEtE,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtE,OAAO,IAAI,CAAA,CAAC,mCAAmC;YACjD,CAAC;YAED,MAAM,gBAAgB,GAAI,aAAa,CAAC,IAAqB,CAAC,QAAQ,CAAA;YACtE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;YAE/D,oDAAoD;YACpD,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAChG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,mEAAmE,EAAE,KAAK,CAAC,CAAA;YACxF,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,QAAQ,CAAC,QAAe;QACpC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAA;YACnD,MAAM,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,MAAM,yBAAyB,CAAC,CAAA;YAE5E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACvC,MAAM,CAAC,IAAI,CAAC,4EAA4E,CAAC,CAAA;gBACzF,uEAAuE;gBACvE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;YAC7E,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YACpC,MAAM,eAAe,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAe,EAAE;gBAC7D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAA4B,CAAA;gBAEtE,+CAA+C;gBAC/C,IAAI,IAAI,GAAa,EAAE,CAAA;gBACvB,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpC,IAAI,GAAG,WAAW,CAAC,IAAI,CAAA;gBACzB,CAAC;qBAAM,IACL,WAAW,CAAC,IAAI;oBAChB,OAAQ,WAAW,CAAC,IAAqC,CAAC,OAAO,KAAK,UAAU,EAChF,CAAC;oBACD,IAAI,GAAI,WAAW,CAAC,IAAoC,CAAC,OAAO,EAAE,CAAA;gBACpE,CAAC;gBAED,MAAM,gBAAgB,GAAqB;oBACzC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,IAAI,SAAS,CAAC;oBACnD,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,CAAC;oBAC3C,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,IAAI,SAAS,CAAC;oBACnD,QAAQ,EAAG,WAAW,CAAC,QAAmB,IAAI,SAAS;oBACvD,UAAU,EAAG,WAAW,CAAC,UAAsC,IAAI,SAAS;oBAC5E,IAAI;oBACJ,OAAO,EAAG,WAAW,CAAC,OAAkB,IAAI,SAAS;oBACrD,SAAS,EAAG,WAAW,CAAC,SAAoB,IAAI,SAAS;oBACzD,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;oBACnE,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;oBACnE,SAAS,EAAG,WAAW,CAAC,SAAoB,IAAI,SAAS;iBAC1D,CAAA;gBAED,OAAO;oBACL,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC;oBACjC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;oBACrC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACzB,MAAM,EAAE,KAAK,CAAC,IAAI,CAAE,MAAM,CAAC,MAAuB,IAAI,EAAE,CAAC;oBACzD,QAAQ,EAAE,gBAAgB;oBAC1B,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;iBAC3C,CAAA;YACH,CAAC,CAAC,CAAA;YAEF,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACvC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;YAE9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CACxC,IAAI,CAAC,SAAS,EACd,eAAuD,EACvD,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CACjC,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,MAAM,wCAAwC,CAAC,CAAA;YAClH,OAAO,QAAQ,CAAA;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,sCAAsC,CAAC,EAAE,CAAC;gBAC7F,yFAAyF;gBACzF,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,aAAa,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAA;QACnE,CAAC;IACH,CAAC;CACF"}

package/dist/local-rag/vectordb/retriever.d.ts

@@ -0,0 +1,51 @@
+import type { GroupingMode, QueryFilters, QueryResult } from '../types.js';
+import type { Table } from '@lancedb/lancedb';
+export interface RetrieverConfig {
+    hybridWeight?: number;
+    maxDistance?: number;
+    grouping?: GroupingMode;
+    ftsEnabled: boolean;
+}
+/**
+ * Handles the complex logic of searching and retrieving data from the vector store.
+ */
+export declare class Retriever {
+    private readonly table;
+    private readonly config;
+    constructor(table: Table, config: RetrieverConfig);
+    setFtsEnabled(enabled: boolean): void;
+    /**
+     * Executes a search query, handling hybrid search, filtering, and result processing.
+     * @param queryVector - The vector representation of the query.
+     * @param queryText - The original text of the query for FTS.
+     * @param limit - The maximum number of results to return.
+     * @param filters - Optional filters to apply to the search.
+     * @returns A promise that resolves to an array of query results.
+     */
+    search(queryVector: number[], queryText: string, limit?: number, filters?: QueryFilters): Promise<QueryResult[]>;
+    /**
+     * Performs a hybrid search by combining FTS and vector search results.
+     */
+    private _performHybridSearch;
+    /**
+     * Performs a standard vector-only search.
+     */
+    private _performVectorSearch;
+    /**
+     * Filters results based on the configured maximum distance.
+     */
+    private _applyDistanceFilter;
+    /**
+     * Groups results by relevance gaps to improve quality.
+     */
+    private _applyGrouping;
+    /**
+     * Reranks and merges FTS and vector search results.
+     */
+    private _hybridRerank;
+}
+/**
+ * Constructs a SQL WHERE clause from the filter object.
+ */
+export declare function buildWhereClause(filters?: QueryFilters): string | undefined;
+//# sourceMappingURL=retriever.d.ts.map

package/dist/local-rag/vectordb/retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retriever.d.ts","sourceRoot":"","sources":["../../../src/local-rag/vectordb/retriever.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAC1E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAO7C,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB,UAAU,EAAE,OAAO,CAAA;CACpB;AAED;;GAEG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAO;IAC7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;gBAE5B,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe;IAK1C,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAI5C;;;;;;;OAOG;IACU,MAAM,CACjB,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,KAAK,SAAK,EACV,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,WAAW,EAAE,CAAC;IA2BzB;;OAEG;YACW,oBAAoB;IAmBlC;;OAEG;YACW,oBAAoB;IAUlC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;OAEG;IACH,OAAO,CAAC,cAAc;IAwBtB;;OAEG;IACH,OAAO,CAAC,aAAa;CAiCtB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,MAAM,GAAG,SAAS,CAa3E"}

package/dist/local-rag/vectordb/retriever.js

@@ -0,0 +1,157 @@
+import { DatabaseError, ValidationError } from '../errors.js';
+import { DataMapper } from './schema.js';
+// Constants for quality retrieval logic
+const GROUPING_BOUNDARY_STD_MULTIPLIER = 1.5;
+const HYBRID_SEARCH_CANDIDATE_MULTIPLIER = 3;
+const DOT_PRODUCT_MAX_DISTANCE = 2; // Theoretical max for dot product on normalized vectors
+/**
+ * Handles the complex logic of searching and retrieving data from the vector store.
+ */
+export class Retriever {
+    table;
+    config;
+    constructor(table, config) {
+        this.table = table;
+        this.config = config;
+    }
+    setFtsEnabled(enabled) {
+        this.config.ftsEnabled = enabled;
+    }
+    /**
+     * Executes a search query, handling hybrid search, filtering, and result processing.
+     * @param queryVector - The vector representation of the query.
+     * @param queryText - The original text of the query for FTS.
+     * @param limit - The maximum number of results to return.
+     * @param filters - Optional filters to apply to the search.
+     * @returns A promise that resolves to an array of query results.
+     */
+    async search(queryVector, queryText, limit = 10, filters) {
+        if (limit < 1 || limit > 50) {
+            throw new ValidationError(`Invalid limit: expected 1-50, got ${limit}`);
+        }
+        try {
+            const whereClause = buildWhereClause(filters);
+            let rawResults;
+            const useHybrid = this.config.ftsEnabled && queryText.trim().length > 0 && (this.config.hybridWeight ?? 0.6) > 0;
+            if (useHybrid) {
+                rawResults = await this._performHybridSearch(queryVector, queryText, limit, whereClause);
+            }
+            else {
+                rawResults = await this._performVectorSearch(queryVector, limit, whereClause);
+            }
+            let results = rawResults.map(DataMapper.toQueryResult);
+            results = this._applyDistanceFilter(results);
+            results = this._applyGrouping(results);
+            return results;
+        }
+        catch (error) {
+            throw new DatabaseError('Failed to search vectors.', error);
+        }
+    }
+    /**
+     * Performs a hybrid search by combining FTS and vector search results.
+     */
+    async _performHybridSearch(queryVector, queryText, limit, whereClause) {
+        const candidateLimit = limit * HYBRID_SEARCH_CANDIDATE_MULTIPLIER;
+        const ftsQuery = this.table.search(queryText, 'fts', 'text').limit(candidateLimit);
+        if (whereClause)
+            ftsQuery.where(whereClause);
+        const ftsResults = await ftsQuery.toArray();
+        const vectorQuery = this.table.vectorSearch(queryVector).distanceType('dot').limit(candidateLimit);
+        if (whereClause)
+            vectorQuery.where(whereClause);
+        const vectorResults = await vectorQuery.toArray();
+        return this._hybridRerank(ftsResults, vectorResults, limit);
+    }
+    /**
+     * Performs a standard vector-only search.
+     */
+    async _performVectorSearch(queryVector, limit, whereClause) {
+        const query = this.table.vectorSearch(queryVector).distanceType('dot').limit(limit);
+        if (whereClause)
+            query.where(whereClause);
+        return query.toArray();
+    }
+    /**
+     * Filters results based on the configured maximum distance.
+     */
+    _applyDistanceFilter(results) {
+        const maxDistance = this.config.maxDistance;
+        if (maxDistance !== undefined) {
+            return results.filter((r) => r.score <= maxDistance);
+        }
+        return results;
+    }
+    /**
+     * Groups results by relevance gaps to improve quality.
+     */
+    _applyGrouping(results) {
+        if (!this.config.grouping || results.length <= 1) {
+            return results;
+        }
+        const gaps = [];
+        for (let i = 0; i < results.length - 1; i++) {
+            gaps.push({ index: i + 1, gap: results[i + 1].score - results[i].score });
+        }
+        const gapValues = gaps.map((g) => g.gap);
+        const mean = gapValues.reduce((a, b) => a + b, 0) / gapValues.length;
+        const std = Math.sqrt(gapValues.map((x) => (x - mean) ** 2).reduce((a, b) => a + b, 0) / gapValues.length);
+        const threshold = mean + GROUPING_BOUNDARY_STD_MULTIPLIER * std;
+        const boundaries = gaps.filter((g) => g.gap > threshold).map((g) => g.index);
+        if (boundaries.length === 0)
+            return results;
+        const groupsToInclude = this.config.grouping === 'similar' ? 1 : 2;
+        const cutoffIndex = boundaries[groupsToInclude - 1];
+        return cutoffIndex !== undefined ? results.slice(0, cutoffIndex) : results;
+    }
+    /**
+     * Reranks and merges FTS and vector search results.
+     */
+    _hybridRerank(ftsResults, vectorResults, limit) {
+        const scoreMap = new Map();
+        const bm25Weight = this.config.hybridWeight ?? 0.6;
+        const vectorWeight = 1.0 - bm25Weight;
+        for (const result of vectorResults) {
+            const key = `${result['filePath']}:${result['chunkIndex']}`;
+            const distance = result['_distance'] ?? DOT_PRODUCT_MAX_DISTANCE;
+            const vectorScore = Math.max(0, 1 - distance / DOT_PRODUCT_MAX_DISTANCE);
+            scoreMap.set(key, { result, score: vectorScore * vectorWeight });
+        }
+        for (let i = 0; i < ftsResults.length; i++) {
+            const result = ftsResults[i];
+            const key = `${result['filePath']}:${result['chunkIndex']}`;
+            const ftsScore = 1 - i / (ftsResults.length || 1);
+            const entry = scoreMap.get(key);
+            if (entry) {
+                entry.score += ftsScore * bm25Weight;
+            }
+            else {
+                scoreMap.set(key, { result, score: ftsScore * bm25Weight });
+            }
+        }
+        return Array.from(scoreMap.values())
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit)
+            .map((item) => ({ ...item.result, _distance: 1 - item.score }));
+    }
+}
+/**
+ * Constructs a SQL WHERE clause from the filter object.
+ */
+export function buildWhereClause(filters) {
+    if (!filters)
+        return undefined;
+    const conditions = [];
+    if (filters.type)
+        conditions.push(`metadata.memoryType = '${filters.type}'`);
+    if (filters.project)
+        conditions.push(`metadata.project = '${filters.project.replace(/'/g, "''")}'`);
+    if (filters.fileName)
+        conditions.push(`metadata.fileName = '${filters.fileName.replace(/'/g, "''")}'`);
+    if (filters.tags && filters.tags.length > 0) {
+        const tagsList = `[${filters.tags.map((t) => `'${t.replace(/'/g, "''")}'`).join(', ')}]`;
+        conditions.push(`array_has_all(metadata.tags, ${tagsList})`);
+    }
+    return conditions.length > 0 ? conditions.join(' AND ') : undefined;
+}
+//# sourceMappingURL=retriever.js.map

package/dist/local-rag/vectordb/retriever.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retriever.js","sourceRoot":"","sources":["../../../src/local-rag/vectordb/retriever.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAKxC,wCAAwC;AACxC,MAAM,gCAAgC,GAAG,GAAG,CAAA;AAC5C,MAAM,kCAAkC,GAAG,CAAC,CAAA;AAC5C,MAAM,wBAAwB,GAAG,CAAC,CAAA,CAAC,wDAAwD;AAS3F;;GAEG;AACH,MAAM,OAAO,SAAS;IACH,KAAK,CAAO;IACZ,MAAM,CAAiB;IAExC,YAAY,KAAY,EAAE,MAAuB;QAC/C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAEM,aAAa,CAAC,OAAgB;QACnC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,OAAO,CAAA;IAClC,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,MAAM,CACjB,WAAqB,EACrB,SAAiB,EACjB,KAAK,GAAG,EAAE,EACV,OAAsB;QAEtB,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,eAAe,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;YAC7C,IAAI,UAAqC,CAAA;YAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAChH,IAAI,SAAS,EAAE,CAAC;gBACd,UAAU,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,CAAC,CAAA;YAC1F,CAAC;iBAAM,CAAC;gBACN,UAAU,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,OAAO,GAAkB,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAA;YAErE,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAA;YAC5C,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YAEtC,OAAO,OAAO,CAAA;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAA;QAC7D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,WAAqB,EACrB,SAAiB,EACjB,KAAa,EACb,WAAoB;QAEpB,MAAM,cAAc,GAAG,KAAK,GAAG,kCAAkC,CAAA;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;QAClF,IAAI,WAAW;YAAE,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QAC5C,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QAE3C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;QAClG,IAAI,WAAW;YAAE,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QAC/C,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,CAAA;QAEjD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,KAAK,CAAC,CAAA;IAC7D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,WAAqB,EACrB,KAAa,EACb,WAAoB;QAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACnF,IAAI,WAAW;YAAE,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QACzC,OAAO,KAAK,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,OAAsB;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAA;QAC3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,WAAW,CAAC,CAAA;QACtD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjD,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,MAAM,IAAI,GAAqC,EAAE,CAAA;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAC3E,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAA;QACpE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;QAC1G,MAAM,SAAS,GAAG,IAAI,GAAG,gCAAgC,GAAG,GAAG,CAAA;QAE/D,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAC5E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,OAAO,CAAA;QAE3C,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAClE,MAAM,WAAW,GAAG,UAAU,CAAC,eAAe,GAAG,CAAC,CAAC,CAAA;QAEnD,OAAO,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;IAC5E,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,UAAqC,EACrC,aAAwC,EACxC,KAAa;QAEb,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA8D,CAAA;QACtF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAA;QAClD,MAAM,YAAY,GAAG,GAAG,GAAG,UAAU,CAAA;QAErC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;YAC3D,MAAM,QAAQ,GAAI,MAAM,CAAC,WAAW,CAAY,IAAI,wBAAwB,CAAA;YAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,GAAG,wBAAwB,CAAC,CAAA;YACxE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE,CAAC,CAAA;QAClE,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;YAC3D,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC,CAAA;YACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC/B,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,KAAK,IAAI,QAAQ,GAAG,UAAU,CAAA;YACtC,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,UAAU,EAAE,CAAC,CAAA;YAC7D,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;aACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAA;IAC9B,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,IAAI,OAAO,CAAC,IAAI;QAAE,UAAU,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,IAAI,GAAG,CAAC,CAAA;IAC5E,IAAI,OAAO,CAAC,OAAO;QAAE,UAAU,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;IACnG,IAAI,OAAO,CAAC,QAAQ;QAAE,UAAU,CAAC,IAAI,CAAC,wBAAwB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;IACtG,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;QACxF,UAAU,CAAC,IAAI,CAAC,gCAAgC,QAAQ,GAAG,CAAC,CAAA;IAC9D,CAAC;IAED,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;AACrE,CAAC"}

package/dist/local-rag/vectordb/schema.d.ts

@@ -0,0 +1,33 @@
+import * as arrow from 'apache-arrow';
+import type { DocumentMetadata, QueryResult, VectorChunk } from '../types.js';
+/**
+ * Defines the robust Arrow schema for the vector table.
+ * This schema serves as the single source of truth for the table structure.
+ * @returns {arrow.Schema} The Apache Arrow schema definition.
+ */
+export declare function getDocumentsSchema(): arrow.Schema;
+/**
+ * A utility class for safely mapping raw database records to strictly-typed application objects.
+ * Handles type conversions and guards against undefined or malformed data.
+ */
+export declare class DataMapper {
+    /**
+     * Safely converts a raw metadata object from the database into a strict DocumentMetadata type.
+     * @param raw - The raw metadata object, typically from a LanceDB record.
+     * @returns A structured DocumentMetadata object.
+     */
+    static toDocumentMetadata(raw: unknown): DocumentMetadata;
+    /**
+     * Converts a raw database record into a QueryResult object.
+     * @param raw - The raw database record.
+     * @returns A structured QueryResult object.
+     */
+    static toQueryResult(raw: Record<string, unknown>): QueryResult;
+    /**
+     * Converts a raw database record into a VectorChunk object.
+     * @param raw - The raw database record.
+     * @returns A structured VectorChunk object.
+     */
+    static toVectorChunk(raw: Record<string, unknown>): VectorChunk;
+}
+//# sourceMappingURL=schema.d.ts.map

package/dist/local-rag/vectordb/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/local-rag/vectordb/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAErC,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAE7E;;;;GAIG;AACH,wBAAgB,kBAAkB,IAAI,KAAK,CAAC,MAAM,CA2BjD;AAED;;;GAGG;AACH,qBAAa,UAAU;IACrB;;;;OAIG;WACW,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,gBAAgB;IA6BhE;;;;OAIG;WACW,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,WAAW;IAUtE;;;;OAIG;WACW,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,WAAW;CAWvE"}

package/dist/local-rag/vectordb/schema.js

@@ -0,0 +1,102 @@
+import * as arrow from 'apache-arrow';
+/**
+ * Defines the robust Arrow schema for the vector table.
+ * This schema serves as the single source of truth for the table structure.
+ * @returns {arrow.Schema} The Apache Arrow schema definition.
+ */
+export function getDocumentsSchema() {
+    const metadataFields = [
+        new arrow.Field('fileName', new arrow.Utf8(), false),
+        new arrow.Field('fileSize', new arrow.Int32(), false),
+        new arrow.Field('fileType', new arrow.Utf8(), false),
+        new arrow.Field('language', new arrow.Utf8(), true),
+        new arrow.Field('memoryType', new arrow.Utf8(), true),
+        new arrow.Field('tags', new arrow.List(new arrow.Field('item', new arrow.Utf8(), false)), true),
+        new arrow.Field('project', new arrow.Utf8(), true),
+        new arrow.Field('expiresAt', new arrow.Utf8(), true),
+        new arrow.Field('createdAt', new arrow.Utf8(), false),
+        new arrow.Field('updatedAt', new arrow.Utf8(), false),
+        new arrow.Field('sourceUrl', new arrow.Utf8(), true),
+        new arrow.Field('author', new arrow.Utf8(), true),
+        new arrow.Field('fileCreatedAt', new arrow.Utf8(), true),
+        new arrow.Field('fileModifiedAt', new arrow.Utf8(), true),
+    ];
+    return new arrow.Schema([
+        new arrow.Field('id', new arrow.Utf8(), false),
+        new arrow.Field('filePath', new arrow.Utf8(), false),
+        new arrow.Field('chunkIndex', new arrow.Int32(), false),
+        new arrow.Field('text', new arrow.Utf8(), false),
+        new arrow.Field('vector', new arrow.FixedSizeList(384, new arrow.Field('item', new arrow.Float32(), false)), false),
+        new arrow.Field('metadata', new arrow.Struct(metadataFields), true),
+        new arrow.Field('timestamp', new arrow.Utf8(), false),
+    ]);
+}
+/**
+ * A utility class for safely mapping raw database records to strictly-typed application objects.
+ * Handles type conversions and guards against undefined or malformed data.
+ */
+export class DataMapper {
+    /**
+     * Safely converts a raw metadata object from the database into a strict DocumentMetadata type.
+     * @param raw - The raw metadata object, typically from a LanceDB record.
+     * @returns A structured DocumentMetadata object.
+     */
+    static toDocumentMetadata(raw) {
+        const data = raw;
+        // LanceDB can return Arrow vectors for lists, so we need to safely convert them.
+        const rawTags = data.tags;
+        let tags = [];
+        if (Array.isArray(rawTags)) {
+            tags = rawTags;
+        }
+        else if (rawTags && typeof rawTags.toArray === 'function') {
+            tags = rawTags.toArray();
+        }
+        return {
+            fileName: String(data.fileName ?? ''),
+            fileSize: typeof data.fileSize === 'number' ? data.fileSize : 0,
+            fileType: String(data.fileType ?? ''),
+            language: typeof data.language === 'string' ? data.language : undefined,
+            tags,
+            project: typeof data.project === 'string' ? data.project : undefined,
+            memoryType: data.memoryType === 'text' || data.memoryType === 'file' || data.memoryType === 'url'
+                ? data.memoryType
+                : undefined,
+            expiresAt: typeof data.expiresAt === 'string' ? data.expiresAt : undefined,
+            createdAt: String(data.createdAt ?? new Date().toISOString()),
+            updatedAt: String(data.updatedAt ?? new Date().toISOString()),
+            sourceUrl: typeof data.sourceUrl === 'string' ? data.sourceUrl : undefined,
+        };
+    }
+    /**
+     * Converts a raw database record into a QueryResult object.
+     * @param raw - The raw database record.
+     * @returns A structured QueryResult object.
+     */
+    static toQueryResult(raw) {
+        return {
+            filePath: String(raw.filePath ?? ''),
+            chunkIndex: raw.chunkIndex,
+            text: String(raw.text ?? ''),
+            score: raw._distance,
+            metadata: DataMapper.toDocumentMetadata(raw.metadata),
+        };
+    }
+    /**
+     * Converts a raw database record into a VectorChunk object.
+     * @param raw - The raw database record.
+     * @returns A structured VectorChunk object.
+     */
+    static toVectorChunk(raw) {
+        return {
+            id: String(raw.id ?? ''),
+            filePath: String(raw.filePath ?? ''),
+            chunkIndex: raw.chunkIndex,
+            text: String(raw.text ?? ''),
+            vector: Array.from(raw.vector ?? []),
+            metadata: this.toDocumentMetadata(raw.metadata),
+            timestamp: String(raw.timestamp ?? ''),
+        };
+    }
+}
+//# sourceMappingURL=schema.js.map

package/dist/local-rag/vectordb/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/local-rag/vectordb/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAIrC;;;;GAIG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,cAAc,GAAG;QACrB,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACnD,IAAI,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QAC/F,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QAClD,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACpD,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACpD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACjD,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;QACxD,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC;KAC1D,CAAA;IAED,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAC9C,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QACpD,IAAI,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC;QACvD,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAChD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,KAAK,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;QACnH,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC;QACnE,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;KACtD,CAAC,CAAA;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,UAAU;IACrB;;;;OAIG;IACI,MAAM,CAAC,kBAAkB,CAAC,GAAY;QAC3C,MAAM,IAAI,GAAG,GAA8B,CAAA;QAC3C,iFAAiF;QACjF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAe,CAAA;QACpC,IAAI,IAAI,GAAa,EAAE,CAAA;QACvB,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,IAAI,GAAG,OAAO,CAAA;QAChB,CAAC;aAAM,IAAI,OAAO,IAAI,OAAQ,OAAwC,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAC9F,IAAI,GAAI,OAAuC,CAAC,OAAO,EAAE,CAAA;QAC3D,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YACrC,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC/D,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YACrC,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACvE,IAAI;YACJ,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;YACpE,UAAU,EACR,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK;gBACnF,CAAC,CAAC,IAAI,CAAC,UAAU;gBACjB,CAAC,CAAC,SAAS;YACf,SAAS,EAAE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YAC1E,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC7D,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC7D,SAAS,EAAE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SAC3E,CAAA;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,aAAa,CAAC,GAA4B;QACtD,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;YACpC,UAAU,EAAE,GAAG,CAAC,UAAoB;YACpC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAC5B,KAAK,EAAE,GAAG,CAAC,SAAmB;YAC9B,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;SACtD,CAAA;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,aAAa,CAAC,GAA4B;QACtD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;YACxB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;YACpC,UAAU,EAAE,GAAG,CAAC,UAAoB;YACpC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,KAAK,CAAC,IAAI,CAAE,GAAG,CAAC,MAAuB,IAAI,EAAE,CAAC;YACtD,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC/C,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;SACvC,CAAA;IACH,CAAC;CACF"}