@tenex-chat/backend 0.9.5 → 0.9.7

package/src/services/rag/RAGCollectionRegistry.ts

@@ -0,0 +1,199 @@
+/**
+ * RAGCollectionRegistry - Tracks collection metadata for scope-aware search.
+ *
+ * Stores scope, project, and agent metadata for each RAG collection as a
+ * JSON sidecar file alongside the LanceDB data directory. This enables
+ * rag_search() to automatically include only relevant collections based on
+ * the agent's context (global, project, personal).
+ *
+ * IMPORTANT: This is NOT access control. No requests are ever denied.
+ * Scoping only determines default visibility when rag_search() runs
+ * without an explicit `collections` parameter.
+ */
+
+import { logger } from "@/utils/logger";
+import { getLanceDBDataDir } from "./rag-utils";
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+/** Scope types for RAG collections */
+export type CollectionScope = "global" | "project" | "personal";
+
+/** Metadata stored for each registered collection */
+export interface CollectionMetadata {
+    /** Scope determines default visibility in rag_search() */
+    scope: CollectionScope;
+
+    /** Project ID (NIP-33 address) that created the collection */
+    projectId?: string;
+
+    /** Agent pubkey that created the collection */
+    agentPubkey?: string;
+
+    /** When the collection was first registered */
+    createdAt: number;
+}
+
+/** Shape of the sidecar JSON file */
+interface RegistryData {
+    /** Schema version for future migrations */
+    version: 1;
+
+    /** Map of collection name → metadata */
+    collections: Record<string, CollectionMetadata>;
+}
+
+const REGISTRY_FILENAME = "collection-registry.json";
+
+export class RAGCollectionRegistry {
+    private static instance: RAGCollectionRegistry | null = null;
+    private data: RegistryData;
+    private readonly filePath: string;
+
+    private constructor() {
+        this.filePath = path.join(getLanceDBDataDir(), REGISTRY_FILENAME);
+        this.data = this.load();
+    }
+
+    public static getInstance(): RAGCollectionRegistry {
+        if (!RAGCollectionRegistry.instance) {
+            RAGCollectionRegistry.instance = new RAGCollectionRegistry();
+        }
+        return RAGCollectionRegistry.instance;
+    }
+
+    /**
+     * Register a collection with scope metadata.
+     * Overwrites existing metadata for the same collection name.
+     */
+    public register(
+        name: string,
+        metadata: Omit<CollectionMetadata, "createdAt">
+    ): void {
+        const existing = this.data.collections[name];
+        this.data.collections[name] = {
+            ...metadata,
+            createdAt: existing?.createdAt ?? Date.now(),
+        };
+        this.save();
+        logger.debug(`[RAGCollectionRegistry] Registered collection '${name}'`, {
+            scope: metadata.scope,
+            projectId: metadata.projectId,
+        });
+    }
+
+    /**
+     * Get metadata for a specific collection.
+     * Returns undefined for unregistered (legacy) collections.
+     */
+    public get(name: string): CollectionMetadata | undefined {
+        return this.data.collections[name];
+    }
+
+    /**
+     * Remove a collection from the registry.
+     */
+    public unregister(name: string): void {
+        if (this.data.collections[name]) {
+            delete this.data.collections[name];
+            this.save();
+            logger.debug(`[RAGCollectionRegistry] Unregistered collection '${name}'`);
+        }
+    }
+
+    /**
+     * Get all collections that match the given context.
+     *
+     * Matching rules:
+     * - `global` collections: always included
+     * - `project` collections: included when projectId matches
+     * - `personal` collections: included when agentPubkey matches
+     * - Unregistered (legacy) collections: treated as global
+     *
+     * @param allCollections - All known collection names (from RAGService.listCollections)
+     * @param projectId - Current project ID
+     * @param agentPubkey - Current agent's pubkey
+     * @returns Collection names that match the context
+     */
+    public getMatchingCollections(
+        allCollections: string[],
+        projectId: string,
+        agentPubkey?: string
+    ): string[] {
+        return allCollections.filter((name) => {
+            const metadata = this.data.collections[name];
+
+            // Unregistered (legacy) collections are treated as global
+            if (!metadata) return true;
+
+            switch (metadata.scope) {
+                case "global":
+                    return true;
+                case "project":
+                    return metadata.projectId === projectId;
+                case "personal":
+                    return metadata.agentPubkey === agentPubkey;
+                default:
+                    // Unknown scope — treat as global for safety
+                    return true;
+            }
+        });
+    }
+
+    /**
+     * Get all registered collection metadata.
+     * Returns a deep copy to prevent callers from mutating internal state.
+     */
+    public getAll(): Record<string, CollectionMetadata> {
+        return JSON.parse(JSON.stringify(this.data.collections));
+    }
+
+    /**
+     * Load registry from the sidecar JSON file.
+     * Returns empty registry if file doesn't exist or is corrupted.
+     */
+    private load(): RegistryData {
+        try {
+            if (fs.existsSync(this.filePath)) {
+                const raw = fs.readFileSync(this.filePath, "utf-8");
+                const parsed = JSON.parse(raw) as RegistryData;
+                if (parsed.version === 1 && parsed.collections) {
+                    return parsed;
+                }
+                logger.warn("[RAGCollectionRegistry] Unknown registry version, starting fresh");
+            }
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.warn("[RAGCollectionRegistry] Failed to load registry, starting fresh", {
+                error: message,
+            });
+        }
+        return { version: 1, collections: {} };
+    }
+
+    /**
+     * Persist registry to the sidecar JSON file.
+     */
+    private save(): void {
+        try {
+            // Ensure directory exists
+            const dir = path.dirname(this.filePath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            fs.writeFileSync(this.filePath, JSON.stringify(this.data, null, 2), "utf-8");
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.error("[RAGCollectionRegistry] Failed to save registry", {
+                error: message,
+            });
+        }
+    }
+
+    /**
+     * Reset singleton (for testing).
+     */
+    public static resetInstance(): void {
+        RAGCollectionRegistry.instance = null;
+    }
+}

package/src/services/rag/RAGDatabaseService.ts

@@ -1,7 +1,6 @@
-import { config } from "@/services/ConfigService";
-import * as path from "node:path";
 import { handleError } from "@/utils/error-handler";
 import { logger } from "@/utils/logger";
+import { getLanceDBDataDir } from "./rag-utils";
 import { type Connection, type Table, connect } from "@lancedb/lancedb";
 
 /**
@@ -27,11 +26,7 @@ export class RAGDatabaseService {
     private tableCache: Map<string, Table> = new Map();
 
     constructor(dataDir?: string) {
-        // Use provided directory or environment variable, fallback to global location
-        this.dataDir =
-            dataDir ||
-            process.env.LANCEDB_DATA_DIR ||
-            path.join(config.getConfigPath("data"), "lancedb");
+        this.dataDir = dataDir || getLanceDBDataDir();
 
         logger.debug(`RAGDatabaseService initialized with data directory: ${this.dataDir}`);
     }

package/src/services/rag/RAGOperations.ts

@@ -1,5 +1,7 @@
 import type { DocumentMetadata, LanceDBResult, LanceDBStoredDocument } from "@/services/rag/rag-utils";
 import { calculateRelevanceScore, mapLanceResultToDocument } from "@/services/rag/rag-utils";
+import { AGENT_PUBKEY_KEYS } from "@/utils/metadataKeys";
+import { SQL_LIKE_ESCAPE_CLAUSE, escapeSqlLikeValue } from "@/utils/sqlEscaping";
 import { handleError } from "@/utils/error-handler";
 import { logger } from "@/utils/logger";
 import type { Table, VectorQuery } from "@lancedb/lancedb";
@@ -469,33 +471,13 @@ export class RAGOperations {
      * @param agentPubkey Optional agent pubkey to count their contributions
      * @returns Object with total count and optional agent-specific count
      */
-    /**
-     * Escape a string for use in SQL LIKE pattern.
-     * Escapes: single quotes ('), double quotes ("), backslashes (\), and LIKE wildcards (%, _).
-     *
-     * IMPORTANT: DataFusion (used by LanceDB) has NO default escape character.
-     * The backslash escapes here only work when paired with ESCAPE '\\' clause.
-     * See: https://github.com/apache/datafusion/issues/13291
-     *
-     * Note: Agent pubkeys are hex strings (0-9, a-f) so most escaping isn't strictly needed,
-     * but we escape properly for defense-in-depth and to handle any future metadata fields.
-     */
-    private escapeSqlLikeValue(value: string): string {
-        return value
-            .replace(/\\/g, "\\\\")  // Escape backslashes first
-            .replace(/'/g, "''")      // SQL standard: escape single quote by doubling
-            .replace(/"/g, '\\"')     // Escape double quotes
-            .replace(/%/g, "\\%")     // Escape LIKE wildcard %
-            .replace(/_/g, "\\_");    // Escape LIKE wildcard _
-    }
-
     async getCollectionStats(
         collectionName: string,
         agentPubkey?: string
     ): Promise<{ totalCount: number; agentCount?: number }> {
-        const table = await this.dbManager.getTable(collectionName);
-
         try {
+            const table = await this.dbManager.getTable(collectionName);
+
             // Get total document count
             const totalCount = await table.countRows();
 
@@ -503,11 +485,11 @@ export class RAGOperations {
             // The metadata field is stored as JSON string, so we use LIKE for matching
             let agentCount: number | undefined;
             if (agentPubkey) {
-                // SQL filter for JSON string field containing agent_pubkey
-                // Format: metadata LIKE '%"agent_pubkey":"<pubkey>"%' ESCAPE '\\'
-                // ESCAPE clause is required because DataFusion has no default escape character
-                const escapedPubkey = this.escapeSqlLikeValue(agentPubkey);
-                const filter = `metadata LIKE '%"agent_pubkey":"${escapedPubkey}"%' ESCAPE '\\\\'`;
+                const escaped = escapeSqlLikeValue(agentPubkey);
+                const clauses = AGENT_PUBKEY_KEYS
+                    .map((key) => `metadata LIKE '%"${key}":"${escaped}"%' ${SQL_LIKE_ESCAPE_CLAUSE}`)
+                    .join(" OR ");
+                const filter = `(${clauses})`;
                 agentCount = await table.countRows(filter);
             }
 
@@ -532,28 +514,26 @@ export class RAGOperations {
     ): Promise<Array<{ name: string; agentDocCount: number; totalDocCount: number }>> {
         const collections = await this.listCollections();
 
-        const stats = await Promise.all(
+        const results = await Promise.allSettled(
             collections.map(async (name) => {
-                try {
-                    const { totalCount, agentCount } = await this.getCollectionStats(name, agentPubkey);
-                    return {
-                        name,
-                        agentDocCount: agentCount ?? 0,
-                        totalDocCount: totalCount,
-                    };
-                } catch (error) {
-                    // Log but don't fail - return zero counts for problematic collections
-                    logger.warn(`Failed to get stats for collection '${name}':`, error);
-                    return {
-                        name,
-                        agentDocCount: 0,
-                        totalDocCount: 0,
-                    };
-                }
+                const { totalCount, agentCount } = await this.getCollectionStats(name, agentPubkey);
+                return {
+                    name,
+                    agentDocCount: agentCount ?? 0,
+                    totalDocCount: totalCount,
+                };
             })
         );
 
-        return stats;
+        return results
+            .filter((r): r is PromiseFulfilledResult<{ name: string; agentDocCount: number; totalDocCount: number }> => {
+                if (r.status === "rejected") {
+                    logger.warn(`Failed to get stats for collection: ${r.reason}`);
+                    return false;
+                }
+                return true;
+            })
+            .map(r => r.value);
     }
 
     /**

package/src/services/rag/RAGService.ts

@@ -5,36 +5,6 @@ import { RAGDatabaseService } from "./RAGDatabaseService";
 import { RAGOperations } from "./RAGOperations";
 import type { BulkUpsertResult, LanceDBSchema, RAGCollection, RAGDocument, RAGQueryResult } from "./RAGOperations";
 
-/**
- * Lightweight check for RAG collections without full service initialization.
- * Returns true if any RAG collections exist in the database.
- *
- * This is designed for use in system prompt building where we want to avoid
- * initializing the embedding provider (which can be expensive) if RAG isn't used.
- */
-export async function hasRagCollections(): Promise<boolean> {
-    let tempDbService: RAGDatabaseService | null = null;
-    try {
-        // Use a temporary DB service just for listing - avoids embedding provider init
-        tempDbService = new RAGDatabaseService();
-        const tables = await tempDbService.listTables();
-        return tables.length > 0;
-    } catch (error) {
-        // Database doesn't exist or can't connect - no collections
-        logger.debug("RAG database not available for collection check:", error);
-        return false;
-    } finally {
-        // Ensure cleanup even on listTables() errors; log but don't override result
-        if (tempDbService) {
-            try {
-                await tempDbService.close();
-            } catch (closeError) {
-                logger.debug("Failed to close temporary RAG database service:", closeError);
-            }
-        }
-    }
-}
-
 /**
  * Facade for RAG functionality
  * Coordinates between database management and operations
@@ -237,4 +207,3 @@ export class RAGService {
 export type { BulkUpsertResult, RAGDocument, RAGCollection, RAGQueryResult } from "./RAGOperations";
 export { RAGValidationError, RAGOperationError } from "./RAGOperations";
 export { RAGDatabaseError } from "./RAGDatabaseService";
-// hasRagCollections is exported at module level (above class)

package/src/services/rag/RagSubscriptionService.ts

@@ -197,72 +197,42 @@ export class RagSubscriptionService {
             });
         };
 
+        // Validate that resourceUri is a proper URI format
         try {
-            // Validate that resourceUri is a proper URI format
-            try {
-                new URL(subscription.resourceUri);
-            } catch {
-                throw new Error(
-                    `Invalid resourceUri: "${subscription.resourceUri}". Resource URI must be a valid URI format (e.g., "nostr://feed/pubkey/kinds", "file:///path/to/file"). This appears to be a tool name or invalid format. If you're using a resource template, you must first expand it with parameters to get the actual URI.`
-                );
-            }
+            new URL(subscription.resourceUri);
+        } catch {
+            throw new Error(
+                `Invalid resourceUri: "${subscription.resourceUri}". Resource URI must be a valid URI format (e.g., "nostr://feed/pubkey/kinds", "file:///path/to/file"). This appears to be a tool name or invalid format. If you're using a resource template, you must first expand it with parameters to get the actual URI.`
+            );
+        }
 
-            // Get mcpManager from project context
-            const projectCtx = getProjectContext();
-            const mcpManager = projectCtx.mcpManager;
-            if (!mcpManager) {
-                throw new Error("MCPManager not available in project context");
-            }
+        // Get mcpManager from project context
+        const projectCtx = getProjectContext();
+        const mcpManager = projectCtx.mcpManager;
+        if (!mcpManager) {
+            throw new Error("MCPManager not available in project context");
+        }
 
-            // Try to subscribe to resource updates
-            let subscriptionSupported = true;
-            try {
-                // CRITICAL: Register handler FIRST, then subscribe
-                const removeHandler = mcpManager.addResourceNotificationHandler(subscription.mcpServerId, listener);
+        // CRITICAL: Register handler FIRST, then subscribe
+        const removeHandler = mcpManager.addResourceNotificationHandler(subscription.mcpServerId, listener);
 
-                // Subscribe to resource updates
-                await mcpManager.subscribeToResource(
-                    subscription.mcpServerId,
-                    subscription.resourceUri
-                );
-
-                // Store listener reference and removal function for cleanup
-                this.resourceListeners.set(
-                    listenerKey,
-                    listener as unknown as (notification: Notification) => void
-                );
-                this.handlerRemovers.set(listenerKey, removeHandler);
+        // Subscribe to resource updates — if server doesn't support subscriptions, let it throw
+        await mcpManager.subscribeToResource(
+            subscription.mcpServerId,
+            subscription.resourceUri
+        );
 
-                logger.info(
-                    `RAG subscription '${subscription.subscriptionId}' active with push notifications. ` +
-                        `Listening for updates from ${subscription.mcpServerId}:${subscription.resourceUri}`
-                );
-            } catch (error) {
-                if (
-                    error instanceof Error &&
-                    error.message.includes("does not support resource subscriptions")
-                ) {
-                    // Server doesn't support subscriptions, gracefully degrade to polling
-                    subscriptionSupported = false;
-                    logger.warn(
-                        `Server '${subscription.mcpServerId}' does not support resource subscriptions. ` +
-                        `Subscription '${subscription.subscriptionId}' will use polling mode. ` +
-                        `Call pollResource() manually or set up a polling interval.`
-                    );
-                    // Don't exit - gracefully degrade to polling mode
-                } else {
-                    throw error;
-                }
-            }
+        // Store listener reference and removal function for cleanup
+        this.resourceListeners.set(
+            listenerKey,
+            listener as unknown as (notification: Notification) => void
+        );
+        this.handlerRemovers.set(listenerKey, removeHandler);
 
-            if (!subscriptionSupported) {
-                subscription.lastError = "Server does not support subscriptions - use polling mode";
-            }
-        } catch (error) {
-            subscription.status = SubscriptionStatus.ERROR;
-            subscription.lastError = error instanceof Error ? error.message : "Unknown error";
-            throw error;
-        }
+        logger.info(
+            `RAG subscription '${subscription.subscriptionId}' active with push notifications. ` +
+                `Listening for updates from ${subscription.mcpServerId}:${subscription.resourceUri}`
+        );
     }
 
     /**
@@ -272,75 +242,54 @@ export class RagSubscriptionService {
         subscription: RagSubscription,
         notification: Notification
     ): Promise<void> {
-        try {
-            // Extract content from notification
-            const content = this.extractContentFromNotification(notification);
+        const content = this.extractContentFromNotification(notification);
 
-            if (!content) {
-                logger.warn(
-                    `Received empty update for subscription '${subscription.subscriptionId}'`
-                );
-                return;
-            }
+        if (!content) {
+            throw new Error(
+                `Received empty update for subscription '${subscription.subscriptionId}'`
+            );
+        }
 
-            // Build metadata with provenance fields for filtering at query time
-            const metadata: DocumentMetadata = {
-                // Auto-inject provenance for filtering at query time
-                agent_pubkey: subscription.agentPubkey,
-                // Subscription-specific metadata
-                subscriptionId: subscription.subscriptionId,
-                mcpServerId: subscription.mcpServerId,
-                resourceUri: subscription.resourceUri,
-                timestamp: Date.now(),
-            };
-
-            // Add project_id if available from project context (uses NIP-33 address format)
-            if (isProjectContextInitialized()) {
-                try {
-                    const projectCtx = getProjectContext();
-                    const projectId = projectCtx.project.tagId();
-                    if (projectId) {
-                        metadata.project_id = projectId;
-                    }
-                } catch (error) {
-                    logger.debug("Project context error during RAG subscription update", { error });
-                }
+        // Build metadata with provenance fields for filtering at query time
+        const metadata: DocumentMetadata = {
+            agent_pubkey: subscription.agentPubkey,
+            subscriptionId: subscription.subscriptionId,
+            mcpServerId: subscription.mcpServerId,
+            resourceUri: subscription.resourceUri,
+            timestamp: Date.now(),
+        };
+
+        // Add project_id if available from project context (uses NIP-33 address format)
+        if (isProjectContextInitialized()) {
+            const projectCtx = getProjectContext();
+            const projectId = projectCtx.project.tagId();
+            if (projectId) {
+                metadata.project_id = projectId;
             }
+        }
 
-            // Add document to RAG collection
-            await this.ragService.addDocuments(subscription.ragCollection, [
-                {
-                    content,
-                    metadata,
-                    source: `${subscription.mcpServerId}:${subscription.resourceUri}`,
-                    timestamp: Date.now(),
-                },
-            ]);
-
-            // Update subscription metrics
-            subscription.documentsProcessed++;
-            subscription.lastDocumentIngested = content.substring(0, 200); // Store snippet
-            subscription.updatedAt = Date.now();
-            subscription.status = SubscriptionStatus.RUNNING;
-            subscription.lastError = undefined;
+        // Add document to RAG collection
+        await this.ragService.addDocuments(subscription.ragCollection, [
+            {
+                content,
+                metadata,
+                source: `${subscription.mcpServerId}:${subscription.resourceUri}`,
+                timestamp: Date.now(),
+            },
+        ]);
 
-            await this.saveSubscriptions();
+        // Update subscription metrics
+        subscription.documentsProcessed++;
+        subscription.lastDocumentIngested = content.substring(0, 200);
+        subscription.updatedAt = Date.now();
+        subscription.status = SubscriptionStatus.RUNNING;
+        subscription.lastError = undefined;
 
-            logger.debug(
-                `Processed update for subscription '${subscription.subscriptionId}', total documents: ${subscription.documentsProcessed}`
-            );
-        } catch (error) {
-            subscription.status = SubscriptionStatus.ERROR;
-            subscription.lastError = error instanceof Error ? error.message : "Unknown error";
-            subscription.updatedAt = Date.now();
-            await this.saveSubscriptions();
+        await this.saveSubscriptions();
 
-            handleError(
-                error,
-                `Failed to process update for subscription '${subscription.subscriptionId}'`,
-                { logLevel: "error" }
-            );
-        }
+        logger.debug(
+            `Processed update for subscription '${subscription.subscriptionId}', total documents: ${subscription.documentsProcessed}`
+        );
     }
 
     /**

package/src/services/rag/rag-utils.ts

@@ -1,4 +1,17 @@
+import { config } from "@/services/ConfigService";
 import { logger } from "@/utils/logger";
+import * as path from "node:path";
+
+/**
+ * Resolve the LanceDB data directory path.
+ * Centralizes env/config fallback logic used by multiple RAG services.
+ */
+export function getLanceDBDataDir(): string {
+    return (
+        process.env.LANCEDB_DATA_DIR ||
+        path.join(config.getConfigPath("data"), "lancedb")
+    );
+}
 
 /**
  * JSON-serializable primitive value.