@tenex-chat/backend 0.9.4 → 0.9.6

package/src/services/ral/RALRegistry.ts

@@ -1,10 +1,8 @@
 import { trace } from "@opentelemetry/api";
 import { EventEmitter, type DefaultEventMap } from "tseep";
-import { getPubkeyService } from "@/services/PubkeyService";
 import { INJECTION_ABORT_REASON, llmOpsRegistry } from "@/services/LLMOperationsRegistry";
 import { shortenConversationId } from "@/utils/conversation-id";
 import { logger } from "@/utils/logger";
-import { PREFIX_LENGTH } from "@/utils/nostr-entity-parser";
 import { ConversationStore } from "@/conversations/ConversationStore";
 // Note: FullEventId type is available via @/types/event-ids for future typed method signatures
 import type {
@@ -1499,181 +1497,6 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
     return rals.find(ral => ral.queuedInjections.length > 0);
   }
 
-  /**
-   * Build a message containing delegation results for injection into the RAL.
-   * Shows complete conversation transcript for each delegation.
-   * Uses shortened delegation IDs (PREFIX_LENGTH chars) for display; agents can use
-   * these prefixes directly with delegate_followup which will resolve them.
-   * Format: [@sender -> @recipient]: message content
-   */
-  /**
-   * Helper to format transcript with error handling for name resolution.
-   */
-  private async formatTranscript(transcript: DelegationMessage[]): Promise<string[]> {
-    const pubkeyService = getPubkeyService();
-    const lines: string[] = [];
-
-    for (const msg of transcript) {
-      try {
-        const senderName = await pubkeyService.getName(msg.senderPubkey);
-        const recipientName = await pubkeyService.getName(msg.recipientPubkey);
-        lines.push(`[@${senderName} -> @${recipientName}]: ${msg.content}`);
-      } catch (error) {
-        // Fallback to shortened pubkeys on error
-        const senderFallback = msg.senderPubkey.substring(0, 12);
-        const recipientFallback = msg.recipientPubkey.substring(0, 12);
-        lines.push(`[@${senderFallback} -> @${recipientFallback}]: ${msg.content}`);
-      }
-    }
-
-    return lines;
-  }
-
-  /**
-   * Helper to render a list of pending delegations with error handling.
-   */
-  private async renderPendingList(pending: PendingDelegation[]): Promise<string[]> {
-    if (pending.length === 0) {
-      return [];
-    }
-
-    const pubkeyService = getPubkeyService();
-    const lines: string[] = [];
-
-    lines.push("## Still Pending");
-    for (const p of pending) {
-      try {
-        const recipientName = await pubkeyService.getName(p.recipientPubkey);
-        lines.push(`- @${recipientName} (${p.delegationConversationId.substring(0, PREFIX_LENGTH)})`);
-      } catch (error) {
-        const fallbackName = p.recipientPubkey.substring(0, 12);
-        lines.push(`- @${fallbackName} (${p.delegationConversationId.substring(0, PREFIX_LENGTH)})`);
-      }
-    }
-    lines.push("");
-
-    return lines;
-  }
-
-  /**
-   * Helper to render delegation header (agent name + ID) with error handling.
-   */
-  private async renderDelegationHeader(
-    recipientPubkey: string,
-    conversationId: string,
-    statusText: string
-  ): Promise<string[]> {
-    const pubkeyService = getPubkeyService();
-    const lines: string[] = [];
-
-    try {
-      const recipientName = await pubkeyService.getName(recipientPubkey);
-      lines.push(`**@${recipientName} ${statusText}**`);
-    } catch (error) {
-      const fallbackName = recipientPubkey.substring(0, 12);
-      lines.push(`**@${fallbackName} ${statusText}**`);
-    }
-
-    lines.push("");
-    lines.push(`## Delegation ID: ${conversationId.substring(0, PREFIX_LENGTH)}`);
-
-    return lines;
-  }
-
-  /**
-   * Build a message for completed delegations.
-   * Shows the agent name, conversation ID, and full transcript.
-   */
-  async buildDelegationResultsMessage(
-    completions: CompletedDelegation[],
-    pending: PendingDelegation[] = []
-  ): Promise<string> {
-    if (completions.length === 0) {
-      return "";
-    }
-
-    const lines: string[] = [];
-
-    lines.push("# DELEGATION COMPLETED");
-    lines.push("");
-
-    for (const c of completions) {
-      lines.push(...await this.renderDelegationHeader(
-        c.recipientPubkey,
-        c.delegationConversationId,
-        "has finished and returned their final response."
-      ));
-      lines.push("");
-      lines.push("### Transcript:");
-      lines.push(...await this.formatTranscript(c.transcript));
-      lines.push("");
-    }
-
-    // Show pending delegations if any remain
-    if (pending.length > 0) {
-      lines.push(...await this.renderPendingList(pending));
-    }
-
-    return lines.join("\n");
-  }
-
-  /**
-   * Build a message for aborted delegations.
-   * Shows the agent name, conversation ID, abort details, and partial transcript.
-   * Format matches buildDelegationResultsMessage for consistency.
-   */
-  async buildDelegationAbortMessage(
-    abortedDelegations: CompletedDelegation[],
-    pending: PendingDelegation[] = []
-  ): Promise<string> {
-    if (abortedDelegations.length === 0) {
-      return "";
-    }
-
-    const lines: string[] = [];
-
-    // Header indicating abort event
-    lines.push("# DELEGATION ABORTED");
-    lines.push("");
-
-    // Format each aborted delegation
-    for (const c of abortedDelegations) {
-      if (c.status !== "aborted") continue; // Type guard for discriminated union
-
-      lines.push(...await this.renderDelegationHeader(
-        c.recipientPubkey,
-        c.delegationConversationId,
-        "was aborted and did not complete their task."
-      ));
-
-      // Add abort-specific metadata with error handling for timestamp
-      try {
-        lines.push(`**Aborted at:** ${new Date(c.completedAt).toISOString()}`);
-      } catch {
-        lines.push(`**Aborted at:** (invalid timestamp)`);
-      }
-      lines.push(`**Reason:** ${c.abortReason}`);
-      lines.push("");
-
-      // Show partial transcript if available
-      if (c.transcript && c.transcript.length > 0) {
-        lines.push("### Partial Progress:");
-        lines.push(...await this.formatTranscript(c.transcript));
-      } else {
-        lines.push("### Partial Progress:");
-        lines.push("(No messages exchanged before abort)");
-      }
-      lines.push("");
-    }
-
-    // Show remaining pending delegations if any
-    if (pending.length > 0) {
-      lines.push(...await this.renderPendingList(pending));
-    }
-
-    return lines.join("\n");
-  }
-
   /**
    * Determine if a new message should wake up an execution.
    *
@@ -2635,13 +2458,17 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
   /**
    * Check if there's any outstanding work for a conversation that would prevent finalization.
    *
-   * This method consolidates checking for both:
+   * This method consolidates checking for:
    * 1. Queued injections (messages waiting to be processed in the next LLM step)
    * 2. Pending delegations (delegations that haven't completed yet)
+   * 3. Completed delegations (delegations that completed but whose results haven't
+   *    been incorporated into the agent's messages via resolveRAL yet)
    *
-   * This is the key guard against the race condition where delegation results arrive
-   * (via debounce) after the last prepareStep but before the executor finalizes.
-   * By checking this before publishing status:completed, we ensure no work is orphaned.
+   * Checking completed delegations is critical for fast-completing delegations:
+   * recordCompletion() moves delegations from pending→completed immediately (no debounce),
+   * but the executor only processes them via resolveRAL() after the debounce fires.
+   * Without this check, the executor sees pendingDelegations=0 and finalizes prematurely,
+   * clearing the RAL before the completed delegation can be processed.
    *
    * @param agentPubkey - The agent's pubkey
    * @param conversationId - The conversation ID
@@ -2657,6 +2484,7 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
     details: {
       queuedInjections: number;
       pendingDelegations: number;
+      completedDelegations: number;
     };
   } {
     const ral = this.getRAL(agentPubkey, conversationId, ralNumber);
@@ -2669,13 +2497,23 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
       ralNumber
     ).length;
 
-    // If RAL doesn't exist, we can't have queued injections but may still have pending delegations
+    // Count completed delegations that haven't been consumed by resolveRAL yet.
+    // These are delegations where recordCompletion() has run but the executor hasn't
+    // processed them into conversation markers yet.
+    const completedDelegations = this.getConversationCompletedDelegations(
+      agentPubkey,
+      conversationId,
+      ralNumber
+    ).length;
+
+    // If RAL doesn't exist, we can't have queued injections but may still have delegations
     if (!ral) {
-      const hasWork = pendingDelegations > 0;
+      const hasWork = pendingDelegations > 0 || completedDelegations > 0;
       if (hasWork) {
         trace.getActiveSpan()?.addEvent("ral.outstanding_work_no_ral", {
           "ral.number": ralNumber,
           "outstanding.pending_delegations": pendingDelegations,
+          "outstanding.completed_delegations": completedDelegations,
           "agent.pubkey": agentPubkey.substring(0, 12),
           "conversation.id": shortenConversationId(conversationId),
         });
@@ -2685,6 +2523,7 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
         details: {
           queuedInjections: 0,
           pendingDelegations,
+          completedDelegations,
         },
       };
     }
@@ -2692,7 +2531,7 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
     // Count queued injections from the RAL entry
     const queuedInjections = ral.queuedInjections.length;
 
-    const hasWork = queuedInjections > 0 || pendingDelegations > 0;
+    const hasWork = queuedInjections > 0 || pendingDelegations > 0 || completedDelegations > 0;
 
     // Add telemetry for debugging race conditions
     if (hasWork) {
@@ -2700,6 +2539,7 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
         "ral.number": ralNumber,
         "outstanding.queued_injections": queuedInjections,
         "outstanding.pending_delegations": pendingDelegations,
+        "outstanding.completed_delegations": completedDelegations,
         "agent.pubkey": agentPubkey.substring(0, 12),
         "conversation.id": shortenConversationId(conversationId),
       });
@@ -2710,6 +2550,7 @@ export class RALRegistry extends EventEmitter<RALRegistryEvents> {
       details: {
         queuedInjections,
         pendingDelegations,
+        completedDelegations,
       },
     };
   }

package/src/services/reports/ReportEmbeddingService.ts

@@ -7,13 +7,13 @@
  * Key features:
  * - Project-scoped: reports are tagged with projectId for isolation
  * - Index on write: called from report_write tool after successful publish
- * - Upsert semantics: re-indexing updates existing documents (by slug + projectId)
- * - Graceful degradation: RAG failures don't break report writes
+ * - Upsert semantics: re-indexing updates existing documents via bulkUpsert
  * - Nostr remains authoritative source; RAG is just a search layer
  */
 
 import { logger } from "@/utils/logger";
 import { RAGService, type RAGDocument, type RAGQueryResult } from "@/services/rag/RAGService";
+import { buildProjectFilter } from "@/services/search/projectFilter";
 import type { ReportInfo } from "./ReportService";
 
 /** Collection name for report embeddings */
@@ -187,59 +187,43 @@ export class ReportEmbeddingService {
     ): Promise<boolean> {
         await this.ensureInitialized();
 
-        try {
-            const documentId = this.buildDocumentId(projectId, report.slug);
-            const embeddingContent = this.buildEmbeddingContent(report);
-
-            if (!embeddingContent.trim()) {
-                logger.debug("No content to embed for report", { slug: report.slug });
-                return false;
-            }
-
-            // Delete existing document before inserting (upsert semantics)
-            try {
-                await this.ragService.deleteDocumentById(REPORT_COLLECTION, documentId);
-            } catch {
-                // Document might not exist - that's fine
-            }
-
-            const document: RAGDocument = {
-                id: documentId,
-                content: embeddingContent,
-                metadata: {
-                    slug: report.slug,
-                    projectId,
-                    title: report.title || "",
-                    summary: report.summary || "",
-                    hashtags: report.hashtags,
-                    agentPubkey,
-                    agentName: agentName || "",
-                    type: "report",
-                    publishedAt: report.publishedAt ?? Math.floor(Date.now() / 1000),
-                },
-                timestamp: Date.now(),
-                source: "report",
-            };
+        const documentId = this.buildDocumentId(projectId, report.slug);
+        const embeddingContent = this.buildEmbeddingContent(report);
 
-            await this.ragService.addDocuments(REPORT_COLLECTION, [document]);
+        if (!embeddingContent.trim()) {
+            logger.debug("No content to embed for report", { slug: report.slug });
+            return false;
+        }
 
-            logger.info("📝 Report indexed in RAG", {
+        const document: RAGDocument = {
+            id: documentId,
+            content: embeddingContent,
+            metadata: {
                 slug: report.slug,
                 projectId,
-                documentId,
-                agentName,
-            });
+                title: report.title || "",
+                summary: report.summary || "",
+                hashtags: report.hashtags,
+                agentPubkey,
+                agentName: agentName || "",
+                type: "report",
+                publishedAt: report.publishedAt ?? Math.floor(Date.now() / 1000),
+            },
+            timestamp: Date.now(),
+            source: "report",
+        };
 
-            return true;
-        } catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            logger.warn("Failed to index report in RAG", {
-                slug: report.slug,
-                projectId,
-                error: message,
-            });
-            return false;
-        }
+        // Atomic upsert via mergeInsert — one LanceDB version per chunk
+        await this.ragService.bulkUpsert(REPORT_COLLECTION, [document]);
+
+        logger.info("📝 Report indexed in RAG", {
+            slug: report.slug,
+            projectId,
+            documentId,
+            agentName,
+        });
+
+        return true;
     }
 
     /**
@@ -247,40 +231,16 @@ export class ReportEmbeddingService {
      * Called when a report is deleted.
      */
     public async removeReport(slug: string, projectId: string): Promise<void> {
-        try {
-            // Check if the collection exists before attempting deletion.
-            // Avoids ensureInitialized() which would create the collection as a side-effect.
-            const collections = await this.ragService.listCollections();
-            if (!collections.includes(REPORT_COLLECTION)) {
-                logger.debug("Report collection does not exist, nothing to remove", {
-                    slug,
-                    projectId,
-                });
-                return;
-            }
-
-            const documentId = this.buildDocumentId(projectId, slug);
-            await this.ragService.deleteDocumentById(REPORT_COLLECTION, documentId);
-            logger.info("🗑️ Report removed from RAG", { slug, projectId, documentId });
-        } catch (error) {
-            logger.debug("Could not remove report from RAG (may not exist)", {
-                slug,
-                projectId,
-                error,
-            });
+        // Check if the collection exists before attempting deletion.
+        // Avoids ensureInitialized() which would create the collection as a side-effect.
+        const collections = await this.ragService.listCollections();
+        if (!collections.includes(REPORT_COLLECTION)) {
+            return;
         }
-    }
 
-    /**
-     * Build SQL prefilter for project isolation.
-     * Applied DURING vector search, not after, to ensure proper project boundaries.
-     */
-    private buildProjectFilter(projectId?: string): string | undefined {
-        if (!projectId || projectId.toLowerCase() === "all") {
-            return undefined;
-        }
-        const escapedProjectId = projectId.replace(/'/g, "''");
-        return `metadata LIKE '%"projectId":"${escapedProjectId}"%'`;
+        const documentId = this.buildDocumentId(projectId, slug);
+        await this.ragService.deleteDocumentById(REPORT_COLLECTION, documentId);
+        logger.info("🗑️ Report removed from RAG", { slug, projectId, documentId });
     }
 
     /**
@@ -298,36 +258,30 @@ export class ReportEmbeddingService {
 
         const { limit = 10, minScore = 0.3, projectId } = options;
 
-        try {
-            logger.info("🔍 Report semantic search", { query, limit, minScore, projectId });
+        logger.info("🔍 Report semantic search", { query, limit, minScore, projectId });
 
-            const filter = this.buildProjectFilter(projectId);
+        const filter = buildProjectFilter(projectId);
 
-            const results = await this.ragService.queryWithFilter(
-                REPORT_COLLECTION,
-                query,
-                limit * 2, // Request more to account for minScore filtering
-                filter
-            );
+        const results = await this.ragService.queryWithFilter(
+            REPORT_COLLECTION,
+            query,
+            limit * 2,
+            filter
+        );
 
-            const searchResults: ReportSearchResult[] = results
-                .filter((result: RAGQueryResult) => result.score >= minScore)
-                .slice(0, limit)
-                .map((result: RAGQueryResult) => this.transformResult(result));
+        const searchResults: ReportSearchResult[] = results
+            .filter((result: RAGQueryResult) => result.score >= minScore)
+            .slice(0, limit)
+            .map((result: RAGQueryResult) => this.transformResult(result));
 
-            logger.info("✅ Report semantic search complete", {
-                query,
-                found: searchResults.length,
-                limit,
-                projectFilter: filter || "none",
-            });
+        logger.info("✅ Report semantic search complete", {
+            query,
+            found: searchResults.length,
+            limit,
+            projectFilter: filter || "none",
+        });
 
-            return searchResults;
-        } catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            logger.error("Report semantic search failed", { query, error: message });
-            return [];
-        }
+        return searchResults;
     }
 
     /**
@@ -403,12 +357,8 @@ export class ReportEmbeddingService {
      * Clear all report embeddings
      */
     public async clearIndex(): Promise<void> {
-        try {
-            await this.ragService.deleteCollection(REPORT_COLLECTION);
-            logger.info("Cleared report embeddings index");
-        } catch (error) {
-            logger.debug("No report index to clear or error clearing", { error });
-        }
+        await this.ragService.deleteCollection(REPORT_COLLECTION);
+        logger.info("Cleared report embeddings index");
 
         this.initialized = false;
         this.initializationPromise = null;

package/src/services/search/UnifiedSearchService.ts

@@ -9,13 +9,17 @@
  * - Parallel queries across all providers
  * - Graceful degradation: one collection failure doesn't block others
  * - Project-scoped isolation via projectId
+ * - Scope-aware collection filtering (global/project/personal)
  * - Optional LLM extraction with configurable prompt
  */
 
 import { logger } from "@/utils/logger";
 import { config as configService } from "@/services/ConfigService";
+import { RAGService } from "@/services/rag/RAGService";
+import { RAGCollectionRegistry } from "@/services/rag/RAGCollectionRegistry";
 import { SearchProviderRegistry } from "./SearchProviderRegistry";
-import type { SearchOptions, SearchResult, UnifiedSearchOutput } from "./types";
+import { GenericCollectionSearchProvider } from "./providers/GenericCollectionSearchProvider";
+import type { SearchOptions, SearchProvider, SearchResult, UnifiedSearchOutput } from "./types";
 
 /** Default search parameters */
 const DEFAULT_LIMIT = 10;
@@ -47,6 +51,7 @@ export class UnifiedSearchService {
             minScore = DEFAULT_MIN_SCORE,
             prompt,
             collections,
+            agentPubkey,
         } = options;
 
         logger.info("🔍 [UnifiedSearch] Starting search", {
@@ -55,11 +60,12 @@ export class UnifiedSearchService {
             limit,
             minScore,
             prompt: prompt ? `${prompt.substring(0, 50)}...` : undefined,
-            collections: collections || "all",
+            collections: collections || "all (scope-aware)",
+            agentPubkey: agentPubkey ? `${agentPubkey.substring(0, 12)}...` : undefined,
         });
 
-        // Get providers to search
-        const providers = this.registry.getByNames(collections);
+        // Resolve providers: static (specialized) + dynamic (generic for discovered RAG collections)
+        const providers = await this.resolveProviders(collections, projectId, agentPubkey);
         if (providers.length === 0) {
             logger.warn("[UnifiedSearch] No search providers available");
             return {
@@ -136,6 +142,111 @@ export class UnifiedSearchService {
         };
     }
 
+    /**
+     * Resolve all providers to query: specialized (from registry) + generic
+     * (dynamically created for any RAG collections not covered by a specialized provider).
+     *
+     * When `requestedCollections` is provided, only those are returned (explicit choice,
+     * no scope filtering). When absent, scope-aware filtering is applied.
+     *
+     * @param requestedCollections - Optional explicit collection filter (bypasses scoping)
+     * @param projectId - Current project ID for scope filtering
+     * @param agentPubkey - Current agent pubkey for personal scope filtering
+     */
+    private async resolveProviders(
+        requestedCollections: string[] | undefined,
+        projectId: string,
+        agentPubkey?: string
+    ): Promise<SearchProvider[]> {
+        const specializedProviders = this.registry.getAll();
+
+        // Derive covered RAG collections from the providers themselves.
+        // A collection is "covered" if a specialized provider declares it via
+        // `collectionName` or if its `name` matches a RAG collection name.
+        const coveredCollections = new Set<string>();
+        for (const provider of specializedProviders) {
+            if (provider.collectionName) {
+                coveredCollections.add(provider.collectionName);
+            }
+            coveredCollections.add(provider.name);
+        }
+
+        // Discover all RAG collections
+        let allCollections: string[];
+        try {
+            const ragService = RAGService.getInstance();
+            allCollections = await ragService.listCollections();
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.warn("[UnifiedSearch] Failed to discover RAG collections", { error: message });
+            allCollections = [];
+        }
+
+        // Apply scope filtering to discovered collections (only when no explicit filter)
+        let scopedCollections = allCollections;
+        if (!requestedCollections) {
+            try {
+                const collectionRegistry = RAGCollectionRegistry.getInstance();
+                scopedCollections = collectionRegistry.getMatchingCollections(
+                    allCollections,
+                    projectId,
+                    agentPubkey
+                );
+
+                if (scopedCollections.length < allCollections.length) {
+                    logger.debug("[UnifiedSearch] Scope filtering applied", {
+                        total: allCollections.length,
+                        matched: scopedCollections.length,
+                        excluded: allCollections.length - scopedCollections.length,
+                    });
+                }
+            } catch (error) {
+                // Registry not available — fall through with all collections
+                const message = error instanceof Error ? error.message : String(error);
+                logger.debug("[UnifiedSearch] Collection registry unavailable, using all collections", {
+                    error: message,
+                });
+            }
+        }
+
+        // Create generic providers for collections not covered by specialized providers
+        const genericProviders = scopedCollections
+            .filter((name) => !coveredCollections.has(name))
+            .map((name) => new GenericCollectionSearchProvider(name));
+
+        const allProviders = [...specializedProviders, ...genericProviders];
+
+        if (genericProviders.length > 0) {
+            logger.debug("[UnifiedSearch] Dynamic collections discovered", {
+                specialized: specializedProviders.map((p) => p.name),
+                generic: genericProviders.map((p) => p.name),
+            });
+        }
+
+        // Apply explicit collection filter by provider name
+        if (requestedCollections && requestedCollections.length > 0) {
+            const requestedSet = new Set(requestedCollections);
+
+            // For explicitly requested collections that aren't in scopedCollections
+            // (because scope would have excluded them), add generic providers anyway.
+            // The agent explicitly asked — no scope filtering.
+            const missingGenericNames = requestedCollections.filter(
+                (name) =>
+                    !specializedProviders.some((p) => p.name === name) &&
+                    !genericProviders.some((p) => p.name === name) &&
+                    allCollections.includes(name)
+            );
+            const extraProviders = missingGenericNames.map(
+                (name) => new GenericCollectionSearchProvider(name)
+            );
+
+            const expandedProviders = [...allProviders, ...extraProviders];
+            return expandedProviders.filter((p) => requestedSet.has(p.name));
+        }
+
+        return allProviders;
+    }
+
     /**
      * Use a fast/cheap LLM to extract focused information from search results.
      *

package/src/services/search/index.ts

@@ -19,6 +19,7 @@ export type {
 export { ReportSearchProvider } from "./providers/ReportSearchProvider";
 export { ConversationSearchProvider } from "./providers/ConversationSearchProvider";
 export { LessonSearchProvider } from "./providers/LessonSearchProvider";
+export { GenericCollectionSearchProvider } from "./providers/GenericCollectionSearchProvider";
 
 import { SearchProviderRegistry } from "./SearchProviderRegistry";
 import { ReportSearchProvider } from "./providers/ReportSearchProvider";