@tenex-chat/backend 0.9.4 → 0.9.6

package/src/services/search/projectFilter.ts

@@ -1,15 +1,28 @@
 /**
  * Centralized SQL project filter for RAG queries.
  *
- * Shared utility for the `metadata LIKE '%"projectId":"..."'` SQL prefilter pattern
- * used across all project-scoped RAG collections (reports, conversations, lessons).
+ * Shared utility for project-scoped metadata filtering across all RAG collections
+ * (reports, conversations, lessons, and generic collections).
  *
  * Applied DURING vector search (prefilter) to ensure proper project isolation.
+ *
+ * Matches both "projectId" (canonical, used by specialized services) and
+ * "project_id" (legacy, used by older rag_add_documents ingestion).
  */
 
+import { PROJECT_ID_KEYS } from "@/utils/metadataKeys";
+import { SQL_LIKE_ESCAPE_CLAUSE, escapeSqlLikeValue } from "@/utils/sqlEscaping";
+
 /**
  * Build a SQL prefilter string for project isolation in LanceDB queries.
  *
+ * Matches documents where metadata contains EITHER:
+ *   - "projectId":"<id>" (canonical camelCase, used by specialized services)
+ *   - "project_id":"<id>" (legacy snake_case, used by older rag_add_documents)
+ *
+ * Uses proper SQL LIKE escaping so that project IDs containing wildcards
+ * (%, _) or quotes don't broaden or break the filter.
+ *
  * @param projectId - The project ID to filter by. Pass 'ALL' or undefined to skip filtering.
  * @returns SQL filter string or undefined if no filtering needed.
  */
@@ -17,6 +30,9 @@ export function buildProjectFilter(projectId?: string): string | undefined {
     if (!projectId || projectId.toLowerCase() === "all") {
         return undefined;
     }
-    const escapedProjectId = projectId.replace(/'/g, "''");
-    return `metadata LIKE '%"projectId":"${escapedProjectId}"%'`;
+    const escaped = escapeSqlLikeValue(projectId);
+    const clauses = PROJECT_ID_KEYS
+        .map((key) => `metadata LIKE '%"${key}":"${escaped}"%' ${SQL_LIKE_ESCAPE_CLAUSE}`)
+        .join(" OR ");
+    return `(${clauses})`;
 }

package/src/services/search/providers/ConversationSearchProvider.ts

@@ -10,6 +10,7 @@ import type { SearchProvider, SearchResult } from "../types";
 
 export class ConversationSearchProvider implements SearchProvider {
     readonly name = "conversations";
+    readonly collectionName = "conversation_embeddings";
     readonly description = "Past conversation threads and discussions";
 
     async search(

package/src/services/search/providers/GenericCollectionSearchProvider.ts

@@ -0,0 +1,81 @@
+/**
+ * GenericCollectionSearchProvider - Search provider for any RAG collection.
+ *
+ * Created dynamically for RAG collections that don't have a dedicated
+ * specialized provider. Queries via RAGService with basic project-scoped
+ * filtering.
+ */
+
+import { logger } from "@/utils/logger";
+import { RAGService, type RAGQueryResult } from "@/services/rag/RAGService";
+import { buildProjectFilter } from "../projectFilter";
+import type { SearchProvider, SearchResult } from "../types";
+
+export class GenericCollectionSearchProvider implements SearchProvider {
+    readonly name: string;
+    readonly description: string;
+
+    /** The actual RAG collection name to query */
+    readonly collectionName: string;
+
+    constructor(collectionName: string) {
+        this.collectionName = collectionName;
+        this.name = collectionName;
+        this.description = `RAG collection: ${collectionName}`;
+    }
+
+    async search(
+        query: string,
+        projectId: string,
+        limit: number,
+        minScore: number
+    ): Promise<SearchResult[]> {
+        const ragService = RAGService.getInstance();
+
+        const filter = buildProjectFilter(projectId);
+
+        const results = await ragService.queryWithFilter(
+            this.collectionName,
+            query,
+            limit * 2, // Request more to account for minScore filtering
+            filter
+        );
+
+        logger.debug(`[GenericSearchProvider:${this.collectionName}] Search complete`, {
+            query,
+            projectId,
+            rawResults: results.length,
+        });
+
+        const filtered = results
+            .filter((result: RAGQueryResult) => result.score >= minScore && !!result.document.id)
+            .slice(0, limit)
+            .map((result: RAGQueryResult) => this.transformResult(result, projectId));
+
+        if (filtered.length < results.length) {
+            const dropped = results.length - filtered.length;
+            logger.debug(`[GenericSearchProvider:${this.collectionName}] Dropped ${dropped} result(s) (low score or missing ID)`);
+        }
+
+        return filtered;
+    }
+
+    private transformResult(result: RAGQueryResult, fallbackProjectId: string): SearchResult {
+        const metadata = result.document.metadata || {};
+
+        return {
+            source: this.collectionName,
+            id: result.document.id || "",
+            projectId: String(metadata.projectId || fallbackProjectId),
+            relevanceScore: result.score,
+            title: String(metadata.title || result.document.id || ""),
+            summary: result.document.content?.substring(0, 200) || "",
+            createdAt: metadata.timestamp ? Number(metadata.timestamp) : undefined,
+            author: metadata.agentPubkey ? String(metadata.agentPubkey) : undefined,
+            authorName: metadata.agentName ? String(metadata.agentName) : undefined,
+            tags: Array.isArray(metadata.hashtags) ? (metadata.hashtags as string[]) : undefined,
+            retrievalTool: "rag_search" as const,
+            retrievalArg: result.document.id || "",
+        };
+    }
+}

package/src/services/search/providers/LessonSearchProvider.ts

@@ -16,6 +16,7 @@ const LESSONS_COLLECTION = "lessons";
 
 export class LessonSearchProvider implements SearchProvider {
     readonly name = "lessons";
+    readonly collectionName = "lessons";
     readonly description = "Agent lessons and insights";
 
     async search(
@@ -25,14 +26,6 @@ export class LessonSearchProvider implements SearchProvider {
         minScore: number
     ): Promise<SearchResult[]> {
         const ragService = RAGService.getInstance();
-
-        // Check if the lessons collection exists
-        const collections = await ragService.listCollections();
-        if (!collections.includes(LESSONS_COLLECTION)) {
-            logger.debug("[LessonSearchProvider] Lessons collection does not exist");
-            return [];
-        }
-
         const filter = buildProjectFilter(projectId);
 
         const results = await ragService.queryWithFilter(

package/src/services/search/providers/ReportSearchProvider.ts

@@ -10,6 +10,7 @@ import type { SearchProvider, SearchResult } from "../types";
 
 export class ReportSearchProvider implements SearchProvider {
     readonly name = "reports";
+    readonly collectionName = "project_reports";
     readonly description = "Project reports and documentation";
 
     async search(

package/src/services/search/types.ts

@@ -51,7 +51,7 @@ export interface SearchResult {
     tags?: string[];
 
     /** Which tool to use to retrieve full content */
-    retrievalTool: "report_read" | "lesson_get" | "conversation_get";
+    retrievalTool: "report_read" | "lesson_get" | "conversation_get" | "rag_search";
 
     /** The argument to pass to the retrieval tool */
     retrievalArg: string;
@@ -79,8 +79,21 @@ export interface SearchOptions {
      */
     prompt?: string;
 
-    /** Which collections to search (defaults to all) */
+    /**
+     * Filter by **provider name** (defaults to all).
+     * Well-known provider names: "reports", "conversations", "lessons".
+     * Dynamically discovered RAG collections use their collection name as provider name
+     * (e.g., "custom_knowledge").
+     *
+     * When provided, searches exactly those collections (no scope filtering).
+     */
     collections?: string[];
+
+    /**
+     * Agent pubkey for scope-aware collection filtering.
+     * Used to include `personal` collections belonging to this agent.
+     */
+    agentPubkey?: string;
 }
 
 /**
@@ -88,12 +101,20 @@ export interface SearchOptions {
  * Each provider wraps a specific RAG collection.
  */
 export interface SearchProvider {
-    /** Unique name for this provider (matches collection concept) */
+    /** Unique name for this provider (used for filtering via `collections` parameter) */
     readonly name: string;
 
     /** Human-readable description */
     readonly description: string;
 
+    /**
+     * The underlying RAG collection name this provider covers.
+     * Used to prevent duplicate generic providers for collections that already
+     * have a specialized provider. When undefined, the provider is not
+     * associated with a specific RAG collection (or uses its `name` directly).
+     */
+    readonly collectionName?: string;
+
     /**
      * Perform semantic search within this provider's collection.
      *

package/src/services/trust-pubkeys/SystemPubkeyListService.ts

@@ -0,0 +1,148 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { agentStorage } from "@/agents/AgentStorage";
+import { ensureDirectory, readFile } from "@/lib/fs";
+import { config } from "@/services/ConfigService";
+import { logger } from "@/utils/logger";
+
+export interface SyncSystemPubkeyListOptions {
+    /**
+     * Extra pubkeys to force-include in the output file.
+     * Useful for "about to publish" contexts where the pubkey must be present
+     * even if other registries haven't synced yet.
+     */
+    additionalPubkeys?: Iterable<string>;
+}
+
+/**
+ * Maintains `$TENEX_BASE_DIR/daemon/whitelist.txt` as the canonical list of
+ * pubkeys known to belong to this TENEX system.
+ *
+ * The file contains one pubkey per line and is rebuilt from:
+ * - daemon/user whitelist from config
+ * - backend pubkey
+ * - all known agent pubkeys from storage
+ * - optional call-site additions (e.g., the pubkey being published right now)
+ */
+export class SystemPubkeyListService {
+    private static instance: SystemPubkeyListService;
+
+    static getInstance(): SystemPubkeyListService {
+        if (!SystemPubkeyListService.instance) {
+            SystemPubkeyListService.instance = new SystemPubkeyListService();
+        }
+        return SystemPubkeyListService.instance;
+    }
+
+    /**
+     * Rebuild and persist daemon whitelist file.
+     * Idempotent: skips writes when content is unchanged.
+     */
+    async syncWhitelistFile(options: SyncSystemPubkeyListOptions = {}): Promise<void> {
+        const daemonDir = config.getConfigPath("daemon");
+        const whitelistPath = path.join(daemonDir, "whitelist.txt");
+        const pubkeys = await this.collectKnownSystemPubkeys(options.additionalPubkeys);
+        const content = this.serialize(pubkeys);
+
+        await ensureDirectory(daemonDir);
+
+        const existing = await this.safeReadFile(whitelistPath);
+        if (existing === content) {
+            return;
+        }
+
+        await this.atomicWrite(whitelistPath, content);
+        logger.debug("[SYSTEM_PUBKEY_LIST] Updated daemon whitelist.txt", {
+            path: whitelistPath,
+            pubkeyCount: pubkeys.length,
+        });
+    }
+
+    private async collectKnownSystemPubkeys(additionalPubkeys?: Iterable<string>): Promise<string[]> {
+        const pubkeys = new Set<string>();
+
+        // Whitelisted daemon pubkeys from loaded config
+        try {
+            const loadedConfig = config.getConfig();
+            const whitelisted = config.getWhitelistedPubkeys(undefined, loadedConfig);
+            for (const pubkey of whitelisted) {
+                this.addPubkey(pubkeys, pubkey);
+            }
+        } catch (error) {
+            logger.debug("[SYSTEM_PUBKEY_LIST] Failed to load whitelisted pubkeys", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+
+        // TENEX backend pubkey
+        try {
+            const backendSigner = await config.getBackendSigner();
+            this.addPubkey(pubkeys, backendSigner.pubkey);
+        } catch (error) {
+            logger.debug("[SYSTEM_PUBKEY_LIST] Failed to load backend pubkey", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+
+        // All known agents (across projects) from storage
+        try {
+            const knownAgentPubkeys = await agentStorage.getAllKnownPubkeys();
+            for (const pubkey of knownAgentPubkeys) {
+                this.addPubkey(pubkeys, pubkey);
+            }
+        } catch (error) {
+            logger.debug("[SYSTEM_PUBKEY_LIST] Failed to load known agent pubkeys", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+
+        // Call-site additions (e.g., pubkey being published right now)
+        if (additionalPubkeys) {
+            for (const pubkey of additionalPubkeys) {
+                this.addPubkey(pubkeys, pubkey);
+            }
+        }
+
+        return Array.from(pubkeys).sort();
+    }
+
+    private addPubkey(pubkeys: Set<string>, candidate: string | undefined): void {
+        if (!candidate) {
+            return;
+        }
+
+        const trimmed = candidate.trim();
+        if (!trimmed) {
+            return;
+        }
+
+        pubkeys.add(trimmed);
+    }
+
+    private serialize(pubkeys: string[]): string {
+        if (pubkeys.length === 0) {
+            return "";
+        }
+        return `${pubkeys.join("\n")}\n`;
+    }
+
+    private async safeReadFile(filePath: string): Promise<string | null> {
+        try {
+            return await readFile(filePath, "utf-8");
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
+                return null;
+            }
+            throw error;
+        }
+    }
+
+    private async atomicWrite(filePath: string, content: string): Promise<void> {
+        const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+        await fs.writeFile(tempPath, content, "utf-8");
+        await fs.rename(tempPath, filePath);
+    }
+}
+
+export const getSystemPubkeyListService = (): SystemPubkeyListService =>
+    SystemPubkeyListService.getInstance();

package/src/services/trust-pubkeys/TrustPubkeyService.ts

@@ -24,9 +24,17 @@ export interface TrustResult {
  * A pubkey is trusted if it is:
  * - In the whitelisted pubkeys from config
  * - The backend's own pubkey
- * - An agent in the system (registered in ProjectContext)
+ * - An agent in the system (registered in ProjectContext or globally across all projects)
  *
  * Trust precedence (highest to lowest): whitelisted > backend > agent
+ *
+ * ## Agent Trust: Two-Tier Lookup + Daemon Seeding
+ * 1. **Project context** (sync): Check the current project's agent registry (fast, scoped)
+ * 2. **Global agent set** (sync): Check daemon-level set of all agent pubkeys across all projects
+ *
+ * The global agent set is seeded by the Daemon at startup from AgentStorage (covering
+ * not-yet-running projects) and kept in sync as projects start/stop. Each sync unions
+ * the active runtime pubkeys with the stored seed, so trust is never dropped.
  */
 export class TrustPubkeyService {
     private static instance: TrustPubkeyService;
@@ -37,6 +45,13 @@ export class TrustPubkeyService {
     /** Cached whitelist Set for O(1) lookups */
     private cachedWhitelistSet?: Set<Hexpubkey>;
 
+    /**
+     * Global set of agent pubkeys across ALL projects (running and discovered).
+     * Pushed by the Daemon whenever projects start/stop or agents are added.
+     * Frozen for safe concurrent reads.
+     */
+    private globalAgentPubkeys: ReadonlySet<Hexpubkey> = Object.freeze(new Set<Hexpubkey>());
+
     private constructor() {}
 
     /**
@@ -165,6 +180,20 @@ export class TrustPubkeyService {
         // Note: getBackendPubkey already logs debug message on failure
     }
 
+    /**
+     * Set the global agent pubkeys set (daemon-level, cross-project).
+     * Called by the Daemon when projects start/stop or agents are dynamically added.
+     * The set is frozen for safe concurrent reads.
+     *
+     * @param pubkeys Set of all known agent pubkeys across all projects
+     */
+    setGlobalAgentPubkeys(pubkeys: Set<Hexpubkey>): void {
+        this.globalAgentPubkeys = Object.freeze(new Set(pubkeys));
+        logger.debug("[TRUST_PUBKEY] Global agent pubkeys updated", {
+            count: pubkeys.size,
+        });
+    }
+
     /**
      * Get all currently trusted pubkeys.
      * Useful for debugging or displaying trust status.
@@ -196,6 +225,7 @@ export class TrustPubkeyService {
         }
 
         // 3. Add agent pubkeys (lowest priority)
+        // 3a. Current project context agents
         const projectCtx = projectContextStore.getContext();
         if (projectCtx) {
             for (const [_slug, agent] of projectCtx.agents) {
@@ -205,6 +235,13 @@ export class TrustPubkeyService {
             }
         }
 
+        // 3b. Global agent pubkeys (cross-project)
+        for (const pubkey of this.globalAgentPubkeys) {
+            if (!trustedMap.has(pubkey)) {
+                trustedMap.set(pubkey, "agent");
+            }
+        }
+
         // Convert map to array
         return Array.from(trustedMap.entries()).map(([pubkey, reason]) => ({
             pubkey,
@@ -296,25 +333,49 @@ export class TrustPubkeyService {
     }
 
     /**
-     * Check if pubkey belongs to an agent in the system.
-     * Returns false if no project context is available (e.g., during daemon startup
-     * or when called outside of projectContextStore.run()).
+     * Check if pubkey belongs to an agent in the system (synchronous, two-tier).
+     *
+     * Tier 1: Current project context (fast, scoped to the active project)
+     * Tier 2: Global agent pubkeys set (daemon-level, covers all projects including non-running)
+     *
+     * The global set is maintained by the Daemon via setGlobalAgentPubkeys(),
+     * which unions active runtime pubkeys with the AgentStorage seed.
      */
     private isAgentPubkey(pubkey: Hexpubkey): boolean {
+        // Tier 1: Current project context (existing fast path)
         const projectCtx = projectContextStore.getContext();
-        if (!projectCtx) {
-            return false;
+        if (projectCtx?.getAgentByPubkey(pubkey) !== undefined) {
+            return true;
         }
-        return projectCtx.getAgentByPubkey(pubkey) !== undefined;
+
+        // Tier 2: Global agent pubkeys (daemon-level, cross-project)
+        if (this.globalAgentPubkeys.has(pubkey)) {
+            return true;
+        }
+
+        return false;
     }
 
     /**
-     * Clear all caches (useful for testing and config reloads)
+     * Clear config-derived caches (useful for testing and config reloads).
+     * Does NOT clear globalAgentPubkeys — that state is managed by the Daemon
+     * via setGlobalAgentPubkeys() and is not derived from config.
      */
     clearCache(): void {
         this.cachedBackendPubkey = undefined;
         this.cachedWhitelistSet = undefined;
-        logger.debug("[TRUST_PUBKEY] Cache cleared");
+        logger.debug("[TRUST_PUBKEY] Config cache cleared");
+    }
+
+    /**
+     * Reset all state including daemon-managed global agent pubkeys.
+     * Use only in tests to fully reset the service.
+     */
+    resetAll(): void {
+        this.cachedBackendPubkey = undefined;
+        this.cachedWhitelistSet = undefined;
+        this.globalAgentPubkeys = Object.freeze(new Set<Hexpubkey>());
+        logger.debug("[TRUST_PUBKEY] Full state reset");
     }
 }
 

package/src/telemetry/setup.ts

@@ -17,10 +17,7 @@ const DEFAULT_ENDPOINT = "http://localhost:4318/v1/traces";
 class ErrorHandlingExporterWrapper implements SpanExporter {
     private disabled = false;
 
-    constructor(
-        private traceExporter: OTLPTraceExporter,
-        private exporterUrl: string
-    ) {}
+    constructor(private traceExporter: OTLPTraceExporter) {}
 
     export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void {
         // Once disabled, drop all spans silently
@@ -31,14 +28,6 @@ class ErrorHandlingExporterWrapper implements SpanExporter {
 
         this.traceExporter.export(spans, (result) => {
             if (result.error && !this.disabled) {
-                const errorMessage = result.error?.message || String(result.error);
-                const isConnectionError = errorMessage.includes("ECONNREFUSED") || errorMessage.includes("connect");
-                if (isConnectionError) {
-                    console.warn(`[Telemetry] ⚠️  Collector not available at ${this.exporterUrl}`);
-                } else {
-                    console.error("[Telemetry] Export error:", errorMessage);
-                }
-                console.warn("[Telemetry] Disabling trace export");
                 this.disabled = true;
             }
             resultCallback(result);
@@ -106,7 +95,7 @@ export function initializeTelemetry(
     });
 
     // Wrap the exporter with error handling
-    const wrappedExporter = new ErrorHandlingExporterWrapper(traceExporter, exporterUrl);
+    const wrappedExporter = new ErrorHandlingExporterWrapper(traceExporter);
 
     sdk = createSDK(serviceName, wrappedExporter);
     sdk.start();

package/src/tools/implementations/ask.ts

@@ -12,7 +12,6 @@ import { z } from "zod";
 import { RALRegistry } from "@/services/ral";
 import type { PendingDelegation } from "@/services/ral/types";
 import { APNsService } from "@/services/apns";
-import { streamPublisher } from "@/llm";
 
 /**
  * Schema for a single-select question.
@@ -303,10 +302,11 @@ async function executeAsk(input: AskInput, context: ToolExecutionContext): Promi
     });
   }
 
-  // Send APNs push notification if user is not connected
+  // Send APNs push notification if APNs is enabled.
+  // Unix socket presence detection was removed with local socket streaming.
   try {
     const apnsService = APNsService.getInstance();
-    if (apnsService.isEnabled() && !streamPublisher.isConnected()) {
+    if (apnsService.isEnabled()) {
       const bodyPreview = askContext.length > 100 ? askContext.substring(0, 100) + "…" : askContext;
       await apnsService.notifyIfNeeded(ownerPubkey, {
         title: "Agent needs your input",