@tenex-chat/backend 0.9.5 → 0.9.7

package/src/tools/implementations/{search.ts → rag_search.ts}

@@ -1,9 +1,10 @@
 /**
- * Unified Search Tool
+ * RAG Search Tool
  *
- * Single search tool that queries across ALL project-scoped RAG collections
- * (conversations, reports, lessons) and returns unified results with metadata
- * that allows agents to dig deeper using other tools.
+ * Single search tool that queries across ALL accessible RAG collections.
+ * Specialized providers handle well-known collections (reports, conversations,
+ * lessons) with smart filtering, while dynamically discovered collections are
+ * queried via generic providers with basic project-scoped filtering.
  *
  * Supports optional prompt-based LLM extraction for focused information retrieval.
  */
@@ -15,11 +16,11 @@ import { logger } from "@/utils/logger";
 import { tool } from "ai";
 import { z } from "zod";
 
-const searchSchema = z.object({
+const ragSearchSchema = z.object({
     query: z.string().describe(
-        "Natural language search query. Searches across all project knowledge: " +
-        "reports (documentation, architecture), conversations (past discussions), " +
-        "and lessons (agent insights). Use descriptive natural language for best results."
+        "Natural language search query. Searches across all project knowledge " +
+        "including reports, conversations, lessons, and any additional RAG collections. " +
+        "Use descriptive natural language for best results."
     ),
     prompt: z
         .string()
@@ -36,18 +37,22 @@ const searchSchema = z.object({
         .default(10)
         .describe("Maximum number of results to return across all collections. Defaults to 10."),
     collections: z
-        .array(z.enum(["reports", "conversations", "lessons"]))
+        .array(z.string())
         .optional()
         .describe(
-            "Which collections to search. Defaults to all. " +
-            "Use to narrow search scope when you know where the information lives."
+            "Filter by provider name. When omitted, searches all collections matching the " +
+            "agent's scope (global + project + personal). When provided, searches exactly " +
+            "those collections (no scope filtering — the agent explicitly chose them). " +
+            "Well-known provider names: 'reports', 'conversations', 'lessons'. " +
+            "Dynamically discovered RAG collections use their collection name as the " +
+            "provider name (e.g., 'custom_knowledge')."
         ),
 });
 
-type SearchInput = z.infer<typeof searchSchema>;
+type RAGSearchInput = z.infer<typeof ragSearchSchema>;
 
-async function executeSearch(
-    input: SearchInput,
+async function executeRAGSearch(
+    input: RAGSearchInput,
     context: ToolExecutionContext
 ): Promise<Record<string, unknown>> {
     const { query, prompt, limit = 10, collections } = input;
@@ -70,7 +75,7 @@ async function executeSearch(
         };
     }
 
-    logger.info("🔍 [SearchTool] Executing unified search", {
+    logger.info("🔍 [RAGSearch] Executing unified search", {
         query,
         prompt: prompt ? `${prompt.substring(0, 50)}...` : undefined,
         limit,
@@ -90,6 +95,7 @@ async function executeSearch(
         limit,
         prompt,
         collections,
+        agentPubkey: context.agent.pubkey,
     });
 
     // Format results for agent consumption
@@ -118,28 +124,28 @@ async function executeSearch(
     };
 }
 
-export function createSearchTool(context: ToolExecutionContext): AISdkTool {
+export function createRAGSearchTool(context: ToolExecutionContext): AISdkTool {
     const aiTool = tool({
         description:
-            "Search across ALL project knowledge — reports, conversations, and lessons — using " +
-            "natural language semantic search. Returns ranked results with metadata and retrieval " +
-            "instructions. Each result includes a `retrievalTool` and `retrievalArg` that you can " +
-            "use to fetch the full document (e.g., call report_read with the slug, lesson_get with " +
-            "the event ID, or conversation_get with the conversation ID).\n\n" +
+            "Search across ALL project knowledge — reports, conversations, lessons, and any " +
+            "additional RAG collections — using natural language semantic search. Returns ranked " +
+            "results with metadata and retrieval instructions. Each result includes a `retrievalTool` " +
+            "and `retrievalArg` that you can use to fetch the full document (e.g., call report_read " +
+            "with the slug, lesson_get with the event ID, or conversation_get with the conversation ID).\n\n" +
             "Optionally provide a `prompt` parameter to have an LLM extract focused information " +
             "from the search results, rather than reviewing them manually.\n\n" +
             "This is the primary discovery tool for finding information across the project. Use " +
             "conversation_search for deep exploration of specific conversation content.",
 
-        inputSchema: searchSchema,
+        inputSchema: ragSearchSchema,
 
-        execute: async (input: SearchInput) => {
-            return await executeSearch(input, context);
+        execute: async (input: RAGSearchInput) => {
+            return await executeRAGSearch(input, context);
         },
     });
 
     Object.defineProperty(aiTool, "getHumanReadableContent", {
-        value: ({ query, prompt, limit, collections }: SearchInput) => {
+        value: ({ query, prompt, limit, collections }: RAGSearchInput) => {
             const parts = [`Searching project knowledge for "${query}"`];
             if (prompt) parts.push(`prompt: "${prompt.substring(0, 40)}..."`);
             if (collections?.length) parts.push(`in: ${collections.join(", ")}`);

package/src/tools/registry.ts

@@ -52,7 +52,7 @@ import { createRAGAddDocumentsTool } from "./implementations/rag_add_documents";
 import { createRAGCreateCollectionTool } from "./implementations/rag_create_collection";
 import { createRAGDeleteCollectionTool } from "./implementations/rag_delete_collection";
 import { createRAGListCollectionsTool } from "./implementations/rag_list_collections";
-import { createRAGQueryTool } from "./implementations/rag_query";
+
 import { createRAGSubscriptionCreateTool } from "./implementations/rag_subscription_create";
 import { createRAGSubscriptionDeleteTool } from "./implementations/rag_subscription_delete";
 import { createRAGSubscriptionGetTool } from "./implementations/rag_subscription_get";
@@ -83,8 +83,8 @@ import { createWebSearchTool } from "./implementations/web_search";
 import { createNostrFetchTool } from "./implementations/nostr_fetch";
 import { createNostrPublishAsUserTool } from "./implementations/nostr_publish_as_user";
 
-// Unified search tool
-import { createSearchTool } from "./implementations/search";
+// Unified RAG search tool
+import { createRAGSearchTool } from "./implementations/rag_search";
 
 // Image generation tools
 import { createGenerateImageTool } from "./implementations/generate_image";
@@ -124,7 +124,7 @@ const toolMetadata: Partial<Record<ToolName, { hasSideEffects: boolean }>> = {
     report_read: { hasSideEffects: false },
     schedule_tasks_list: { hasSideEffects: false },
     rag_list_collections: { hasSideEffects: false },
-    rag_query: { hasSideEffects: false },
+
     rag_subscription_list: { hasSideEffects: false },
     rag_subscription_get: { hasSideEffects: false },
     mcp_resource_read: { hasSideEffects: false },
@@ -132,7 +132,7 @@ const toolMetadata: Partial<Record<ToolName, { hasSideEffects: boolean }>> = {
     web_fetch: { hasSideEffects: false },
     web_search: { hasSideEffects: false },
     nostr_fetch: { hasSideEffects: false },
-    search: { hasSideEffects: false },
+    rag_search: { hasSideEffects: false },
 };
 
 /**
@@ -204,8 +204,8 @@ const toolFactories: Record<ToolName, ToolFactory> = {
     // Conversation search
     conversation_search: createConversationSearchTool,
 
-    // Unified search across all project knowledge
-    search: createSearchTool,
+    // Unified RAG search across all project knowledge
+    rag_search: createRAGSearchTool,
 
     shell: createShellTool,
     kill: createKillTool,
@@ -216,7 +216,6 @@ const toolFactories: Record<ToolName, ToolFactory> = {
     // RAG tools
     rag_create_collection: createRAGCreateCollectionTool,
     rag_add_documents: createRAGAddDocumentsTool,
-    rag_query: createRAGQueryTool,
     rag_delete_collection: createRAGDeleteCollectionTool,
     rag_list_collections: createRAGListCollectionsTool,
 

package/src/tools/types.ts

@@ -5,6 +5,11 @@ import type { MCPManager } from "@/services/mcp/MCPManager";
 import type { NDKEvent } from "@nostr-dev-kit/ndk";
 import type { Tool as CoreTool } from "ai";
 
+export interface ToolTranscriptArgSpec {
+    key: string;
+    attribute?: string;
+}
+
 /**
  * Tool names available in the system.
  * Keep this list in sync with implementations registered in the tool registry.
@@ -47,7 +52,7 @@ export type ToolName =
     | "upload_blob"
     | "rag_create_collection"
     | "rag_add_documents"
-    | "rag_query"
+    | "rag_search"
     | "rag_delete_collection"
     | "rag_list_collections"
     | "rag_subscription_create"
@@ -63,7 +68,6 @@ export type ToolName =
     | "web_search"
     | "nostr_fetch"
     | "nostr_publish_as_user"
-    | "search"
     | "change_model"
     | "kill"
     | "generate_image";
@@ -73,6 +77,11 @@ export type ToolName =
  */
 export type AISdkTool<TInput = unknown, TOutput = unknown> = CoreTool<TInput, TOutput> & {
     getHumanReadableContent?: (args: TInput) => string;
+    /**
+     * Optional list of input argument keys to expose in conversation transcript XML.
+     * Keys are read from tool input args at execution time and serialized as XML attributes.
+     */
+    transcriptArgsToInclude?: ToolTranscriptArgSpec[];
     /**
      * Whether this tool has side effects (modifies state, writes files, sends messages, etc.)
      * Default is true (assume side effects unless explicitly declared false).

package/src/tools/utils/transcript-args.ts

@@ -0,0 +1,13 @@
+import type { AISdkTool, ToolTranscriptArgSpec } from "@/tools/types";
+
+export function attachTranscriptArgs<TInput = unknown, TOutput = unknown>(
+    tool: AISdkTool<TInput, TOutput>,
+    args: ToolTranscriptArgSpec[]
+): AISdkTool<TInput, TOutput> {
+    Object.defineProperty(tool, "transcriptArgsToInclude", {
+        value: args,
+        enumerable: false,
+        configurable: true,
+    });
+    return tool;
+}

package/src/utils/cli-theme.ts

@@ -0,0 +1,13 @@
+import chalk from "chalk";
+
+export const amber = chalk.hex("#FFC107");
+export const amberBold = chalk.hex("#FFC107").bold;
+
+export const inquirerTheme = {
+    prefix: { idle: amber("?"), done: chalk.green("✓") },
+    icon: { cursor: amber("❯") },
+    style: {
+        highlight: (text: string) => amber(text),
+        answer: (text: string) => amber(text),
+    },
+};

package/src/utils/logger.ts

@@ -40,6 +40,8 @@ const emojis = {
 
 // File logging state
 let logFilePath: string | null = null;
+let warnLogFilePath: string | null = null;
+const WARN_LOG_MAX_SIZE = 100 * 1024 * 1024; // 100MB
 
 // Helper to format timestamp for file output
 function formatTimestamp(): string {
@@ -64,6 +66,55 @@ function writeToFile(level: string, message: string, args: unknown[]): void {
     fs.appendFileSync(logFilePath, logLine);
 }
 
+/**
+ * Structured entry written to warn.log for operator troubleshooting.
+ */
+export interface WarnLogEntry {
+    timestamp: string;
+    level: "warn" | "error";
+    component: string;
+    message: string;
+    context?: Record<string, unknown>;
+    error?: string;
+    stack?: string;
+}
+
+/**
+ * Rotate warn.log when it exceeds WARN_LOG_MAX_SIZE.
+ * Renames current file to warn.log.1, discarding any previous .1.
+ */
+function rotateWarnLogIfNeeded(): void {
+    if (!warnLogFilePath) return;
+    try {
+        const stat = fs.statSync(warnLogFilePath);
+        if (stat.size >= WARN_LOG_MAX_SIZE) {
+            const rotatedPath = `${warnLogFilePath}.1`;
+            try {
+                fs.unlinkSync(rotatedPath);
+            } catch {
+                // .1 doesn't exist, fine
+            }
+            fs.renameSync(warnLogFilePath, rotatedPath);
+        }
+    } catch {
+        // File doesn't exist yet, no rotation needed
+    }
+}
+
+/**
+ * Write a structured JSON entry to warn.log.
+ * Only writes if warn.log transport has been initialized.
+ */
+function writeToWarnLog(entry: WarnLogEntry): void {
+    if (!warnLogFilePath) return;
+    try {
+        rotateWarnLogIfNeeded();
+        fs.appendFileSync(warnLogFilePath, JSON.stringify(entry) + "\n");
+    } catch {
+        // Swallow — we can't let warn.log failures crash the system
+    }
+}
+
 // Initialize daemon logging
 async function initDaemonLogging(): Promise<void> {
     // Lazy-load config to avoid circular dependency
@@ -76,11 +127,15 @@ async function initDaemonLogging(): Promise<void> {
     // Ensure directory exists
     const logDir = path.dirname(logFilePath);
     fs.mkdirSync(logDir, { recursive: true });
+
+    // Initialize warn.log in the same directory
+    warnLogFilePath = path.join(logDir, "warn.log");
 }
 
 // Main logger object
 export const logger = {
     initDaemonLogging,
+    writeToWarnLog,
 
     /**
      * Check if a specific log level is enabled

package/src/utils/metadataKeys.ts

@@ -0,0 +1,17 @@
+/**
+ * Canonical metadata key variants for RAG document filtering.
+ *
+ * Documents ingested via different code-paths use different key conventions:
+ *   - Specialized services (reports, lessons, conversations) write **camelCase**.
+ *   - The generic `rag_add_documents` tool historically wrote **snake_case**.
+ *
+ * Every SQL LIKE filter that matches on these keys must check BOTH variants
+ * to avoid silently missing documents.  Keeping the pairs here as shared
+ * constants prevents the two sets from drifting out of sync.
+ */
+
+/** Tuple of [camelCase, snake_case] key names for project ID metadata. */
+export const PROJECT_ID_KEYS = ["projectId", "project_id"] as const;
+
+/** Tuple of [camelCase, snake_case] key names for agent pubkey metadata. */
+export const AGENT_PUBKEY_KEYS = ["agentPubkey", "agent_pubkey"] as const;

package/src/utils/sqlEscaping.ts

@@ -0,0 +1,39 @@
+/**
+ * SQL escaping utilities for LanceDB metadata LIKE queries.
+ *
+ * Shared across all RAG query builders (project filters, agent stats, etc.)
+ * to ensure consistent escaping of user-supplied values in SQL LIKE patterns.
+ *
+ * IMPORTANT: DataFusion (used by LanceDB) has NO default escape character.
+ * Backslash-based escapes only work when paired with an ESCAPE '\\' clause.
+ * See: https://github.com/apache/datafusion/issues/13291
+ */
+
+/**
+ * The ESCAPE clause literal required by every LIKE expression that uses
+ * the backslash escapes produced by {@link escapeSqlLikeValue}.
+ *
+ * Usage: `metadata LIKE '…${escaped}…' ${SQL_LIKE_ESCAPE_CLAUSE}`
+ */
+export const SQL_LIKE_ESCAPE_CLAUSE = "ESCAPE '\\\\'";
+
+/**
+ * Escape a string for use inside a SQL LIKE pattern.
+ *
+ * Escapes: backslashes (\), single quotes ('), double quotes ("),
+ * and LIKE wildcards (%, _).
+ *
+ * The returned value MUST be used together with {@link SQL_LIKE_ESCAPE_CLAUSE}
+ * so that DataFusion recognises the backslash as the escape character.
+ *
+ * @param value - Raw string to escape (e.g. a project ID or agent pubkey).
+ * @returns Escaped string safe for interpolation into a LIKE pattern.
+ */
+export function escapeSqlLikeValue(value: string): string {
+    return value
+        .replace(/\\/g, "\\\\")  // Escape backslashes first
+        .replace(/'/g, "''")     // SQL standard: escape single quote by doubling
+        .replace(/"/g, '\\"')    // Escape double quotes
+        .replace(/%/g, "\\%")    // Escape LIKE wildcard %
+        .replace(/_/g, "\\_");   // Escape LIKE wildcard _
+}

package/src/wrapper.ts

@@ -193,13 +193,17 @@ class DaemonWrapper {
      */
     private spawnDaemon(args: string[]): Promise<number> {
         return new Promise((resolve) => {
-            // Build command: bun <entry-point> daemon --supervised [args]
             const indexPath = resolveEntryPoint();
             const daemonArgs = [indexPath, "daemon", "--supervised", ...args];
 
-            console.log(`[Wrapper] Spawning: bun ${daemonArgs.join(" ")}`);
+            // Use Bun when running TypeScript sources; otherwise run compiled JS via Node.
+            const runtimeBinary = indexPath.endsWith(".ts")
+                ? (process.env.TENEX_BUN_BIN || "bun")
+                : process.execPath;
 
-            this.child = spawn("bun", daemonArgs, {
+            console.log(`[Wrapper] Spawning: ${runtimeBinary} ${daemonArgs.join(" ")}`);
+
+            this.child = spawn(runtimeBinary, daemonArgs, {
                 stdio: "inherit",
                 env: process.env,
             });