@tenex-chat/backend 0.9.1

package/src/prompts/fragments/25-rag-instructions.ts

@@ -0,0 +1,333 @@
+import { fragmentRegistry } from "../core/FragmentRegistry";
+import type { PromptFragment } from "../core/types";
+
+/**
+ * RAG (Retrieval-Augmented Generation) system instructions fragment
+ */
+export const ragInstructionsFragment: PromptFragment = {
+    id: "rag-instructions",
+    priority: 25,
+    template: () => `# RAG (Retrieval-Augmented Generation) System
+
+The RAG system provides semantic search and vector-based retrieval capabilities for enhanced agent memory and context management.
+
+## Architecture Overview
+
+The RAG system follows clean architecture principles with clear separation of concerns:
+- **RAGService**: Facade that coordinates all operations
+- **RAGDatabaseService**: Handles database lifecycle and connections
+- **RAGOperations**: Implements business logic for CRUD operations
+- **EmbeddingProvider**: Abstraction for embedding generation (local or cloud-based)
+
+## Configuration
+
+### Setting Up Embedding Models
+
+Configure your preferred embedding model using the CLI:
+
+\`\`\`bash
+# Global configuration (applies to all projects)
+tenex setup embed
+
+# Project-specific configuration
+tenex setup embed --project
+\`\`\`
+
+The system supports:
+- **Local Transformers**: Run models directly on your machine (no API key required)
+  - all-MiniLM-L6-v2 (default, fast)
+  - all-mpnet-base-v2 (better quality)
+  - Custom HuggingFace models
+- **OpenAI**: Cloud-based embeddings (requires API key)
+  - text-embedding-3-small (fast, good quality)
+  - text-embedding-3-large (best quality)
+
+## Available Tools
+
+### 1. rag_create_collection
+Create a new vector database collection for storing embeddings.
+
+\`\`\`typescript
+rag_create_collection({
+  name: "project_knowledge",  // Alphanumeric with underscores only
+  schema: {                    // Optional custom schema
+    category: "string"
+  }
+})
+\`\`\`
+
+### 2. rag_add_documents
+Add documents to a collection with automatic embedding generation.
+
+\`\`\`typescript
+rag_add_documents({
+  collection: "project_knowledge",
+  documents: [
+    {
+      content: "Document text content",
+      metadata: { type: "documentation", tags: ["api", "rest"] },
+      source: "api-docs.md",
+      id: "doc_001"  // Optional custom ID
+    },
+    {
+      file_path: "./docs/README.md",  // Can read from files
+      metadata: { type: "readme" }
+    }
+  ]
+})
+\`\`\`
+
+### 3. rag_query
+Perform semantic search on a collection.
+
+\`\`\`typescript
+rag_query({
+  collection: "project_knowledge",
+  query_text: "How does authentication work?",
+  top_k: 5,  // Number of results (1-100)
+  include_metadata: true  // Include document metadata
+})
+\`\`\`
+
+### 4. rag_delete_collection
+Remove a collection and all its documents.
+
+\`\`\`typescript
+rag_delete_collection({
+  name: "project_knowledge",
+  confirm: true  // Required safety flag
+})
+\`\`\`
+
+### 5. rag_list_collections
+List all available collections with optional statistics.
+
+\`\`\`typescript
+rag_list_collections({
+  include_stats: true  // Include document counts per collection
+})
+\`\`\`
+
+## Best Practices
+
+### Collection Design
+- **Single Purpose**: Create focused collections for specific domains
+- **Naming Convention**: Use descriptive lowercase names with underscores
+  - ✅ \`agent_memory\`, \`code_snippets\`, \`user_preferences\`
+  - ❌ \`MyCollection\`, \`data-store\`, \`collection#1\`
+
+### Document Management
+- **Metadata Strategy**: Always include relevant metadata for filtering
+  \`\`\`typescript
+  metadata: {
+    type: "code" | "documentation" | "conversation",
+    language?: string,
+    timestamp?: number,
+    tags?: string[],
+    source?: string
+  }
+  \`\`\`
+- **Content Size**: Keep individual documents under 1MB for optimal performance
+- **Batch Operations**: Add multiple documents in a single call for efficiency
+
+### Query Optimization
+- **Natural Language**: Use conversational queries for best results
+  - ✅ "How to implement user authentication with JWT tokens"
+  - ❌ "auth jwt impl func"
+- **Result Limits**: Use appropriate \`top_k\` values (5-10 for most cases)
+- **Relevance Scores**: Results include scores (0-1) indicating similarity
+
+### Error Handling
+All tools use standardized error responses:
+\`\`\`json
+{
+  "success": false,
+  "error": "Descriptive error message",
+  "toolName": "rag_query"
+}
+\`\`\`
+
+## Use Cases
+
+### 1. Agent Self-Reflection
+Build persistent memory across conversations:
+
+\`\`\`typescript
+// Store insights and decisions
+rag_create_collection({ name: "agent_insights" })
+
+rag_add_documents({
+  collection: "agent_insights",
+  documents: [{
+    content: "User prefers TypeScript over JavaScript for all new projects",
+    metadata: { 
+      type: "preference",
+      confidence: 0.9,
+      learned_from: "conversation_123"
+    }
+  }]
+})
+
+// Later, retrieve relevant context
+rag_query({
+  collection: "agent_insights",
+  query_text: "What are the user's programming language preferences?"
+})
+\`\`\`
+
+### 2. Project Knowledge Base
+Index project documentation and code:
+
+\`\`\`typescript
+rag_create_collection({ name: "project_docs" })
+
+// Index all markdown files
+rag_add_documents({
+  collection: "project_docs",
+  documents: [
+    { file_path: "README.md" },
+    { file_path: "docs/api.md" },
+    { file_path: "docs/architecture.md" }
+  ]
+})
+
+// Query for specific information
+rag_query({
+  collection: "project_docs",
+  query_text: "API authentication methods"
+})
+\`\`\`
+
+### 3. Enhanced Lesson Learning
+Combine with lesson_learn for semantic retrieval:
+
+\`\`\`typescript
+// After learning a lesson
+lesson_learn({
+  title: "Async error handling",
+  lesson: "Always use try-catch with async/await"
+})
+
+// Store in RAG for semantic search
+rag_add_documents({
+  collection: "lessons",
+  documents: [{
+    content: lesson.detailed || lesson.lesson,
+    metadata: {
+      title: lesson.title,
+      category: lesson.category,
+      hashtags: lesson.hashtags
+    }
+  }]
+})
+
+// Find related lessons semantically
+rag_query({
+  collection: "lessons",
+  query_text: "How to handle promise rejections"
+})
+\`\`\`
+
+### 4. Code Pattern Recognition
+Store and retrieve code patterns:
+
+\`\`\`typescript
+rag_create_collection({ name: "code_patterns" })
+
+rag_add_documents({
+  collection: "code_patterns",
+  documents: [{
+    content: "const useAuth = () => { const [user, setUser] = useState(null); ... }",
+    metadata: {
+      pattern: "React Hook",
+      language: "TypeScript",
+      framework: "React",
+      complexity: "medium"
+    }
+  }]
+})
+
+rag_query({
+  collection: "code_patterns",
+  query_text: "authentication hook implementation"
+})
+\`\`\`
+
+## Integration with Other Tools
+
+### With grep
+Index search results for faster future retrieval:
+\`\`\`typescript
+// After grep finds relevant files
+rag_add_documents({
+  collection: "indexed_code",
+  documents: searchResults.map(file => ({
+    file_path: file,
+    metadata: { type: "code" }
+  }))
+})
+\`\`\`
+
+### With delegate
+Share collections between agents:
+\`\`\`typescript
+delegate({
+  task: "Analyze the project documentation",
+  tools: ["rag_query"],
+  context: "Use collection 'project_docs' for analysis"
+})
+\`\`\`
+
+### With report_write
+Store reports for easy retrieval:
+\`\`\`typescript
+report_write({ title: "Performance Analysis", content: "..." })
+
+rag_add_documents({
+  collection: "reports",
+  documents: [{
+    content: report.content,
+    metadata: { 
+      title: report.title,
+      type: "report",
+      created_at: Date.now()
+    }
+  }]
+})
+\`\`\`
+
+## Performance Considerations
+
+1. **Embedding Generation**: First-time model loading may take a few seconds
+2. **Batch Size**: Documents are processed in batches of 100 for optimal performance
+3. **Vector Dimensions**: Varies by model (384 for MiniLM, 768 for mpnet)
+4. **Storage**: LanceDB uses efficient columnar storage with compression
+5. **Query Speed**: Sub-second for collections under 100K documents
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Collection Already Exists**
+   - Solution: Use unique names or delete existing collection first
+
+2. **Empty Query Results**
+   - Check if documents were successfully added
+   - Verify collection name is correct
+   - Try broader query terms
+
+3. **Slow Embedding Generation**
+   - First run downloads model (one-time)
+   - Consider using smaller model for speed
+   - Use cloud-based embeddings for better performance
+
+4. **Configuration Not Found**
+   - Run \`tenex setup embed\` to configure
+   - Check \`.tenex/embed.json\` exists
+   - Verify environment variables for API keys
+
+Remember: RAG empowers agents with persistent, searchable knowledge that enhances capabilities across conversations!`,
+};
+
+// Register the fragment
+fragmentRegistry.register(ragInstructionsFragment);

package/src/prompts/fragments/26-mcp-resources.ts

@@ -0,0 +1,237 @@
+import type { Resource, ResourceTemplate } from "@modelcontextprotocol/sdk/types.js";
+import type { MCPManager } from "@/services/mcp/MCPManager";
+import { logger } from "@/utils/logger";
+import { fragmentRegistry } from "../core/FragmentRegistry";
+import type { PromptFragment } from "../core/types";
+
+// Use official SDK types
+type MCPResource = Resource;
+type MCPResourceTemplate = ResourceTemplate;
+
+interface ResourcesPerServer {
+    serverName: string;
+    resources: MCPResource[];
+    templates: MCPResourceTemplate[];
+}
+
+interface McpResourcesFragmentArgs {
+    agentPubkey: string;
+    mcpEnabled: boolean;
+    resourcesPerServer: ResourcesPerServer[];
+}
+
+/**
+ * Extract unique MCP server names from agent's tool list.
+ * MCP tools are namespaced as: mcp__serverName__toolName
+ */
+export function extractAgentMcpServers(agentTools: string[]): string[] {
+    const servers = new Set<string>();
+    for (const tool of agentTools) {
+        if (tool.startsWith("mcp__")) {
+            const parts = tool.split("__");
+            if (parts.length >= 3) {
+                servers.add(parts[1]); // parts[1] is server name
+            }
+        }
+    }
+    return Array.from(servers);
+}
+
+/**
+ * Fetch resources from MCP servers that the agent has access to.
+ * Agent has access to a server if it has any tools from that server (mcp__serverName__*).
+ */
+export async function fetchAgentMcpResources(
+    agentTools: string[],
+    mcpManager: MCPManager
+): Promise<ResourcesPerServer[]> {
+    const agentMcpServers = extractAgentMcpServers(agentTools);
+
+    if (agentMcpServers.length === 0) {
+        return [];
+    }
+
+    const runningServers = mcpManager.getRunningServers();
+
+    // Only show resources from servers the agent has tools for AND are running
+    const agentRunningServers = agentMcpServers.filter(s => runningServers.includes(s));
+
+    if (agentRunningServers.length === 0) {
+        return [];
+    }
+
+    const resourcesPerServer = await Promise.all(
+        agentRunningServers.map(async (serverName: string) => {
+            try {
+                const [resources, templates] = await Promise.all([
+                    mcpManager.listResources(serverName),
+                    mcpManager.listResourceTemplates(serverName),
+                ]);
+                logger.debug(
+                    `Fetched ${resources.length} resources and ${templates.length} templates from '${serverName}'`
+                );
+                return { serverName, resources, templates };
+            } catch (error) {
+                logger.warn(`Failed to fetch MCP resources from '${serverName}':`, error);
+                return { serverName, resources: [], templates: [] };
+            }
+        })
+    );
+
+    return resourcesPerServer;
+}
+
+/**
+ * Format a single resource for display
+ */
+function formatResource(
+    resource: { uri: string; name: string; description?: string; mimeType?: string },
+    serverName: string
+): string {
+    const lines: string[] = [];
+    lines.push(`- **${resource.name}** (\`${resource.uri}\`)`);
+    if (resource.description) {
+        lines.push(`  ${resource.description}`);
+    }
+    lines.push(`  Server: ${serverName}`);
+    if (resource.mimeType) {
+        lines.push(`  Type: ${resource.mimeType}`);
+    }
+    return lines.join("\n");
+}
+
+/**
+ * Format a resource template for display
+ */
+function formatTemplate(
+    template: { uriTemplate: string; name: string; description?: string; mimeType?: string },
+    serverName: string
+): string {
+    const lines: string[] = [];
+    lines.push(`- **${template.name}** (\`${template.uriTemplate}\`) *[Template]*`);
+    if (template.description) {
+        lines.push(`  ${template.description}`);
+    }
+    lines.push(`  Server: ${serverName}`);
+
+    // Extract parameter names from template
+    const params = template.uriTemplate.match(/\{([^}]+)\}/g);
+    if (params) {
+        const paramNames = params.map((p) => p.slice(1, -1)).join(", ");
+        lines.push(`  **Required parameters:** ${paramNames}`);
+        lines.push("  **Note:** Expand this template with actual values before subscribing");
+    }
+
+    if (template.mimeType) {
+        lines.push(`  Type: ${template.mimeType}`);
+    }
+    return lines.join("\n");
+}
+
+/**
+ * MCP Resources fragment - shows available resources for RAG subscription
+ */
+export const mcpResourcesFragment: PromptFragment<McpResourcesFragmentArgs> = {
+    id: "mcp-resources",
+    priority: 26,
+
+    template: (args: McpResourcesFragmentArgs): string => {
+        if (!args.mcpEnabled || args.resourcesPerServer.length === 0) {
+            return ""; // Don't show if MCP is disabled or no resources
+        }
+
+        // Check if there are any actual resources or templates across all servers
+        const hasAnyResources = args.resourcesPerServer.some(
+            (server) => server.resources.length > 0 || server.templates.length > 0
+        );
+
+        if (!hasAnyResources) {
+            return ""; // Don't show fragment if no resources available
+        }
+
+        const sections: string[] = [];
+        const serverNames = args.resourcesPerServer.map((s) => s.serverName);
+        sections.push("# Available MCP Resources for RAG Subscription\n");
+        sections.push(
+            `You have access to MCP resources from ${serverNames.length} server${serverNames.length === 1 ? "" : "s"}: ${serverNames.join(", ")}\n`
+        );
+
+        let totalResources = 0;
+        let totalTemplates = 0;
+
+        for (const serverData of args.resourcesPerServer) {
+            const { serverName, resources, templates } = serverData;
+
+            if (resources.length === 0 && templates.length === 0) {
+                continue; // Skip servers with no resources
+            }
+
+            sections.push(`## Server: ${serverName}\n`);
+
+            // Direct resources
+            if (resources.length > 0) {
+                sections.push("### Direct Resources (ready to subscribe)\n");
+                for (const resource of resources) {
+                    sections.push(formatResource(resource, serverName));
+                    sections.push("");
+                }
+                totalResources += resources.length;
+            }
+
+            // Resource templates
+            if (templates.length > 0) {
+                sections.push("### Resource Templates (require parameter expansion)\n");
+                for (const template of templates) {
+                    sections.push(formatTemplate(template, serverName));
+                    sections.push("");
+                }
+                totalTemplates += templates.length;
+            }
+
+            sections.push("---\n");
+        }
+
+        // Add usage instructions
+        sections.push("## How to Use MCP Resources\n");
+
+        sections.push("### On-Demand Reading\n");
+        sections.push("Use the `mcp_resource_read` tool to fetch resource content immediately:\n");
+        sections.push('- **serverName**: The server name shown above (e.g., "nostr-provider")');
+        sections.push("- **resourceUri**: The exact URI shown in parentheses");
+        sections.push("- **description**: Why you're reading this resource\n");
+        sections.push("For templates, provide `templateParams` to expand `{placeholders}`.\n");
+
+        sections.push("### Persistent Subscriptions\n");
+        sections.push("Use the `rag_subscription_create` tool to stream updates into RAG collections:\n");
+        sections.push('- **subscriptionId**: Unique ID for your subscription (e.g., "global-feed")');
+        sections.push('- **mcpServerId**: The server name shown above (e.g., "nostr-provider")');
+        sections.push("- **resourceUri**: The exact URI (templates must be expanded first)");
+        sections.push("- **ragCollection**: RAG collection name");
+        sections.push("- **description**: What this subscription does\n");
+
+        sections.push(
+            `**Summary:** ${totalResources} direct resource${totalResources === 1 ? "" : "s"}, ${totalTemplates} template${totalTemplates === 1 ? "" : "s"} available`
+        );
+
+        return sections.join("\n");
+    },
+
+    validateArgs: (args: unknown): args is McpResourcesFragmentArgs => {
+        return (
+            typeof args === "object" &&
+            args !== null &&
+            "agentPubkey" in args &&
+            "mcpEnabled" in args &&
+            "resourcesPerServer" in args &&
+            typeof (args as McpResourcesFragmentArgs).agentPubkey === "string" &&
+            typeof (args as McpResourcesFragmentArgs).mcpEnabled === "boolean" &&
+            Array.isArray((args as McpResourcesFragmentArgs).resourcesPerServer)
+        );
+    },
+
+    expectedArgs:
+        "{ agentPubkey: string, mcpEnabled: boolean, resourcesPerServer: ResourcesPerServer[] }",
+};
+
+// Auto-register the fragment
+fragmentRegistry.register(mcpResourcesFragment);

package/src/prompts/fragments/27-memorized-reports.ts

@@ -0,0 +1,77 @@
+import type { ReportInfo } from "@/services/reports/ReportService";
+import { fragmentRegistry } from "../core/FragmentRegistry";
+import type { PromptFragment } from "../core/types";
+
+/**
+ * Memorized reports fragment - injects reports marked with memorize=true or memorize_team=true
+ * into the agent's system prompt for persistent context.
+ *
+ * - memorize=true: Report is only injected into the authoring agent's system prompt
+ * - memorize_team=true: Report is injected into ALL agents' system prompts in the project
+ */
+interface MemorizedReportsArgs {
+    reports: ReportInfo[];
+}
+
+export const memorizedReportsFragment: PromptFragment<MemorizedReportsArgs> = {
+    id: "memorized-reports",
+    priority: 27, // After RAG instructions, before worktree context
+    template: ({ reports }) => {
+        if (!reports || reports.length === 0) {
+            return ""; // No memorized reports
+        }
+
+        // Separate team-wide reports from agent-specific reports
+        const teamReports = reports.filter((r) => r.isMemorizedTeam);
+        const agentReports = reports.filter((r) => !r.isMemorizedTeam);
+
+        const parts: string[] = [];
+        parts.push("## Memorized Knowledge\n");
+
+        // Add team-wide reports first (if any)
+        if (teamReports.length > 0) {
+            parts.push("### Team-Wide Knowledge\n");
+            parts.push("*The following knowledge is shared across ALL agents in this project:*\n");
+
+            for (const report of teamReports) {
+                parts.push(`#### ${report.title || report.slug}`);
+                if (report.summary) {
+                    parts.push(`*${report.summary}*\n`);
+                }
+                if (report.content) {
+                    parts.push(report.content);
+                }
+                parts.push(""); // Empty line between reports
+            }
+        }
+
+        // Add agent-specific reports (if any)
+        if (agentReports.length > 0) {
+            if (teamReports.length > 0) {
+                parts.push("### Agent-Specific Knowledge\n");
+                parts.push("*The following knowledge is specific to your role:*\n");
+            } else {
+                parts.push(
+                    "The following reports have been memorized and represent persistent knowledge for your role:\n"
+                );
+            }
+
+            for (const report of agentReports) {
+                const heading = teamReports.length > 0 ? "####" : "###";
+                parts.push(`${heading} ${report.title || report.slug}`);
+                if (report.summary) {
+                    parts.push(`*${report.summary}*\n`);
+                }
+                if (report.content) {
+                    parts.push(report.content);
+                }
+                parts.push(""); // Empty line between reports
+            }
+        }
+
+        return parts.join("\n");
+    },
+};
+
+// Register the fragment
+fragmentRegistry.register(memorizedReportsFragment);

package/src/prompts/fragments/28-agent-directed-monitoring.ts

@@ -0,0 +1,32 @@
+/**
+ * Agent-Directed Monitoring Fragment
+ *
+ * Provides guidance on how agents can monitor delegated work using existing tools.
+ * Agents control when and how often to check on delegated tasks.
+ */
+
+import type { PromptFragment } from "../core/types";
+
+const MONITORING_GUIDANCE = `## Monitoring Delegated Work
+
+When you delegate tasks to other agents, you can monitor their progress using existing tools:
+
+1. **Check Progress**: Use \`conversation_get\` with the delegation conversation ID. Optionally pass a \`prompt\` parameter to have the tool summarize the delegatee's progress.
+
+2. **Wait Between Checks**: Use \`shell()\` with \`sleep <seconds>\` to wait between progress checks. Choose intervals based on task complexity:
+   - Quick tasks (< 2 min expected): Check every 30 seconds
+   - Medium tasks (2-10 min): Check every 1-2 minutes
+   - Long tasks (> 10 min): Check every 3-5 minutes
+
+3. **When to Monitor**: You decide whether active monitoring is needed based on:
+   - Task criticality and deadline pressure
+   - Whether you need intermediate results
+   - The delegatee's reliability for the task type
+
+Most delegations complete and return results automatically. Active monitoring is optional and primarily useful for long-running tasks where you want progress visibility.`;
+
+export const agentDirectedMonitoringFragment: PromptFragment = {
+    id: "agent-directed-monitoring",
+    priority: 28, // After memorized reports (27), before worktree context (30)
+    template: () => MONITORING_GUIDANCE,
+};