@tenex-chat/backend 0.9.4 → 0.9.6

package/src/prompts/fragments/25-rag-instructions.ts

@@ -77,15 +77,16 @@ rag_add_documents({
 })
 \`\`\`
 
-### 3. rag_query
-Perform semantic search on a collection.
+### 3. rag_search
+Search across ALL project knowledge — reports, conversations, lessons, and any
+additional RAG collections — using natural language semantic search.
 
 \`\`\`typescript
-rag_query({
-  collection: "project_knowledge",
-  query_text: "How does authentication work?",
-  top_k: 5,  // Number of results (1-100)
-  include_metadata: true  // Include document metadata
+rag_search({
+  query: "How does authentication work?",
+  limit: 10,  // Maximum results (default: 10)
+  collections: ["reports", "lessons"],  // Optional: filter to specific collections
+  prompt: "Summarize the auth approach"  // Optional: LLM extraction
 })
 \`\`\`
 
@@ -143,7 +144,7 @@ All tools use standardized error responses:
 {
   "success": false,
   "error": "Descriptive error message",
-  "toolName": "rag_query"
+  "toolName": "rag_search"
 }
 \`\`\`
 
@@ -169,9 +170,9 @@ rag_add_documents({
 })
 
 // Later, retrieve relevant context
-rag_query({
-  collection: "agent_insights",
-  query_text: "What are the user's programming language preferences?"
+rag_search({
+  query: "What are the user's programming language preferences?",
+  collections: ["agent_insights"]
 })
 \`\`\`
 
@@ -192,9 +193,9 @@ rag_add_documents({
 })
 
 // Query for specific information
-rag_query({
-  collection: "project_docs",
-  query_text: "API authentication methods"
+rag_search({
+  query: "API authentication methods",
+  collections: ["project_docs"]
 })
 \`\`\`
 
@@ -222,9 +223,9 @@ rag_add_documents({
 })
 
 // Find related lessons semantically
-rag_query({
-  collection: "lessons",
-  query_text: "How to handle promise rejections"
+rag_search({
+  query: "How to handle promise rejections",
+  collections: ["lessons"]
 })
 \`\`\`
 
@@ -247,9 +248,9 @@ rag_add_documents({
   }]
 })
 
-rag_query({
-  collection: "code_patterns",
-  query_text: "authentication hook implementation"
+rag_search({
+  query: "authentication hook implementation",
+  collections: ["code_patterns"]
 })
 \`\`\`
 
@@ -273,7 +274,7 @@ Share collections between agents:
 \`\`\`typescript
 delegate({
   task: "Analyze the project documentation",
-  tools: ["rag_query"],
+  tools: ["rag_search"],
   context: "Use collection 'project_docs' for analysis"
 })
 \`\`\`

package/src/prompts/fragments/31-agents-md-guidance.ts

@@ -57,27 +57,13 @@ export const agentsMdGuidanceFragment: PromptFragment<AgentsMdGuidanceArgs> = {
 - Deeper, more specific AGENTS.md files override general root instructions.
 
 ### Writing AGENTS.md Files
-When working in a directory that needs specific agent guidance:
-1. Create an AGENTS.md file in that directory.
-2. Focus on **executable commands** (test/build) and **strict conventions**.
-3. Do not duplicate generic info; focus on what is unique to this directory.
-
-### AGENTS.md Format
-\`\`\`markdown
-# Directory Context
-Specific architectural details or business logic for this directory.
-
-## Commands
-- Test: \`npm test path/to/dir\`
-- Lint: \`npm run lint:specific\`
-
-## Conventions
-- Code Style: Functional patterns preferred
-- Naming: CamelCase for files, PascalCase for classes
-
-## Related
-- [API Docs](./docs/api.md)
-\`\`\``);
+When creating or updating an AGENTS.md file:
+1. **Maximum 50 lines.** If it's longer, you're including too much. Cut aggressively.
+2. **No code examples.** Don't include correct/incorrect patterns, usage snippets, or testing templates. Just state the rule in plain English.
+3. **No boilerplate sections.** Don't add Anti-Patterns, Testing, Dependencies, or Related sections. Only include what's genuinely unique to this directory.
+4. **Don't repeat parent info.** Import patterns, naming conventions, layer rules — if it's in a parent directory's AGENTS.md, don't restate it.
+5. **List actual files.** Name the real files that exist, not idealized directory trees. Keep it to key files only, not every file.
+6. **State rules, not ideology.** "Tools delegate to services" is a rule. A 10-line example showing correct vs incorrect delegation is ideology.`);
 
         // If root AGENTS.md content is available and short, include it
         if (rootAgentsMdContent && rootAgentsMdContent.length < MAX_ROOT_CONTENT_LENGTH_FOR_SYSTEM_PROMPT) {

package/src/prompts/fragments/index.ts

@@ -37,6 +37,7 @@ import { agentDirectedMonitoringFragment } from "./28-agent-directed-monitoring"
 import { ragCollectionsFragment } from "./29-rag-collections";
 import { worktreeContextFragment } from "./30-worktree-context";
 import { agentsMdGuidanceFragment } from "./31-agents-md-guidance";
+import { processMetricsFragment } from "./32-process-metrics";
 import { debugModeFragment } from "./debug-mode";
 import { delegationCompletionFragment } from "./delegation-completion";
 
@@ -85,6 +86,7 @@ export function registerAllFragments(): void {
     fragmentRegistry.register(ragCollectionsFragment);
     fragmentRegistry.register(worktreeContextFragment);
     fragmentRegistry.register(agentsMdGuidanceFragment);
+    fragmentRegistry.register(processMetricsFragment);
 }
 
 // Auto-register all fragments on import

package/src/prompts/utils/systemPromptBuilder.ts

@@ -250,37 +250,24 @@ async function addCoreAgentFragments(
 
     // Add RAG collection attribution - shows agents their contributions to RAG collections
     // This uses the provenance tracking metadata (agent_pubkey) from document ingestion
-    //
-    // OPTIMIZATION: First check if any collections exist using lightweight check
-    // to avoid initializing embedding provider when RAG isn't used.
     try {
-        const { hasRagCollections, RAGService } = await import("@/services/rag/RAGService");
-
-        // Fast path: skip full initialization if no collections exist
-        // Note: hasRagCollections() returns false on errors and logs them internally
-        if (!(await hasRagCollections())) {
-            logger.debug("📊 Skipping RAG collection stats - no collections available");
-        } else {
-            // Collections exist - now we need full service for stats
-            const ragService = RAGService.getInstance();
-            const collections = await ragService.getAllCollectionStats(agent.pubkey);
-
-            // Only add the fragment if we have any collection data
-            if (collections.length > 0) {
-                builder.add("rag-collections", {
-                    agentPubkey: agent.pubkey,
-                    collections,
-                });
-                logger.debug("📊 Added RAG collection stats to system prompt", {
-                    agent: agent.name,
-                    collectionsWithContributions: collections.filter(c => c.agentDocCount > 0).length,
-                    totalCollections: collections.length,
-                });
-            }
+        const { RAGService } = await import("@/services/rag/RAGService");
+        const ragService = RAGService.getInstance();
+        const collections = await ragService.getAllCollectionStats(agent.pubkey);
+
+        if (collections.length > 0) {
+            builder.add("rag-collections", {
+                agentPubkey: agent.pubkey,
+                collections,
+            });
+            logger.debug("📊 Added RAG collection stats to system prompt", {
+                agent: agent.name,
+                collectionsWithContributions: collections.filter(c => c.agentDocCount > 0).length,
+                totalCollections: collections.length,
+            });
         }
     } catch (error) {
-        // RAG service might not be available - skip gracefully
-        logger.debug("Could not fetch RAG collection stats for prompt:", error);
+        logger.debug("Could not get RAG collection stats:", error);
     }
 }
 
@@ -654,6 +641,9 @@ async function buildMainSystemPrompt(options: BuildSystemPromptOptions): Promise
     // Add relay configuration context
     systemPromptBuilder.add("relay-configuration", {});
 
+    // Add process metrics (PID, uptime, CPU/memory usage)
+    systemPromptBuilder.add("process-metrics", {});
+
     // Add meta-project context (other projects this agent belongs to)
     // This gives agents cross-project awareness without overwhelming them
     systemPromptBuilder.add("meta-project-context", {

package/src/services/AgentDefinitionMonitor.ts

@@ -1,6 +1,7 @@
 import type { StoredAgent } from "@/agents/AgentStorage";
 import { agentStorage } from "@/agents/AgentStorage";
 import { NDKAgentDefinition } from "@/events/NDKAgentDefinition";
+import { isValidCategory } from "@/agents/role-categories";
 import { logger } from "@/utils/logger";
 import type NDK from "@nostr-dev-kit/ndk";
 import type { NDKEvent, NDKFilter, NDKSubscription } from "@nostr-dev-kit/ndk";
@@ -545,6 +546,13 @@ export class AgentDefinitionMonitor {
             changedFields.push("role");
         }
 
+        const rawCategory = agentDef.category || undefined;
+        const newCategory = rawCategory && isValidCategory(rawCategory) ? rawCategory : undefined;
+        if (newCategory !== storedAgent.category) {
+            storedAgent.category = newCategory;
+            changedFields.push("category");
+        }
+
         const newDescription = agentDef.description || undefined;
         if (newDescription !== storedAgent.description) {
             storedAgent.description = newDescription;

package/src/services/ConfigService.ts

@@ -367,6 +367,40 @@ export class ConfigService {
         return loadedConfig.llms.configurations[name];
     }
 
+    /**
+     * Get all resolved LLM configurations, skipping meta models.
+     * Returns the default config first, followed by the rest in declaration order.
+     */
+    getAllLLMConfigs(): LLMConfiguration[] {
+        if (!this.loadedConfig) {
+            throw new Error("Config not loaded. Call loadConfig() first.");
+        }
+
+        const llms = this.loadedConfig.llms;
+        const configs: LLMConfiguration[] = [];
+        const seen = new Set<string>();
+
+        // Default first
+        if (llms.default && llms.configurations[llms.default]) {
+            const config = llms.configurations[llms.default];
+            if (!isMetaModelConfiguration(config)) {
+                configs.push(config);
+                seen.add(`${config.provider}:${config.model}`);
+            }
+        }
+
+        // Then the rest, deduplicating by provider:model
+        for (const config of Object.values(llms.configurations)) {
+            if (isMetaModelConfiguration(config)) continue;
+            const key = `${config.provider}:${config.model}`;
+            if (seen.has(key)) continue;
+            seen.add(key);
+            configs.push(config);
+        }
+
+        return configs;
+    }
+
     /**
      * Check if a configuration name refers to a meta model.
      */

package/src/services/PubkeyService.ts

@@ -85,7 +85,13 @@ export class PubkeyService {
      * Uses AgentRegistry's getAgentByPubkey for efficient O(1) lookup.
      */
     private getAgentSlug(pubkey: Hexpubkey): string | undefined {
-        const projectCtx = getProjectContext();
+        let projectCtx;
+        try {
+            projectCtx = getProjectContext();
+        } catch {
+            // Name resolution must remain available even outside a project-scoped context
+            return undefined;
+        }
 
         // Use direct pubkey lookup from AgentRegistry (O(1) instead of O(n))
         const agent = projectCtx.getAgentByPubkey(pubkey);

package/src/services/compression/CompressionService.ts

@@ -2,6 +2,7 @@ import type { LLMService } from "@/llm/service";
 import { shortenConversationId } from "@/utils/conversation-id";
 import type { ConversationStore } from "@/conversations/ConversationStore";
 import type { ConversationEntry } from "@/conversations/types";
+import { renderConversationXml } from "@/conversations/formatters/utils/conversation-transcript-formatter";
 import { trace, SpanStatusCode, type Span } from "@opentelemetry/api";
 import { logger } from "@/utils/logger";
 import { config } from "@/services/ConfigService";
@@ -18,9 +19,52 @@ import {
   validateSegmentsForEntries,
   applySegmentsToEntries,
   createFallbackSegmentForEntries,
+  computeTokenAwareWindowSize,
 } from "./compression-utils.js";
 
 const tracer = trace.getTracer("tenex.compression");
+const CONTEXT_SEGMENT_LIMIT = 3;
+
+function buildContextPreamble(existingSegments: CompressionSegment[]): string {
+  if (existingSegments.length === 0) {
+    return "";
+  }
+
+  const recentSegments = existingSegments.slice(-CONTEXT_SEGMENT_LIMIT);
+  const contextLines = recentSegments.map(
+    (segment, index) => `[Previous context ${index + 1}]: ${segment.compressed}`
+  );
+
+  return `Previous conversation context (already compressed):\n${contextLines.join("\n")}\n\n`;
+}
+
+function buildCompressionPrompt(
+  transcriptXml: string,
+  firstShortId: string,
+  lastShortId: string,
+  existingSegments: CompressionSegment[]
+): string {
+  const contextPreamble = buildContextPreamble(existingSegments);
+
+  return `You are compressing conversation history represented as XML. Analyze the conversation and create 1-3 compressed segments that preserve key information while being concise.
+
+For each segment, provide:
+- fromEventId: starting message id from XML (the id attribute)
+- toEventId: ending message id from XML (the id attribute)
+- compressed: a concise summary (2-4 sentences) of the key points
+
+Rules:
+- Preserve attribution: who said or did what.
+- Preserve recipient targeting when present.
+- Preserve temporal flow using the time="+seconds" indicators and conversation t0.
+- Use IDs exactly as shown in XML id attributes (do not invent IDs).
+- The first segment must start at id "${firstShortId}" and the last segment must end at id "${lastShortId}".
+
+${contextPreamble}Messages to compress:
+${transcriptXml}
+
+Create segments that group related topics together. Preserve important decisions, errors, and outcomes.`;
+}
 
 /**
  * CompressionService - Orchestrates conversation history compression.
@@ -153,17 +197,57 @@ export class CompressionService {
               conversationId,
               entries,
               compressionConfig.slidingWindowSize,
-              span
+              span,
+              effectiveBudget
             );
           }
           span.setStatus({ code: SpanStatusCode.OK });
           return;
         }
 
+        // Compute range impact before attempting LLM call
+        const rangeEntries = entries.slice(range.startIndex, range.endIndex);
+        const rangeTokens = estimateTokensFromEntries(rangeEntries);
+
+        // Skip LLM when range can't meaningfully help
+        if (blocking) {
+          const tokenOverage = currentTokens - effectiveBudget;
+          if (rangeTokens < tokenOverage) {
+            // Even compressing the entire range to zero can't close the gap
+            span.addEvent("compression.skip_to_fallback", {
+              "reason": "range_too_small_for_overage",
+              "range.tokens": rangeTokens,
+              "token.overage": tokenOverage,
+            });
+            await this.useFallback(
+              conversationId,
+              entries,
+              compressionConfig.slidingWindowSize,
+              span,
+              effectiveBudget
+            );
+            span.setStatus({ code: SpanStatusCode.OK });
+            return;
+          }
+        } else {
+          // Proactive mode: skip if range is tiny (not worth an LLM call)
+          if (rangeTokens < 500) {
+            span.addEvent("compression.skip_proactive", {
+              "reason": "range_too_small",
+              "range.tokens": rangeTokens,
+            });
+            span.setStatus({ code: SpanStatusCode.OK });
+            return;
+          }
+        }
+
         // Attempt LLM compression
         try {
-          const rangeEntries = entries.slice(range.startIndex, range.endIndex);
-          const newSegments = await this.compressEntries(rangeEntries);
+          const newSegments = await this.compressEntries(
+            conversationId,
+            rangeEntries,
+            existingSegments
+          );
 
           // Emit telemetry for successful summary generation
           span.addEvent("compression.summary_generated", {
@@ -195,7 +279,8 @@ export class CompressionService {
                 conversationId,
                 entries,
                 compressionConfig.slidingWindowSize,
-                span
+                span,
+                effectiveBudget
               );
             }
             return;
@@ -239,7 +324,8 @@ export class CompressionService {
               conversationId,
               entries,
               compressionConfig.slidingWindowSize,
-              span
+              span,
+              effectiveBudget
             );
           } else {
             // Proactive mode: fail silently, don't throw
@@ -263,7 +349,9 @@ export class CompressionService {
    * Compress a range of entries using LLM.
    */
   private async compressEntries(
-    entries: ConversationEntry[]
+    conversationId: string,
+    entries: ConversationEntry[],
+    existingSegments: CompressionSegment[]
   ): Promise<CompressionSegment[]> {
     return tracer.startActiveSpan(
       "compression.llm_compress",
@@ -271,71 +359,30 @@ export class CompressionService {
         try {
           span.setAttribute("entries.count", entries.length);
 
-          // Format entries for LLM, including tool payloads
-          const formattedEntries = entries
-            .map((e) => {
-              let formatted = `[${e.messageType}]`;
-
-              // Add text content if present
-              if (e.content) {
-                formatted += ` ${e.content}`;
-              }
-
-              // Add tool payload summary for tool-call/tool-result entries
-              if (e.toolData && e.toolData.length > 0) {
-                const toolSummary = e.toolData
-                  .map((tool) => {
-                    if ('toolName' in tool) {
-                      // ToolCallPart
-                      return `Tool: ${tool.toolName}`;
-                    } else if ('toolCallId' in tool) {
-                      // ToolResultPart - cast to any to avoid type narrowing issues
-                      const toolResult = tool as any;
-                      const resultPreview = typeof toolResult.result === 'string'
-                        ? toolResult.result.substring(0, 100)
-                        : JSON.stringify(toolResult.result).substring(0, 100);
-                      return `Result: ${resultPreview}${resultPreview.length >= 100 ? '...' : ''}`;
-                    }
-                    return '';
-                  })
-                  .filter(Boolean)
-                  .join(', ');
-
-                if (toolSummary) {
-                  formatted += ` [${toolSummary}]`;
-                }
-              }
-
-              return formatted;
-            })
-            .join("\n\n");
-
-          const eventIds = entries
-            .filter((e) => e.eventId)
-            .map((e) => e.eventId!);
-
-          if (eventIds.length === 0) {
+          const {
+            xml: transcriptXml,
+            shortIdToEventId,
+            firstShortId,
+            lastShortId,
+          } = renderConversationXml(entries, { conversationId });
+
+          if (shortIdToEventId.size === 0 || !firstShortId || !lastShortId) {
             throw new Error("No eventIds found in entries to compress");
           }
 
+          const prompt = buildCompressionPrompt(
+            transcriptXml,
+            firstShortId,
+            lastShortId,
+            existingSegments
+          );
+
           // Call LLM to compress
           const result = await this.effectiveLlmService.generateObject(
             [
               {
                 role: "user",
-                content: `You are compressing conversation history. Analyze the following messages and create 1-3 compressed segments that preserve key information while being concise.
-
-For each segment, provide:
-- fromEventId: starting message event ID
-- toEventId: ending message event ID
-- compressed: a concise summary (2-4 sentences) of the key points
-
-Messages to compress:
-${formattedEntries}
-
-Event IDs in order: ${eventIds.join(", ")}
-
-Create segments that group related topics together. Preserve important decisions, errors, and outcomes.`,
+                content: prompt,
               },
             ],
             CompressionSegmentsSchema
@@ -343,13 +390,17 @@ Create segments that group related topics together. Preserve important decisions
 
           // Convert LLM output to CompressionSegment format
           const segments: CompressionSegment[] = result.object.map(
-            (seg: CompressionSegmentInput) => ({
-              fromEventId: seg.fromEventId,
-              toEventId: seg.toEventId,
-              compressed: seg.compressed,
-              createdAt: Date.now(),
-              model: this.effectiveLlmService.model,
-            })
+            (seg: CompressionSegmentInput) => {
+              const fromId = seg.fromEventId.trim();
+              const toId = seg.toEventId.trim();
+              return {
+                fromEventId: shortIdToEventId.get(fromId) ?? fromId,
+                toEventId: shortIdToEventId.get(toId) ?? toId,
+                compressed: seg.compressed,
+                createdAt: Date.now(),
+                model: this.effectiveLlmService.model,
+              };
+            }
           );
 
           span.setAttribute("segments.count", segments.length);
@@ -375,19 +426,27 @@ Create segments that group related topics together. Preserve important decisions
     conversationId: string,
     entries: ConversationEntry[],
     windowSize: number,
-    span: Span
+    span: Span,
+    tokenBudget: number
   ): Promise<void> {
+    const tokenAwareWindowSize = computeTokenAwareWindowSize(entries, tokenBudget);
+    const effectiveWindowSize = Math.min(windowSize, tokenAwareWindowSize);
+
     span.setAttribute("fallback.used", true);
-    span.setAttribute("fallback.window_size", windowSize);
+    span.setAttribute("fallback.configured_window", windowSize);
+    span.setAttribute("fallback.token_aware_window", tokenAwareWindowSize);
+    span.setAttribute("fallback.effective_window", effectiveWindowSize);
 
     logger.warn("Compression fallback triggered - using sliding window truncation", {
       conversationId,
       entriesCount: entries.length,
-      windowSize,
+      configuredWindow: windowSize,
+      tokenAwareWindow: tokenAwareWindowSize,
+      effectiveWindow: effectiveWindowSize,
     });
 
     // Delegate to pure utility function
-    const fallbackSegment = createFallbackSegmentForEntries(entries, windowSize);
+    const fallbackSegment = createFallbackSegmentForEntries(entries, effectiveWindowSize);
 
     if (!fallbackSegment) {
       // Can't create a valid segment (too few entries or insufficient event IDs)