@tenex-chat/backend 0.9.1

package/src/conversations/search/index.ts

@@ -0,0 +1,49 @@
+/**
+ * Conversation Search Module
+ *
+ * Provides full-text, keyword, and semantic search capabilities for conversations.
+ */
+
+// Types
+export type {
+    AdvancedSearchResult,
+    ConversationIndex,
+    ConversationIndexEntry,
+    IndexMetadata,
+    MessageIndexEntry,
+    MessageMatch,
+    SearchFilters,
+    SearchQuery,
+    SearchResult,
+} from "./types";
+
+// Query Parser
+export {
+    getEffectiveSinceTimestamp,
+    parseQuery,
+    parseTimestamp,
+    type RawSearchInput,
+} from "./QueryParser";
+
+// Snippet Extractor
+export {
+    extractAllSnippets,
+    extractSnippet,
+    type SnippetResult,
+} from "./SnippetExtractor";
+
+// Search Engine
+export {
+    search,
+    searchByTitleOnly,
+} from "./SearchEngine";
+
+// Index Manager
+export {
+    clearIndexManagerInstances,
+    ConversationIndexManager,
+    getIndexManager,
+} from "./ConversationIndexManager";
+
+// Embeddings and Semantic Search
+export * from "./embeddings";

package/src/conversations/search/types.ts

@@ -0,0 +1,124 @@
+/**
+ * Type definitions for the conversation search module.
+ */
+
+/**
+ * Search filters for narrowing down results.
+ * Note: 'after' in input is a pure alias for 'since' - only 'since' is stored after parsing.
+ */
+export interface SearchFilters {
+    /** Filter by agent slugs or pubkeys - conversation must include at least one */
+    agents?: string[];
+    /** Return conversations with activity since this timestamp (Unix seconds) */
+    since?: number;
+}
+
+/**
+ * Parsed and validated search query.
+ */
+export interface SearchQuery {
+    /** The text to search for in message content */
+    text: string;
+    /** Optional filters to narrow results */
+    filters: SearchFilters;
+}
+
+/**
+ * A single message match with snippet context.
+ */
+export interface MessageMatch {
+    /** The ID of the matching message */
+    messageId: string;
+    /** Snippet showing the match with 50-75 char context */
+    snippet: string;
+}
+
+/**
+ * A conversation search result.
+ */
+export interface SearchResult {
+    /** The conversation ID */
+    conversationId: string;
+    /** The conversation title */
+    title?: string;
+    /** Total number of messages in the conversation */
+    messageCount: number;
+    /** When the conversation was created (Unix timestamp in seconds) */
+    createdAt?: number;
+    /** When the last activity occurred (Unix timestamp in seconds) */
+    lastActivity?: number;
+    /** Array of matching messages with snippets */
+    matches: MessageMatch[];
+}
+
+/**
+ * Index entry for a single message.
+ */
+export interface MessageIndexEntry {
+    /** Unique message identifier (index within conversation) */
+    messageId: string;
+    /** Full message text content for searching */
+    content: string;
+    /** Message timestamp (Unix seconds) */
+    timestamp?: number;
+    /** Sender identifier (pubkey or agent slug) */
+    from?: string;
+    /** Target recipient identifier */
+    to?: string;
+}
+
+/**
+ * Index entry for a conversation.
+ */
+export interface ConversationIndexEntry {
+    /** The conversation ID */
+    conversationId: string;
+    /** Project slug/ID this conversation belongs to */
+    slug: string;
+    /** Conversation title */
+    title?: string;
+    /** Total message count */
+    messageCount: number;
+    /** Last message timestamp (Unix seconds) */
+    lastMessageAt?: number;
+    /** List of agent pubkeys that participated */
+    agents: string[];
+    /** Indexed messages */
+    messages: MessageIndexEntry[];
+}
+
+/**
+ * The full search index structure.
+ */
+export interface ConversationIndex {
+    /** Index format version for migration compatibility */
+    version: string;
+    /** When the index was last updated (ISO 8601) */
+    lastUpdated: string;
+    /** All indexed conversations */
+    conversations: ConversationIndexEntry[];
+}
+
+/**
+ * Index file metadata for staleness checks.
+ */
+export interface IndexMetadata {
+    version: string;
+    lastUpdated: string;
+    conversationCount: number;
+}
+
+/**
+ * Result of an advanced search operation.
+ * Includes explicit error information when applicable.
+ */
+export interface AdvancedSearchResult {
+    /** Whether the search completed successfully */
+    success: boolean;
+    /** Search results (empty array if error) */
+    results: SearchResult[];
+    /** Error message if success is false */
+    error?: string;
+    /** Whether fallback to title-only search was used */
+    usedFallback?: boolean;
+}

package/src/conversations/services/CategoryManager.ts

@@ -0,0 +1,160 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { ensureDirectory, fileExists } from "@/lib/fs";
+
+/**
+ * CategoryTally represents the stored format: category name -> usage count
+ * @example { "authentication": 15, "storage": 8, "testing": 12 }
+ */
+export type CategoryTally = Record<string, number>;
+
+/**
+ * CategoryManager - Manages conversation categories with usage tracking
+ *
+ * ## Responsibility
+ * Maintains a tally of conversation categories stored in a local JSON file.
+ * Categories track usage counts to identify frequently used categorizations.
+ *
+ * ## Storage Location
+ * Categories are stored in `data/conversation-categories.json` relative to the
+ * project root. The data directory is created automatically if it doesn't exist.
+ *
+ * ## Category Format Requirements
+ * - Lowercase (e.g., "authentication" not "Authentication")
+ * - Singular nouns (e.g., "authentication" not "auth")
+ * - No special characters or spaces (e.g., "error-handling" not "error handling")
+ *
+ * ## Thread Safety
+ * All file operations use async/await to ensure proper sequencing.
+ * The class maintains no in-memory cache to avoid stale data issues.
+ *
+ * @example
+ * const manager = new CategoryManager('/path/to/project');
+ * await manager.initialize();
+ *
+ * // Get existing categories
+ * const categories = await manager.getCategories();
+ * // Returns: ["authentication", "storage", "testing"]
+ *
+ * // Update with new categories
+ * await manager.updateCategories(["authentication", "database"]);
+ * // Increments "authentication" count, adds "database" with count 1
+ */
+export class CategoryManager {
+    private dataDir: string;
+    private categoriesFilePath: string;
+
+    /**
+     * Creates a new CategoryManager instance
+     * @param projectRoot - The root directory of the project
+     */
+    constructor(projectRoot: string) {
+        this.dataDir = path.join(projectRoot, "data");
+        this.categoriesFilePath = path.join(this.dataDir, "conversation-categories.json");
+    }
+
+    /**
+     * Initialize the CategoryManager by ensuring the data directory exists
+     */
+    async initialize(): Promise<void> {
+        await ensureDirectory(this.dataDir);
+    }
+
+    /**
+     * Load the category tally from disk
+     * @returns The current category tally, or empty object if file doesn't exist
+     */
+    private async loadTally(): Promise<CategoryTally> {
+        try {
+            if (!(await fileExists(this.categoriesFilePath))) {
+                return {};
+            }
+            const content = await fs.readFile(this.categoriesFilePath, "utf-8");
+            return JSON.parse(content) as CategoryTally;
+        } catch (error) {
+            // If file is corrupted or unreadable, start fresh
+            console.error("Failed to load category tally, starting fresh:", error);
+            return {};
+        }
+    }
+
+    /**
+     * Save the category tally to disk
+     * @param tally - The category tally to save
+     */
+    private async saveTally(tally: CategoryTally): Promise<void> {
+        await ensureDirectory(this.dataDir);
+        await fs.writeFile(this.categoriesFilePath, JSON.stringify(tally, null, 2), "utf-8");
+    }
+
+    /**
+     * Get all existing categories as an array
+     * Categories are sorted by usage count (highest first)
+     * @returns Array of category names
+     */
+    async getCategories(): Promise<string[]> {
+        const tally = await this.loadTally();
+        // Sort by usage count (descending) to prioritize frequently used categories
+        return Object.entries(tally)
+            .sort(([, countA], [, countB]) => countB - countA)
+            .map(([category]) => category);
+    }
+
+    /**
+     * Get the full category tally including usage counts
+     * @returns The category tally object
+     */
+    async getCategoryTally(): Promise<CategoryTally> {
+        return this.loadTally();
+    }
+
+    /**
+     * Update categories by adding new ones and incrementing counts for existing ones
+     *
+     * @param newCategories - Array of category names to add/increment
+     *
+     * @example
+     * // If tally is { "authentication": 5, "storage": 2 }
+     * await manager.updateCategories(["authentication", "database"]);
+     * // Tally becomes { "authentication": 6, "storage": 2, "database": 1 }
+     */
+    async updateCategories(newCategories: string[]): Promise<void> {
+        if (newCategories.length === 0) {
+            return;
+        }
+
+        const tally = await this.loadTally();
+
+        for (const category of newCategories) {
+            // Normalize: lowercase, trim whitespace
+            const normalizedCategory = category.toLowerCase().trim();
+
+            if (!normalizedCategory) {
+                continue;
+            }
+
+            // Increment existing or initialize to 1
+            tally[normalizedCategory] = (tally[normalizedCategory] || 0) + 1;
+        }
+
+        await this.saveTally(tally);
+    }
+
+    /**
+     * Remove a category from the tally
+     * @param category - The category to remove
+     * @returns true if the category was removed, false if it didn't exist
+     */
+    async removeCategory(category: string): Promise<boolean> {
+        const tally = await this.loadTally();
+        const normalizedCategory = category.toLowerCase().trim();
+
+        if (!(normalizedCategory in tally)) {
+            return false;
+        }
+
+        delete tally[normalizedCategory];
+        await this.saveTally(tally);
+        return true;
+    }
+}

package/src/conversations/services/ConversationResolver.ts

@@ -0,0 +1,296 @@
+import { ConversationStore } from "../ConversationStore";
+import type { ConversationMetadata } from "../types";
+import { AgentEventDecoder } from "@/nostr/AgentEventDecoder";
+import { shortenConversationId } from "@/utils/conversation-id";
+import { getNDK } from "@/nostr/ndkClient";
+import { getProjectContext } from "@/services/projects";
+import { logger } from "@/utils/logger";
+import { buildDelegationChain } from "@/utils/delegation-chain";
+import type { NDKEvent } from "@nostr-dev-kit/ndk";
+import { NDKArticle } from "@nostr-dev-kit/ndk";
+import { trace } from "@opentelemetry/api";
+import chalk from "chalk";
+import { TagExtractor } from "@/nostr/TagExtractor";
+import { formatAnyError } from "@/lib/error-formatter";
+
+/**
+ * Fetch a kind 30023 (NDKArticle) from an a-tag reference.
+ * @param aTagValue - The a-tag value in format "30023:pubkey:d-tag"
+ * @returns The article metadata or null if not found
+ */
+async function fetchReferencedArticle(
+    aTagValue: string
+): Promise<ConversationMetadata["referencedArticle"] | null> {
+    try {
+        const parts = aTagValue.split(":");
+        if (parts.length < 3 || parts[0] !== "30023") {
+            return null;
+        }
+
+        const [, pubkey, ...dTagParts] = parts;
+        const dTag = dTagParts.join(":"); // Handle d-tags that contain colons
+
+        const ndk = getNDK();
+        const filter = {
+            kinds: [30023],
+            authors: [pubkey],
+            "#d": [dTag],
+        };
+
+        const events = await ndk.fetchEvents(filter);
+        if (events.size === 0) {
+            logger.debug(chalk.yellow(`Referenced article not found: ${aTagValue}`));
+            return null;
+        }
+
+        const event = Array.from(events)[0];
+        const article = NDKArticle.from(event);
+
+        logger.info(chalk.cyan(`📄 Fetched referenced article: "${article.title || dTag}"`));
+
+        return {
+            title: article.title || dTag,
+            content: article.content || "",
+            dTag,
+        };
+    } catch (error) {
+        logger.debug(chalk.yellow(`Failed to fetch referenced article: ${formatAnyError(error)}`));
+        return null;
+    }
+}
+
+/**
+ * Extract and fetch the first kind 30023 article reference from an event's a-tags.
+ * @param event - The event to extract article references from
+ * @returns The article metadata or null if none found
+ */
+async function extractReferencedArticle(
+    event: NDKEvent
+): Promise<ConversationMetadata["referencedArticle"] | null> {
+    const aTags = TagExtractor.getATags(event);
+
+    // Find the first a-tag referencing a kind 30023 (article)
+    const articleATag = aTags.find((tag) => tag.startsWith("30023:"));
+    if (!articleATag) {
+        return null;
+    }
+
+    return fetchReferencedArticle(articleATag);
+}
+
+export interface ConversationResolutionResult {
+    conversation: ConversationStore | undefined;
+    isNew?: boolean;
+}
+
+/**
+ * ConversationResolver encapsulates all logic for finding or creating conversations
+ * based on incoming Nostr events.
+ */
+export class ConversationResolver {
+
+    /**
+     * Resolve the conversation for an incoming event.
+     */
+    async resolveConversationForEvent(event: NDKEvent): Promise<ConversationResolutionResult> {
+        const activeSpan = trace.getActiveSpan();
+        const replyTarget = AgentEventDecoder.getReplyTarget(event);
+
+        // If event has an 'e' tag (reply), try to find existing conversation
+        if (replyTarget) {
+            const conversation = ConversationStore.findByEventId(replyTarget);
+            if (conversation) {
+                activeSpan?.addEvent("conversation.resolved", {
+                    "resolution.type": "found_existing",
+                    "conversation.id": shortenConversationId(conversation.id),
+                    "conversation.message_count": conversation.getAllMessages().length,
+                });
+                return { conversation };
+            }
+
+            // Has e tag but conversation not found - try orphaned reply handling
+            const mentionedPubkeys = AgentEventDecoder.getMentionedPubkeys(event);
+            const newConversation = await this.handleOrphanedReply(event, replyTarget, mentionedPubkeys);
+            if (newConversation) {
+                activeSpan?.addEvent("conversation.resolved", {
+                    "resolution.type": "created_from_orphan",
+                    "conversation.id": shortenConversationId(newConversation.id),
+                    "conversation.message_count": newConversation.getAllMessages().length,
+                });
+                return { conversation: newConversation, isNew: true };
+            }
+
+            activeSpan?.addEvent("conversation.resolution_failed", {
+                reason: "conversation_not_found_for_reply_target",
+                "reply_target.id": replyTarget,
+            });
+            return { conversation: undefined };
+        }
+
+        // No 'e' tag - this is a NEW conversation, create it
+        const mentionedPubkeys = AgentEventDecoder.getMentionedPubkeys(event);
+        const projectCtx = getProjectContext();
+        const isDirectedToAgent = mentionedPubkeys.some((pubkey) =>
+            Array.from(projectCtx.agents.values()).some((a) => a.pubkey === pubkey)
+        );
+
+        if (!isDirectedToAgent) {
+            activeSpan?.addEvent("conversation.resolution_failed", {
+                reason: "not_directed_to_agent",
+            });
+            return { conversation: undefined };
+        }
+
+        const conversation = await ConversationStore.create(event);
+        if (conversation) {
+            // Check for referenced kind 30023 articles and populate metadata
+            const referencedArticle = await extractReferencedArticle(event);
+            if (referencedArticle) {
+                conversation.updateMetadata({ referencedArticle });
+                await conversation.save();
+
+                activeSpan?.addEvent("referenced_article_loaded", {
+                    "article.title": referencedArticle.title,
+                    "article.dTag": referencedArticle.dTag,
+                    "article.content_length": referencedArticle.content.length,
+                });
+            }
+
+            // Build and store delegation chain if this is a delegated conversation
+            // For delegation events, the first p-tag is always the intended recipient.
+            // getMentionedPubkeys returns p-tags in order, so the first agent pubkey
+            // found is the correct target.
+            const targetAgentPubkey = mentionedPubkeys.find((pubkey) =>
+                Array.from(projectCtx.agents.values()).some((a) => a.pubkey === pubkey)
+            );
+
+            if (targetAgentPubkey) {
+                const delegationChain = buildDelegationChain(
+                    event,
+                    targetAgentPubkey,
+                    projectCtx.project.pubkey, // Project owner is the human user
+                    conversation.id // The conversation being created for the current agent
+                );
+
+                if (delegationChain && delegationChain.length > 0) {
+                    conversation.updateMetadata({ delegationChain });
+                    await conversation.save();
+
+                    activeSpan?.addEvent("delegation_chain_built", {
+                        "chain.length": delegationChain.length,
+                        "chain.display": delegationChain.map(c => c.displayName).join(" → "),
+                    });
+
+                    logger.debug("[ConversationResolver] Built delegation chain for new conversation", {
+                        conversationId: conversation.id.substring(0, 8),
+                        chainLength: delegationChain.length,
+                        chain: delegationChain.map(c => c.displayName).join(" → "),
+                    });
+                }
+            }
+
+            logger.info(chalk.green(`Created new conversation ${conversation.id.substring(0, 8)} from kind:1 event`));
+            activeSpan?.addEvent("conversation.resolved", {
+                "resolution.type": "created_new",
+                "conversation.id": shortenConversationId(conversation.id),
+            });
+            return { conversation, isNew: true };
+        }
+
+        activeSpan?.addEvent("conversation.resolution_failed", {
+            reason: "failed_to_create_conversation",
+        });
+        return { conversation: undefined };
+    }
+
+    /**
+     * Handle orphaned replies by fetching the thread from the network
+     */
+    private async handleOrphanedReply(
+        event: NDKEvent,
+        replyTargetId: string,
+        mentionedPubkeys: string[]
+    ): Promise<ConversationStore | undefined> {
+        if (mentionedPubkeys.length === 0) {
+            return undefined;
+        }
+
+        const projectCtx = getProjectContext();
+        const isDirectedToAgent = mentionedPubkeys.some((pubkey) =>
+            Array.from(projectCtx.agents.values()).some((a) => a.pubkey === pubkey)
+        );
+
+        if (!isDirectedToAgent) {
+            return undefined;
+        }
+
+        logger.info(
+            chalk.yellow(
+                `Fetching conversation thread for orphaned reply, target: ${replyTargetId.substring(0, 8)}`
+            )
+        );
+
+        const activeSpan = trace.getActiveSpan();
+        activeSpan?.addEvent("conversation.fetching_orphaned_thread", {
+            reply_target_id: replyTargetId,
+        });
+
+        const ndk = getNDK();
+
+        // Fetch the reply target event and any events that also reply to it
+        const events = await ndk.fetchEvents([
+            { ids: [replyTargetId] },
+            { "#e": [replyTargetId] },
+        ]);
+
+        const eventsArray = Array.from(events);
+        const rootEvent = eventsArray.find((e) => e.id === replyTargetId);
+
+        if (!rootEvent) {
+            logger.warn(chalk.yellow(`Could not fetch target event ${replyTargetId.substring(0, 8)} from network`));
+            activeSpan?.addEvent("conversation.fetch_failed", {
+                reason: "target_event_not_found",
+                reply_target_id: replyTargetId,
+            });
+            return undefined;
+        }
+
+        const replies = eventsArray.filter((e) => e.id !== replyTargetId);
+
+        logger.info(chalk.green(`Fetched target event and ${replies.length} replies`));
+        activeSpan?.addEvent("conversation.thread_fetched", {
+            "fetched.reply_count": replies.length,
+            "fetched.total_events": eventsArray.length,
+        });
+
+        const conversation = await ConversationStore.create(rootEvent);
+        if (!conversation) {
+            return undefined;
+        }
+
+        // Check for referenced kind 30023 articles in the root event and populate metadata
+        const referencedArticle = await extractReferencedArticle(rootEvent);
+        if (referencedArticle) {
+            conversation.updateMetadata({ referencedArticle });
+            await conversation.save();
+
+            activeSpan?.addEvent("referenced_article_loaded", {
+                "article.title": referencedArticle.title,
+                "article.dTag": referencedArticle.dTag,
+                "article.content_length": referencedArticle.content.length,
+            });
+        }
+
+        replies.sort((a, b) => (a.created_at || 0) - (b.created_at || 0));
+
+        for (const reply of replies) {
+            await ConversationStore.addEvent(conversation.id, reply);
+        }
+
+        if (event.id !== rootEvent.id && !replies.some((r) => r.id === event.id)) {
+            await ConversationStore.addEvent(conversation.id, event);
+        }
+
+        return conversation;
+    }
+}