@tenex-chat/backend 0.9.1

package/src/services/rag/rag-utils.ts

@@ -0,0 +1,174 @@
+import { logger } from "@/utils/logger";
+
+/**
+ * JSON-serializable primitive value.
+ */
+export type JsonPrimitive = string | number | boolean | null;
+
+/**
+ * JSON-serializable value (recursive).
+ */
+export type JsonValue = JsonPrimitive | JsonObject | JsonArray;
+
+/**
+ * JSON-serializable object.
+ */
+export interface JsonObject {
+    [key: string]: JsonValue;
+}
+
+/**
+ * JSON-serializable array.
+ */
+export type JsonArray = JsonValue[];
+
+/**
+ * Document metadata with known common fields and extensibility.
+ */
+export interface DocumentMetadata {
+    language?: string;
+    type?: string;
+    category?: string;
+    tags?: string[];
+    author?: string;
+    title?: string;
+    uri?: string;
+    [key: string]: JsonValue | undefined;
+}
+
+/**
+ * LanceDB stored document structure (what gets stored in the DB).
+ */
+export interface LanceDBStoredDocument {
+    id: string;
+    content: string;
+    vector: number[];
+    metadata: string;
+    timestamp: number;
+    source: string;
+}
+
+/**
+ * LanceDB query result structure (what comes back from queries).
+ */
+export interface LanceDBResult {
+    id: string | undefined;
+    content: string | undefined;
+    metadata?: string | Record<string, unknown>;
+    timestamp: number | undefined;
+    source: string | undefined;
+    vector?: number[];
+    _distance?: number;
+}
+
+/**
+ * RAG-specific document interface for mapping.
+ */
+export interface MappedRAGDocument {
+    id: string;
+    content: string;
+    metadata: DocumentMetadata;
+    timestamp: number;
+    source: string;
+}
+
+/**
+ * Type guard to validate JSON object structure.
+ */
+function isJsonObject(value: unknown): value is JsonObject {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Type guard to validate DocumentMetadata structure.
+ */
+export function isDocumentMetadata(value: unknown): value is DocumentMetadata {
+    if (!isJsonObject(value)) return false;
+
+    // Validate optional known fields if present
+    if (value.language !== undefined && typeof value.language !== "string") return false;
+    if (value.type !== undefined && typeof value.type !== "string") return false;
+    if (value.category !== undefined && typeof value.category !== "string") return false;
+    if (value.author !== undefined && typeof value.author !== "string") return false;
+    if (value.title !== undefined && typeof value.title !== "string") return false;
+    if (value.uri !== undefined && typeof value.uri !== "string") return false;
+
+    // Deep validation for tags array - ensure all elements are strings
+    if (value.tags !== undefined) {
+        if (!Array.isArray(value.tags)) return false;
+        if (!value.tags.every((tag): tag is string => typeof tag === "string")) return false;
+    }
+
+    return true;
+}
+
+/**
+ * Parse document metadata from JSON string or object with type validation.
+ */
+export function parseDocumentMetadata(
+    metadata: string | DocumentMetadata | Record<string, unknown> | undefined
+): DocumentMetadata {
+    if (!metadata) return {};
+
+    if (typeof metadata === "string") {
+        try {
+            const parsed = JSON.parse(metadata);
+            if (!isDocumentMetadata(parsed)) {
+                logger.warn("Parsed metadata does not match DocumentMetadata schema", { parsed });
+                return {};
+            }
+            return parsed;
+        } catch (error) {
+            logger.warn("Failed to parse document metadata", { error, metadata });
+            return {};
+        }
+    }
+
+    if (!isDocumentMetadata(metadata)) {
+        logger.warn("Metadata object does not match DocumentMetadata schema", { metadata });
+        return {};
+    }
+
+    return metadata;
+}
+
+/**
+ * Map LanceDB query result to RAG document format.
+ * Handles metadata parsing and field extraction.
+ */
+export function mapLanceResultToDocument(result: LanceDBResult): MappedRAGDocument {
+    return {
+        id: result.id ?? "",
+        content: result.content ?? "",
+        metadata: parseDocumentMetadata(result.metadata),
+        timestamp: result.timestamp ?? Date.now(),
+        source: result.source ?? "unknown",
+    };
+}
+
+/**
+ * Calculate relevance score from vector distance.
+ * Converts L2 (Euclidean) distance to similarity score (0-1 range).
+ *
+ * METRIC ASSUMPTION: This formula assumes L2 (Euclidean) distance metric.
+ * If LanceDB is configured to use a different metric (e.g., cosine similarity),
+ * this conversion would need adjustment:
+ *   - L2 distance: Use 1/(1+d) as implemented here
+ *   - Cosine similarity: Already in [0,1] or [-1,1], may need different mapping
+ *   - Cosine distance: Use 1-d for distances in [0,2] range
+ *
+ * OpenAI's text-embedding-3-large returns L2 distances typically in
+ * the ~1.2-1.8 range, NOT normalized [0,1]. The formula 1/(1+distance)
+ * properly handles this:
+ *   - distance = 0   → similarity = 1.0
+ *   - distance = 1   → similarity = 0.5
+ *   - distance = 1.5 → similarity = 0.4
+ *   - distance = 2   → similarity = 0.33
+ */
+export function calculateRelevanceScore(distance: number | undefined): number {
+    if (distance === undefined || distance === null) return 0;
+    if (!Number.isFinite(distance)) return 0;
+    if (distance < 0) return 0;
+    // Convert L2 distance to similarity: closer distance = higher similarity
+    return 1 / (1 + distance);
+}

package/src/services/ral/PendingDelegationsRegistry.ts

@@ -0,0 +1,168 @@
+/**
+ * PendingDelegationsRegistry - Tracks event references for tag correlation
+ *
+ * This registry stores event references at creation time (in AgentPublisher/tools)
+ * and retrieves them at tool completion time (in ToolExecutionTracker).
+ *
+ * This solves the problem of MCP result transformation stripping reference info:
+ * - Registration happens at the source before any transformation
+ * - Consumption happens in ToolExecutionTracker.completeExecution
+ * - Works identically for both MCP and non-MCP code paths
+ *
+ * Supports:
+ * - Delegation event IDs (q-tags) - for ask/delegate tools
+ * - Addressable events (a-tags) - for report_write tool
+ *
+ * Key: (agentPubkey, conversationId)
+ */
+
+import { logger } from "@/utils/logger";
+
+type RegistryKey = `${string}:${string}`; // agentPubkey:conversationId
+
+interface PendingReferences {
+    delegations: string[];
+    addressableEvents: string[];
+}
+
+class PendingDelegationsRegistryImpl {
+    private pending = new Map<RegistryKey, PendingReferences>();
+
+    private makeKey(agentPubkey: string, conversationId: string): RegistryKey {
+        return `${agentPubkey}:${conversationId}`;
+    }
+
+    private getOrCreate(key: RegistryKey): PendingReferences {
+        let refs = this.pending.get(key);
+        if (!refs) {
+            refs = { delegations: [], addressableEvents: [] };
+            this.pending.set(key, refs);
+        }
+        return refs;
+    }
+
+    /**
+     * Register a delegation event ID (for q-tags).
+     * Called by AgentPublisher when creating ask/delegate events.
+     */
+    register(agentPubkey: string, conversationId: string, delegationEventId: string): void {
+        const key = this.makeKey(agentPubkey, conversationId);
+        const refs = this.getOrCreate(key);
+        refs.delegations.push(delegationEventId);
+
+        logger.debug("[PendingDelegationsRegistry] Registered delegation", {
+            agentPubkey: agentPubkey.substring(0, 8),
+            conversationId: conversationId.substring(0, 8),
+            delegationEventId: delegationEventId.substring(0, 8),
+            totalPending: refs.delegations.length,
+        });
+    }
+
+    /**
+     * Register an addressable event reference (for a-tags).
+     * Called by tools that create addressable events (e.g., report_write).
+     * Format: "30023:pubkey:d-tag"
+     */
+    registerAddressable(agentPubkey: string, conversationId: string, addressableRef: string): void {
+        const key = this.makeKey(agentPubkey, conversationId);
+        const refs = this.getOrCreate(key);
+        refs.addressableEvents.push(addressableRef);
+
+        logger.debug("[PendingDelegationsRegistry] Registered addressable event", {
+            agentPubkey: agentPubkey.substring(0, 8),
+            conversationId: conversationId.substring(0, 8),
+            addressableRef: addressableRef.substring(0, 30),
+            totalPending: refs.addressableEvents.length,
+        });
+    }
+
+    /**
+     * Consume (retrieve and clear) all pending delegation event IDs.
+     * Called by ToolExecutionTracker when completing a delegation tool.
+     */
+    consume(agentPubkey: string, conversationId: string): string[] {
+        const key = this.makeKey(agentPubkey, conversationId);
+        const refs = this.pending.get(key);
+        if (!refs) return [];
+
+        const delegations = refs.delegations;
+        refs.delegations = [];
+
+        // Clean up if both arrays are empty
+        if (refs.addressableEvents.length === 0) {
+            this.pending.delete(key);
+        }
+
+        if (delegations.length > 0) {
+            logger.debug("[PendingDelegationsRegistry] Consumed delegations", {
+                agentPubkey: agentPubkey.substring(0, 8),
+                conversationId: conversationId.substring(0, 8),
+                count: delegations.length,
+                eventIds: delegations.map((id) => id.substring(0, 8)),
+            });
+        }
+
+        return delegations;
+    }
+
+    /**
+     * Consume (retrieve and clear) all pending addressable event references.
+     * Called by ToolExecutionTracker when completing an addressable event tool.
+     */
+    consumeAddressable(agentPubkey: string, conversationId: string): string[] {
+        const key = this.makeKey(agentPubkey, conversationId);
+        const refs = this.pending.get(key);
+        if (!refs) return [];
+
+        const addressableEvents = refs.addressableEvents;
+        refs.addressableEvents = [];
+
+        // Clean up if both arrays are empty
+        if (refs.delegations.length === 0) {
+            this.pending.delete(key);
+        }
+
+        if (addressableEvents.length > 0) {
+            logger.debug("[PendingDelegationsRegistry] Consumed addressable events", {
+                agentPubkey: agentPubkey.substring(0, 8),
+                conversationId: conversationId.substring(0, 8),
+                count: addressableEvents.length,
+            });
+        }
+
+        return addressableEvents;
+    }
+
+    /**
+     * Peek at pending delegations without consuming them.
+     */
+    peek(agentPubkey: string, conversationId: string): string[] {
+        const key = this.makeKey(agentPubkey, conversationId);
+        return [...(this.pending.get(key)?.delegations || [])];
+    }
+
+    /**
+     * Peek at pending addressable events without consuming them.
+     */
+    peekAddressable(agentPubkey: string, conversationId: string): string[] {
+        const key = this.makeKey(agentPubkey, conversationId);
+        return [...(this.pending.get(key)?.addressableEvents || [])];
+    }
+
+    /**
+     * Clear all pending references.
+     */
+    clear(): void {
+        this.pending.clear();
+    }
+
+    /**
+     * Get the number of conversations with pending references.
+     */
+    get size(): number {
+        return this.pending.size;
+    }
+}
+
+// Export singleton instance
+export const PendingDelegationsRegistry = new PendingDelegationsRegistryImpl();