@gethmy/mcp 1.0.0 → 2.1.0

package/dist/lib/context-assembly.js

@@ -0,0 +1,606 @@
+/**
+ * Context Assembly Engine
+ *
+ * Token-budget-aware context constructor that assembles relevant memories
+ * for a given task, producing a manifest of what was included/excluded.
+ */
+import { checkPromotion } from "@harmony/memory";
+// Constants
+const DEFAULT_TOKEN_BUDGET = 4000;
+const MAX_TOKENS_PER_ENTITY = 500;
+const MIN_RELEVANCE_THRESHOLD = 0.1;
+// Tier weight multipliers for relevance scoring
+const TIER_WEIGHTS = {
+    reference: 1.0,
+    episode: 0.7,
+    draft: 0.4,
+};
+// Dedicated procedure budget as a fraction of total budget
+const PROCEDURE_BUDGET_FRACTION = 0.15;
+// Tier budget allocation percentages (of remaining budget after procedure reservation)
+const TIER_BUDGET_ALLOCATION = {
+    reference: 0.6,
+    episode: 0.3,
+    draft: 0.1,
+};
+// Minimum guaranteed slots per tier
+const MIN_REFERENCE_SLOTS = 3;
+/**
+ * Estimate token count (rough: 1 token per 4 chars)
+ */
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Generate a unique assembly ID
+ */
+function generateAssemblyId() {
+    return `ctx_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+/**
+ * Truncate entity content to fit within token limit.
+ * Keeps first paragraph + bullet points if present.
+ */
+function truncateContent(content, maxTokens) {
+    const currentTokens = estimateTokens(content);
+    if (currentTokens <= maxTokens) {
+        return { text: content, truncated: false };
+    }
+    // Try to keep first paragraph
+    const paragraphs = content.split(/\n\n+/);
+    let result = paragraphs[0];
+    // Add bullet points from subsequent paragraphs if they fit
+    for (let i = 1; i < paragraphs.length; i++) {
+        const lines = paragraphs[i]
+            .split("\n")
+            .filter((l) => l.startsWith("- ") || l.startsWith("* "));
+        if (lines.length > 0) {
+            const bulletSection = lines.join("\n");
+            if (estimateTokens(result + "\n\n" + bulletSection) <= maxTokens) {
+                result += "\n\n" + bulletSection;
+            }
+        }
+    }
+    // Hard truncate if still too long
+    if (estimateTokens(result) > maxTokens) {
+        const maxChars = maxTokens * 4;
+        result = result.slice(0, maxChars - 3) + "...";
+    }
+    return { text: result, truncated: true };
+}
+/**
+ * Compute relevance score for an entity against task context.
+ */
+export function computeRelevanceScore(entity, taskContext, cardLabels) {
+    const reasons = [];
+    let score = 0;
+    // 0. DB hybrid search signal (RRF score from FTS + vector fusion)
+    //    Scaled to 0-0.3 contribution; when present, reduces reliance on word-overlap
+    const hasRrfScore = entity.rrf_score !== undefined && entity.rrf_score > 0;
+    if (hasRrfScore) {
+        // RRF scores are typically 0-0.04; normalize to 0-1 range then scale
+        const normalizedRrf = Math.min(entity.rrf_score / 0.04, 1.0);
+        const rrfContribution = normalizedRrf * 0.3;
+        score += rrfContribution;
+        reasons.push(`hybrid_search(rrf=${entity.rrf_score.toFixed(4)})`);
+    }
+    // 1. Text match: simple word overlap scoring (reduced weight when RRF available)
+    const textMatchWeight = hasRrfScore ? 0.15 : 0.4;
+    const taskWords = new Set(taskContext
+        .toLowerCase()
+        .split(/\W+/)
+        .filter((w) => w.length > 2));
+    const entityWords = new Set(`${entity.title} ${entity.content}`
+        .toLowerCase()
+        .split(/\W+/)
+        .filter((w) => w.length > 2));
+    const overlap = [...taskWords].filter((w) => entityWords.has(w));
+    if (overlap.length > 0) {
+        const textScore = Math.min(overlap.length / Math.max(taskWords.size, 1), 1.0) *
+            textMatchWeight;
+        score += textScore;
+        reasons.push(`text_match(${overlap.length} words)`);
+    }
+    // 2. Tag overlap with card labels
+    if (cardLabels.length > 0 && entity.tags.length > 0) {
+        const labelSet = new Set(cardLabels.map((l) => l.toLowerCase()));
+        const tagOverlap = entity.tags.filter((t) => labelSet.has(t.toLowerCase()));
+        if (tagOverlap.length > 0) {
+            const tagScore = (tagOverlap.length / cardLabels.length) * 0.3;
+            score += tagScore;
+            reasons.push(`tag_match(${tagOverlap.join(",")})`);
+        }
+    }
+    // 3. Confidence as a quality signal
+    score += entity.confidence * 0.15;
+    if (entity.confidence >= 0.9) {
+        reasons.push("high_confidence");
+    }
+    // 4. Recency: decay based on last access with tier-specific half-lives
+    if (entity.last_accessed_at) {
+        const daysSinceAccess = (Date.now() - new Date(entity.last_accessed_at).getTime()) /
+            (1000 * 60 * 60 * 24);
+        const halfLife = { draft: 7, episode: 30, reference: 180 }[entity.memory_tier];
+        const recencyScore = 0.5 ** (daysSinceAccess / halfLife) * 0.1;
+        score += recencyScore;
+        if (daysSinceAccess < 7)
+            reasons.push("recently_accessed");
+    }
+    // 5. Access frequency (log-scaled)
+    if (entity.access_count > 0) {
+        const freqScore = Math.log10(entity.access_count + 1) * 0.05;
+        score += Math.min(freqScore, 0.1);
+        if (entity.access_count >= 5)
+            reasons.push(`frequently_used(${entity.access_count})`);
+    }
+    // 6. Usefulness score from feedback loop (0-0.15 weight)
+    const usefulnessScore = entity.metadata?.usefulness_score ?? 0;
+    if (usefulnessScore >= 3) {
+        const usefulnessBoost = Math.min(usefulnessScore / 20, 0.15);
+        score += usefulnessBoost;
+        reasons.push(`useful(${usefulnessScore})`);
+    }
+    else if (usefulnessScore === 0 && entity.access_count >= 5) {
+        // Accessed many times but never marked useful — slight penalty
+        score -= 0.02;
+        reasons.push("low_usefulness");
+    }
+    // Procedure boost: actionable step-by-step instructions are highly valuable
+    if (entity.type === "procedure") {
+        score += 0.1;
+        reasons.push("procedure_boost");
+    }
+    // Clamp raw score to 0-1 range before applying tier weight
+    score = Math.min(score, 1.0);
+    // Apply tier weight
+    const tierWeight = TIER_WEIGHTS[entity.memory_tier];
+    score *= tierWeight;
+    return { score, reasons };
+}
+/**
+ * Assemble context from knowledge graph entities with token budget management.
+ */
+export async function assembleContext(options) {
+    const { workspaceId, projectId, taskContext, cardLabels = [], tokenBudget = DEFAULT_TOKEN_BUDGET, client, } = options;
+    const assemblyId = generateAssemblyId();
+    const manifest = {
+        assemblyId,
+        timestamp: new Date().toISOString(),
+        included: [],
+        excluded: [],
+        budgetUsed: 0,
+        budgetTotal: tokenBudget,
+        tierBreakdown: {
+            draft: { count: 0, tokens: 0 },
+            episode: { count: 0, tokens: 0 },
+            reference: { count: 0, tokens: 0 },
+        },
+    };
+    // Fetch candidate entities: search by task context + list by project
+    let candidates = [];
+    try {
+        // Full-text search by task context
+        const searchResult = await client.searchMemoryEntities(workspaceId, taskContext, { project_id: projectId, limit: 30 });
+        if (searchResult.entities?.length > 0) {
+            candidates = searchResult.entities.map(mapToContextEntity);
+        }
+    }
+    catch {
+        // Search failed, fall back to listing
+    }
+    // Also fetch by project scope if we have few candidates
+    if (candidates.length < 10 && projectId) {
+        try {
+            const listResult = await client.listMemoryEntities({
+                workspace_id: workspaceId,
+                project_id: projectId,
+                limit: 30,
+            });
+            if (listResult.entities?.length > 0) {
+                const existingIds = new Set(candidates.map((c) => c.id));
+                const additional = listResult.entities
+                    .map(mapToContextEntity)
+                    .filter((e) => !existingIds.has(e.id));
+                candidates.push(...additional);
+            }
+        }
+        catch {
+            // List failed, continue with what we have
+        }
+    }
+    if (candidates.length === 0) {
+        return {
+            context: "",
+            manifest,
+            memories: [],
+        };
+    }
+    // Score all candidates
+    const scored = candidates.map((entity) => {
+        const { score, reasons } = computeRelevanceScore(entity, taskContext, cardLabels);
+        return { entity, score, reasons };
+    });
+    // Sort by score descending
+    scored.sort((a, b) => b.score - a.score);
+    // Reserve dedicated procedure budget, allocate remaining to tiers
+    const procedureBudget = Math.floor(tokenBudget * PROCEDURE_BUDGET_FRACTION);
+    const remainingBudget = tokenBudget - procedureBudget;
+    const tierBudgets = {
+        reference: Math.floor(remainingBudget * TIER_BUDGET_ALLOCATION.reference),
+        episode: Math.floor(remainingBudget * TIER_BUDGET_ALLOCATION.episode),
+        draft: Math.floor(remainingBudget * TIER_BUDGET_ALLOCATION.draft),
+    };
+    const tierUsed = {
+        reference: 0,
+        episode: 0,
+        draft: 0,
+    };
+    let procedureUsed = 0;
+    const included = [];
+    let totalUsed = 0;
+    // First pass: guarantee minimum reference slots
+    let referenceCount = 0;
+    for (const item of scored) {
+        if (item.entity.memory_tier === "reference" &&
+            item.entity.type !== "procedure" &&
+            referenceCount < MIN_REFERENCE_SLOTS) {
+            const { text, truncated } = truncateContent(item.entity.content, MAX_TOKENS_PER_ENTITY);
+            const tokens = estimateTokens(`### ${item.entity.title}\n${text}`);
+            if (totalUsed + tokens <= tokenBudget) {
+                included.push({ ...item, tokens, truncated });
+                item.entity.content = text;
+                totalUsed += tokens;
+                tierUsed.reference += tokens;
+                referenceCount++;
+            }
+        }
+    }
+    // Second pass: include procedure entities with dedicated budget
+    const includedIds = new Set(included.map((i) => i.entity.id));
+    const procedureCandidates = scored.filter((item) => item.entity.type === "procedure" && !includedIds.has(item.entity.id));
+    for (const item of procedureCandidates) {
+        if (item.score < MIN_RELEVANCE_THRESHOLD) {
+            manifest.excluded.push({
+                entityId: item.entity.id,
+                title: item.entity.title,
+                type: item.entity.type,
+                tier: item.entity.memory_tier,
+                relevanceScore: item.score,
+                reason: "below_relevance_threshold",
+            });
+            continue;
+        }
+        const { text, truncated } = truncateContent(item.entity.content, MAX_TOKENS_PER_ENTITY);
+        const tokens = estimateTokens(`### ${item.entity.title}\n${text}`);
+        // Check dedicated procedure budget, allow overflow to total remaining
+        if (procedureUsed + tokens > procedureBudget) {
+            const totalRemaining = tokenBudget - totalUsed;
+            if (tokens > totalRemaining) {
+                manifest.excluded.push({
+                    entityId: item.entity.id,
+                    title: item.entity.title,
+                    type: item.entity.type,
+                    tier: item.entity.memory_tier,
+                    relevanceScore: item.score,
+                    reason: "procedure_budget_exceeded",
+                });
+                continue;
+            }
+        }
+        if (totalUsed + tokens > tokenBudget) {
+            manifest.excluded.push({
+                entityId: item.entity.id,
+                title: item.entity.title,
+                type: item.entity.type,
+                tier: item.entity.memory_tier,
+                relevanceScore: item.score,
+                reason: "total_budget_exceeded",
+            });
+            continue;
+        }
+        included.push({ ...item, tokens, truncated });
+        item.entity.content = text;
+        totalUsed += tokens;
+        procedureUsed += tokens;
+        includedIds.add(item.entity.id);
+    }
+    // Third pass: fill remaining budget by score (non-procedure entities)
+    for (const item of scored) {
+        if (includedIds.has(item.entity.id))
+            continue;
+        if (item.entity.type === "procedure")
+            continue; // Already handled
+        if (item.score < MIN_RELEVANCE_THRESHOLD) {
+            manifest.excluded.push({
+                entityId: item.entity.id,
+                title: item.entity.title,
+                type: item.entity.type,
+                tier: item.entity.memory_tier,
+                relevanceScore: item.score,
+                reason: "below_relevance_threshold",
+            });
+            continue;
+        }
+        const tier = item.entity.memory_tier;
+        const { text, truncated } = truncateContent(item.entity.content, MAX_TOKENS_PER_ENTITY);
+        const tokens = estimateTokens(`### ${item.entity.title}\n${text}`);
+        // Check tier budget (allow overflow to unused tiers)
+        if (tierUsed[tier] + tokens > tierBudgets[tier]) {
+            // Check if there's unused budget from other tiers
+            const totalRemaining = tokenBudget - totalUsed;
+            if (tokens > totalRemaining) {
+                manifest.excluded.push({
+                    entityId: item.entity.id,
+                    title: item.entity.title,
+                    type: item.entity.type,
+                    tier,
+                    relevanceScore: item.score,
+                    reason: "budget_exceeded",
+                });
+                continue;
+            }
+        }
+        if (totalUsed + tokens > tokenBudget) {
+            manifest.excluded.push({
+                entityId: item.entity.id,
+                title: item.entity.title,
+                type: item.entity.type,
+                tier,
+                relevanceScore: item.score,
+                reason: "total_budget_exceeded",
+            });
+            continue;
+        }
+        included.push({ ...item, tokens, truncated });
+        item.entity.content = text;
+        totalUsed += tokens;
+        tierUsed[tier] += tokens;
+        includedIds.add(item.entity.id);
+    }
+    // Build manifest
+    manifest.budgetUsed = totalUsed;
+    const procedureItems = included.filter((i) => i.entity.type === "procedure");
+    manifest.tierBreakdown = {
+        reference: {
+            count: included.filter((i) => i.entity.memory_tier === "reference" && i.entity.type !== "procedure").length,
+            tokens: tierUsed.reference,
+        },
+        episode: {
+            count: included.filter((i) => i.entity.memory_tier === "episode" && i.entity.type !== "procedure").length,
+            tokens: tierUsed.episode,
+        },
+        draft: {
+            count: included.filter((i) => i.entity.memory_tier === "draft" && i.entity.type !== "procedure").length,
+            tokens: tierUsed.draft,
+        },
+    };
+    manifest.procedureBreakdown = {
+        count: procedureItems.length,
+        tokens: procedureUsed,
+        budget: procedureBudget,
+    };
+    for (const item of included) {
+        manifest.included.push({
+            entityId: item.entity.id,
+            title: item.entity.title,
+            type: item.entity.type,
+            tier: item.entity.memory_tier,
+            relevanceScore: item.score,
+            reasons: item.reasons,
+            tokenCount: item.tokens,
+            truncated: item.truncated,
+        });
+    }
+    // Build context string — procedures in their own section
+    const contextSections = [];
+    const nonProcedureItems = included.filter((i) => i.entity.type !== "procedure");
+    if (included.length > 0) {
+        // Procedure section first (actionable instructions)
+        if (procedureItems.length > 0) {
+            contextSections.push(`## Procedures (${procedureItems.length} loaded, ${procedureUsed}/${procedureBudget} tokens)`);
+            for (const item of procedureItems) {
+                const tags = item.entity.tags.length > 0
+                    ? ` [${item.entity.tags.join(", ")}]`
+                    : "";
+                const tierLabel = item.entity.memory_tier !== "reference"
+                    ? ` (${item.entity.memory_tier})`
+                    : "";
+                contextSections.push(`\n### ${item.entity.title} (confidence: ${item.entity.confidence})${tierLabel}${tags}`);
+                contextSections.push(item.entity.content);
+            }
+        }
+        // Non-procedure memories
+        if (nonProcedureItems.length > 0) {
+            contextSections.push(`\n## Relevant Memories (${nonProcedureItems.length} loaded, ${manifest.excluded.length} excluded)`);
+            contextSections.push(`*Assembly: ${assemblyId} | Budget: ${totalUsed}/${tokenBudget} tokens*`);
+            for (const item of nonProcedureItems) {
+                const tags = item.entity.tags.length > 0
+                    ? ` [${item.entity.tags.join(", ")}]`
+                    : "";
+                const tierLabel = item.entity.memory_tier !== "reference"
+                    ? ` (${item.entity.memory_tier})`
+                    : "";
+                contextSections.push(`\n### ${item.entity.title} (${item.entity.type}, confidence: ${item.entity.confidence})${tierLabel}${tags}`);
+                contextSections.push(item.entity.content);
+            }
+        }
+    }
+    // Increment access_count for included entities (fire-and-forget)
+    incrementAccessCounts(client, included.map((i) => i.entity.id)).catch(() => { });
+    // Auto-promote entities that cross access thresholds after the bump (fire-and-forget)
+    promoteEligibleEntities(client, included.map((i) => i.entity)).catch(() => { });
+    return {
+        context: contextSections.join("\n"),
+        manifest,
+        memories: included.map((i) => i.entity),
+    };
+}
+/**
+ * Map raw API entity to ContextEntity
+ */
+export function mapToContextEntity(raw) {
+    const e = raw;
+    return {
+        id: e.id,
+        type: e.type,
+        title: e.title,
+        content: e.content,
+        confidence: e.confidence ?? 1.0,
+        tags: e.tags || [],
+        memory_tier: e.memory_tier || "reference",
+        access_count: e.access_count || 0,
+        last_accessed_at: e.last_accessed_at || null,
+        created_at: e.created_at || "",
+        updated_at: e.updated_at || "",
+        metadata: e.metadata ?? undefined,
+        // Hybrid search signals (present when results come from RPC)
+        rrf_score: e.rrf_score ?? undefined,
+        fts_rank: e.fts_rank ?? undefined,
+        semantic_rank: e.semantic_rank ?? undefined,
+    };
+}
+/**
+ * Increment access counts for entities loaded into context.
+ * Uses batch_touch_knowledge_entities RPC for a single-roundtrip update.
+ * Falls back to individual touches if the batch endpoint is unavailable.
+ */
+async function incrementAccessCounts(client, entityIds) {
+    if (entityIds.length === 0)
+        return;
+    try {
+        await client.batchTouchMemoryEntities(entityIds);
+    }
+    catch {
+        // Fallback: individual touches (e.g. older server version)
+        await Promise.allSettled(entityIds.map((id) => client.touchMemoryEntity(id)));
+    }
+}
+/**
+ * Check included entities for promotion eligibility after access count bump.
+ * Uses access_count + 1 to reflect the touch that just happened.
+ */
+async function promoteEligibleEntities(client, entities) {
+    for (const entity of entities) {
+        if (entity.memory_tier === "reference")
+            continue;
+        if (!entity.created_at)
+            continue;
+        // +1 because incrementAccessCounts just bumped it
+        const promotion = checkPromotion(entity.memory_tier, entity.access_count + 1, entity.confidence, entity.created_at);
+        if (promotion.eligible && promotion.targetTier) {
+            try {
+                await client.updateMemoryEntity(entity.id, {
+                    memory_tier: promotion.targetTier,
+                    metadata: {
+                        ...(entity.metadata || {}),
+                        promoted_at: new Date().toISOString(),
+                        promotion_reason: promotion.reason,
+                        promoted_from: entity.memory_tier,
+                    },
+                });
+            }
+            catch {
+                // Non-fatal: promotion is best-effort
+            }
+        }
+    }
+}
+// In-memory manifest cache (keyed by assemblyId)
+const manifestCache = new Map();
+const MAX_CACHE_SIZE = 50;
+/**
+ * Store a manifest for later retrieval.
+ */
+export function cacheManifest(manifest) {
+    if (manifestCache.size >= MAX_CACHE_SIZE) {
+        // Remove oldest entry
+        const firstKey = manifestCache.keys().next().value;
+        if (firstKey)
+            manifestCache.delete(firstKey);
+    }
+    manifestCache.set(manifest.assemblyId, manifest);
+}
+/**
+ * Retrieve a cached manifest by assembly ID.
+ */
+export function getCachedManifest(assemblyId) {
+    return manifestCache.get(assemblyId);
+}
+// --- Feedback-Driven Scoring ---
+/** Track which assemblyId was used for which card session */
+const sessionAssemblyMap = new Map();
+const MAX_SESSION_MAP_SIZE = 100;
+/**
+ * Associate an assemblyId with a card session for later feedback.
+ * Called when context is assembled during session start or prompt generation.
+ */
+export function trackSessionAssembly(cardId, assemblyId) {
+    if (sessionAssemblyMap.size >= MAX_SESSION_MAP_SIZE) {
+        const firstKey = sessionAssemblyMap.keys().next().value;
+        if (firstKey)
+            sessionAssemblyMap.delete(firstKey);
+    }
+    sessionAssemblyMap.set(cardId, assemblyId);
+}
+/**
+ * Get the assemblyId associated with a card session.
+ */
+export function getSessionAssemblyId(cardId) {
+    return sessionAssemblyMap.get(cardId);
+}
+/**
+ * Record context feedback based on session outcome.
+ * Adjusts entity confidence based on whether the session completed successfully.
+ *
+ * - Completed successfully (status=completed, progress>=100): boost included entities
+ * - Paused/blocked: neutral or slight penalty for included entities
+ */
+export async function recordContextFeedback(client, cardId, sessionStatus, progressPercent, hadBlockers) {
+    const assemblyId = sessionAssemblyMap.get(cardId);
+    if (!assemblyId)
+        return { adjusted: 0 };
+    const manifest = manifestCache.get(assemblyId);
+    if (!manifest || manifest.included.length === 0)
+        return { adjusted: 0 };
+    let adjusted = 0;
+    const isSuccess = sessionStatus === "completed" && (progressPercent ?? 0) >= 100;
+    for (const entry of manifest.included) {
+        try {
+            if (isSuccess) {
+                // Boost confidence by +0.05 (max 1.0) and increment usefulness_score
+                const { entity } = await client.getMemoryEntity(entry.entityId);
+                const e = entity;
+                const currentUsefulness = e.metadata?.usefulness_score ?? 0;
+                const newConfidence = Math.min((e.confidence ?? 0.5) + 0.05, 1.0);
+                await client.updateMemoryEntity(entry.entityId, {
+                    confidence: newConfidence,
+                    metadata: {
+                        usefulness_score: currentUsefulness + 1,
+                        last_feedback_at: new Date().toISOString(),
+                    },
+                });
+                adjusted++;
+            }
+            else if (hadBlockers) {
+                // Slight penalty for entities included when session had blockers
+                const { entity } = await client.getMemoryEntity(entry.entityId);
+                const e = entity;
+                const newConfidence = Math.max((e.confidence ?? 0.5) - 0.02, 0.1);
+                await client.updateMemoryEntity(entry.entityId, {
+                    confidence: newConfidence,
+                    metadata: {
+                        last_feedback_at: new Date().toISOString(),
+                    },
+                });
+                adjusted++;
+            }
+            // Paused without blockers: no change (neutral signal)
+        }
+        catch {
+            // Non-fatal: individual entity update failure
+        }
+    }
+    // Clean up tracking
+    sessionAssemblyMap.delete(cardId);
+    return { adjusted };
+}