@hoverlover/cc-discord 0.1.0

package/memory/core/MemoryCoordinator.ts

@@ -0,0 +1,703 @@
+import { clamp } from "./types.ts";
+
+interface MemoryCoordinatorDefaults {
+  activeWindowSize: number;
+  maxCards: number;
+  maxRecallTurns: number;
+  maxTurnScan: number;
+}
+
+interface MemoryCoordinatorOptions {
+  store: any;
+  logger?: Pick<Console, "log" | "warn" | "error">;
+  defaults?: Partial<MemoryCoordinatorDefaults>;
+}
+
+export class MemoryCoordinator {
+  store: any;
+  logger: Pick<Console, "log" | "warn" | "error">;
+  defaults: MemoryCoordinatorDefaults;
+
+  constructor(options: MemoryCoordinatorOptions) {
+    if (!options?.store) throw new Error("MemoryCoordinator requires a store");
+
+    this.store = options.store;
+    this.logger = options.logger || console;
+    this.defaults = {
+      activeWindowSize: 12,
+      maxCards: 6,
+      maxRecallTurns: 8,
+      maxTurnScan: 400,
+      ...(options.defaults || {}),
+    };
+  }
+
+  async init() {
+    await this.store.init();
+  }
+
+  async appendTurn({
+    sessionKey,
+    agentId = null,
+    role,
+    content,
+    metadata = null,
+    turnIndex = null,
+  }: {
+    sessionKey: string;
+    agentId?: string | null;
+    role: string;
+    content: string;
+    metadata?: any;
+    turnIndex?: number | null;
+  }) {
+    return this.store.writeBatch({
+      batchId: makeBatchId("turn"),
+      sessionKey,
+      agentId,
+      turns: [{ role, content, metadata, turnIndex }],
+    });
+  }
+
+  async appendTurns({
+    sessionKey,
+    agentId = null,
+    turns = [],
+  }: {
+    sessionKey: string;
+    agentId?: string | null;
+    turns?: any[];
+  }) {
+    return this.store.writeBatch({
+      batchId: makeBatchId("turns"),
+      sessionKey,
+      agentId,
+      turns,
+    });
+  }
+
+  async ensureRuntimeContext({
+    sessionKey,
+    runtimeContextId = null,
+    runtimeEpoch = null,
+  }: {
+    sessionKey: string;
+    runtimeContextId?: string | null;
+    runtimeEpoch?: number | null;
+  }) {
+    if (!sessionKey) throw new Error("ensureRuntimeContext() requires sessionKey");
+
+    const existing = await this.store.readRuntimeState(sessionKey);
+    if (existing) return existing;
+
+    return this.store.upsertRuntimeState({
+      sessionKey,
+      runtimeContextId: runtimeContextId || makeRuntimeContextId("init"),
+      runtimeEpoch: Number.isInteger(runtimeEpoch) ? runtimeEpoch : 1,
+    });
+  }
+
+  async beginNewRuntimeContext({
+    sessionKey,
+    runtimeContextId = null,
+  }: {
+    sessionKey: string;
+    runtimeContextId?: string | null;
+  }) {
+    if (!sessionKey) throw new Error("beginNewRuntimeContext() requires sessionKey");
+
+    return this.store.bumpRuntimeContext({
+      sessionKey,
+      runtimeContextId: runtimeContextId || makeRuntimeContextId("sessionstart"),
+    });
+  }
+
+  /**
+   * Build context-aware memory payload for this turn.
+   *
+   * Strategy:
+   * 1) Determine active window (what model likely already has)
+   * 2) Filter out overlapping memories
+   * 3) Rank out-of-window cards/turns by relevance + novelty
+   */
+  async assembleContext(input: any) {
+    const {
+      sessionKey,
+      queryText = "",
+      runtimeContextId = null,
+      runtimeEpoch = null,
+      activeWindowTurnIds = [],
+      includeSnapshot = true,
+      avoidCurrentRuntime = true,
+      activeWindowSize = this.defaults.activeWindowSize,
+      maxCards = this.defaults.maxCards,
+      maxRecallTurns = this.defaults.maxRecallTurns,
+      maxTurnScan = this.defaults.maxTurnScan,
+    } = input || {};
+
+    if (!sessionKey) throw new Error("assembleContext() requires sessionKey");
+
+    const safeActiveWindowSize = clamp(Number(activeWindowSize) || this.defaults.activeWindowSize, 1, 100);
+    const safeMaxCards = clamp(Number(maxCards) || this.defaults.maxCards, 0, 30);
+    const safeMaxRecallTurns = clamp(Number(maxRecallTurns) || this.defaults.maxRecallTurns, 0, 30);
+    const safeMaxTurnScan = clamp(Number(maxTurnScan) || this.defaults.maxTurnScan, 50, 2000);
+
+    const resolvedRuntimeState = await resolveRuntimeState({
+      store: this.store,
+      sessionKey,
+      runtimeContextId,
+      runtimeEpoch,
+    });
+
+    const snapshot = includeSnapshot ? await this.store.readSessionSnapshot(sessionKey) : null;
+
+    const compactionState = await this.store.readCompactionState(sessionKey);
+    const compactedCutoff = await resolveCompactedCutoff({
+      store: this.store,
+      sessionKey,
+      compactionState,
+    });
+
+    const candidateTurns = await this.store.listRecentTurns({
+      sessionKey,
+      limit: safeMaxTurnScan,
+    });
+
+    const runtimeActiveTurns = avoidCurrentRuntime
+      ? candidateTurns.filter((turn: any) => isTurnInActiveRuntimeWindow(turn, resolvedRuntimeState, compactedCutoff))
+      : [];
+
+    const activeIds = new Set([
+      ...activeWindowTurnIds.map(String),
+      ...runtimeActiveTurns.map((t: any) => String(t.id)),
+    ]);
+
+    const activeWindowTurns = runtimeActiveTurns.slice(-safeActiveWindowSize);
+    const activeText = activeWindowTurns.map((t: any) => `${t.role}: ${t.content}`).join("\n");
+    const activeTokens = tokenize(activeText);
+    const queryTokens = tokenize(queryText);
+
+    const allCards = await this.store.queryCards({
+      sessionKey,
+      includeExpired: false,
+      limit: 500,
+    });
+
+    const rankedCards = allCards
+      .filter((card: any) => !overlapsActiveWindow(card, activeIds))
+      .map((card: any) => {
+        const text = `${card.title || ""}\n${card.body || ""}`;
+        const tokens = tokenize(text);
+        const overlap = tokenOverlapScore(queryTokens, tokens);
+        const novelty = 1 - jaccard(tokens, activeTokens);
+        const pinnedBoost = card.pinned ? 0.8 : 0;
+        const confidenceBoost = clamp(Number(card.confidence ?? 0.5), 0, 1) * 0.4;
+        const score = overlap * 1.2 + novelty * 0.5 + pinnedBoost + confidenceBoost;
+        return { card, score, overlap, novelty };
+      })
+      .filter((x: any) => x.score > 0.35 || x.card.pinned)
+      .sort((a: any, b: any) => b.score - a.score)
+      .slice(0, safeMaxCards)
+      .map((x: any) => x.card);
+
+    // Build turn index map for adjacency lookups
+    const turnsByIndex = new Map();
+    for (const turn of candidateTurns) {
+      turnsByIndex.set(turn.turnIndex, turn);
+    }
+
+    // Identify rare query tokens (appear in few candidate turns)
+    const tokenFrequency = new Map();
+    const candidateTokenSets = new Map();
+    for (const turn of candidateTurns) {
+      const tokens = tokenize(`${turn.role}: ${turn.content}`);
+      candidateTokenSets.set(turn.id, tokens);
+      for (const tok of tokens) {
+        tokenFrequency.set(tok, (tokenFrequency.get(tok) || 0) + 1);
+      }
+    }
+    const rareQueryTokens = new Set<string>();
+    for (const tok of queryTokens) {
+      const freq = tokenFrequency.get(tok) || 0;
+      // Token appears in <20% of turns = rare/specific
+      if (freq > 0 && freq <= Math.max(2, candidateTurns.length * 0.2)) {
+        rareQueryTokens.add(tok);
+      }
+    }
+    // Also check fuzzy matches for rare tokens
+    const rareQueryTrigrams = rareQueryTokens.size > 0 ? trigramSet(rareQueryTokens) : new Set<string>();
+
+    const scoredTurns = candidateTurns
+      .filter((turn: any) => !activeIds.has(String(turn.id)))
+      .map((turn: any) => {
+        const tokens = candidateTokenSets.get(turn.id) || tokenize(`${turn.role}: ${turn.content}`);
+        const overlap = tokenOverlapScore(queryTokens, tokens);
+        const novelty = 1 - jaccard(tokens, activeTokens);
+        const recency = turn.turnIndex / Math.max(1, candidateTurns.length);
+        const negationPenalty = isNegationTurn(turn.content) ? 0.3 : 1.0;
+
+        // Rare keyword boost: if turn contains a rare query term (exact or fuzzy), boost it
+        let rareBoost = 0;
+        if (rareQueryTokens.size > 0) {
+          for (const tok of tokens) {
+            if (rareQueryTokens.has(tok)) {
+              rareBoost = 0.3;
+              break;
+            }
+          }
+          // Fuzzy check via trigrams if no exact match
+          if (rareBoost === 0 && rareQueryTrigrams.size > 0) {
+            const turnTrigrams = trigramSet(tokens);
+            let triIntersect = 0;
+            for (const tri of rareQueryTrigrams) {
+              if (turnTrigrams.has(tri)) triIntersect++;
+            }
+            const triUnion = rareQueryTrigrams.size + turnTrigrams.size - triIntersect;
+            const sim = triUnion > 0 ? triIntersect / triUnion : 0;
+            if (sim > 0.15) rareBoost = 0.2;
+          }
+        }
+
+        // Novelty only contributes when there's some keyword signal.
+        // Without overlap or rareBoost, novelty alone shouldn't qualify a turn.
+        const hasKeywordSignal = overlap > 0.05 || rareBoost > 0;
+        const effectiveNovelty = hasKeywordSignal ? novelty * 0.4 : 0;
+
+        const score = (overlap * 1.0 + effectiveNovelty + recency * 0.1 + rareBoost) * negationPenalty;
+        return { turn, score };
+      })
+      .filter((x: any) => x.score > 0.4)
+      .sort((a: any, b: any) => b.score - a.score);
+
+    // Deduplicate near-identical turns (e.g. user asking the same question repeatedly)
+    const deduped: any[] = [];
+    const seenContentHashes = new Set();
+    for (const scored of scoredTurns) {
+      const hash = contentHash(scored.turn.content);
+      if (seenContentHashes.has(hash)) continue;
+      seenContentHashes.add(hash);
+      deduped.push(scored);
+    }
+
+    // Select top turns, then expand with adjacent response turns (conversation threading)
+    const selectedIds = new Set();
+    const selectedTurns: any[] = [];
+
+    for (const { turn } of deduped) {
+      if (selectedTurns.length >= safeMaxRecallTurns) break;
+      if (selectedIds.has(turn.id)) continue;
+
+      selectedIds.add(turn.id);
+      selectedTurns.push(turn);
+
+      // If this is a user turn, pull the next assistant turn (the actual response)
+      if (turn.role === "user" && selectedTurns.length < safeMaxRecallTurns) {
+        const nextTurn = turnsByIndex.get(turn.turnIndex + 1);
+        if (
+          nextTurn &&
+          nextTurn.role === "assistant" &&
+          !activeIds.has(String(nextTurn.id)) &&
+          !selectedIds.has(nextTurn.id)
+        ) {
+          // Only include if not a negation turn
+          if (!isNegationTurn(nextTurn.content)) {
+            selectedIds.add(nextTurn.id);
+            selectedTurns.push(nextTurn);
+          }
+        }
+      }
+
+      // If this is an assistant turn, pull the preceding user turn for context
+      if (turn.role === "assistant" && selectedTurns.length < safeMaxRecallTurns) {
+        const prevTurn = turnsByIndex.get(turn.turnIndex - 1);
+        if (
+          prevTurn &&
+          prevTurn.role === "user" &&
+          !activeIds.has(String(prevTurn.id)) &&
+          !selectedIds.has(prevTurn.id)
+        ) {
+          selectedIds.add(prevTurn.id);
+          selectedTurns.push(prevTurn);
+        }
+      }
+    }
+
+    const recalledTurns = selectedTurns.sort((a: any, b: any) => a.turnIndex - b.turnIndex);
+
+    return {
+      sessionKey,
+      queryText,
+      snapshot,
+      activeWindowTurns,
+      recalledTurns,
+      cards: rankedCards,
+      debug: {
+        runtimeContextId: resolvedRuntimeState?.runtimeContextId || null,
+        runtimeEpoch: resolvedRuntimeState?.runtimeEpoch || null,
+        compactedCutoffTurnId: compactionState?.lastCompactedTurnId || null,
+        compactedCutoffTurnIndex: compactedCutoff?.turnIndex ?? null,
+        totalCardsScanned: allCards.length,
+        totalTurnsScanned: candidateTurns.length,
+        activeTurnCount: runtimeActiveTurns.length,
+      },
+    };
+  }
+
+  formatContextPacket(payload: any) {
+    const parts: string[] = [];
+
+    if (payload.snapshot?.summaryText) {
+      parts.push(`Session summary:\n${truncate(payload.snapshot.summaryText, 800)}`);
+
+      if (Array.isArray(payload.snapshot.openTasks) && payload.snapshot.openTasks.length > 0) {
+        parts.push(`Open tasks:\n- ${payload.snapshot.openTasks.join("\n- ")}`);
+      }
+    }
+
+    if (Array.isArray(payload.cards) && payload.cards.length > 0) {
+      const lines = payload.cards.map((card: any) => {
+        const label = card.cardType ? `[${card.cardType}]` : "";
+        const title = card.title ? `${truncate(card.title, 120)}: ` : "";
+        return `- ${label} ${title}${truncate(card.body, 320)}`.trim();
+      });
+      parts.push(`Relevant long-term memory (outside current window):\n${lines.join("\n")}`);
+    }
+
+    if (Array.isArray(payload.recalledTurns) && payload.recalledTurns.length > 0) {
+      const lines = payload.recalledTurns.map((turn: any) => `- (${turn.role}) ${truncate(turn.content, 280)}`);
+      parts.push(`Relevant prior turns (outside current window):\n${lines.join("\n")}`);
+    }
+
+    if (parts.length === 0) return "";
+    return `MEMORY CONTEXT:\n${parts.join("\n\n")}`;
+  }
+}
+
+function makeBatchId(prefix: string) {
+  const ts = Date.now().toString(36);
+  const rand = Math.random().toString(36).slice(2, 10);
+  return `${prefix}_${ts}_${rand}`;
+}
+
+function makeRuntimeContextId(prefix: string = "runtime") {
+  const ts = Date.now().toString(36);
+  const rand = Math.random().toString(36).slice(2, 10);
+  return `${prefix}_${ts}_${rand}`;
+}
+
+function overlapsActiveWindow(card: any, activeIds: Set<string>) {
+  if (card.sourceTurnFrom && activeIds.has(String(card.sourceTurnFrom))) return true;
+  if (card.sourceTurnTo && activeIds.has(String(card.sourceTurnTo))) return true;
+  return false;
+}
+
+async function resolveRuntimeState({
+  store,
+  sessionKey,
+  runtimeContextId,
+  runtimeEpoch,
+}: {
+  store: any;
+  sessionKey: string;
+  runtimeContextId: string | null;
+  runtimeEpoch: number | null;
+}) {
+  if (runtimeContextId || runtimeEpoch) {
+    return {
+      runtimeContextId: runtimeContextId || null,
+      runtimeEpoch: Number.isInteger(runtimeEpoch) ? runtimeEpoch : null,
+    };
+  }
+
+  if (!store?.readRuntimeState) return null;
+
+  try {
+    return await store.readRuntimeState(sessionKey);
+  } catch {
+    return null;
+  }
+}
+
+async function resolveCompactedCutoff({
+  store,
+  sessionKey,
+  compactionState,
+}: {
+  store: any;
+  sessionKey: string;
+  compactionState: any;
+}) {
+  const turnId = compactionState?.lastCompactedTurnId;
+  if (!turnId) return null;
+
+  if (!store?.getTurnById) return null;
+
+  try {
+    const turn = await store.getTurnById({ sessionKey, turnId });
+    return turn || null;
+  } catch {
+    return null;
+  }
+}
+
+function isTurnInActiveRuntimeWindow(turn: any, runtimeState: any, compactedCutoff: any) {
+  if (!runtimeState?.runtimeContextId) return false;
+
+  const turnRuntimeContextId = turn?.metadata?.runtimeContextId;
+  if (!turnRuntimeContextId || turnRuntimeContextId !== runtimeState.runtimeContextId) {
+    return false;
+  }
+
+  if (compactedCutoff?.turnIndex === undefined || compactedCutoff?.turnIndex === null) {
+    // No compaction cutoff available: treat all current-runtime turns as active.
+    return true;
+  }
+
+  return turn.turnIndex > compactedCutoff.turnIndex;
+}
+
+const STOP_WORDS = new Set([
+  "the",
+  "and",
+  "for",
+  "are",
+  "but",
+  "not",
+  "you",
+  "all",
+  "any",
+  "can",
+  "has",
+  "her",
+  "his",
+  "how",
+  "its",
+  "may",
+  "our",
+  "out",
+  "was",
+  "who",
+  "did",
+  "get",
+  "got",
+  "had",
+  "him",
+  "let",
+  "say",
+  "she",
+  "too",
+  "use",
+  "yes",
+  "yet",
+  "been",
+  "each",
+  "have",
+  "from",
+  "into",
+  "just",
+  "like",
+  "make",
+  "many",
+  "more",
+  "most",
+  "much",
+  "must",
+  "name",
+  "only",
+  "over",
+  "such",
+  "take",
+  "than",
+  "that",
+  "them",
+  "then",
+  "they",
+  "this",
+  "very",
+  "what",
+  "when",
+  "will",
+  "with",
+  "your",
+  "also",
+  "back",
+  "been",
+  "come",
+  "could",
+  "does",
+  "don",
+  "dont",
+  "even",
+  "give",
+  "goes",
+  "gone",
+  "good",
+  "here",
+  "high",
+  "keep",
+  "know",
+  "last",
+  "long",
+  "look",
+  "made",
+  "some",
+  "sure",
+  "tell",
+  "told",
+  "want",
+  "well",
+  "were",
+  "which",
+  "would",
+  "about",
+  "after",
+  "again",
+  "being",
+  "below",
+  "could",
+  "every",
+  "first",
+  "found",
+  "going",
+  "great",
+  "might",
+  "never",
+  "other",
+  "right",
+  "shall",
+  "since",
+  "still",
+  "their",
+  "there",
+  "these",
+  "thing",
+  "think",
+  "those",
+  "until",
+  "where",
+  "while",
+  "would",
+  "should",
+  "really",
+  "before",
+  "because",
+  "between",
+  "time",
+  "one",
+  "more",
+  "see",
+  "now",
+  "way",
+  "new",
+  "said",
+  "asked",
+  "user",
+  "assistant",
+  "hoverlover617",
+]);
+
+function tokenize(text: string): Set<string> {
+  return new Set(
+    String(text || "")
+      .toLowerCase()
+      .replace(/[^a-z0-9\s]/g, " ")
+      .split(/\s+/)
+      .filter((t) => t.length > 2 && !STOP_WORDS.has(t)),
+  );
+}
+
+/**
+ * Generate character trigrams for a token set.
+ * Used for fuzzy matching (typos, near-misses).
+ */
+function trigramSet(tokens: Set<string>): Set<string> {
+  const trigrams = new Set<string>();
+  for (const tok of tokens) {
+    const padded = `  ${tok} `;
+    for (let i = 0; i < padded.length - 2; i++) {
+      trigrams.add(padded.slice(i, i + 3));
+    }
+  }
+  return trigrams;
+}
+
+function trigramSimilarity(aTokens: Set<string>, bTokens: Set<string>): number {
+  if (!aTokens || !bTokens || aTokens.size === 0 || bTokens.size === 0) return 0;
+  const aTri = trigramSet(aTokens);
+  const bTri = trigramSet(bTokens);
+  if (aTri.size === 0 || bTri.size === 0) return 0;
+
+  let intersection = 0;
+  for (const tri of aTri) {
+    if (bTri.has(tri)) intersection++;
+  }
+  const union = aTri.size + bTri.size - intersection;
+  return union > 0 ? intersection / union : 0;
+}
+
+function tokenOverlapScore(aTokens: Set<string>, bTokens: Set<string>): number {
+  if (!aTokens || !bTokens || aTokens.size === 0 || bTokens.size === 0) return 0;
+
+  // Exact match component
+  let exactOverlap = 0;
+  for (const tok of aTokens) {
+    if (bTokens.has(tok)) exactOverlap++;
+  }
+  const exactScore = exactOverlap / Math.max(1, Math.min(aTokens.size, 10));
+
+  // Fuzzy trigram component (catches typos like matterpost/mattermost)
+  const fuzzyScore = trigramSimilarity(aTokens, bTokens);
+
+  // Blend: exact match weighted higher, fuzzy as fallback
+  return Math.max(exactScore, exactScore * 0.7 + fuzzyScore * 0.5);
+}
+
+function jaccard(aTokens: Set<string>, bTokens: Set<string>): number {
+  if ((!aTokens || aTokens.size === 0) && (!bTokens || bTokens.size === 0)) return 1;
+  if (!aTokens || !bTokens || aTokens.size === 0 || bTokens.size === 0) return 0;
+
+  let intersection = 0;
+  for (const tok of aTokens) {
+    if (bTokens.has(tok)) intersection++;
+  }
+
+  const union = aTokens.size + bTokens.size - intersection;
+  return union > 0 ? intersection / union : 0;
+}
+
+/**
+ * Detect "I don't know/remember" meta-turns that contain query terms
+ * but carry no actual information. These get penalized in scoring.
+ */
+const NEGATION_PATTERNS = [
+  /\bdon'?t have (the |any )?(specific )?(details|context|record|information|memory)\b/i,
+  /\bdon'?t (remember|recall|know)\b/i,
+  /\bunfortunately.{0,40}(don'?t|no |not ).{0,30}(context|detail|record|memory|information)\b/i,
+  /\bmy memory.{0,30}(limited|doesn'?t|does not)\b/i,
+  /\bcould you remind me\b/i,
+  /\bcan'?t recall\b/i,
+  /\bno record of\b/i,
+  /\bbefore my.{0,20}(memory|persistent|context)\b/i,
+];
+
+function isNegationTurn(content: string): boolean {
+  const text = String(content || "");
+  return NEGATION_PATTERNS.some((p) => p.test(text));
+}
+
+/**
+ * Simple content hash for deduplication.
+ * Normalizes whitespace/punctuation and takes first ~100 significant chars.
+ */
+function contentHash(content: string): string {
+  return String(content || "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, "")
+    .slice(0, 100);
+}
+
+function truncate(text: string, maxLen: number): string {
+  const str = String(text || "");
+  if (str.length <= maxLen) return str;
+  return `${str.slice(0, maxLen - 1)}…`;
+}