agent-mockingbird 0.0.1

package/apps/server/src/backend/memory/qmdPort.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, test } from "bun:test";
+
+import { blendRrfAndRerank, hasStrongBm25Signal, reciprocalRankFusion } from "./qmdPort";
+
+describe("hasStrongBm25Signal", () => {
+  test("returns true when score and gap thresholds are met", () => {
+    const ranked = [
+      { id: "a", score: 0.91 },
+      { id: "b", score: 0.72 },
+    ];
+    expect(hasStrongBm25Signal(ranked, { minScore: 0.85, minGap: 0.15 })).toBe(true);
+  });
+
+  test("returns false when top gap is too small", () => {
+    const ranked = [
+      { id: "a", score: 0.91 },
+      { id: "b", score: 0.8 },
+    ];
+    expect(hasStrongBm25Signal(ranked, { minScore: 0.85, minGap: 0.15 })).toBe(false);
+  });
+});
+
+describe("reciprocalRankFusion", () => {
+  test("boosts items that rank well across multiple lists", () => {
+    const results = reciprocalRankFusion(
+      [
+        [
+          { id: "x", score: 1 },
+          { id: "y", score: 0.9 },
+        ],
+        [
+          { id: "y", score: 1 },
+          { id: "x", score: 0.8 },
+        ],
+      ],
+      [1, 1],
+      60,
+    );
+
+    expect(results.length).toBe(2);
+    expect(results[0]?.id).toBe("x");
+    expect(results[1]?.id).toBe("y");
+  });
+});
+
+describe("blendRrfAndRerank", () => {
+  test("protects top-ranked items with stronger rrf weight", () => {
+    const top = blendRrfAndRerank(1, 0.1);
+    const mid = blendRrfAndRerank(8, 0.1);
+    const tail = blendRrfAndRerank(20, 0.1);
+    expect(top).toBeGreaterThan(mid);
+    expect(mid).toBeGreaterThan(tail);
+  });
+});

package/apps/server/src/backend/memory/qmdPort.ts

@@ -0,0 +1,61 @@
+// Adapted from qmd (MIT): reciprocal-rank fusion and strong-signal gating.
+// Source: https://github.com/tobi/qmd
+
+interface RankedResult {
+  id: string;
+  score: number;
+}
+
+export interface ExpandedQuery {
+  type: "lex" | "vec" | "hyde";
+  text: string;
+}
+
+export function hasStrongBm25Signal(
+  ranked: RankedResult[],
+  thresholds: { minScore: number; minGap: number },
+): boolean {
+  if (!ranked.length) return false;
+  const topScore = ranked[0]?.score ?? 0;
+  const secondScore = ranked[1]?.score ?? 0;
+  return topScore >= thresholds.minScore && topScore - secondScore >= thresholds.minGap;
+}
+
+export function reciprocalRankFusion(resultLists: RankedResult[][], weights: number[] = [], k = 60): RankedResult[] {
+  const scores = new Map<string, { score: number; topRank: number }>();
+
+  for (let listIdx = 0; listIdx < resultLists.length; listIdx += 1) {
+    const list = resultLists[listIdx];
+    if (!list) continue;
+    const weight = weights[listIdx] ?? 1;
+    for (let rank = 0; rank < list.length; rank += 1) {
+      const result = list[rank];
+      if (!result) continue;
+      const contribution = weight / (k + rank + 1);
+      const existing = scores.get(result.id);
+      if (existing) {
+        existing.score += contribution;
+        existing.topRank = Math.min(existing.topRank, rank);
+        continue;
+      }
+      scores.set(result.id, { score: contribution, topRank: rank });
+    }
+  }
+
+  for (const entry of scores.values()) {
+    if (entry.topRank === 0) entry.score += 0.05;
+    else if (entry.topRank <= 2) entry.score += 0.02;
+  }
+
+  return [...scores.entries()]
+    .map(([id, value]) => ({ id, score: value.score }))
+    .sort((a, b) => b.score - a.score);
+}
+
+export function blendRrfAndRerank(rrfRank: number, rerankScore: number) {
+  let rrfWeight = 0.4;
+  if (rrfRank <= 3) rrfWeight = 0.75;
+  else if (rrfRank <= 10) rrfWeight = 0.6;
+  const rrfScore = 1 / Math.max(1, rrfRank);
+  return rrfWeight * rrfScore + (1 - rrfWeight) * rerankScore;
+}

package/apps/server/src/backend/memory/records.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, test } from "bun:test";
+
+import { migrateLegacyMemoryMarkdownToV2, parseMemoryRecordBlocks, parseMemoryRecords } from "./records";
+
+describe("parseMemoryRecordBlocks", () => {
+  test("extracts structured memory record blocks with line ranges", () => {
+    const content = [
+      "# Daily Notes",
+      "",
+      "### [memory:memory_a1] 2026-03-02T20:38:29.490Z",
+      "meta: source=user",
+      "",
+      "User is currently in the NICU with Lucy Lee.",
+      "",
+      "### [memory:memory_b2] 2026-03-02T20:41:01.037Z",
+      "meta: source=user; entities=Vulpix",
+      "",
+      "User's favorite Pokemon is Vulpix.",
+      "",
+    ].join("\n");
+
+    const blocks = parseMemoryRecordBlocks(content);
+    expect(blocks.length).toBe(2);
+    expect(blocks[0]?.recordId).toBe("memory_a1");
+    expect(blocks[0]?.startLine).toBe(3);
+    expect(blocks[0]?.endLine).toBe(6);
+    expect(blocks[1]?.recordId).toBe("memory_b2");
+    expect(blocks[1]?.startLine).toBe(8);
+    expect(blocks[1]?.endLine).toBe(11);
+  });
+});
+
+describe("parseMemoryRecords", () => {
+  test("continues parsing record metadata/body normally", () => {
+    const content = [
+      "### [memory:memory_z9] 2026-03-02T20:41:33.914Z",
+      "meta: source=user; entities=Android; confidence=1",
+      "",
+      "User has an Android phone.",
+    ].join("\n");
+    const records = parseMemoryRecords(content);
+    expect(records.length).toBe(1);
+    expect(records[0]?.id).toBe("memory_z9");
+    expect(records[0]?.content).toBe("User has an Android phone.");
+    expect(records[0]?.source).toBe("user");
+    expect(records[0]?.entities).toEqual(["Android"]);
+  });
+});
+
+describe("migrateLegacyMemoryMarkdownToV2", () => {
+  test("rewrites legacy JSON records into compact v2 format", () => {
+    const content = [
+      "### [memory:memory_z9] 2026-03-02T20:41:33.914Z",
+      "```json",
+      '{"id":"memory_z9","source":"user","recordedAt":"2026-03-02T20:41:33.914Z","entities":["Android"],"confidence":1}',
+      "```",
+      "User has an Android phone.",
+      "",
+    ].join("\n");
+
+    const migrated = migrateLegacyMemoryMarkdownToV2(content);
+    expect(migrated.migrated).toBe(1);
+    expect(migrated.content).toContain("meta: source=user; entities=Android");
+    expect(migrated.content).not.toContain("```json");
+  });
+});

package/apps/server/src/backend/memory/records.ts

@@ -0,0 +1,229 @@
+import crypto from "node:crypto";
+
+import type { MemoryRecord, MemoryRecordInput } from "./types";
+
+const RECORD_PREFIX = "memory";
+const RECORD_HEADING_RE = /^###\s+\[memory:([a-zA-Z0-9_-]+)\].*$/m;
+const RECORD_BLOCK_RE =
+  /###\s+\[memory:([a-zA-Z0-9_-]+)\]\s+([^\n]+)\n(?:meta:\s*([^\n]*)\n)?(?:\n)?([\s\S]*?)(?=\n###\s+\[memory:|$)/g;
+const LEGACY_RECORD_BLOCK_RE =
+  /###\s+\[memory:([a-zA-Z0-9_-]+)\][^\n]*\n```json\n([\s\S]*?)\n```\n([\s\S]*?)(?=\n###\s+\[memory:|$)/g;
+
+interface ParsedMemoryRecordBlock {
+  recordId: string;
+  startLine: number;
+  endLine: number;
+  text: string;
+}
+
+function normalizeList(values: string[] | undefined): string[] {
+  if (!values?.length) {
+    return [];
+  }
+  return [...new Set(values.map(value => value.trim()).filter(Boolean))];
+}
+
+function clampConfidence(value: number | undefined): number {
+  if (typeof value !== "number" || Number.isNaN(value)) {
+    return 0.75;
+  }
+  return Math.max(0, Math.min(1, value));
+}
+
+function normalizeContent(content: string): string {
+  return content.trim().replace(/\r\n/g, "\n");
+}
+
+function parseMetaLine(metaLine: string | undefined): Partial<MemoryRecordInput> {
+  if (!metaLine) return {};
+  const parsed: Partial<MemoryRecordInput> = {};
+  const entries = metaLine
+    .split(";")
+    .map(item => item.trim())
+    .filter(Boolean);
+
+  for (const entry of entries) {
+    const delimiter = entry.indexOf("=");
+    if (delimiter <= 0) continue;
+    const key = entry.slice(0, delimiter).trim().toLowerCase();
+    const value = entry.slice(delimiter + 1).trim();
+    if (!value) continue;
+    if (key === "source") {
+      if (value === "user" || value === "assistant" || value === "system") {
+        parsed.source = value;
+      }
+      continue;
+    }
+    if (key === "confidence") {
+      const numeric = Number(value);
+      if (Number.isFinite(numeric)) {
+        parsed.confidence = clampConfidence(numeric);
+      }
+      continue;
+    }
+    if (key === "entities") {
+      parsed.entities = normalizeList(value.split(","));
+      continue;
+    }
+    if (key === "supersedes") {
+      parsed.supersedes = normalizeList(value.split(","));
+    }
+  }
+  return parsed;
+}
+
+function buildMetaLine(record: MemoryRecord): string {
+  const parts = [`source=${record.source}`];
+  const confidence = clampConfidence(record.confidence);
+  const entities = normalizeList(record.entities);
+  const supersedes = normalizeList(record.supersedes);
+  if (confidence !== 1) {
+    parts.push(`confidence=${Number(confidence.toFixed(4))}`);
+  }
+  if (entities.length > 0) {
+    parts.push(`entities=${entities.join(", ")}`);
+  }
+  if (supersedes.length > 0) {
+    parts.push(`supersedes=${supersedes.join(",")}`);
+  }
+  return `meta: ${parts.join("; ")}`;
+}
+
+export function createMemoryRecord(input: MemoryRecordInput): MemoryRecord {
+  const createdAt = new Date().toISOString();
+  const id = `${RECORD_PREFIX}_${crypto.randomUUID().replaceAll("-", "").slice(0, 12)}`;
+  return {
+    id,
+    recordedAt: createdAt,
+    source: input.source,
+    content: normalizeContent(input.content),
+    entities: normalizeList(input.entities),
+    confidence: clampConfidence(input.confidence),
+    supersedes: normalizeList(input.supersedes),
+  };
+}
+
+export function formatMemoryRecord(record: MemoryRecord): string {
+  return [
+    `### [memory:${record.id}] ${record.recordedAt}`,
+    buildMetaLine(record),
+    "",
+    record.content,
+    "",
+  ].join("\n");
+}
+
+export function parseMemoryRecords(content: string): MemoryRecord[] {
+  const records: MemoryRecord[] = [];
+  const normalized = content.replace(/\r\n/g, "\n");
+
+  let match: RegExpExecArray | null = RECORD_BLOCK_RE.exec(normalized);
+  while (match) {
+    const parsedId = match[1]?.trim();
+    const recordedAt = match[2]?.trim();
+    const metaRaw = match[3]?.trim();
+    const bodyRaw = match[4]?.trim() ?? "";
+    if (!parsedId || !recordedAt) {
+      match = RECORD_BLOCK_RE.exec(normalized);
+      continue;
+    }
+
+    const metadata = parseMetaLine(metaRaw);
+    records.push({
+      id: parsedId,
+      source: metadata.source ?? "system",
+      recordedAt,
+      content: bodyRaw,
+      entities: normalizeList(metadata.entities),
+      supersedes: normalizeList(metadata.supersedes),
+      confidence: clampConfidence(metadata.confidence),
+    });
+
+    match = RECORD_BLOCK_RE.exec(normalized);
+  }
+
+  return records;
+}
+
+export function parseMemoryRecordBlocks(content: string): ParsedMemoryRecordBlock[] {
+  const blocks: ParsedMemoryRecordBlock[] = [];
+  const normalized = content.replace(/\r\n/g, "\n");
+  let match: RegExpExecArray | null = RECORD_BLOCK_RE.exec(normalized);
+  while (match) {
+    const full = match[0] ?? "";
+    const recordId = match[1]?.trim();
+    const startOffset = match.index;
+    const endOffsetExclusive = startOffset + full.length;
+    if (!recordId || startOffset < 0 || !full.trim()) {
+      match = RECORD_BLOCK_RE.exec(normalized);
+      continue;
+    }
+    const startLine = normalized.slice(0, startOffset).split("\n").length;
+    const endLine = normalized.slice(0, Math.max(startOffset, endOffsetExclusive - 1)).split("\n").length;
+    blocks.push({
+      recordId,
+      startLine,
+      endLine,
+      text: full.trimEnd(),
+    });
+    match = RECORD_BLOCK_RE.exec(normalized);
+  }
+  return blocks;
+}
+
+export function extractRecordIdFromChunk(text: string): string | null {
+  const headingMatch = text.match(RECORD_HEADING_RE);
+  const rawId = headingMatch?.[1]?.trim();
+  if (!rawId) {
+    return null;
+  }
+  return rawId;
+}
+
+export function migrateLegacyMemoryMarkdownToV2(content: string): { content: string; migrated: number } {
+  const normalized = content.replace(/\r\n/g, "\n");
+  const migrated: string[] = [];
+  let migratedCount = 0;
+  let match: RegExpExecArray | null = LEGACY_RECORD_BLOCK_RE.exec(normalized);
+  while (match) {
+    const parsedId = match[1]?.trim();
+    const metaRaw = match[2]?.trim();
+    const bodyRaw = match[3]?.trim() ?? "";
+    if (!parsedId || !metaRaw) {
+      match = LEGACY_RECORD_BLOCK_RE.exec(normalized);
+      continue;
+    }
+
+    try {
+      const metadata = JSON.parse(metaRaw) as Partial<MemoryRecord>;
+      const source = metadata.source ?? "system";
+      const recordedAt =
+        typeof metadata.recordedAt === "string" && metadata.recordedAt.trim()
+          ? metadata.recordedAt.trim()
+          : new Date().toISOString();
+      const record: MemoryRecord = {
+        id: parsedId,
+        source: source === "user" || source === "assistant" || source === "system" ? source : "system",
+        recordedAt,
+        content: bodyRaw,
+        entities: normalizeList(metadata.entities),
+        supersedes: normalizeList(metadata.supersedes),
+        confidence: clampConfidence(metadata.confidence),
+      };
+      migrated.push(formatMemoryRecord(record).trimEnd());
+      migratedCount += 1;
+    } catch {
+      // ignore malformed legacy records
+    }
+
+    match = LEGACY_RECORD_BLOCK_RE.exec(normalized);
+  }
+
+  if (migratedCount === 0) {
+    return { content: normalized, migrated: 0 };
+  }
+  return {
+    content: `${migrated.join("\n\n")}\n`,
+    migrated: migratedCount,
+  };
+}