@teammates/recall 0.4.1 → 0.5.1

package/src/indexer.test.ts

@@ -107,9 +107,11 @@ describe("Indexer", () => {
       expect(uris).toContain("beacon/memory/project_goals.md");
     });
 
-    it("skips daily logs (YYYY-MM-DD.md pattern)", async () => {
+    it("includes older daily logs but skips today's", async () => {
       const memDir = join(testDir, "beacon", "memory");
       await mkdir(memDir, { recursive: true });
+      const today = new Date().toISOString().slice(0, 10);
+      await writeFile(join(memDir, `${today}.md`), "# Today");
       await writeFile(join(memDir, "2026-03-14.md"), "# Day 1");
       await writeFile(join(memDir, "2026-03-15.md"), "# Day 2");
       await writeFile(join(memDir, "feedback_testing.md"), "# Feedback");
@@ -118,8 +120,9 @@ describe("Indexer", () => {
       const { files } = await indexer.collectFiles("beacon");
 
       const uris = files.map((f) => f.uri);
-      expect(uris).not.toContain("beacon/memory/2026-03-14.md");
-      expect(uris).not.toContain("beacon/memory/2026-03-15.md");
+      expect(uris).not.toContain(`beacon/memory/${today}.md`);
+      expect(uris).toContain("beacon/memory/2026-03-14.md");
+      expect(uris).toContain("beacon/memory/2026-03-15.md");
       expect(uris).toContain("beacon/memory/feedback_testing.md");
     });
 
@@ -230,6 +233,78 @@ describe("Indexer", () => {
     });
   });
 
+  describe("upsertFile", () => {
+    it("upserts a single file into a new index", async () => {
+      const beacon = join(testDir, "beacon");
+      await mkdir(beacon, { recursive: true });
+      const filePath = join(beacon, "WISDOM.md");
+      await writeFile(filePath, "# Upsert test wisdom");
+
+      const indexer = createIndexer(testDir);
+      await indexer.upsertFile("beacon", filePath);
+
+      // Verify index was created by syncing (which reads the index)
+      const count = await indexer.syncTeammate("beacon");
+      expect(count).toBeGreaterThanOrEqual(1);
+    });
+
+    it("upserts into an existing index without rebuilding", async () => {
+      const beacon = join(testDir, "beacon");
+      const memDir = join(beacon, "memory");
+      await mkdir(memDir, { recursive: true });
+      await writeFile(join(beacon, "WISDOM.md"), "# Wisdom");
+
+      const indexer = createIndexer(testDir);
+      // Build initial index
+      await indexer.indexTeammate("beacon");
+
+      // Upsert a new file
+      const newFile = join(memDir, "feedback_test.md");
+      await writeFile(newFile, "# New feedback content");
+      await indexer.upsertFile("beacon", newFile);
+
+      // Sync should see both files
+      const count = await indexer.syncTeammate("beacon");
+      expect(count).toBe(2);
+    });
+
+    it("skips empty files", async () => {
+      const beacon = join(testDir, "beacon");
+      await mkdir(beacon, { recursive: true });
+      const filePath = join(beacon, "WISDOM.md");
+      await writeFile(filePath, "   "); // whitespace only
+
+      const indexer = createIndexer(testDir);
+      // Should not throw, just skip
+      await indexer.upsertFile("beacon", filePath);
+    });
+  });
+
+  describe("syncAll", () => {
+    it("syncs all discovered teammates", async () => {
+      const beacon = join(testDir, "beacon");
+      const scribe = join(testDir, "scribe");
+      await mkdir(beacon, { recursive: true });
+      await mkdir(scribe, { recursive: true });
+      await writeFile(join(beacon, "SOUL.md"), "# Beacon");
+      await writeFile(join(beacon, "WISDOM.md"), "# Beacon wisdom");
+      await writeFile(join(scribe, "SOUL.md"), "# Scribe");
+      await writeFile(join(scribe, "WISDOM.md"), "# Scribe wisdom");
+
+      const indexer = createIndexer(testDir);
+      const results = await indexer.syncAll();
+
+      expect(results.get("beacon")).toBe(1);
+      expect(results.get("scribe")).toBe(1);
+    });
+
+    it("returns empty map when no teammates exist", async () => {
+      const indexer = createIndexer(testDir);
+      const results = await indexer.syncAll();
+      expect(results.size).toBe(0);
+    });
+  });
+
   describe("syncTeammate", () => {
     it("falls back to full index when no index exists", async () => {
       const beacon = join(testDir, "beacon");

package/src/indexer.ts

@@ -74,15 +74,19 @@ export class Indexer {
       // No WISDOM.md
     }
 
-    // memory/*.md — typed memories only (skip raw daily logs, they're in prompt context)
+    // memory/*.md — typed memories + daily logs (day 2+)
+    // Today's daily log is excluded (still being written). Older dailies are
+    // indexed so recall can surface high-resolution episodic context beyond
+    // the 7-day prompt window. Dailies older than 30 days are purged elsewhere.
     const memoryDir = path.join(teammateDir, "memory");
+    const today = new Date().toISOString().slice(0, 10);
     try {
       const memoryEntries = await fs.readdir(memoryDir);
       for (const entry of memoryEntries) {
         if (!entry.endsWith(".md")) continue;
         const stem = path.basename(entry, ".md");
-        // Skip daily logs (YYYY-MM-DD) — they're already in prompt context
-        if (/^\d{4}-\d{2}-\d{2}$/.test(stem)) continue;
+        // Skip today's daily log — it's still being written and already in prompt context
+        if (stem === today) continue;
         files.push({
           uri: `${teammate}/memory/${entry}`,
           absolutePath: path.join(memoryDir, entry),
@@ -222,6 +226,25 @@ export class Indexer {
     return count;
   }
 
+  /**
+   * Delete a document from a teammate's index by URI.
+   * Used to purge stale daily logs after they age out on disk.
+   */
+  async deleteDocument(teammate: string, uri: string): Promise<void> {
+    const indexPath = this.indexPath(teammate);
+    const index = new LocalDocumentIndex({
+      folderPath: indexPath,
+      embeddings: this._embeddings,
+    });
+
+    if (!(await index.isIndexCreated())) return;
+
+    const docId = await index.getDocumentId(uri);
+    if (docId) {
+      await index.deleteDocument(uri);
+    }
+  }
+
   /**
    * Sync indexes for all teammates.
    */

package/src/memory-index.test.ts

@@ -0,0 +1,149 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { matchMemoryCatalog, scanMemoryCatalog } from "./memory-index.js";
+
+const TEST_DIR = path.join(process.cwd(), ".test-memory-index");
+const TEAMMATE = "testmate";
+
+beforeAll(async () => {
+  const memoryDir = path.join(TEST_DIR, TEAMMATE, "memory");
+  await fs.mkdir(memoryDir, { recursive: true });
+
+  // Create typed memory files with frontmatter
+  await fs.writeFile(
+    path.join(memoryDir, "project_goals.md"),
+    `---
+name: project_goals
+description: Stack-ranked feature goals for the teammates project
+type: project
+---
+
+## Goals
+
+1. Recall query architecture
+2. CLI improvements
+`,
+  );
+
+  await fs.writeFile(
+    path.join(memoryDir, "feedback_testing.md"),
+    `---
+name: feedback_testing
+description: Integration tests must hit a real database, not mocks
+type: feedback
+---
+
+Use real databases in tests. Mocks hide migration bugs.
+`,
+  );
+
+  // Create a file without frontmatter (should be skipped)
+  await fs.writeFile(
+    path.join(memoryDir, "notes.md"),
+    "Just some notes without frontmatter.\n",
+  );
+
+  // Create a daily log (should be skipped)
+  await fs.writeFile(
+    path.join(memoryDir, "2026-03-21.md"),
+    "# 2026-03-21\nDaily log content.\n",
+  );
+});
+
+afterAll(async () => {
+  await fs.rm(TEST_DIR, { recursive: true, force: true });
+});
+
+describe("scanMemoryCatalog", () => {
+  it("returns entries with frontmatter", async () => {
+    const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
+    expect(entries.length).toBe(2);
+    const names = entries.map((e) => e.name);
+    expect(names).toContain("project_goals");
+    expect(names).toContain("feedback_testing");
+  });
+
+  it("skips files without frontmatter", async () => {
+    const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
+    const uris = entries.map((e) => e.uri);
+    expect(uris).not.toContain(`${TEAMMATE}/memory/notes.md`);
+  });
+
+  it("skips daily logs", async () => {
+    const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
+    const uris = entries.map((e) => e.uri);
+    expect(uris).not.toContain(`${TEAMMATE}/memory/2026-03-21.md`);
+  });
+
+  it("returns empty array for nonexistent teammate", async () => {
+    const entries = await scanMemoryCatalog(TEST_DIR, "nobody");
+    expect(entries).toEqual([]);
+  });
+});
+
+describe("matchMemoryCatalog", () => {
+  it("matches files whose frontmatter overlaps with the query", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "what are our project goals and features?",
+    );
+    expect(results.length).toBeGreaterThanOrEqual(1);
+    expect(results[0].uri).toContain("project_goals");
+  });
+
+  it("matches on description keywords", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "database testing integration",
+    );
+    expect(results.length).toBeGreaterThanOrEqual(1);
+    const uris = results.map((r) => r.uri);
+    expect(uris).toContain(`${TEAMMATE}/memory/feedback_testing.md`);
+  });
+
+  it("returns empty for unrelated queries", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "quantum physics dark matter",
+    );
+    expect(results.length).toBe(0);
+  });
+
+  it("strips frontmatter from result text", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "project goals features",
+    );
+    expect(results.length).toBeGreaterThanOrEqual(1);
+    expect(results[0].text).not.toContain("---");
+    expect(results[0].text).toContain("Goals");
+  });
+
+  it("assigns scores in the 0.85-0.95 range", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "project goals features teammates",
+    );
+    for (const r of results) {
+      expect(r.score).toBeGreaterThanOrEqual(0.85);
+      expect(r.score).toBeLessThanOrEqual(0.95);
+    }
+  });
+
+  it("sets contentType to typed_memory", async () => {
+    const results = await matchMemoryCatalog(
+      TEST_DIR,
+      TEAMMATE,
+      "project goals",
+    );
+    for (const r of results) {
+      expect(r.contentType).toBe("typed_memory");
+    }
+  });
+});

package/src/memory-index.ts

@@ -0,0 +1,151 @@
+/**
+ * Memory frontmatter scanning for Pass 1 recall queries.
+ *
+ * Reads the teammate's memory file catalog (name + description from frontmatter)
+ * and does fast text matching against the task prompt. This is a lightweight,
+ * no-embedding relevance signal — "here's a menu of what I might know about."
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import type { SearchResult } from "./search.js";
+
+interface MemoryEntry {
+  /** Relative URI (e.g. "beacon/memory/project_goals.md") */
+  uri: string;
+  /** Absolute file path */
+  absolutePath: string;
+  /** Frontmatter name field */
+  name: string;
+  /** Frontmatter description field */
+  description: string;
+}
+
+/**
+ * Parse YAML-ish frontmatter from a markdown file's content.
+ * Returns name and description fields, or null if no frontmatter found.
+ */
+function parseFrontmatter(content: string): { name: string; description: string } | null {
+  const match = content.match(/^---\s*\n([\s\S]*?)\n---/);
+  if (!match) return null;
+
+  const fm = match[1];
+  const nameMatch = fm.match(/^name:\s*(.+)$/m);
+  const descMatch = fm.match(/^description:\s*(.+)$/m);
+
+  if (!nameMatch) return null;
+
+  return {
+    name: nameMatch[1].trim(),
+    description: descMatch?.[1]?.trim() ?? "",
+  };
+}
+
+/**
+ * Scan a teammate's memory directory and build a catalog of memory entries
+ * with their frontmatter metadata.
+ */
+export async function scanMemoryCatalog(
+  teammatesDir: string,
+  teammate: string,
+): Promise<MemoryEntry[]> {
+  const memoryDir = path.join(teammatesDir, teammate, "memory");
+  const entries: MemoryEntry[] = [];
+
+  try {
+    const files = await fs.readdir(memoryDir);
+    for (const file of files) {
+      if (!file.endsWith(".md")) continue;
+      // Skip daily logs (YYYY-MM-DD.md)
+      const stem = path.basename(file, ".md");
+      if (/^\d{4}-\d{2}-\d{2}$/.test(stem)) continue;
+
+      const absolutePath = path.join(memoryDir, file);
+      const content = await fs.readFile(absolutePath, "utf-8");
+      const fm = parseFrontmatter(content);
+      if (!fm) continue;
+
+      entries.push({
+        uri: `${teammate}/memory/${file}`,
+        absolutePath,
+        name: fm.name,
+        description: fm.description,
+      });
+    }
+  } catch {
+    // No memory/ directory
+  }
+
+  return entries;
+}
+
+/**
+ * Match task prompt text against memory catalog entries.
+ * Returns memory files whose name or description has significant word overlap
+ * with the task prompt. Each match is returned as a SearchResult with the
+ * file's full content.
+ *
+ * Matching is case-insensitive. A match requires at least one word from the
+ * task prompt appearing in the name or description.
+ */
+export async function matchMemoryCatalog(
+  teammatesDir: string,
+  teammate: string,
+  taskPrompt: string,
+  maxTokens = 500,
+): Promise<SearchResult[]> {
+  const catalog = await scanMemoryCatalog(teammatesDir, teammate);
+  if (catalog.length === 0) return [];
+
+  // Tokenize the task prompt into lowercase words (3+ chars)
+  const promptWords = new Set(
+    taskPrompt
+      .toLowerCase()
+      .replace(/[^\w\s@/-]/g, " ")
+      .split(/\s+/)
+      .filter((w) => w.length > 2),
+  );
+
+  const results: SearchResult[] = [];
+
+  for (const entry of catalog) {
+    const catalogText = `${entry.name} ${entry.description}`.toLowerCase();
+    const catalogWords = catalogText
+      .replace(/[^\w\s@/_-]/g, " ")
+      .split(/\s+/)
+      .filter((w) => w.length > 2);
+
+    // Count overlapping words
+    let overlap = 0;
+    for (const w of catalogWords) {
+      if (promptWords.has(w)) overlap++;
+    }
+
+    // Also check if prompt words appear as substrings in the catalog text
+    // (e.g., "goal" matches "project_goals")
+    for (const pw of promptWords) {
+      if (catalogText.includes(pw) && !catalogWords.includes(pw)) {
+        overlap += 0.5;
+      }
+    }
+
+    if (overlap >= 1) {
+      // Read full file content for matched entries
+      const content = await fs.readFile(entry.absolutePath, "utf-8");
+      // Strip frontmatter from the content
+      const body = content.replace(/^---\s*\n[\s\S]*?\n---\s*\n?/, "").trim();
+
+      results.push({
+        teammate,
+        uri: entry.uri,
+        text: body.slice(0, maxTokens * 4), // rough token limit
+        score: 0.85 + Math.min(overlap * 0.02, 0.1), // 0.85-0.95 range
+        contentType: "typed_memory",
+      });
+    }
+  }
+
+  // Sort by score descending
+  results.sort((a, b) => b.score - a.score);
+  return results;
+}

package/src/query-expansion.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it } from "vitest";
+import { buildQueryVariations, extractKeywords } from "./query-expansion.js";
+
+describe("extractKeywords", () => {
+  it("removes stopwords", () => {
+    const result = extractKeywords("the quick brown fox jumps over the lazy dog");
+    expect(result).toContain("quick");
+    expect(result).toContain("brown");
+    expect(result).toContain("fox");
+    expect(result).toContain("jumps");
+    expect(result).toContain("lazy");
+    expect(result).toContain("dog");
+    expect(result).not.toContain("the");
+    expect(result).not.toContain("over");
+  });
+
+  it("filters short tokens (length <= 2)", () => {
+    const result = extractKeywords("an AI is a type of ML system");
+    expect(result).not.toContain("an");
+    expect(result).not.toContain("is");
+    // "type" and "system" stay, "AI" and "ML" filtered (length 2)
+    expect(result).toContain("type");
+    expect(result).toContain("system");
+  });
+
+  it("deduplicates while preserving order", () => {
+    const result = extractKeywords("recall search recall index search");
+    expect(result).toEqual(["recall", "search", "index"]);
+  });
+
+  it("lowercases all output", () => {
+    const result = extractKeywords("Update the HOOKS spec");
+    expect(result).toContain("update");
+    expect(result).toContain("hooks");
+    expect(result).toContain("spec");
+  });
+
+  it("returns empty array for all-stopword input", () => {
+    const result = extractKeywords("the is a");
+    expect(result).toEqual([]);
+  });
+
+  it("preserves @mentions and paths", () => {
+    const result = extractKeywords("deploy @pipeline src/hooks");
+    expect(result).toContain("deploy");
+    expect(result).toContain("@pipeline");
+    expect(result).toContain("src/hooks");
+  });
+});
+
+describe("buildQueryVariations", () => {
+  it("always includes the original prompt as the first query", () => {
+    const result = buildQueryVariations("fix the authentication bug");
+    expect(result[0]).toBe("fix the authentication bug");
+  });
+
+  it("generates a keyword-focused query when prompt is verbose", () => {
+    const verbose = "I want you to please update the recall search system so that it handles multiple queries at the same time and deduplicates the results properly";
+    const result = buildQueryVariations(verbose);
+    expect(result.length).toBeGreaterThanOrEqual(2);
+    // The keyword query should be shorter than the original
+    if (result.length > 1) {
+      expect(result[1].length).toBeLessThan(verbose.length);
+    }
+  });
+
+  it("adds a conversation-derived query when context is provided", () => {
+    const conversationContext = `## Conversation History
+
+**stevenic:** lets talk about the CI pipeline and hooks
+
+**pipeline:** CI Pipeline Hooks — Analysis`;
+    const result = buildQueryVariations("what should we do next?", conversationContext);
+    // Should have at least the original + conversation query
+    expect(result.length).toBeGreaterThanOrEqual(2);
+  });
+
+  it("skips conversation query when no context", () => {
+    const result = buildQueryVariations("short task");
+    // Short prompts with few keywords may only produce 1 query
+    expect(result.length).toBeGreaterThanOrEqual(1);
+    expect(result[0]).toBe("short task");
+  });
+
+  it("does not generate keyword query when prompt is already concise", () => {
+    const result = buildQueryVariations("recall search");
+    // Very short — keyword query wouldn't differ meaningfully
+    expect(result[0]).toBe("recall search");
+  });
+});

package/src/query-expansion.ts

@@ -0,0 +1,105 @@
+/**
+ * Lightweight query expansion for Pass 1 recall queries.
+ *
+ * No LLM needed — uses stopword removal and basic text analysis
+ * to generate multiple query variations from a task prompt.
+ */
+
+/** Common English stopwords to filter from queries. */
+const STOPWORDS = new Set([
+  "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+  "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
+  "being", "have", "has", "had", "do", "does", "did", "will", "would",
+  "could", "should", "may", "might", "shall", "can", "need", "must",
+  "it", "its", "this", "that", "these", "those", "i", "you", "he", "she",
+  "we", "they", "me", "him", "her", "us", "them", "my", "your", "his",
+  "our", "their", "what", "which", "who", "whom", "where", "when", "how",
+  "why", "if", "then", "so", "not", "no", "just", "also", "very", "too",
+  "some", "any", "all", "each", "every", "both", "few", "more", "most",
+  "other", "into", "over", "after", "before", "between", "through",
+  "about", "up", "out", "off", "down", "here", "there", "again", "once",
+  "let", "lets", "let's", "get", "got", "go", "going", "make", "made",
+  "take", "took", "come", "came", "see", "saw", "know", "knew", "think",
+  "thought", "say", "said", "tell", "told", "ask", "asked", "want",
+  "wanted", "like", "look", "use", "used", "find", "give", "work",
+]);
+
+/**
+ * Extract meaningful keywords from text by removing stopwords and short tokens.
+ * Returns lowercase keywords in order of appearance.
+ */
+export function extractKeywords(text: string): string[] {
+  const words = text
+    .toLowerCase()
+    .replace(/[^\w\s@/-]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 2 && !STOPWORDS.has(w));
+
+  // Deduplicate while preserving order
+  const seen = new Set<string>();
+  const result: string[] = [];
+  for (const w of words) {
+    if (!seen.has(w)) {
+      seen.add(w);
+      result.push(w);
+    }
+  }
+  return result;
+}
+
+/**
+ * Build multiple query variations from a task prompt and optional conversation context.
+ *
+ * Returns 1-3 queries:
+ * 1. The original task prompt (always)
+ * 2. A focused keyword query (if keywords differ meaningfully from the original)
+ * 3. A conversation-derived query (if recent conversation context is provided)
+ */
+export function buildQueryVariations(
+  taskPrompt: string,
+  conversationContext?: string,
+): string[] {
+  const queries: string[] = [taskPrompt];
+
+  // Query 2: Focused keywords from the task prompt
+  const keywords = extractKeywords(taskPrompt);
+  if (keywords.length >= 2 && keywords.length <= 20) {
+    const keywordQuery = keywords.slice(0, 8).join(" ");
+    // Only add if meaningfully different from original
+    if (keywordQuery.length < taskPrompt.length * 0.7) {
+      queries.push(keywordQuery);
+    }
+  }
+
+  // Query 3: Recent conversation topic
+  if (conversationContext) {
+    const recentTopic = extractRecentTopic(conversationContext);
+    if (recentTopic) {
+      queries.push(recentTopic);
+    }
+  }
+
+  return queries;
+}
+
+/**
+ * Extract the most recent topic/theme from conversation context.
+ * Takes the last 1-2 meaningful entries and extracts keywords.
+ */
+function extractRecentTopic(conversationContext: string): string | null {
+  // Split on common conversation entry patterns
+  const entries = conversationContext
+    .split(/\n\*\*\w+:\*\*\s*/g)
+    .filter((e) => e.trim().length > 10);
+
+  if (entries.length === 0) return null;
+
+  // Take the last 1-2 entries (most recent conversation)
+  const recent = entries.slice(-2).join(" ");
+  const keywords = extractKeywords(recent);
+
+  if (keywords.length < 2) return null;
+
+  // Build a focused query from the recent conversation keywords
+  return keywords.slice(0, 6).join(" ");
+}