@disco_trooper/apple-notes-mcp 1.2.0 → 1.3.0

package/src/search/chunk-indexer.test.ts

@@ -0,0 +1,353 @@
+import { describe, it, expect, vi, beforeEach, type Mock } from "vitest";
+import type { NoteDetails } from "../notes/read.js";
+import type { ChunkRecord } from "../db/lancedb.js";
+
+// Mock dependencies before importing the module under test
+vi.mock("../embeddings/index.js", () => ({
+  getEmbeddingBatch: vi.fn(),
+}));
+
+vi.mock("../db/lancedb.js", () => ({
+  getChunkStore: vi.fn(() => ({
+    indexChunks: vi.fn(),
+    count: vi.fn().mockResolvedValue(0),
+  })),
+}));
+
+vi.mock("../notes/read.js", () => ({
+  getAllNotesWithContent: vi.fn(),
+}));
+
+vi.mock("../utils/debug.js", () => ({
+  createDebugLogger: vi.fn(() => vi.fn()),
+}));
+
+// Import after mocking
+import { chunkNote, fullChunkIndex, hasChunkIndex } from "./chunk-indexer.js";
+import { getEmbeddingBatch } from "../embeddings/index.js";
+import { getChunkStore } from "../db/lancedb.js";
+import { getAllNotesWithContent } from "../notes/read.js";
+
+describe("chunk-indexer", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe("chunkNote", () => {
+    // Note: content must be at least 50 chars to pass the content filter
+    it("creates chunks for a note with content", () => {
+      const content = "This is a test note content that is long enough to pass the minimum content length requirement for indexing.";
+      const note: NoteDetails = {
+        id: "note-123",
+        title: "Test Note",
+        folder: "Work",
+        content,
+        htmlContent: `<p>${content}</p>`,
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-02T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]).toMatchObject({
+        chunk_id: "note-123_chunk_0",
+        note_id: "note-123",
+        note_title: "Test Note",
+        folder: "Work",
+        chunk_index: 0,
+        total_chunks: 1,
+        content,
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-02T00:00:00.000Z",
+      });
+      // Vector should be empty - not generated yet
+      expect(chunks[0].vector).toEqual([]);
+      // indexed_at should be empty - set during batch processing
+      expect(chunks[0].indexed_at).toBe("");
+      // Tags and outlinks should be extracted
+      expect(chunks[0].tags).toEqual([]);
+      expect(chunks[0].outlinks).toEqual([]);
+    });
+
+    it("returns single chunk for notes under chunk size", () => {
+      const note: NoteDetails = {
+        id: "short-note",
+        title: "Short",
+        folder: "Notes",
+        content: "This is a shorter note but still has enough content to pass the minimum length filter requirement.",
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0].chunk_index).toBe(0);
+      expect(chunks[0].total_chunks).toBe(1);
+    });
+
+    it("returns empty array for empty notes", () => {
+      const note: NoteDetails = {
+        id: "empty-note",
+        title: "Empty",
+        folder: "Notes",
+        content: "",
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(0);
+    });
+
+    it("returns empty array for whitespace-only notes", () => {
+      const note: NoteDetails = {
+        id: "whitespace-note",
+        title: "Whitespace",
+        folder: "Notes",
+        content: "   \n\n   ",
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(0);
+    });
+
+    it("extracts tags from note content", () => {
+      const note: NoteDetails = {
+        id: "tagged-note",
+        title: "Tagged Note",
+        folder: "Work",
+        content: "This note has #important and #work tags. It also contains enough text to pass the minimum content length requirement.",
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0].tags).toContain("important");
+      expect(chunks[0].tags).toContain("work");
+    });
+
+    it("extracts outlinks from note content", () => {
+      const note: NoteDetails = {
+        id: "linked-note",
+        title: "Linked Note",
+        folder: "Work",
+        content: "This links to [[Other Note]] and [[Another Note]]. This is additional content to meet the minimum length requirement.",
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0].outlinks).toContain("Other Note");
+      expect(chunks[0].outlinks).toContain("Another Note");
+    });
+
+    it("creates multiple chunks for long notes", () => {
+      // Create a long note that will produce multiple chunks
+      const longContent = "This is paragraph one. ".repeat(50) + "\n\n" +
+                          "This is paragraph two. ".repeat(50);
+      const note: NoteDetails = {
+        id: "long-note",
+        title: "Long Note",
+        folder: "Work",
+        content: longContent,
+        htmlContent: "",
+        created: "2024-01-01T00:00:00.000Z",
+        modified: "2024-01-01T00:00:00.000Z",
+      };
+
+      const chunks = chunkNote(note);
+
+      expect(chunks.length).toBeGreaterThan(1);
+      // Verify chunk IDs are unique
+      const chunkIds = chunks.map(c => c.chunk_id);
+      expect(new Set(chunkIds).size).toBe(chunks.length);
+      // Verify indices are correct
+      chunks.forEach((chunk, i) => {
+        expect(chunk.chunk_index).toBe(i);
+        expect(chunk.total_chunks).toBe(chunks.length);
+        expect(chunk.chunk_id).toBe(`long-note_chunk_${i}`);
+      });
+    });
+  });
+
+  describe("fullChunkIndex", () => {
+    it("processes notes and creates chunks with embeddings", async () => {
+      // Note: content must be at least 50 chars to pass content filter
+      const mockNotes: NoteDetails[] = [
+        {
+          id: "note-1",
+          title: "Note 1",
+          folder: "Work",
+          content: "This is the first note content with enough text to pass the minimum length filter requirement.",
+          htmlContent: "",
+          created: "2024-01-01T00:00:00.000Z",
+          modified: "2024-01-02T00:00:00.000Z",
+        },
+        {
+          id: "note-2",
+          title: "Note 2",
+          folder: "Personal",
+          content: "This is the second note content with enough text to pass the minimum length filter requirement.",
+          htmlContent: "",
+          created: "2024-01-01T00:00:00.000Z",
+          modified: "2024-01-02T00:00:00.000Z",
+        },
+      ];
+
+      const mockVectors = [
+        [0.1, 0.2, 0.3],
+        [0.4, 0.5, 0.6],
+      ];
+
+      const mockIndexChunks = vi.fn();
+      (getAllNotesWithContent as Mock).mockResolvedValue(mockNotes);
+      (getEmbeddingBatch as Mock).mockResolvedValue(mockVectors);
+      (getChunkStore as Mock).mockReturnValue({
+        indexChunks: mockIndexChunks,
+        count: vi.fn().mockResolvedValue(0),
+      });
+
+      const result = await fullChunkIndex();
+
+      // Verify all notes were fetched
+      expect(getAllNotesWithContent).toHaveBeenCalledOnce();
+
+      // Verify embeddings were generated for chunks
+      expect(getEmbeddingBatch).toHaveBeenCalledOnce();
+      const embeddingTexts = (getEmbeddingBatch as Mock).mock.calls[0][0];
+      expect(embeddingTexts).toHaveLength(2);
+
+      // Verify chunks were stored with vectors
+      expect(mockIndexChunks).toHaveBeenCalledOnce();
+      const storedChunks = mockIndexChunks.mock.calls[0][0] as ChunkRecord[];
+      expect(storedChunks).toHaveLength(2);
+      expect(storedChunks[0].vector).toEqual([0.1, 0.2, 0.3]);
+      expect(storedChunks[1].vector).toEqual([0.4, 0.5, 0.6]);
+
+      // Verify indexed_at is set
+      storedChunks.forEach(chunk => {
+        expect(chunk.indexed_at).toBeTruthy();
+        // Should be valid ISO date
+        expect(new Date(chunk.indexed_at).toISOString()).toBe(chunk.indexed_at);
+      });
+
+      // Verify result
+      expect(result).toMatchObject({
+        totalNotes: 2,
+        totalChunks: 2,
+        indexed: 2,
+      });
+      expect(result.timeMs).toBeGreaterThanOrEqual(0);
+    });
+
+    it("handles empty note list", async () => {
+      (getAllNotesWithContent as Mock).mockResolvedValue([]);
+      const mockIndexChunks = vi.fn();
+      (getChunkStore as Mock).mockReturnValue({
+        indexChunks: mockIndexChunks,
+        count: vi.fn().mockResolvedValue(0),
+      });
+
+      const result = await fullChunkIndex();
+
+      expect(result).toMatchObject({
+        totalNotes: 0,
+        totalChunks: 0,
+        indexed: 0,
+      });
+      // Should not call embedding or indexing for empty list
+      expect(getEmbeddingBatch).not.toHaveBeenCalled();
+      expect(mockIndexChunks).not.toHaveBeenCalled();
+    });
+
+    it("skips empty notes in chunking", async () => {
+      // Note: content must be at least 50 chars to pass content filter
+      const mockNotes: NoteDetails[] = [
+        {
+          id: "note-1",
+          title: "Note 1",
+          folder: "Work",
+          content: "This is valid note content with enough characters to pass the minimum length requirement for indexing.",
+          htmlContent: "",
+          created: "2024-01-01T00:00:00.000Z",
+          modified: "2024-01-02T00:00:00.000Z",
+        },
+        {
+          id: "note-2",
+          title: "Empty Note",
+          folder: "Work",
+          content: "", // Empty!
+          htmlContent: "",
+          created: "2024-01-01T00:00:00.000Z",
+          modified: "2024-01-02T00:00:00.000Z",
+        },
+      ];
+
+      const mockIndexChunks = vi.fn();
+      (getAllNotesWithContent as Mock).mockResolvedValue(mockNotes);
+      (getEmbeddingBatch as Mock).mockResolvedValue([[0.1, 0.2, 0.3]]);
+      (getChunkStore as Mock).mockReturnValue({
+        indexChunks: mockIndexChunks,
+        count: vi.fn().mockResolvedValue(0),
+      });
+
+      const result = await fullChunkIndex();
+
+      // Only 1 chunk should be created (empty note skipped)
+      expect(result.totalNotes).toBe(2);
+      expect(result.totalChunks).toBe(1);
+      expect(result.indexed).toBe(1);
+
+      const storedChunks = mockIndexChunks.mock.calls[0][0] as ChunkRecord[];
+      expect(storedChunks).toHaveLength(1);
+      expect(storedChunks[0].note_title).toBe("Note 1");
+    });
+  });
+
+  describe("hasChunkIndex", () => {
+    it("returns true when chunk index exists", async () => {
+      (getChunkStore as Mock).mockReturnValue({
+        count: vi.fn().mockResolvedValue(10),
+      });
+
+      const result = await hasChunkIndex();
+
+      expect(result).toBe(true);
+    });
+
+    it("returns false when chunk index is empty", async () => {
+      (getChunkStore as Mock).mockReturnValue({
+        count: vi.fn().mockResolvedValue(0),
+      });
+
+      const result = await hasChunkIndex();
+
+      expect(result).toBe(false);
+    });
+
+    it("returns false when chunk store throws (table not found)", async () => {
+      (getChunkStore as Mock).mockReturnValue({
+        count: vi.fn().mockRejectedValue(new Error("Chunk index not found")),
+      });
+
+      const result = await hasChunkIndex();
+
+      expect(result).toBe(false);
+    });
+  });
+});

package/src/search/chunk-indexer.ts

@@ -0,0 +1,207 @@
+/**
+ * Chunk indexer for Parent Document Retriever pattern.
+ * Splits notes into overlapping chunks, generates embeddings, and stores in LanceDB.
+ */
+
+import { getEmbeddingBatch } from "../embeddings/index.js";
+import { getChunkStore, type ChunkRecord } from "../db/lancedb.js";
+import { getAllNotesWithContent, type NoteDetails } from "../notes/read.js";
+import { chunkText } from "../utils/chunker.js";
+import { extractMetadata } from "../graph/extract.js";
+import { DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP } from "../config/constants.js";
+import { createDebugLogger } from "../utils/debug.js";
+import { filterContent, shouldIndexContent } from "../utils/content-filter.js";
+
+// Debug logging
+const debug = createDebugLogger("CHUNK-INDEXER");
+
+/**
+ * Result of a chunk indexing operation.
+ */
+export interface ChunkIndexResult {
+  /** Total number of notes processed */
+  totalNotes: number;
+  /** Total number of chunks created */
+  totalChunks: number;
+  /** Number of chunks indexed (with embeddings) */
+  indexed: number;
+  /** Time taken in milliseconds */
+  timeMs: number;
+}
+
+/** Chunk record for internal processing - explicit types to avoid index signature issues */
+interface InternalChunkRecord {
+  chunk_id: string;
+  note_id: string;
+  note_title: string;
+  folder: string;
+  chunk_index: number;
+  total_chunks: number;
+  content: string;
+  vector: number[];
+  created: string;
+  modified: string;
+  indexed_at: string;
+  tags: string[];
+  outlinks: string[];
+}
+
+/**
+ * Convert a note into chunk records WITHOUT vectors.
+ * Vectors are added later during batch embedding generation.
+ *
+ * Filters out Base64/binary content before chunking to improve search quality.
+ *
+ * @param note - The note to chunk
+ * @returns Array of ChunkRecord with empty vectors
+ */
+export function chunkNote(note: NoteDetails): InternalChunkRecord[] {
+  // Quick check - skip notes with mostly encoded content
+  if (!shouldIndexContent(note.content)) {
+    debug(`Note "${note.title}" skipped: contains mostly encoded/binary content`);
+    return [];
+  }
+
+  // Filter content to remove Base64 blocks and redact secrets
+  const filterResult = filterContent(note.content);
+
+  if (filterResult.action === "skip") {
+    debug(`Note "${note.title}" skipped: ${filterResult.reasons.join(", ")}`);
+    return [];
+  }
+
+  const contentToChunk = filterResult.cleanedContent || note.content;
+
+  if (filterResult.action === "filter") {
+    debug(`Note "${note.title}" filtered: ${filterResult.reasons.join(", ")}`);
+  }
+
+  // Extract metadata from the ORIGINAL content (tags/links should be preserved)
+  const { tags, outlinks } = extractMetadata(note.content);
+
+  // Chunk the filtered content
+  const chunks = chunkText(contentToChunk, {
+    chunkSize: DEFAULT_CHUNK_SIZE,
+    overlap: DEFAULT_CHUNK_OVERLAP,
+  });
+
+  // Return empty array for empty notes (chunkText handles this)
+  if (chunks.length === 0) {
+    debug(`Note "${note.title}" has no content to chunk`);
+    return [];
+  }
+
+  debug(`Note "${note.title}" chunked into ${chunks.length} chunks`);
+
+  // Convert to ChunkRecord format
+  return chunks.map((chunk) => ({
+    chunk_id: `${note.id}_chunk_${chunk.index}`,
+    note_id: note.id,
+    note_title: note.title,
+    folder: note.folder,
+    chunk_index: chunk.index,
+    total_chunks: chunk.totalChunks,
+    content: chunk.content,
+    vector: [], // Empty - to be filled during embedding generation
+    created: note.created,
+    modified: note.modified,
+    indexed_at: "", // Empty - to be set during batch processing
+    tags,
+    outlinks,
+  }));
+}
+
+/**
+ * Perform a full chunk index of all notes.
+ *
+ * Phases:
+ * 1. Fetch all notes via getAllNotesWithContent
+ * 2. Chunk all notes using chunkNote
+ * 3. Generate embeddings in batch using getEmbeddingBatch
+ * 4. Combine chunks with vectors and set indexed_at
+ * 5. Store via getChunkStore().indexChunks()
+ *
+ * @returns ChunkIndexResult with stats
+ */
+export async function fullChunkIndex(): Promise<ChunkIndexResult> {
+  const startTime = Date.now();
+
+  // Phase 1: Fetch all notes
+  debug("Phase 1: Fetching all notes...");
+  const notes = await getAllNotesWithContent();
+  debug(`Fetched ${notes.length} notes`);
+
+  if (notes.length === 0) {
+    return {
+      totalNotes: 0,
+      totalChunks: 0,
+      indexed: 0,
+      timeMs: Date.now() - startTime,
+    };
+  }
+
+  // Phase 2: Chunk all notes
+  debug("Phase 2: Chunking all notes...");
+  const allChunks: InternalChunkRecord[] = [];
+  for (const note of notes) {
+    const noteChunks = chunkNote(note);
+    allChunks.push(...noteChunks);
+  }
+  debug(`Created ${allChunks.length} chunks from ${notes.length} notes`);
+
+  if (allChunks.length === 0) {
+    return {
+      totalNotes: notes.length,
+      totalChunks: 0,
+      indexed: 0,
+      timeMs: Date.now() - startTime,
+    };
+  }
+
+  // Phase 3: Generate embeddings in batch
+  debug("Phase 3: Generating embeddings...");
+  const chunkTexts: string[] = allChunks.map((chunk) => chunk.content);
+  const vectors = await getEmbeddingBatch(chunkTexts);
+  debug(`Generated ${vectors.length} embeddings`);
+
+  // Phase 4: Combine chunks with vectors and set indexed_at
+  debug("Phase 4: Combining chunks with vectors...");
+  const indexedAt = new Date().toISOString();
+  const completeChunks: ChunkRecord[] = allChunks.map((chunk, i) => ({
+    ...chunk,
+    vector: vectors[i],
+    indexed_at: indexedAt,
+  }));
+
+  // Phase 5: Store in LanceDB
+  debug("Phase 5: Storing chunks...");
+  const chunkStore = getChunkStore();
+  await chunkStore.indexChunks(completeChunks);
+  debug(`Stored ${completeChunks.length} chunks`);
+
+  const timeMs = Date.now() - startTime;
+  debug(`Chunk indexing completed in ${timeMs}ms`);
+
+  return {
+    totalNotes: notes.length,
+    totalChunks: allChunks.length,
+    indexed: completeChunks.length,
+    timeMs,
+  };
+}
+
+/**
+ * Check if a chunk index exists.
+ *
+ * @returns true if chunk index has records, false otherwise
+ */
+export async function hasChunkIndex(): Promise<boolean> {
+  try {
+    const chunkStore = getChunkStore();
+    const count = await chunkStore.count();
+    return count > 0;
+  } catch {
+    // Table doesn't exist or error - no index
+    return false;
+  }
+}