@disco_trooper/apple-notes-mcp 1.0.0

package/src/search/index.ts

@@ -0,0 +1,283 @@
+/**
+ * Hybrid search combining vector similarity and full-text search.
+ *
+ * Uses Reciprocal Rank Fusion (RRF) to merge results from:
+ * - Vector search (semantic similarity)
+ * - Full-text search (keyword matching)
+ */
+
+import { getEmbedding } from "../embeddings/index.js";
+import { getVectorStore } from "../db/lancedb.js";
+import type { DBSearchResult, SearchResult } from "../types/index.js";
+import { DEFAULT_SEARCH_LIMIT, FOLDER_FILTER_MULTIPLIER, HYBRID_SEARCH_MIN_FETCH, PREVIEW_LENGTH, PREVIEW_TRUNCATE_RATIO, RRF_K } from "../config/constants.js";
+import { createDebugLogger } from "../utils/debug.js";
+
+// Debug logging
+const debug = createDebugLogger("SEARCH");
+
+/**
+ * Search mode options.
+ * - hybrid: Combine vector + FTS with RRF (default)
+ * - keyword: Full-text search only
+ * - semantic: Vector search only
+ */
+export type SearchMode = "hybrid" | "keyword" | "semantic";
+
+/**
+ * Options for search operations.
+ */
+export interface SearchOptions {
+  /** Filter by folder name */
+  folder?: string;
+  /** Maximum number of results (default: 20) */
+  limit?: number;
+  /** Search mode: hybrid, keyword, or semantic (default: hybrid) */
+  mode?: SearchMode;
+  /** Include full content instead of preview (default: false) */
+  include_content?: boolean;
+}
+
+// SearchResult is imported from ../types/index.js
+// RRF_K is imported from ../config/constants.js
+
+/**
+ * Calculate RRF score for a result at a given rank.
+ * Formula: 1 / (k + rank)
+ * where k is a constant (typically 60) and rank is 0-indexed.
+ */
+function rrfScore(rank: number): number {
+  return 1 / (RRF_K + rank);
+}
+
+/**
+ * Generate a preview of content (first N characters, default from PREVIEW_LENGTH constant).
+ */
+function generatePreview(content: string, maxLength = PREVIEW_LENGTH): string {
+  if (!content) {
+    return "";
+  }
+
+  // Clean up whitespace
+  const cleaned = content.replace(/\s+/g, " ").trim();
+
+  if (cleaned.length <= maxLength) {
+    return cleaned;
+  }
+
+  // Truncate at word boundary
+  const truncated = cleaned.slice(0, maxLength);
+  const lastSpace = truncated.lastIndexOf(" ");
+
+  if (lastSpace > maxLength * PREVIEW_TRUNCATE_RATIO) {
+    return truncated.slice(0, lastSpace) + "...";
+  }
+
+  return truncated + "...";
+}
+
+/**
+ * Filter results by folder if specified.
+ */
+function filterByFolder<T extends { folder: string }>(
+  results: T[],
+  folder?: string
+): T[] {
+  if (!folder) {
+    return results;
+  }
+
+  const normalizedFolder = folder.toLowerCase();
+  return results.filter(
+    (r) => r.folder.toLowerCase() === normalizedFolder
+  );
+}
+
+/**
+ * Perform vector-only search.
+ */
+async function vectorSearch(
+  query: string,
+  limit: number,
+  folder?: string
+): Promise<DBSearchResult[]> {
+  debug(`Vector search: "${query}" (limit: ${limit})`);
+
+  const store = getVectorStore();
+  const queryVector = await getEmbedding(query);
+
+  // Fetch more results than needed if filtering by folder
+  const fetchLimit = folder ? limit * FOLDER_FILTER_MULTIPLIER : limit;
+  const results = await store.search(queryVector, fetchLimit);
+
+  const filtered = filterByFolder(results, folder);
+  return filtered.slice(0, limit);
+}
+
+/**
+ * Perform full-text search only.
+ */
+async function keywordSearch(
+  query: string,
+  limit: number,
+  folder?: string
+): Promise<DBSearchResult[]> {
+  debug(`FTS search: "${query}" (limit: ${limit})`);
+
+  const store = getVectorStore();
+
+  // Fetch more results than needed if filtering by folder
+  const fetchLimit = folder ? limit * FOLDER_FILTER_MULTIPLIER : limit;
+  const results = await store.searchFTS(query, fetchLimit);
+
+  const filtered = filterByFolder(results, folder);
+  return filtered.slice(0, limit);
+}
+
+/**
+ * Perform hybrid search combining vector and FTS results using RRF.
+ */
+async function hybridSearch(
+  query: string,
+  limit: number,
+  folder?: string
+): Promise<DBSearchResult[]> {
+  debug(`Hybrid search: "${query}" (limit: ${limit})`);
+
+  const store = getVectorStore();
+
+  // Fetch more results for RRF merging
+  const fetchLimit = Math.max(limit * 2, HYBRID_SEARCH_MIN_FETCH);
+
+  // Run both searches in parallel
+  const [queryVector, ftsResults] = await Promise.all([
+    getEmbedding(query),
+    store.searchFTS(query, fetchLimit).catch(() => [] as DBSearchResult[]),
+  ]);
+
+  const vectorResults = await store.search(queryVector, fetchLimit);
+
+  debug(`Vector results: ${vectorResults.length}, FTS results: ${ftsResults.length}`);
+
+  // Merge results using Reciprocal Rank Fusion
+  const scoreMap = new Map<string, number>();
+  const contentMap = new Map<string, DBSearchResult>();
+
+  // Process vector search results
+  vectorResults.forEach((item, rank) => {
+    const key = item.title;
+    scoreMap.set(key, (scoreMap.get(key) || 0) + rrfScore(rank));
+    contentMap.set(key, item);
+  });
+
+  // Process FTS results
+  ftsResults.forEach((item, rank) => {
+    const key = item.title;
+    scoreMap.set(key, (scoreMap.get(key) || 0) + rrfScore(rank));
+    if (!contentMap.has(key)) {
+      contentMap.set(key, item);
+    }
+  });
+
+  // Sort by combined RRF score
+  const merged = Array.from(scoreMap.entries())
+    .sort((a, b) => b[1] - a[1])
+    .map(([key, score]) => ({
+      ...contentMap.get(key)!,
+      score,
+    }));
+
+  // Apply folder filter and limit
+  const filtered = filterByFolder(merged, folder);
+  return filtered.slice(0, limit);
+}
+
+/**
+ * Search notes with configurable search mode.
+ *
+ * @param query - Search query string
+ * @param options - Search configuration options
+ * @returns Array of search results sorted by relevance
+ *
+ * @example
+ * ```typescript
+ * // Basic hybrid search
+ * const results = await searchNotes("project ideas");
+ *
+ * // Keyword-only search in specific folder
+ * const results = await searchNotes("meeting", {
+ *   mode: "keyword",
+ *   folder: "Work",
+ *   limit: 10,
+ * });
+ *
+ * // Semantic search with full content
+ * const results = await searchNotes("concepts similar to machine learning", {
+ *   mode: "semantic",
+ *   include_content: true,
+ * });
+ * ```
+ */
+export async function searchNotes(
+  query: string,
+  options: SearchOptions = {}
+): Promise<SearchResult[]> {
+  const {
+    folder,
+    limit = DEFAULT_SEARCH_LIMIT,
+    mode = "hybrid",
+    include_content = false,
+  } = options;
+
+  if (!query || query.trim().length === 0) {
+    debug("Empty query, returning empty results");
+    return [];
+  }
+
+  const trimmedQuery = query.trim();
+
+  debug(`searchNotes: "${trimmedQuery}" mode=${mode} folder=${folder || "all"} limit=${limit}`);
+
+  let dbResults: DBSearchResult[];
+
+  switch (mode) {
+    case "keyword":
+      dbResults = await keywordSearch(trimmedQuery, limit, folder);
+      break;
+
+    case "semantic":
+      dbResults = await vectorSearch(trimmedQuery, limit, folder);
+      break;
+
+    case "hybrid":
+    default:
+      dbResults = await hybridSearch(trimmedQuery, limit, folder);
+      break;
+  }
+
+  // Transform to SearchResult format
+  const results: SearchResult[] = dbResults.map((r) => {
+    const result: SearchResult = {
+      title: r.title,
+      folder: r.folder,
+      preview: generatePreview(r.content),
+      modified: r.modified,
+      score: r.score,
+    };
+
+    if (include_content) {
+      result.content = r.content;
+    }
+
+    return result;
+  });
+
+  debug(`Returning ${results.length} results`);
+  return results;
+}
+
+// Re-export types for convenience
+export type { SearchMode as Mode };
+export type { SearchResult } from "../types/index.js";
+
+// Export utility functions for testing
+export { rrfScore, generatePreview, filterByFolder };

package/src/search/indexer.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect } from "vitest";
+import { extractTitleFromKey } from "./indexer.js";
+
+describe("extractTitleFromKey", () => {
+  it("extracts title from simple folder/title key", () => {
+    expect(extractTitleFromKey("Work/My Note")).toBe("My Note");
+  });
+
+  it("extracts title from nested folder key", () => {
+    expect(extractTitleFromKey("Work/Projects/My Note")).toBe("My Note");
+  });
+
+  it("extracts title from deeply nested folder key", () => {
+    expect(extractTitleFromKey("Personal/Archive/2024/January/Meeting Notes")).toBe("Meeting Notes");
+  });
+
+  it("handles single segment (no folder)", () => {
+    expect(extractTitleFromKey("My Note")).toBe("My Note");
+  });
+});
+
+describe("delete key parsing", () => {
+  it("should correctly parse folder and title from key", () => {
+    const key = "Work/Projects/My Note";
+    const lastSlash = key.lastIndexOf("/");
+    const folder = key.substring(0, lastSlash);
+    const title = key.substring(lastSlash + 1);
+
+    expect(folder).toBe("Work/Projects");
+    expect(title).toBe("My Note");
+  });
+
+  it("should handle simple folder/title", () => {
+    const key = "Personal/My Note";
+    const lastSlash = key.lastIndexOf("/");
+    const folder = key.substring(0, lastSlash);
+    const title = key.substring(lastSlash + 1);
+
+    expect(folder).toBe("Personal");
+    expect(title).toBe("My Note");
+  });
+});

package/src/search/indexer.ts

@@ -0,0 +1,335 @@
+/**
+ * Note indexing pipeline for semantic search.
+ *
+ * Supports:
+ * - Full reindexing (drop all, reindex everything)
+ * - Incremental indexing (only changed notes)
+ * - Single note reindexing
+ */
+
+import { getEmbedding } from "../embeddings/index.js";
+import { getVectorStore, type NoteRecord } from "../db/lancedb.js";
+import { getAllNotes, getNoteByFolderAndTitle, getNoteByTitle, type NoteInfo } from "../notes/read.js";
+import { createDebugLogger } from "../utils/debug.js";
+import { truncateForEmbedding } from "../utils/text.js";
+import { EMBEDDING_DELAY_MS } from "../config/constants.js";
+
+/**
+ * Extract note title from folder/title key.
+ * Handles nested folders correctly by taking the last segment.
+ */
+export function extractTitleFromKey(key: string): string {
+  const parts = key.split("/");
+  return parts[parts.length - 1];
+}
+
+// Debug logging
+const debug = createDebugLogger("INDEX");
+
+/**
+ * Result of an indexing operation.
+ */
+export interface IndexResult {
+  /** Total notes processed */
+  total: number;
+  /** Notes successfully indexed */
+  indexed: number;
+  /** Notes that failed to index */
+  errors: number;
+  /** Time taken in milliseconds */
+  timeMs: number;
+  /** Breakdown for incremental indexing */
+  breakdown?: {
+    added: number;
+    updated: number;
+    deleted: number;
+    skipped: number;
+  };
+  /** List of notes that failed to index (for debugging) */
+  failedNotes?: string[];
+}
+
+/**
+ * Sleep for a specified duration.
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Perform full reindexing of all notes.
+ * Drops existing index and rebuilds from scratch.
+ */
+export async function fullIndex(): Promise<IndexResult> {
+  const startTime = Date.now();
+  debug("Starting full index...");
+
+  // Get all notes from Apple Notes
+  const notes = await getAllNotes();
+  debug(`Found ${notes.length} notes in Apple Notes`);
+
+  const records: NoteRecord[] = [];
+  let errors = 0;
+  const failedNotes: string[] = [];
+
+  for (let i = 0; i < notes.length; i++) {
+    const noteInfo = notes[i];
+    debug(`Processing ${i + 1}/${notes.length}: ${noteInfo.title}`);
+
+    try {
+      // Get full note content using folder and title separately
+      // to handle notes with "/" in their titles
+      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
+      if (!noteDetails) {
+        debug(`Could not fetch note: ${noteInfo.title}`);
+        failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+        errors++;
+        continue;
+      }
+
+      // Skip empty notes
+      if (!noteDetails.content.trim()) {
+        debug(`Skipping empty note: ${noteInfo.title}`);
+        continue;
+      }
+
+      // Generate embedding
+      const content = truncateForEmbedding(noteDetails.content);
+      const vector = await getEmbedding(content);
+
+      const record: NoteRecord = {
+        title: noteDetails.title,
+        content: noteDetails.content,
+        vector,
+        folder: noteDetails.folder,
+        created: noteDetails.created,
+        modified: noteDetails.modified,
+        indexed_at: new Date().toISOString(),
+      };
+
+      records.push(record);
+
+      // Delay to avoid rate limiting
+      if (i < notes.length - 1) {
+        await sleep(EMBEDDING_DELAY_MS);
+      }
+    } catch (error) {
+      debug(`Error processing ${noteInfo.title}:`, error);
+      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+      errors++;
+    }
+  }
+
+  // Store all records in vector database
+  const store = getVectorStore();
+  await store.index(records);
+
+  const timeMs = Date.now() - startTime;
+  debug(`Full index complete: ${records.length} indexed, ${errors} errors, ${timeMs}ms`);
+
+  return {
+    total: notes.length,
+    indexed: records.length,
+    errors,
+    timeMs,
+    failedNotes: failedNotes.length > 0 ? failedNotes : undefined,
+  };
+}
+
+/**
+ * Perform incremental indexing.
+ * Only processes notes that have changed since last index.
+ */
+export async function incrementalIndex(): Promise<IndexResult> {
+  const startTime = Date.now();
+  debug("Starting incremental index...");
+
+  const store = getVectorStore();
+
+  // Get all notes from Apple Notes
+  const currentNotes = await getAllNotes();
+  debug(`Found ${currentNotes.length} notes in Apple Notes`);
+
+  // Get existing indexed notes
+  let existingRecords: NoteRecord[];
+  try {
+    existingRecords = await store.getAll();
+  } catch (error) {
+    // No existing index, fall back to full index
+    debug("No existing index found, performing full index. Error:", error);
+    return fullIndex();
+  }
+
+  // Build lookup maps
+  const existingByTitle = new Map<string, NoteRecord>();
+  for (const record of existingRecords) {
+    const key = `${record.folder}/${record.title}`;
+    existingByTitle.set(key, record);
+  }
+
+  const currentByTitle = new Map<string, NoteInfo>();
+  for (const note of currentNotes) {
+    const key = `${note.folder}/${note.title}`;
+    currentByTitle.set(key, note);
+  }
+
+  // Determine what needs to be done
+  const toAdd: NoteInfo[] = [];
+  const toUpdate: NoteInfo[] = [];
+  const toDelete: string[] = [];
+  const toSkip: string[] = [];
+
+  // Check current notes
+  for (const note of currentNotes) {
+    const key = `${note.folder}/${note.title}`;
+    const existing = existingByTitle.get(key);
+
+    if (!existing) {
+      toAdd.push(note);
+    } else {
+      // Compare modified date with indexed_at timestamp
+      const currentModified = new Date(note.modified).getTime();
+      const indexedAt = new Date(existing.indexed_at).getTime();
+
+      if (currentModified > indexedAt) {
+        toUpdate.push(note);
+      } else {
+        toSkip.push(key);
+      }
+    }
+  }
+
+  // Check for deleted notes
+  for (const [key] of existingByTitle) {
+    if (!currentByTitle.has(key)) {
+      toDelete.push(key);
+    }
+  }
+
+  debug(`Incremental: add=${toAdd.length}, update=${toUpdate.length}, delete=${toDelete.length}, skip=${toSkip.length}`);
+
+  let errors = 0;
+  const failedNotes: string[] = [];
+
+  // Process additions and updates
+  const toProcess = [...toAdd, ...toUpdate];
+  for (let i = 0; i < toProcess.length; i++) {
+    const noteInfo = toProcess[i];
+    debug(`Processing ${i + 1}/${toProcess.length}: ${noteInfo.title}`);
+
+    try {
+      // Use folder and title separately to handle "/" in titles
+      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
+      if (!noteDetails) {
+        failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+        errors++;
+        continue;
+      }
+
+      if (!noteDetails.content.trim()) {
+        continue;
+      }
+
+      const content = truncateForEmbedding(noteDetails.content);
+      const vector = await getEmbedding(content);
+
+      const record: NoteRecord = {
+        title: noteDetails.title,
+        content: noteDetails.content,
+        vector,
+        folder: noteDetails.folder,
+        created: noteDetails.created,
+        modified: noteDetails.modified,
+        indexed_at: new Date().toISOString(),
+      };
+
+      await store.update(record);
+
+      if (i < toProcess.length - 1) {
+        await sleep(EMBEDDING_DELAY_MS);
+      }
+    } catch (error) {
+      debug(`Error processing ${noteInfo.title}:`, error);
+      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+      errors++;
+    }
+  }
+
+  // Process deletions
+  for (const key of toDelete) {
+    try {
+      // Parse folder and title from key (e.g., "Work/Projects/My Note")
+      const lastSlash = key.lastIndexOf("/");
+      const folder = key.substring(0, lastSlash);
+      const title = key.substring(lastSlash + 1);
+      await store.deleteByFolderAndTitle(folder, title);
+    } catch (error) {
+      debug(`Error deleting ${key}:`, error);
+      failedNotes.push(`DELETE: ${key}`);
+      errors++;
+    }
+  }
+
+  const timeMs = Date.now() - startTime;
+  debug(`Incremental index complete: ${timeMs}ms`);
+
+  return {
+    total: currentNotes.length,
+    indexed: toAdd.length + toUpdate.length,
+    errors,
+    timeMs,
+    breakdown: {
+      added: toAdd.length,
+      updated: toUpdate.length,
+      deleted: toDelete.length,
+      skipped: toSkip.length,
+    },
+    failedNotes: failedNotes.length > 0 ? failedNotes : undefined,
+  };
+}
+
+/**
+ * Reindex a single note by title.
+ */
+export async function reindexNote(title: string): Promise<void> {
+  debug(`Reindexing single note: ${title}`);
+
+  const noteDetails = await getNoteByTitle(title);
+  if (!noteDetails) {
+    throw new Error(`Note not found: "${title}"`);
+  }
+
+  if (!noteDetails.content.trim()) {
+    throw new Error(`Note is empty: "${title}"`);
+  }
+
+  const content = truncateForEmbedding(noteDetails.content);
+  const vector = await getEmbedding(content);
+
+  const record: NoteRecord = {
+    title: noteDetails.title,
+    content: noteDetails.content,
+    vector,
+    folder: noteDetails.folder,
+    created: noteDetails.created,
+    modified: noteDetails.modified,
+    indexed_at: new Date().toISOString(),
+  };
+
+  const store = getVectorStore();
+  await store.update(record);
+
+  debug(`Reindexed: ${title}`);
+}
+
+/**
+ * Index notes based on mode.
+ */
+export async function indexNotes(
+  mode: "full" | "incremental" = "incremental"
+): Promise<IndexResult> {
+  if (mode === "full") {
+    return fullIndex();
+  }
+  return incrementalIndex();
+}