@disco_trooper/apple-notes-mcp 1.1.0 → 1.3.0

package/src/search/indexer.ts

@@ -7,20 +7,20 @@
  * - Single note reindexing
  */
 
-import { getEmbedding } from "../embeddings/index.js";
+import { getEmbedding, getEmbeddingBatch } from "../embeddings/index.js";
 import { getVectorStore, type NoteRecord } from "../db/lancedb.js";
-import { getAllNotes, getNoteByFolderAndTitle, getNoteByTitle, type NoteInfo } from "../notes/read.js";
+import { getAllNotes, getAllNotesWithContent, getNoteByFolderAndTitle, getNoteByTitle, type NoteInfo } from "../notes/read.js";
 import { createDebugLogger } from "../utils/debug.js";
 import { truncateForEmbedding } from "../utils/text.js";
-import { EMBEDDING_DELAY_MS } from "../config/constants.js";
+import { NoteNotFoundError } from "../errors/index.js";
+import { extractMetadata } from "../graph/extract.js";
 
 /**
  * Extract note title from folder/title key.
  * Handles nested folders correctly by taking the last segment.
  */
 export function extractTitleFromKey(key: string): string {
-  const parts = key.split("/");
-  return parts[parts.length - 1];
+  return key.split("/").at(-1) ?? key;
 }
 
 // Debug logging
@@ -50,89 +50,127 @@ export interface IndexResult {
 }
 
 /**
- * Sleep for a specified duration.
+ * Note data prepared for embedding.
  */
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+interface PreparedNote {
+  id: string;
+  title: string;
+  content: string;
+  truncatedContent: string;
+  folder: string;
+  created: string;
+  modified: string;
+  tags: string[];
+  outlinks: string[];
+}
+
+/**
+ * Prepare a note for embedding by extracting metadata and truncating content.
+ * Returns null if the note content is empty.
+ */
+function prepareNoteForEmbedding(note: {
+  id: string;
+  title: string;
+  content: string;
+  folder: string;
+  created: string;
+  modified: string;
+}): PreparedNote | null {
+  if (!note.content.trim()) {
+    return null;
+  }
+
+  const metadata = extractMetadata(note.content);
+
+  return {
+    id: note.id,
+    title: note.title,
+    content: note.content,
+    truncatedContent: truncateForEmbedding(note.content),
+    folder: note.folder,
+    created: note.created,
+    modified: note.modified,
+    tags: metadata.tags,
+    outlinks: metadata.outlinks,
+  };
+}
+
+/**
+ * Build a NoteRecord from a PreparedNote and its embedding vector.
+ */
+function buildNoteRecord(
+  note: PreparedNote,
+  vector: number[],
+  indexedAt: string
+): NoteRecord {
+  return {
+    id: note.id,
+    title: note.title,
+    content: note.content,
+    vector,
+    folder: note.folder,
+    created: note.created,
+    modified: note.modified,
+    indexed_at: indexedAt,
+    tags: note.tags,
+    outlinks: note.outlinks,
+  };
 }
 
 /**
  * Perform full reindexing of all notes.
  * Drops existing index and rebuilds from scratch.
+ * Uses single JXA call + batch embedding for maximum speed.
  */
 export async function fullIndex(): Promise<IndexResult> {
   const startTime = Date.now();
   debug("Starting full index...");
 
-  // Get all notes from Apple Notes
-  const notes = await getAllNotes();
-  debug(`Found ${notes.length} notes in Apple Notes`);
-
-  const records: NoteRecord[] = [];
-  let errors = 0;
-  const failedNotes: string[] = [];
+  // Phase 1: Fetch all notes with content in single JXA call
+  debug("Phase 1: Fetching all notes with content (single JXA call)...");
+  const allNotes = await getAllNotesWithContent();
+  debug(`Fetched ${allNotes.length} notes from Apple Notes`);
 
-  for (let i = 0; i < notes.length; i++) {
-    const noteInfo = notes[i];
-    debug(`Processing ${i + 1}/${notes.length}: ${noteInfo.title}`);
+  // Filter empty notes and prepare for embedding
+  const preparedNotes = allNotes
+    .map(prepareNoteForEmbedding)
+    .filter((note): note is PreparedNote => note !== null);
 
-    try {
-      // Get full note content using folder and title separately
-      // to handle notes with "/" in their titles
-      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
-      if (!noteDetails) {
-        debug(`Could not fetch note: ${noteInfo.title}`);
-        failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-        errors++;
-        continue;
-      }
+  debug(`Prepared ${preparedNotes.length} notes for embedding`);
 
-      // Skip empty notes
-      if (!noteDetails.content.trim()) {
-        debug(`Skipping empty note: ${noteInfo.title}`);
-        continue;
-      }
+  // Phase 2: Generate embeddings in batch (with concurrent API calls)
+  debug("Phase 2: Generating embeddings in batch...");
+  const textsToEmbed = preparedNotes.map(n => n.truncatedContent);
 
-      // Generate embedding
-      const content = truncateForEmbedding(noteDetails.content);
-      const vector = await getEmbedding(content);
-
-      const record: NoteRecord = {
-        title: noteDetails.title,
-        content: noteDetails.content,
-        vector,
-        folder: noteDetails.folder,
-        created: noteDetails.created,
-        modified: noteDetails.modified,
-        indexed_at: new Date().toISOString(),
-      };
-
-      records.push(record);
-
-      // Delay to avoid rate limiting
-      if (i < notes.length - 1) {
-        await sleep(EMBEDDING_DELAY_MS);
-      }
-    } catch (error) {
-      debug(`Error processing ${noteInfo.title}:`, error);
-      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-      errors++;
-    }
+  let vectors: number[][];
+  try {
+    vectors = await getEmbeddingBatch(textsToEmbed);
+  } catch (error) {
+    debug("Batch embedding failed:", error);
+    throw error;
   }
 
-  // Store all records in vector database
+  debug(`Generated ${vectors.length} embeddings`);
+
+  // Phase 3: Build records and store
+  debug("Phase 3: Storing in database...");
+  const indexedAt = new Date().toISOString();
+  const records = preparedNotes.map((note, i) =>
+    buildNoteRecord(note, vectors[i], indexedAt)
+  );
+
   const store = getVectorStore();
   await store.index(records);
 
   const timeMs = Date.now() - startTime;
-  debug(`Full index complete: ${records.length} indexed, ${errors} errors, ${timeMs}ms`);
+  const skipped = allNotes.length - preparedNotes.length;
+  debug(`Full index complete: ${records.length} indexed, ${skipped} empty/skipped, ${timeMs}ms`);
 
   return {
-    total: notes.length,
+    total: allNotes.length,
     indexed: records.length,
-    errors,
+    errors: 0,
     timeMs,
-    failedNotes: failedNotes.length > 0 ? failedNotes : undefined,
   };
 }
 
@@ -211,47 +249,63 @@ export async function incrementalIndex(): Promise<IndexResult> {
   let errors = 0;
   const failedNotes: string[] = [];
 
-  // Process additions and updates
+  // Process additions and updates in batch
   const toProcess = [...toAdd, ...toUpdate];
-  for (let i = 0; i < toProcess.length; i++) {
-    const noteInfo = toProcess[i];
-    debug(`Processing ${i + 1}/${toProcess.length}: ${noteInfo.title}`);
 
-    try {
-      // Use folder and title separately to handle "/" in titles
-      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
-      if (!noteDetails) {
+  if (toProcess.length > 0) {
+    // Phase 1: Fetch all note content
+    debug(`Phase 1: Fetching ${toProcess.length} notes content...`);
+    const preparedNotes: PreparedNote[] = [];
+
+    for (const noteInfo of toProcess) {
+      try {
+        const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
+        if (!noteDetails) {
+          failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+          errors++;
+          continue;
+        }
+
+        const prepared = prepareNoteForEmbedding(noteDetails);
+        if (prepared) {
+          preparedNotes.push(prepared);
+        }
+      } catch (error) {
+        debug(`Error fetching ${noteInfo.title}:`, error);
         failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
         errors++;
-        continue;
       }
+    }
 
-      if (!noteDetails.content.trim()) {
-        continue;
+    if (preparedNotes.length > 0) {
+      // Phase 2: Generate embeddings in batch
+      debug(`Phase 2: Generating ${preparedNotes.length} embeddings in batch...`);
+      const textsToEmbed = preparedNotes.map(n => n.truncatedContent);
+
+      let vectors: number[][];
+      try {
+        vectors = await getEmbeddingBatch(textsToEmbed);
+      } catch (error) {
+        debug("Batch embedding failed:", error);
+        throw error;
       }
 
-      const content = truncateForEmbedding(noteDetails.content);
-      const vector = await getEmbedding(content);
-
-      const record: NoteRecord = {
-        title: noteDetails.title,
-        content: noteDetails.content,
-        vector,
-        folder: noteDetails.folder,
-        created: noteDetails.created,
-        modified: noteDetails.modified,
-        indexed_at: new Date().toISOString(),
-      };
-
-      await store.update(record);
-
-      if (i < toProcess.length - 1) {
-        await sleep(EMBEDDING_DELAY_MS);
+      // Phase 3: Update database
+      debug("Phase 3: Updating database...");
+      const indexedAt = new Date().toISOString();
+
+      for (let i = 0; i < preparedNotes.length; i++) {
+        const note = preparedNotes[i];
+        const record = buildNoteRecord(note, vectors[i], indexedAt);
+
+        try {
+          await store.update(record);
+        } catch (error) {
+          debug(`Error updating ${note.title}:`, error);
+          failedNotes.push(`${note.folder}/${note.title}`);
+          errors++;
+        }
       }
-    } catch (error) {
-      debug(`Error processing ${noteInfo.title}:`, error);
-      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-      errors++;
     }
   }
 
@@ -270,6 +324,12 @@ export async function incrementalIndex(): Promise<IndexResult> {
     }
   }
 
+  // Rebuild FTS index if any changes were made
+  if (toAdd.length > 0 || toUpdate.length > 0 || toDelete.length > 0) {
+    debug("Rebuilding FTS index after incremental changes");
+    await store.rebuildFtsIndex();
+  }
+
   const timeMs = Date.now() - startTime;
   debug(`Incremental index complete: ${timeMs}ms`);
 
@@ -296,29 +356,24 @@ export async function reindexNote(title: string): Promise<void> {
 
   const noteDetails = await getNoteByTitle(title);
   if (!noteDetails) {
-    throw new Error(`Note not found: "${title}"`);
+    throw new NoteNotFoundError(title);
   }
 
-  if (!noteDetails.content.trim()) {
+  const prepared = prepareNoteForEmbedding(noteDetails);
+  if (!prepared) {
     throw new Error(`Note is empty: "${title}"`);
   }
 
-  const content = truncateForEmbedding(noteDetails.content);
-  const vector = await getEmbedding(content);
-
-  const record: NoteRecord = {
-    title: noteDetails.title,
-    content: noteDetails.content,
-    vector,
-    folder: noteDetails.folder,
-    created: noteDetails.created,
-    modified: noteDetails.modified,
-    indexed_at: new Date().toISOString(),
-  };
+  const vector = await getEmbedding(prepared.truncatedContent);
+  const record = buildNoteRecord(prepared, vector, new Date().toISOString());
 
   const store = getVectorStore();
   await store.update(record);
 
+  // Rebuild FTS index after single note update
+  debug("Rebuilding FTS index after single note reindex");
+  await store.rebuildFtsIndex();
+
   debug(`Reindexed: ${title}`);
 }
 

package/src/setup.ts

@@ -12,14 +12,23 @@
 
 import * as p from "@clack/prompts";
 import * as fs from "node:fs";
-import * as path from "node:path";
+import {
+  getEnvPath,
+  ensureConfigDir,
+  hasLegacyConfig,
+  getLegacyEnvPath,
+  hasConfig,
+  isNpmInstall,
+} from "./config/paths.js";
+import {
+  getClaudeConfigEntry,
+  writeClaudeConfig,
+  getExistingInstallMethod,
+} from "./config/claude.js";
+import { checkBunRuntime } from "./utils/runtime.js";
+
 // Paths
-const PROJECT_DIR = path.dirname(new URL(import.meta.url).pathname);
-const ENV_FILE = path.join(PROJECT_DIR, "..", ".env");
-const CLAUDE_CONFIG_PATH = path.join(
-  process.env.HOME || "~",
-  ".claude.json"
-);
+const ENV_FILE = getEnvPath();
 
 interface Config {
   provider: "local" | "openrouter";
@@ -69,6 +78,7 @@ function readExistingEnv(): Record<string, string> {
  * Write configuration to .env file
  */
 function writeEnvFile(config: Config): void {
+  ensureConfigDir();
   const lines: string[] = [
     "# apple-notes-mcp configuration",
     "# Generated by setup wizard",
@@ -114,80 +124,29 @@ function writeEnvFile(config: Config): void {
   fs.writeFileSync(ENV_FILE, lines.join("\n") + "\n");
 }
 
-/**
- * Read Claude Code config if it exists
- */
-function readClaudeConfig(): Record<string, unknown> | null {
-  if (!fs.existsSync(CLAUDE_CONFIG_PATH)) {
-    return null;
-  }
-
-  try {
-    const content = fs.readFileSync(CLAUDE_CONFIG_PATH, "utf-8");
-    return JSON.parse(content);
-  } catch (error) {
-    // Config doesn't exist or is invalid JSON
-    if (process.env.DEBUG === "true") {
-      console.error("[SETUP] Could not read Claude config:", error);
-    }
-    return null;
-  }
-}
-
 /**
  * Add MCP server to Claude Code config
  */
 function addToClaudeConfig(): boolean {
-  const projectPath = path.resolve(PROJECT_DIR, "..");
-  const serverEntry = {
-    command: "bun",
-    args: ["run", path.join(projectPath, "src", "index.ts")],
-    env: {},
-  };
+  const entry = getClaudeConfigEntry();
 
-  let config = readClaudeConfig();
+  // Check for install method change
+  const existingMethod = getExistingInstallMethod();
+  const currentMethod = isNpmInstall() ? "npm" : "source";
 
-  if (!config) {
-    // Create new config
-    config = {
-      mcpServers: {
-        "apple-notes": serverEntry,
-      },
-    };
-  } else {
-    // Add to existing config
-    const mcpServers = (config.mcpServers || {}) as Record<string, unknown>;
-    mcpServers["apple-notes"] = serverEntry;
-    config.mcpServers = mcpServers;
+  if (existingMethod && existingMethod !== currentMethod) {
+    p.log.info(`Updating Claude config from ${existingMethod} to ${currentMethod} installation`);
   }
 
-  try {
-    fs.writeFileSync(CLAUDE_CONFIG_PATH, JSON.stringify(config, null, 2) + "\n");
-    return true;
-  } catch (error) {
-    if (process.env.DEBUG === "true") {
-      console.error("[SETUP] Failed to write Claude config:", error);
-    }
-    return false;
-  }
+  return writeClaudeConfig(entry);
 }
 
 /**
  * Generate config snippet for manual setup
  */
 function getConfigSnippet(): string {
-  const projectPath = path.resolve(PROJECT_DIR, "..");
-  return JSON.stringify(
-    {
-      "apple-notes": {
-        command: "bun",
-        args: ["run", path.join(projectPath, "src", "index.ts")],
-        env: {},
-      },
-    },
-    null,
-    2
-  );
+  const entry = getClaudeConfigEntry();
+  return JSON.stringify({ "apple-notes": entry }, null, 2);
 }
 
 /**
@@ -217,6 +176,7 @@ async function downloadLocalModel(): Promise<void> {
  * Main setup wizard
  */
 async function main(): Promise<void> {
+  checkBunRuntime();
   console.clear();
 
   p.intro("apple-notes-mcp Setup Wizard");
@@ -233,6 +193,25 @@ async function main(): Promise<void> {
     );
   }
 
+  // Check for legacy config migration
+  if (hasLegacyConfig() && !hasConfig()) {
+    const migrate = await p.confirm({
+      message: "Found config in project directory. Migrate to ~/.apple-notes-mcp/?",
+      initialValue: true,
+    });
+
+    if (p.isCancel(migrate)) {
+      p.cancel("Setup cancelled.");
+      process.exit(0);
+    }
+
+    if (migrate) {
+      ensureConfigDir();
+      fs.copyFileSync(getLegacyEnvPath(), getEnvPath());
+      p.log.success("Config migrated to ~/.apple-notes-mcp/.env");
+    }
+  }
+
   // Provider selection
   const provider = await p.select({
     message: "Which embedding provider would you like to use?",

package/src/types/index.ts

@@ -7,6 +7,8 @@
  * Contains the full content of the note.
  */
 export interface DBSearchResult {
+  /** Apple Notes unique identifier */
+  id?: string;
   /** Note title */
   title: string;
   /** Folder containing the note */
@@ -24,6 +26,8 @@ export interface DBSearchResult {
  * Contains a preview instead of full content by default.
  */
 export interface SearchResult {
+  /** Apple Notes unique identifier */
+  id?: string;
   /** Note title */
   title: string;
   /** Folder containing the note */