@disco_trooper/apple-notes-mcp 1.2.0 → 1.3.0

package/src/search/indexer.ts

@@ -7,13 +7,13 @@
  * - Single note reindexing
  */
 
-import { getEmbedding } from "../embeddings/index.js";
+import { getEmbedding, getEmbeddingBatch } from "../embeddings/index.js";
 import { getVectorStore, type NoteRecord } from "../db/lancedb.js";
-import { getAllNotes, getNoteByFolderAndTitle, getNoteByTitle, type NoteInfo } from "../notes/read.js";
+import { getAllNotes, getAllNotesWithContent, getNoteByFolderAndTitle, getNoteByTitle, type NoteInfo } from "../notes/read.js";
 import { createDebugLogger } from "../utils/debug.js";
 import { truncateForEmbedding } from "../utils/text.js";
-import { EMBEDDING_DELAY_MS } from "../config/constants.js";
 import { NoteNotFoundError } from "../errors/index.js";
+import { extractMetadata } from "../graph/extract.js";
 
 /**
  * Extract note title from folder/title key.
@@ -50,90 +50,127 @@ export interface IndexResult {
 }
 
 /**
- * Sleep for a specified duration.
+ * Note data prepared for embedding.
  */
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+interface PreparedNote {
+  id: string;
+  title: string;
+  content: string;
+  truncatedContent: string;
+  folder: string;
+  created: string;
+  modified: string;
+  tags: string[];
+  outlinks: string[];
+}
+
+/**
+ * Prepare a note for embedding by extracting metadata and truncating content.
+ * Returns null if the note content is empty.
+ */
+function prepareNoteForEmbedding(note: {
+  id: string;
+  title: string;
+  content: string;
+  folder: string;
+  created: string;
+  modified: string;
+}): PreparedNote | null {
+  if (!note.content.trim()) {
+    return null;
+  }
+
+  const metadata = extractMetadata(note.content);
+
+  return {
+    id: note.id,
+    title: note.title,
+    content: note.content,
+    truncatedContent: truncateForEmbedding(note.content),
+    folder: note.folder,
+    created: note.created,
+    modified: note.modified,
+    tags: metadata.tags,
+    outlinks: metadata.outlinks,
+  };
+}
+
+/**
+ * Build a NoteRecord from a PreparedNote and its embedding vector.
+ */
+function buildNoteRecord(
+  note: PreparedNote,
+  vector: number[],
+  indexedAt: string
+): NoteRecord {
+  return {
+    id: note.id,
+    title: note.title,
+    content: note.content,
+    vector,
+    folder: note.folder,
+    created: note.created,
+    modified: note.modified,
+    indexed_at: indexedAt,
+    tags: note.tags,
+    outlinks: note.outlinks,
+  };
 }
 
 /**
  * Perform full reindexing of all notes.
  * Drops existing index and rebuilds from scratch.
+ * Uses single JXA call + batch embedding for maximum speed.
  */
 export async function fullIndex(): Promise<IndexResult> {
   const startTime = Date.now();
   debug("Starting full index...");
 
-  // Get all notes from Apple Notes
-  const notes = await getAllNotes();
-  debug(`Found ${notes.length} notes in Apple Notes`);
+  // Phase 1: Fetch all notes with content in single JXA call
+  debug("Phase 1: Fetching all notes with content (single JXA call)...");
+  const allNotes = await getAllNotesWithContent();
+  debug(`Fetched ${allNotes.length} notes from Apple Notes`);
 
-  const records: NoteRecord[] = [];
-  let errors = 0;
-  const failedNotes: string[] = [];
+  // Filter empty notes and prepare for embedding
+  const preparedNotes = allNotes
+    .map(prepareNoteForEmbedding)
+    .filter((note): note is PreparedNote => note !== null);
 
-  for (let i = 0; i < notes.length; i++) {
-    const noteInfo = notes[i];
-    debug(`Processing ${i + 1}/${notes.length}: ${noteInfo.title}`);
+  debug(`Prepared ${preparedNotes.length} notes for embedding`);
 
-    try {
-      // Get full note content using folder and title separately
-      // to handle notes with "/" in their titles
-      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
-      if (!noteDetails) {
-        debug(`Could not fetch note: ${noteInfo.title}`);
-        failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-        errors++;
-        continue;
-      }
+  // Phase 2: Generate embeddings in batch (with concurrent API calls)
+  debug("Phase 2: Generating embeddings in batch...");
+  const textsToEmbed = preparedNotes.map(n => n.truncatedContent);
 
-      // Skip empty notes
-      if (!noteDetails.content.trim()) {
-        debug(`Skipping empty note: ${noteInfo.title}`);
-        continue;
-      }
-
-      // Generate embedding
-      const content = truncateForEmbedding(noteDetails.content);
-      const vector = await getEmbedding(content);
-
-      const record: NoteRecord = {
-        id: noteDetails.id,
-        title: noteDetails.title,
-        content: noteDetails.content,
-        vector,
-        folder: noteDetails.folder,
-        created: noteDetails.created,
-        modified: noteDetails.modified,
-        indexed_at: new Date().toISOString(),
-      };
-
-      records.push(record);
-
-      // Delay to avoid rate limiting
-      if (i < notes.length - 1) {
-        await sleep(EMBEDDING_DELAY_MS);
-      }
-    } catch (error) {
-      debug(`Error processing ${noteInfo.title}:`, error);
-      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-      errors++;
-    }
+  let vectors: number[][];
+  try {
+    vectors = await getEmbeddingBatch(textsToEmbed);
+  } catch (error) {
+    debug("Batch embedding failed:", error);
+    throw error;
   }
 
-  // Store all records in vector database
+  debug(`Generated ${vectors.length} embeddings`);
+
+  // Phase 3: Build records and store
+  debug("Phase 3: Storing in database...");
+  const indexedAt = new Date().toISOString();
+  const records = preparedNotes.map((note, i) =>
+    buildNoteRecord(note, vectors[i], indexedAt)
+  );
+
   const store = getVectorStore();
   await store.index(records);
 
   const timeMs = Date.now() - startTime;
-  debug(`Full index complete: ${records.length} indexed, ${errors} errors, ${timeMs}ms`);
+  const skipped = allNotes.length - preparedNotes.length;
+  debug(`Full index complete: ${records.length} indexed, ${skipped} empty/skipped, ${timeMs}ms`);
 
   return {
-    total: notes.length,
+    total: allNotes.length,
     indexed: records.length,
-    errors,
+    errors: 0,
     timeMs,
-    failedNotes: failedNotes.length > 0 ? failedNotes : undefined,
   };
 }
 
@@ -212,48 +249,63 @@ export async function incrementalIndex(): Promise<IndexResult> {
   let errors = 0;
   const failedNotes: string[] = [];
 
-  // Process additions and updates
+  // Process additions and updates in batch
   const toProcess = [...toAdd, ...toUpdate];
-  for (let i = 0; i < toProcess.length; i++) {
-    const noteInfo = toProcess[i];
-    debug(`Processing ${i + 1}/${toProcess.length}: ${noteInfo.title}`);
 
-    try {
-      // Use folder and title separately to handle "/" in titles
-      const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
-      if (!noteDetails) {
+  if (toProcess.length > 0) {
+    // Phase 1: Fetch all note content
+    debug(`Phase 1: Fetching ${toProcess.length} notes content...`);
+    const preparedNotes: PreparedNote[] = [];
+
+    for (const noteInfo of toProcess) {
+      try {
+        const noteDetails = await getNoteByFolderAndTitle(noteInfo.folder, noteInfo.title);
+        if (!noteDetails) {
+          failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
+          errors++;
+          continue;
+        }
+
+        const prepared = prepareNoteForEmbedding(noteDetails);
+        if (prepared) {
+          preparedNotes.push(prepared);
+        }
+      } catch (error) {
+        debug(`Error fetching ${noteInfo.title}:`, error);
         failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
         errors++;
-        continue;
       }
+    }
 
-      if (!noteDetails.content.trim()) {
-        continue;
+    if (preparedNotes.length > 0) {
+      // Phase 2: Generate embeddings in batch
+      debug(`Phase 2: Generating ${preparedNotes.length} embeddings in batch...`);
+      const textsToEmbed = preparedNotes.map(n => n.truncatedContent);
+
+      let vectors: number[][];
+      try {
+        vectors = await getEmbeddingBatch(textsToEmbed);
+      } catch (error) {
+        debug("Batch embedding failed:", error);
+        throw error;
       }
 
-      const content = truncateForEmbedding(noteDetails.content);
-      const vector = await getEmbedding(content);
-
-      const record: NoteRecord = {
-        id: noteDetails.id,
-        title: noteDetails.title,
-        content: noteDetails.content,
-        vector,
-        folder: noteDetails.folder,
-        created: noteDetails.created,
-        modified: noteDetails.modified,
-        indexed_at: new Date().toISOString(),
-      };
-
-      await store.update(record);
-
-      if (i < toProcess.length - 1) {
-        await sleep(EMBEDDING_DELAY_MS);
+      // Phase 3: Update database
+      debug("Phase 3: Updating database...");
+      const indexedAt = new Date().toISOString();
+
+      for (let i = 0; i < preparedNotes.length; i++) {
+        const note = preparedNotes[i];
+        const record = buildNoteRecord(note, vectors[i], indexedAt);
+
+        try {
+          await store.update(record);
+        } catch (error) {
+          debug(`Error updating ${note.title}:`, error);
+          failedNotes.push(`${note.folder}/${note.title}`);
+          errors++;
+        }
       }
-    } catch (error) {
-      debug(`Error processing ${noteInfo.title}:`, error);
-      failedNotes.push(`${noteInfo.folder}/${noteInfo.title}`);
-      errors++;
     }
   }
 
@@ -307,23 +359,13 @@ export async function reindexNote(title: string): Promise<void> {
     throw new NoteNotFoundError(title);
   }
 
-  if (!noteDetails.content.trim()) {
+  const prepared = prepareNoteForEmbedding(noteDetails);
+  if (!prepared) {
     throw new Error(`Note is empty: "${title}"`);
   }
 
-  const content = truncateForEmbedding(noteDetails.content);
-  const vector = await getEmbedding(content);
-
-  const record: NoteRecord = {
-    id: noteDetails.id,
-    title: noteDetails.title,
-    content: noteDetails.content,
-    vector,
-    folder: noteDetails.folder,
-    created: noteDetails.created,
-    modified: noteDetails.modified,
-    indexed_at: new Date().toISOString(),
-  };
+  const vector = await getEmbedding(prepared.truncatedContent);
+  const record = buildNoteRecord(prepared, vector, new Date().toISOString());
 
   const store = getVectorStore();
   await store.update(record);

package/src/setup.ts

@@ -12,14 +12,23 @@
 
 import * as p from "@clack/prompts";
 import * as fs from "node:fs";
-import * as path from "node:path";
+import {
+  getEnvPath,
+  ensureConfigDir,
+  hasLegacyConfig,
+  getLegacyEnvPath,
+  hasConfig,
+  isNpmInstall,
+} from "./config/paths.js";
+import {
+  getClaudeConfigEntry,
+  writeClaudeConfig,
+  getExistingInstallMethod,
+} from "./config/claude.js";
+import { checkBunRuntime } from "./utils/runtime.js";
+
 // Paths
-const PROJECT_DIR = path.dirname(new URL(import.meta.url).pathname);
-const ENV_FILE = path.join(PROJECT_DIR, "..", ".env");
-const CLAUDE_CONFIG_PATH = path.join(
-  process.env.HOME || "~",
-  ".claude.json"
-);
+const ENV_FILE = getEnvPath();
 
 interface Config {
   provider: "local" | "openrouter";
@@ -69,6 +78,7 @@ function readExistingEnv(): Record<string, string> {
  * Write configuration to .env file
  */
 function writeEnvFile(config: Config): void {
+  ensureConfigDir();
   const lines: string[] = [
     "# apple-notes-mcp configuration",
     "# Generated by setup wizard",
@@ -114,80 +124,29 @@ function writeEnvFile(config: Config): void {
   fs.writeFileSync(ENV_FILE, lines.join("\n") + "\n");
 }
 
-/**
- * Read Claude Code config if it exists
- */
-function readClaudeConfig(): Record<string, unknown> | null {
-  if (!fs.existsSync(CLAUDE_CONFIG_PATH)) {
-    return null;
-  }
-
-  try {
-    const content = fs.readFileSync(CLAUDE_CONFIG_PATH, "utf-8");
-    return JSON.parse(content);
-  } catch (error) {
-    // Config doesn't exist or is invalid JSON
-    if (process.env.DEBUG === "true") {
-      console.error("[SETUP] Could not read Claude config:", error);
-    }
-    return null;
-  }
-}
-
 /**
  * Add MCP server to Claude Code config
  */
 function addToClaudeConfig(): boolean {
-  const projectPath = path.resolve(PROJECT_DIR, "..");
-  const serverEntry = {
-    command: "bun",
-    args: ["run", path.join(projectPath, "src", "index.ts")],
-    env: {},
-  };
+  const entry = getClaudeConfigEntry();
 
-  let config = readClaudeConfig();
+  // Check for install method change
+  const existingMethod = getExistingInstallMethod();
+  const currentMethod = isNpmInstall() ? "npm" : "source";
 
-  if (!config) {
-    // Create new config
-    config = {
-      mcpServers: {
-        "apple-notes": serverEntry,
-      },
-    };
-  } else {
-    // Add to existing config
-    const mcpServers = (config.mcpServers || {}) as Record<string, unknown>;
-    mcpServers["apple-notes"] = serverEntry;
-    config.mcpServers = mcpServers;
+  if (existingMethod && existingMethod !== currentMethod) {
+    p.log.info(`Updating Claude config from ${existingMethod} to ${currentMethod} installation`);
   }
 
-  try {
-    fs.writeFileSync(CLAUDE_CONFIG_PATH, JSON.stringify(config, null, 2) + "\n");
-    return true;
-  } catch (error) {
-    if (process.env.DEBUG === "true") {
-      console.error("[SETUP] Failed to write Claude config:", error);
-    }
-    return false;
-  }
+  return writeClaudeConfig(entry);
 }
 
 /**
  * Generate config snippet for manual setup
  */
 function getConfigSnippet(): string {
-  const projectPath = path.resolve(PROJECT_DIR, "..");
-  return JSON.stringify(
-    {
-      "apple-notes": {
-        command: "bun",
-        args: ["run", path.join(projectPath, "src", "index.ts")],
-        env: {},
-      },
-    },
-    null,
-    2
-  );
+  const entry = getClaudeConfigEntry();
+  return JSON.stringify({ "apple-notes": entry }, null, 2);
 }
 
 /**
@@ -217,6 +176,7 @@ async function downloadLocalModel(): Promise<void> {
  * Main setup wizard
  */
 async function main(): Promise<void> {
+  checkBunRuntime();
   console.clear();
 
   p.intro("apple-notes-mcp Setup Wizard");
@@ -233,6 +193,25 @@ async function main(): Promise<void> {
     );
   }
 
+  // Check for legacy config migration
+  if (hasLegacyConfig() && !hasConfig()) {
+    const migrate = await p.confirm({
+      message: "Found config in project directory. Migrate to ~/.apple-notes-mcp/?",
+      initialValue: true,
+    });
+
+    if (p.isCancel(migrate)) {
+      p.cancel("Setup cancelled.");
+      process.exit(0);
+    }
+
+    if (migrate) {
+      ensureConfigDir();
+      fs.copyFileSync(getLegacyEnvPath(), getEnvPath());
+      p.log.success("Config migrated to ~/.apple-notes-mcp/.env");
+    }
+  }
+
   // Provider selection
   const provider = await p.select({
     message: "Which embedding provider would you like to use?",

package/src/utils/chunker.test.ts

@@ -0,0 +1,182 @@
+import { describe, expect, it } from "vitest";
+import {
+  chunkText,
+  type ChunkOptions,
+  DEFAULT_CHUNK_OPTIONS,
+  SEPARATORS,
+  findSplitPoint,
+} from "./chunker.js";
+
+describe("chunker", () => {
+  describe("exports", () => {
+    it("exports SEPARATORS array with correct order", () => {
+      expect(SEPARATORS).toEqual([
+        "\n\n",
+        "\n",
+        ". ",
+        "! ",
+        "? ",
+        "; ",
+        ", ",
+        " ",
+        "",
+      ]);
+    });
+
+    it("exports DEFAULT_CHUNK_OPTIONS with correct values", () => {
+      expect(DEFAULT_CHUNK_OPTIONS).toEqual({
+        chunkSize: 500,
+        overlap: 100,
+      });
+    });
+  });
+
+  describe("findSplitPoint", () => {
+    it("finds paragraph boundary near target", () => {
+      const text = "First paragraph.\n\nSecond paragraph.";
+      const target = 20;
+      const result = findSplitPoint(text, target);
+      // Should find the \n\n at position 16
+      expect(result).toBe(18); // After \n\n
+    });
+
+    it("falls back to sentence boundary", () => {
+      const text = "First sentence. Second sentence.";
+      const target = 18;
+      const result = findSplitPoint(text, target);
+      // Should find ". " at position 14-16
+      expect(result).toBe(16); // After ". "
+    });
+
+    it("falls back to word boundary", () => {
+      const text = "oneword anotherword";
+      const target = 10;
+      const result = findSplitPoint(text, target);
+      // Should find space at position 7
+      expect(result).toBe(8); // After " "
+    });
+
+    it("returns target when no separator found", () => {
+      const text = "noseparatorshere";
+      const target = 8;
+      const result = findSplitPoint(text, target);
+      expect(result).toBe(8);
+    });
+  });
+
+  describe("chunkText", () => {
+    it("returns single chunk for short text", () => {
+      const text = "Short text";
+      const options: ChunkOptions = { chunkSize: 100, overlap: 20 };
+
+      const result = chunkText(text, options);
+
+      expect(result).toHaveLength(1);
+      expect(result[0]).toEqual({
+        content: "Short text",
+        index: 0,
+        totalChunks: 1,
+        startPos: 0,
+        endPos: 10,
+      });
+    });
+
+    it("creates multiple chunks for long text", () => {
+      const text = "Word ".repeat(50).trim(); // 249 chars
+      const options: ChunkOptions = { chunkSize: 50, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      expect(result.length).toBeGreaterThan(1);
+      // Each chunk should have content
+      result.forEach((chunk) => {
+        expect(chunk.content.length).toBeGreaterThan(0);
+        expect(chunk.content.length).toBeLessThanOrEqual(options.chunkSize);
+      });
+    });
+
+    it("includes overlap between chunks", () => {
+      const text = "First part. Second part. Third part. Fourth part.";
+      const options: ChunkOptions = { chunkSize: 25, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      // Check that chunks overlap - endPos of chunk N should be > startPos of chunk N+1
+      for (let i = 0; i < result.length - 1; i++) {
+        const currentChunk = result[i];
+        const nextChunk = result[i + 1];
+        // Overlap means next chunk starts before current chunk ends
+        expect(nextChunk.startPos).toBeLessThan(currentChunk.endPos);
+      }
+    });
+
+    it("respects paragraph boundaries when splitting", () => {
+      const text = "First paragraph here.\n\nSecond paragraph here.\n\nThird paragraph.";
+      const options: ChunkOptions = { chunkSize: 30, overlap: 5 };
+
+      const result = chunkText(text, options);
+
+      // At least one chunk should end at a paragraph boundary
+      const hasParaBoundary = result.some((chunk) => {
+        const endContent = text.slice(chunk.startPos, chunk.endPos);
+        return endContent.endsWith("\n\n") || chunk.endPos === text.length;
+      });
+      expect(hasParaBoundary).toBe(true);
+    });
+
+    it("sets correct totalChunks on all chunks", () => {
+      const text = "A ".repeat(100).trim(); // Create text that will be chunked
+      const options: ChunkOptions = { chunkSize: 20, overlap: 5 };
+
+      const result = chunkText(text, options);
+
+      const expectedTotal = result.length;
+      result.forEach((chunk, idx) => {
+        expect(chunk.totalChunks).toBe(expectedTotal);
+        expect(chunk.index).toBe(idx);
+      });
+    });
+
+    it("handles empty text", () => {
+      const result = chunkText("", { chunkSize: 100, overlap: 20 });
+
+      expect(result).toHaveLength(0);
+    });
+
+    it("handles whitespace-only text", () => {
+      const result = chunkText("   \n\n   ", { chunkSize: 100, overlap: 20 });
+
+      expect(result).toHaveLength(0);
+    });
+
+    it("uses default options when not provided", () => {
+      const text = "Test";
+      const result = chunkText(text);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe("Test");
+    });
+
+    it("covers all original text with chunks", () => {
+      const text = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.";
+      const options: ChunkOptions = { chunkSize: 30, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      // Verify chunks cover the entire text
+      expect(result[0].startPos).toBe(0);
+      expect(result[result.length - 1].endPos).toBe(text.length);
+
+      // Verify each chunk's content matches its position in original text
+      for (const chunk of result) {
+        expect(chunk.content).toBe(text.slice(chunk.startPos, chunk.endPos));
+      }
+
+      // Verify chunks are contiguous (no gaps)
+      for (let i = 0; i < result.length - 1; i++) {
+        // Next chunk should start before or at current chunk's end (overlap)
+        expect(result[i + 1].startPos).toBeLessThanOrEqual(result[i].endPos);
+      }
+    });
+  });
+});