@disco_trooper/apple-notes-mcp 1.7.0 → 1.8.2

package/README.md

@@ -18,13 +18,14 @@ MCP server for Apple Notes with semantic search and CRUD operations. Claude sear
 - **Semantic Search** - Find notes by meaning, not keywords
 - **Full CRUD** - Create, read, update, delete, and move notes
 - **Incremental Indexing** - Re-embed only changed notes
+- **Background Index Jobs** - Async full/incremental indexing with progress polling
 - **Dual Embedding** - Local HuggingFace or OpenRouter API
 
-## What's New in 1.7
+## What's New in 1.8.1
 
-- **Hybrid Fallback Indexing** - Recovers from failures by falling back: single call → folder batch → note-by-note
-- **Streaming Batches** - Processes embeddings in batches to reduce peak memory
-- **Skipped Notes Reporting** - Shows which notes failed (locked, syncing, corrupted)
+- **Faster `list-notes` folder filtering** - `list-notes` now queries only the requested folder instead of scanning all notes first
+- **Duplicate folder name correctness** - Folder filtering now aggregates matching folders across accounts
+- **Large vault performance** - Folder-scoped listing is significantly faster on larger note libraries
 
 ## Installation
 
@@ -76,10 +77,19 @@ Configuration stored in `~/.apple-notes-mcp/.env`:
 | `EMBEDDING_MODEL` | Model name (local or OpenRouter) | `Xenova/multilingual-e5-small` |
 | `EMBEDDING_DIMS` | Embedding dimensions | `4096` |
 | `READONLY_MODE` | Block all write operations | `false` |
-| `INDEX_TTL` | Auto-reindex interval in seconds | - |
+| `INDEX_TTL` | Auto-refresh-on-search interval in seconds (disabled when unset) | - |
+| `SEARCH_REFRESH_TIMEOUT_MS` | Max time search waits for refresh before using stale index | `2000` |
+| `INDEX_JOB_RETENTION_SECONDS` | How long completed/failed index jobs remain queryable | `3600` |
 | `EMBEDDING_BATCH_SIZE` | Batch size for embedding generation | `50` |
 | `DEBUG` | Enable debug logging | `false` |
 
+### Search Auto-Refresh Policy
+
+- `search-notes` does **not** force refresh on every request.
+- If `INDEX_TTL` is unset, auto-refresh is disabled and search uses the current index.
+- If `INDEX_TTL` is set, refresh runs only after TTL expiration.
+- If refresh fails or takes longer than `SEARCH_REFRESH_TIMEOUT_MS`, search falls back to stale index results instead of timing out.
+
 To reconfigure:
 
 ```bash
@@ -121,6 +131,8 @@ limit: 10                # max notes to return (1-100)
 folder: "Work"           # filter by folder (case-insensitive)
 ```
 
+When `folder` is provided, the server fetches only matching folders from Apple Notes. This keeps folder-scoped requests fast even when your vault has hundreds of notes.
+
 **Examples:**
 - Get 5 newest notes: `{ sort_by: "created", order: "desc", limit: 5 }`
 - Recently modified: `{ sort_by: "modified", limit: 10 }`
@@ -164,10 +176,48 @@ Index notes for semantic search.
 ```
 mode: "incremental"       # incremental (default) or full
 force: false              # force reindex even if TTL hasn't expired
+background: false         # optional; defaults to false (synchronous mode)
 ```
 
 Use `mode: "full"` to create the chunk index for better long-note search. First full index takes longer as it generates chunks, but subsequent searches run fast.
 
+For large vaults, prefer background indexing:
+
+#### `start-index-job`
+
+```
+mode: "full"               # full or incremental
+```
+
+Returns a job snapshot with `id`, `status`, and `progress`.
+Progress updates in smaller steps across fetch, embed, and persist phases.
+
+#### `get-index-job`
+
+```
+job_id: "<job-id>"
+```
+
+Poll until status is `completed`, `failed`, or `cancelled`.
+You may see `cancelling` as a transitional status.
+
+#### `list-index-jobs`
+
+```
+limit: 10                  # optional, 1-50
+```
+
+#### `cancel-index-job`
+
+```
+job_id: "<job-id>"
+```
+
+Requests best-effort cancellation for a running job. Cancellation is cooperative:
+- A long-running step must reach a cancellation checkpoint.
+- Partial work may remain.
+- Start a new job after the current one reaches `cancelled`.
+
 #### `reindex-note`
 Re-index a single note after manual edits.
 
@@ -186,6 +236,9 @@ content: "# Heading\n\nMarkdown content..."
 folder: "Work"            # optional, defaults to Notes
 ```
 
+After create, update, delete, or move, the server auto-syncs vector and chunk indexes in best-effort mode.
+If sync partly fails, the tool response includes an `index sync warning`. Run `reindex-note` or `index-notes`.
+
 #### `update-note`
 Update an existing note.
 
@@ -333,6 +386,15 @@ Set `READONLY_MODE=false` in `.env` to enable write operations.
 ### Notes missing from search
 Run `index-notes` to update the search index. Use `mode: full` if incremental misses changes.
 
+### "iCloud account not available" / `Can't get account "iCloud"`
+This error comes from a different Apple Notes MCP implementation that uses tool `search_notes` and argument `Keywords`.
+
+This project uses:
+- tool: `search-notes`
+- argument: `query`
+
+If your client calls `search_notes` with `Keywords`, point your MCP config to `apple-notes-mcp` and restart the client.
+
 ### JXA errors
 Ensure Apple Notes runs and contains notes. Grant automation permissions when prompted.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@disco_trooper/apple-notes-mcp",
-  "version": "1.7.0",
+  "version": "1.8.2",
   "description": "MCP server for Apple Notes with semantic search and CRUD operations",
   "type": "module",
   "main": "src/index.ts",

package/src/config/constants.ts

@@ -19,6 +19,7 @@ export const RRF_K = 60; // Reciprocal Rank Fusion constant
 
 // Timeouts and retries
 export const OPENROUTER_TIMEOUT_MS = 30000;
+export const DEFAULT_SEARCH_REFRESH_TIMEOUT_MS = 2000;
 
 // Cache settings
 export const EMBEDDING_CACHE_MAX_SIZE = 1000;
@@ -52,6 +53,10 @@ export const DEFAULT_CHUNK_OVERLAP = 100;
 // Batch processing
 export const DEFAULT_EMBEDDING_BATCH_SIZE = 50;
 
+// Background indexing jobs
+export const DEFAULT_INDEX_JOB_RETENTION_SECONDS = 3600;
+export const MAX_INDEX_JOB_HISTORY = 100;
+
 /**
  * Get embedding batch size from environment or use default.
  * Lower values reduce peak memory usage but increase processing time.

package/src/config/env.test.ts

@@ -28,6 +28,8 @@ describe("validateEnv", () => {
     delete process.env.READONLY_MODE;
     delete process.env.EMBEDDING_DIMS;
     delete process.env.INDEX_TTL;
+    delete process.env.SEARCH_REFRESH_TIMEOUT_MS;
+    delete process.env.INDEX_JOB_RETENTION_SECONDS;
     const { validateEnv } = await import("./env.js");
     expect(() => validateEnv()).not.toThrow();
   });
@@ -55,4 +57,16 @@ describe("validateEnv", () => {
     const { validateEnv } = await import("./env.js");
     expect(() => validateEnv()).toThrow();
   });
+
+  it("validates SEARCH_REFRESH_TIMEOUT_MS is numeric", async () => {
+    process.env.SEARCH_REFRESH_TIMEOUT_MS = "not-a-number";
+    const { validateEnv } = await import("./env.js");
+    expect(() => validateEnv()).toThrow();
+  });
+
+  it("validates INDEX_JOB_RETENTION_SECONDS is numeric", async () => {
+    process.env.INDEX_JOB_RETENTION_SECONDS = "abc";
+    const { validateEnv } = await import("./env.js");
+    expect(() => validateEnv()).toThrow();
+  });
 });

package/src/config/env.ts

@@ -6,6 +6,8 @@ const EnvSchema = z.object({
   EMBEDDING_DIMS: z.string().regex(/^\d+$/).optional(),
   READONLY_MODE: z.enum(["true", "false"]).optional(),
   INDEX_TTL: z.string().regex(/^\d+$/).optional(),
+  SEARCH_REFRESH_TIMEOUT_MS: z.string().regex(/^\d+$/).optional(),
+  INDEX_JOB_RETENTION_SECONDS: z.string().regex(/^\d+$/).optional(),
   DEBUG: z.enum(["true", "false"]).optional(),
 });
 

package/src/db/lancedb.test.ts

@@ -72,6 +72,20 @@ describe("LanceDBStore", () => {
       await store.index([createTestRecord("Existing")]);
       await expect(store.delete("Non Existent")).resolves.not.toThrow();
     });
+
+    it("deleteByIdAndFolderAndTitle removes only matching record", async () => {
+      const base = createTestRecord("Shared");
+      const other = { ...createTestRecord("Shared"), id: "other-id" };
+
+      await store.index([base, other]);
+      expect(await store.count()).toBe(2);
+
+      await store.deleteByIdAndFolderAndTitle(base.id, base.folder, base.title);
+
+      expect(await store.count()).toBe(1);
+      const remaining = await store.getByTitle("Shared");
+      expect(remaining?.id).toBe("other-id");
+    });
   });
 
   describe("getAll", () => {

package/src/db/lancedb.ts

@@ -20,6 +20,13 @@ export interface NoteRecord {
   [key: string]: unknown; // Index signature for LanceDB compatibility
 }
 
+export interface IndexMetadataRecord {
+  id: string;
+  title: string;
+  folder: string;
+  indexed_at: string;
+}
+
 // Schema for chunked notes (Parent Document Retriever pattern)
 export interface ChunkRecord {
   chunk_id: string;      // `${note_id}_chunk_${index}`
@@ -48,9 +55,11 @@ export interface VectorStore {
   update(record: NoteRecord): Promise<void>;
   delete(title: string): Promise<void>;
   deleteByFolderAndTitle(folder: string, title: string): Promise<void>;
+  deleteByIdAndFolderAndTitle(id: string, folder: string, title: string): Promise<void>;
   search(queryVector: number[], limit: number): Promise<SearchResult[]>;
   searchFTS(query: string, limit: number): Promise<SearchResult[]>;
   getByTitle(title: string): Promise<NoteRecord | null>;
+  getIndexMetadata(): Promise<IndexMetadataRecord[]>;
   getAll(): Promise<NoteRecord[]>;
   count(): Promise<number>;
   clear(): Promise<void>;
@@ -251,6 +260,18 @@ export class LanceDBStore implements VectorStore {
     debug(`Deleted record: ${folder}/${title}`);
   }
 
+  async deleteByIdAndFolderAndTitle(id: string, folder: string, title: string): Promise<void> {
+    const table = await this.ensureTable();
+    const validTitle = validateTitle(title);
+    const escapedId = escapeForFilter(id);
+    const escapedTitle = escapeForFilter(validTitle);
+    const escapedFolder = escapeForFilter(folder);
+    await table.delete(
+      `id = '${escapedId}' AND folder = '${escapedFolder}' AND title = '${escapedTitle}'`
+    );
+    debug(`Deleted record by id: ${id} (${folder}/${title})`);
+  }
+
   async search(queryVector: number[], limit: number): Promise<SearchResult[]> {
     const table = await this.ensureTable();
 
@@ -295,6 +316,22 @@ export class LanceDBStore implements VectorStore {
     return results[0] as unknown as NoteRecord;
   }
 
+  async getIndexMetadata(): Promise<IndexMetadataRecord[]> {
+    const table = await this.ensureTable();
+
+    const results = await table
+      .query()
+      .select(["id", "title", "folder", "indexed_at"])
+      .toArray();
+
+    return results.map((row): IndexMetadataRecord => ({
+      id: (row.id as string) ?? "",
+      title: row.title as string,
+      folder: row.folder as string,
+      indexed_at: row.indexed_at as string,
+    }));
+  }
+
   async getAll(): Promise<NoteRecord[]> {
     const table = await this.ensureTable();
 

package/src/index.ts

@@ -32,6 +32,13 @@ import { indexNotes, reindexNote } from "./search/indexer.js";
 import { fullChunkIndex, hasChunkIndex } from "./search/chunk-indexer.js";
 import { searchChunks } from "./search/chunk-search.js";
 import { refreshIfNeeded } from "./search/refresh.js";
+import { getIndexJobManager } from "./indexing/job-manager.js";
+import {
+  syncAfterCreate,
+  syncAfterUpdate,
+  syncAfterDelete,
+  syncAfterMove,
+} from "./search/write-sync.js";
 import { listTags, searchByTag, findRelatedNotes } from "./graph/queries.js";
 import { exportGraph } from "./graph/export.js";
 import { findTables, parseTable } from "./notes/tables.js";
@@ -85,6 +92,23 @@ const SearchNotesSchema = z.object({
 const IndexNotesSchema = z.object({
   mode: z.enum(["full", "incremental"]).default("incremental"),
   force: z.boolean().default(false),
+  background: z.boolean().optional(),
+});
+
+const StartIndexJobSchema = z.object({
+  mode: z.enum(["full", "incremental"]).default("incremental"),
+});
+
+const GetIndexJobSchema = z.object({
+  job_id: z.string().min(1),
+});
+
+const ListIndexJobsSchema = z.object({
+  limit: z.number().min(1).max(50).default(10),
+});
+
+const CancelIndexJobSchema = z.object({
+  job_id: z.string().min(1),
 });
 
 const ReindexNoteSchema = z.object({
@@ -254,10 +278,71 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
               type: "boolean",
               description: "Force reindex even if TTL hasn't expired (default: false)"
             },
+            background: {
+              type: "boolean",
+              description: "Run indexing as async background job (default: false)"
+            },
+          },
+          required: [],
+        },
+      },
+      {
+        name: "start-index-job",
+        description: "Start background indexing job and return job id for polling",
+        inputSchema: {
+          type: "object",
+          properties: {
+            mode: {
+              type: "string",
+              enum: ["full", "incremental"],
+              description: "Index mode (default: incremental)",
+            },
+          },
+          required: [],
+        },
+      },
+      {
+        name: "get-index-job",
+        description: "Get background indexing job status and progress",
+        inputSchema: {
+          type: "object",
+          properties: {
+            job_id: {
+              type: "string",
+              description: "Job ID returned from start-index-job or index-notes",
+            },
+          },
+          required: ["job_id"],
+        },
+      },
+      {
+        name: "list-index-jobs",
+        description: "List recent background indexing jobs",
+        inputSchema: {
+          type: "object",
+          properties: {
+            limit: {
+              type: "number",
+              description: "Max jobs to return (default: 10)",
+            },
           },
           required: [],
         },
       },
+      {
+        name: "cancel-index-job",
+        description: "Request cancellation for a running background indexing job",
+        inputSchema: {
+          type: "object",
+          properties: {
+            job_id: {
+              type: "string",
+              description: "Job ID returned from start-index-job or index-notes",
+            },
+          },
+          required: ["job_id"],
+        },
+      },
       {
         name: "reindex-note",
         description: "Re-index a single note after manual edits",
@@ -586,6 +671,25 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
       case "index-notes": {
         const params = IndexNotesSchema.parse(args);
+        const jobs = getIndexJobManager();
+
+        const runInBackground = params.background ?? false;
+        if (runInBackground) {
+          const job = jobs.start({ mode: params.mode });
+          return textResponse(
+            JSON.stringify(
+              {
+                message: `Started ${params.mode} index job`,
+                job_id: job.id,
+                status: job.status,
+                progress: job.progress,
+              },
+              null,
+              2
+            )
+          );
+        }
+
         const result = await indexNotes(params.mode);
 
         let message = `Indexed ${result.indexed} notes in ${(result.timeMs / 1000).toFixed(1)}s`;
@@ -626,6 +730,40 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         return textResponse(message);
       }
 
+      case "start-index-job": {
+        const params = StartIndexJobSchema.parse(args);
+        const jobs = getIndexJobManager();
+        const job = jobs.start({ mode: params.mode });
+        return textResponse(JSON.stringify(job, null, 2));
+      }
+
+      case "get-index-job": {
+        const params = GetIndexJobSchema.parse(args);
+        const jobs = getIndexJobManager();
+        const job = jobs.get(params.job_id);
+        if (!job) {
+          return errorResponse(`Index job not found: "${params.job_id}"`);
+        }
+        return textResponse(JSON.stringify(job, null, 2));
+      }
+
+      case "list-index-jobs": {
+        const params = ListIndexJobsSchema.parse(args);
+        const jobs = getIndexJobManager();
+        const list = jobs.list(params.limit);
+        return textResponse(JSON.stringify(list, null, 2));
+      }
+
+      case "cancel-index-job": {
+        const params = CancelIndexJobSchema.parse(args);
+        const jobs = getIndexJobManager();
+        const job = jobs.cancel(params.job_id);
+        if (!job) {
+          return errorResponse(`Index job not found: "${params.job_id}"`);
+        }
+        return textResponse(JSON.stringify(job, null, 2));
+      }
+
       case "reindex-note": {
         const params = ReindexNoteSchema.parse(args);
         await reindexNote(params.title);
@@ -726,9 +864,15 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       // Write tools
       case "create-note": {
         const params = CreateNoteSchema.parse(args);
-        await createNote(params.title, params.content, params.folder);
-        const location = params.folder ? `${params.folder}/${params.title}` : params.title;
-        return textResponse(`Created note: "${location}"`);
+        const result = await createNote(params.title, params.content, params.folder);
+        const location = `${result.folder}/${result.title}`;
+
+        const syncResult = await syncAfterCreate(result);
+        const syncWarning = syncResult.warnings.length > 0
+          ? ` (index sync warning: ${syncResult.warnings.join("; ")})`
+          : "";
+
+        return textResponse(`Created note: "${location}"${syncWarning}`);
       }
 
       case "update-note": {
@@ -742,15 +886,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           : "";
 
         if (params.reindex) {
-          try {
-            // Use new title for reindexing (Apple Notes may have renamed it)
-            const reindexTitle = `${result.folder}/${result.newTitle}`;
-            await reindexNote(reindexTitle);
-            return textResponse(`Updated and reindexed note: "${location}"${renamedMsg}`);
-          } catch (reindexError) {
-            debug("Reindex after update failed:", reindexError);
-            return textResponse(`Updated note: "${location}"${renamedMsg} (reindexing failed, run index-notes to update)`);
-          }
+          const syncResult = await syncAfterUpdate(result);
+          const syncWarning = syncResult.warnings.length > 0
+            ? ` (index sync warning: ${syncResult.warnings.join("; ")})`
+            : "";
+          return textResponse(`Updated and reindexed note: "${location}"${renamedMsg}${syncWarning}`);
         }
 
         return textResponse(`Updated note: "${location}"${renamedMsg}`);
@@ -761,14 +901,22 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         if (!params.confirm) {
           return errorResponse("Add confirm: true to delete the note");
         }
-        await deleteNote(params.title);
-        return textResponse(`Deleted note: "${params.title}"`);
+        const result = await deleteNote(params.title);
+        const syncResult = await syncAfterDelete(result);
+        const syncWarning = syncResult.warnings.length > 0
+          ? ` (index sync warning: ${syncResult.warnings.join("; ")})`
+          : "";
+        return textResponse(`Deleted note: "${params.title}"${syncWarning}`);
       }
 
       case "move-note": {
         const params = MoveNoteSchema.parse(args);
-        await moveNote(params.title, params.folder);
-        return textResponse(`Moved note: "${params.title}" to folder "${params.folder}"`);
+        const result = await moveNote(params.title, params.folder);
+        const syncResult = await syncAfterMove(result);
+        const syncWarning = syncResult.warnings.length > 0
+          ? ` (index sync warning: ${syncResult.warnings.join("; ")})`
+          : "";
+        return textResponse(`Moved note: "${params.title}" to folder "${params.folder}"${syncWarning}`);
       }
 
       case "edit-table": {

package/src/indexing/contracts.test.ts

@@ -0,0 +1,13 @@
+import { describe, it, expect } from "vitest";
+import { IndexCancelledError, isIndexCancelledError } from "./contracts.js";
+
+describe("indexing contracts", () => {
+  it("creates cancellable error type", () => {
+    const err = new IndexCancelledError("user requested cancel");
+    expect(isIndexCancelledError(err)).toBe(true);
+  });
+
+  it("does not treat generic Error as cancelled", () => {
+    expect(isIndexCancelledError(new Error("x"))).toBe(false);
+  });
+});

package/src/indexing/contracts.ts

@@ -0,0 +1,28 @@
+export interface IndexProgressEvent {
+  stage: "fetch" | "prepare" | "embed" | "persist" | "delete" | "rebuild-fts" | "done";
+  current: number;
+  total: number;
+  message: string;
+}
+
+export interface IndexRunOptions {
+  signal?: AbortSignal;
+  onProgress?: (event: IndexProgressEvent) => void;
+}
+
+export class IndexCancelledError extends Error {
+  constructor(message = "Indexing cancelled") {
+    super(message);
+    this.name = "IndexCancelledError";
+  }
+}
+
+export function isIndexCancelledError(error: unknown): error is IndexCancelledError {
+  return error instanceof IndexCancelledError;
+}
+
+export function throwIfCancelled(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw new IndexCancelledError();
+  }
+}