@disco_trooper/apple-notes-mcp 1.7.0 → 1.8.2

package/src/search/refresh.test.ts

@@ -1,24 +1,36 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 
 vi.mock("../notes/read.js", () => ({
   getAllNotes: vi.fn(),
+  getNoteByFolderAndTitle: vi.fn(),
 }));
 
 vi.mock("../db/lancedb.js", () => ({
   getVectorStore: vi.fn(),
+  getChunkStore: vi.fn(),
 }));
 
 vi.mock("./indexer.js", () => ({
   incrementalIndex: vi.fn(),
 }));
 
+vi.mock("./chunk-indexer.js", () => ({
+  hasChunkIndex: vi.fn().mockResolvedValue(false),
+  updateChunksForNotes: vi.fn(),
+}));
+
 describe("checkForChanges", () => {
   beforeEach(() => {
     vi.resetModules();
     vi.clearAllMocks();
   });
 
-  it("should return true if notes were modified after indexing", async () => {
+  afterEach(() => {
+    delete process.env.INDEX_TTL;
+    delete process.env.SEARCH_REFRESH_TIMEOUT_MS;
+  });
+
+  it("returns true if notes were modified after indexing", async () => {
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
@@ -27,8 +39,8 @@ describe("checkForChanges", () => {
     ]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([
-        { title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
       ]),
     } as any);
 
@@ -38,7 +50,7 @@ describe("checkForChanges", () => {
     expect(hasChanges).toBe(true);
   });
 
-  it("should return false if no changes", async () => {
+  it("returns false if no changes", async () => {
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
@@ -47,8 +59,8 @@ describe("checkForChanges", () => {
     ]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([
-        { title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
       ]),
     } as any);
 
@@ -58,7 +70,7 @@ describe("checkForChanges", () => {
     expect(hasChanges).toBe(false);
   });
 
-  it("should return true if new note added", async () => {
+  it("returns true if a new note was added", async () => {
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
@@ -68,8 +80,8 @@ describe("checkForChanges", () => {
     ]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([
-        { title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
       ]),
     } as any);
 
@@ -79,15 +91,15 @@ describe("checkForChanges", () => {
     expect(hasChanges).toBe(true);
   });
 
-  it("should return true if note deleted", async () => {
+  it("returns true if a note was deleted", async () => {
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
     vi.mocked(getAllNotes).mockResolvedValue([]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([
-        { title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
       ]),
     } as any);
 
@@ -97,7 +109,7 @@ describe("checkForChanges", () => {
     expect(hasChanges).toBe(true);
   });
 
-  it("should return true if no index exists and notes exist", async () => {
+  it("returns true if no index exists and notes exist", async () => {
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
@@ -106,7 +118,7 @@ describe("checkForChanges", () => {
     ]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockRejectedValue(new Error("Table not found")),
+      getIndexMetadata: vi.fn().mockRejectedValue(new Error("Table not found")),
     } as any);
 
     const { checkForChanges } = await import("./refresh.js");
@@ -120,19 +132,41 @@ describe("refreshIfNeeded", () => {
   beforeEach(() => {
     vi.resetModules();
     vi.clearAllMocks();
+    delete process.env.INDEX_TTL;
+    delete process.env.SEARCH_REFRESH_TIMEOUT_MS;
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("skips refresh when INDEX_TTL is not configured", async () => {
+    const { incrementalIndex } = await import("./indexer.js");
+    const { getVectorStore } = await import("../db/lancedb.js");
+
+    const { refreshIfNeeded } = await import("./refresh.js");
+    const refreshed = await refreshIfNeeded();
+
+    expect(refreshed).toBe(false);
+    expect(incrementalIndex).not.toHaveBeenCalled();
+    expect(getVectorStore).not.toHaveBeenCalled();
   });
 
-  it("should trigger incremental index if changes detected", async () => {
+  it("triggers incremental index when TTL expired and changes detected", async () => {
+    process.env.INDEX_TTL = "1";
+
     const { incrementalIndex } = await import("./indexer.js");
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
     vi.mocked(getAllNotes).mockResolvedValue([
-      { title: "New Note", folder: "Work", created: "2026-01-10", modified: "2026-01-10T12:00:00Z" },
+      { title: "Note 1", folder: "Work", created: "2026-01-01", modified: "2026-01-10T12:00:00Z" },
     ]);
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([]),
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2020-01-01T00:00:00Z" },
+      ]),
     } as any);
 
     vi.mocked(incrementalIndex).mockResolvedValue({
@@ -146,21 +180,45 @@ describe("refreshIfNeeded", () => {
     const refreshed = await refreshIfNeeded();
 
     expect(refreshed).toBe(true);
-    expect(incrementalIndex).toHaveBeenCalled();
+    expect(incrementalIndex).toHaveBeenCalledOnce();
   });
 
-  it("should not trigger index if no changes", async () => {
+  it("skips refresh when TTL is not expired", async () => {
+    process.env.INDEX_TTL = "86400";
+
+    const { incrementalIndex } = await import("./indexer.js");
+    const { getVectorStore } = await import("../db/lancedb.js");
+
+    vi.mocked(getVectorStore).mockReturnValue({
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        {
+          id: "1",
+          title: "Note 1",
+          folder: "Work",
+          indexed_at: new Date().toISOString(),
+        },
+      ]),
+    } as any);
+
+    const { refreshIfNeeded } = await import("./refresh.js");
+    const refreshed = await refreshIfNeeded();
+
+    expect(refreshed).toBe(false);
+    expect(incrementalIndex).not.toHaveBeenCalled();
+  });
+
+  it("returns false when refresh throws", async () => {
+    process.env.INDEX_TTL = "1";
+
     const { incrementalIndex } = await import("./indexer.js");
     const { getAllNotes } = await import("../notes/read.js");
     const { getVectorStore } = await import("../db/lancedb.js");
 
-    vi.mocked(getAllNotes).mockResolvedValue([
-      { title: "Note 1", folder: "Work", created: "2026-01-01", modified: "2026-01-08T12:00:00Z" },
-    ]);
+    vi.mocked(getAllNotes).mockRejectedValue(new Error("JXA failed"));
 
     vi.mocked(getVectorStore).mockReturnValue({
-      getAll: vi.fn().mockResolvedValue([
-        { title: "Note 1", folder: "Work", indexed_at: "2026-01-09T12:00:00Z" },
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2020-01-01T00:00:00Z" },
       ]),
     } as any);
 
@@ -170,4 +228,67 @@ describe("refreshIfNeeded", () => {
     expect(refreshed).toBe(false);
     expect(incrementalIndex).not.toHaveBeenCalled();
   });
+
+  it("returns false when refresh exceeds timeout budget", async () => {
+    vi.useFakeTimers();
+
+    process.env.INDEX_TTL = "1";
+    process.env.SEARCH_REFRESH_TIMEOUT_MS = "5";
+
+    const { incrementalIndex } = await import("./indexer.js");
+    const { getAllNotes } = await import("../notes/read.js");
+    const { getVectorStore } = await import("../db/lancedb.js");
+
+    vi.mocked(getAllNotes).mockResolvedValue([
+      { title: "Note 1", folder: "Work", created: "2026-01-01", modified: "2026-01-10T12:00:00Z" },
+    ]);
+
+    vi.mocked(getVectorStore).mockReturnValue({
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2020-01-01T00:00:00Z" },
+      ]),
+    } as any);
+
+    vi.mocked(incrementalIndex).mockImplementation(
+      () => new Promise(() => {
+        // Intentionally never resolves
+      })
+    );
+
+    const { refreshIfNeeded } = await import("./refresh.js");
+    const pending = refreshIfNeeded();
+
+    await vi.advanceTimersByTimeAsync(10);
+    await expect(pending).resolves.toBe(false);
+  });
+
+  it("coalesces concurrent refresh calls into a single run", async () => {
+    process.env.INDEX_TTL = "1";
+
+    const { incrementalIndex } = await import("./indexer.js");
+    const { getAllNotes } = await import("../notes/read.js");
+    const { getVectorStore } = await import("../db/lancedb.js");
+
+    vi.mocked(getAllNotes).mockResolvedValue([
+      { title: "Note 1", folder: "Work", created: "2026-01-01", modified: "2026-01-10T12:00:00Z" },
+    ]);
+
+    vi.mocked(getVectorStore).mockReturnValue({
+      getIndexMetadata: vi.fn().mockResolvedValue([
+        { id: "1", title: "Note 1", folder: "Work", indexed_at: "2020-01-01T00:00:00Z" },
+      ]),
+    } as any);
+
+    vi.mocked(incrementalIndex).mockImplementation(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 20));
+      return { total: 1, indexed: 1, errors: 0, timeMs: 100 };
+    });
+
+    const { refreshIfNeeded } = await import("./refresh.js");
+    const [first, second] = await Promise.all([refreshIfNeeded(), refreshIfNeeded()]);
+
+    expect(first).toBe(true);
+    expect(second).toBe(true);
+    expect(incrementalIndex).toHaveBeenCalledTimes(1);
+  });
 });

package/src/search/refresh.ts

@@ -5,12 +5,26 @@
  */
 
 import { getAllNotes, getNoteByFolderAndTitle, type NoteInfo } from "../notes/read.js";
-import { getVectorStore, getChunkStore } from "../db/lancedb.js";
+import {
+  getVectorStore,
+  getChunkStore,
+  type IndexMetadataRecord,
+} from "../db/lancedb.js";
 import { incrementalIndex } from "./indexer.js";
 import { updateChunksForNotes, hasChunkIndex } from "./chunk-indexer.js";
 import { createDebugLogger } from "../utils/debug.js";
+import { DEFAULT_SEARCH_REFRESH_TIMEOUT_MS } from "../config/constants.js";
+import { shouldAutoRefreshByTtl } from "./refresh-policy.js";
 
 const debug = createDebugLogger("REFRESH");
+let refreshInFlight: Promise<boolean> | null = null;
+
+interface LegacyIndexRecord {
+  id?: string;
+  title: string;
+  folder: string;
+  indexed_at: string;
+}
 
 /**
  * Detected changes in notes.
@@ -22,27 +36,144 @@ interface DetectedChanges {
   deleted: string[]; // note IDs
 }
 
+async function getIndexMetadata(): Promise<IndexMetadataRecord[]> {
+  const store = getVectorStore() as {
+    getIndexMetadata?: () => Promise<IndexMetadataRecord[]>;
+    getAll?: () => Promise<LegacyIndexRecord[]>;
+  };
+
+  if (typeof store.getIndexMetadata === "function") {
+    return store.getIndexMetadata();
+  }
+
+  if (typeof store.getAll === "function") {
+    const legacyRecords = await store.getAll();
+    return legacyRecords.map((record) => ({
+      id: record.id ?? "",
+      title: record.title,
+      folder: record.folder,
+      indexed_at: record.indexed_at,
+    }));
+  }
+
+  throw new Error("Vector store does not expose index metadata methods");
+}
+
+function getLatestIndexedAtMs(records: IndexMetadataRecord[]): number | null {
+  let latestMs: number | null = null;
+
+  for (const record of records) {
+    const indexedAtMs = Date.parse(record.indexed_at);
+    if (Number.isNaN(indexedAtMs)) {
+      continue;
+    }
+    if (latestMs === null || indexedAtMs > latestMs) {
+      latestMs = indexedAtMs;
+    }
+  }
+
+  return latestMs;
+}
+
+function getSearchRefreshTimeoutMs(): number {
+  const raw = process.env.SEARCH_REFRESH_TIMEOUT_MS;
+  if (!raw) {
+    return DEFAULT_SEARCH_REFRESH_TIMEOUT_MS;
+  }
+
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return DEFAULT_SEARCH_REFRESH_TIMEOUT_MS;
+  }
+
+  return parsed;
+}
+
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
+  let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
+
+  const timeoutPromise = new Promise<null>((resolve) => {
+    timeoutHandle = setTimeout(() => resolve(null), timeoutMs);
+  });
+
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    if (timeoutHandle !== undefined) {
+      clearTimeout(timeoutHandle);
+    }
+  }
+}
+
+async function runRefresh(existingRecords: IndexMetadataRecord[]): Promise<boolean> {
+  const changes = await detectChanges(existingRecords);
+
+  if (!changes.hasChanges) {
+    return false;
+  }
+
+  // Update main index
+  debug("Changes detected, running incremental index...");
+  const result = await incrementalIndex();
+  debug(`Main index refresh: ${result.indexed} notes updated in ${result.timeMs}ms`);
+
+  // Update chunk index if it exists and there are changes
+  const hasChunks = await hasChunkIndex();
+  if (hasChunks && (changes.added.length > 0 || changes.modified.length > 0)) {
+    debug("Updating chunk index for changed notes...");
+
+    // Fetch full content for changed notes
+    const changedNotes = [...changes.added, ...changes.modified];
+    const notesWithContent = await Promise.all(
+      changedNotes.map(async (n) => {
+        const note = await getNoteByFolderAndTitle(n.folder, n.title);
+        return note;
+      })
+    );
+
+    // Filter out nulls (notes that couldn't be fetched)
+    const validNotes = notesWithContent.filter((n) => n !== null);
+
+    if (validNotes.length > 0) {
+      const chunksCreated = await updateChunksForNotes(validNotes);
+      debug(`Chunk index refresh: ${chunksCreated} chunks for ${validNotes.length} notes`);
+    }
+
+    // Delete chunks for deleted notes
+    if (changes.deleted.length > 0) {
+      const chunkStore = getChunkStore();
+      await chunkStore.deleteChunksByNoteIds(changes.deleted);
+      debug(`Deleted chunks for ${changes.deleted.length} notes`);
+    }
+  }
+
+  return true;
+}
+
 /**
  * Check for note changes and return details about what changed.
  */
-export async function detectChanges(): Promise<DetectedChanges> {
+export async function detectChanges(
+  existingIndexMetadata?: IndexMetadataRecord[]
+): Promise<DetectedChanges> {
   debug("Checking for changes...");
 
   const currentNotes = await getAllNotes();
-  const store = getVectorStore();
 
-  let existingRecords;
-  try {
-    existingRecords = await store.getAll();
-  } catch {
-    // No index exists yet
-    debug("No existing index found");
-    return {
-      hasChanges: currentNotes.length > 0,
-      added: currentNotes,
-      modified: [],
-      deleted: [],
-    };
+  let existingRecords = existingIndexMetadata;
+  if (!existingRecords) {
+    try {
+      existingRecords = await getIndexMetadata();
+    } catch {
+      // No index exists yet
+      debug("No existing index found");
+      return {
+        hasChanges: currentNotes.length > 0,
+        added: currentNotes,
+        modified: [],
+        deleted: [],
+      };
+    }
   }
 
   // Build lookup maps
@@ -106,46 +237,75 @@ export async function checkForChanges(): Promise<boolean> {
  * @returns true if index was refreshed, false if no changes
  */
 export async function refreshIfNeeded(): Promise<boolean> {
-  const changes = await detectChanges();
+  try {
+    const ttlRaw = process.env.INDEX_TTL;
+    if (!ttlRaw) {
+      debug("Auto-refresh disabled (INDEX_TTL not set)");
+      return false;
+    }
 
-  if (!changes.hasChanges) {
-    return false;
-  }
+    const parsedTtlSeconds = Number.parseInt(ttlRaw, 10);
+    if (!Number.isFinite(parsedTtlSeconds) || parsedTtlSeconds <= 0) {
+      debug("Auto-refresh disabled (INDEX_TTL invalid)");
+      return false;
+    }
 
-  // Update main index
-  debug("Changes detected, running incremental index...");
-  const result = await incrementalIndex();
-  debug(`Main index refresh: ${result.indexed} notes updated in ${result.timeMs}ms`);
+    const timeoutMs = getSearchRefreshTimeoutMs();
+    if (refreshInFlight) {
+      debug("Auto-refresh already in progress, waiting for existing run");
+      const sharedResult = await withTimeout(refreshInFlight, timeoutMs);
+      if (sharedResult === null) {
+        debug(`Auto-refresh wait timed out after ${timeoutMs}ms; returning stale index results`);
+        return false;
+      }
+      return sharedResult;
+    }
 
-  // Update chunk index if it exists and there are changes
-  const hasChunks = await hasChunkIndex();
-  if (hasChunks && (changes.added.length > 0 || changes.modified.length > 0)) {
-    debug("Updating chunk index for changed notes...");
+    const refreshTask = (async () => {
+      let existingRecords: IndexMetadataRecord[] = [];
 
-    // Fetch full content for changed notes
-    const changedNotes = [...changes.added, ...changes.modified];
-    const notesWithContent = await Promise.all(
-      changedNotes.map(async (n) => {
-        const note = await getNoteByFolderAndTitle(n.folder, n.title);
-        return note;
+      try {
+        existingRecords = await getIndexMetadata();
+      } catch {
+        // No index yet - treat as empty metadata
+        existingRecords = [];
+      }
+
+      const lastIndexedAtMs = getLatestIndexedAtMs(existingRecords);
+      const shouldRefresh = shouldAutoRefreshByTtl(
+        ttlRaw,
+        Date.now(),
+        lastIndexedAtMs
+      );
+
+      if (!shouldRefresh) {
+        debug("Auto-refresh skipped by TTL policy");
+        return false;
+      }
+
+      return runRefresh(existingRecords);
+    })()
+      .catch((error) => {
+        debug("Auto-refresh failed; returning stale index results", error);
+        return false;
       })
-    );
+      .finally(() => {
+        if (refreshInFlight === refreshTask) {
+          refreshInFlight = null;
+        }
+      });
 
-    // Filter out nulls (notes that couldn't be fetched)
-    const validNotes = notesWithContent.filter((n) => n !== null);
+    refreshInFlight = refreshTask;
 
-    if (validNotes.length > 0) {
-      const chunksCreated = await updateChunksForNotes(validNotes);
-      debug(`Chunk index refresh: ${chunksCreated} chunks for ${validNotes.length} notes`);
+    const refreshed = await withTimeout(refreshTask, timeoutMs);
+    if (refreshed === null) {
+      debug(`Auto-refresh timed out after ${timeoutMs}ms; returning stale index results`);
+      return false;
     }
 
-    // Delete chunks for deleted notes
-    if (changes.deleted.length > 0) {
-      const chunkStore = getChunkStore();
-      await chunkStore.deleteChunksByNoteIds(changes.deleted);
-      debug(`Deleted chunks for ${changes.deleted.length} notes`);
-    }
+    return refreshed;
+  } catch (error) {
+    debug("Unexpected auto-refresh failure; returning stale index results", error);
+    return false;
   }
-
-  return true;
 }