@disco_trooper/apple-notes-mcp 1.7.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -5
- package/package.json +1 -1
- package/src/config/constants.ts +5 -0
- package/src/config/env.test.ts +14 -0
- package/src/config/env.ts +2 -0
- package/src/db/lancedb.test.ts +14 -0
- package/src/db/lancedb.ts +37 -0
- package/src/index.ts +164 -16
- package/src/indexing/contracts.test.ts +13 -0
- package/src/indexing/contracts.ts +28 -0
- package/src/indexing/job-manager.test.ts +185 -0
- package/src/indexing/job-manager.ts +377 -0
- package/src/notes/crud.test.ts +33 -6
- package/src/notes/crud.ts +62 -7
- package/src/notes/read.test.ts +139 -5
- package/src/notes/read.ts +58 -5
- package/src/search/chunk-indexer.ts +69 -4
- package/src/search/indexer.progress.test.ts +75 -0
- package/src/search/indexer.ts +149 -38
- package/src/search/refresh-policy.test.ts +25 -0
- package/src/search/refresh-policy.ts +33 -0
- package/src/search/refresh.test.ts +146 -25
- package/src/search/refresh.ts +207 -47
- package/src/search/write-sync.test.ts +133 -0
- package/src/search/write-sync.ts +155 -0
package/src/notes/crud.ts
CHANGED
|
@@ -72,7 +72,7 @@ export async function createNote(
|
|
|
72
72
|
title: string,
|
|
73
73
|
content: string,
|
|
74
74
|
folder?: string
|
|
75
|
-
): Promise<
|
|
75
|
+
): Promise<CreateResult> {
|
|
76
76
|
checkReadOnly();
|
|
77
77
|
|
|
78
78
|
debug(`Creating note: "${title}" in folder: "${folder || "Notes"}"`);
|
|
@@ -85,7 +85,7 @@ export async function createNote(
|
|
|
85
85
|
|
|
86
86
|
debug(`HTML content length: ${htmlContent.length}`);
|
|
87
87
|
|
|
88
|
-
await runJxa(`
|
|
88
|
+
const result = await runJxa(`
|
|
89
89
|
const app = Application('Notes');
|
|
90
90
|
const title = ${escapedTitle};
|
|
91
91
|
const content = ${escapedContent};
|
|
@@ -109,10 +109,36 @@ export async function createNote(
|
|
|
109
109
|
const note = app.Note({name: title, body: content});
|
|
110
110
|
targetFolder.notes.push(note);
|
|
111
111
|
|
|
112
|
-
return
|
|
113
|
-
|
|
112
|
+
return JSON.stringify({
|
|
113
|
+
id: note.id(),
|
|
114
|
+
title: note.name(),
|
|
115
|
+
folder: targetFolder.name(),
|
|
116
|
+
});
|
|
117
|
+
`) as string;
|
|
118
|
+
|
|
119
|
+
const created = JSON.parse(result) as {
|
|
120
|
+
id: string;
|
|
121
|
+
title: string;
|
|
122
|
+
folder: string;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
debug(`Note created: "${created.folder}/${created.title}"`);
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
id: created.id,
|
|
129
|
+
title: created.title,
|
|
130
|
+
folder: created.folder,
|
|
131
|
+
requestedTitle: title,
|
|
132
|
+
titleChanged: created.title !== title,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
114
135
|
|
|
115
|
-
|
|
136
|
+
export interface CreateResult {
|
|
137
|
+
id: string;
|
|
138
|
+
title: string;
|
|
139
|
+
folder: string;
|
|
140
|
+
requestedTitle: string;
|
|
141
|
+
titleChanged: boolean;
|
|
116
142
|
}
|
|
117
143
|
|
|
118
144
|
/**
|
|
@@ -120,6 +146,8 @@ export async function createNote(
|
|
|
120
146
|
* Apple Notes may rename the note based on content (first h1 heading).
|
|
121
147
|
*/
|
|
122
148
|
export interface UpdateResult {
|
|
149
|
+
/** Note ID */
|
|
150
|
+
id: string;
|
|
123
151
|
/** Original title before update */
|
|
124
152
|
originalTitle: string;
|
|
125
153
|
/** Current title after update (may differ if Apple Notes renamed it) */
|
|
@@ -189,6 +217,7 @@ export async function updateNote(title: string, content: string): Promise<Update
|
|
|
189
217
|
}
|
|
190
218
|
|
|
191
219
|
return {
|
|
220
|
+
id: note.id,
|
|
192
221
|
originalTitle,
|
|
193
222
|
newTitle,
|
|
194
223
|
folder,
|
|
@@ -206,7 +235,7 @@ export async function updateNote(title: string, content: string): Promise<Update
|
|
|
206
235
|
* @throws Error if READONLY_MODE is enabled
|
|
207
236
|
* @throws Error if note not found or duplicate titles without folder prefix
|
|
208
237
|
*/
|
|
209
|
-
export async function deleteNote(title: string): Promise<
|
|
238
|
+
export async function deleteNote(title: string): Promise<DeleteResult> {
|
|
210
239
|
checkReadOnly();
|
|
211
240
|
|
|
212
241
|
debug(`Deleting note: "${title}"`);
|
|
@@ -232,6 +261,18 @@ export async function deleteNote(title: string): Promise<void> {
|
|
|
232
261
|
`);
|
|
233
262
|
|
|
234
263
|
debug(`Note deleted: "${title}"`);
|
|
264
|
+
|
|
265
|
+
return {
|
|
266
|
+
id: note.id,
|
|
267
|
+
title: note.title,
|
|
268
|
+
folder: note.folder,
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export interface DeleteResult {
|
|
273
|
+
id: string;
|
|
274
|
+
title: string;
|
|
275
|
+
folder: string;
|
|
235
276
|
}
|
|
236
277
|
|
|
237
278
|
/**
|
|
@@ -242,7 +283,7 @@ export async function deleteNote(title: string): Promise<void> {
|
|
|
242
283
|
* @throws Error if READONLY_MODE is enabled
|
|
243
284
|
* @throws Error if note not found or target folder not found
|
|
244
285
|
*/
|
|
245
|
-
export async function moveNote(title: string, folder: string): Promise<
|
|
286
|
+
export async function moveNote(title: string, folder: string): Promise<MoveResult> {
|
|
246
287
|
checkReadOnly();
|
|
247
288
|
|
|
248
289
|
debug(`Moving note: "${title}" to folder: "${folder}"`);
|
|
@@ -277,6 +318,20 @@ export async function moveNote(title: string, folder: string): Promise<void> {
|
|
|
277
318
|
`);
|
|
278
319
|
|
|
279
320
|
debug(`Note moved: "${title}" -> "${folder}"`);
|
|
321
|
+
|
|
322
|
+
return {
|
|
323
|
+
id: note.id,
|
|
324
|
+
title: note.title,
|
|
325
|
+
fromFolder: note.folder,
|
|
326
|
+
toFolder: folder,
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export interface MoveResult {
|
|
331
|
+
id: string;
|
|
332
|
+
title: string;
|
|
333
|
+
fromFolder: string;
|
|
334
|
+
toFolder: string;
|
|
280
335
|
}
|
|
281
336
|
|
|
282
337
|
export interface TableEdit {
|
package/src/notes/read.test.ts
CHANGED
|
@@ -6,7 +6,7 @@ vi.mock("run-jxa", () => ({
|
|
|
6
6
|
}));
|
|
7
7
|
|
|
8
8
|
import { runJxa } from "run-jxa";
|
|
9
|
-
import { getAllNotes, getNoteByTitle, getAllFolders, resolveNoteTitle, listNotes } from "./read.js";
|
|
9
|
+
import { getAllNotes, getNoteByTitle, getAllFolders, resolveNoteTitle, listNotes, getNoteMetadataByFolder } from "./read.js";
|
|
10
10
|
|
|
11
11
|
describe("getAllNotes", () => {
|
|
12
12
|
beforeEach(() => {
|
|
@@ -243,8 +243,12 @@ describe("listNotes", () => {
|
|
|
243
243
|
expect(notes[2].title).toBe("Alpha");
|
|
244
244
|
});
|
|
245
245
|
|
|
246
|
-
it("should filter by folder", async () => {
|
|
247
|
-
|
|
246
|
+
it("should filter by folder (uses getNoteMetadataByFolder)", async () => {
|
|
247
|
+
const workNotes = [
|
|
248
|
+
{ title: "Alpha", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-10T00:00:00Z" },
|
|
249
|
+
{ title: "Gamma", folder: "Work", created: "2024-01-02T00:00:00Z", modified: "2024-01-15T00:00:00Z" },
|
|
250
|
+
];
|
|
251
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(workNotes));
|
|
248
252
|
|
|
249
253
|
const notes = await listNotes({ folder: "Work" });
|
|
250
254
|
expect(notes).toHaveLength(2);
|
|
@@ -259,7 +263,11 @@ describe("listNotes", () => {
|
|
|
259
263
|
});
|
|
260
264
|
|
|
261
265
|
it("should combine folder filter and limit", async () => {
|
|
262
|
-
|
|
266
|
+
const workNotes = [
|
|
267
|
+
{ title: "Alpha", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-10T00:00:00Z" },
|
|
268
|
+
{ title: "Gamma", folder: "Work", created: "2024-01-02T00:00:00Z", modified: "2024-01-15T00:00:00Z" },
|
|
269
|
+
];
|
|
270
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(workNotes));
|
|
263
271
|
|
|
264
272
|
const notes = await listNotes({ folder: "Work", limit: 1 });
|
|
265
273
|
expect(notes).toHaveLength(1);
|
|
@@ -267,7 +275,7 @@ describe("listNotes", () => {
|
|
|
267
275
|
});
|
|
268
276
|
|
|
269
277
|
it("should return empty array when folder has no notes", async () => {
|
|
270
|
-
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(
|
|
278
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify([]));
|
|
271
279
|
|
|
272
280
|
const notes = await listNotes({ folder: "NonExistent" });
|
|
273
281
|
expect(notes).toHaveLength(0);
|
|
@@ -345,3 +353,129 @@ describe("listNotes", () => {
|
|
|
345
353
|
expect(notes[2].title).toBe("Recent");
|
|
346
354
|
});
|
|
347
355
|
});
|
|
356
|
+
|
|
357
|
+
describe("getNoteMetadataByFolder", () => {
|
|
358
|
+
beforeEach(() => {
|
|
359
|
+
vi.clearAllMocks();
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
it("should return notes from the specified folder", async () => {
|
|
363
|
+
const folderNotes = [
|
|
364
|
+
{ title: "Note A", folder: "Projects", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
365
|
+
{ title: "Note B", folder: "Projects", created: "2024-01-03T00:00:00Z", modified: "2024-01-04T00:00:00Z" },
|
|
366
|
+
];
|
|
367
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(folderNotes));
|
|
368
|
+
|
|
369
|
+
const notes = await getNoteMetadataByFolder("Projects");
|
|
370
|
+
expect(notes).toHaveLength(2);
|
|
371
|
+
expect(notes[0].title).toBe("Note A");
|
|
372
|
+
expect(notes[1].folder).toBe("Projects");
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
it("should return empty array when folder has no notes", async () => {
|
|
376
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify([]));
|
|
377
|
+
|
|
378
|
+
const notes = await getNoteMetadataByFolder("Empty");
|
|
379
|
+
expect(notes).toHaveLength(0);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
it("should return metadata without content fields", async () => {
|
|
383
|
+
const folderNotes = [
|
|
384
|
+
{ title: "Note", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
385
|
+
];
|
|
386
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(folderNotes));
|
|
387
|
+
|
|
388
|
+
const notes = await getNoteMetadataByFolder("Work");
|
|
389
|
+
expect(notes[0]).toHaveProperty("title");
|
|
390
|
+
expect(notes[0]).toHaveProperty("folder");
|
|
391
|
+
expect(notes[0]).toHaveProperty("created");
|
|
392
|
+
expect(notes[0]).toHaveProperty("modified");
|
|
393
|
+
expect(notes[0]).not.toHaveProperty("content");
|
|
394
|
+
expect(notes[0]).not.toHaveProperty("htmlContent");
|
|
395
|
+
expect(notes[0]).not.toHaveProperty("id");
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
it("should aggregate notes from duplicate folder names", async () => {
|
|
399
|
+
const duplicateFolderNotes = [
|
|
400
|
+
{ title: "A1", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
401
|
+
{ title: "A2", folder: "Work", created: "2024-01-03T00:00:00Z", modified: "2024-01-04T00:00:00Z" },
|
|
402
|
+
];
|
|
403
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(duplicateFolderNotes));
|
|
404
|
+
|
|
405
|
+
const notes = await getNoteMetadataByFolder("Work");
|
|
406
|
+
expect(notes).toHaveLength(2);
|
|
407
|
+
expect(notes.map((n) => n.title)).toEqual(["A1", "A2"]);
|
|
408
|
+
|
|
409
|
+
const jxaCode = vi.mocked(runJxa).mock.calls[0][0] as string;
|
|
410
|
+
expect(jxaCode).not.toContain("break;");
|
|
411
|
+
});
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
describe("listNotes folder optimization", () => {
|
|
415
|
+
beforeEach(() => {
|
|
416
|
+
vi.clearAllMocks();
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
it("should make only one JXA call when folder is specified", async () => {
|
|
420
|
+
const folderNotes = [
|
|
421
|
+
{ title: "Note 1", folder: "xx", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
422
|
+
];
|
|
423
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(folderNotes));
|
|
424
|
+
|
|
425
|
+
await listNotes({ folder: "xx" });
|
|
426
|
+
expect(runJxa).toHaveBeenCalledTimes(1);
|
|
427
|
+
|
|
428
|
+
const jxaCode = vi.mocked(runJxa).mock.calls[0][0] as string;
|
|
429
|
+
expect(jxaCode).toContain("targetFolder");
|
|
430
|
+
expect(jxaCode).toContain("toLowerCase");
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
it("should not fetch all notes when folder is specified", async () => {
|
|
434
|
+
const folderNotes = [
|
|
435
|
+
{ title: "Only One", folder: "Tiny", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
436
|
+
];
|
|
437
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(folderNotes));
|
|
438
|
+
|
|
439
|
+
const notes = await listNotes({ folder: "Tiny" });
|
|
440
|
+
expect(notes).toHaveLength(1);
|
|
441
|
+
expect(notes[0].title).toBe("Only One");
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
it("should use getAllNotes when no folder is specified", async () => {
|
|
445
|
+
const allNotes = [
|
|
446
|
+
{ title: "A", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
447
|
+
{ title: "B", folder: "Personal", created: "2024-01-03T00:00:00Z", modified: "2024-01-04T00:00:00Z" },
|
|
448
|
+
];
|
|
449
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(allNotes));
|
|
450
|
+
|
|
451
|
+
const notes = await listNotes();
|
|
452
|
+
expect(notes).toHaveLength(2);
|
|
453
|
+
expect(runJxa).toHaveBeenCalledTimes(1);
|
|
454
|
+
|
|
455
|
+
const jxaCode = vi.mocked(runJxa).mock.calls[0][0] as string;
|
|
456
|
+
expect(jxaCode).not.toContain("targetFolder");
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
it("should still sort folder-filtered results", async () => {
|
|
460
|
+
const folderNotes = [
|
|
461
|
+
{ title: "Old", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-05T00:00:00Z" },
|
|
462
|
+
{ title: "New", folder: "Work", created: "2024-01-02T00:00:00Z", modified: "2024-01-15T00:00:00Z" },
|
|
463
|
+
];
|
|
464
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(folderNotes));
|
|
465
|
+
|
|
466
|
+
const notes = await listNotes({ folder: "Work", sort_by: "modified", order: "desc" });
|
|
467
|
+
expect(notes[0].title).toBe("New");
|
|
468
|
+
expect(notes[1].title).toBe("Old");
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
it("should match folder case-insensitively in optimized path", async () => {
|
|
472
|
+
const workNotes = [
|
|
473
|
+
{ title: "Case Note", folder: "Work", created: "2024-01-01T00:00:00Z", modified: "2024-01-02T00:00:00Z" },
|
|
474
|
+
];
|
|
475
|
+
vi.mocked(runJxa).mockResolvedValueOnce(JSON.stringify(workNotes));
|
|
476
|
+
|
|
477
|
+
const notes = await listNotes({ folder: "work" });
|
|
478
|
+
expect(notes).toHaveLength(1);
|
|
479
|
+
expect(notes[0].folder).toBe("Work");
|
|
480
|
+
});
|
|
481
|
+
});
|
package/src/notes/read.ts
CHANGED
|
@@ -619,9 +619,65 @@ export type { ListNotesOptions } from "../index.js";
|
|
|
619
619
|
// Import the type for internal use
|
|
620
620
|
import type { ListNotesOptions } from "../index.js";
|
|
621
621
|
|
|
622
|
+
/**
|
|
623
|
+
* Get notes metadata from a specific folder (no content).
|
|
624
|
+
* Much faster than getAllNotes() when only one folder is needed,
|
|
625
|
+
* because it skips iterating all other folders in JXA.
|
|
626
|
+
*
|
|
627
|
+
* @param folderName - The folder name (case-insensitive matching)
|
|
628
|
+
* @returns Array of note metadata from the specified folder
|
|
629
|
+
*/
|
|
630
|
+
export async function getNoteMetadataByFolder(folderName: string): Promise<NoteInfo[]> {
|
|
631
|
+
debug(`Getting note metadata for folder: ${folderName}`);
|
|
632
|
+
|
|
633
|
+
const escapedFolder = JSON.stringify(folderName);
|
|
634
|
+
|
|
635
|
+
const jxaCode = `
|
|
636
|
+
const app = Application('Notes');
|
|
637
|
+
app.includeStandardAdditions = true;
|
|
638
|
+
|
|
639
|
+
const targetFolder = ${escapedFolder}.toLowerCase();
|
|
640
|
+
const result = [];
|
|
641
|
+
const folders = app.folders();
|
|
642
|
+
|
|
643
|
+
for (const folder of folders) {
|
|
644
|
+
const folderName = folder.name();
|
|
645
|
+
if (folderName.toLowerCase() !== targetFolder) continue;
|
|
646
|
+
|
|
647
|
+
const notes = folder.notes();
|
|
648
|
+
|
|
649
|
+
for (const note of notes) {
|
|
650
|
+
try {
|
|
651
|
+
const props = note.properties();
|
|
652
|
+
result.push({
|
|
653
|
+
title: props.name || '',
|
|
654
|
+
folder: folderName,
|
|
655
|
+
created: props.creationDate ? props.creationDate.toISOString() : '',
|
|
656
|
+
modified: props.modificationDate ? props.modificationDate.toISOString() : ''
|
|
657
|
+
});
|
|
658
|
+
} catch (e) {
|
|
659
|
+
// Skip notes that can't be accessed
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
return JSON.stringify(result);
|
|
665
|
+
`;
|
|
666
|
+
|
|
667
|
+
const result = await executeJxa<string>(jxaCode);
|
|
668
|
+
const notes = JSON.parse(result) as NoteInfo[];
|
|
669
|
+
|
|
670
|
+
debug(`Found ${notes.length} notes in folder: ${folderName}`);
|
|
671
|
+
return notes;
|
|
672
|
+
}
|
|
673
|
+
|
|
622
674
|
/**
|
|
623
675
|
* List notes with sorting and filtering.
|
|
624
676
|
*
|
|
677
|
+
* When a folder filter is provided, only that folder is queried via JXA
|
|
678
|
+
* instead of fetching all notes first. This is significantly faster for
|
|
679
|
+
* users with many notes spread across folders.
|
|
680
|
+
*
|
|
625
681
|
* @param options - Sorting and filtering options
|
|
626
682
|
* @returns Array of note metadata sorted and filtered as specified
|
|
627
683
|
*/
|
|
@@ -630,12 +686,9 @@ export async function listNotes(options: ListNotesOptions = {}): Promise<NoteInf
|
|
|
630
686
|
|
|
631
687
|
debug(`Listing notes: sort_by=${sort_by}, order=${order}, limit=${limit}, folder=${folder}`);
|
|
632
688
|
|
|
633
|
-
const allNotes = await getAllNotes();
|
|
634
|
-
|
|
635
|
-
// Filter by folder (case-insensitive for better UX)
|
|
636
689
|
const filtered = folder
|
|
637
|
-
?
|
|
638
|
-
:
|
|
690
|
+
? await getNoteMetadataByFolder(folder)
|
|
691
|
+
: await getAllNotes();
|
|
639
692
|
|
|
640
693
|
filtered.sort((a, b) => {
|
|
641
694
|
let comparison: number;
|
|
@@ -8,9 +8,18 @@ import { getChunkStore, type ChunkRecord } from "../db/lancedb.js";
|
|
|
8
8
|
import { getAllNotesWithFallback, type NoteDetails } from "../notes/read.js";
|
|
9
9
|
import { chunkText } from "../utils/chunker.js";
|
|
10
10
|
import { extractMetadata } from "../graph/extract.js";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
DEFAULT_CHUNK_SIZE,
|
|
13
|
+
DEFAULT_CHUNK_OVERLAP,
|
|
14
|
+
getEmbeddingBatchSize,
|
|
15
|
+
} from "../config/constants.js";
|
|
12
16
|
import { createDebugLogger } from "../utils/debug.js";
|
|
13
17
|
import { filterContent, shouldIndexContent } from "../utils/content-filter.js";
|
|
18
|
+
import {
|
|
19
|
+
type IndexRunOptions,
|
|
20
|
+
type IndexProgressEvent,
|
|
21
|
+
throwIfCancelled,
|
|
22
|
+
} from "../indexing/contracts.js";
|
|
14
23
|
|
|
15
24
|
// Debug logging
|
|
16
25
|
const debug = createDebugLogger("CHUNK-INDEXER");
|
|
@@ -48,6 +57,29 @@ interface InternalChunkRecord {
|
|
|
48
57
|
outlinks: string[];
|
|
49
58
|
}
|
|
50
59
|
|
|
60
|
+
function chunks<T>(array: T[], size: number): T[][] {
|
|
61
|
+
const result: T[][] = [];
|
|
62
|
+
for (let i = 0; i < array.length; i += size) {
|
|
63
|
+
result.push(array.slice(i, i + size));
|
|
64
|
+
}
|
|
65
|
+
return result;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function emitProgress(
|
|
69
|
+
options: IndexRunOptions,
|
|
70
|
+
stage: IndexProgressEvent["stage"],
|
|
71
|
+
current: number,
|
|
72
|
+
total: number,
|
|
73
|
+
message: string
|
|
74
|
+
): void {
|
|
75
|
+
options.onProgress?.({
|
|
76
|
+
stage,
|
|
77
|
+
current,
|
|
78
|
+
total,
|
|
79
|
+
message,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
51
83
|
/**
|
|
52
84
|
* Convert a note into chunk records WITHOUT vectors.
|
|
53
85
|
* Vectors are added later during batch embedding generation.
|
|
@@ -125,13 +157,17 @@ export function chunkNote(note: NoteDetails): InternalChunkRecord[] {
|
|
|
125
157
|
*
|
|
126
158
|
* @returns ChunkIndexResult with stats
|
|
127
159
|
*/
|
|
128
|
-
export async function fullChunkIndex(): Promise<ChunkIndexResult> {
|
|
160
|
+
export async function fullChunkIndex(options: IndexRunOptions = {}): Promise<ChunkIndexResult> {
|
|
129
161
|
const startTime = Date.now();
|
|
162
|
+
throwIfCancelled(options.signal);
|
|
163
|
+
emitProgress(options, "fetch", 0, 1, "Fetching notes for chunk index");
|
|
130
164
|
|
|
131
165
|
// Phase 1: Fetch all notes with hybrid fallback
|
|
132
166
|
debug("Phase 1: Fetching all notes with fallback...");
|
|
133
167
|
const { notes, skipped: skippedNotes } = await getAllNotesWithFallback();
|
|
134
168
|
debug(`Fetched ${notes.length} notes, skipped ${skippedNotes.length}`);
|
|
169
|
+
emitProgress(options, "fetch", 1, 1, `Fetched ${notes.length} notes`);
|
|
170
|
+
throwIfCancelled(options.signal);
|
|
135
171
|
|
|
136
172
|
if (notes.length === 0) {
|
|
137
173
|
return {
|
|
@@ -147,10 +183,13 @@ export async function fullChunkIndex(): Promise<ChunkIndexResult> {
|
|
|
147
183
|
debug("Phase 2: Chunking all notes...");
|
|
148
184
|
const allChunks: InternalChunkRecord[] = [];
|
|
149
185
|
for (const note of notes) {
|
|
186
|
+
throwIfCancelled(options.signal);
|
|
150
187
|
const noteChunks = chunkNote(note);
|
|
151
188
|
allChunks.push(...noteChunks);
|
|
152
189
|
}
|
|
153
190
|
debug(`Created ${allChunks.length} chunks from ${notes.length} notes`);
|
|
191
|
+
emitProgress(options, "prepare", allChunks.length, Math.max(notes.length, 1), "Prepared note chunks");
|
|
192
|
+
throwIfCancelled(options.signal);
|
|
154
193
|
|
|
155
194
|
if (allChunks.length === 0) {
|
|
156
195
|
return {
|
|
@@ -163,9 +202,32 @@ export async function fullChunkIndex(): Promise<ChunkIndexResult> {
|
|
|
163
202
|
|
|
164
203
|
// Phase 3: Generate embeddings in batch
|
|
165
204
|
debug("Phase 3: Generating embeddings...");
|
|
166
|
-
const
|
|
167
|
-
const vectors =
|
|
205
|
+
const chunkBatches = chunks(allChunks, getEmbeddingBatchSize());
|
|
206
|
+
const vectors: number[][] = [];
|
|
207
|
+
for (let batchIdx = 0; batchIdx < chunkBatches.length; batchIdx++) {
|
|
208
|
+
throwIfCancelled(options.signal);
|
|
209
|
+
emitProgress(
|
|
210
|
+
options,
|
|
211
|
+
"embed",
|
|
212
|
+
batchIdx,
|
|
213
|
+
chunkBatches.length,
|
|
214
|
+
`Embedding chunk batch ${batchIdx + 1}/${chunkBatches.length}`
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
const batchTexts = chunkBatches[batchIdx].map((chunk) => chunk.content);
|
|
218
|
+
const batchVectors = await getEmbeddingBatch(batchTexts);
|
|
219
|
+
vectors.push(...batchVectors);
|
|
220
|
+
|
|
221
|
+
emitProgress(
|
|
222
|
+
options,
|
|
223
|
+
"embed",
|
|
224
|
+
batchIdx + 1,
|
|
225
|
+
chunkBatches.length,
|
|
226
|
+
`Embedded chunk batch ${batchIdx + 1}/${chunkBatches.length}`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
168
229
|
debug(`Generated ${vectors.length} embeddings`);
|
|
230
|
+
throwIfCancelled(options.signal);
|
|
169
231
|
|
|
170
232
|
// Phase 4: Combine chunks with vectors and set indexed_at
|
|
171
233
|
debug("Phase 4: Combining chunks with vectors...");
|
|
@@ -179,11 +241,14 @@ export async function fullChunkIndex(): Promise<ChunkIndexResult> {
|
|
|
179
241
|
// Phase 5: Store in LanceDB
|
|
180
242
|
debug("Phase 5: Storing chunks...");
|
|
181
243
|
const chunkStore = getChunkStore();
|
|
244
|
+
emitProgress(options, "persist", 0, 1, "Storing chunk vectors");
|
|
182
245
|
await chunkStore.indexChunks(completeChunks);
|
|
246
|
+
emitProgress(options, "persist", 1, 1, "Stored chunk vectors");
|
|
183
247
|
debug(`Stored ${completeChunks.length} chunks`);
|
|
184
248
|
|
|
185
249
|
const timeMs = Date.now() - startTime;
|
|
186
250
|
debug(`Chunk indexing completed in ${timeMs}ms`);
|
|
251
|
+
emitProgress(options, "done", 1, 1, "Chunk index completed");
|
|
187
252
|
|
|
188
253
|
return {
|
|
189
254
|
totalNotes: notes.length,
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
|
|
3
|
+
const mockStore = {
|
|
4
|
+
clear: vi.fn(),
|
|
5
|
+
index: vi.fn(),
|
|
6
|
+
addRecords: vi.fn(),
|
|
7
|
+
rebuildFtsIndex: vi.fn(),
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
vi.mock("../db/lancedb.js", () => ({
|
|
11
|
+
getVectorStore: vi.fn(() => mockStore),
|
|
12
|
+
}));
|
|
13
|
+
|
|
14
|
+
vi.mock("../notes/read.js", () => ({
|
|
15
|
+
getAllNotesWithFallback: vi.fn().mockResolvedValue({
|
|
16
|
+
notes: [
|
|
17
|
+
{
|
|
18
|
+
id: "n1",
|
|
19
|
+
title: "Note 1",
|
|
20
|
+
folder: "Work",
|
|
21
|
+
content: "content",
|
|
22
|
+
created: "2026-01-01T00:00:00.000Z",
|
|
23
|
+
modified: "2026-01-01T00:00:00.000Z",
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
skipped: [],
|
|
27
|
+
}),
|
|
28
|
+
getNoteByTitle: vi.fn(),
|
|
29
|
+
}));
|
|
30
|
+
|
|
31
|
+
vi.mock("../embeddings/index.js", () => ({
|
|
32
|
+
getEmbedding: vi.fn(),
|
|
33
|
+
getEmbeddingBatch: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]),
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
vi.mock("../config/constants.js", () => ({
|
|
37
|
+
getEmbeddingBatchSize: vi.fn(() => 50),
|
|
38
|
+
}));
|
|
39
|
+
|
|
40
|
+
vi.mock("../graph/extract.js", () => ({
|
|
41
|
+
extractMetadata: vi.fn(() => ({ tags: [], outlinks: [] })),
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
vi.mock("../utils/text.js", () => ({
|
|
45
|
+
truncateForEmbedding: vi.fn((content: string) => content),
|
|
46
|
+
}));
|
|
47
|
+
|
|
48
|
+
vi.mock("../utils/debug.js", () => ({
|
|
49
|
+
createDebugLogger: vi.fn(() => vi.fn()),
|
|
50
|
+
}));
|
|
51
|
+
|
|
52
|
+
describe("indexer progress/cancel safety", () => {
|
|
53
|
+
beforeEach(() => {
|
|
54
|
+
vi.clearAllMocks();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("does not call clear during full index", async () => {
|
|
58
|
+
const { fullIndex } = await import("./indexer.js");
|
|
59
|
+
|
|
60
|
+
await fullIndex();
|
|
61
|
+
|
|
62
|
+
expect(mockStore.clear).not.toHaveBeenCalled();
|
|
63
|
+
expect(mockStore.index).toHaveBeenCalledTimes(1);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("throws cancellation before indexing when signal is already aborted", async () => {
|
|
67
|
+
const { fullIndex } = await import("./indexer.js");
|
|
68
|
+
const controller = new AbortController();
|
|
69
|
+
controller.abort();
|
|
70
|
+
|
|
71
|
+
await expect(fullIndex({ signal: controller.signal })).rejects.toThrow("Indexing cancelled");
|
|
72
|
+
expect(mockStore.clear).not.toHaveBeenCalled();
|
|
73
|
+
expect(mockStore.index).not.toHaveBeenCalled();
|
|
74
|
+
});
|
|
75
|
+
});
|