apple-tools-mcp 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +53 -83
- package/indexer.js +26 -21
- package/lib/audit.js +22 -14
- package/lib/indexGate.js +49 -0
- package/lib/lancedbTables.js +307 -0
- package/package.json +1 -1
- package/search.js +4 -18
package/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
import fs from "fs";
|
|
10
10
|
import path from "path";
|
|
11
11
|
import { validateEmailPath, stripHtmlTags, unfoldRfc822Headers, validateLimit, validateDaysBack, validateWeekOffset, toUnixMillis } from "./lib/validators.js";
|
|
12
|
-
import {
|
|
12
|
+
import { cycleEndFlags, indexUnavailableMessage, indexQueryGate } from "./lib/indexGate.js";
|
|
13
13
|
import { isIndexerMode } from "./lib/processMode.js";
|
|
14
14
|
import { loadResolvedIndexInterval, logResolvedInterval } from "./lib/config.js";
|
|
15
15
|
import { createIndexerLock, DEFAULT_LOCK_HEARTBEAT_MS } from "./lib/indexerLock.js";
|
|
@@ -180,14 +180,6 @@ async function checkIfFirstRun() {
|
|
|
180
180
|
return !(emailsReady || messagesReady || calendarReady);
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
-
// Get appropriate status message based on indexing state
|
|
184
|
-
function getIndexingMessage() {
|
|
185
|
-
if (isFirstEverRun) {
|
|
186
|
-
return "Building initial index. This may take several minutes on first run. Please try again shortly.";
|
|
187
|
-
}
|
|
188
|
-
return "Indexing new data. Please try again in a moment.";
|
|
189
|
-
}
|
|
190
|
-
|
|
191
183
|
// Run a single indexing cycle (called by background timer)
|
|
192
184
|
function runIndexCycle() {
|
|
193
185
|
const cycle = beginIndexCycle(indexingInProgress);
|
|
@@ -318,10 +310,30 @@ function applyCycleEnd(success) {
|
|
|
318
310
|
// Index-backed tools wait only while THIS process owns the lock and has not
|
|
319
311
|
// finished its cycle. Lost-lock secondaries fall through to isIndexReady().
|
|
320
312
|
function stillIndexingMessage() {
|
|
321
|
-
|
|
322
|
-
|
|
313
|
+
const gate = indexQueryGate({
|
|
314
|
+
sessionIndexComplete,
|
|
315
|
+
ownsIndexLock,
|
|
316
|
+
indexReady: true,
|
|
317
|
+
isFirstEverRun
|
|
318
|
+
});
|
|
319
|
+
return gate.ok ? null : gate.message;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Shared preflight for index-backed query tools (on-disk tables, not local cycle). */
|
|
323
|
+
async function requireIndex(type) {
|
|
324
|
+
const indexing = stillIndexingMessage();
|
|
325
|
+
if (indexing) {
|
|
326
|
+
return indexing;
|
|
323
327
|
}
|
|
324
|
-
|
|
328
|
+
const ready = await isIndexReady(type);
|
|
329
|
+
const gate = indexQueryGate({
|
|
330
|
+
sessionIndexComplete,
|
|
331
|
+
ownsIndexLock,
|
|
332
|
+
indexReady: ready,
|
|
333
|
+
type,
|
|
334
|
+
isFirstEverRun
|
|
335
|
+
});
|
|
336
|
+
return gate.ok ? null : gate.message;
|
|
325
337
|
}
|
|
326
338
|
|
|
327
339
|
function waitForLockAndStartDaemon() {
|
|
@@ -355,7 +367,8 @@ async function initializeIndexing() {
|
|
|
355
367
|
if (!startup.startBackground) {
|
|
356
368
|
console.error("Another apple-tools-mcp instance is indexing. Server will run without background indexing.");
|
|
357
369
|
// Lost lock is not "still indexing": this process will never complete a
|
|
358
|
-
// local cycle. Searches proceed whenever isIndexReady() is true
|
|
370
|
+
// local cycle. Searches proceed whenever isIndexReady() is true (initDB
|
|
371
|
+
// re-lists shared on-disk tables; it must not cache an empty first connect).
|
|
359
372
|
ownsIndexLock = false;
|
|
360
373
|
return;
|
|
361
374
|
}
|
|
@@ -379,14 +392,9 @@ async function mailSearch(query, options = {}) {
|
|
|
379
392
|
return "Error: query parameter is required for mail_search";
|
|
380
393
|
}
|
|
381
394
|
|
|
382
|
-
const
|
|
383
|
-
if (
|
|
384
|
-
return
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
const ready = await isIndexReady("emails");
|
|
388
|
-
if (!ready) {
|
|
389
|
-
return indexUnavailableMessage("emails");
|
|
395
|
+
const blocked = await requireIndex("emails");
|
|
396
|
+
if (blocked) {
|
|
397
|
+
return blocked;
|
|
390
398
|
}
|
|
391
399
|
|
|
392
400
|
const result = await searchEmails(query, options);
|
|
@@ -394,14 +402,9 @@ async function mailSearch(query, options = {}) {
|
|
|
394
402
|
}
|
|
395
403
|
|
|
396
404
|
async function mailRecent(limit = 30, daysBack = 7, unreadOnly = false, includeJunk = false) {
|
|
397
|
-
const
|
|
398
|
-
if (
|
|
399
|
-
return
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
const ready = await isIndexReady("emails");
|
|
403
|
-
if (!ready) {
|
|
404
|
-
return indexUnavailableMessage("emails");
|
|
405
|
+
const blocked = await requireIndex("emails");
|
|
406
|
+
if (blocked) {
|
|
407
|
+
return blocked;
|
|
405
408
|
}
|
|
406
409
|
|
|
407
410
|
const result = await getRecentEmailResults(limit, daysBack, unreadOnly, includeJunk);
|
|
@@ -409,14 +412,9 @@ async function mailRecent(limit = 30, daysBack = 7, unreadOnly = false, includeJ
|
|
|
409
412
|
}
|
|
410
413
|
|
|
411
414
|
async function mailDate(date, includeJunk = false) {
|
|
412
|
-
const
|
|
413
|
-
if (
|
|
414
|
-
return
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
const ready = await isIndexReady("emails");
|
|
418
|
-
if (!ready) {
|
|
419
|
-
return indexUnavailableMessage("emails");
|
|
415
|
+
const blocked = await requireIndex("emails");
|
|
416
|
+
if (blocked) {
|
|
417
|
+
return blocked;
|
|
420
418
|
}
|
|
421
419
|
|
|
422
420
|
const result = await getEmailDateResults(date, includeJunk);
|
|
@@ -424,14 +422,9 @@ async function mailDate(date, includeJunk = false) {
|
|
|
424
422
|
}
|
|
425
423
|
|
|
426
424
|
async function messagesSearch(query, options = {}) {
|
|
427
|
-
const
|
|
428
|
-
if (
|
|
429
|
-
return
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
const ready = await isIndexReady("messages");
|
|
433
|
-
if (!ready) {
|
|
434
|
-
return indexUnavailableMessage("messages");
|
|
425
|
+
const blocked = await requireIndex("messages");
|
|
426
|
+
if (blocked) {
|
|
427
|
+
return blocked;
|
|
435
428
|
}
|
|
436
429
|
|
|
437
430
|
const result = await searchMessages(query, options);
|
|
@@ -439,14 +432,9 @@ async function messagesSearch(query, options = {}) {
|
|
|
439
432
|
}
|
|
440
433
|
|
|
441
434
|
async function messagesRecent(limit = 10, daysBack = 1) {
|
|
442
|
-
const
|
|
443
|
-
if (
|
|
444
|
-
return
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
const ready = await isIndexReady("messages");
|
|
448
|
-
if (!ready) {
|
|
449
|
-
return indexUnavailableMessage("messages");
|
|
435
|
+
const blocked = await requireIndex("messages");
|
|
436
|
+
if (blocked) {
|
|
437
|
+
return blocked;
|
|
450
438
|
}
|
|
451
439
|
|
|
452
440
|
const result = await getRecentMessageResults(limit, daysBack);
|
|
@@ -454,14 +442,9 @@ async function messagesRecent(limit = 10, daysBack = 1) {
|
|
|
454
442
|
}
|
|
455
443
|
|
|
456
444
|
async function messagesConversation(contact, limit = 50) {
|
|
457
|
-
const
|
|
458
|
-
if (
|
|
459
|
-
return
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
const ready = await isIndexReady("messages");
|
|
463
|
-
if (!ready) {
|
|
464
|
-
return indexUnavailableMessage("messages");
|
|
445
|
+
const blocked = await requireIndex("messages");
|
|
446
|
+
if (blocked) {
|
|
447
|
+
return blocked;
|
|
465
448
|
}
|
|
466
449
|
|
|
467
450
|
const result = await getConversationResults(contact, limit);
|
|
@@ -469,14 +452,9 @@ async function messagesConversation(contact, limit = 50) {
|
|
|
469
452
|
}
|
|
470
453
|
|
|
471
454
|
async function calendarSearch(query, options = {}) {
|
|
472
|
-
const
|
|
473
|
-
if (
|
|
474
|
-
return
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
const ready = await isIndexReady("calendar");
|
|
478
|
-
if (!ready) {
|
|
479
|
-
return indexUnavailableMessage("calendar");
|
|
455
|
+
const blocked = await requireIndex("calendar");
|
|
456
|
+
if (blocked) {
|
|
457
|
+
return blocked;
|
|
480
458
|
}
|
|
481
459
|
|
|
482
460
|
const result = await searchCalendar(query, options);
|
|
@@ -1397,16 +1375,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1397
1375
|
// Mail tools
|
|
1398
1376
|
case "mail_senders":
|
|
1399
1377
|
{
|
|
1400
|
-
const
|
|
1401
|
-
if (
|
|
1402
|
-
result =
|
|
1378
|
+
const blocked = await requireIndex("emails");
|
|
1379
|
+
if (blocked) {
|
|
1380
|
+
result = blocked;
|
|
1403
1381
|
break;
|
|
1404
1382
|
}
|
|
1405
1383
|
}
|
|
1406
|
-
if (!(await isIndexReady("emails"))) {
|
|
1407
|
-
result = indexUnavailableMessage("emails");
|
|
1408
|
-
break;
|
|
1409
|
-
}
|
|
1410
1384
|
result = formatSendersResults(await getFrequentSenders(
|
|
1411
1385
|
validateLimit(args?.limit, 30),
|
|
1412
1386
|
validateDaysBack(args?.days_back),
|
|
@@ -1481,16 +1455,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1481
1455
|
|
|
1482
1456
|
case "mail_thread":
|
|
1483
1457
|
{
|
|
1484
|
-
const
|
|
1485
|
-
if (
|
|
1486
|
-
result =
|
|
1458
|
+
const blocked = await requireIndex("emails");
|
|
1459
|
+
if (blocked) {
|
|
1460
|
+
result = blocked;
|
|
1487
1461
|
break;
|
|
1488
1462
|
}
|
|
1489
1463
|
}
|
|
1490
|
-
if (!(await isIndexReady("emails"))) {
|
|
1491
|
-
result = indexUnavailableMessage("emails");
|
|
1492
|
-
break;
|
|
1493
|
-
}
|
|
1494
1464
|
result = formatEmailThreadResults(await getEmailThread(args.file_path, validateLimit(args?.limit, 30)));
|
|
1495
1465
|
break;
|
|
1496
1466
|
|
package/indexer.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "./lib/validators.js";
|
|
17
17
|
import { safeSqlite3Json, safeOsascript, safeFind } from "./lib/shell.js";
|
|
18
18
|
import { indexUnavailableMessage } from "./lib/indexGate.js";
|
|
19
|
+
import { createLanceTableCache, LANCE_CONNECT_OPTIONS } from "./lib/lancedbTables.js";
|
|
19
20
|
|
|
20
21
|
// Re-export contact functions for use by other modules
|
|
21
22
|
export {
|
|
@@ -70,8 +71,17 @@ const MAC_ABSOLUTE_EPOCH = 978307200;
|
|
|
70
71
|
export const CALENDAR_TMP_PREFIX = "apple-tools-cal-";
|
|
71
72
|
|
|
72
73
|
let embeddingPipeline = null;
|
|
74
|
+
|
|
75
|
+
// One connection per process. Do not early-return a first connect that saw an
|
|
76
|
+
// empty catalog — MCP stdio must pick up tables the indexer daemon already wrote.
|
|
77
|
+
const lanceCache = createLanceTableCache({
|
|
78
|
+
connect: (uri, options) => lancedb.connect(uri, options),
|
|
79
|
+
indexDir: INDEX_DIR,
|
|
80
|
+
mkdirSync: (p, o) => fs.mkdirSync(p, o),
|
|
81
|
+
connectOptions: LANCE_CONNECT_OPTIONS
|
|
82
|
+
});
|
|
83
|
+
const tables = lanceCache.tables;
|
|
73
84
|
let db = null;
|
|
74
|
-
let tables = {};
|
|
75
85
|
|
|
76
86
|
async function getEmbedder() {
|
|
77
87
|
if (!embeddingPipeline) {
|
|
@@ -751,23 +761,19 @@ function getCalendarEvents() {
|
|
|
751
761
|
// ============ DATABASE FUNCTIONS ============
|
|
752
762
|
|
|
753
763
|
export async function initDB() {
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
764
|
+
const result = await lanceCache.initDB();
|
|
765
|
+
db = result.db;
|
|
766
|
+
return { db, tables };
|
|
767
|
+
}
|
|
758
768
|
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
if (tableNames.includes("messages")) {
|
|
764
|
-
tables.messages = await db.openTable("messages");
|
|
765
|
-
}
|
|
766
|
-
if (tableNames.includes("calendar")) {
|
|
767
|
-
tables.calendar = await db.openTable("calendar");
|
|
768
|
-
}
|
|
769
|
+
export async function getOpenTable(type) {
|
|
770
|
+
await initDB();
|
|
771
|
+
return tables[type] || null;
|
|
772
|
+
}
|
|
769
773
|
|
|
770
|
-
|
|
774
|
+
function resetLanceConnection() {
|
|
775
|
+
lanceCache.reset();
|
|
776
|
+
db = null;
|
|
771
777
|
}
|
|
772
778
|
|
|
773
779
|
export async function clearEmailsTable() {
|
|
@@ -828,10 +834,9 @@ export async function rebuildIndex(sources = ["emails", "messages", "calendar"],
|
|
|
828
834
|
}
|
|
829
835
|
}
|
|
830
836
|
|
|
831
|
-
// Reset
|
|
832
|
-
//
|
|
833
|
-
|
|
834
|
-
tables = {};
|
|
837
|
+
// Reset connection cache after dropping tables so initDB() re-opens
|
|
838
|
+
// (or recreates) them instead of holding stale/empty handles.
|
|
839
|
+
resetLanceConnection();
|
|
835
840
|
|
|
836
841
|
// Re-index requested sources
|
|
837
842
|
for (const source of sources) {
|
|
@@ -1451,7 +1456,7 @@ export async function indexAll(progressCallback = null) {
|
|
|
1451
1456
|
|
|
1452
1457
|
export async function isIndexReady(type = "emails") {
|
|
1453
1458
|
await initDB();
|
|
1454
|
-
return tables[type]
|
|
1459
|
+
return tables[type] != null;
|
|
1455
1460
|
}
|
|
1456
1461
|
|
|
1457
1462
|
// ============ DIRECT QUERIES (for recent items) ============
|
package/lib/audit.js
CHANGED
|
@@ -15,6 +15,7 @@ import fs from "fs";
|
|
|
15
15
|
import path from "path";
|
|
16
16
|
import { safeSqlite3Json, safeFind } from "./shell.js";
|
|
17
17
|
import * as lancedb from "@lancedb/lancedb";
|
|
18
|
+
import { LANCE_CONNECT_OPTIONS, openMissingIndexTables } from "./lancedbTables.js";
|
|
18
19
|
|
|
19
20
|
// ============================================================================
|
|
20
21
|
// CONSTANTS AND PATHS
|
|
@@ -55,22 +56,29 @@ let tables = {};
|
|
|
55
56
|
// ============================================================================
|
|
56
57
|
|
|
57
58
|
async function initDB() {
|
|
58
|
-
if (db)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (tableNames.includes("emails")) {
|
|
65
|
-
tables.emails = await db.openTable("emails");
|
|
66
|
-
}
|
|
67
|
-
if (tableNames.includes("messages")) {
|
|
68
|
-
tables.messages = await db.openTable("messages");
|
|
69
|
-
}
|
|
70
|
-
if (tableNames.includes("calendar")) {
|
|
71
|
-
tables.calendar = await db.openTable("calendar");
|
|
59
|
+
if (!db) {
|
|
60
|
+
try {
|
|
61
|
+
db = await lancedb.connect(INDEX_DIR, LANCE_CONNECT_OPTIONS);
|
|
62
|
+
} catch (e) {
|
|
63
|
+
console.error("Error initializing database:", e.message);
|
|
64
|
+
return { db: null, tables: {} };
|
|
72
65
|
}
|
|
66
|
+
}
|
|
73
67
|
|
|
68
|
+
try {
|
|
69
|
+
const result = await openMissingIndexTables({
|
|
70
|
+
db,
|
|
71
|
+
tables,
|
|
72
|
+
indexDir: INDEX_DIR,
|
|
73
|
+
reconnect: async () => {
|
|
74
|
+
try {
|
|
75
|
+
db?.close?.();
|
|
76
|
+
} catch {}
|
|
77
|
+
db = await lancedb.connect(INDEX_DIR, LANCE_CONNECT_OPTIONS);
|
|
78
|
+
return db;
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
db = result.db;
|
|
74
82
|
return { db, tables };
|
|
75
83
|
} catch (e) {
|
|
76
84
|
console.error("Error initializing database:", e.message);
|
package/lib/indexGate.js
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* index cycle." A second MCP instance that lost the indexer lock never runs
|
|
6
6
|
* a cycle, so that flag stays false. Lost-lock must not be treated as
|
|
7
7
|
* "still indexing" — callers still check isIndexReady() for a missing index.
|
|
8
|
+
* isIndexReady() / initDB() must re-list LanceDB tables on each check so a
|
|
9
|
+
* first empty catalog (daemon writer in flight) is not cached forever.
|
|
8
10
|
*/
|
|
9
11
|
|
|
10
12
|
/**
|
|
@@ -57,3 +59,50 @@ export function indexUnavailableMessage(type) {
|
|
|
57
59
|
}
|
|
58
60
|
return "Index not available. Please try again shortly.";
|
|
59
61
|
}
|
|
62
|
+
|
|
63
|
+
export const BUILDING_INITIAL_INDEX_MESSAGE =
|
|
64
|
+
"Building initial index. This may take several minutes on first run. Please try again shortly.";
|
|
65
|
+
|
|
66
|
+
export const INDEXING_NEW_DATA_MESSAGE =
|
|
67
|
+
"Indexing new data. Please try again in a moment.";
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Still-indexing copy. Only for a process that owns the lock and has not
|
|
71
|
+
* finished a local cycle — never for a lost-lock MCP reader.
|
|
72
|
+
*
|
|
73
|
+
* @param {boolean} isFirstEverRun
|
|
74
|
+
* @returns {string}
|
|
75
|
+
*/
|
|
76
|
+
export function indexingInProgressMessage(isFirstEverRun) {
|
|
77
|
+
return isFirstEverRun ? BUILDING_INITIAL_INDEX_MESSAGE : INDEXING_NEW_DATA_MESSAGE;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Preflight for index-backed MCP query tools.
|
|
82
|
+
* Readiness is `indexReady` (usable on-disk tables), not sessionIndexComplete.
|
|
83
|
+
* Lost-lock never returns "building initial index" / still-indexing.
|
|
84
|
+
*
|
|
85
|
+
* @param {{
|
|
86
|
+
* sessionIndexComplete: boolean,
|
|
87
|
+
* ownsIndexLock: boolean,
|
|
88
|
+
* indexReady: boolean,
|
|
89
|
+
* type?: "emails"|"messages"|"calendar",
|
|
90
|
+
* isFirstEverRun?: boolean
|
|
91
|
+
* }} args
|
|
92
|
+
* @returns {{ ok: boolean, message: string|null }}
|
|
93
|
+
*/
|
|
94
|
+
export function indexQueryGate({
|
|
95
|
+
sessionIndexComplete,
|
|
96
|
+
ownsIndexLock,
|
|
97
|
+
indexReady,
|
|
98
|
+
type,
|
|
99
|
+
isFirstEverRun = false
|
|
100
|
+
}) {
|
|
101
|
+
if (isSearchBlockedByIndexing(sessionIndexComplete, ownsIndexLock)) {
|
|
102
|
+
return { ok: false, message: indexingInProgressMessage(isFirstEverRun) };
|
|
103
|
+
}
|
|
104
|
+
if (!indexReady) {
|
|
105
|
+
return { ok: false, message: indexUnavailableMessage(type) };
|
|
106
|
+
}
|
|
107
|
+
return { ok: true, message: null };
|
|
108
|
+
}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared LanceDB table cache for the vector index.
|
|
3
|
+
*
|
|
4
|
+
* MCP stdio and the indexer daemon are separate processes. The daemon keeps a
|
|
5
|
+
* writer connection for its lifetime. A short-lived MCP reader must still see
|
|
6
|
+
* on-disk emails/messages/calendar tables — including when the first connect
|
|
7
|
+
* listed an empty catalog (writer commit in flight, stale snapshot, or a
|
|
8
|
+
* concurrent initDB that returned before openTable finished).
|
|
9
|
+
*
|
|
10
|
+
* Default LanceDB consistency does not re-check other processes. Readers pass
|
|
11
|
+
* readConsistencyInterval: 0 so tableNames()/openTable() see committed catalog.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import fs from "fs";
|
|
15
|
+
import path from "path";
|
|
16
|
+
|
|
17
|
+
export const INDEX_TABLE_NAMES = ["emails", "messages", "calendar"];
|
|
18
|
+
|
|
19
|
+
/** Strong cross-process read freshness (seconds). 0 = check on every read. */
|
|
20
|
+
export const LANCE_READ_CONSISTENCY_INTERVAL = 0;
|
|
21
|
+
|
|
22
|
+
export const LANCE_CONNECT_OPTIONS = {
|
|
23
|
+
readConsistencyInterval: LANCE_READ_CONSISTENCY_INTERVAL
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* LanceDB stores each table as `<name>.lance` under the index directory.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} indexDir
|
|
30
|
+
* @param {string} name
|
|
31
|
+
* @param {(p: string) => boolean} [existsSync]
|
|
32
|
+
* @returns {boolean}
|
|
33
|
+
*/
|
|
34
|
+
export function lanceTableExistsOnDisk(indexDir, name, existsSync = fs.existsSync) {
|
|
35
|
+
if (!indexDir || !INDEX_TABLE_NAMES.includes(name)) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
const tableDir = path.join(indexDir, `${name}.lance`);
|
|
40
|
+
const resolvedIndex = path.resolve(indexDir);
|
|
41
|
+
const resolvedTable = path.resolve(tableDir);
|
|
42
|
+
if (resolvedTable !== path.join(resolvedIndex, `${name}.lance`)) {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
return existsSync(tableDir);
|
|
46
|
+
} catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Open any INDEX_TABLE_NAMES entries not already in `tables`.
|
|
53
|
+
* Always re-lists tableNames — never assume a prior empty catalog is final.
|
|
54
|
+
*
|
|
55
|
+
* @param {{
|
|
56
|
+
* db: { tableNames: () => Promise<string[]>, openTable: (name: string) => Promise<unknown>, close?: () => void },
|
|
57
|
+
* tables: Record<string, unknown>,
|
|
58
|
+
* indexDir: string,
|
|
59
|
+
* existsSync?: (p: string) => boolean,
|
|
60
|
+
* reconnect?: () => Promise<{ tableNames: () => Promise<string[]>, openTable: (name: string) => Promise<unknown>, close?: () => void }>,
|
|
61
|
+
* isCurrent?: () => boolean,
|
|
62
|
+
* log?: (msg: string) => void
|
|
63
|
+
* }} args
|
|
64
|
+
* @returns {Promise<{ db: object, tables: Record<string, unknown>, reconnected: boolean, aborted: boolean }>}
|
|
65
|
+
*/
|
|
66
|
+
export async function openMissingIndexTables({
|
|
67
|
+
db,
|
|
68
|
+
tables,
|
|
69
|
+
indexDir,
|
|
70
|
+
existsSync = fs.existsSync,
|
|
71
|
+
reconnect,
|
|
72
|
+
isCurrent = () => true,
|
|
73
|
+
log = (msg) => console.error(msg)
|
|
74
|
+
}) {
|
|
75
|
+
const stillCurrent = () => {
|
|
76
|
+
try {
|
|
77
|
+
return isCurrent() !== false;
|
|
78
|
+
} catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const missing = INDEX_TABLE_NAMES.filter((name) => tables[name] == null);
|
|
84
|
+
if (missing.length === 0) {
|
|
85
|
+
return { db, tables, reconnected: false, aborted: false };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let tableNames = [];
|
|
89
|
+
try {
|
|
90
|
+
tableNames = await db.tableNames();
|
|
91
|
+
} catch (e) {
|
|
92
|
+
log(`Failed to list index tables: ${e.message}`);
|
|
93
|
+
return { db, tables, reconnected: false, aborted: !stillCurrent() };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (!stillCurrent()) {
|
|
97
|
+
return { db, tables, reconnected: false, aborted: true };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const listed = new Set(tableNames);
|
|
101
|
+
const onDiskButUnlisted = missing.filter(
|
|
102
|
+
(name) => !listed.has(name) && lanceTableExistsOnDisk(indexDir, name, existsSync)
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
let reconnected = false;
|
|
106
|
+
let conn = db;
|
|
107
|
+
if (onDiskButUnlisted.length > 0 && typeof reconnect === "function") {
|
|
108
|
+
log("Index catalog missed on-disk tables; reconnecting to shared vector-index.");
|
|
109
|
+
for (const name of INDEX_TABLE_NAMES) {
|
|
110
|
+
delete tables[name];
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
conn = await reconnect();
|
|
114
|
+
reconnected = true;
|
|
115
|
+
if (!stillCurrent()) {
|
|
116
|
+
return { db: conn, tables, reconnected, aborted: true };
|
|
117
|
+
}
|
|
118
|
+
tableNames = await conn.tableNames();
|
|
119
|
+
} catch (e) {
|
|
120
|
+
log(`Failed to reconnect to index: ${e.message}`);
|
|
121
|
+
return { db: conn, tables, reconnected, aborted: !stillCurrent() };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (!stillCurrent()) {
|
|
126
|
+
return { db: conn, tables, reconnected, aborted: true };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const stillMissing = INDEX_TABLE_NAMES.filter((name) => tables[name] == null);
|
|
130
|
+
for (const name of stillMissing) {
|
|
131
|
+
if (!tableNames.includes(name)) {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
tables[name] = await conn.openTable(name);
|
|
136
|
+
} catch (e) {
|
|
137
|
+
log(`Failed to open ${name} table: ${e.message}`);
|
|
138
|
+
}
|
|
139
|
+
if (!stillCurrent()) {
|
|
140
|
+
return { db: conn, tables, reconnected, aborted: true };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
return { db: conn, tables, reconnected, aborted: false };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Process-local LanceDB connection + table handles.
|
|
149
|
+
* `tables` is a stable object (reset deletes keys, does not replace the object)
|
|
150
|
+
* so indexer.js can alias it for createTable/dropTable mutation.
|
|
151
|
+
*
|
|
152
|
+
* @param {{
|
|
153
|
+
* connect: (uri: string, options: object) => Promise<object>,
|
|
154
|
+
* indexDir: string,
|
|
155
|
+
* mkdirSync?: (p: string, opts?: object) => void,
|
|
156
|
+
* existsSync?: (p: string) => boolean,
|
|
157
|
+
* log?: (msg: string) => void,
|
|
158
|
+
* connectOptions?: object
|
|
159
|
+
* }} options
|
|
160
|
+
*/
|
|
161
|
+
export function createLanceTableCache(options) {
|
|
162
|
+
const {
|
|
163
|
+
connect,
|
|
164
|
+
indexDir,
|
|
165
|
+
mkdirSync,
|
|
166
|
+
existsSync = fs.existsSync,
|
|
167
|
+
log = (msg) => console.error(msg),
|
|
168
|
+
connectOptions = LANCE_CONNECT_OPTIONS
|
|
169
|
+
} = options;
|
|
170
|
+
|
|
171
|
+
const tables = {};
|
|
172
|
+
let db = null;
|
|
173
|
+
let connecting = null;
|
|
174
|
+
let refreshing = null;
|
|
175
|
+
let generation = 0;
|
|
176
|
+
|
|
177
|
+
function rememberPromise(getSlot, setSlot, pending) {
|
|
178
|
+
// Store the .finally() chain, not the raw pending. Discarding the derived
|
|
179
|
+
// promise leaves connect() failures as unhandledRejection (index.js exits).
|
|
180
|
+
const tracked = pending.finally(() => {
|
|
181
|
+
if (getSlot() === tracked) {
|
|
182
|
+
setSlot(null);
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
setSlot(tracked);
|
|
186
|
+
return tracked;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async function ensureDb() {
|
|
190
|
+
if (db) {
|
|
191
|
+
return db;
|
|
192
|
+
}
|
|
193
|
+
if (!connecting) {
|
|
194
|
+
const gen = generation;
|
|
195
|
+
rememberPromise(
|
|
196
|
+
() => connecting,
|
|
197
|
+
(value) => {
|
|
198
|
+
connecting = value;
|
|
199
|
+
},
|
|
200
|
+
(async () => {
|
|
201
|
+
if (typeof mkdirSync === "function") {
|
|
202
|
+
mkdirSync(indexDir, { recursive: true });
|
|
203
|
+
}
|
|
204
|
+
const conn = await connect(indexDir, connectOptions);
|
|
205
|
+
if (gen !== generation) {
|
|
206
|
+
try {
|
|
207
|
+
conn?.close?.();
|
|
208
|
+
} catch {
|
|
209
|
+
// stale connect after reset
|
|
210
|
+
}
|
|
211
|
+
return ensureDb();
|
|
212
|
+
}
|
|
213
|
+
db = conn;
|
|
214
|
+
return db;
|
|
215
|
+
})()
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
return connecting;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async function initDB() {
|
|
222
|
+
if (!refreshing) {
|
|
223
|
+
const gen = generation;
|
|
224
|
+
rememberPromise(
|
|
225
|
+
() => refreshing,
|
|
226
|
+
(value) => {
|
|
227
|
+
refreshing = value;
|
|
228
|
+
},
|
|
229
|
+
(async () => {
|
|
230
|
+
await ensureDb();
|
|
231
|
+
if (gen !== generation) {
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const result = await openMissingIndexTables({
|
|
235
|
+
db,
|
|
236
|
+
tables,
|
|
237
|
+
indexDir,
|
|
238
|
+
existsSync,
|
|
239
|
+
log,
|
|
240
|
+
isCurrent: () => gen === generation,
|
|
241
|
+
reconnect: async () => {
|
|
242
|
+
if (gen !== generation) {
|
|
243
|
+
return ensureDb();
|
|
244
|
+
}
|
|
245
|
+
try {
|
|
246
|
+
db?.close?.();
|
|
247
|
+
} catch {
|
|
248
|
+
// ignore close errors on a stale handle
|
|
249
|
+
}
|
|
250
|
+
if (gen !== generation) {
|
|
251
|
+
return ensureDb();
|
|
252
|
+
}
|
|
253
|
+
db = null;
|
|
254
|
+
return ensureDb();
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
if (gen !== generation || result.aborted) {
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
db = result.db;
|
|
261
|
+
})()
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
await refreshing;
|
|
265
|
+
return { db, tables };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function isIndexReady(type = "emails") {
|
|
269
|
+
await initDB();
|
|
270
|
+
return tables[type] != null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function getOpenTable(type) {
|
|
274
|
+
await initDB();
|
|
275
|
+
return tables[type] || null;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function reset() {
|
|
279
|
+
generation += 1;
|
|
280
|
+
const toClose = db;
|
|
281
|
+
db = null;
|
|
282
|
+
connecting = null;
|
|
283
|
+
refreshing = null;
|
|
284
|
+
for (const key of Object.keys(tables)) {
|
|
285
|
+
delete tables[key];
|
|
286
|
+
}
|
|
287
|
+
try {
|
|
288
|
+
toClose?.close?.();
|
|
289
|
+
} catch {
|
|
290
|
+
// ignore
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
return {
|
|
295
|
+
initDB,
|
|
296
|
+
isIndexReady,
|
|
297
|
+
getOpenTable,
|
|
298
|
+
reset,
|
|
299
|
+
get db() {
|
|
300
|
+
return db;
|
|
301
|
+
},
|
|
302
|
+
set db(value) {
|
|
303
|
+
db = value;
|
|
304
|
+
},
|
|
305
|
+
tables
|
|
306
|
+
};
|
|
307
|
+
}
|
package/package.json
CHANGED
package/search.js
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
import * as lancedb from "@lancedb/lancedb";
|
|
2
1
|
import * as chrono from "chrono-node";
|
|
3
2
|
import { safeOsascript } from "./lib/shell.js";
|
|
4
3
|
import { safeMatch, validateSearchQuery, toUnixMillis } from "./lib/validators.js";
|
|
5
|
-
import { embed,
|
|
4
|
+
import { embed, getOpenTable, getRecentEmails, getEmailsByDateRange, getRecentMessages, getConversation, getEventsOnDate, resolveEmail, resolvePhone, formatContact } from "./indexer.js";
|
|
6
5
|
import { indexUnavailableMessage } from "./lib/indexGate.js";
|
|
7
6
|
|
|
8
|
-
let db = null;
|
|
9
|
-
let tables = {};
|
|
10
|
-
|
|
11
7
|
// ============ CACHING ============
|
|
12
8
|
|
|
13
9
|
// Embedding cache with TTL (5 minutes)
|
|
@@ -458,19 +454,9 @@ async function searchWithRetry(_searchFn, tbl, query, options, keyField) {
|
|
|
458
454
|
}
|
|
459
455
|
|
|
460
456
|
async function getTable(type) {
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
db = await lancedb.connect(INDEX_DIR);
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
const tableNames = await db.tableNames();
|
|
468
|
-
if (!tableNames.includes(type)) {
|
|
469
|
-
return null;
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
tables[type] = await db.openTable(type);
|
|
473
|
-
return tables[type];
|
|
457
|
+
// Share indexer.js's connection so a first empty catalog is refreshed the
|
|
458
|
+
// same way isIndexReady() is — not a second handle that can stay empty.
|
|
459
|
+
return getOpenTable(type);
|
|
474
460
|
}
|
|
475
461
|
|
|
476
462
|
// Pre-warm all tables to eliminate first-query latency
|