apple-tools-mcp 3.2.4 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/indexer.js +135 -19
- package/lib/indexMaintenance.js +223 -0
- package/lib/lancedbTables.js +59 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -184,6 +184,8 @@ The refresh interval is chosen once, when the process starts. The default is 5 m
|
|
|
184
184
|
|
|
185
185
|
Values are clamped to 15 seconds through 6 hours. A missing config file is fine; the 5-minute default is used.
|
|
186
186
|
|
|
187
|
+
The indexer keeps its memory bounded for long runs. The vector database's caches are capped at a 64 MB index cache and a 256 MB metadata cache; set `APPLE_TOOLS_LANCE_INDEX_CACHE_MB` or `APPLE_TOOLS_LANCE_METADATA_CACHE_MB` (whole megabytes, 1 to 8192) in the LaunchAgent's `EnvironmentVariables` to change them. Indexed IDs stay in memory between passes, so a pass with no new mail does not re-read the index. Once a day, after a pass that added or removed something, it compacts the index files and removes table versions older than an hour.
|
|
188
|
+
|
|
187
189
|
If you do not run the indexer, the MCP process indexes when it starts, then exits when the client disconnects. Writes then run inside that process, and macOS attributes them to the app that launched it.
|
|
188
190
|
|
|
189
191
|
### 5. Another computer (optional)
|
package/indexer.js
CHANGED
|
@@ -15,7 +15,18 @@ import {
|
|
|
15
15
|
} from "./lib/validators.js";
|
|
16
16
|
import { safeSqlite3Json, safeSqlite3JsonAsync, safeOsascript, safeFindAsync } from "./lib/shell.js";
|
|
17
17
|
import { indexUnavailableMessage } from "./lib/indexGate.js";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
createLanceTableCache,
|
|
20
|
+
createLanceSession,
|
|
21
|
+
INDEX_TABLE_NAMES,
|
|
22
|
+
LANCE_CONNECT_OPTIONS
|
|
23
|
+
} from "./lib/lancedbTables.js";
|
|
24
|
+
import {
|
|
25
|
+
createIndexedIdCache,
|
|
26
|
+
createDuplicatePathTracker,
|
|
27
|
+
isIndexOptimizeDue,
|
|
28
|
+
optimizeIndexTables
|
|
29
|
+
} from "./lib/indexMaintenance.js";
|
|
19
30
|
import { createMessageContactMatcher } from "./lib/messageContact.js";
|
|
20
31
|
import { resolveByExactName, normalizePhone } from "./contacts.js";
|
|
21
32
|
|
|
@@ -77,15 +88,30 @@ let embeddingPipeline = null;
|
|
|
77
88
|
|
|
78
89
|
// One connection per process. Do not early-return a first connect that saw an
|
|
79
90
|
// empty catalog — MCP stdio must pick up tables the indexer daemon already wrote.
|
|
91
|
+
// The capped Session keeps LanceDB's caches from growing to their 6 GB / 1 GB
|
|
92
|
+
// defaults in the long-lived indexer daemon.
|
|
80
93
|
const lanceCache = createLanceTableCache({
|
|
81
|
-
connect: (uri, options) => lancedb.connect(uri, options),
|
|
94
|
+
connect: (uri, options, session) => lancedb.connect(uri, options, session),
|
|
82
95
|
indexDir: INDEX_DIR,
|
|
83
96
|
mkdirSync: (p, o) => fs.mkdirSync(p, o),
|
|
84
|
-
connectOptions: LANCE_CONNECT_OPTIONS
|
|
97
|
+
connectOptions: LANCE_CONNECT_OPTIONS,
|
|
98
|
+
session: createLanceSession(lancedb.Session)
|
|
85
99
|
});
|
|
86
100
|
const tables = lanceCache.tables;
|
|
87
101
|
let db = null;
|
|
88
102
|
|
|
103
|
+
// Indexed ID sets stay in memory between passes; a table is re-read only when
|
|
104
|
+
// its version moves. Email files skipped as Message-ID duplicates are
|
|
105
|
+
// remembered so the same IMAP copies are not parsed every pass.
|
|
106
|
+
const EMAIL_ID_FIELDS = ["filePath", "messageId"];
|
|
107
|
+
const indexedIdCache = createIndexedIdCache({
|
|
108
|
+
getTable: async (name) => {
|
|
109
|
+
await initDB();
|
|
110
|
+
return tables[name] || null;
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
const duplicateEmailPaths = createDuplicatePathTracker();
|
|
114
|
+
|
|
89
115
|
async function getEmbedder() {
|
|
90
116
|
if (!embeddingPipeline) {
|
|
91
117
|
console.error("Loading embedding model (first time may take a minute)...");
|
|
@@ -782,6 +808,8 @@ export async function getOpenTable(type) {
|
|
|
782
808
|
function resetLanceConnection() {
|
|
783
809
|
lanceCache.reset();
|
|
784
810
|
db = null;
|
|
811
|
+
indexedIdCache.invalidate();
|
|
812
|
+
duplicateEmailPaths.clear();
|
|
785
813
|
}
|
|
786
814
|
|
|
787
815
|
export async function clearEmailsTable() {
|
|
@@ -789,6 +817,8 @@ export async function clearEmailsTable() {
|
|
|
789
817
|
if (tables.emails) {
|
|
790
818
|
await db.dropTable("emails");
|
|
791
819
|
tables.emails = null;
|
|
820
|
+
indexedIdCache.invalidate("emails");
|
|
821
|
+
duplicateEmailPaths.clear();
|
|
792
822
|
console.error("Emails table dropped. Re-run indexing to rebuild.");
|
|
793
823
|
return { cleared: true };
|
|
794
824
|
}
|
|
@@ -800,6 +830,7 @@ export async function clearMessagesTable() {
|
|
|
800
830
|
if (tables.messages) {
|
|
801
831
|
await db.dropTable("messages");
|
|
802
832
|
tables.messages = null;
|
|
833
|
+
indexedIdCache.invalidate("messages");
|
|
803
834
|
console.error("Messages table dropped. Re-run indexing to rebuild.");
|
|
804
835
|
return { cleared: true };
|
|
805
836
|
}
|
|
@@ -910,6 +941,31 @@ async function getIndexedIdsWithRetry(tableName, idField, maxRetries = 3) {
|
|
|
910
941
|
throw lastError;
|
|
911
942
|
}
|
|
912
943
|
|
|
944
|
+
/**
|
|
945
|
+
* Indexed ID sets for several columns of one table, read in one query and
|
|
946
|
+
* kept in memory between passes (re-read only when the table version moves).
|
|
947
|
+
* Same retry policy as getIndexedIdsWithRetry.
|
|
948
|
+
*/
|
|
949
|
+
async function getCachedIdSetsWithRetry(tableName, fields, maxRetries = 3) {
|
|
950
|
+
let lastError;
|
|
951
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
952
|
+
try {
|
|
953
|
+
return await indexedIdCache.getIdSets(tableName, fields);
|
|
954
|
+
} catch (e) {
|
|
955
|
+
// THROW (after retries) instead of returning empty Sets to prevent duplicate entries
|
|
956
|
+
console.error(`Error getting indexed IDs from ${tableName}: ${e.message}`);
|
|
957
|
+
lastError = new Error(`Failed to get indexed IDs from ${tableName}: ${e.message}`);
|
|
958
|
+
indexedIdCache.invalidate(tableName);
|
|
959
|
+
if (attempt < maxRetries) {
|
|
960
|
+
const backoff = Math.min(1000 * Math.pow(2, attempt - 1), 5000);
|
|
961
|
+
console.error(`Retry ${attempt}/${maxRetries} after ${backoff}ms...`);
|
|
962
|
+
await new Promise(resolve => setTimeout(resolve, backoff));
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
throw lastError;
|
|
967
|
+
}
|
|
968
|
+
|
|
913
969
|
// ============ INDEX ALL CONTENT ============
|
|
914
970
|
|
|
915
971
|
export async function indexEmails(progressCallback = null, forceFullScan = false) {
|
|
@@ -957,12 +1013,27 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
957
1013
|
}
|
|
958
1014
|
console.error(`Found ${newFiles.length} new/modified email files (${Date.now() - startTime}ms)`);
|
|
959
1015
|
|
|
960
|
-
|
|
961
|
-
const
|
|
1016
|
+
// One read of both ID columns, served from memory while the table is unchanged.
|
|
1017
|
+
const emailIdSets = await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS);
|
|
1018
|
+
const indexedPaths = emailIdSets.filePath;
|
|
1019
|
+
const indexedMessageIds = emailIdSets.messageId;
|
|
962
1020
|
console.error(`Already indexed: ${indexedPaths.size} emails`);
|
|
963
1021
|
|
|
964
|
-
// Filter out already-indexed files
|
|
965
|
-
|
|
1022
|
+
// Filter out already-indexed files and copies already known to duplicate an
|
|
1023
|
+
// indexed Message-ID (forget copies that left the lookback window).
|
|
1024
|
+
duplicateEmailPaths.retainOnly(newFiles);
|
|
1025
|
+
let knownDuplicates = 0;
|
|
1026
|
+
let toIndex = newFiles.filter(f => {
|
|
1027
|
+
if (indexedPaths.has(f)) return false;
|
|
1028
|
+
if (duplicateEmailPaths.has(f)) {
|
|
1029
|
+
knownDuplicates++;
|
|
1030
|
+
return false;
|
|
1031
|
+
}
|
|
1032
|
+
return true;
|
|
1033
|
+
});
|
|
1034
|
+
if (knownDuplicates > 0) {
|
|
1035
|
+
console.error(`Known duplicate copies (already indexed by Message-ID): ${knownDuplicates}`);
|
|
1036
|
+
}
|
|
966
1037
|
console.error(`Need to index: ${toIndex.length} emails`);
|
|
967
1038
|
|
|
968
1039
|
// Sort by mailbox priority so higher-priority folders are indexed first
|
|
@@ -976,7 +1047,8 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
976
1047
|
}
|
|
977
1048
|
|
|
978
1049
|
let processed = 0;
|
|
979
|
-
let
|
|
1050
|
+
let indexedCount = 0;
|
|
1051
|
+
let skippedCount = { parseNull: 0, shortSearchText: 0, duplicateMessageId: 0, alreadyIndexed: 0 };
|
|
980
1052
|
|
|
981
1053
|
// Progress tracking
|
|
982
1054
|
const totalBatches = Math.ceil(toIndex.length / BATCH_SIZE);
|
|
@@ -989,6 +1061,9 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
989
1061
|
// Parse all emails in batch first
|
|
990
1062
|
const parsedEmails = [];
|
|
991
1063
|
const searchTexts = [];
|
|
1064
|
+
// Message-IDs taken earlier in this batch (two copies of one new email can
|
|
1065
|
+
// land in the same batch before either is in indexedMessageIds)
|
|
1066
|
+
const batchMessageIds = new Set();
|
|
992
1067
|
|
|
993
1068
|
for (const filePath of batch) {
|
|
994
1069
|
const parsed = parseEmlx(filePath);
|
|
@@ -1005,13 +1080,18 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
1005
1080
|
continue;
|
|
1006
1081
|
}
|
|
1007
1082
|
// Skip duplicate messageIds - same email in multiple folders (IMAP behavior)
|
|
1008
|
-
if (parsed.messageId && indexedMessageIds.has(parsed.messageId)) {
|
|
1083
|
+
if (parsed.messageId && (indexedMessageIds.has(parsed.messageId) || batchMessageIds.has(parsed.messageId))) {
|
|
1009
1084
|
skippedCount.duplicateMessageId++;
|
|
1085
|
+
// Remember the copy's path so later passes skip it before parsing.
|
|
1086
|
+
duplicateEmailPaths.add(filePath);
|
|
1010
1087
|
continue; // Skip this duplicate
|
|
1011
1088
|
}
|
|
1012
1089
|
// NOTE: Don't filter by email header date - mdfind already filters by file modification time
|
|
1013
1090
|
// The DAYS_BACK filter is applied at the mdfind level (finding recently modified files),
|
|
1014
1091
|
// not at the email content level (email header dates can be old for recently modified messages)
|
|
1092
|
+
if (parsed.messageId) {
|
|
1093
|
+
batchMessageIds.add(parsed.messageId);
|
|
1094
|
+
}
|
|
1015
1095
|
parsedEmails.push({ filePath, parsed });
|
|
1016
1096
|
searchTexts.push(parsed.searchText);
|
|
1017
1097
|
}
|
|
@@ -1068,6 +1148,7 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
1068
1148
|
if (uniqueRecords.length > 0) {
|
|
1069
1149
|
if (!tables.emails) {
|
|
1070
1150
|
tables.emails = await db.createTable("emails", uniqueRecords, { mode: "overwrite" });
|
|
1151
|
+
indexedCount += uniqueRecords.length;
|
|
1071
1152
|
// Track messageIds from the first batch so later batches can
|
|
1072
1153
|
// dedupe IMAP copies (INBOX vs Junk) of the same Message-ID.
|
|
1073
1154
|
for (const record of uniqueRecords) {
|
|
@@ -1077,16 +1158,18 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
1077
1158
|
}
|
|
1078
1159
|
} else {
|
|
1079
1160
|
// Double-check: verify these IDs truly aren't in the index
|
|
1080
|
-
const currentIndexed = await
|
|
1161
|
+
const currentIndexed = (await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS)).filePath;
|
|
1081
1162
|
const trulyNew = uniqueRecords.filter(r => !currentIndexed.has(r.filePath));
|
|
1082
1163
|
|
|
1083
1164
|
if (trulyNew.length < uniqueRecords.length) {
|
|
1165
|
+
skippedCount.alreadyIndexed += uniqueRecords.length - trulyNew.length;
|
|
1084
1166
|
console.error(`WARNING: ${uniqueRecords.length - trulyNew.length} records were already indexed. Filtering them out.`);
|
|
1085
1167
|
}
|
|
1086
1168
|
|
|
1087
1169
|
if (trulyNew.length > 0) {
|
|
1088
1170
|
const beforeCount = await tables.emails.countRows();
|
|
1089
1171
|
await tables.emails.add(trulyNew);
|
|
1172
|
+
indexedCount += trulyNew.length;
|
|
1090
1173
|
const afterCount = await tables.emails.countRows();
|
|
1091
1174
|
const actualAdded = afterCount - beforeCount;
|
|
1092
1175
|
|
|
@@ -1101,6 +1184,7 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
1101
1184
|
indexedMessageIds.add(record.messageId);
|
|
1102
1185
|
}
|
|
1103
1186
|
}
|
|
1187
|
+
await indexedIdCache.noteAdded("emails", trulyNew);
|
|
1104
1188
|
}
|
|
1105
1189
|
}
|
|
1106
1190
|
}
|
|
@@ -1134,20 +1218,23 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
|
|
|
1134
1218
|
|
|
1135
1219
|
// Log skip summary with timing
|
|
1136
1220
|
const totalProcessingTime = Date.now() - startProcessTime;
|
|
1137
|
-
const totalSkipped = skippedCount.parseNull + skippedCount.shortSearchText +
|
|
1221
|
+
const totalSkipped = skippedCount.parseNull + skippedCount.shortSearchText +
|
|
1222
|
+
skippedCount.duplicateMessageId + skippedCount.alreadyIndexed;
|
|
1138
1223
|
console.error(`\nEmail indexing summary:`);
|
|
1139
1224
|
console.error(` Files to index: ${toIndex.length}`);
|
|
1140
|
-
console.error(` Successfully indexed: ${
|
|
1225
|
+
console.error(` Successfully indexed: ${indexedCount}`);
|
|
1141
1226
|
console.error(` Skipped - parse errors: ${skippedCount.parseNull}`);
|
|
1142
1227
|
console.error(` Skipped - short searchText: ${skippedCount.shortSearchText}`);
|
|
1143
1228
|
console.error(` Skipped - duplicate messageId: ${skippedCount.duplicateMessageId}`);
|
|
1229
|
+
console.error(` Skipped - already indexed: ${skippedCount.alreadyIndexed}`);
|
|
1144
1230
|
console.error(` Total skipped: ${totalSkipped}`);
|
|
1145
|
-
console.error(` Discrepancy: ${toIndex.length -
|
|
1231
|
+
console.error(` Discrepancy: ${toIndex.length - indexedCount - totalSkipped}`);
|
|
1146
1232
|
console.error(` Total time: ${Math.round(totalProcessingTime / 1000)}s (${Math.round(totalProcessingTime / 60000)}m)\n`);
|
|
1147
1233
|
|
|
1148
|
-
//
|
|
1149
|
-
|
|
1150
|
-
|
|
1234
|
+
// Indexed count from the version-checked ID cache (re-reads only if the
|
|
1235
|
+
// table moved by something other than this pass's own adds)
|
|
1236
|
+
const finalIndexedPaths = (await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS)).filePath;
|
|
1237
|
+
return { indexed: finalIndexedPaths.size, added: indexedCount };
|
|
1151
1238
|
}
|
|
1152
1239
|
|
|
1153
1240
|
export async function indexMessages(forceFullScan = false) {
|
|
@@ -1173,7 +1260,7 @@ export async function indexMessages(forceFullScan = false) {
|
|
|
1173
1260
|
}
|
|
1174
1261
|
console.error(`Found ${messages.length} messages${lastMessageIndexTime ? ' (incremental)' : ' (full scan)'}`);
|
|
1175
1262
|
|
|
1176
|
-
const indexed = await
|
|
1263
|
+
const indexed = (await getCachedIdSetsWithRetry("messages", ["id"])).id;
|
|
1177
1264
|
console.error(`Already indexed: ${indexed.size} messages`);
|
|
1178
1265
|
|
|
1179
1266
|
const toIndex = messages.filter(m => !indexed.has(String(m.id)));
|
|
@@ -1186,6 +1273,7 @@ export async function indexMessages(forceFullScan = false) {
|
|
|
1186
1273
|
}
|
|
1187
1274
|
|
|
1188
1275
|
let processed = 0;
|
|
1276
|
+
let addedCount = 0;
|
|
1189
1277
|
|
|
1190
1278
|
// Progress tracking
|
|
1191
1279
|
const totalBatches = Math.ceil(toIndex.length / BATCH_SIZE);
|
|
@@ -1260,9 +1348,10 @@ export async function indexMessages(forceFullScan = false) {
|
|
|
1260
1348
|
if (uniqueRecords.length > 0) {
|
|
1261
1349
|
if (!tables.messages) {
|
|
1262
1350
|
tables.messages = await db.createTable("messages", uniqueRecords, { mode: "overwrite" });
|
|
1351
|
+
addedCount += uniqueRecords.length;
|
|
1263
1352
|
} else {
|
|
1264
1353
|
// Double-check: verify these IDs truly aren't in the index
|
|
1265
|
-
const currentIndexed = await
|
|
1354
|
+
const currentIndexed = (await getCachedIdSetsWithRetry("messages", ["id"])).id;
|
|
1266
1355
|
const trulyNew = uniqueRecords.filter(r => !currentIndexed.has(r.id));
|
|
1267
1356
|
|
|
1268
1357
|
if (trulyNew.length < uniqueRecords.length) {
|
|
@@ -1272,12 +1361,14 @@ export async function indexMessages(forceFullScan = false) {
|
|
|
1272
1361
|
if (trulyNew.length > 0) {
|
|
1273
1362
|
const beforeCount = await tables.messages.countRows();
|
|
1274
1363
|
await tables.messages.add(trulyNew);
|
|
1364
|
+
addedCount += trulyNew.length;
|
|
1275
1365
|
const afterCount = await tables.messages.countRows();
|
|
1276
1366
|
const actualAdded = afterCount - beforeCount;
|
|
1277
1367
|
|
|
1278
1368
|
if (actualAdded !== trulyNew.length) {
|
|
1279
1369
|
console.error(`WARNING: Expected to add ${trulyNew.length} records but index grew by ${actualAdded}. Possible duplicate issue.`);
|
|
1280
1370
|
}
|
|
1371
|
+
await indexedIdCache.noteAdded("messages", trulyNew);
|
|
1281
1372
|
}
|
|
1282
1373
|
}
|
|
1283
1374
|
}
|
|
@@ -1304,7 +1395,7 @@ export async function indexMessages(forceFullScan = false) {
|
|
|
1304
1395
|
// Save timestamp for next incremental scan
|
|
1305
1396
|
saveIndexMeta({ ...meta, lastMessageIndexTime: indexStartTime });
|
|
1306
1397
|
|
|
1307
|
-
return { indexed: indexed.size, added:
|
|
1398
|
+
return { indexed: indexed.size, added: addedCount };
|
|
1308
1399
|
}
|
|
1309
1400
|
|
|
1310
1401
|
export async function indexCalendar() {
|
|
@@ -1458,10 +1549,35 @@ export async function indexAll(progressCallback = null) {
|
|
|
1458
1549
|
const calendarResult = await indexCalendar();
|
|
1459
1550
|
if (progressCallback) progressCallback('calendar-complete');
|
|
1460
1551
|
|
|
1552
|
+
await maybeOptimizeIndex({ emails: emailResult, messages: messageResult, calendar: calendarResult });
|
|
1553
|
+
|
|
1461
1554
|
console.error("Full index complete.");
|
|
1462
1555
|
return { emails: emailResult, messages: messageResult, calendar: calendarResult };
|
|
1463
1556
|
}
|
|
1464
1557
|
|
|
1558
|
+
/**
|
|
1559
|
+
* Daily LanceDB compaction + old-version cleanup, only after a pass that wrote
|
|
1560
|
+
* something. Every add() leaves a small data file and a new version; without
|
|
1561
|
+
* this the tables grow thousands of fragments that every full read opens.
|
|
1562
|
+
*/
|
|
1563
|
+
async function maybeOptimizeIndex(results) {
|
|
1564
|
+
const wroteSomething =
|
|
1565
|
+
(results.emails?.added || 0) > 0 ||
|
|
1566
|
+
(results.messages?.added || 0) > 0 ||
|
|
1567
|
+
(results.calendar?.added || 0) > 0 ||
|
|
1568
|
+
(results.calendar?.removed || 0) > 0;
|
|
1569
|
+
const meta = loadIndexMeta();
|
|
1570
|
+
if (!isIndexOptimizeDue({ lastOptimizeTime: meta.lastIndexOptimizeTime, wroteSomething })) {
|
|
1571
|
+
return null;
|
|
1572
|
+
}
|
|
1573
|
+
console.error("Optimizing index tables (compaction + old version cleanup)...");
|
|
1574
|
+
const result = await optimizeIndexTables({ tables, tableNames: INDEX_TABLE_NAMES });
|
|
1575
|
+
// Compaction makes new table versions; re-read IDs once next pass.
|
|
1576
|
+
indexedIdCache.invalidate();
|
|
1577
|
+
saveIndexMeta({ ...loadIndexMeta(), lastIndexOptimizeTime: Date.now() });
|
|
1578
|
+
return result;
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1465
1581
|
export async function isIndexReady(type = "emails") {
|
|
1466
1582
|
await initDB();
|
|
1467
1583
|
return tables[type] != null;
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Indexer-daemon upkeep that keeps a long-lived process light:
|
|
3
|
+
*
|
|
4
|
+
* - createIndexedIdCache: keeps the indexed ID sets (email filePath and
|
|
5
|
+
* messageId, message id) in memory between passes and re-reads a table only
|
|
6
|
+
* when its LanceDB version moved. A quiet pass costs one version check
|
|
7
|
+
* instead of a full column scan.
|
|
8
|
+
* - createDuplicatePathTracker: remembers email files skipped because their
|
|
9
|
+
* Message-ID is already indexed (IMAP copies in another folder), so the
|
|
10
|
+
* same copies are not parsed again every pass.
|
|
11
|
+
* - optimizeIndexTables / isIndexOptimizeDue: periodic LanceDB compaction and
|
|
12
|
+
* old-version cleanup, run after a pass that wrote something.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export const INDEX_OPTIMIZE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
16
|
+
/** Keep this much table history when cleaning up old versions. */
|
|
17
|
+
export const INDEX_OPTIMIZE_KEEP_VERSIONS_MS = 60 * 60 * 1000;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {{
|
|
21
|
+
* getTable: (tableName: string) => Promise<object | null>,
|
|
22
|
+
* log?: (msg: string) => void
|
|
23
|
+
* }} options
|
|
24
|
+
*/
|
|
25
|
+
export function createIndexedIdCache({ getTable, log = (msg) => console.error(msg) }) {
|
|
26
|
+
/** @type {Map<string, { table: object, version: number, sets: Record<string, Set<unknown>> }>} */
|
|
27
|
+
const entries = new Map();
|
|
28
|
+
|
|
29
|
+
function emptySets(fields) {
|
|
30
|
+
return Object.fromEntries(fields.map((field) => [field, new Set()]));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* ID sets for `fields`, read in one query. Served from memory while the
|
|
35
|
+
* table handle and version are unchanged.
|
|
36
|
+
*
|
|
37
|
+
* @param {string} tableName
|
|
38
|
+
* @param {string[]} fields
|
|
39
|
+
* @returns {Promise<Record<string, Set<unknown>>>}
|
|
40
|
+
*/
|
|
41
|
+
async function getIdSets(tableName, fields) {
|
|
42
|
+
const table = await getTable(tableName);
|
|
43
|
+
if (!table) {
|
|
44
|
+
entries.delete(tableName);
|
|
45
|
+
return emptySets(fields);
|
|
46
|
+
}
|
|
47
|
+
const version = await table.version();
|
|
48
|
+
const cached = entries.get(tableName);
|
|
49
|
+
if (
|
|
50
|
+
cached &&
|
|
51
|
+
cached.table === table &&
|
|
52
|
+
cached.version === version &&
|
|
53
|
+
fields.every((field) => cached.sets[field])
|
|
54
|
+
) {
|
|
55
|
+
return cached.sets;
|
|
56
|
+
}
|
|
57
|
+
const rows = await table.query().select(fields).toArray();
|
|
58
|
+
const sets = emptySets(fields);
|
|
59
|
+
for (const row of rows) {
|
|
60
|
+
for (const field of fields) {
|
|
61
|
+
sets[field].add(row[field]);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
entries.set(tableName, { table, version, sets });
|
|
65
|
+
return sets;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Fold rows this process just added into the cached sets. Call right after a
|
|
70
|
+
* table.add() whose ID sets were read with getIdSets just before it. If the
|
|
71
|
+
* table moved by anything other than that one commit (another writer), the
|
|
72
|
+
* entry is dropped and the next getIdSets re-reads.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} tableName
|
|
75
|
+
* @param {object[]} records
|
|
76
|
+
*/
|
|
77
|
+
async function noteAdded(tableName, records) {
|
|
78
|
+
const cached = entries.get(tableName);
|
|
79
|
+
if (!cached) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const table = await getTable(tableName);
|
|
83
|
+
let version;
|
|
84
|
+
try {
|
|
85
|
+
version = table ? await table.version() : null;
|
|
86
|
+
} catch (e) {
|
|
87
|
+
log(`Index ID cache: could not read ${tableName} version (${e.message}); will re-read.`);
|
|
88
|
+
entries.delete(tableName);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (table !== cached.table || version !== cached.version + 1) {
|
|
92
|
+
entries.delete(tableName);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
for (const record of records) {
|
|
96
|
+
for (const field of Object.keys(cached.sets)) {
|
|
97
|
+
if (record[field] !== undefined && record[field] !== null && record[field] !== "") {
|
|
98
|
+
cached.sets[field].add(record[field]);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
cached.version = version;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function invalidate(tableName) {
|
|
106
|
+
if (tableName === undefined) {
|
|
107
|
+
entries.clear();
|
|
108
|
+
} else {
|
|
109
|
+
entries.delete(tableName);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
getIdSets,
|
|
115
|
+
noteAdded,
|
|
116
|
+
invalidate,
|
|
117
|
+
/** @returns {boolean} */
|
|
118
|
+
has(tableName) {
|
|
119
|
+
return entries.has(tableName);
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Paths of email files that duplicate an already-indexed Message-ID.
|
|
126
|
+
* retainOnly() drops paths that left the incremental lookback window, so the
|
|
127
|
+
* set stays bounded by the window's duplicates.
|
|
128
|
+
*/
|
|
129
|
+
export function createDuplicatePathTracker() {
|
|
130
|
+
const paths = new Set();
|
|
131
|
+
return {
|
|
132
|
+
add(filePath) {
|
|
133
|
+
paths.add(filePath);
|
|
134
|
+
},
|
|
135
|
+
has(filePath) {
|
|
136
|
+
return paths.has(filePath);
|
|
137
|
+
},
|
|
138
|
+
/** @param {Iterable<string>} currentPaths */
|
|
139
|
+
retainOnly(currentPaths) {
|
|
140
|
+
const keep = currentPaths instanceof Set ? currentPaths : new Set(currentPaths);
|
|
141
|
+
for (const filePath of paths) {
|
|
142
|
+
if (!keep.has(filePath)) {
|
|
143
|
+
paths.delete(filePath);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
clear() {
|
|
148
|
+
paths.clear();
|
|
149
|
+
},
|
|
150
|
+
get size() {
|
|
151
|
+
return paths.size;
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* @param {{ lastOptimizeTime?: number | null, now?: number, wroteSomething: boolean, intervalMs?: number }} args
|
|
158
|
+
*/
|
|
159
|
+
export function isIndexOptimizeDue({
|
|
160
|
+
lastOptimizeTime,
|
|
161
|
+
now = Date.now(),
|
|
162
|
+
wroteSomething,
|
|
163
|
+
intervalMs = INDEX_OPTIMIZE_INTERVAL_MS
|
|
164
|
+
}) {
|
|
165
|
+
if (!wroteSomething) {
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
if (typeof lastOptimizeTime !== "number" || !Number.isFinite(lastOptimizeTime)) {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
if (lastOptimizeTime > now) {
|
|
172
|
+
return true;
|
|
173
|
+
}
|
|
174
|
+
return now - lastOptimizeTime >= intervalMs;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Compact each open table and remove versions older than keepVersionsMs.
|
|
179
|
+
* Failures are logged per table and never thrown: an index pass must not fail
|
|
180
|
+
* because upkeep did.
|
|
181
|
+
*
|
|
182
|
+
* @param {{
|
|
183
|
+
* tables: Record<string, { optimize: (opts: object) => Promise<any> } | null | undefined>,
|
|
184
|
+
* tableNames: string[],
|
|
185
|
+
* now?: number,
|
|
186
|
+
* keepVersionsMs?: number,
|
|
187
|
+
* log?: (msg: string) => void
|
|
188
|
+
* }} args
|
|
189
|
+
* @returns {Promise<{ optimized: string[], failed: string[] }>}
|
|
190
|
+
*/
|
|
191
|
+
export async function optimizeIndexTables({
|
|
192
|
+
tables,
|
|
193
|
+
tableNames,
|
|
194
|
+
now = Date.now(),
|
|
195
|
+
keepVersionsMs = INDEX_OPTIMIZE_KEEP_VERSIONS_MS,
|
|
196
|
+
log = (msg) => console.error(msg)
|
|
197
|
+
}) {
|
|
198
|
+
const optimized = [];
|
|
199
|
+
const failed = [];
|
|
200
|
+
const cleanupOlderThan = new Date(now - keepVersionsMs);
|
|
201
|
+
for (const name of tableNames) {
|
|
202
|
+
const table = tables[name];
|
|
203
|
+
if (!table || typeof table.optimize !== "function") {
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const started = Date.now();
|
|
207
|
+
try {
|
|
208
|
+
const stats = await table.optimize({ cleanupOlderThan });
|
|
209
|
+
const c = stats?.compaction || {};
|
|
210
|
+
const p = stats?.prune || {};
|
|
211
|
+
log(
|
|
212
|
+
`Optimized ${name}: fragments -${c.fragmentsRemoved ?? 0} +${c.fragmentsAdded ?? 0}, ` +
|
|
213
|
+
`old versions removed ${p.oldVersionsRemoved ?? 0}, ` +
|
|
214
|
+
`${Math.round(Number(p.bytesRemoved ?? 0) / (1024 * 1024))} MB freed (${Date.now() - started}ms)`
|
|
215
|
+
);
|
|
216
|
+
optimized.push(name);
|
|
217
|
+
} catch (e) {
|
|
218
|
+
log(`Optimize ${name} failed (index still usable): ${e.message}`);
|
|
219
|
+
failed.push(name);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return { optimized, failed };
|
|
223
|
+
}
|
package/lib/lancedbTables.js
CHANGED
|
@@ -23,6 +23,58 @@ export const LANCE_CONNECT_OPTIONS = {
|
|
|
23
23
|
readConsistencyInterval: LANCE_READ_CONSISTENCY_INTERVAL
|
|
24
24
|
};
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* LanceDB session cache caps. Without a Session, every connection gets the
|
|
28
|
+
* library defaults: a 6 GB index cache and a 1 GB metadata cache. The indexer
|
|
29
|
+
* daemon runs for days and fills them, so cap both. Override with
|
|
30
|
+
* APPLE_TOOLS_LANCE_INDEX_CACHE_MB / APPLE_TOOLS_LANCE_METADATA_CACHE_MB.
|
|
31
|
+
*/
|
|
32
|
+
export const LANCE_INDEX_CACHE_MB_DEFAULT = 64;
|
|
33
|
+
export const LANCE_METADATA_CACHE_MB_DEFAULT = 256;
|
|
34
|
+
const LANCE_CACHE_MB_MAX = 8192;
|
|
35
|
+
|
|
36
|
+
function cacheMbFromEnv(raw, fallback) {
|
|
37
|
+
if (raw === undefined || raw === null || String(raw).trim() === "") {
|
|
38
|
+
return fallback;
|
|
39
|
+
}
|
|
40
|
+
const text = String(raw).trim();
|
|
41
|
+
if (!/^[0-9]+$/.test(text)) {
|
|
42
|
+
return fallback;
|
|
43
|
+
}
|
|
44
|
+
const mb = Number(text);
|
|
45
|
+
if (!Number.isSafeInteger(mb) || mb < 1 || mb > LANCE_CACHE_MB_MAX) {
|
|
46
|
+
return fallback;
|
|
47
|
+
}
|
|
48
|
+
return mb;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {Record<string, string | undefined>} [env]
|
|
53
|
+
* @returns {{ indexCacheMb: number, metadataCacheMb: number, indexCacheBytes: bigint, metadataCacheBytes: bigint }}
|
|
54
|
+
*/
|
|
55
|
+
export function resolveLanceCacheSizes(env = process.env) {
|
|
56
|
+
const indexCacheMb = cacheMbFromEnv(env.APPLE_TOOLS_LANCE_INDEX_CACHE_MB, LANCE_INDEX_CACHE_MB_DEFAULT);
|
|
57
|
+
const metadataCacheMb = cacheMbFromEnv(env.APPLE_TOOLS_LANCE_METADATA_CACHE_MB, LANCE_METADATA_CACHE_MB_DEFAULT);
|
|
58
|
+
return {
|
|
59
|
+
indexCacheMb,
|
|
60
|
+
metadataCacheMb,
|
|
61
|
+
indexCacheBytes: BigInt(indexCacheMb) * 1024n * 1024n,
|
|
62
|
+
metadataCacheBytes: BigInt(metadataCacheMb) * 1024n * 1024n
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Build a capped LanceDB Session. `SessionClass` is lancedb's `Session`
|
|
68
|
+
* (injected so this module stays free of the native import).
|
|
69
|
+
*
|
|
70
|
+
* @param {new (indexBytes: bigint, metadataBytes: bigint) => object} SessionClass
|
|
71
|
+
* @param {Record<string, string | undefined>} [env]
|
|
72
|
+
*/
|
|
73
|
+
export function createLanceSession(SessionClass, env = process.env) {
|
|
74
|
+
const sizes = resolveLanceCacheSizes(env);
|
|
75
|
+
return new SessionClass(sizes.indexCacheBytes, sizes.metadataCacheBytes);
|
|
76
|
+
}
|
|
77
|
+
|
|
26
78
|
/**
|
|
27
79
|
* LanceDB stores each table as `<name>.lance` under the index directory.
|
|
28
80
|
*
|
|
@@ -150,8 +202,9 @@ export async function openMissingIndexTables({
|
|
|
150
202
|
* so indexer.js can alias it for createTable/dropTable mutation.
|
|
151
203
|
*
|
|
152
204
|
* @param {{
|
|
153
|
-
* connect: (uri: string, options: object) => Promise<object>,
|
|
205
|
+
* connect: (uri: string, options: object, session?: object) => Promise<object>,
|
|
154
206
|
* indexDir: string,
|
|
207
|
+
* session?: object,
|
|
155
208
|
* mkdirSync?: (p: string, opts?: object) => void,
|
|
156
209
|
* existsSync?: (p: string) => boolean,
|
|
157
210
|
* log?: (msg: string) => void,
|
|
@@ -165,7 +218,8 @@ export function createLanceTableCache(options) {
|
|
|
165
218
|
mkdirSync,
|
|
166
219
|
existsSync = fs.existsSync,
|
|
167
220
|
log = (msg) => console.error(msg),
|
|
168
|
-
connectOptions = LANCE_CONNECT_OPTIONS
|
|
221
|
+
connectOptions = LANCE_CONNECT_OPTIONS,
|
|
222
|
+
session
|
|
169
223
|
} = options;
|
|
170
224
|
|
|
171
225
|
const tables = {};
|
|
@@ -201,7 +255,9 @@ export function createLanceTableCache(options) {
|
|
|
201
255
|
if (typeof mkdirSync === "function") {
|
|
202
256
|
mkdirSync(indexDir, { recursive: true });
|
|
203
257
|
}
|
|
204
|
-
const conn =
|
|
258
|
+
const conn = session
|
|
259
|
+
? await connect(indexDir, connectOptions, session)
|
|
260
|
+
: await connect(indexDir, connectOptions);
|
|
205
261
|
if (gen !== generation) {
|
|
206
262
|
try {
|
|
207
263
|
conn?.close?.();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apple-tools-mcp",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "MCP server for semantic search and write actions across Apple Mail, Messages, Calendar, and Contacts. Speaks stdio (local) or bearer-token-authenticated Streamable HTTP (LAN / Tailscale).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|