apple-tools-mcp 3.2.4 → 3.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -184,6 +184,8 @@ The refresh interval is chosen once, when the process starts. The default is 5 m
184
184
 
185
185
  Values are clamped to 15 seconds through 6 hours. A missing config file is fine; the 5-minute default is used.
186
186
 
187
+ The indexer keeps its memory bounded for long runs. The vector database's caches are capped at a 64 MB index cache and a 256 MB metadata cache; set `APPLE_TOOLS_LANCE_INDEX_CACHE_MB` or `APPLE_TOOLS_LANCE_METADATA_CACHE_MB` (whole megabytes, 1 to 8192) in the LaunchAgent's `EnvironmentVariables` to change them. Indexed IDs stay in memory between passes, so a pass with no new mail does not re-read the index. Once a day, after a pass that added or removed something, it compacts the index files and removes table versions older than an hour.
188
+
187
189
  If you do not run the indexer, the MCP process indexes when it starts, then exits when the client disconnects. Writes then run inside that process, and macOS attributes them to the app that launched it.
188
190
 
189
191
  ### 5. Another computer (optional)
package/indexer.js CHANGED
@@ -15,7 +15,18 @@ import {
15
15
  } from "./lib/validators.js";
16
16
  import { safeSqlite3Json, safeSqlite3JsonAsync, safeOsascript, safeFindAsync } from "./lib/shell.js";
17
17
  import { indexUnavailableMessage } from "./lib/indexGate.js";
18
- import { createLanceTableCache, LANCE_CONNECT_OPTIONS } from "./lib/lancedbTables.js";
18
+ import {
19
+ createLanceTableCache,
20
+ createLanceSession,
21
+ INDEX_TABLE_NAMES,
22
+ LANCE_CONNECT_OPTIONS
23
+ } from "./lib/lancedbTables.js";
24
+ import {
25
+ createIndexedIdCache,
26
+ createDuplicatePathTracker,
27
+ isIndexOptimizeDue,
28
+ optimizeIndexTables
29
+ } from "./lib/indexMaintenance.js";
19
30
  import { createMessageContactMatcher } from "./lib/messageContact.js";
20
31
  import { resolveByExactName, normalizePhone } from "./contacts.js";
21
32
 
@@ -77,15 +88,30 @@ let embeddingPipeline = null;
77
88
 
78
89
  // One connection per process. Do not early-return a first connect that saw an
79
90
  // empty catalog — MCP stdio must pick up tables the indexer daemon already wrote.
91
+ // The capped Session keeps LanceDB's caches from growing to their 6 GB / 1 GB
92
+ // defaults in the long-lived indexer daemon.
80
93
  const lanceCache = createLanceTableCache({
81
- connect: (uri, options) => lancedb.connect(uri, options),
94
+ connect: (uri, options, session) => lancedb.connect(uri, options, session),
82
95
  indexDir: INDEX_DIR,
83
96
  mkdirSync: (p, o) => fs.mkdirSync(p, o),
84
- connectOptions: LANCE_CONNECT_OPTIONS
97
+ connectOptions: LANCE_CONNECT_OPTIONS,
98
+ session: createLanceSession(lancedb.Session)
85
99
  });
86
100
  const tables = lanceCache.tables;
87
101
  let db = null;
88
102
 
103
+ // Indexed ID sets stay in memory between passes; a table is re-read only when
104
+ // its version moves. Email files skipped as Message-ID duplicates are
105
+ // remembered so the same IMAP copies are not parsed every pass.
106
+ const EMAIL_ID_FIELDS = ["filePath", "messageId"];
107
+ const indexedIdCache = createIndexedIdCache({
108
+ getTable: async (name) => {
109
+ await initDB();
110
+ return tables[name] || null;
111
+ }
112
+ });
113
+ const duplicateEmailPaths = createDuplicatePathTracker();
114
+
89
115
  async function getEmbedder() {
90
116
  if (!embeddingPipeline) {
91
117
  console.error("Loading embedding model (first time may take a minute)...");
@@ -782,6 +808,8 @@ export async function getOpenTable(type) {
782
808
  function resetLanceConnection() {
783
809
  lanceCache.reset();
784
810
  db = null;
811
+ indexedIdCache.invalidate();
812
+ duplicateEmailPaths.clear();
785
813
  }
786
814
 
787
815
  export async function clearEmailsTable() {
@@ -789,6 +817,8 @@ export async function clearEmailsTable() {
789
817
  if (tables.emails) {
790
818
  await db.dropTable("emails");
791
819
  tables.emails = null;
820
+ indexedIdCache.invalidate("emails");
821
+ duplicateEmailPaths.clear();
792
822
  console.error("Emails table dropped. Re-run indexing to rebuild.");
793
823
  return { cleared: true };
794
824
  }
@@ -800,6 +830,7 @@ export async function clearMessagesTable() {
800
830
  if (tables.messages) {
801
831
  await db.dropTable("messages");
802
832
  tables.messages = null;
833
+ indexedIdCache.invalidate("messages");
803
834
  console.error("Messages table dropped. Re-run indexing to rebuild.");
804
835
  return { cleared: true };
805
836
  }
@@ -910,6 +941,31 @@ async function getIndexedIdsWithRetry(tableName, idField, maxRetries = 3) {
910
941
  throw lastError;
911
942
  }
912
943
 
944
+ /**
945
+ * Indexed ID sets for several columns of one table, read in one query and
946
+ * kept in memory between passes (re-read only when the table version moves).
947
+ * Same retry policy as getIndexedIdsWithRetry.
948
+ */
949
+ async function getCachedIdSetsWithRetry(tableName, fields, maxRetries = 3) {
950
+ let lastError;
951
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
952
+ try {
953
+ return await indexedIdCache.getIdSets(tableName, fields);
954
+ } catch (e) {
955
+ // THROW (after retries) instead of returning empty Sets to prevent duplicate entries
956
+ console.error(`Error getting indexed IDs from ${tableName}: ${e.message}`);
957
+ lastError = new Error(`Failed to get indexed IDs from ${tableName}: ${e.message}`);
958
+ indexedIdCache.invalidate(tableName);
959
+ if (attempt < maxRetries) {
960
+ const backoff = Math.min(1000 * Math.pow(2, attempt - 1), 5000);
961
+ console.error(`Retry ${attempt}/${maxRetries} after ${backoff}ms...`);
962
+ await new Promise(resolve => setTimeout(resolve, backoff));
963
+ }
964
+ }
965
+ }
966
+ throw lastError;
967
+ }
968
+
913
969
  // ============ INDEX ALL CONTENT ============
914
970
 
915
971
  export async function indexEmails(progressCallback = null, forceFullScan = false) {
@@ -957,12 +1013,27 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
957
1013
  }
958
1014
  console.error(`Found ${newFiles.length} new/modified email files (${Date.now() - startTime}ms)`);
959
1015
 
960
- const indexedPaths = await getIndexedIdsWithRetry("emails", "filePath");
961
- const indexedMessageIds = await getIndexedIdsWithRetry("emails", "messageId");
1016
+ // One read of both ID columns, served from memory while the table is unchanged.
1017
+ const emailIdSets = await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS);
1018
+ const indexedPaths = emailIdSets.filePath;
1019
+ const indexedMessageIds = emailIdSets.messageId;
962
1020
  console.error(`Already indexed: ${indexedPaths.size} emails`);
963
1021
 
964
- // Filter out already-indexed files
965
- let toIndex = newFiles.filter(f => !indexedPaths.has(f));
1022
+ // Filter out already-indexed files and copies already known to duplicate an
1023
+ // indexed Message-ID (forget copies that left the lookback window).
1024
+ duplicateEmailPaths.retainOnly(newFiles);
1025
+ let knownDuplicates = 0;
1026
+ let toIndex = newFiles.filter(f => {
1027
+ if (indexedPaths.has(f)) return false;
1028
+ if (duplicateEmailPaths.has(f)) {
1029
+ knownDuplicates++;
1030
+ return false;
1031
+ }
1032
+ return true;
1033
+ });
1034
+ if (knownDuplicates > 0) {
1035
+ console.error(`Known duplicate copies (already indexed by Message-ID): ${knownDuplicates}`);
1036
+ }
966
1037
  console.error(`Need to index: ${toIndex.length} emails`);
967
1038
 
968
1039
  // Sort by mailbox priority so higher-priority folders are indexed first
@@ -976,7 +1047,8 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
976
1047
  }
977
1048
 
978
1049
  let processed = 0;
979
- let skippedCount = { parseNull: 0, shortSearchText: 0, duplicateMessageId: 0 };
1050
+ let indexedCount = 0;
1051
+ let skippedCount = { parseNull: 0, shortSearchText: 0, duplicateMessageId: 0, alreadyIndexed: 0 };
980
1052
 
981
1053
  // Progress tracking
982
1054
  const totalBatches = Math.ceil(toIndex.length / BATCH_SIZE);
@@ -989,6 +1061,9 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
989
1061
  // Parse all emails in batch first
990
1062
  const parsedEmails = [];
991
1063
  const searchTexts = [];
1064
+ // Message-IDs taken earlier in this batch (two copies of one new email can
1065
+ // land in the same batch before either is in indexedMessageIds)
1066
+ const batchMessageIds = new Set();
992
1067
 
993
1068
  for (const filePath of batch) {
994
1069
  const parsed = parseEmlx(filePath);
@@ -1005,13 +1080,18 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
1005
1080
  continue;
1006
1081
  }
1007
1082
  // Skip duplicate messageIds - same email in multiple folders (IMAP behavior)
1008
- if (parsed.messageId && indexedMessageIds.has(parsed.messageId)) {
1083
+ if (parsed.messageId && (indexedMessageIds.has(parsed.messageId) || batchMessageIds.has(parsed.messageId))) {
1009
1084
  skippedCount.duplicateMessageId++;
1085
+ // Remember the copy's path so later passes skip it before parsing.
1086
+ duplicateEmailPaths.add(filePath);
1010
1087
  continue; // Skip this duplicate
1011
1088
  }
1012
1089
  // NOTE: Don't filter by email header date - mdfind already filters by file modification time
1013
1090
  // The DAYS_BACK filter is applied at the mdfind level (finding recently modified files),
1014
1091
  // not at the email content level (email header dates can be old for recently modified messages)
1092
+ if (parsed.messageId) {
1093
+ batchMessageIds.add(parsed.messageId);
1094
+ }
1015
1095
  parsedEmails.push({ filePath, parsed });
1016
1096
  searchTexts.push(parsed.searchText);
1017
1097
  }
@@ -1068,6 +1148,7 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
1068
1148
  if (uniqueRecords.length > 0) {
1069
1149
  if (!tables.emails) {
1070
1150
  tables.emails = await db.createTable("emails", uniqueRecords, { mode: "overwrite" });
1151
+ indexedCount += uniqueRecords.length;
1071
1152
  // Track messageIds from the first batch so later batches can
1072
1153
  // dedupe IMAP copies (INBOX vs Junk) of the same Message-ID.
1073
1154
  for (const record of uniqueRecords) {
@@ -1077,16 +1158,18 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
1077
1158
  }
1078
1159
  } else {
1079
1160
  // Double-check: verify these IDs truly aren't in the index
1080
- const currentIndexed = await getIndexedIdsWithRetry("emails", "filePath");
1161
+ const currentIndexed = (await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS)).filePath;
1081
1162
  const trulyNew = uniqueRecords.filter(r => !currentIndexed.has(r.filePath));
1082
1163
 
1083
1164
  if (trulyNew.length < uniqueRecords.length) {
1165
+ skippedCount.alreadyIndexed += uniqueRecords.length - trulyNew.length;
1084
1166
  console.error(`WARNING: ${uniqueRecords.length - trulyNew.length} records were already indexed. Filtering them out.`);
1085
1167
  }
1086
1168
 
1087
1169
  if (trulyNew.length > 0) {
1088
1170
  const beforeCount = await tables.emails.countRows();
1089
1171
  await tables.emails.add(trulyNew);
1172
+ indexedCount += trulyNew.length;
1090
1173
  const afterCount = await tables.emails.countRows();
1091
1174
  const actualAdded = afterCount - beforeCount;
1092
1175
 
@@ -1101,6 +1184,7 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
1101
1184
  indexedMessageIds.add(record.messageId);
1102
1185
  }
1103
1186
  }
1187
+ await indexedIdCache.noteAdded("emails", trulyNew);
1104
1188
  }
1105
1189
  }
1106
1190
  }
@@ -1134,20 +1218,23 @@ export async function indexEmails(progressCallback = null, forceFullScan = false
1134
1218
 
1135
1219
  // Log skip summary with timing
1136
1220
  const totalProcessingTime = Date.now() - startProcessTime;
1137
- const totalSkipped = skippedCount.parseNull + skippedCount.shortSearchText + skippedCount.duplicateMessageId;
1221
+ const totalSkipped = skippedCount.parseNull + skippedCount.shortSearchText +
1222
+ skippedCount.duplicateMessageId + skippedCount.alreadyIndexed;
1138
1223
  console.error(`\nEmail indexing summary:`);
1139
1224
  console.error(` Files to index: ${toIndex.length}`);
1140
- console.error(` Successfully indexed: ${processed}`);
1225
+ console.error(` Successfully indexed: ${indexedCount}`);
1141
1226
  console.error(` Skipped - parse errors: ${skippedCount.parseNull}`);
1142
1227
  console.error(` Skipped - short searchText: ${skippedCount.shortSearchText}`);
1143
1228
  console.error(` Skipped - duplicate messageId: ${skippedCount.duplicateMessageId}`);
1229
+ console.error(` Skipped - already indexed: ${skippedCount.alreadyIndexed}`);
1144
1230
  console.error(` Total skipped: ${totalSkipped}`);
1145
- console.error(` Discrepancy: ${toIndex.length - processed - totalSkipped}`);
1231
+ console.error(` Discrepancy: ${toIndex.length - indexedCount - totalSkipped}`);
1146
1232
  console.error(` Total time: ${Math.round(totalProcessingTime / 1000)}s (${Math.round(totalProcessingTime / 60000)}m)\n`);
1147
1233
 
1148
- // Return actual indexed count from database, not stale cache
1149
- const finalIndexedPaths = await getIndexedIdsWithRetry("emails", "filePath");
1150
- return { indexed: finalIndexedPaths.size, added: processed };
1234
+ // Indexed count from the version-checked ID cache (re-reads only if the
1235
+ // table moved by something other than this pass's own adds)
1236
+ const finalIndexedPaths = (await getCachedIdSetsWithRetry("emails", EMAIL_ID_FIELDS)).filePath;
1237
+ return { indexed: finalIndexedPaths.size, added: indexedCount };
1151
1238
  }
1152
1239
 
1153
1240
  export async function indexMessages(forceFullScan = false) {
@@ -1173,7 +1260,7 @@ export async function indexMessages(forceFullScan = false) {
1173
1260
  }
1174
1261
  console.error(`Found ${messages.length} messages${lastMessageIndexTime ? ' (incremental)' : ' (full scan)'}`);
1175
1262
 
1176
- const indexed = await getIndexedIdsWithRetry("messages", "id");
1263
+ const indexed = (await getCachedIdSetsWithRetry("messages", ["id"])).id;
1177
1264
  console.error(`Already indexed: ${indexed.size} messages`);
1178
1265
 
1179
1266
  const toIndex = messages.filter(m => !indexed.has(String(m.id)));
@@ -1186,6 +1273,7 @@ export async function indexMessages(forceFullScan = false) {
1186
1273
  }
1187
1274
 
1188
1275
  let processed = 0;
1276
+ let addedCount = 0;
1189
1277
 
1190
1278
  // Progress tracking
1191
1279
  const totalBatches = Math.ceil(toIndex.length / BATCH_SIZE);
@@ -1260,9 +1348,10 @@ export async function indexMessages(forceFullScan = false) {
1260
1348
  if (uniqueRecords.length > 0) {
1261
1349
  if (!tables.messages) {
1262
1350
  tables.messages = await db.createTable("messages", uniqueRecords, { mode: "overwrite" });
1351
+ addedCount += uniqueRecords.length;
1263
1352
  } else {
1264
1353
  // Double-check: verify these IDs truly aren't in the index
1265
- const currentIndexed = await getIndexedIdsWithRetry("messages", "id");
1354
+ const currentIndexed = (await getCachedIdSetsWithRetry("messages", ["id"])).id;
1266
1355
  const trulyNew = uniqueRecords.filter(r => !currentIndexed.has(r.id));
1267
1356
 
1268
1357
  if (trulyNew.length < uniqueRecords.length) {
@@ -1272,12 +1361,14 @@ export async function indexMessages(forceFullScan = false) {
1272
1361
  if (trulyNew.length > 0) {
1273
1362
  const beforeCount = await tables.messages.countRows();
1274
1363
  await tables.messages.add(trulyNew);
1364
+ addedCount += trulyNew.length;
1275
1365
  const afterCount = await tables.messages.countRows();
1276
1366
  const actualAdded = afterCount - beforeCount;
1277
1367
 
1278
1368
  if (actualAdded !== trulyNew.length) {
1279
1369
  console.error(`WARNING: Expected to add ${trulyNew.length} records but index grew by ${actualAdded}. Possible duplicate issue.`);
1280
1370
  }
1371
+ await indexedIdCache.noteAdded("messages", trulyNew);
1281
1372
  }
1282
1373
  }
1283
1374
  }
@@ -1304,7 +1395,7 @@ export async function indexMessages(forceFullScan = false) {
1304
1395
  // Save timestamp for next incremental scan
1305
1396
  saveIndexMeta({ ...meta, lastMessageIndexTime: indexStartTime });
1306
1397
 
1307
- return { indexed: indexed.size, added: processed };
1398
+ return { indexed: indexed.size, added: addedCount };
1308
1399
  }
1309
1400
 
1310
1401
  export async function indexCalendar() {
@@ -1458,10 +1549,35 @@ export async function indexAll(progressCallback = null) {
1458
1549
  const calendarResult = await indexCalendar();
1459
1550
  if (progressCallback) progressCallback('calendar-complete');
1460
1551
 
1552
+ await maybeOptimizeIndex({ emails: emailResult, messages: messageResult, calendar: calendarResult });
1553
+
1461
1554
  console.error("Full index complete.");
1462
1555
  return { emails: emailResult, messages: messageResult, calendar: calendarResult };
1463
1556
  }
1464
1557
 
1558
+ /**
1559
+ * Daily LanceDB compaction + old-version cleanup, only after a pass that wrote
1560
+ * something. Every add() leaves a small data file and a new version; without
1561
+ * this the tables grow thousands of fragments that every full read opens.
1562
+ */
1563
+ async function maybeOptimizeIndex(results) {
1564
+ const wroteSomething =
1565
+ (results.emails?.added || 0) > 0 ||
1566
+ (results.messages?.added || 0) > 0 ||
1567
+ (results.calendar?.added || 0) > 0 ||
1568
+ (results.calendar?.removed || 0) > 0;
1569
+ const meta = loadIndexMeta();
1570
+ if (!isIndexOptimizeDue({ lastOptimizeTime: meta.lastIndexOptimizeTime, wroteSomething })) {
1571
+ return null;
1572
+ }
1573
+ console.error("Optimizing index tables (compaction + old version cleanup)...");
1574
+ const result = await optimizeIndexTables({ tables, tableNames: INDEX_TABLE_NAMES });
1575
+ // Compaction makes new table versions; re-read IDs once next pass.
1576
+ indexedIdCache.invalidate();
1577
+ saveIndexMeta({ ...loadIndexMeta(), lastIndexOptimizeTime: Date.now() });
1578
+ return result;
1579
+ }
1580
+
1465
1581
  export async function isIndexReady(type = "emails") {
1466
1582
  await initDB();
1467
1583
  return tables[type] != null;
@@ -0,0 +1,223 @@
1
+ /**
2
+ * Indexer-daemon upkeep that keeps a long-lived process light:
3
+ *
4
+ * - createIndexedIdCache: keeps the indexed ID sets (email filePath and
5
+ * messageId, message id) in memory between passes and re-reads a table only
6
+ * when its LanceDB version moved. A quiet pass costs one version check
7
+ * instead of a full column scan.
8
+ * - createDuplicatePathTracker: remembers email files skipped because their
9
+ * Message-ID is already indexed (IMAP copies in another folder), so the
10
+ * same copies are not parsed again every pass.
11
+ * - optimizeIndexTables / isIndexOptimizeDue: periodic LanceDB compaction and
12
+ * old-version cleanup, run after a pass that wrote something.
13
+ */
14
+
15
+ export const INDEX_OPTIMIZE_INTERVAL_MS = 24 * 60 * 60 * 1000;
16
+ /** Keep this much table history when cleaning up old versions. */
17
+ export const INDEX_OPTIMIZE_KEEP_VERSIONS_MS = 60 * 60 * 1000;
18
+
19
+ /**
20
+ * @param {{
21
+ * getTable: (tableName: string) => Promise<object | null>,
22
+ * log?: (msg: string) => void
23
+ * }} options
24
+ */
25
+ export function createIndexedIdCache({ getTable, log = (msg) => console.error(msg) }) {
26
+ /** @type {Map<string, { table: object, version: number, sets: Record<string, Set<unknown>> }>} */
27
+ const entries = new Map();
28
+
29
+ function emptySets(fields) {
30
+ return Object.fromEntries(fields.map((field) => [field, new Set()]));
31
+ }
32
+
33
+ /**
34
+ * ID sets for `fields`, read in one query. Served from memory while the
35
+ * table handle and version are unchanged.
36
+ *
37
+ * @param {string} tableName
38
+ * @param {string[]} fields
39
+ * @returns {Promise<Record<string, Set<unknown>>>}
40
+ */
41
+ async function getIdSets(tableName, fields) {
42
+ const table = await getTable(tableName);
43
+ if (!table) {
44
+ entries.delete(tableName);
45
+ return emptySets(fields);
46
+ }
47
+ const version = await table.version();
48
+ const cached = entries.get(tableName);
49
+ if (
50
+ cached &&
51
+ cached.table === table &&
52
+ cached.version === version &&
53
+ fields.every((field) => cached.sets[field])
54
+ ) {
55
+ return cached.sets;
56
+ }
57
+ const rows = await table.query().select(fields).toArray();
58
+ const sets = emptySets(fields);
59
+ for (const row of rows) {
60
+ for (const field of fields) {
61
+ sets[field].add(row[field]);
62
+ }
63
+ }
64
+ entries.set(tableName, { table, version, sets });
65
+ return sets;
66
+ }
67
+
68
+ /**
69
+ * Fold rows this process just added into the cached sets. Call right after a
70
+ * table.add() whose ID sets were read with getIdSets just before it. If the
71
+ * table moved by anything other than that one commit (another writer), the
72
+ * entry is dropped and the next getIdSets re-reads.
73
+ *
74
+ * @param {string} tableName
75
+ * @param {object[]} records
76
+ */
77
+ async function noteAdded(tableName, records) {
78
+ const cached = entries.get(tableName);
79
+ if (!cached) {
80
+ return;
81
+ }
82
+ const table = await getTable(tableName);
83
+ let version;
84
+ try {
85
+ version = table ? await table.version() : null;
86
+ } catch (e) {
87
+ log(`Index ID cache: could not read ${tableName} version (${e.message}); will re-read.`);
88
+ entries.delete(tableName);
89
+ return;
90
+ }
91
+ if (table !== cached.table || version !== cached.version + 1) {
92
+ entries.delete(tableName);
93
+ return;
94
+ }
95
+ for (const record of records) {
96
+ for (const field of Object.keys(cached.sets)) {
97
+ if (record[field] !== undefined && record[field] !== null && record[field] !== "") {
98
+ cached.sets[field].add(record[field]);
99
+ }
100
+ }
101
+ }
102
+ cached.version = version;
103
+ }
104
+
105
+ function invalidate(tableName) {
106
+ if (tableName === undefined) {
107
+ entries.clear();
108
+ } else {
109
+ entries.delete(tableName);
110
+ }
111
+ }
112
+
113
+ return {
114
+ getIdSets,
115
+ noteAdded,
116
+ invalidate,
117
+ /** @returns {boolean} */
118
+ has(tableName) {
119
+ return entries.has(tableName);
120
+ }
121
+ };
122
+ }
123
+
124
+ /**
125
+ * Paths of email files that duplicate an already-indexed Message-ID.
126
+ * retainOnly() drops paths that left the incremental lookback window, so the
127
+ * set stays bounded by the window's duplicates.
128
+ */
129
+ export function createDuplicatePathTracker() {
130
+ const paths = new Set();
131
+ return {
132
+ add(filePath) {
133
+ paths.add(filePath);
134
+ },
135
+ has(filePath) {
136
+ return paths.has(filePath);
137
+ },
138
+ /** @param {Iterable<string>} currentPaths */
139
+ retainOnly(currentPaths) {
140
+ const keep = currentPaths instanceof Set ? currentPaths : new Set(currentPaths);
141
+ for (const filePath of paths) {
142
+ if (!keep.has(filePath)) {
143
+ paths.delete(filePath);
144
+ }
145
+ }
146
+ },
147
+ clear() {
148
+ paths.clear();
149
+ },
150
+ get size() {
151
+ return paths.size;
152
+ }
153
+ };
154
+ }
155
+
156
+ /**
157
+ * @param {{ lastOptimizeTime?: number | null, now?: number, wroteSomething: boolean, intervalMs?: number }} args
158
+ */
159
+ export function isIndexOptimizeDue({
160
+ lastOptimizeTime,
161
+ now = Date.now(),
162
+ wroteSomething,
163
+ intervalMs = INDEX_OPTIMIZE_INTERVAL_MS
164
+ }) {
165
+ if (!wroteSomething) {
166
+ return false;
167
+ }
168
+ if (typeof lastOptimizeTime !== "number" || !Number.isFinite(lastOptimizeTime)) {
169
+ return true;
170
+ }
171
+ if (lastOptimizeTime > now) {
172
+ return true;
173
+ }
174
+ return now - lastOptimizeTime >= intervalMs;
175
+ }
176
+
177
+ /**
178
+ * Compact each open table and remove versions older than keepVersionsMs.
179
+ * Failures are logged per table and never thrown: an index pass must not fail
180
+ * because upkeep did.
181
+ *
182
+ * @param {{
183
+ * tables: Record<string, { optimize: (opts: object) => Promise<any> } | null | undefined>,
184
+ * tableNames: string[],
185
+ * now?: number,
186
+ * keepVersionsMs?: number,
187
+ * log?: (msg: string) => void
188
+ * }} args
189
+ * @returns {Promise<{ optimized: string[], failed: string[] }>}
190
+ */
191
+ export async function optimizeIndexTables({
192
+ tables,
193
+ tableNames,
194
+ now = Date.now(),
195
+ keepVersionsMs = INDEX_OPTIMIZE_KEEP_VERSIONS_MS,
196
+ log = (msg) => console.error(msg)
197
+ }) {
198
+ const optimized = [];
199
+ const failed = [];
200
+ const cleanupOlderThan = new Date(now - keepVersionsMs);
201
+ for (const name of tableNames) {
202
+ const table = tables[name];
203
+ if (!table || typeof table.optimize !== "function") {
204
+ continue;
205
+ }
206
+ const started = Date.now();
207
+ try {
208
+ const stats = await table.optimize({ cleanupOlderThan });
209
+ const c = stats?.compaction || {};
210
+ const p = stats?.prune || {};
211
+ log(
212
+ `Optimized ${name}: fragments -${c.fragmentsRemoved ?? 0} +${c.fragmentsAdded ?? 0}, ` +
213
+ `old versions removed ${p.oldVersionsRemoved ?? 0}, ` +
214
+ `${Math.round(Number(p.bytesRemoved ?? 0) / (1024 * 1024))} MB freed (${Date.now() - started}ms)`
215
+ );
216
+ optimized.push(name);
217
+ } catch (e) {
218
+ log(`Optimize ${name} failed (index still usable): ${e.message}`);
219
+ failed.push(name);
220
+ }
221
+ }
222
+ return { optimized, failed };
223
+ }
@@ -23,6 +23,58 @@ export const LANCE_CONNECT_OPTIONS = {
23
23
  readConsistencyInterval: LANCE_READ_CONSISTENCY_INTERVAL
24
24
  };
25
25
 
26
+ /**
27
+ * LanceDB session cache caps. Without a Session, every connection gets the
28
+ * library defaults: a 6 GB index cache and a 1 GB metadata cache. The indexer
29
+ * daemon runs for days and fills them, so cap both. Override with
30
+ * APPLE_TOOLS_LANCE_INDEX_CACHE_MB / APPLE_TOOLS_LANCE_METADATA_CACHE_MB.
31
+ */
32
+ export const LANCE_INDEX_CACHE_MB_DEFAULT = 64;
33
+ export const LANCE_METADATA_CACHE_MB_DEFAULT = 256;
34
+ const LANCE_CACHE_MB_MAX = 8192;
35
+
36
+ function cacheMbFromEnv(raw, fallback) {
37
+ if (raw === undefined || raw === null || String(raw).trim() === "") {
38
+ return fallback;
39
+ }
40
+ const text = String(raw).trim();
41
+ if (!/^[0-9]+$/.test(text)) {
42
+ return fallback;
43
+ }
44
+ const mb = Number(text);
45
+ if (!Number.isSafeInteger(mb) || mb < 1 || mb > LANCE_CACHE_MB_MAX) {
46
+ return fallback;
47
+ }
48
+ return mb;
49
+ }
50
+
51
+ /**
52
+ * @param {Record<string, string | undefined>} [env]
53
+ * @returns {{ indexCacheMb: number, metadataCacheMb: number, indexCacheBytes: bigint, metadataCacheBytes: bigint }}
54
+ */
55
+ export function resolveLanceCacheSizes(env = process.env) {
56
+ const indexCacheMb = cacheMbFromEnv(env.APPLE_TOOLS_LANCE_INDEX_CACHE_MB, LANCE_INDEX_CACHE_MB_DEFAULT);
57
+ const metadataCacheMb = cacheMbFromEnv(env.APPLE_TOOLS_LANCE_METADATA_CACHE_MB, LANCE_METADATA_CACHE_MB_DEFAULT);
58
+ return {
59
+ indexCacheMb,
60
+ metadataCacheMb,
61
+ indexCacheBytes: BigInt(indexCacheMb) * 1024n * 1024n,
62
+ metadataCacheBytes: BigInt(metadataCacheMb) * 1024n * 1024n
63
+ };
64
+ }
65
+
66
+ /**
67
+ * Build a capped LanceDB Session. `SessionClass` is lancedb's `Session`
68
+ * (injected so this module stays free of the native import).
69
+ *
70
+ * @param {new (indexBytes: bigint, metadataBytes: bigint) => object} SessionClass
71
+ * @param {Record<string, string | undefined>} [env]
72
+ */
73
+ export function createLanceSession(SessionClass, env = process.env) {
74
+ const sizes = resolveLanceCacheSizes(env);
75
+ return new SessionClass(sizes.indexCacheBytes, sizes.metadataCacheBytes);
76
+ }
77
+
26
78
  /**
27
79
  * LanceDB stores each table as `<name>.lance` under the index directory.
28
80
  *
@@ -150,8 +202,9 @@ export async function openMissingIndexTables({
150
202
  * so indexer.js can alias it for createTable/dropTable mutation.
151
203
  *
152
204
  * @param {{
153
- * connect: (uri: string, options: object) => Promise<object>,
205
+ * connect: (uri: string, options: object, session?: object) => Promise<object>,
154
206
  * indexDir: string,
207
+ * session?: object,
155
208
  * mkdirSync?: (p: string, opts?: object) => void,
156
209
  * existsSync?: (p: string) => boolean,
157
210
  * log?: (msg: string) => void,
@@ -165,7 +218,8 @@ export function createLanceTableCache(options) {
165
218
  mkdirSync,
166
219
  existsSync = fs.existsSync,
167
220
  log = (msg) => console.error(msg),
168
- connectOptions = LANCE_CONNECT_OPTIONS
221
+ connectOptions = LANCE_CONNECT_OPTIONS,
222
+ session
169
223
  } = options;
170
224
 
171
225
  const tables = {};
@@ -201,7 +255,9 @@ export function createLanceTableCache(options) {
201
255
  if (typeof mkdirSync === "function") {
202
256
  mkdirSync(indexDir, { recursive: true });
203
257
  }
204
- const conn = await connect(indexDir, connectOptions);
258
+ const conn = session
259
+ ? await connect(indexDir, connectOptions, session)
260
+ : await connect(indexDir, connectOptions);
205
261
  if (gen !== generation) {
206
262
  try {
207
263
  conn?.close?.();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "apple-tools-mcp",
3
- "version": "3.2.4",
3
+ "version": "3.2.5",
4
4
  "description": "MCP server for semantic search and write actions across Apple Mail, Messages, Calendar, and Contacts. Speaks stdio (local) or bearer-token-authenticated Streamable HTTP (LAN / Tailscale).",
5
5
  "type": "module",
6
6
  "main": "index.js",