@cortexkit/pi-magic-context 0.47.0 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/auto-search-worker.js +27 -22
  2. package/dist/embedding-activity-ew6dshfd.js +10 -0
  3. package/dist/embedding-backfill-bnnytehy.js +20 -0
  4. package/dist/embedding-cache-7rp4gqzr.js +16 -0
  5. package/dist/embedding-worker.js +11 -8
  6. package/dist/harness-9872e0ym.js +10 -0
  7. package/dist/historian-calibration-extension.js +3 -2
  8. package/dist/index-0am8a7k1.js +152 -0
  9. package/dist/index-0h2mkab2.js +17 -0
  10. package/dist/index-135e8hvg.js +43 -0
  11. package/dist/{index-0qejsd7f.js → index-3cgbv47m.js} +289 -45
  12. package/dist/index-419bndtm.js +2309 -0
  13. package/dist/index-544c0e0x.js +3026 -0
  14. package/dist/index-5hd7h4rr.js +3098 -0
  15. package/dist/index-5jp1h0bz.js +750 -0
  16. package/dist/index-603mt6nm.js +171 -0
  17. package/dist/index-674f6qs2.js +16 -0
  18. package/dist/{index-vgcemg0n.js → index-6pgq5ns2.js} +151 -2452
  19. package/dist/{index-xpbh1ypk.js → index-72bdc1bd.js} +38 -741
  20. package/dist/index-8976mezv.js +209 -0
  21. package/dist/{index-td885aah.js → index-8y5knskr.js} +98 -231
  22. package/dist/{index-k3ezsjng.js → index-98z1c1r0.js} +14574 -13942
  23. package/dist/index-9p40dgkm.js +22 -0
  24. package/dist/index-9x1f96r8.js +11 -0
  25. package/dist/{index-266qrgss.js → index-9xb0axhc.js} +301 -27
  26. package/dist/{index-spk5vrhy.js → index-bgmv2kaf.js} +1 -16
  27. package/dist/index-dkx0b1kt.js +514 -0
  28. package/dist/{index-7n8gb6j3.js → index-f956htf3.js} +28 -31
  29. package/dist/index-fapenpch.js +13 -0
  30. package/dist/index-fm3ksh5e.js +1060 -0
  31. package/dist/{index-2p1e8706.js → index-fp37he9a.js} +3 -3
  32. package/dist/index-hrf13mn3.js +2388 -0
  33. package/dist/index-pvm4pd5d.js +3069 -0
  34. package/dist/{index-kztsbkqq.js → index-r7xg4hvw.js} +171 -168
  35. package/dist/index-rt5jgzp1.js +1127 -0
  36. package/dist/{index-n3g5mwa8.js → index-wrm4nbj7.js} +3 -3
  37. package/dist/index-xjqheg7a.js +931 -0
  38. package/dist/index-yhzhdp3m.js +143 -0
  39. package/dist/{index-w76x3af7.js → index-zrr5t5a7.js} +1 -3
  40. package/dist/index.js +4203 -2183
  41. package/dist/logger-v3dsxdnm.js +19 -0
  42. package/dist/message-index-bounded-t7vvty76.js +120 -0
  43. package/dist/{message-time-backfill-d3db2etr.js → message-time-backfill-4ndf7hrq.js} +7 -6
  44. package/dist/migration-worker.js +20 -14
  45. package/dist/project-embedding-registry-jy2hmq28.js +85 -0
  46. package/dist/raw-ordinal-warmup-worker.js +60 -0
  47. package/dist/{read-session-chunk-atxsyfqv.js → read-session-chunk-47pjn7fz.js} +12 -6
  48. package/dist/read-session-raw-yn8nawav.js +55 -0
  49. package/dist/{safe-notification-target-6cn3z99t.js → safe-notification-target-3c1ns931.js} +3 -2
  50. package/dist/search-42szfhwp.js +40 -0
  51. package/dist/sqlite-3yye76ht.js +44 -0
  52. package/dist/sqlite-context-pragmas-gcgch6qz.js +6 -0
  53. package/dist/storage-memory-1z20y55x.js +71 -0
  54. package/dist/storage-tags-jym5z6yk.js +127 -0
  55. package/dist/subagent-entry.js +34 -17
  56. package/dist/transformers-node-wasm.js +18415 -18415
  57. package/package.json +3 -3
  58. package/dist/index-7s8vw2y0.js +0 -5207
  59. package/dist/index-qaa5j277.js +0 -8544
  60. /package/dist/{index-en1k8h8m.js → index-5prbx026.js} +0 -0
@@ -0,0 +1,3026 @@
1
+ import {
2
+ getHarness2
3
+ } from "./index-0h2mkab2.js";
4
+ import {
5
+ sanitizeDiagnosticText,
6
+ log2
7
+ } from "./index-f956htf3.js";
8
+ import {
9
+ isUserHomeDirectory,
10
+ setBootQuietPeriodForTests
11
+ } from "./index-dkx0b1kt.js";
12
+ import {
13
+ estimateTokens
14
+ } from "./index-9xb0axhc.js";
15
+ import {
16
+ SYNAPSE_DEFAULT_MODEL,
17
+ isSynapseEmbeddingTruncated,
18
+ getSynapseBatchRequestKey,
19
+ SynapseEmbeddingProvider
20
+ } from "./index-419bndtm.js";
21
+ import {
22
+ withSqliteBackgroundWriter2,
23
+ withoutSqliteTransformPass2,
24
+ isTransientSqliteError2
25
+ } from "./index-3cgbv47m.js";
26
+ import {
27
+ CHUNK_WINDOW_SAFETY_RATIO,
28
+ normalizeCompartmentChunkMaxInputTokens,
29
+ buildCanonicalChunkTextFromFts,
30
+ buildCompartmentSummaryFallbackText,
31
+ chunkCanonicalText,
32
+ chunkEmbeddingWindowsAreCurrent,
33
+ replaceCompartmentChunkEmbeddings,
34
+ loadUnembeddedShadowChunkCandidates,
35
+ loadUnembeddedCompartmentChunkCandidatesPolite,
36
+ recordChunkEmbedBackoff,
37
+ chunkWindowSourceKey,
38
+ loadUnembeddedSessionChunkCandidatesPolite,
39
+ countSessionCompartmentEmbedCoverage,
40
+ countSessionCompartmentEmbedCoveragePolite,
41
+ countUnembeddedSessionCompartmentsPolite
42
+ } from "./index-fm3ksh5e.js";
43
+ import {
44
+ DEFAULT_LOCAL_EMBEDDING_MODEL,
45
+ embeddingModelsMatch,
46
+ resolveEmbeddingTextPrefixes,
47
+ getEmbeddingProviderIdentity,
48
+ dominantEmbeddingFailure,
49
+ LocalEmbeddingProvider
50
+ } from "./index-6pgq5ns2.js";
51
+ import {
52
+ isEmbeddingHostBusy2
53
+ } from "./index-9p40dgkm.js";
54
+ import {
55
+ saveEmbeddingIfHashMatches,
56
+ getMemoryEmbedCoverage,
57
+ invalidateProject2
58
+ } from "./index-yhzhdp3m.js";
59
+ import {
60
+ logSlowWriteTransaction
61
+ } from "./index-fp37he9a.js";
62
+
63
+ // ../plugin/src/features/magic-context/session-project-storage.ts
64
+ var SESSION_CHUNK_REPAIR_BATCH_SIZE = 100;
65
+ var upsertSessionProjectStatements = new WeakMap;
66
+ var repairSessionChunkProjectStatements = new WeakMap;
67
+ var misScopedProjectChunkStatements = new WeakMap;
68
+ var MIS_SCOPED_PROJECT_CHUNK_IDS_SQL = `
69
+ SELECT e.id FROM compartment_chunk_embeddings e
70
+ JOIN session_projects sp ON sp.session_id = e.session_id AND sp.harness = e.harness
71
+ WHERE e.project_path = ? AND sp.project_path <> e.project_path
72
+ UNION ALL
73
+ SELECT e.id FROM session_projects sp
74
+ JOIN compartment_chunk_embeddings e ON e.session_id = sp.session_id
75
+ WHERE sp.project_path = ? AND e.harness = sp.harness AND e.project_path <> sp.project_path`;
76
+ function getUpsertSessionProjectStatement(db) {
77
+ let stmt = upsertSessionProjectStatements.get(db);
78
+ if (!stmt) {
79
+ stmt = db.prepare(`INSERT INTO session_projects (session_id, harness, project_path, updated_at)
80
+ VALUES (?, ?, ?, ?)
81
+ ON CONFLICT(session_id, harness) DO UPDATE SET
82
+ project_path = excluded.project_path,
83
+ updated_at = excluded.updated_at
84
+ WHERE session_projects.project_path <> excluded.project_path`);
85
+ upsertSessionProjectStatements.set(db, stmt);
86
+ }
87
+ return stmt;
88
+ }
89
+ function getRepairSessionChunkProjectStatement(db) {
90
+ let stmt = repairSessionChunkProjectStatements.get(db);
91
+ if (!stmt) {
92
+ stmt = db.prepare(`UPDATE compartment_chunk_embeddings
93
+ SET project_path = ?
94
+ WHERE id IN (
95
+ SELECT id
96
+ FROM compartment_chunk_embeddings
97
+ WHERE session_id = ?
98
+ AND harness = ?
99
+ AND project_path <> ?
100
+ LIMIT ?
101
+ )`);
102
+ repairSessionChunkProjectStatements.set(db, stmt);
103
+ }
104
+ return stmt;
105
+ }
106
+ function findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectPath) {
107
+ return db.prepare(`SELECT id FROM (${MIS_SCOPED_PROJECT_CHUNK_IDS_SQL}) LIMIT 25`).all(projectPath, projectPath).map(({ id }) => id);
108
+ }
109
+ function recordSessionProjectIdentity(db, sessionId, projectPath) {
110
+ if (!sessionId || !projectPath)
111
+ return;
112
+ if (!projectPath.startsWith("git:") && !projectPath.startsWith("dir:") && isUserHomeDirectory(projectPath))
113
+ return;
114
+ const harness = getHarness2();
115
+ const now = Date.now();
116
+ let previousProjectPath;
117
+ db.transaction(() => {
118
+ const existing = db.prepare("SELECT project_path FROM session_projects WHERE session_id = ? AND harness = ?").get(sessionId, harness);
119
+ previousProjectPath = existing?.project_path;
120
+ getUpsertSessionProjectStatement(db).run(sessionId, harness, projectPath, now);
121
+ getRepairSessionChunkProjectStatement(db).run(projectPath, sessionId, harness, projectPath, SESSION_CHUNK_REPAIR_BATCH_SIZE);
122
+ }).immediate();
123
+ return previousProjectPath === projectPath ? undefined : { previousProjectPath, projectPath };
124
+ }
125
+ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath, ids = findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectPath)) {
126
+ if (!projectPath || ids.length === 0)
127
+ return 0;
128
+ return db.prepare(`UPDATE compartment_chunk_embeddings
129
+ SET project_path = (SELECT sp.project_path FROM session_projects sp
130
+ WHERE sp.session_id = compartment_chunk_embeddings.session_id
131
+ AND sp.harness = compartment_chunk_embeddings.harness)
132
+ WHERE id IN (${ids.map(() => "?").join(",")}) AND EXISTS (
133
+ SELECT 1 FROM session_projects sp
134
+ WHERE sp.session_id = compartment_chunk_embeddings.session_id
135
+ AND sp.harness = compartment_chunk_embeddings.harness
136
+ AND sp.project_path <> compartment_chunk_embeddings.project_path
137
+ AND (sp.project_path = ? OR compartment_chunk_embeddings.project_path = ?)
138
+ )`).run(...ids, projectPath, projectPath).changes;
139
+ }
140
+
141
+ // ../plugin/src/features/magic-context/storage-embedding-measurements.ts
142
+ import { createHash } from "node:crypto";
143
+ function normalizedQueryHash(query) {
144
+ const normalized = query.trim().replace(/\s+/g, " ").toLowerCase();
145
+ return createHash("sha256").update(normalized).digest("hex");
146
+ }
147
+ var MEASUREMENT_CORPUS_SESSION_ROW_CAP = 2000;
148
+ function recordEmbeddingMeasurement(db, input, cap = MEASUREMENT_CORPUS_SESSION_ROW_CAP) {
149
+ const queryTextHash = normalizedQueryHash(input.queryText);
150
+ const dedupKey = queryTextHash;
151
+ const result = db.prepare(`INSERT OR IGNORE INTO embedding_measurement_corpus
152
+ (session_id, project_path, dedup_key, cohort_key, query_text_hash,
153
+ primary_result_ids_json, shadow_result_ids_json, primary_latency_ms, shadow_latency_ms,
154
+ primary_failed, shadow_failed, primary_model_id, shadow_model_id,
155
+ primary_fingerprint, shadow_fingerprint, primary_epoch, shadow_epoch,
156
+ corpus_hash, coverage_json, created_at)
157
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(input.sessionId, input.projectPath, dedupKey, input.cohortKey, queryTextHash, JSON.stringify(input.primaryResultIds.slice(0, 10)), JSON.stringify(input.shadowResultIds.slice(0, 10)), input.primaryLatencyMs, input.shadowLatencyMs, input.primaryFailed ? 1 : 0, input.shadowFailed ? 1 : 0, input.primaryModelId, input.shadowModelId, input.primaryFingerprint, input.shadowFingerprint, input.primaryEpoch, input.shadowEpoch, input.corpusHash, JSON.stringify(input.coverage), Date.now());
158
+ if (result.changes > 0) {
159
+ const rowCount = db.prepare("SELECT COUNT(*) AS count FROM embedding_measurement_corpus WHERE session_id = ?").get(input.sessionId).count;
160
+ const overflow = rowCount - cap;
161
+ if (overflow > 0) {
162
+ db.prepare(`DELETE FROM embedding_measurement_corpus
163
+ WHERE session_id = ?
164
+ AND id IN (
165
+ SELECT id FROM embedding_measurement_corpus
166
+ WHERE session_id = ?
167
+ ORDER BY id ASC
168
+ LIMIT ?
169
+ )`).run(input.sessionId, input.sessionId, overflow);
170
+ }
171
+ }
172
+ return result.changes > 0;
173
+ }
174
+ function beginSynapseBatchLedger(db, input, now = Date.now()) {
175
+ db.prepare(`INSERT INTO synapse_batch_ledger
176
+ (session_id, project_path, scope, manifest_json, request_key, status, created_at, updated_at)
177
+ VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
178
+ ON CONFLICT(session_id, request_key) DO UPDATE SET
179
+ manifest_json = excluded.manifest_json,
180
+ updated_at = excluded.updated_at`).run(input.sessionId, input.projectPath, input.scope, JSON.stringify(input.manifest), input.requestKey, now, now);
181
+ }
182
+ function finishSynapseBatchLedger(db, sessionId, requestKey, status, now = Date.now()) {
183
+ db.prepare("UPDATE synapse_batch_ledger SET status = ?, updated_at = ? WHERE session_id = ? AND request_key = ?").run(status, now, sessionId, requestKey);
184
+ }
185
+ var SYNAPSE_BATCH_LEDGER_TTL_MS = 14 * 24 * 60 * 60 * 1000;
186
+ function pruneSynapseBatchLedgerForProject(db, projectIdentity, ttlMs = SYNAPSE_BATCH_LEDGER_TTL_MS) {
187
+ const ledgerTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'synapse_batch_ledger'").get();
188
+ if (!ledgerTable)
189
+ return 0;
190
+ const shadow = `shadow:${projectIdentity}`;
191
+ const cutoff = Date.now() - ttlMs;
192
+ const predicate = "session_id IN (?, ?) AND updated_at < ?";
193
+ if (!db.prepare(`SELECT 1 FROM synapse_batch_ledger WHERE ${predicate} LIMIT 1`).get(projectIdentity, shadow, cutoff))
194
+ return 0;
195
+ return db.prepare(`DELETE FROM synapse_batch_ledger WHERE ${predicate}`).run(projectIdentity, shadow, cutoff).changes;
196
+ }
197
+
198
+ // ../plugin/src/features/magic-context/memory/embedding-ssrf.ts
199
+ var METADATA_HOSTNAMES = new Set(["metadata.google.internal", "metadata.goog"]);
200
+ var IPV6_METADATA_HOSTS = new Set(["fd00:ec2::254"]);
201
+ function isLinkLocalIpv4(host) {
202
+ return /^169\.254\.\d{1,3}\.\d{1,3}$/.test(host);
203
+ }
204
+ function ipv4FromMappedIpv6(host) {
205
+ const m = /^::ffff:(.+)$/.exec(host);
206
+ if (!m)
207
+ return null;
208
+ const tail = m[1];
209
+ if (/^\d{1,3}(\.\d{1,3}){3}$/.test(tail))
210
+ return tail;
211
+ const hex = /^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(tail);
212
+ if (hex) {
213
+ const hi = Number.parseInt(hex[1], 16);
214
+ const lo = Number.parseInt(hex[2], 16);
215
+ if (Number.isNaN(hi) || Number.isNaN(lo))
216
+ return null;
217
+ return `${hi >> 8 & 255}.${hi & 255}.${lo >> 8 & 255}.${lo & 255}`;
218
+ }
219
+ return null;
220
+ }
221
+ function blockedEmbeddingEndpointReason(endpoint) {
222
+ const trimmed = endpoint.trim();
223
+ if (trimmed.length === 0)
224
+ return null;
225
+ let url;
226
+ try {
227
+ url = new URL(trimmed);
228
+ } catch {
229
+ return `embedding endpoint is not a valid URL: ${trimmed}`;
230
+ }
231
+ const host = url.hostname.toLowerCase().replace(/^\[/, "").replace(/\]$/, "");
232
+ if (METADATA_HOSTNAMES.has(host)) {
233
+ return `embedding endpoint host ${host} is a cloud metadata service (blocked)`;
234
+ }
235
+ if (IPV6_METADATA_HOSTS.has(host)) {
236
+ return `embedding endpoint host ${host} is the AWS IPv6 metadata service (blocked)`;
237
+ }
238
+ if (isLinkLocalIpv4(host)) {
239
+ return `embedding endpoint host ${host} is link-local / cloud metadata (blocked)`;
240
+ }
241
+ const mappedV4 = ipv4FromMappedIpv6(host);
242
+ if (mappedV4 && isLinkLocalIpv4(mappedV4)) {
243
+ return `embedding endpoint host ${host} (IPv4-mapped ${mappedV4}) is link-local / cloud metadata (blocked)`;
244
+ }
245
+ if (host.startsWith("fe80:")) {
246
+ return `embedding endpoint host ${host} is link-local / cloud metadata (blocked)`;
247
+ }
248
+ return null;
249
+ }
250
+
251
+ // ../plugin/src/features/magic-context/memory/embedding-openai.ts
252
+ function responseSlots(items, inputCount) {
253
+ if (items.every((item) => item?.index === undefined)) {
254
+ return items.map((_, position) => position);
255
+ }
256
+ const slots = [];
257
+ const seen = new Set;
258
+ for (const item of items) {
259
+ const index = item?.index;
260
+ if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= inputCount || seen.has(index)) {
261
+ return null;
262
+ }
263
+ seen.add(index);
264
+ slots.push(index);
265
+ }
266
+ return slots;
267
+ }
268
+ function capEmbeddingInput(input, maxInputTokens, purpose) {
269
+ const ratio = purpose === "query" ? CHUNK_WINDOW_SAFETY_RATIO : 1;
270
+ const budget = Math.max(1, Math.floor(maxInputTokens * ratio));
271
+ if (estimateTokens(input) <= budget)
272
+ return input;
273
+ let low = 0;
274
+ let high = input.length;
275
+ while (low < high) {
276
+ const mid = Math.ceil((low + high) / 2);
277
+ if (estimateTokens(input.slice(0, mid)) <= budget)
278
+ low = mid;
279
+ else
280
+ high = mid - 1;
281
+ }
282
+ if (low > 0 && low < input.length) {
283
+ const last = input.charCodeAt(low - 1);
284
+ const next = input.charCodeAt(low);
285
+ if (last >= 55296 && last <= 56319 && next >= 56320 && next <= 57343)
286
+ low -= 1;
287
+ }
288
+ return input.slice(0, low);
289
+ }
290
+ function normalizeEndpoint(endpoint) {
291
+ return endpoint?.trim().replace(/\/+$/, "") ?? "";
292
+ }
293
+ var FAILURE_THRESHOLD = 3;
294
+ var FAILURE_WINDOW_MS = 60000;
295
+ var OPEN_DURATION_MS = 5 * 60000;
296
+ var FETCH_TIMEOUT_MS = 30000;
297
+
298
+ class OpenAICompatibleEmbeddingProvider {
299
+ modelId;
300
+ maxInputTokens;
301
+ endpoint;
302
+ model;
303
+ apiKey;
304
+ inputType;
305
+ queryInputType;
306
+ queryPrefix;
307
+ documentPrefix;
308
+ truncate;
309
+ initialized = false;
310
+ failureTimes = [];
311
+ circuitOpenUntil = 0;
312
+ openLogged = false;
313
+ modelMismatchLogged = false;
314
+ lastFailureReason = null;
315
+ halfOpenProbeInFlight = false;
316
+ queryTruncationLogged = false;
317
+ constructor(options) {
318
+ this.endpoint = normalizeEndpoint(options.endpoint);
319
+ this.model = options.model?.trim() ?? "";
320
+ this.apiKey = options.apiKey?.trim() ?? "";
321
+ this.inputType = options.inputType?.trim() ?? "";
322
+ this.queryInputType = options.queryInputType?.trim() ?? "";
323
+ const prefixes = resolveEmbeddingTextPrefixes(this.model, options.queryInstruction, options.documentPrefix);
324
+ this.queryPrefix = prefixes.queryPrefix;
325
+ this.documentPrefix = prefixes.documentPrefix;
326
+ this.truncate = options.truncate?.trim() ?? "";
327
+ this.maxInputTokens = typeof options.maxInputTokens === "number" && Number.isFinite(options.maxInputTokens) ? Math.max(1, Math.floor(options.maxInputTokens)) : 512;
328
+ this.modelId = getEmbeddingProviderIdentity({
329
+ provider: "openai-compatible",
330
+ endpoint: this.endpoint,
331
+ model: this.model,
332
+ ...this.apiKey ? { api_key: this.apiKey } : {},
333
+ ...this.inputType ? { input_type: this.inputType } : {},
334
+ ...options.documentPrefix !== undefined ? { document_prefix: options.documentPrefix } : {},
335
+ ...this.truncate ? { truncate: this.truncate } : {}
336
+ });
337
+ }
338
+ async initialize() {
339
+ if (this.initialized)
340
+ return true;
341
+ if (!this.endpoint || !this.model) {
342
+ log2("[magic-context] openai-compatible embedding provider is missing endpoint or model");
343
+ this.initialized = false;
344
+ return false;
345
+ }
346
+ const blockedReason = blockedEmbeddingEndpointReason(this.endpoint);
347
+ if (blockedReason) {
348
+ log2(`[magic-context] embedding endpoint blocked: ${blockedReason}`);
349
+ this.initialized = false;
350
+ return false;
351
+ }
352
+ this.initialized = true;
353
+ return true;
354
+ }
355
+ resolveInputTypeForPurpose(purpose = "passage") {
356
+ if (purpose === "query") {
357
+ return this.queryInputType || this.inputType;
358
+ }
359
+ return this.inputType;
360
+ }
361
+ async embed(text, signal, purpose) {
362
+ const [embedding] = await this.embedBatch([text], signal, purpose);
363
+ return embedding ?? null;
364
+ }
365
+ async embedBatch(texts, signal, purpose) {
366
+ if (texts.length === 0) {
367
+ return [];
368
+ }
369
+ const textPrefix = purpose === "query" ? this.queryPrefix : this.documentPrefix;
370
+ const requestTexts = texts.map((text) => {
371
+ const input = `${textPrefix}${text.trim().length === 0 ? " " : text}`;
372
+ const capped = capEmbeddingInput(input, this.maxInputTokens, purpose);
373
+ if (purpose === "query" && capped.length < input.length && !this.queryTruncationLogged) {
374
+ log2(`[magic-context] embedding query truncated from ${input.length} to ${capped.length} characters to fit max_input_tokens`);
375
+ this.queryTruncationLogged = true;
376
+ }
377
+ return capped;
378
+ });
379
+ if (!await this.initialize()) {
380
+ return Array.from({ length: texts.length }, () => null);
381
+ }
382
+ if (signal?.aborted) {
383
+ return Array.from({ length: texts.length }, () => null);
384
+ }
385
+ let isProbe = false;
386
+ let internalController;
387
+ let timeoutHandle;
388
+ let onOuterAbort;
389
+ try {
390
+ const claim = this.claimProbeOrShortCircuit();
391
+ if (claim === "short_circuit") {
392
+ return Array.from({ length: texts.length }, () => null);
393
+ }
394
+ isProbe = claim === "probe";
395
+ internalController = new AbortController;
396
+ timeoutHandle = setTimeout(() => internalController?.abort(), FETCH_TIMEOUT_MS);
397
+ onOuterAbort = () => internalController?.abort();
398
+ if (signal) {
399
+ signal.addEventListener("abort", onOuterAbort, { once: true });
400
+ }
401
+ const inputTypeForRequest = this.resolveInputTypeForPurpose(purpose);
402
+ const response = await fetch(`${this.endpoint}/embeddings`, {
403
+ method: "POST",
404
+ headers: {
405
+ "content-type": "application/json",
406
+ ...this.apiKey ? { authorization: `Bearer ${this.apiKey}` } : {}
407
+ },
408
+ body: JSON.stringify({
409
+ model: this.model,
410
+ input: requestTexts,
411
+ ...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
412
+ ...this.truncate ? { truncate: this.truncate } : {}
413
+ }),
414
+ redirect: "error",
415
+ signal: internalController.signal
416
+ });
417
+ if (!response.ok) {
418
+ const excerpt = await response.text().catch(() => "");
419
+ const failure = this.failure("http_error", `HTTP ${response.status} from endpoint${this.bodyExcerpt(excerpt)}`, response.status >= 500 || response.status === 408 || response.status === 429);
420
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
421
+ this.recordFailure(isProbe, failure);
422
+ return Array.from({ length: texts.length }, () => null);
423
+ }
424
+ const rawBody = await response.text();
425
+ if (rawBody.trim().length === 0) {
426
+ const failure = this.failure("invalid_envelope", "response body was empty", false);
427
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
428
+ this.recordFailure(isProbe, failure);
429
+ return Array.from({ length: texts.length }, () => null);
430
+ }
431
+ let body;
432
+ try {
433
+ body = JSON.parse(rawBody);
434
+ } catch {
435
+ const failure = this.failure("invalid_envelope", `response body was not valid JSON${this.bodyExcerpt(rawBody)}`, false);
436
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
437
+ this.recordFailure(isProbe, failure);
438
+ return Array.from({ length: texts.length }, () => null);
439
+ }
440
+ const servedModel = typeof body.model === "string" ? body.model : "";
441
+ if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
442
+ if (!this.modelMismatchLogged) {
443
+ log2(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${sanitizeDiagnosticText(this.model)}" served="${sanitizeDiagnosticText(servedModel)}". Check that the endpoint serves the requested model; variant suffixes and vendor prefixes are matched automatically.`);
444
+ this.modelMismatchLogged = true;
445
+ }
446
+ this.recordFailure(isProbe, this.failure("substitution_rejected", `served model '${sanitizeDiagnosticText(servedModel)}' does not match requested '${sanitizeDiagnosticText(this.model)}' (substitution guard)`, false));
447
+ return Array.from({ length: texts.length }, () => null);
448
+ }
449
+ const responseKeys = Object.keys(body).sort();
450
+ if (!Array.isArray(body.data)) {
451
+ const failure = this.failure("invalid_envelope", `response had keys [${responseKeys.join(", ")}] but data[] was absent`, false);
452
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
453
+ this.recordFailure(isProbe, failure);
454
+ return Array.from({ length: texts.length }, () => null);
455
+ }
456
+ if (body.data.length === 0) {
457
+ const failure = this.failure("empty_result", "response data[] was empty", true);
458
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
459
+ this.recordFailure(isProbe, failure);
460
+ return Array.from({ length: texts.length }, () => null);
461
+ }
462
+ const slots = responseSlots(body.data, texts.length);
463
+ if (!slots) {
464
+ const failure = this.failure("invalid_envelope", "response data[].index values were missing, duplicated or out of range", false);
465
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
466
+ this.recordFailure(isProbe, failure);
467
+ return Array.from({ length: texts.length }, () => null);
468
+ }
469
+ const results = Array.from({ length: texts.length }, () => null);
470
+ body.data.forEach((item, position) => {
471
+ const embedding = item?.embedding;
472
+ const slot = slots[position];
473
+ if (slot !== undefined && slot < results.length && Array.isArray(embedding)) {
474
+ results[slot] = Float32Array.from(embedding);
475
+ }
476
+ });
477
+ if (results.every((r) => r === null)) {
478
+ const failure = this.failure("invalid_envelope", `response had keys [${responseKeys.join(", ")}] but data[].embedding was absent`, false);
479
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
480
+ this.recordFailure(isProbe, failure);
481
+ } else {
482
+ this.recordSuccess();
483
+ }
484
+ return results;
485
+ } catch (error) {
486
+ const isAbort = error instanceof Error && (error.name === "AbortError" || error.message.includes("aborted"));
487
+ if (isAbort) {
488
+ if (signal?.aborted) {} else {
489
+ log2(`[magic-context] openai-compatible embedding request timed out after ${FETCH_TIMEOUT_MS}ms`);
490
+ const failure = this.failure("transport_error", `request timed out after ${FETCH_TIMEOUT_MS}ms`, true);
491
+ this.recordFailure(isProbe, failure);
492
+ }
493
+ } else {
494
+ const detail = error instanceof Error ? error.message : String(error);
495
+ const failure = this.failure("transport_error", `transport error: ${sanitizeDiagnosticText(detail)}`, true);
496
+ log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
497
+ this.recordFailure(isProbe, failure);
498
+ }
499
+ return Array.from({ length: texts.length }, () => null);
500
+ } finally {
501
+ if (timeoutHandle !== undefined) {
502
+ clearTimeout(timeoutHandle);
503
+ }
504
+ if (signal && onOuterAbort) {
505
+ signal.removeEventListener("abort", onOuterAbort);
506
+ }
507
+ if (isProbe) {
508
+ this.halfOpenProbeInFlight = false;
509
+ }
510
+ }
511
+ }
512
+ async dispose() {
513
+ this.initialized = false;
514
+ }
515
+ isLoaded() {
516
+ return this.initialized;
517
+ }
518
+ claimProbeOrShortCircuit() {
519
+ if (this.circuitOpenUntil === 0) {
520
+ return "allow";
521
+ }
522
+ if (Date.now() < this.circuitOpenUntil) {
523
+ return "short_circuit";
524
+ }
525
+ if (this.halfOpenProbeInFlight) {
526
+ return "short_circuit";
527
+ }
528
+ this.halfOpenProbeInFlight = true;
529
+ log2(`[magic-context] openai-compatible embedding: circuit half-open, probing endpoint after ${this.lastFailureReason?.reason ?? "unknown failure"}`);
530
+ return "probe";
531
+ }
532
+ failure(failureClass, reason, retryable) {
533
+ return { class: failureClass, reason, retryable };
534
+ }
535
+ bodyExcerpt(body) {
536
+ const excerpt = sanitizeDiagnosticText(body).replace(/\s+/g, " ").trim().slice(0, 200);
537
+ return excerpt ? `: ${excerpt}` : "";
538
+ }
539
+ recordFailure(isProbe, failure) {
540
+ if (failure)
541
+ this.lastFailureReason = failure;
542
+ if (isProbe) {
543
+ this.circuitOpenUntil = Date.now() + OPEN_DURATION_MS;
544
+ if (!this.openLogged) {
545
+ log2(`[magic-context] openai-compatible embedding: probe failed (${failure?.reason ?? this.lastFailureReason?.reason ?? "unknown failure"}), re-opening circuit for ${OPEN_DURATION_MS / 60000}min`);
546
+ this.openLogged = true;
547
+ }
548
+ this.failureTimes = [];
549
+ return;
550
+ }
551
+ const now = Date.now();
552
+ const cutoff = now - FAILURE_WINDOW_MS;
553
+ this.failureTimes = this.failureTimes.filter((t) => t > cutoff);
554
+ this.failureTimes.push(now);
555
+ if (this.failureTimes.length >= FAILURE_THRESHOLD) {
556
+ this.circuitOpenUntil = now + OPEN_DURATION_MS;
557
+ if (!this.openLogged) {
558
+ log2(`[magic-context] openai-compatible embedding: opening circuit for ${OPEN_DURATION_MS / 60000}min after ${this.failureTimes.length} failures in ${FAILURE_WINDOW_MS / 1000}s (${failure?.reason ?? this.lastFailureReason?.reason ?? "unknown failure"})`);
559
+ this.openLogged = true;
560
+ }
561
+ this.failureTimes = [];
562
+ }
563
+ }
564
+ recordSuccess() {
565
+ if (this.failureTimes.length > 0 || this.circuitOpenUntil > 0 || this.openLogged) {
566
+ log2("[magic-context] openai-compatible embedding: endpoint recovered, circuit closed");
567
+ }
568
+ this.failureTimes = [];
569
+ this.circuitOpenUntil = 0;
570
+ this.openLogged = false;
571
+ this.lastFailureReason = null;
572
+ }
573
+ getLastFailureReason() {
574
+ return this.lastFailureReason;
575
+ }
576
+ _getCircuitState() {
577
+ if (this.circuitOpenUntil === 0)
578
+ return "closed";
579
+ if (Date.now() < this.circuitOpenUntil) {
580
+ return this.halfOpenProbeInFlight ? "half_open" : "open";
581
+ }
582
+ return "half_open";
583
+ }
584
+ _getFailureCount() {
585
+ return this.failureTimes.length;
586
+ }
587
+ _resetCircuit() {
588
+ this.failureTimes = [];
589
+ this.circuitOpenUntil = 0;
590
+ this.openLogged = false;
591
+ this.halfOpenProbeInFlight = false;
592
+ this.lastFailureReason = null;
593
+ }
594
+ }
595
+
596
+ // ../plugin/src/features/magic-context/project-embedding-registry.ts
597
+ import { createHash as createHash2, randomUUID } from "node:crypto";
598
+
599
+ // ../plugin/src/shared/background-batch-drain.ts
600
+ async function drainBackgroundBatches(step, options = {}) {
601
+ const now = options.now ?? (() => performance.now());
602
+ const deadline = now() + (options.budgetMs ?? 2000);
603
+ const yieldFn = options.yieldFn ?? (() => new Promise((resolve) => setTimeout(resolve, 0)));
604
+ let batches = 0;
605
+ while (now() < deadline) {
606
+ try {
607
+ if (!withoutSqliteTransformPass2(() => withSqliteBackgroundWriter2(step)))
608
+ break;
609
+ } catch (error) {
610
+ if (isTransientSqliteError2(error))
611
+ break;
612
+ throw error;
613
+ }
614
+ batches++;
615
+ await yieldFn();
616
+ }
617
+ return batches;
618
+ }
619
+
620
+ // ../plugin/src/features/magic-context/git-commits/storage-git-commit-embeddings.ts
621
+ var saveStatements = new WeakMap;
622
+ var loadProjectStatements = new WeakMap;
623
+ var loadUnembeddedStatements = new WeakMap;
624
+ var countEmbeddedStatements = new WeakMap;
625
+ function getSaveStatement(db) {
626
+ let stmt = saveStatements.get(db);
627
+ if (!stmt) {
628
+ stmt = db.prepare(`INSERT INTO git_commit_embeddings (sha, embedding, model_id, created_at)
629
+ VALUES (?, ?, ?, ?)
630
+ ON CONFLICT(sha, model_id) DO UPDATE SET
631
+ embedding = excluded.embedding,
632
+ created_at = excluded.created_at`);
633
+ saveStatements.set(db, stmt);
634
+ }
635
+ return stmt;
636
+ }
637
+ function getLoadProjectStatement(db) {
638
+ let stmt = loadProjectStatements.get(db);
639
+ if (!stmt) {
640
+ stmt = db.prepare(`SELECT e.sha AS sha, e.embedding AS embedding, e.model_id AS model_id
641
+ FROM git_commit_embeddings e
642
+ JOIN git_commits c ON c.sha = e.sha
643
+ WHERE c.project_path = ? AND e.model_id = ?`);
644
+ loadProjectStatements.set(db, stmt);
645
+ }
646
+ return stmt;
647
+ }
648
+ function getLoadUnembeddedStatement(db) {
649
+ let stmt = loadUnembeddedStatements.get(db);
650
+ if (!stmt) {
651
+ stmt = db.prepare(`SELECT c.sha AS sha, c.message AS message
652
+ FROM git_commits c
653
+ LEFT JOIN git_commit_embeddings e ON c.sha = e.sha AND e.model_id = ?
654
+ WHERE c.project_path = ? AND e.sha IS NULL
655
+ ORDER BY c.committed_at DESC
656
+ LIMIT ?`);
657
+ loadUnembeddedStatements.set(db, stmt);
658
+ }
659
+ return stmt;
660
+ }
661
+ function getCountEmbeddedStatement(db) {
662
+ let stmt = countEmbeddedStatements.get(db);
663
+ if (!stmt) {
664
+ stmt = db.prepare(`SELECT COUNT(*) AS count FROM git_commit_embeddings e
665
+ JOIN git_commits c ON c.sha = e.sha WHERE c.project_path = ? AND e.model_id = ?`);
666
+ countEmbeddedStatements.set(db, stmt);
667
+ }
668
+ return stmt;
669
+ }
670
+ function saveCommitEmbedding(db, sha, embedding, modelId) {
671
+ const bytes = new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength);
672
+ getSaveStatement(db).run(sha, bytes, modelId, Date.now());
673
+ }
674
+ function loadProjectCommitEmbeddings(db, projectPath, modelId) {
675
+ const rows = getLoadProjectStatement(db).all(projectPath, modelId);
676
+ const map = new Map;
677
+ for (const row of rows) {
678
+ const buffer = row.embedding.buffer.slice(row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength);
679
+ map.set(row.sha, new Float32Array(buffer));
680
+ }
681
+ return map;
682
+ }
683
+ function loadUnembeddedCommits(db, projectPath, modelId, limit) {
684
+ return getLoadUnembeddedStatement(db).all(modelId, projectPath, limit);
685
+ }
686
+ function countEmbeddedCommits(db, projectPath, modelId) {
687
+ const row = getCountEmbeddedStatement(db).get(projectPath, modelId);
688
+ return row?.count ?? 0;
689
+ }
690
+
691
+ // ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
692
+ var insertStatements = new WeakMap;
693
+ var existingShasStatements = new WeakMap;
694
+ var projectCountStatements = new WeakMap;
695
+ var evictOverflowStatements = new WeakMap;
696
+ var latestCommitTimeStatements = new WeakMap;
697
+ var latestCommitShaStatements = new WeakMap;
698
+ function getInsertStatement(db) {
699
+ let stmt = insertStatements.get(db);
700
+ if (!stmt) {
701
+ stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
702
+ VALUES (?, ?, ?, ?, ?, ?, ?)
703
+ ON CONFLICT(sha) DO UPDATE SET
704
+ project_path = excluded.project_path,
705
+ short_sha = excluded.short_sha,
706
+ message = excluded.message,
707
+ author = excluded.author,
708
+ committed_at = excluded.committed_at,
709
+ indexed_at = excluded.indexed_at
710
+ WHERE git_commits.message != excluded.message`);
711
+ insertStatements.set(db, stmt);
712
+ }
713
+ return stmt;
714
+ }
715
+ function getExistingShasStatement(db) {
716
+ let stmt = existingShasStatements.get(db);
717
+ if (!stmt) {
718
+ stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
719
+ existingShasStatements.set(db, stmt);
720
+ }
721
+ return stmt;
722
+ }
723
+ function getProjectCountStatement(db) {
724
+ let stmt = projectCountStatements.get(db);
725
+ if (!stmt) {
726
+ stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
727
+ projectCountStatements.set(db, stmt);
728
+ }
729
+ return stmt;
730
+ }
731
+ function getLatestCommitShaStatement(db) {
732
+ let stmt = latestCommitShaStatements.get(db);
733
+ if (!stmt) {
734
+ stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ? ORDER BY committed_at DESC, sha DESC LIMIT 1");
735
+ latestCommitShaStatements.set(db, stmt);
736
+ }
737
+ return stmt;
738
+ }
739
+ function getEvictOverflowStatement(db) {
740
+ let stmt = evictOverflowStatements.get(db);
741
+ if (!stmt) {
742
+ stmt = db.prepare(`DELETE FROM git_commits
743
+ WHERE rowid IN (
744
+ SELECT rowid FROM git_commits
745
+ WHERE project_path = ?
746
+ ORDER BY committed_at DESC, sha DESC
747
+ LIMIT -1 OFFSET ?
748
+ )`);
749
+ evictOverflowStatements.set(db, stmt);
750
+ }
751
+ return stmt;
752
+ }
753
+ function upsertCommits(db, projectPath, commits) {
754
+ if (commits.length === 0)
755
+ return { inserted: 0, updated: 0 };
756
+ const existing = new Set;
757
+ for (const row of getExistingShasStatement(db).all(projectPath)) {
758
+ existing.add(row.sha);
759
+ }
760
+ let inserted = 0;
761
+ let updated = 0;
762
+ const now = Date.now();
763
+ const insertStmt = getInsertStatement(db);
764
+ db.transaction(() => {
765
+ for (const commit of commits) {
766
+ const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
767
+ if (result.changes > 0) {
768
+ if (existing.has(commit.sha)) {
769
+ updated++;
770
+ } else {
771
+ inserted++;
772
+ existing.add(commit.sha);
773
+ }
774
+ }
775
+ }
776
+ }).immediate();
777
+ return { inserted, updated };
778
+ }
779
+ function getCommitCount(db, projectPath) {
780
+ const row = getProjectCountStatement(db).get(projectPath);
781
+ return row?.count ?? 0;
782
+ }
783
+ function getLatestIndexedCommitSha(db, projectPath) {
784
+ const row = getLatestCommitShaStatement(db).get(projectPath);
785
+ return row?.sha ?? null;
786
+ }
787
+ function enforceProjectCap(db, projectPath, maxCommits) {
788
+ if (maxCommits <= 0)
789
+ return 0;
790
+ const count = getCommitCount(db, projectPath);
791
+ if (count <= maxCommits)
792
+ return 0;
793
+ getEvictOverflowStatement(db).run(projectPath, maxCommits);
794
+ const after = getCommitCount(db, projectPath);
795
+ const evicted = Math.max(0, count - after);
796
+ if (evicted > 0) {
797
+ log2(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
798
+ }
799
+ return evicted;
800
+ }
801
+
802
+ // ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
803
+ var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
804
+ var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
805
+ var GIT_SWEEP_NON_INDEXABLE_REPROBE_MS = 24 * 60 * 60 * 1000;
806
+ var GIT_SWEEP_LEASE_RENEWAL_MS = 60 * 1000;
807
+ function runImmediate(db, body) {
808
+ db.exec("BEGIN IMMEDIATE");
809
+ const transactionStartedAt = performance.now();
810
+ let committed = false;
811
+ try {
812
+ const result = body();
813
+ db.exec("COMMIT");
814
+ committed = true;
815
+ logSlowWriteTransaction("git_sweep_lease", transactionStartedAt);
816
+ return result;
817
+ } finally {
818
+ if (!committed) {
819
+ try {
820
+ db.exec("ROLLBACK");
821
+ } catch {}
822
+ }
823
+ }
824
+ }
825
+ function rowToState(row) {
826
+ return {
827
+ projectPath: row.project_path,
828
+ leaseHolder: row.lease_holder,
829
+ leaseExpiresAt: row.lease_expires_at,
830
+ lastSweptAt: row.last_swept_at
831
+ };
832
+ }
833
+ function getGitSweepCoordinatorState(db, projectPath) {
834
+ const row = db.prepare(`SELECT project_path, lease_holder, lease_expires_at, last_swept_at
835
+ FROM git_sweep_coordinator
836
+ WHERE project_path = ?`).get(projectPath);
837
+ return row ? rowToState(row) : null;
838
+ }
839
+ function acquireGitSweepLease(db, projectPath, holderId, options = {}) {
840
+ const cooldownMs = options.cooldownMs ?? GIT_SWEEP_COOLDOWN_MS;
841
+ const leaseTtlMs = options.leaseTtlMs ?? GIT_SWEEP_LEASE_TTL_MS;
842
+ return runImmediate(db, () => {
843
+ const now = Date.now();
844
+ const row = getGitSweepCoordinatorState(db, projectPath);
845
+ if (row?.leaseHolder && row.leaseExpiresAt !== null && row.leaseExpiresAt > now) {
846
+ return {
847
+ acquired: false,
848
+ projectPath,
849
+ reason: "lease_active",
850
+ leaseHolder: row.leaseHolder,
851
+ leaseExpiresAt: row.leaseExpiresAt,
852
+ lastSweptAt: row.lastSweptAt,
853
+ nextAllowedAt: null
854
+ };
855
+ }
856
+ if (!options.ignoreCooldown && row?.lastSweptAt !== null && row?.lastSweptAt !== undefined) {
857
+ const nextAllowedAt = row.lastSweptAt + cooldownMs;
858
+ if (nextAllowedAt > now) {
859
+ return {
860
+ acquired: false,
861
+ projectPath,
862
+ reason: "cooldown_active",
863
+ leaseHolder: row.leaseHolder,
864
+ leaseExpiresAt: row.leaseExpiresAt,
865
+ lastSweptAt: row.lastSweptAt,
866
+ nextAllowedAt
867
+ };
868
+ }
869
+ }
870
+ const leaseExpiresAt = now + leaseTtlMs;
871
+ db.prepare(`INSERT INTO git_sweep_coordinator (
872
+ project_path,
873
+ lease_holder,
874
+ lease_expires_at,
875
+ last_swept_at
876
+ ) VALUES (?, ?, ?, NULL)
877
+ ON CONFLICT(project_path) DO UPDATE SET
878
+ lease_holder = excluded.lease_holder,
879
+ lease_expires_at = excluded.lease_expires_at`).run(projectPath, holderId, leaseExpiresAt);
880
+ return {
881
+ acquired: true,
882
+ projectPath,
883
+ holderId,
884
+ acquiredAt: now,
885
+ leaseExpiresAt
886
+ };
887
+ });
888
+ }
889
+ function renewGitSweepLease(db, projectPath, holderId, leaseTtlMs = GIT_SWEEP_LEASE_TTL_MS) {
890
+ return runImmediate(db, () => {
891
+ const now = Date.now();
892
+ const leaseExpiresAt = now + leaseTtlMs;
893
+ const result = db.prepare(`UPDATE git_sweep_coordinator
894
+ SET lease_expires_at = ?
895
+ WHERE project_path = ?
896
+ AND lease_holder = ?
897
+ AND lease_expires_at > ?`).run(leaseExpiresAt, projectPath, holderId, now);
898
+ return result.changes === 1;
899
+ });
900
+ }
901
+ function markGitSweepSuccessAndRelease(db, projectPath, holderId) {
902
+ return runImmediate(db, () => {
903
+ const now = Date.now();
904
+ const result = db.prepare(`UPDATE git_sweep_coordinator
905
+ SET lease_holder = NULL,
906
+ lease_expires_at = NULL,
907
+ last_swept_at = ?
908
+ WHERE project_path = ?
909
+ AND lease_holder = ?
910
+ AND lease_expires_at > ?`).run(now, projectPath, holderId, now);
911
+ return result.changes === 1;
912
+ });
913
+ }
914
+ function parkGitSweepNonIndexable(db, projectPath, holderId, reprobeMs = GIT_SWEEP_NON_INDEXABLE_REPROBE_MS) {
915
+ return runImmediate(db, () => {
916
+ const now = Date.now();
917
+ const sweptAt = now + reprobeMs - GIT_SWEEP_COOLDOWN_MS;
918
+ const result = db.prepare(`UPDATE git_sweep_coordinator
919
+ SET lease_holder = NULL,
920
+ lease_expires_at = NULL,
921
+ last_swept_at = ?
922
+ WHERE project_path = ?
923
+ AND lease_holder = ?
924
+ AND lease_expires_at > ?`).run(sweptAt, projectPath, holderId, now);
925
+ return result.changes === 1;
926
+ });
927
+ }
928
+ function releaseGitSweepLease(db, projectPath, holderId) {
929
+ runImmediate(db, () => {
930
+ db.prepare(`UPDATE git_sweep_coordinator
931
+ SET lease_holder = NULL,
932
+ lease_expires_at = NULL
933
+ WHERE project_path = ?
934
+ AND lease_holder = ?`).run(projectPath, holderId);
935
+ });
936
+ }
937
+
938
+ // ../plugin/src/features/magic-context/shadow-backfill-state.ts
939
+ var SHADOW_BACKFILL_PROVENANCE_KEY = "_magicContextShadowBackfill";
940
+ function parseJsonRecord(value) {
941
+ try {
942
+ const parsed = JSON.parse(value);
943
+ return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : {};
944
+ } catch {
945
+ return {};
946
+ }
947
+ }
948
+ function parsePersistedShadowBackfillState(provenanceJson) {
949
+ const raw = parseJsonRecord(provenanceJson)[SHADOW_BACKFILL_PROVENANCE_KEY];
950
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw))
951
+ return;
952
+ const state = raw;
953
+ if (state.version !== 1)
954
+ return;
955
+ return state;
956
+ }
957
+ function hasShadowEmbeddingRegistrationsTable(db) {
958
+ return Boolean(db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get());
959
+ }
960
+ function describeShadowBackfillWriteRefusal2(reason) {
961
+ switch (reason) {
962
+ case "provider_returned_no_vectors":
963
+ return "the provider returned no vectors";
964
+ case "memory_hash_guard_rejected":
965
+ return "the memory normalized-hash guard rejected vectors because content changed in flight";
966
+ case "candidate_rows_changed":
967
+ return "the selected source rows changed before the writer loaded them";
968
+ case "registration_retired_during_embed":
969
+ return "the shadow registration was retired or replaced while the provider call was in flight, so the vectors were discarded";
970
+ case "chunk_fts_mapping_incomplete":
971
+ return "the chunk writer refused rows whose transcript ordinals are not fully mapped in FTS";
972
+ case "chunk_empty_canonical_text":
973
+ return "the chunk writer produced no canonical windows";
974
+ case "chunk_partial_vector_set":
975
+ return "the chunk writer refused a partial provider result so it would not replace a compartment incompletely";
976
+ case "chunk_window_contract_mismatch":
977
+ return "written chunk window keys or hashes did not satisfy the selector's window contract";
978
+ case "duplicate_submission_budget":
979
+ return "the same content batch was already submitted within the one-hour provider budget";
980
+ default:
981
+ return "the write completed without satisfying the selected candidate";
982
+ }
983
+ }
984
+ function listShadowBackfillStalls2(db, projectIdentity) {
985
+ if (!hasShadowEmbeddingRegistrationsTable(db))
986
+ return [];
987
+ const rows = db.prepare(`SELECT project_path AS projectIdentity, scope, model_id AS modelId,
988
+ provenance_json AS provenanceJson
989
+ FROM shadow_embedding_registrations
990
+ ${projectIdentity ? "WHERE project_path = ?" : ""}
991
+ ORDER BY updated_at DESC, generation DESC`).all(...projectIdentity ? [projectIdentity] : []);
992
+ const latestScopes = new Set;
993
+ const stalls = [];
994
+ for (const row of rows) {
995
+ const scopeKey = `${row.projectIdentity}:${row.scope}`;
996
+ if (latestScopes.has(scopeKey))
997
+ continue;
998
+ latestScopes.add(scopeKey);
999
+ const state = parsePersistedShadowBackfillState(row.provenanceJson);
1000
+ if (state?.stopReason !== "stalled_no_progress")
1001
+ continue;
1002
+ stalls.push({
1003
+ projectIdentity: row.projectIdentity,
1004
+ scope: row.scope,
1005
+ modelId: row.modelId,
1006
+ candidateSignature: state.candidateSignature ?? "unknown",
1007
+ writeRefusalReason: state.writeRefusalReason ?? "unknown_write_rejection",
1008
+ stoppedAt: state.stoppedAt ?? 0
1009
+ });
1010
+ }
1011
+ return stalls;
1012
+ }
1013
+
1014
+ // ../plugin/src/features/magic-context/project-embedding-registry.ts
1015
+ var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
1016
+ var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
1017
+ var SWEEP_MAX_CONSECUTIVE_EMPTY = 3;
1018
+ var COMMIT_DRAIN_BATCH_SIZE = 16;
1019
+ var COMMIT_DRAIN_MAX_PER_SWEEP = 500;
1020
+ var CHUNK_DRAIN_BATCH_SIZE = 8;
1021
+ var CHUNK_DRAIN_MAX_PER_SWEEP = CHUNK_DRAIN_BATCH_SIZE;
1022
+ var EMBEDDING_IDENTITY_GC_GRACE_MS = 14 * 24 * 60 * 60 * 1000;
1023
+ var STALE_EMBEDDING_GC_BATCH_SIZE = 25;
1024
+ var MAX_WINDOWS_PER_EMBED_CALL = 2;
1025
+ var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
1026
+ var EMBED_SLICE_RETRY_ATTEMPTS = 3;
1027
+ var EMBED_SLICE_RETRY_BASE_MS = 250;
1028
+ var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
1029
+ var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
1030
+ var projectRegistrations = new Map;
1031
+ var shadowRegistrations = new Map;
1032
+ var shadowQueue = [];
1033
+ var shadowWorker = null;
1034
+ var SHADOW_MAX_ITEMS_PER_TICK = 64;
1035
+ var SHADOW_MAX_BYTES_PER_TICK = 512 * 1024;
1036
+ var SHADOW_MAX_WALL_CLOCK_MS = 2000;
1037
+ var SHADOW_RESUBMIT_WINDOW_MS = 60 * 60 * 1000;
1038
+ var SHADOW_BACKFILL_PROVENANCE_KEY2 = "_magicContextShadowBackfill";
1039
+ var pendingShadowBackfills = new Map;
1040
+ var shadowBackfillLastIds = new Map;
1041
+ var shadowBackfillStopReasons = new Map;
1042
+ var shadowBackfillLastWriteOutcomes = new Map;
1043
+ var shadowBackfillNow = () => Date.now();
1044
+ function getShadowBackfillStopReason2(projectIdentity, scope) {
1045
+ return shadowBackfillStopReasons.get(`${projectIdentity}:${scope}`);
1046
+ }
1047
+ var loadUnembeddedMemoriesStatements = new WeakMap;
1048
+ var upsertActiveIdentityStatements = new WeakMap;
1049
+ var backfillActiveIdentityStatements = new Map;
1050
+ var staleIdentityStatements = new Map;
1051
+ var deleteActiveIdentityStatements = new WeakMap;
1052
+ var globalRegistrationGeneration = 0;
1053
+ var untrustedLoadProjects = new Set;
1054
+ function markProjectLoadUntrusted2(projectIdentity) {
1055
+ untrustedLoadProjects.add(projectIdentity);
1056
+ }
1057
+ var projectSweepInProgress = false;
1058
+ var testProviderFactory = null;
1059
+
1060
+ class TestProviderFactoryRequiredError extends Error {
1061
+ constructor() {
1062
+ super("test constructed a network-capable embedding provider without a test factory — install _setTestProviderFactoryForProject or set embedding.provider off in the fixture");
1063
+ this.name = "TestProviderFactoryRequiredError";
1064
+ }
1065
+ }
1066
+ function synapseDescriptorFromConfig(config) {
1067
+ const raw = config;
1068
+ const candidate = raw.synapse_descriptor;
1069
+ if (!candidate || typeof candidate !== "object" || Array.isArray(candidate))
1070
+ return;
1071
+ const descriptor = candidate;
1072
+ if (typeof descriptor.lane !== "string" || descriptor.lane.length === 0 || typeof descriptor.max_tokens !== "number" || !Number.isInteger(descriptor.max_tokens) || descriptor.max_tokens <= 0 || descriptor.max_tokens_source !== "runtime_bucket" && descriptor.max_tokens_source !== "worker_bucket" && descriptor.max_tokens_source !== "catalog" && descriptor.max_tokens_source !== "catalog_unloaded" || typeof descriptor.warm !== "boolean") {
1073
+ return;
1074
+ }
1075
+ return descriptor;
1076
+ }
1077
+ function synapseConfigFields(config) {
1078
+ const raw = config;
1079
+ const descriptor = synapseDescriptorFromConfig(config);
1080
+ const rawProvenance = raw.synapse_provenance;
1081
+ const provenance = descriptor ? {
1082
+ ...rawProvenance && typeof rawProvenance === "object" && !Array.isArray(rawProvenance) ? rawProvenance : rawProvenance === undefined ? {} : { synapse_provenance: rawProvenance },
1083
+ capability_descriptor: descriptor
1084
+ } : rawProvenance;
1085
+ return {
1086
+ ...typeof raw.model === "string" ? { model: raw.model } : {},
1087
+ ...typeof raw.synapse_fingerprint === "string" ? { fingerprint: raw.synapse_fingerprint } : {},
1088
+ ...typeof raw.synapse_table_epoch === "number" ? { tableEpoch: raw.synapse_table_epoch } : {},
1089
+ ...typeof raw.synapse_dims === "number" ? { dims: raw.synapse_dims } : {},
1090
+ ...provenance !== undefined ? { provenance } : {},
1091
+ ...descriptor ? { descriptor } : {}
1092
+ };
1093
+ }
1094
+ function parseJsonRecord2(value) {
1095
+ try {
1096
+ const parsed = JSON.parse(value);
1097
+ return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : { synapse_provenance: parsed };
1098
+ } catch {
1099
+ return {};
1100
+ }
1101
+ }
1102
+ function hasShadowEmbeddingRegistrationsTable2(db) {
1103
+ return Boolean(db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get());
1104
+ }
1105
+ function getPersistedShadowBackfillState(db, projectIdentity, scope, modelId) {
1106
+ if (!hasShadowEmbeddingRegistrationsTable2(db))
1107
+ return;
1108
+ const row = db.prepare(`SELECT provenance_json AS provenanceJson
1109
+ FROM shadow_embedding_registrations
1110
+ WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
1111
+ return typeof row?.provenanceJson === "string" ? parsePersistedShadowBackfillState(row.provenanceJson) : undefined;
1112
+ }
1113
+ function updatePersistedShadowBackfillState(db, projectIdentity, scope, modelId, update) {
1114
+ if (!hasShadowEmbeddingRegistrationsTable2(db))
1115
+ return;
1116
+ const row = db.prepare(`SELECT provenance_json AS provenanceJson
1117
+ FROM shadow_embedding_registrations
1118
+ WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
1119
+ if (typeof row?.provenanceJson !== "string")
1120
+ return;
1121
+ const provenance = parseJsonRecord2(row.provenanceJson);
1122
+ const current = parsePersistedShadowBackfillState(row.provenanceJson) ?? { version: 1 };
1123
+ provenance[SHADOW_BACKFILL_PROVENANCE_KEY2] = update(current);
1124
+ db.prepare(`UPDATE shadow_embedding_registrations
1125
+ SET provenance_json = ?
1126
+ WHERE project_path = ? AND scope = ? AND model_id = ?`).run(JSON.stringify(provenance), projectIdentity, scope, modelId);
1127
+ }
1128
+ function shadowDescriptorProvenanceJson(db, projectIdentity, scope, modelId, synapseProvenance) {
1129
+ const provenance = typeof synapseProvenance === "object" && synapseProvenance !== null && !Array.isArray(synapseProvenance) ? { ...synapseProvenance } : synapseProvenance === undefined ? {} : { synapse_provenance: synapseProvenance };
1130
+ const existing = getPersistedShadowBackfillState(db, projectIdentity, scope, modelId);
1131
+ if (existing)
1132
+ provenance[SHADOW_BACKFILL_PROVENANCE_KEY2] = existing;
1133
+ return JSON.stringify(provenance);
1134
+ }
1135
+ function persistPrimaryDescriptor(db, registration) {
1136
+ const descriptorTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'embedding_registrations'").get();
1137
+ if (!descriptorTable)
1138
+ return;
1139
+ const fields = synapseConfigFields(registration.config);
1140
+ db.prepare(`INSERT INTO embedding_registrations
1141
+ (project_path, provider_identity, model_id, chunk_model_id, fingerprint, table_epoch, dims, provenance_json, generation, updated_at)
1142
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1143
+ ON CONFLICT(project_path) DO UPDATE SET
1144
+ provider_identity = excluded.provider_identity,
1145
+ model_id = excluded.model_id,
1146
+ chunk_model_id = excluded.chunk_model_id,
1147
+ fingerprint = excluded.fingerprint,
1148
+ table_epoch = excluded.table_epoch,
1149
+ dims = excluded.dims,
1150
+ provenance_json = excluded.provenance_json,
1151
+ generation = excluded.generation,
1152
+ updated_at = excluded.updated_at`).run(registration.projectIdentity, registration.providerIdentity, registration.modelId, registration.chunkModelId, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, JSON.stringify(fields.provenance ?? {}), registration.generation, Date.now());
1153
+ }
1154
+ function persistShadowDescriptor(db, registration) {
1155
+ const descriptorTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get();
1156
+ if (!descriptorTable)
1157
+ return;
1158
+ const fields = synapseConfigFields(registration.config);
1159
+ const now = Date.now();
1160
+ db.prepare(`INSERT INTO shadow_embedding_registrations
1161
+ (project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
1162
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1163
+ ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
1164
+ generation = excluded.generation,
1165
+ fingerprint = excluded.fingerprint,
1166
+ table_epoch = excluded.table_epoch,
1167
+ dims = excluded.dims,
1168
+ provenance_json = excluded.provenance_json,
1169
+ updated_at = excluded.updated_at`).run(registration.projectIdentity, "memory", registration.modelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "memory", registration.modelId, fields.provenance), now);
1170
+ db.prepare(`INSERT INTO shadow_embedding_registrations
1171
+ (project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
1172
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1173
+ ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
1174
+ generation = excluded.generation,
1175
+ fingerprint = excluded.fingerprint,
1176
+ table_epoch = excluded.table_epoch,
1177
+ dims = excluded.dims,
1178
+ provenance_json = excluded.provenance_json,
1179
+ updated_at = excluded.updated_at`).run(registration.projectIdentity, "commit", registration.modelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "commit", registration.modelId, fields.provenance), now);
1180
+ db.prepare(`INSERT INTO shadow_embedding_registrations
1181
+ (project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
1182
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1183
+ ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
1184
+ generation = excluded.generation,
1185
+ fingerprint = excluded.fingerprint,
1186
+ table_epoch = excluded.table_epoch,
1187
+ dims = excluded.dims,
1188
+ provenance_json = excluded.provenance_json,
1189
+ updated_at = excluded.updated_at`).run(registration.projectIdentity, "chunk", registration.chunkModelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "chunk", registration.chunkModelId, fields.provenance), now);
1190
+ }
1191
+ function resolveEmbeddingConfig(config) {
1192
+ if (!config || config.provider === "local") {
1193
+ return {
1194
+ provider: "local",
1195
+ model: config?.model?.trim() || DEFAULT_LOCAL_EMBEDDING_MODEL,
1196
+ local_runtime: config?.local_runtime ?? "auto",
1197
+ ...config?.max_input_tokens ? {
1198
+ max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config.max_input_tokens)
1199
+ } : {},
1200
+ ...config?.local_dtype ? { local_dtype: config.local_dtype } : {}
1201
+ };
1202
+ }
1203
+ if (config.provider === "openai-compatible") {
1204
+ const apiKey = config.api_key?.trim();
1205
+ const inputType = config.input_type?.trim();
1206
+ const queryInputType = config.query_input_type?.trim();
1207
+ const truncate = config.truncate?.trim();
1208
+ return {
1209
+ provider: "openai-compatible",
1210
+ model: config.model.trim(),
1211
+ endpoint: config.endpoint.trim(),
1212
+ ...apiKey ? { api_key: apiKey } : {},
1213
+ ...inputType ? { input_type: inputType } : {},
1214
+ ...queryInputType ? { query_input_type: queryInputType } : {},
1215
+ ...config.query_instruction !== undefined ? { query_instruction: config.query_instruction } : {},
1216
+ ...config.document_prefix !== undefined ? { document_prefix: config.document_prefix } : {},
1217
+ ...truncate ? { truncate } : {},
1218
+ ...config.max_input_tokens ? {
1219
+ max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config.max_input_tokens)
1220
+ } : {}
1221
+ };
1222
+ }
1223
+ if (config.provider === "off") {
1224
+ return { provider: "off" };
1225
+ }
1226
+ if (config.provider === "synapse") {
1227
+ const synapse = config;
1228
+ const descriptor = synapseDescriptorFromConfig(config);
1229
+ return {
1230
+ provider: "synapse",
1231
+ model: synapse.model?.trim() || "gte-modernbert-base-f16",
1232
+ max_input_tokens: normalizeCompartmentChunkMaxInputTokens(descriptor?.max_tokens ?? synapse.max_input_tokens),
1233
+ ...synapse.synapse_connection_file ? { synapse_connection_file: synapse.synapse_connection_file } : {},
1234
+ ...synapse.synapse_fingerprint ? { synapse_fingerprint: synapse.synapse_fingerprint } : {},
1235
+ ...typeof synapse.synapse_table_epoch === "number" ? { synapse_table_epoch: synapse.synapse_table_epoch } : {},
1236
+ ...typeof synapse.synapse_dims === "number" ? { synapse_dims: synapse.synapse_dims } : {},
1237
+ ...typeof synapse.synapse_recommended_batch === "number" ? { synapse_recommended_batch: synapse.synapse_recommended_batch } : {},
1238
+ ...typeof synapse.synapse_recommended_token_budget === "number" ? {
1239
+ synapse_recommended_token_budget: synapse.synapse_recommended_token_budget
1240
+ } : {},
1241
+ ...descriptor ? { synapse_descriptor: descriptor } : {},
1242
+ ...synapse.synapse_provenance !== undefined ? { synapse_provenance: synapse.synapse_provenance } : {}
1243
+ };
1244
+ }
1245
+ throw new Error("Unknown embedding provider");
1246
+ }
1247
+ function createProvider(config, context) {
1248
+ if (config.provider === "off") {
1249
+ return null;
1250
+ }
1251
+ if (testProviderFactory) {
1252
+ return testProviderFactory(config);
1253
+ }
1254
+ if (process.env.MAGIC_CONTEXT_TEST_DATA_DIR?.trim()) {
1255
+ throw new TestProviderFactoryRequiredError;
1256
+ }
1257
+ if (config.provider === "openai-compatible") {
1258
+ return new OpenAICompatibleEmbeddingProvider({
1259
+ endpoint: config.endpoint,
1260
+ model: config.model,
1261
+ apiKey: config.api_key,
1262
+ inputType: config.input_type,
1263
+ queryInputType: config.query_input_type,
1264
+ queryInstruction: config.query_instruction,
1265
+ documentPrefix: config.document_prefix,
1266
+ truncate: config.truncate,
1267
+ maxInputTokens: config.max_input_tokens
1268
+ });
1269
+ }
1270
+ if (config.provider === "local") {
1271
+ return new LocalEmbeddingProvider(config.model, config.max_input_tokens, config.local_dtype, config.local_runtime);
1272
+ }
1273
+ if (config.provider === "synapse") {
1274
+ const synapse = config;
1275
+ return new SynapseEmbeddingProvider({
1276
+ connectionFile: synapse.synapse_connection_file ?? "",
1277
+ projectRoot: context?.projectRoot ?? "",
1278
+ session: context?.session ?? "embedding",
1279
+ model: synapse.model,
1280
+ fingerprint: synapse.synapse_fingerprint,
1281
+ tableEpoch: synapse.synapse_table_epoch,
1282
+ dims: synapse.synapse_dims,
1283
+ recommendedBatch: synapse.synapse_recommended_batch,
1284
+ recommendedTokenBudget: synapse.synapse_recommended_token_budget,
1285
+ descriptor: synapse.synapse_descriptor,
1286
+ provenance: synapse.synapse_provenance
1287
+ });
1288
+ }
1289
+ throw new Error("Unknown embedding provider");
1290
+ }
1291
+ function stableStringify(value) {
1292
+ if (Array.isArray(value)) {
1293
+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
1294
+ }
1295
+ if (value && typeof value === "object") {
1296
+ const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
1297
+ return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
1298
+ }
1299
+ return JSON.stringify(value);
1300
+ }
1301
+ function sha256Prefix(value, length = 16) {
1302
+ return createHash2("sha256").update(value).digest("hex").slice(0, length);
1303
+ }
1304
+ function contentSha2562(value) {
1305
+ return createHash2("sha256").update(value).digest("hex");
1306
+ }
1307
+ function getRuntimeFingerprint(config) {
1308
+ if (config.provider === "off") {
1309
+ return OFF_PROVIDER_IDENTITY;
1310
+ }
1311
+ return `${getEmbeddingProviderIdentity(config)}:${sha256Prefix(stableStringify(config))}`;
1312
+ }
1313
+ function getChunkEmbeddingModelId(config, providerIdentity) {
1314
+ if (config.provider === "off") {
1315
+ return OFF_PROVIDER_IDENTITY;
1316
+ }
1317
+ const chunkIdentity = {
1318
+ providerIdentity,
1319
+ chunkerVersion: 2,
1320
+ maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config ? config.max_input_tokens : undefined),
1321
+ truncate: config.provider === "openai-compatible" ? config.truncate ?? "" : ""
1322
+ };
1323
+ return `${providerIdentity}:chunk:${sha256Prefix(stableStringify(chunkIdentity))}`;
1324
+ }
1325
+ function sameFeatures(a, b) {
1326
+ return a.memoryEnabled === b.memoryEnabled && a.gitCommitEnabled === b.gitCommitEnabled;
1327
+ }
1328
+ function snapshotFor(registration) {
1329
+ const providerIsOn = registration.providerIdentity !== OFF_PROVIDER_IDENTITY;
1330
+ const historyEnabled = !registration.observationMode && providerIsOn;
1331
+ const enabled = historyEnabled && registration.features.memoryEnabled;
1332
+ const gitCommitEnabled = !registration.observationMode && providerIsOn && registration.features.gitCommitEnabled;
1333
+ const configuredModel = "model" in registration.config && typeof registration.config.model === "string" ? registration.config.model.trim() : "";
1334
+ const synapseDescriptor = !registration.observationMode && registration.config.provider === "synapse" ? synapseDescriptorFromConfig(registration.config) : undefined;
1335
+ return {
1336
+ projectIdentity: registration.projectIdentity,
1337
+ sourceDirectory: registration.sourceDirectory,
1338
+ providerIdentity: registration.providerIdentity,
1339
+ runtimeFingerprint: registration.runtimeFingerprint,
1340
+ generation: registration.generation,
1341
+ features: { ...registration.features },
1342
+ enabled,
1343
+ historyEnabled,
1344
+ gitCommitEnabled,
1345
+ modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
1346
+ chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
1347
+ model: registration.observationMode || !providerIsOn ? "off" : configuredModel ? configuredModel : registration.modelId,
1348
+ provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local",
1349
+ ...synapseDescriptor ? { synapseDescriptor } : {}
1350
+ };
1351
+ }
1352
+ function disposeProvider(provider) {
1353
+ if (!provider)
1354
+ return;
1355
+ provider.dispose().catch((error) => {
1356
+ log2("[magic-context] embedding provider dispose failed:", error);
1357
+ });
1358
+ }
1359
+ function getUpsertActiveIdentityStatement(db) {
1360
+ let stmt = upsertActiveIdentityStatements.get(db);
1361
+ if (!stmt) {
1362
+ stmt = db.prepare(`INSERT INTO embedding_identity_active (project_path, scope, model_id, last_active_at)
1363
+ VALUES (?, ?, ?, ?)
1364
+ ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
1365
+ last_active_at = excluded.last_active_at`);
1366
+ upsertActiveIdentityStatements.set(db, stmt);
1367
+ }
1368
+ return stmt;
1369
+ }
1370
+ function statementMapFor(maps, key) {
1371
+ let map = maps.get(key);
1372
+ if (!map) {
1373
+ map = new WeakMap;
1374
+ maps.set(key, map);
1375
+ }
1376
+ return map;
1377
+ }
1378
+ function getBackfillActiveIdentityStatement(db, scope) {
1379
+ const map = statementMapFor(backfillActiveIdentityStatements, scope);
1380
+ let stmt = map.get(db);
1381
+ if (!stmt) {
1382
+ const selectByScope = {
1383
+ memory: `SELECT DISTINCT e.model_id AS model_id
1384
+ FROM memory_embeddings e
1385
+ JOIN memories m ON m.id = e.memory_id
1386
+ WHERE m.project_path = ?`,
1387
+ commit: `SELECT DISTINCT e.model_id AS model_id
1388
+ FROM git_commit_embeddings e
1389
+ JOIN git_commits c ON c.sha = e.sha
1390
+ WHERE c.project_path = ?`,
1391
+ chunk: `SELECT DISTINCT e.model_id AS model_id
1392
+ FROM compartment_chunk_embeddings e
1393
+ WHERE e.project_path = ?`
1394
+ };
1395
+ stmt = db.prepare(`SELECT model_id FROM (${selectByScope[scope]}) legacy
1396
+ WHERE model_id IS NOT NULL AND NOT EXISTS (
1397
+ SELECT 1 FROM embedding_identity_active active
1398
+ WHERE active.project_path = ? AND active.scope = ?
1399
+ AND active.model_id = legacy.model_id
1400
+ ) LIMIT 25`);
1401
+ map.set(db, stmt);
1402
+ }
1403
+ return stmt;
1404
+ }
1405
+ function recordScopeActiveIdentity(db, projectIdentity, scope, modelId, now) {
1406
+ getUpsertActiveIdentityStatement(db).run(projectIdentity, scope, modelId, now);
1407
+ }
1408
+ var legacyDiscoveryComplete = new WeakMap;
1409
+ function recordActiveEmbeddingIdentity(db, projectIdentity, currentProviderIdentity, currentChunkIdentity, features) {
1410
+ if (currentProviderIdentity === OFF_PROVIDER_IDENTITY) {
1411
+ return false;
1412
+ }
1413
+ const scopes = [["chunk", currentChunkIdentity]];
1414
+ if (features.memoryEnabled)
1415
+ scopes.push(["memory", currentProviderIdentity]);
1416
+ if (features.gitCommitEnabled)
1417
+ scopes.push(["commit", currentProviderIdentity]);
1418
+ const active = db.prepare("SELECT 1 FROM embedding_identity_active WHERE project_path = ? AND scope = ? AND model_id = ?");
1419
+ let discovered = legacyDiscoveryComplete.get(db);
1420
+ if (!discovered) {
1421
+ discovered = new Set;
1422
+ legacyDiscoveryComplete.set(db, discovered);
1423
+ }
1424
+ const discoveryKey = (scope, model) => JSON.stringify([projectIdentity, scope, model]);
1425
+ const legacy = scopes.filter(([scope, model]) => !discovered.has(discoveryKey(scope, model)) || !active.get(projectIdentity, scope, model)).map(([scope, model]) => ({
1426
+ scope,
1427
+ model,
1428
+ rows: getBackfillActiveIdentityStatement(db, scope).all(projectIdentity, projectIdentity, scope)
1429
+ }));
1430
+ const repairIds = findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectIdentity);
1431
+ if (legacy.every(({ rows }) => rows.length === 0) && scopes.every(([scope, model]) => active.get(projectIdentity, scope, model)) && repairIds.length === 0) {
1432
+ for (const { scope, model } of legacy)
1433
+ discovered.add(discoveryKey(scope, model));
1434
+ return false;
1435
+ }
1436
+ const now = Date.now();
1437
+ db.exec("BEGIN IMMEDIATE");
1438
+ const transactionStartedAt = performance.now();
1439
+ try {
1440
+ if (features.memoryEnabled) {
1441
+ recordScopeActiveIdentity(db, projectIdentity, "memory", currentProviderIdentity, now);
1442
+ }
1443
+ if (features.gitCommitEnabled) {
1444
+ recordScopeActiveIdentity(db, projectIdentity, "commit", currentProviderIdentity, now);
1445
+ }
1446
+ for (const { scope, rows } of legacy) {
1447
+ for (const { model_id } of rows) {
1448
+ db.prepare(`INSERT OR IGNORE INTO embedding_identity_active
1449
+ (project_path, scope, model_id, last_active_at) VALUES (?, ?, ?, ?)`).run(projectIdentity, scope, model_id, now);
1450
+ }
1451
+ }
1452
+ repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectIdentity, repairIds);
1453
+ recordScopeActiveIdentity(db, projectIdentity, "chunk", currentChunkIdentity, now);
1454
+ db.exec("COMMIT");
1455
+ logSlowWriteTransaction("embedding_identity_record", transactionStartedAt);
1456
+ for (const { scope, model, rows } of legacy) {
1457
+ if (rows.length < 25)
1458
+ discovered.add(discoveryKey(scope, model));
1459
+ }
1460
+ return true;
1461
+ } catch (error) {
1462
+ try {
1463
+ db.exec("ROLLBACK");
1464
+ } catch {}
1465
+ throw error;
1466
+ }
1467
+ }
1468
+ function getStaleIdentityStatement(db, scope) {
1469
+ const map = statementMapFor(staleIdentityStatements, scope);
1470
+ let stmt = map.get(db);
1471
+ if (!stmt) {
1472
+ stmt = db.prepare(`SELECT model_id AS modelId
1473
+ FROM embedding_identity_active
1474
+ WHERE project_path = ?
1475
+ AND scope = ?
1476
+ AND model_id <> ?
1477
+ AND last_active_at < ?`);
1478
+ map.set(db, stmt);
1479
+ }
1480
+ return stmt;
1481
+ }
1482
+ function getDeleteActiveIdentityStatement(db) {
1483
+ let stmt = deleteActiveIdentityStatements.get(db);
1484
+ if (!stmt) {
1485
+ stmt = db.prepare(`DELETE FROM embedding_identity_active
1486
+ WHERE project_path = ? AND scope = ? AND model_id = ?`);
1487
+ deleteActiveIdentityStatements.set(db, stmt);
1488
+ }
1489
+ return stmt;
1490
+ }
1491
+ function staleModelsForScope(db, projectIdentity, scope, currentModelId, cutoff, protectedModelIds = new Set([currentModelId])) {
1492
+ const rows = getStaleIdentityStatement(db, scope).all(projectIdentity, scope, currentModelId, cutoff);
1493
+ return rows.map((row) => row.modelId).filter((modelId) => typeof modelId === "string" && !protectedModelIds.has(modelId));
1494
+ }
1495
+ function deleteStaleEmbeddingBatch(db, scope, projectIdentity, modelId, limit) {
1496
+ if (scope === "memory") {
1497
+ return db.prepare(`DELETE FROM memory_embeddings
1498
+ WHERE rowid IN (
1499
+ SELECT me.rowid
1500
+ FROM memory_embeddings me
1501
+ WHERE me.memory_id IN (SELECT id FROM memories WHERE project_path = ?)
1502
+ AND me.model_id = ?
1503
+ LIMIT ?
1504
+ )`).run(projectIdentity, modelId, limit).changes;
1505
+ }
1506
+ if (scope === "commit") {
1507
+ return db.prepare(`DELETE FROM git_commit_embeddings
1508
+ WHERE rowid IN (
1509
+ SELECT gce.rowid
1510
+ FROM git_commit_embeddings gce
1511
+ WHERE gce.sha IN (SELECT sha FROM git_commits WHERE project_path = ?)
1512
+ AND gce.model_id = ?
1513
+ LIMIT ?
1514
+ )`).run(projectIdentity, modelId, limit).changes;
1515
+ }
1516
+ return db.prepare(`DELETE FROM compartment_chunk_embeddings
1517
+ WHERE id IN (
1518
+ SELECT id
1519
+ FROM compartment_chunk_embeddings
1520
+ WHERE project_path = ? AND model_id = ?
1521
+ LIMIT ?
1522
+ )`).run(projectIdentity, modelId, limit).changes;
1523
+ }
1524
+ function hasStaleEmbeddingRows(db, scope, projectIdentity, modelId) {
1525
+ if (scope === "memory") {
1526
+ return Boolean(db.prepare(`SELECT 1
1527
+ FROM memory_embeddings me
1528
+ WHERE me.memory_id IN (SELECT id FROM memories WHERE project_path = ?)
1529
+ AND me.model_id = ?
1530
+ LIMIT 1`).get(projectIdentity, modelId));
1531
+ }
1532
+ if (scope === "commit") {
1533
+ return Boolean(db.prepare(`SELECT 1
1534
+ FROM git_commit_embeddings gce
1535
+ WHERE gce.sha IN (SELECT sha FROM git_commits WHERE project_path = ?)
1536
+ AND gce.model_id = ?
1537
+ LIMIT 1`).get(projectIdentity, modelId));
1538
+ }
1539
+ return Boolean(db.prepare(`SELECT 1
1540
+ FROM compartment_chunk_embeddings
1541
+ WHERE project_path = ? AND model_id = ?
1542
+ LIMIT 1`).get(projectIdentity, modelId));
1543
+ }
1544
+ function sweepStaleEmbeddingIdentitiesForProject(db, projectIdentity, now = Date.now()) {
1545
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
1546
+ const result = {
1547
+ memoryRowsDeleted: 0,
1548
+ commitRowsDeleted: 0,
1549
+ chunkRowsDeleted: 0,
1550
+ trackingRowsDeleted: 0
1551
+ };
1552
+ if (!snapshot)
1553
+ return result;
1554
+ if (untrustedLoadProjects.has(projectIdentity))
1555
+ return result;
1556
+ const cutoff = now - EMBEDDING_IDENTITY_GC_GRACE_MS;
1557
+ const deleteTracking = getDeleteActiveIdentityStatement(db);
1558
+ const shadow = shadowRegistrations.get(projectIdentity);
1559
+ const protectedModels = {
1560
+ memory: new Set([snapshot.modelId, ...shadow ? [shadow.modelId] : []]),
1561
+ commit: new Set([snapshot.modelId, ...shadow ? [shadow.modelId] : []]),
1562
+ chunk: new Set([snapshot.chunkModelId, ...shadow ? [shadow.chunkModelId] : []])
1563
+ };
1564
+ const scopes = [
1565
+ {
1566
+ scope: "memory",
1567
+ enabled: snapshot.enabled && snapshot.modelId !== "off",
1568
+ currentModelId: snapshot.modelId
1569
+ },
1570
+ {
1571
+ scope: "commit",
1572
+ enabled: snapshot.gitCommitEnabled && snapshot.modelId !== "off",
1573
+ currentModelId: snapshot.modelId
1574
+ },
1575
+ {
1576
+ scope: "chunk",
1577
+ enabled: snapshot.historyEnabled && snapshot.chunkModelId !== "off",
1578
+ currentModelId: snapshot.chunkModelId
1579
+ }
1580
+ ];
1581
+ let remainingBudget = STALE_EMBEDDING_GC_BATCH_SIZE;
1582
+ const candidates = scopes.flatMap(({ scope, enabled, currentModelId }) => enabled ? staleModelsForScope(db, projectIdentity, scope, currentModelId, cutoff, protectedModels[scope]).slice(0, STALE_EMBEDDING_GC_BATCH_SIZE).map((modelId) => ({ scope, modelId })) : []).slice(0, STALE_EMBEDDING_GC_BATCH_SIZE);
1583
+ if (candidates.length === 0)
1584
+ return result;
1585
+ db.exec("BEGIN IMMEDIATE");
1586
+ const transactionStartedAt = performance.now();
1587
+ try {
1588
+ for (const { scope, modelId } of candidates) {
1589
+ if (remainingBudget === 0)
1590
+ break;
1591
+ const marker = db.prepare(`SELECT last_active_at FROM embedding_identity_active
1592
+ WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
1593
+ if (!marker || marker.last_active_at >= cutoff)
1594
+ continue;
1595
+ const deleted = deleteStaleEmbeddingBatch(db, scope, projectIdentity, modelId, remainingBudget);
1596
+ remainingBudget -= deleted;
1597
+ if (scope === "memory")
1598
+ result.memoryRowsDeleted += deleted;
1599
+ else if (scope === "commit")
1600
+ result.commitRowsDeleted += deleted;
1601
+ else
1602
+ result.chunkRowsDeleted += deleted;
1603
+ if (!hasStaleEmbeddingRows(db, scope, projectIdentity, modelId)) {
1604
+ result.trackingRowsDeleted += deleteTracking.run(projectIdentity, scope, modelId).changes;
1605
+ } else if (deleted === 0) {
1606
+ remainingBudget = 0;
1607
+ }
1608
+ }
1609
+ db.exec("COMMIT");
1610
+ logSlowWriteTransaction("embedding_stale_gc", transactionStartedAt);
1611
+ } catch (error) {
1612
+ try {
1613
+ db.exec("ROLLBACK");
1614
+ } catch {}
1615
+ throw error;
1616
+ }
1617
+ if (result.memoryRowsDeleted > 0)
1618
+ invalidateProject2(projectIdentity);
1619
+ return result;
1620
+ }
1621
+ async function drainProjectEmbeddingIdentityMaintenance2(db, projectIdentity, budgetMs = 2000) {
1622
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
1623
+ if (!snapshot || untrustedLoadProjects.has(projectIdentity))
1624
+ return 0;
1625
+ return drainBackgroundBatches(() => recordActiveEmbeddingIdentity(db, projectIdentity, snapshot.modelId, snapshot.chunkModelId, { memoryEnabled: snapshot.enabled, gitCommitEnabled: snapshot.gitCommitEnabled }), { budgetMs });
1626
+ }
1627
+ async function drainStaleEmbeddingIdentitiesForProject2(db, projectIdentity, budgetMs = 2000) {
1628
+ const total = {
1629
+ memoryRowsDeleted: 0,
1630
+ commitRowsDeleted: 0,
1631
+ chunkRowsDeleted: 0,
1632
+ trackingRowsDeleted: 0
1633
+ };
1634
+ await drainBackgroundBatches(() => {
1635
+ const result = sweepStaleEmbeddingIdentitiesForProject(db, projectIdentity);
1636
+ total.memoryRowsDeleted += result.memoryRowsDeleted;
1637
+ total.commitRowsDeleted += result.commitRowsDeleted;
1638
+ total.chunkRowsDeleted += result.chunkRowsDeleted;
1639
+ total.trackingRowsDeleted += result.trackingRowsDeleted;
1640
+ return result.memoryRowsDeleted + result.commitRowsDeleted + result.chunkRowsDeleted + result.trackingRowsDeleted > 0;
1641
+ }, { budgetMs });
1642
+ return total;
1643
+ }
1644
+ function registerProjectEmbedding2(db, projectIdentity, config, features, sourceDirectory) {
1645
+ const resolvedConfig = resolveEmbeddingConfig(config);
1646
+ const providerIdentity = getEmbeddingProviderIdentity(resolvedConfig);
1647
+ const runtimeFingerprint = getRuntimeFingerprint(resolvedConfig);
1648
+ const chunkModelId = getChunkEmbeddingModelId(resolvedConfig, providerIdentity);
1649
+ const prior = projectRegistrations.get(projectIdentity);
1650
+ const canReuseProvider = prior !== undefined && !prior.observationMode && prior.runtimeFingerprint === runtimeFingerprint && prior.providerIdentity === providerIdentity;
1651
+ recordActiveEmbeddingIdentity(db, projectIdentity, providerIdentity, chunkModelId, features);
1652
+ pruneSynapseBatchLedgerForProject(db, projectIdentity);
1653
+ untrustedLoadProjects.delete(projectIdentity);
1654
+ const generationChanged = prior === undefined || prior.observationMode || prior.runtimeFingerprint !== runtimeFingerprint || prior.chunkModelId !== chunkModelId || !sameFeatures(prior.features, features);
1655
+ const generation = generationChanged ? ++globalRegistrationGeneration : prior.generation;
1656
+ const registration = {
1657
+ projectIdentity,
1658
+ sourceDirectory,
1659
+ config: resolvedConfig,
1660
+ providerIdentity,
1661
+ runtimeFingerprint,
1662
+ provider: canReuseProvider ? prior.provider : null,
1663
+ generation,
1664
+ features: { ...features },
1665
+ modelId: providerIdentity === OFF_PROVIDER_IDENTITY ? "off" : providerIdentity,
1666
+ chunkModelId: providerIdentity === OFF_PROVIDER_IDENTITY ? "off" : chunkModelId,
1667
+ observationMode: false
1668
+ };
1669
+ projectRegistrations.set(projectIdentity, registration);
1670
+ if (generationChanged || !db.prepare("SELECT 1 FROM embedding_registrations WHERE project_path = ? AND provider_identity = ? AND chunk_model_id = ? AND generation = ?").get(projectIdentity, providerIdentity, registration.chunkModelId, generation)) {
1671
+ persistPrimaryDescriptor(db, registration);
1672
+ }
1673
+ if (!canReuseProvider) {
1674
+ disposeProvider(prior?.provider ?? null);
1675
+ }
1676
+ return snapshotFor(registration);
1677
+ }
1678
+ function registerProjectShadowEmbedding2(db, projectIdentity, config, sourceDirectory, options = {}) {
1679
+ const resolvedConfig = resolveEmbeddingConfig(config);
1680
+ if (resolvedConfig.provider !== "synapse") {
1681
+ throw new Error("Shadow embedding registration requires the synapse provider");
1682
+ }
1683
+ const providerIdentity = getEmbeddingProviderIdentity(resolvedConfig);
1684
+ const chunkModelId = getChunkEmbeddingModelId(resolvedConfig, providerIdentity);
1685
+ const provider = createProvider(resolvedConfig, {
1686
+ projectRoot: sourceDirectory,
1687
+ session: `shadow:${projectIdentity}`
1688
+ });
1689
+ if (!provider)
1690
+ return null;
1691
+ const prior = shadowRegistrations.get(projectIdentity);
1692
+ if (prior && prior.providerIdentity === providerIdentity) {
1693
+ provider.dispose();
1694
+ dbForShadowQueue.set(projectIdentity, db);
1695
+ persistShadowDescriptor(db, prior);
1696
+ const backfillAlreadyArmed = hasPendingShadowBackfill(projectIdentity) || shadowQueue.some((item) => item.projectIdentity === projectIdentity);
1697
+ if (!backfillAlreadyArmed || options.manualBackfill === true) {
1698
+ maybeArmShadowBackfill(db, projectIdentity, prior, options.manualBackfill === true);
1699
+ }
1700
+ return {
1701
+ ...snapshotFor({
1702
+ projectIdentity,
1703
+ sourceDirectory,
1704
+ config: prior.config,
1705
+ providerIdentity: prior.providerIdentity,
1706
+ runtimeFingerprint: `shadow:${prior.providerIdentity}`,
1707
+ provider: prior.provider,
1708
+ generation: prior.generation,
1709
+ features: { memoryEnabled: true, gitCommitEnabled: true },
1710
+ modelId: prior.modelId,
1711
+ chunkModelId: prior.chunkModelId,
1712
+ observationMode: false
1713
+ }),
1714
+ provider: "synapse"
1715
+ };
1716
+ }
1717
+ const generation = ++globalRegistrationGeneration;
1718
+ const registration = {
1719
+ projectIdentity,
1720
+ sourceDirectory,
1721
+ config: resolvedConfig,
1722
+ provider,
1723
+ providerIdentity,
1724
+ modelId: providerIdentity,
1725
+ chunkModelId,
1726
+ generation
1727
+ };
1728
+ shadowRegistrations.set(projectIdentity, registration);
1729
+ dbForShadowQueue.set(projectIdentity, db);
1730
+ if (prior) {
1731
+ disposeProvider(prior.provider);
1732
+ for (const scope of ["memory", "commit", "chunk"]) {
1733
+ const scopeKey = `${projectIdentity}:${scope}`;
1734
+ shadowBackfillLastIds.delete(scopeKey);
1735
+ shadowBackfillStopReasons.delete(scopeKey);
1736
+ shadowBackfillLastWriteOutcomes.delete(scopeKey);
1737
+ }
1738
+ }
1739
+ db.transaction(() => {
1740
+ const now = Date.now();
1741
+ recordScopeActiveIdentity(db, projectIdentity, "memory", registration.modelId, now);
1742
+ recordScopeActiveIdentity(db, projectIdentity, "commit", registration.modelId, now);
1743
+ recordScopeActiveIdentity(db, projectIdentity, "chunk", registration.chunkModelId, now);
1744
+ persistShadowDescriptor(db, registration);
1745
+ }).immediate();
1746
+ maybeArmShadowBackfill(db, projectIdentity, registration, options.manualBackfill === true);
1747
+ return {
1748
+ projectIdentity,
1749
+ sourceDirectory,
1750
+ providerIdentity,
1751
+ runtimeFingerprint: `shadow:${providerIdentity}`,
1752
+ generation,
1753
+ features: { memoryEnabled: true, gitCommitEnabled: true },
1754
+ enabled: true,
1755
+ historyEnabled: true,
1756
+ gitCommitEnabled: true,
1757
+ modelId: registration.modelId,
1758
+ chunkModelId: registration.chunkModelId,
1759
+ model: "model" in resolvedConfig && typeof resolvedConfig.model === "string" ? resolvedConfig.model : registration.modelId,
1760
+ provider: "synapse",
1761
+ ...synapseDescriptorFromConfig(resolvedConfig) ? { synapseDescriptor: synapseDescriptorFromConfig(resolvedConfig) } : {}
1762
+ };
1763
+ }
1764
+ function startShadowWorker() {
1765
+ if (shadowWorker)
1766
+ return;
1767
+ shadowWorker = runShadowWorker().finally(() => {
1768
+ shadowWorker = null;
1769
+ if (shadowQueue.length > 0 || hasPendingShadowBackfill())
1770
+ startShadowWorker();
1771
+ });
1772
+ }
1773
+ function getShadowEmbeddingMeasurementCohort2(projectIdentity) {
1774
+ const registration = shadowRegistrations.get(projectIdentity);
1775
+ if (!registration)
1776
+ return null;
1777
+ const fields = synapseConfigFields(registration.config);
1778
+ return {
1779
+ modelId: registration.modelId,
1780
+ chunkModelId: registration.chunkModelId,
1781
+ fingerprint: fields.fingerprint ?? "",
1782
+ epoch: fields.tableEpoch ?? 0,
1783
+ dims: fields.dims ?? 0
1784
+ };
1785
+ }
1786
+ function getPrimaryEmbeddingMeasurementCohort2(projectIdentity) {
1787
+ const registration = projectRegistrations.get(projectIdentity);
1788
+ if (!registration)
1789
+ return null;
1790
+ const fields = synapseConfigFields(registration.config);
1791
+ return {
1792
+ modelId: registration.modelId,
1793
+ chunkModelId: registration.chunkModelId,
1794
+ fingerprint: fields.fingerprint ?? "",
1795
+ epoch: fields.tableEpoch ?? 0,
1796
+ dims: fields.dims ?? 0
1797
+ };
1798
+ }
1799
+ async function embedShadowTextForProject2(projectIdentity, text, signal) {
1800
+ const registration = shadowRegistrations.get(projectIdentity);
1801
+ if (!registration)
1802
+ return null;
1803
+ try {
1804
+ const vector = await registration.provider.embed(text, signal, "query");
1805
+ return vector && !isSynapseEmbeddingTruncated(vector) ? vector : null;
1806
+ } catch (error) {
1807
+ log2("[magic-context] Synapse shadow query failed:", error);
1808
+ return null;
1809
+ }
1810
+ }
1811
+ function enqueueShadowEmbeddingItems2(projectIdentity, scope, ids) {
1812
+ if (ids.length === 0 || !shadowRegistrations.has(projectIdentity))
1813
+ return;
1814
+ shadowQueue.push({ projectIdentity, scope, ids: [...ids] });
1815
+ startShadowWorker();
1816
+ }
1817
+ function shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity) {
1818
+ if (scope === "memory") {
1819
+ return {
1820
+ sql: `SELECT m.id AS id
1821
+ FROM memories m
1822
+ JOIN memory_embeddings mp ON mp.memory_id = m.id AND mp.model_id = ?
1823
+ LEFT JOIN memory_embeddings ms ON ms.memory_id = m.id AND ms.model_id = ?
1824
+ WHERE m.project_path = ? AND m.status = 'active' AND ms.memory_id IS NULL`,
1825
+ params: [primaryModelId, shadowModelId, projectIdentity],
1826
+ orderBy: " ORDER BY m.id"
1827
+ };
1828
+ }
1829
+ return {
1830
+ sql: `SELECT gc.sha AS id
1831
+ FROM git_commits gc
1832
+ JOIN git_commit_embeddings gp ON gp.sha = gc.sha AND gp.model_id = ?
1833
+ LEFT JOIN git_commit_embeddings gs ON gs.sha = gc.sha AND gs.model_id = ?
1834
+ WHERE gc.project_path = ? AND gs.sha IS NULL`,
1835
+ params: [primaryModelId, shadowModelId, projectIdentity],
1836
+ orderBy: " ORDER BY gc.committed_at DESC, gc.sha"
1837
+ };
1838
+ }
1839
+ function shadowBackfillMissingIds(db, projectIdentity, scope, primaryModelId, shadowModelId, limit, shadowMaxInputTokens) {
1840
+ if (scope === "chunk") {
1841
+ return loadUnembeddedShadowChunkCandidates(db, projectIdentity, primaryModelId, shadowModelId, limit, shadowMaxInputTokens).map((candidate) => String(candidate.id));
1842
+ }
1843
+ const { sql, params, orderBy } = shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity);
1844
+ const rows = db.prepare(`${sql}${orderBy} LIMIT ?`).all(...params, limit);
1845
+ return rows.map((row) => String(row.id));
1846
+ }
1847
+ function countShadowBackfillMissing(db, projectIdentity, scope, primaryModelId, shadowModelId, shadowMaxInputTokens) {
1848
+ if (scope === "chunk") {
1849
+ return loadUnembeddedShadowChunkCandidates(db, projectIdentity, primaryModelId, shadowModelId, Number.MAX_SAFE_INTEGER, shadowMaxInputTokens).length;
1850
+ }
1851
+ const { sql, params } = shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity);
1852
+ return db.prepare(`SELECT COUNT(*) AS count FROM (${sql})`).get(...params).count;
1853
+ }
1854
+ function shadowModelIdForScope(registration, scope) {
1855
+ return scope === "chunk" ? registration.chunkModelId : registration.modelId;
1856
+ }
1857
+ function shadowMaxInputTokensFor(registration) {
1858
+ return normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in registration.config ? registration.config.max_input_tokens : undefined);
1859
+ }
1860
+ function shadowBackfillCohortFingerprint(db, projectIdentity, scope, primaryModelId) {
1861
+ if (scope === "memory") {
1862
+ return db.prepare(`SELECT COUNT(*) AS rowCount, MAX(m.id) AS maxId, MAX(m.updated_at) AS maxUpdatedAt
1863
+ FROM memories m
1864
+ JOIN memory_embeddings me ON me.memory_id = m.id AND me.model_id = ?
1865
+ WHERE m.project_path = ? AND m.status = 'active'`).get(primaryModelId, projectIdentity);
1866
+ }
1867
+ if (scope === "commit") {
1868
+ return db.prepare(`SELECT COUNT(*) AS rowCount, MAX(gc.sha) AS maxId, MAX(gc.committed_at) AS maxUpdatedAt
1869
+ FROM git_commits gc
1870
+ JOIN git_commit_embeddings gce ON gce.sha = gc.sha AND gce.model_id = ?
1871
+ WHERE gc.project_path = ?`).get(primaryModelId, projectIdentity);
1872
+ }
1873
+ return db.prepare(`SELECT COUNT(*) AS rowCount,
1874
+ COUNT(DISTINCT compartment_id) AS compartmentCount,
1875
+ MAX(compartment_id) AS maxId,
1876
+ MAX(created_at) AS maxUpdatedAt
1877
+ FROM compartment_chunk_embeddings
1878
+ WHERE project_path = ? AND model_id = ?`).get(projectIdentity, primaryModelId);
1879
+ }
1880
+ function shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, limit) {
1881
+ const shadowModelId = shadowModelIdForScope(shadow, scope);
1882
+ const ids = shadowBackfillMissingIds(db, projectIdentity, scope, primaryModelId, shadowModelId, limit, shadowMaxInputTokensFor(shadow));
1883
+ return {
1884
+ ids,
1885
+ signature: sha256Prefix(stableStringify({
1886
+ scope,
1887
+ primaryModelId,
1888
+ shadowModelId,
1889
+ ids,
1890
+ cohort: shadowBackfillCohortFingerprint(db, projectIdentity, scope, primaryModelId)
1891
+ }), 32)
1892
+ };
1893
+ }
1894
+ function hasPendingShadowBackfill(projectIdentity) {
1895
+ if (projectIdentity === undefined)
1896
+ return pendingShadowBackfills.size > 0;
1897
+ const scopes = pendingShadowBackfills.get(projectIdentity);
1898
+ return scopes !== undefined && scopes.size > 0;
1899
+ }
1900
+ function pumpShadowBackfill() {
1901
+ for (const [projectIdentity, scopes] of pendingShadowBackfills) {
1902
+ const db = dbForShadowQueue.get(projectIdentity);
1903
+ const shadow = shadowRegistrations.get(projectIdentity);
1904
+ const primary = projectRegistrations.get(projectIdentity);
1905
+ if (!db || !shadow || !primary) {
1906
+ pendingShadowBackfills.delete(projectIdentity);
1907
+ continue;
1908
+ }
1909
+ for (const scope of [...scopes]) {
1910
+ const primaryModelId = shadowModelIdForScope(primary, scope);
1911
+ const shadowModelId = shadowModelIdForScope(shadow, scope);
1912
+ const stallKey = `${projectIdentity}:${scope}`;
1913
+ if (primaryModelId === "off" || shadowModelId === "off") {
1914
+ scopes.delete(scope);
1915
+ shadowBackfillLastIds.delete(stallKey);
1916
+ shadowBackfillLastWriteOutcomes.delete(stallKey);
1917
+ continue;
1918
+ }
1919
+ const batch = shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, SHADOW_MAX_ITEMS_PER_TICK);
1920
+ if (batch.ids.length === 0) {
1921
+ shadowBackfillStopReasons.set(stallKey, "drained");
1922
+ scopes.delete(scope);
1923
+ shadowBackfillLastIds.delete(stallKey);
1924
+ shadowBackfillLastWriteOutcomes.delete(stallKey);
1925
+ updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
1926
+ ...state,
1927
+ stopReason: "drained",
1928
+ candidateSignature: undefined,
1929
+ writeRefusalReason: undefined,
1930
+ stoppedAt: shadowBackfillNow()
1931
+ }));
1932
+ continue;
1933
+ }
1934
+ if (shadowBackfillLastIds.get(stallKey) === batch.signature) {
1935
+ const lastOutcome = shadowBackfillLastWriteOutcomes.get(stallKey);
1936
+ const writeRefusalReason = lastOutcome?.refusalReason ?? (lastOutcome && lastOutcome.writes > 0 ? "chunk_window_contract_mismatch" : "unknown_write_rejection");
1937
+ shadowBackfillStopReasons.set(stallKey, "stalled_no_progress");
1938
+ updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
1939
+ ...state,
1940
+ stopReason: "stalled_no_progress",
1941
+ candidateSignature: batch.signature,
1942
+ writeRefusalReason,
1943
+ stoppedAt: shadowBackfillNow()
1944
+ }));
1945
+ log2(`[shadow] backfill scope ${scope} for ${projectIdentity} retired without progress — ` + `${describeShadowBackfillWriteRefusal2(writeRefusalReason)}; ` + `${batch.ids.length}+ items remain and automatic registration will not retry unchanged candidates`);
1946
+ scopes.delete(scope);
1947
+ shadowBackfillLastIds.delete(stallKey);
1948
+ shadowBackfillLastWriteOutcomes.delete(stallKey);
1949
+ continue;
1950
+ }
1951
+ shadowBackfillLastIds.set(stallKey, batch.signature);
1952
+ shadowQueue.push({ projectIdentity, scope, ids: batch.ids });
1953
+ }
1954
+ if (scopes.size === 0)
1955
+ pendingShadowBackfills.delete(projectIdentity);
1956
+ }
1957
+ }
1958
+ function maybeArmShadowBackfill(db, projectIdentity, shadow, manualBackfill = false) {
1959
+ if (untrustedLoadProjects.has(projectIdentity))
1960
+ return;
1961
+ const primary = projectRegistrations.get(projectIdentity);
1962
+ if (!primary)
1963
+ return;
1964
+ const pending = new Set;
1965
+ for (const scope of ["memory", "commit", "chunk"]) {
1966
+ const primaryModelId = shadowModelIdForScope(primary, scope);
1967
+ const shadowModelId = shadowModelIdForScope(shadow, scope);
1968
+ const stallKey = `${projectIdentity}:${scope}`;
1969
+ if (primaryModelId === "off" || shadowModelId === "off")
1970
+ continue;
1971
+ if (scope === "memory" && !primary.features.memoryEnabled)
1972
+ continue;
1973
+ const batch = shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, SHADOW_MAX_ITEMS_PER_TICK);
1974
+ const persisted = getPersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId);
1975
+ if (batch.ids.length === 0) {
1976
+ if (persisted?.stopReason === "stalled_no_progress") {
1977
+ shadowBackfillStopReasons.set(stallKey, "drained");
1978
+ updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
1979
+ ...state,
1980
+ stopReason: "drained",
1981
+ candidateSignature: undefined,
1982
+ writeRefusalReason: undefined,
1983
+ stoppedAt: shadowBackfillNow()
1984
+ }));
1985
+ }
1986
+ continue;
1987
+ }
1988
+ if (!manualBackfill && persisted?.stopReason === "stalled_no_progress" && persisted.candidateSignature === batch.signature) {
1989
+ shadowBackfillStopReasons.set(stallKey, "stalled_no_progress");
1990
+ continue;
1991
+ }
1992
+ shadowBackfillStopReasons.delete(stallKey);
1993
+ shadowBackfillLastIds.delete(stallKey);
1994
+ shadowBackfillLastWriteOutcomes.delete(stallKey);
1995
+ updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
1996
+ ...state,
1997
+ stopReason: undefined,
1998
+ candidateSignature: undefined,
1999
+ writeRefusalReason: undefined,
2000
+ stoppedAt: undefined
2001
+ }));
2002
+ pending.add(scope);
2003
+ }
2004
+ if (pending.size === 0)
2005
+ return;
2006
+ pendingShadowBackfills.set(projectIdentity, pending);
2007
+ pumpShadowBackfill();
2008
+ startShadowWorker();
2009
+ }
2010
+ function getShadowBackfillRemaining2(db, projectIdentity) {
2011
+ const remaining = { memory: 0, commit: 0, chunk: 0 };
2012
+ const primary = projectRegistrations.get(projectIdentity);
2013
+ const shadow = shadowRegistrations.get(projectIdentity);
2014
+ if (!primary || !shadow)
2015
+ return remaining;
2016
+ for (const scope of ["memory", "commit", "chunk"]) {
2017
+ const primaryModelId = shadowModelIdForScope(primary, scope);
2018
+ const shadowModelId = shadowModelIdForScope(shadow, scope);
2019
+ if (primaryModelId === "off" || shadowModelId === "off")
2020
+ continue;
2021
+ remaining[scope] = countShadowBackfillMissing(db, projectIdentity, scope, primaryModelId, shadowModelId, shadowMaxInputTokensFor(shadow));
2022
+ }
2023
+ return remaining;
2024
+ }
2025
+ async function flushShadowEmbeddingBacklog2(projectIdentity, onSettled) {
2026
+ if (shadowQueue.length > 0 || hasPendingShadowBackfill(projectIdentity)) {
2027
+ startShadowWorker();
2028
+ }
2029
+ for (;; ) {
2030
+ const worker = shadowWorker;
2031
+ if (worker) {
2032
+ await worker;
2033
+ await onSettled?.();
2034
+ continue;
2035
+ }
2036
+ if (shadowQueue.length === 0 && !hasPendingShadowBackfill(projectIdentity))
2037
+ return;
2038
+ startShadowWorker();
2039
+ if (!shadowWorker)
2040
+ await new Promise((resolve) => setTimeout(resolve, 10));
2041
+ await onSettled?.();
2042
+ }
2043
+ }
2044
+ async function embedShadowItems(registration, items, db, scope) {
2045
+ const raw = registration.config;
2046
+ const fingerprint = typeof raw.synapse_fingerprint === "string" ? raw.synapse_fingerprint : "";
2047
+ const tableEpoch = typeof raw.synapse_table_epoch === "number" ? raw.synapse_table_epoch : 0;
2048
+ const requestKey = getSynapseBatchRequestKey({
2049
+ model: typeof raw.model === "string" ? raw.model : SYNAPSE_DEFAULT_MODEL,
2050
+ fingerprint,
2051
+ tableEpoch,
2052
+ items
2053
+ });
2054
+ const sessionId = `shadow:${registration.projectIdentity}`;
2055
+ const now = shadowBackfillNow();
2056
+ const prior = db.prepare(`SELECT updated_at AS updatedAt
2057
+ FROM synapse_batch_ledger
2058
+ WHERE session_id = ? AND request_key = ?`).get(sessionId, requestKey);
2059
+ if (typeof prior?.updatedAt === "number" && now - prior.updatedAt >= 0 && now - prior.updatedAt < SHADOW_RESUBMIT_WINDOW_MS) {
2060
+ const modelId = shadowModelIdForScope(registration, scope);
2061
+ const state = getPersistedShadowBackfillState(db, registration.projectIdentity, scope, modelId);
2062
+ if (state?.budgetLogRequestKey !== requestKey || typeof state.budgetLoggedAt !== "number" || now - state.budgetLoggedAt >= SHADOW_RESUBMIT_WINDOW_MS) {
2063
+ log2(`[shadow] skipped duplicate ${scope} batch for ${registration.projectIdentity}; ` + "the same content was submitted within the one-hour provider budget");
2064
+ updatePersistedShadowBackfillState(db, registration.projectIdentity, scope, modelId, (current) => ({
2065
+ ...current,
2066
+ budgetLogRequestKey: requestKey,
2067
+ budgetLoggedAt: now
2068
+ }));
2069
+ }
2070
+ return {
2071
+ vectors: new Map,
2072
+ refusalReason: "duplicate_submission_budget"
2073
+ };
2074
+ }
2075
+ beginSynapseBatchLedger(db, {
2076
+ sessionId,
2077
+ projectPath: registration.projectIdentity,
2078
+ scope,
2079
+ manifest: items.map(({ id, contentSha256 }) => ({ id, contentSha256 })),
2080
+ requestKey
2081
+ }, now);
2082
+ try {
2083
+ if (registration.provider.embedItems) {
2084
+ const returned = await registration.provider.embedItems(items);
2085
+ const vectors = new Map([...returned].filter(([, vector]) => !isSynapseEmbeddingTruncated(vector)));
2086
+ finishSynapseBatchLedger(db, sessionId, requestKey, vectors.size === items.length ? "complete" : "partial", shadowBackfillNow());
2087
+ return {
2088
+ vectors,
2089
+ ...vectors.size === 0 ? { refusalReason: "provider_returned_no_vectors" } : {}
2090
+ };
2091
+ }
2092
+ const positional = await registration.provider.embedBatch(items.map((item) => item.text));
2093
+ const vectors = new Map(items.flatMap((item, index) => {
2094
+ const vector = positional[index];
2095
+ return vector && !isSynapseEmbeddingTruncated(vector) ? [[item.id, vector]] : [];
2096
+ }));
2097
+ finishSynapseBatchLedger(db, sessionId, requestKey, vectors.size === items.length ? "complete" : "partial", shadowBackfillNow());
2098
+ return {
2099
+ vectors,
2100
+ ...vectors.size === 0 ? { refusalReason: "provider_returned_no_vectors" } : {}
2101
+ };
2102
+ } catch (error) {
2103
+ finishSynapseBatchLedger(db, sessionId, requestKey, "failed", shadowBackfillNow());
2104
+ throw error;
2105
+ }
2106
+ }
2107
+ async function processShadowQueueItem(item) {
2108
+ const registration = shadowRegistrations.get(item.projectIdentity);
2109
+ if (!registration)
2110
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2111
+ const boundedIds = item.ids.slice(0, SHADOW_MAX_ITEMS_PER_TICK);
2112
+ if (item.scope === "memory") {
2113
+ const db = dbForShadowQueue.get(item.projectIdentity);
2114
+ if (!db)
2115
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2116
+ const placeholders = boundedIds.map(() => "?").join(",");
2117
+ const rows = db.prepare(`SELECT id, content, normalized_hash FROM memories
2118
+ WHERE project_path = ? AND id IN (${placeholders}) AND status = 'active'`).all(item.projectIdentity, ...boundedIds.map((id) => Number(id)));
2119
+ if (rows.length === 0)
2120
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2121
+ const embedded = await embedShadowItems(registration, rows.map((row) => ({
2122
+ id: `memory:${row.id}`,
2123
+ text: row.content,
2124
+ contentSha256: contentSha2562(row.content)
2125
+ })), db, "memory");
2126
+ const live = shadowRegistrations.get(item.projectIdentity);
2127
+ if (!live || live.generation !== registration.generation) {
2128
+ return { writes: 0, refusalReason: "registration_retired_during_embed" };
2129
+ }
2130
+ let writes = 0;
2131
+ let hashGuardRejected = false;
2132
+ db.transaction(() => {
2133
+ for (const row of rows) {
2134
+ const vector = embedded.vectors.get(`memory:${row.id}`);
2135
+ if (!vector)
2136
+ continue;
2137
+ if (saveEmbeddingIfHashMatches(db, row.id, vector, registration.modelId, row.normalized_hash)) {
2138
+ writes += 1;
2139
+ } else {
2140
+ hashGuardRejected = true;
2141
+ }
2142
+ }
2143
+ }).immediate();
2144
+ return {
2145
+ writes,
2146
+ ...writes === 0 ? {
2147
+ refusalReason: embedded.refusalReason ?? (hashGuardRejected ? "memory_hash_guard_rejected" : "provider_returned_no_vectors")
2148
+ } : {}
2149
+ };
2150
+ }
2151
+ if (item.scope === "commit") {
2152
+ const db = dbForShadowQueue.get(item.projectIdentity);
2153
+ if (!db)
2154
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2155
+ const placeholders = boundedIds.map(() => "?").join(",");
2156
+ const rows = db.prepare(`SELECT sha, message FROM git_commits WHERE project_path = ? AND sha IN (${placeholders})`).all(item.projectIdentity, ...boundedIds);
2157
+ if (rows.length === 0)
2158
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2159
+ const embedded = await embedShadowItems(registration, rows.map((row) => ({
2160
+ id: `commit:${row.sha}`,
2161
+ text: row.message,
2162
+ contentSha256: contentSha2562(row.message)
2163
+ })), db, "commit");
2164
+ const live = shadowRegistrations.get(item.projectIdentity);
2165
+ if (!live || live.generation !== registration.generation) {
2166
+ return { writes: 0, refusalReason: "registration_retired_during_embed" };
2167
+ }
2168
+ let writes = 0;
2169
+ db.transaction(() => {
2170
+ for (const row of rows) {
2171
+ const vector = embedded.vectors.get(`commit:${row.sha}`);
2172
+ if (!vector)
2173
+ continue;
2174
+ saveCommitEmbedding(db, row.sha, vector, registration.modelId);
2175
+ writes += 1;
2176
+ }
2177
+ }).immediate();
2178
+ return {
2179
+ writes,
2180
+ ...writes === 0 ? { refusalReason: embedded.refusalReason ?? "provider_returned_no_vectors" } : {}
2181
+ };
2182
+ }
2183
+ const db = dbForShadowQueue.get(item.projectIdentity);
2184
+ if (!db)
2185
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2186
+ const placeholders = boundedIds.map(() => "?").join(",");
2187
+ const candidates = db.prepare(`SELECT id, session_id, start_message, end_message
2188
+ FROM compartments WHERE id IN (${placeholders})`).all(...boundedIds.map((id) => Number(id)));
2189
+ if (candidates.length === 0) {
2190
+ return { writes: 0, refusalReason: "candidate_rows_changed" };
2191
+ }
2192
+ const prepared = [];
2193
+ let ftsMappingIncomplete = false;
2194
+ let emptyCanonicalText = false;
2195
+ for (const candidate of candidates) {
2196
+ const mappedText = buildCanonicalChunkTextFromFts(db, candidate.session_id, candidate.start_message, candidate.end_message);
2197
+ if (mappedText === null) {
2198
+ ftsMappingIncomplete = true;
2199
+ } else {
2200
+ const text = mappedText || buildCompartmentSummaryFallbackText(db, candidate.id);
2201
+ const shadowMaxInputTokens = shadowMaxInputTokensFor(registration);
2202
+ const windows = chunkCanonicalText(text, candidate.start_message, candidate.end_message, shadowMaxInputTokens);
2203
+ const windowSourceKey = chunkWindowSourceKey(text, candidate.start_message, candidate.end_message, shadowMaxInputTokens);
2204
+ if (windows.length > 0)
2205
+ prepared.push({ candidate, windows, windowSourceKey });
2206
+ else
2207
+ emptyCanonicalText = true;
2208
+ }
2209
+ await new Promise((resolve) => setTimeout(resolve, 0));
2210
+ }
2211
+ if (prepared.length === 0) {
2212
+ return {
2213
+ writes: 0,
2214
+ refusalReason: ftsMappingIncomplete ? "chunk_fts_mapping_incomplete" : "chunk_empty_canonical_text"
2215
+ };
2216
+ }
2217
+ const items = prepared.flatMap((item) => item.windows.map((window) => ({
2218
+ id: `chunk:${item.candidate.id}:${window.windowIndex}`,
2219
+ text: window.text,
2220
+ contentSha256: contentSha2562(window.text)
2221
+ })));
2222
+ const embedded = await embedShadowItems(registration, items, db, "chunk");
2223
+ const live = shadowRegistrations.get(item.projectIdentity);
2224
+ if (!live || live.generation !== registration.generation) {
2225
+ return { writes: 0, refusalReason: "registration_retired_during_embed" };
2226
+ }
2227
+ let writes = 0;
2228
+ let partialVectorSet = false;
2229
+ for (const item of prepared) {
2230
+ const rows = item.windows.flatMap((window) => {
2231
+ const vector = embedded.vectors.get(`chunk:${item.candidate.id}:${window.windowIndex}`);
2232
+ return vector ? [
2233
+ {
2234
+ compartmentId: item.candidate.id,
2235
+ sessionId: item.candidate.session_id,
2236
+ projectPath: registration.projectIdentity,
2237
+ window,
2238
+ modelId: registration.chunkModelId,
2239
+ vector
2240
+ }
2241
+ ] : [];
2242
+ });
2243
+ if (rows.length === item.windows.length) {
2244
+ replaceCompartmentChunkEmbeddings(db, rows, item.windowSourceKey);
2245
+ writes += 1;
2246
+ } else {
2247
+ partialVectorSet = true;
2248
+ }
2249
+ }
2250
+ return {
2251
+ writes,
2252
+ ...writes === 0 ? {
2253
+ refusalReason: embedded.refusalReason ?? (partialVectorSet ? "chunk_partial_vector_set" : ftsMappingIncomplete ? "chunk_fts_mapping_incomplete" : emptyCanonicalText ? "chunk_empty_canonical_text" : "unknown_write_rejection")
2254
+ } : {}
2255
+ };
2256
+ }
2257
+ var dbForShadowQueue = new Map;
2258
+ async function runShadowWorker() {
2259
+ const startedAt = Date.now();
2260
+ let processed = 0;
2261
+ let processedBytes = 0;
2262
+ for (;; ) {
2263
+ if (shadowQueue.length === 0) {
2264
+ pumpShadowBackfill();
2265
+ if (shadowQueue.length === 0)
2266
+ break;
2267
+ }
2268
+ if (processed >= SHADOW_MAX_ITEMS_PER_TICK || Date.now() - startedAt >= SHADOW_MAX_WALL_CLOCK_MS) {
2269
+ break;
2270
+ }
2271
+ const item = shadowQueue.shift();
2272
+ if (!item)
2273
+ break;
2274
+ const itemBytes = item.ids.reduce((total, id) => total + id.length, 0);
2275
+ if (processed > 0 && processedBytes + itemBytes > SHADOW_MAX_BYTES_PER_TICK) {
2276
+ shadowQueue.unshift(item);
2277
+ break;
2278
+ }
2279
+ const generationAtStart = shadowRegistrations.get(item.projectIdentity)?.generation;
2280
+ const isStillCurrent = () => generationAtStart !== undefined && shadowRegistrations.get(item.projectIdentity)?.generation === generationAtStart;
2281
+ try {
2282
+ const outcome = await processShadowQueueItem(item);
2283
+ if (isStillCurrent()) {
2284
+ shadowBackfillLastWriteOutcomes.set(`${item.projectIdentity}:${item.scope}`, outcome);
2285
+ }
2286
+ } catch (error) {
2287
+ if (isStillCurrent()) {
2288
+ shadowBackfillLastWriteOutcomes.set(`${item.projectIdentity}:${item.scope}`, {
2289
+ writes: 0,
2290
+ refusalReason: "provider_returned_no_vectors"
2291
+ });
2292
+ }
2293
+ log2("[magic-context] Synapse shadow write failed:", error);
2294
+ }
2295
+ processed += item.ids.length;
2296
+ processedBytes += itemBytes;
2297
+ await new Promise((resolve) => setTimeout(resolve, 0));
2298
+ }
2299
+ }
2300
+ function registerProjectInObservationMode2(db, projectIdentity, sourceDirectory, failedConfig, failureSummary) {
2301
+ const prior = projectRegistrations.get(projectIdentity);
2302
+ const runtimeFingerprint = `observation:${sha256Prefix(failureSummary)}`;
2303
+ const generation = prior?.runtimeFingerprint === runtimeFingerprint && prior.observationMode ? prior.generation : ++globalRegistrationGeneration;
2304
+ const registration = {
2305
+ projectIdentity,
2306
+ sourceDirectory,
2307
+ config: resolveEmbeddingConfig(failedConfig),
2308
+ providerIdentity: OFF_PROVIDER_IDENTITY,
2309
+ runtimeFingerprint,
2310
+ provider: null,
2311
+ generation,
2312
+ features: { memoryEnabled: false, gitCommitEnabled: false },
2313
+ modelId: "off",
2314
+ chunkModelId: "off",
2315
+ observationMode: true
2316
+ };
2317
+ projectRegistrations.set(projectIdentity, registration);
2318
+ disposeProvider(prior?.provider ?? null);
2319
+ return snapshotFor(registration);
2320
+ }
2321
+ function unregisterProjectShadowEmbedding2(projectIdentity) {
2322
+ const shadow = shadowRegistrations.get(projectIdentity);
2323
+ shadowRegistrations.delete(projectIdentity);
2324
+ dbForShadowQueue.delete(projectIdentity);
2325
+ pendingShadowBackfills.delete(projectIdentity);
2326
+ for (let index = shadowQueue.length - 1;index >= 0; index -= 1) {
2327
+ if (shadowQueue[index].projectIdentity === projectIdentity)
2328
+ shadowQueue.splice(index, 1);
2329
+ }
2330
+ for (const scope of ["memory", "commit", "chunk"]) {
2331
+ const key = `${projectIdentity}:${scope}`;
2332
+ shadowBackfillLastIds.delete(key);
2333
+ shadowBackfillStopReasons.delete(key);
2334
+ shadowBackfillLastWriteOutcomes.delete(key);
2335
+ }
2336
+ const primaryProvider = projectRegistrations.get(projectIdentity)?.provider ?? null;
2337
+ if (shadow?.provider && shadow.provider !== primaryProvider) {
2338
+ disposeProvider(shadow.provider);
2339
+ }
2340
+ }
2341
+ function unregisterProjectEmbedding2(projectIdentity) {
2342
+ const prior = projectRegistrations.get(projectIdentity);
2343
+ const shadow = shadowRegistrations.get(projectIdentity);
2344
+ if (!prior && !shadow)
2345
+ return;
2346
+ projectRegistrations.delete(projectIdentity);
2347
+ shadowRegistrations.delete(projectIdentity);
2348
+ dbForShadowQueue.delete(projectIdentity);
2349
+ globalRegistrationGeneration += 1;
2350
+ disposeProvider(prior?.provider ?? null);
2351
+ disposeProvider(shadow?.provider ?? null);
2352
+ }
2353
+ function getProjectEmbeddingSnapshot2(projectIdentity) {
2354
+ const registration = projectRegistrations.get(projectIdentity);
2355
+ return registration ? snapshotFor(registration) : null;
2356
+ }
2357
+ function installProjectEmbeddingSearchBridge2(snapshot, provider) {
2358
+ projectRegistrations.set(snapshot.projectIdentity, {
2359
+ projectIdentity: snapshot.projectIdentity,
2360
+ sourceDirectory: snapshot.sourceDirectory,
2361
+ providerIdentity: snapshot.providerIdentity,
2362
+ runtimeFingerprint: snapshot.runtimeFingerprint,
2363
+ generation: snapshot.generation,
2364
+ features: { ...snapshot.features },
2365
+ modelId: snapshot.modelId,
2366
+ chunkModelId: snapshot.chunkModelId,
2367
+ config: { provider: "off" },
2368
+ provider,
2369
+ observationMode: !snapshot.historyEnabled
2370
+ });
2371
+ }
2372
+ function getProjectChunkEmbeddingModelId2(projectIdentity) {
2373
+ const registration = projectRegistrations.get(projectIdentity);
2374
+ return registration && !registration.observationMode ? registration.chunkModelId : "off";
2375
+ }
2376
+ function getProjectEmbeddingMaxInputTokens2(projectIdentity) {
2377
+ const registration = projectRegistrations.get(projectIdentity);
2378
+ const configMax = registration?.config && "max_input_tokens" in registration.config ? registration.config.max_input_tokens : undefined;
2379
+ return normalizeCompartmentChunkMaxInputTokens(registration?.provider?.maxInputTokens ?? configMax);
2380
+ }
2381
+ function getOrCreateProjectProvider(registration) {
2382
+ if (registration.providerIdentity === OFF_PROVIDER_IDENTITY || registration.observationMode) {
2383
+ return null;
2384
+ }
2385
+ if (registration.provider) {
2386
+ return registration.provider;
2387
+ }
2388
+ const provider = createProvider(registration.config, {
2389
+ projectRoot: registration.sourceDirectory,
2390
+ session: `project:${registration.projectIdentity}`
2391
+ });
2392
+ registration.provider = provider;
2393
+ return provider;
2394
+ }
2395
+ async function embedTextForProject2(projectIdentity, text, signal, purpose = "passage") {
2396
+ const registration = projectRegistrations.get(projectIdentity);
2397
+ if (!registration)
2398
+ return null;
2399
+ const generation = registration.generation;
2400
+ const modelId = registration.modelId;
2401
+ const provider = getOrCreateProjectProvider(registration);
2402
+ if (!provider)
2403
+ return null;
2404
+ const vector = await provider.embed(text, signal, purpose);
2405
+ if (!vector || isSynapseEmbeddingTruncated(vector))
2406
+ return null;
2407
+ const current = projectRegistrations.get(projectIdentity);
2408
+ if (!current || current.generation !== generation || current.runtimeFingerprint !== registration.runtimeFingerprint) {
2409
+ return null;
2410
+ }
2411
+ return { vector, modelId, chunkModelId: registration.chunkModelId, generation };
2412
+ }
2413
+ async function embedBatchForProject2(projectIdentity, texts, signal, purpose = "passage") {
2414
+ if (texts.length === 0) {
2415
+ const registration = projectRegistrations.get(projectIdentity);
2416
+ if (!registration || registration.observationMode)
2417
+ return null;
2418
+ return { vectors: [], modelId: registration.modelId, generation: registration.generation };
2419
+ }
2420
+ const registration = projectRegistrations.get(projectIdentity);
2421
+ if (!registration)
2422
+ return null;
2423
+ const generation = registration.generation;
2424
+ const modelId = registration.modelId;
2425
+ const runtimeFingerprint = registration.runtimeFingerprint;
2426
+ const provider = getOrCreateProjectProvider(registration);
2427
+ if (!provider)
2428
+ return null;
2429
+ const vectors = (await provider.embedBatch(texts, signal, purpose)).map((vector) => vector && !isSynapseEmbeddingTruncated(vector) ? vector : null);
2430
+ const current = projectRegistrations.get(projectIdentity);
2431
+ if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
2432
+ return null;
2433
+ }
2434
+ return { vectors, modelId, generation };
2435
+ }
2436
+ async function embedItemsForProject2(projectIdentity, items, signal, db, sessionId = projectIdentity) {
2437
+ const registration = projectRegistrations.get(projectIdentity);
2438
+ if (!registration || registration.observationMode || items.length === 0)
2439
+ return null;
2440
+ const generation = registration.generation;
2441
+ const modelId = registration.modelId;
2442
+ const runtimeFingerprint = registration.runtimeFingerprint;
2443
+ const provider = getOrCreateProjectProvider(registration);
2444
+ if (!provider)
2445
+ return null;
2446
+ const rawConfig = registration.config;
2447
+ const fingerprint = typeof rawConfig.synapse_fingerprint === "string" ? rawConfig.synapse_fingerprint : "";
2448
+ const tableEpoch = typeof rawConfig.synapse_table_epoch === "number" ? rawConfig.synapse_table_epoch : undefined;
2449
+ const isSynapse = registration.providerIdentity.startsWith("synapse:v1:") && fingerprint && tableEpoch !== undefined;
2450
+ const ledgerKey = isSynapse ? getSynapseBatchRequestKey({
2451
+ model: typeof rawConfig.model === "string" ? rawConfig.model : registration.modelId,
2452
+ fingerprint,
2453
+ tableEpoch,
2454
+ items
2455
+ }) : null;
2456
+ if (db && ledgerKey) {
2457
+ beginSynapseBatchLedger(db, {
2458
+ sessionId,
2459
+ projectPath: projectIdentity,
2460
+ scope: items[0].id.split(":", 1)[0],
2461
+ manifest: items.map(({ id, contentSha256 }) => ({ id, contentSha256 })),
2462
+ requestKey: ledgerKey
2463
+ });
2464
+ }
2465
+ let vectors;
2466
+ try {
2467
+ if (provider.embedItems) {
2468
+ const returned = await provider.embedItems(items, signal);
2469
+ vectors = new Map([...returned].filter(([, vector]) => !isSynapseEmbeddingTruncated(vector)));
2470
+ } else {
2471
+ const positional = await provider.embedBatch(items.map((item) => item.text), signal, "passage");
2472
+ vectors = new Map(items.flatMap((item, index) => {
2473
+ const vector = positional[index];
2474
+ return vector && !isSynapseEmbeddingTruncated(vector) ? [[item.id, vector]] : [];
2475
+ }));
2476
+ }
2477
+ } catch (error) {
2478
+ if (db && ledgerKey)
2479
+ finishSynapseBatchLedger(db, sessionId, ledgerKey, "failed");
2480
+ throw error;
2481
+ }
2482
+ if (db && ledgerKey) {
2483
+ finishSynapseBatchLedger(db, sessionId, ledgerKey, vectors.size === items.length ? "complete" : "partial");
2484
+ }
2485
+ const current = projectRegistrations.get(projectIdentity);
2486
+ if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
2487
+ return null;
2488
+ }
2489
+ return { vectors, modelId, generation };
2490
+ }
2491
+ async function embedItemsWindowBounded(projectIdentity, items, signal, db) {
2492
+ if (items.length <= MAX_WINDOWS_PER_EMBED_CALL) {
2493
+ return embedItemsForProject2(projectIdentity, items, signal, db, projectIdentity);
2494
+ }
2495
+ const vectors = new Map;
2496
+ let modelId = null;
2497
+ let generation = null;
2498
+ for (let start = 0;start < items.length; start += MAX_WINDOWS_PER_EMBED_CALL) {
2499
+ const result = await embedItemsForProject2(projectIdentity, items.slice(start, start + MAX_WINDOWS_PER_EMBED_CALL), signal, db, projectIdentity);
2500
+ if (!result)
2501
+ return null;
2502
+ if (modelId === null) {
2503
+ modelId = result.modelId;
2504
+ generation = result.generation;
2505
+ } else if (modelId !== result.modelId || generation !== result.generation) {
2506
+ return null;
2507
+ }
2508
+ for (const [id, vector] of result.vectors)
2509
+ vectors.set(id, vector);
2510
+ }
2511
+ return modelId === null || generation === null ? null : { vectors, modelId, generation };
2512
+ }
2513
+ function isUnembeddedMemoryRow(row) {
2514
+ if (row === null || typeof row !== "object")
2515
+ return false;
2516
+ const candidate = row;
2517
+ return typeof candidate.id === "number" && typeof candidate.content === "string" && typeof candidate.normalized_hash === "string";
2518
+ }
2519
+ function getLoadUnembeddedMemoriesStatement(db) {
2520
+ let stmt = loadUnembeddedMemoriesStatements.get(db);
2521
+ if (!stmt) {
2522
+ stmt = db.prepare("SELECT m.id AS id, m.content AS content, m.normalized_hash AS normalized_hash FROM memories m LEFT JOIN memory_embeddings me ON m.id = me.memory_id AND me.model_id = ? WHERE m.project_path = ? AND m.status = 'active' AND me.memory_id IS NULL LIMIT ?");
2523
+ loadUnembeddedMemoriesStatements.set(db, stmt);
2524
+ }
2525
+ return stmt;
2526
+ }
2527
+ async function embedUnembeddedMemoriesForProject2(db, projectIdentity, batchSize = 10) {
2528
+ const outcome = await embedUnembeddedMemoriesForProjectOutcome(db, projectIdentity, batchSize);
2529
+ return outcome.kind === "embedded" ? outcome.count : 0;
2530
+ }
2531
+ async function embedUnembeddedMemoriesForProjectOutcome(db, projectIdentity, batchSize = 10) {
2532
+ if (isEmbeddingHostBusy2())
2533
+ return { kind: "unavailable" };
2534
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2535
+ if (!snapshot?.enabled)
2536
+ return { kind: "disabled" };
2537
+ const normalizedBatchSize = Math.max(1, Math.floor(batchSize));
2538
+ const memories = getLoadUnembeddedMemoriesStatement(db).all(snapshot.modelId, projectIdentity, normalizedBatchSize).filter(isUnembeddedMemoryRow);
2539
+ if (memories.length === 0)
2540
+ return { kind: "done" };
2541
+ try {
2542
+ const result = await embedItemsForProject2(projectIdentity, memories.map((memory) => ({
2543
+ id: `memory:${memory.id}`,
2544
+ text: memory.content,
2545
+ contentSha256: contentSha2562(memory.content)
2546
+ })), undefined, db, projectIdentity);
2547
+ if (!result)
2548
+ return { kind: "unavailable" };
2549
+ let embeddedCount = 0;
2550
+ db.transaction(() => {
2551
+ for (const memory of memories) {
2552
+ const embedding = result.vectors.get(`memory:${memory.id}`);
2553
+ if (!embedding)
2554
+ continue;
2555
+ if (saveEmbeddingIfHashMatches(db, memory.id, embedding, result.modelId, memory.normalized_hash)) {
2556
+ embeddedCount += 1;
2557
+ }
2558
+ }
2559
+ }).immediate();
2560
+ enqueueShadowEmbeddingItems2(projectIdentity, "memory", memories.filter((memory) => result.vectors.has(`memory:${memory.id}`)).map((memory) => String(memory.id)));
2561
+ return { kind: "embedded", count: embeddedCount };
2562
+ } catch (error) {
2563
+ log2("[magic-context] failed to proactively embed missing memories:", error);
2564
+ return { kind: "failed", error };
2565
+ }
2566
+ }
2567
+ async function embedCommitBatch(db, projectIdentity, batchSize) {
2568
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2569
+ if (!snapshot?.gitCommitEnabled)
2570
+ return 0;
2571
+ const commits = loadUnembeddedCommits(db, projectIdentity, snapshot.modelId, Math.max(1, Math.floor(batchSize)));
2572
+ if (commits.length === 0)
2573
+ return 0;
2574
+ const result = await embedItemsForProject2(projectIdentity, commits.map((commit) => ({
2575
+ id: `commit:${commit.sha}`,
2576
+ text: commit.message,
2577
+ contentSha256: contentSha2562(commit.message)
2578
+ })), undefined, db, projectIdentity);
2579
+ if (!result)
2580
+ return 0;
2581
+ let embeddedCount = 0;
2582
+ db.transaction(() => {
2583
+ for (const commit of commits) {
2584
+ const embedding = result.vectors.get(`commit:${commit.sha}`);
2585
+ if (!embedding)
2586
+ continue;
2587
+ saveCommitEmbedding(db, commit.sha, embedding, result.modelId);
2588
+ embeddedCount += 1;
2589
+ }
2590
+ }).immediate();
2591
+ enqueueShadowEmbeddingItems2(projectIdentity, "commit", commits.filter((commit) => result.vectors.has(`commit:${commit.sha}`)).map((commit) => commit.sha));
2592
+ return embeddedCount;
2593
+ }
2594
+ async function drainCommitBacklogForProject2(db, projectIdentity, deadline) {
2595
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2596
+ if (!snapshot?.gitCommitEnabled)
2597
+ return 0;
2598
+ const holderId = `embed-sweep-${randomUUID()}`;
2599
+ const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
2600
+ if (!lease.acquired) {
2601
+ return 0;
2602
+ }
2603
+ let total = 0;
2604
+ let leaseLost = false;
2605
+ const renewal = setInterval(() => {
2606
+ try {
2607
+ if (!renewGitSweepLease(db, projectIdentity, holderId))
2608
+ leaseLost = true;
2609
+ } catch {}
2610
+ }, SESSION_EMBED_LEASE_RENEWAL_MS);
2611
+ renewal.unref?.();
2612
+ try {
2613
+ while (!leaseLost && !isEmbeddingHostBusy2() && Date.now() < deadline && total < COMMIT_DRAIN_MAX_PER_SWEEP) {
2614
+ const embedded = await embedCommitBatch(db, projectIdentity, COMMIT_DRAIN_BATCH_SIZE);
2615
+ if (!renewGitSweepLease(db, projectIdentity, holderId))
2616
+ leaseLost = true;
2617
+ if (leaseLost || embedded === 0)
2618
+ break;
2619
+ total += embedded;
2620
+ if (embedded < COMMIT_DRAIN_BATCH_SIZE)
2621
+ break;
2622
+ }
2623
+ } finally {
2624
+ clearInterval(renewal);
2625
+ releaseGitSweepLease(db, projectIdentity, holderId);
2626
+ }
2627
+ return total;
2628
+ }
2629
+ async function embedCompartmentChunkBatch(db, projectIdentity, batchSize) {
2630
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2631
+ if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off")
2632
+ return 0;
2633
+ repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectIdentity);
2634
+ const candidates = await loadUnembeddedCompartmentChunkCandidatesPolite(db, projectIdentity, snapshot.chunkModelId, batchSize, getProjectEmbeddingMaxInputTokens2(projectIdentity), true);
2635
+ if (candidates.length === 0)
2636
+ return 0;
2637
+ const { embedded } = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates);
2638
+ return embedded;
2639
+ }
2640
+ function getLastEmbeddingFailureForProject(projectIdentity) {
2641
+ const registration = projectRegistrations.get(projectIdentity);
2642
+ return registration?.provider?.getLastFailureReason?.() ?? null;
2643
+ }
2644
+ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
2645
+ const noWork = [];
2646
+ const failed = [];
2647
+ const failureReasons = [];
2648
+ if (candidates.length === 0)
2649
+ return { embedded: 0, noWork, failed, failureReasons };
2650
+ const maxInputTokens = getProjectEmbeddingMaxInputTokens2(projectIdentity);
2651
+ const prepared = [];
2652
+ for (const candidate of candidates) {
2653
+ await new Promise((resolve) => setTimeout(resolve, 0));
2654
+ const mappedText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage);
2655
+ if (mappedText === null)
2656
+ continue;
2657
+ const canonicalText = mappedText || buildCompartmentSummaryFallbackText(db, candidate.id);
2658
+ if (canonicalText.length === 0) {
2659
+ recordChunkEmbedBackoff(db, candidate, projectIdentity, modelId);
2660
+ noWork.push(candidate.id);
2661
+ continue;
2662
+ }
2663
+ const windows = chunkCanonicalText(canonicalText, candidate.startMessage, candidate.endMessage, maxInputTokens);
2664
+ if (windows.length === 0 || chunkEmbeddingWindowsAreCurrent(db, candidate.id, modelId, windows, projectIdentity)) {
2665
+ noWork.push(candidate.id);
2666
+ continue;
2667
+ }
2668
+ prepared.push({
2669
+ candidate,
2670
+ windows,
2671
+ windowSourceKey: chunkWindowSourceKey(canonicalText, candidate.startMessage, candidate.endMessage, maxInputTokens)
2672
+ });
2673
+ }
2674
+ if (prepared.length === 0)
2675
+ return { embedded: 0, noWork, failed, failureReasons };
2676
+ let embedded = 0;
2677
+ let i = 0;
2678
+ while (i < prepared.length) {
2679
+ if (signal?.aborted)
2680
+ break;
2681
+ const slice = [];
2682
+ let windowCount = 0;
2683
+ do {
2684
+ const item = prepared[i];
2685
+ slice.push(item);
2686
+ windowCount += item.windows.length;
2687
+ i += 1;
2688
+ } while (i < prepared.length && windowCount + prepared[i].windows.length <= MAX_WINDOWS_PER_EMBED_CALL);
2689
+ const items = slice.flatMap((item) => item.windows.map((window) => ({
2690
+ id: `chunk:${item.candidate.id}:${window.windowIndex}`,
2691
+ text: window.text,
2692
+ contentSha256: contentSha2562(window.text)
2693
+ })));
2694
+ const persistedIds = new Set;
2695
+ for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
2696
+ if (signal?.aborted)
2697
+ break;
2698
+ await new Promise((resolve) => setTimeout(resolve, 15));
2699
+ let result = null;
2700
+ const attemptStart = Date.now();
2701
+ try {
2702
+ result = await embedItemsWindowBounded(projectIdentity, items, signal, db);
2703
+ } catch (error) {
2704
+ log2("[magic-context] failed to proactively embed compartment chunks:", error);
2705
+ }
2706
+ if (signal?.aborted)
2707
+ break;
2708
+ if (!result || result.vectors.size === 0) {
2709
+ const failure = getLastEmbeddingFailureForProject(projectIdentity);
2710
+ if (failure)
2711
+ failureReasons.push(failure);
2712
+ }
2713
+ if (result) {
2714
+ let writeSliceStarted = performance.now();
2715
+ for (const item of slice) {
2716
+ if (persistedIds.has(item.candidate.id))
2717
+ continue;
2718
+ const vectors = item.windows.map((window) => result.vectors.get(`chunk:${item.candidate.id}:${window.windowIndex}`));
2719
+ if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
2720
+ continue;
2721
+ }
2722
+ const rows = item.windows.map((window, index) => ({
2723
+ compartmentId: item.candidate.id,
2724
+ sessionId: item.candidate.sessionId,
2725
+ projectPath: projectIdentity,
2726
+ window,
2727
+ modelId,
2728
+ vector: vectors[index]
2729
+ }));
2730
+ replaceCompartmentChunkEmbeddings(db, rows, item.windowSourceKey);
2731
+ persistedIds.add(item.candidate.id);
2732
+ enqueueShadowEmbeddingItems2(projectIdentity, "chunk", [
2733
+ String(item.candidate.id)
2734
+ ]);
2735
+ if (performance.now() - writeSliceStarted >= 35) {
2736
+ await new Promise((resolve) => setTimeout(resolve, 0));
2737
+ writeSliceStarted = performance.now();
2738
+ }
2739
+ }
2740
+ }
2741
+ if (persistedIds.size === slice.length)
2742
+ break;
2743
+ if (persistedIds.size > 0)
2744
+ break;
2745
+ if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
2746
+ break;
2747
+ if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
2748
+ await new Promise((resolve) => setTimeout(resolve, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
2749
+ }
2750
+ }
2751
+ embedded += persistedIds.size;
2752
+ if (!signal?.aborted) {
2753
+ for (const item of slice) {
2754
+ if (!persistedIds.has(item.candidate.id)) {
2755
+ recordChunkEmbedBackoff(db, item.candidate, projectIdentity, modelId);
2756
+ failed.push(item.candidate.id);
2757
+ }
2758
+ }
2759
+ }
2760
+ }
2761
+ return { embedded, noWork, failed, failureReasons };
2762
+ }
2763
+ async function drainCompartmentChunkBacklogForProject(db, projectIdentity, deadline) {
2764
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2765
+ if (!snapshot?.historyEnabled)
2766
+ return 0;
2767
+ const holderId = `chunk-embed-sweep-${randomUUID()}`;
2768
+ const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
2769
+ if (!lease.acquired) {
2770
+ return 0;
2771
+ }
2772
+ let total = 0;
2773
+ let leaseLost = false;
2774
+ const renewal = setInterval(() => {
2775
+ try {
2776
+ if (!renewGitSweepLease(db, projectIdentity, holderId))
2777
+ leaseLost = true;
2778
+ } catch {}
2779
+ }, SESSION_EMBED_LEASE_RENEWAL_MS);
2780
+ renewal.unref?.();
2781
+ try {
2782
+ while (!leaseLost && !isEmbeddingHostBusy2() && Date.now() < deadline && total < CHUNK_DRAIN_MAX_PER_SWEEP) {
2783
+ const embedded = await embedCompartmentChunkBatch(db, projectIdentity, CHUNK_DRAIN_BATCH_SIZE);
2784
+ if (!renewGitSweepLease(db, projectIdentity, holderId))
2785
+ leaseLost = true;
2786
+ if (leaseLost || embedded === 0)
2787
+ break;
2788
+ total += embedded;
2789
+ if (embedded < CHUNK_DRAIN_BATCH_SIZE)
2790
+ break;
2791
+ }
2792
+ } finally {
2793
+ clearInterval(renewal);
2794
+ releaseGitSweepLease(db, projectIdentity, holderId);
2795
+ }
2796
+ return total;
2797
+ }
2798
+ async function embedUnembeddedCompartmentChunksForProject2(db, projectIdentity) {
2799
+ return drainCompartmentChunkBacklogForProject(db, projectIdentity, Date.now() + SWEEP_MAX_WALL_CLOCK_MS);
2800
+ }
2801
+ async function embedSessionCompartmentChunks2(db, projectIdentity, sessionId, options) {
2802
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2803
+ if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
2804
+ return { status: "disabled", embedded: 0, total: 0 };
2805
+ }
2806
+ recordSessionProjectIdentity(db, sessionId, projectIdentity);
2807
+ const maxInputTokens = getProjectEmbeddingMaxInputTokens2(projectIdentity);
2808
+ const total = await countUnembeddedSessionCompartmentsPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, maxInputTokens);
2809
+ const holderId = `session-embed-${randomUUID()}`;
2810
+ const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
2811
+ if (!lease.acquired)
2812
+ return { status: "busy", embedded: 0, total };
2813
+ let leaseLost = false;
2814
+ const drainAbort = new AbortController;
2815
+ const forwardCallerAbort = () => drainAbort.abort();
2816
+ if (options?.signal?.aborted)
2817
+ drainAbort.abort();
2818
+ else
2819
+ options?.signal?.addEventListener("abort", forwardCallerAbort, { once: true });
2820
+ const renewal = setInterval(() => {
2821
+ try {
2822
+ if (!renewGitSweepLease(db, projectIdentity, holderId)) {
2823
+ leaseLost = true;
2824
+ drainAbort.abort();
2825
+ }
2826
+ } catch {}
2827
+ }, SESSION_EMBED_LEASE_RENEWAL_MS);
2828
+ renewal.unref?.();
2829
+ const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
2830
+ const skipIds = [];
2831
+ const failedIds = [];
2832
+ let embedded = 0;
2833
+ let aborted = false;
2834
+ let providerDown = false;
2835
+ let consecutiveFailedBatches = 0;
2836
+ const failureReasons = [];
2837
+ try {
2838
+ options?.onProgress?.({ embedded, total });
2839
+ for (;; ) {
2840
+ if (leaseLost || drainAbort.signal.aborted || isEmbeddingHostBusy2()) {
2841
+ aborted = true;
2842
+ break;
2843
+ }
2844
+ const candidates = await loadUnembeddedSessionChunkCandidatesPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds], maxInputTokens, true);
2845
+ if (candidates.length === 0)
2846
+ break;
2847
+ const {
2848
+ embedded: n,
2849
+ noWork,
2850
+ failed,
2851
+ failureReasons: batchFailureReasons
2852
+ } = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, drainAbort.signal);
2853
+ if (leaseLost || !renewGitSweepLease(db, projectIdentity, holderId)) {
2854
+ leaseLost = true;
2855
+ drainAbort.abort();
2856
+ aborted = true;
2857
+ break;
2858
+ }
2859
+ failureReasons.push(...batchFailureReasons);
2860
+ for (const id of noWork)
2861
+ skipIds.push(id);
2862
+ for (const id of failed)
2863
+ failedIds.push(id);
2864
+ if (n === 0 && noWork.length === 0) {
2865
+ consecutiveFailedBatches += 1;
2866
+ if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
2867
+ providerDown = true;
2868
+ break;
2869
+ }
2870
+ } else {
2871
+ consecutiveFailedBatches = 0;
2872
+ }
2873
+ embedded += n;
2874
+ options?.onProgress?.({ embedded: Math.min(embedded, total), total });
2875
+ await new Promise((resolve) => setTimeout(resolve, 15));
2876
+ }
2877
+ } finally {
2878
+ clearInterval(renewal);
2879
+ options?.signal?.removeEventListener("abort", forwardCallerAbort);
2880
+ try {
2881
+ releaseGitSweepLease(db, projectIdentity, holderId);
2882
+ } catch (error) {
2883
+ log2("[magic-context] embed drain: lease release failed (will TTL-expire):", error);
2884
+ }
2885
+ }
2886
+ if (total === 0)
2887
+ return { status: "nothing", embedded: 0, total: 0 };
2888
+ if (aborted)
2889
+ return { status: "aborted", embedded, total, failed: failedIds.length };
2890
+ if (providerDown || failedIds.length > 0) {
2891
+ const remaining = Math.max(failedIds.length, await countUnembeddedSessionCompartmentsPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, maxInputTokens) - skipIds.length);
2892
+ if (remaining > 0) {
2893
+ return {
2894
+ status: "stalled",
2895
+ embedded,
2896
+ total,
2897
+ remaining,
2898
+ failed: failedIds.length,
2899
+ failure: dominantEmbeddingFailure(failureReasons)
2900
+ };
2901
+ }
2902
+ }
2903
+ return { status: "done", embedded, total, failed: failedIds.length };
2904
+ }
2905
+ function getEmbeddingCoverageStatus2(db, projectIdentity, sessionId) {
2906
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2907
+ if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
2908
+ return {
2909
+ enabled: false,
2910
+ model: snapshot?.model ?? "off",
2911
+ provider: snapshot?.provider ?? "off",
2912
+ ...snapshot?.synapseDescriptor ? { synapseDescriptor: snapshot.synapseDescriptor } : {},
2913
+ session: { embedded: 0, total: 0 },
2914
+ memories: { embedded: 0, total: 0 },
2915
+ commits: { embedded: 0, total: 0, gitEnabled: false },
2916
+ shadowBackfillStalls: listShadowBackfillStalls2(db, projectIdentity)
2917
+ };
2918
+ }
2919
+ const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId, getProjectEmbeddingMaxInputTokens2(projectIdentity));
2920
+ const memories = snapshot.enabled ? getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId) : { embedded: 0, total: 0, memoryEnabled: false };
2921
+ const gitEnabled = snapshot.gitCommitEnabled;
2922
+ const commits = gitEnabled ? {
2923
+ embedded: countEmbeddedCommits(db, projectIdentity, snapshot.modelId),
2924
+ total: getCommitCount(db, projectIdentity),
2925
+ gitEnabled: true
2926
+ } : { embedded: 0, total: 0, gitEnabled: false };
2927
+ return {
2928
+ enabled: true,
2929
+ model: snapshot.model,
2930
+ provider: snapshot.provider,
2931
+ ...snapshot.synapseDescriptor ? { synapseDescriptor: snapshot.synapseDescriptor } : {},
2932
+ session,
2933
+ memories,
2934
+ commits,
2935
+ shadowBackfillStalls: listShadowBackfillStalls2(db, projectIdentity)
2936
+ };
2937
+ }
2938
+ async function getAutoEmbeddingSessionCoverage2(db, projectIdentity, sessionId) {
2939
+ const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
2940
+ if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
2941
+ return { enabled: false, embedded: 0, total: 0 };
2942
+ }
2943
+ const coverage = await countSessionCompartmentEmbedCoveragePolite(db, projectIdentity, sessionId, snapshot.chunkModelId, getProjectEmbeddingMaxInputTokens2(projectIdentity));
2944
+ return { enabled: true, ...coverage };
2945
+ }
2946
+ async function sweepAllRegisteredProjects2(db, batchSize = 10) {
2947
+ if (projectSweepInProgress || isEmbeddingHostBusy2()) {
2948
+ log2("[magic-context] project embedding sweep already in progress, skipping this tick");
2949
+ return {
2950
+ memoriesEmbedded: 0,
2951
+ commitsEmbedded: 0,
2952
+ chunksEmbedded: 0,
2953
+ perProject: new Map
2954
+ };
2955
+ }
2956
+ projectSweepInProgress = true;
2957
+ const startedAt = Date.now();
2958
+ const deadline = startedAt + SWEEP_MAX_WALL_CLOCK_MS;
2959
+ const perProject = new Map;
2960
+ let memoriesEmbedded = 0;
2961
+ let commitsEmbedded = 0;
2962
+ let chunksEmbedded = 0;
2963
+ try {
2964
+ for (const projectIdentity of projectRegistrations.keys()) {
2965
+ let memories = 0;
2966
+ let commits = 0;
2967
+ let chunks = 0;
2968
+ let consecutiveEmpty = 0;
2969
+ while (Date.now() < deadline && !isEmbeddingHostBusy2()) {
2970
+ const count = await embedUnembeddedMemoriesForProject2(db, projectIdentity, batchSize);
2971
+ if (count === 0) {
2972
+ consecutiveEmpty += 1;
2973
+ if (consecutiveEmpty >= SWEEP_MAX_CONSECUTIVE_EMPTY)
2974
+ break;
2975
+ break;
2976
+ }
2977
+ consecutiveEmpty = 0;
2978
+ memories += count;
2979
+ memoriesEmbedded += count;
2980
+ if (count < batchSize)
2981
+ break;
2982
+ }
2983
+ if (Date.now() < deadline) {
2984
+ commits = await drainCommitBacklogForProject2(db, projectIdentity, deadline);
2985
+ commitsEmbedded += commits;
2986
+ }
2987
+ if (Date.now() < deadline) {
2988
+ chunks = await drainCompartmentChunkBacklogForProject(db, projectIdentity, deadline);
2989
+ chunksEmbedded += chunks;
2990
+ }
2991
+ perProject.set(projectIdentity, { memories, commits, chunks });
2992
+ if (Date.now() >= deadline)
2993
+ break;
2994
+ }
2995
+ } finally {
2996
+ projectSweepInProgress = false;
2997
+ }
2998
+ return { memoriesEmbedded, commitsEmbedded, chunksEmbedded, perProject };
2999
+ }
3000
+ function _setTestProviderFactoryForProject2(factory) {
3001
+ testProviderFactory = factory;
3002
+ }
3003
+ function _resetProjectEmbeddingRegistryForTests2() {
3004
+ for (const registration of projectRegistrations.values()) {
3005
+ disposeProvider(registration.provider);
3006
+ }
3007
+ for (const registration of shadowRegistrations.values()) {
3008
+ disposeProvider(registration.provider);
3009
+ }
3010
+ projectRegistrations.clear();
3011
+ shadowRegistrations.clear();
3012
+ shadowQueue.length = 0;
3013
+ pendingShadowBackfills.clear();
3014
+ shadowBackfillLastIds.clear();
3015
+ shadowBackfillStopReasons.clear();
3016
+ shadowBackfillLastWriteOutcomes.clear();
3017
+ dbForShadowQueue.clear();
3018
+ untrustedLoadProjects.clear();
3019
+ globalRegistrationGeneration = 0;
3020
+ projectSweepInProgress = false;
3021
+ testProviderFactory = null;
3022
+ shadowBackfillNow = () => Date.now();
3023
+ setBootQuietPeriodForTests(null);
3024
+ }
3025
+
3026
+ export { recordSessionProjectIdentity, recordEmbeddingMeasurement, OpenAICompatibleEmbeddingProvider, drainBackgroundBatches, saveCommitEmbedding, loadProjectCommitEmbeddings, loadUnembeddedCommits, countEmbeddedCommits, upsertCommits, getLatestIndexedCommitSha, enforceProjectCap, GIT_SWEEP_LEASE_RENEWAL_MS, acquireGitSweepLease, renewGitSweepLease, markGitSweepSuccessAndRelease, parkGitSweepNonIndexable, releaseGitSweepLease, describeShadowBackfillWriteRefusal2, listShadowBackfillStalls2, getShadowBackfillStopReason2, markProjectLoadUntrusted2, contentSha2562, drainProjectEmbeddingIdentityMaintenance2, drainStaleEmbeddingIdentitiesForProject2, registerProjectEmbedding2, registerProjectShadowEmbedding2, getShadowEmbeddingMeasurementCohort2, getPrimaryEmbeddingMeasurementCohort2, embedShadowTextForProject2, enqueueShadowEmbeddingItems2, getShadowBackfillRemaining2, flushShadowEmbeddingBacklog2, registerProjectInObservationMode2, unregisterProjectShadowEmbedding2, unregisterProjectEmbedding2, getProjectEmbeddingSnapshot2, installProjectEmbeddingSearchBridge2, getProjectChunkEmbeddingModelId2, getProjectEmbeddingMaxInputTokens2, embedTextForProject2, embedBatchForProject2, embedItemsForProject2, embedUnembeddedMemoriesForProject2, drainCommitBacklogForProject2, embedUnembeddedCompartmentChunksForProject2, embedSessionCompartmentChunks2, getEmbeddingCoverageStatus2, getAutoEmbeddingSessionCoverage2, sweepAllRegisteredProjects2, _setTestProviderFactoryForProject2, _resetProjectEmbeddingRegistryForTests2 };