@cortexkit/pi-magic-context 0.49.11 → 0.49.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +4 -4
- package/dist/{compaction-marker-candidates-n4dphyz4.js → compaction-marker-candidates-k4tqg4jb.js} +2 -2
- package/dist/compaction-marker-candidates-worker.js +1 -1
- package/dist/embedding-backfill-14tqcfrn.js +20 -0
- package/dist/embedding-worker-process.js +1 -1
- package/dist/embedding-worker.js +5 -5
- package/dist/historian-calibration-extension.js +1 -1
- package/dist/{index-ta0ed2nc.js → index-1k729e4f.js} +15 -15
- package/dist/{index-g13fb2sj.js → index-3cnarwf8.js} +4 -4
- package/dist/{index-729kkhmf.js → index-46q4e104.js} +1 -1
- package/dist/{index-m8aaepft.js → index-53xdqb3d.js} +7 -7
- package/dist/{index-szh001qb.js → index-6dz2ed1v.js} +2 -2
- package/dist/{index-gqxsds3q.js → index-6ex901g3.js} +187 -27
- package/dist/{index-s3c9dskk.js → index-6hex51s9.js} +8 -8
- package/dist/{index-gycf3sp7.js → index-7bqrfyny.js} +46 -2
- package/dist/{index-drah07tf.js → index-af9nb5db.js} +5 -5
- package/dist/{index-3veejgp8.js → index-apgwdby5.js} +1 -1
- package/dist/{index-43zqh10z.js → index-d4dt19md.js} +1 -1
- package/dist/{index-fdxzpnbj.js → index-djxc8w8c.js} +1 -1
- package/dist/{index-x7426vy4.js → index-ed9d2ded.js} +18 -5
- package/dist/{index-kh3jt7t9.js → index-f18ec69t.js} +1 -1
- package/dist/{index-78c864c8.js → index-f6e8jsrh.js} +140 -17
- package/dist/{index-4pxydwvg.js → index-g4jf772p.js} +1 -1
- package/dist/{index-2df03yc3.js → index-gykw2zhn.js} +8 -8
- package/dist/{index-4vj37yw9.js → index-he9s4qsj.js} +16 -6
- package/dist/{index-8yjhda46.js → index-j24916mq.js} +3 -3
- package/dist/{index-ttxsmaj5.js → index-jrap43gc.js} +1 -1
- package/dist/{index-s9kwv5pp.js → index-xpsht877.js} +5 -5
- package/dist/{index-9azmm7cg.js → index-z5ftst3e.js} +9 -9
- package/dist/{index-0vq6ebae.js → index-zapgkmp7.js} +29 -24
- package/dist/index.js +340 -203
- package/dist/{logger-qg51hmp7.js → logger-dygt96rn.js} +1 -1
- package/dist/{message-index-bounded-chgr6dn3.js → message-index-bounded-ws37m1nt.js} +13 -13
- package/dist/{message-index-5zjmsg57.js → message-index-dfs6k35a.js} +13 -13
- package/dist/{message-time-backfill-q9s88zdv.js → message-time-backfill-r2ar1qvk.js} +2 -2
- package/dist/migration-worker.js +9 -9
- package/dist/{project-embedding-registry-dk5axctd.js → project-embedding-registry-ymvne11v.js} +10 -10
- package/dist/raw-ordinal-warmup-worker.js +12 -7
- package/dist/{read-session-chunk-3kb7r5pq.js → read-session-chunk-83864b37.js} +8 -8
- package/dist/{read-session-raw-hm7ptnrh.js → read-session-raw-azkqjgzg.js} +6 -2
- package/dist/{safe-notification-target-62ty6rvp.js → safe-notification-target-2rkvkmw8.js} +2 -2
- package/dist/{search-8pczyejh.js → search-85e21wb8.js} +19 -19
- package/dist/{storage-tags-zh23ynsh.js → storage-tags-zmf30zqp.js} +6 -4
- package/dist/subagent-entry.js +22 -22
- package/package.json +1 -1
- package/dist/embedding-backfill-1tknh2jn.js +0 -20
|
@@ -4,11 +4,11 @@ import {
|
|
|
4
4
|
import {
|
|
5
5
|
sanitizeDiagnosticText,
|
|
6
6
|
log2
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-7bqrfyny.js";
|
|
8
8
|
import {
|
|
9
9
|
isUserHomeDirectory,
|
|
10
10
|
setBootQuietPeriodForTests
|
|
11
|
-
} from "./index-
|
|
11
|
+
} from "./index-6ex901g3.js";
|
|
12
12
|
import {
|
|
13
13
|
inRootAsyncContext2,
|
|
14
14
|
withSqliteBackgroundWriter2,
|
|
@@ -17,14 +17,15 @@ import {
|
|
|
17
17
|
} from "./index-asrxgdt4.js";
|
|
18
18
|
import {
|
|
19
19
|
estimateTokens
|
|
20
|
-
} from "./index-
|
|
20
|
+
} from "./index-g4jf772p.js";
|
|
21
21
|
import {
|
|
22
22
|
SYNAPSE_DEFAULT_MODEL,
|
|
23
23
|
isSynapseEmbeddingTruncated,
|
|
24
24
|
getSynapseBatchRequestKey,
|
|
25
25
|
SynapseEmbeddingProvider
|
|
26
|
-
} from "./index-
|
|
26
|
+
} from "./index-6dz2ed1v.js";
|
|
27
27
|
import {
|
|
28
|
+
DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS,
|
|
28
29
|
CHUNK_WINDOW_SAFETY_RATIO,
|
|
29
30
|
normalizeCompartmentChunkMaxInputTokens,
|
|
30
31
|
buildCanonicalChunkTextFromFts,
|
|
@@ -40,17 +41,17 @@ import {
|
|
|
40
41
|
countSessionCompartmentEmbedCoverage,
|
|
41
42
|
countSessionCompartmentEmbedCoveragePolite,
|
|
42
43
|
countUnembeddedSessionCompartmentsPolite
|
|
43
|
-
} from "./index-
|
|
44
|
+
} from "./index-af9nb5db.js";
|
|
44
45
|
import {
|
|
45
46
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
46
|
-
} from "./index-
|
|
47
|
+
} from "./index-1k729e4f.js";
|
|
47
48
|
import {
|
|
48
49
|
embeddingModelsMatch,
|
|
49
50
|
resolveEmbeddingTextPrefixes,
|
|
50
51
|
getEmbeddingProviderIdentity,
|
|
51
52
|
dominantEmbeddingFailure,
|
|
52
53
|
LocalEmbeddingProvider
|
|
53
|
-
} from "./index-
|
|
54
|
+
} from "./index-j24916mq.js";
|
|
54
55
|
import {
|
|
55
56
|
isEmbeddingHostBusy2
|
|
56
57
|
} from "./index-9p40dgkm.js";
|
|
@@ -61,7 +62,7 @@ import {
|
|
|
61
62
|
} from "./index-yhzhdp3m.js";
|
|
62
63
|
import {
|
|
63
64
|
logSlowWriteTransaction
|
|
64
|
-
} from "./index-
|
|
65
|
+
} from "./index-d4dt19md.js";
|
|
65
66
|
|
|
66
67
|
// ../plugin/src/features/magic-context/session-project-storage.ts
|
|
67
68
|
var SESSION_CHUNK_REPAIR_BATCH_SIZE = 100;
|
|
@@ -219,6 +220,9 @@ function pruneSynapseBatchLedgerForProject(db, projectIdentity, ttlMs = SYNAPSE_
|
|
|
219
220
|
return db.prepare(`DELETE FROM synapse_batch_ledger WHERE ${predicate}`).run(projectIdentity, shadow, cutoff).changes;
|
|
220
221
|
}
|
|
221
222
|
|
|
223
|
+
// ../plugin/src/features/magic-context/memory/embedding-openai.ts
|
|
224
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
225
|
+
|
|
222
226
|
// ../plugin/src/features/magic-context/memory/embedding-ssrf.ts
|
|
223
227
|
var METADATA_HOSTNAMES = new Set(["metadata.google.internal", "metadata.goog"]);
|
|
224
228
|
var IPV6_METADATA_HOSTS = new Set(["fd00:ec2::254"]);
|
|
@@ -318,6 +322,55 @@ var FAILURE_THRESHOLD = 3;
|
|
|
318
322
|
var FAILURE_WINDOW_MS = 60000;
|
|
319
323
|
var OPEN_DURATION_MS = 5 * 60000;
|
|
320
324
|
var FETCH_TIMEOUT_MS = 30000;
|
|
325
|
+
var RETRY_AFTER_CAP_MS = 60000;
|
|
326
|
+
var RATE_LIMIT_BACKOFF_BASE_MS = 1000;
|
|
327
|
+
var RATE_LIMIT_BACKOFF_CAP_MS = 30000;
|
|
328
|
+
var RATE_LIMIT_BACKOFF_JITTER = 0.25;
|
|
329
|
+
var rateLimitStates = new Map;
|
|
330
|
+
function rateLimitStateKey(endpoint, apiKey) {
|
|
331
|
+
const keyHash = apiKey ? createHash2("sha256").update(apiKey).digest("hex").slice(0, 16) : "";
|
|
332
|
+
return `${endpoint}\x00${keyHash}`;
|
|
333
|
+
}
|
|
334
|
+
function parseRetryAfterMs(value, nowMs) {
|
|
335
|
+
const trimmed = value?.trim() ?? "";
|
|
336
|
+
if (!trimmed)
|
|
337
|
+
return null;
|
|
338
|
+
let delayMs;
|
|
339
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
340
|
+
delayMs = Number(trimmed) * 1000;
|
|
341
|
+
} else {
|
|
342
|
+
const at = Date.parse(trimmed);
|
|
343
|
+
if (!Number.isFinite(at))
|
|
344
|
+
return null;
|
|
345
|
+
delayMs = at - nowMs;
|
|
346
|
+
}
|
|
347
|
+
if (!(delayMs > 0))
|
|
348
|
+
return null;
|
|
349
|
+
return Math.min(Math.ceil(delayMs), RETRY_AFTER_CAP_MS);
|
|
350
|
+
}
|
|
351
|
+
function rateLimitBackoffMs(consecutive, random = Math.random) {
|
|
352
|
+
const exponent = Math.min(Math.max(0, Math.floor(consecutive)), 30);
|
|
353
|
+
const nominal = Math.min(RATE_LIMIT_BACKOFF_CAP_MS, RATE_LIMIT_BACKOFF_BASE_MS * 2 ** exponent);
|
|
354
|
+
const jitter = Math.min(1, Math.max(0, random())) * RATE_LIMIT_BACKOFF_JITTER;
|
|
355
|
+
return Math.min(RATE_LIMIT_BACKOFF_CAP_MS, Math.round(nominal * (1 + jitter)));
|
|
356
|
+
}
|
|
357
|
+
function sleepUnlessAborted(ms, signal) {
|
|
358
|
+
if (signal?.aborted)
|
|
359
|
+
return Promise.resolve(false);
|
|
360
|
+
return new Promise((resolve) => {
|
|
361
|
+
const onAbort = () => {
|
|
362
|
+
clearTimeout(timer);
|
|
363
|
+
resolve(false);
|
|
364
|
+
};
|
|
365
|
+
const timer = setTimeout(() => {
|
|
366
|
+
signal?.removeEventListener("abort", onAbort);
|
|
367
|
+
resolve(true);
|
|
368
|
+
}, ms);
|
|
369
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
var RATE_LIMITED = Symbol("rate-limited");
|
|
373
|
+
var maxInputTokensWarnings = new Set;
|
|
321
374
|
|
|
322
375
|
class OpenAICompatibleEmbeddingProvider {
|
|
323
376
|
modelId;
|
|
@@ -338,6 +391,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
338
391
|
lastFailureReason = null;
|
|
339
392
|
halfOpenProbeInFlight = false;
|
|
340
393
|
queryTruncationLogged = false;
|
|
394
|
+
rateLimitKey;
|
|
341
395
|
constructor(options) {
|
|
342
396
|
this.endpoint = normalizeEndpoint(options.endpoint);
|
|
343
397
|
this.model = options.model?.trim() ?? "";
|
|
@@ -348,7 +402,16 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
348
402
|
this.queryPrefix = prefixes.queryPrefix;
|
|
349
403
|
this.documentPrefix = prefixes.documentPrefix;
|
|
350
404
|
this.truncate = options.truncate?.trim() ?? "";
|
|
351
|
-
this.
|
|
405
|
+
this.rateLimitKey = rateLimitStateKey(this.endpoint, this.apiKey);
|
|
406
|
+
const maxInputTokensSet = typeof options.maxInputTokens === "number" && Number.isFinite(options.maxInputTokens);
|
|
407
|
+
this.maxInputTokens = maxInputTokensSet ? Math.max(1, Math.floor(options.maxInputTokens)) : DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS;
|
|
408
|
+
if (!maxInputTokensSet && this.endpoint && this.model) {
|
|
409
|
+
const warningKey = `${this.endpoint}\x00${this.model}`;
|
|
410
|
+
if (!maxInputTokensWarnings.has(warningKey)) {
|
|
411
|
+
maxInputTokensWarnings.add(warningKey);
|
|
412
|
+
log2(`[magic-context] embedding.max_input_tokens is not set for openai-compatible model "${sanitizeDiagnosticText(this.model)}"; chunk embeddings use the ${DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS}-token default, which is sized for the bundled local model and splits history into small pieces. Set embedding.max_input_tokens to the model's input limit (changing it re-embeds stored chunks once).`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
352
415
|
this.modelId = getEmbeddingProviderIdentity({
|
|
353
416
|
provider: "openai-compatible",
|
|
354
417
|
endpoint: this.endpoint,
|
|
@@ -406,6 +469,23 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
406
469
|
if (signal?.aborted) {
|
|
407
470
|
return Array.from({ length: texts.length }, () => null);
|
|
408
471
|
}
|
|
472
|
+
const isQuery = purpose === "query";
|
|
473
|
+
for (;; ) {
|
|
474
|
+
if (Date.now() < this.rateLimit.until) {
|
|
475
|
+
if (isQuery)
|
|
476
|
+
return Array.from({ length: texts.length }, () => null);
|
|
477
|
+
if (!await this.waitForRateLimitCooldown(signal)) {
|
|
478
|
+
return Array.from({ length: texts.length }, () => null);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
const outcome = await this.requestOnce(texts, requestTexts, signal, purpose);
|
|
482
|
+
if (outcome !== RATE_LIMITED)
|
|
483
|
+
return outcome;
|
|
484
|
+
if (isQuery)
|
|
485
|
+
return Array.from({ length: texts.length }, () => null);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
async requestOnce(texts, requestTexts, signal, purpose) {
|
|
409
489
|
let isProbe = false;
|
|
410
490
|
let internalController;
|
|
411
491
|
let timeoutHandle;
|
|
@@ -438,9 +518,14 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
438
518
|
redirect: "error",
|
|
439
519
|
signal: internalController.signal
|
|
440
520
|
});
|
|
521
|
+
if (response.status === 429) {
|
|
522
|
+
await response.text().catch(() => "");
|
|
523
|
+
this.noteRateLimit(response.headers.get("retry-after"));
|
|
524
|
+
return RATE_LIMITED;
|
|
525
|
+
}
|
|
441
526
|
if (!response.ok) {
|
|
442
527
|
const excerpt = await response.text().catch(() => "");
|
|
443
|
-
const failure = this.failure("http_error", `HTTP ${response.status} from endpoint${this.bodyExcerpt(excerpt)}`, response.status >= 500 || response.status === 408
|
|
528
|
+
const failure = this.failure("http_error", `HTTP ${response.status} from endpoint${this.bodyExcerpt(excerpt)}`, response.status >= 500 || response.status === 408);
|
|
444
529
|
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
445
530
|
this.recordFailure(isProbe, failure);
|
|
446
531
|
return Array.from({ length: texts.length }, () => null);
|
|
@@ -539,6 +624,36 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
539
624
|
isLoaded() {
|
|
540
625
|
return this.initialized;
|
|
541
626
|
}
|
|
627
|
+
get rateLimit() {
|
|
628
|
+
let state = rateLimitStates.get(this.rateLimitKey);
|
|
629
|
+
if (!state) {
|
|
630
|
+
state = { until: 0, consecutive: 0 };
|
|
631
|
+
rateLimitStates.set(this.rateLimitKey, state);
|
|
632
|
+
}
|
|
633
|
+
return state;
|
|
634
|
+
}
|
|
635
|
+
async waitForRateLimitCooldown(signal) {
|
|
636
|
+
for (;; ) {
|
|
637
|
+
const remaining = this.rateLimit.until - Date.now();
|
|
638
|
+
if (remaining <= 0)
|
|
639
|
+
return true;
|
|
640
|
+
if (!await sleepUnlessAborted(remaining, signal))
|
|
641
|
+
return false;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
noteRateLimit(retryAfter) {
|
|
645
|
+
const state = this.rateLimit;
|
|
646
|
+
const now = Date.now();
|
|
647
|
+
const headerDelayMs = parseRetryAfterMs(retryAfter, now);
|
|
648
|
+
const delayMs = headerDelayMs ?? rateLimitBackoffMs(state.consecutive);
|
|
649
|
+
state.consecutive += 1;
|
|
650
|
+
const alreadyCoolingDown = now < state.until;
|
|
651
|
+
state.until = Math.max(state.until, now + delayMs);
|
|
652
|
+
const seconds = Math.ceil(delayMs / 1000);
|
|
653
|
+
if (!alreadyCoolingDown) {
|
|
654
|
+
log2(`[magic-context] openai-compatible embedding rate limited (HTTP 429): pausing requests for ${seconds}s (${headerDelayMs !== null ? "Retry-After" : "backoff"}); background embedding resumes after the pause and searches run lexical-only until then`);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
542
657
|
claimProbeOrShortCircuit() {
|
|
543
658
|
if (this.circuitOpenUntil === 0) {
|
|
544
659
|
return "allow";
|
|
@@ -593,6 +708,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
593
708
|
this.circuitOpenUntil = 0;
|
|
594
709
|
this.openLogged = false;
|
|
595
710
|
this.lastFailureReason = null;
|
|
711
|
+
this.rateLimit.consecutive = 0;
|
|
596
712
|
}
|
|
597
713
|
getLastFailureReason() {
|
|
598
714
|
return this.lastFailureReason;
|
|
@@ -608,6 +724,9 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
608
724
|
_getFailureCount() {
|
|
609
725
|
return this.failureTimes.length;
|
|
610
726
|
}
|
|
727
|
+
_getRateLimitedUntil() {
|
|
728
|
+
return this.rateLimit.until;
|
|
729
|
+
}
|
|
611
730
|
_resetCircuit() {
|
|
612
731
|
this.failureTimes = [];
|
|
613
732
|
this.circuitOpenUntil = 0;
|
|
@@ -618,7 +737,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
618
737
|
}
|
|
619
738
|
|
|
620
739
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
621
|
-
import { createHash as
|
|
740
|
+
import { createHash as createHash3, randomUUID } from "node:crypto";
|
|
622
741
|
|
|
623
742
|
// ../plugin/src/features/magic-context/git-commits/storage-git-commit-embeddings.ts
|
|
624
743
|
var saveStatements = new WeakMap;
|
|
@@ -1301,10 +1420,10 @@ function stableStringify(value) {
|
|
|
1301
1420
|
return JSON.stringify(value);
|
|
1302
1421
|
}
|
|
1303
1422
|
function sha256Prefix(value, length = 16) {
|
|
1304
|
-
return
|
|
1423
|
+
return createHash3("sha256").update(value).digest("hex").slice(0, length);
|
|
1305
1424
|
}
|
|
1306
1425
|
function contentSha2562(value) {
|
|
1307
|
-
return
|
|
1426
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
1308
1427
|
}
|
|
1309
1428
|
function getRuntimeFingerprint(config) {
|
|
1310
1429
|
if (config.provider === "off") {
|
|
@@ -2628,7 +2747,7 @@ async function drainCommitBacklogForProject2(db, projectIdentity, deadline) {
|
|
|
2628
2747
|
}
|
|
2629
2748
|
return total;
|
|
2630
2749
|
}
|
|
2631
|
-
async function embedCompartmentChunkBatch(db, projectIdentity, batchSize) {
|
|
2750
|
+
async function embedCompartmentChunkBatch(db, projectIdentity, batchSize, signal) {
|
|
2632
2751
|
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2633
2752
|
if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off")
|
|
2634
2753
|
return 0;
|
|
@@ -2636,7 +2755,7 @@ async function embedCompartmentChunkBatch(db, projectIdentity, batchSize) {
|
|
|
2636
2755
|
const candidates = await loadUnembeddedCompartmentChunkCandidatesPolite(db, projectIdentity, snapshot.chunkModelId, batchSize, getProjectEmbeddingMaxInputTokens2(projectIdentity), true);
|
|
2637
2756
|
if (candidates.length === 0)
|
|
2638
2757
|
return 0;
|
|
2639
|
-
const { embedded } = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates);
|
|
2758
|
+
const { embedded } = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, signal);
|
|
2640
2759
|
return embedded;
|
|
2641
2760
|
}
|
|
2642
2761
|
function getLastEmbeddingFailureForProject(projectIdentity) {
|
|
@@ -2773,6 +2892,9 @@ async function drainCompartmentChunkBacklogForProject(db, projectIdentity, deadl
|
|
|
2773
2892
|
}
|
|
2774
2893
|
let total = 0;
|
|
2775
2894
|
let leaseLost = false;
|
|
2895
|
+
const drainAbort = new AbortController;
|
|
2896
|
+
const deadlineTimer = setTimeout(() => drainAbort.abort(), Math.max(0, deadline - Date.now()));
|
|
2897
|
+
deadlineTimer.unref?.();
|
|
2776
2898
|
const renewal = inRootAsyncContext2(() => setInterval(() => {
|
|
2777
2899
|
try {
|
|
2778
2900
|
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
@@ -2781,8 +2903,8 @@ async function drainCompartmentChunkBacklogForProject(db, projectIdentity, deadl
|
|
|
2781
2903
|
}, SESSION_EMBED_LEASE_RENEWAL_MS));
|
|
2782
2904
|
renewal.unref?.();
|
|
2783
2905
|
try {
|
|
2784
|
-
while (!leaseLost && !isEmbeddingHostBusy2() && Date.now() < deadline && total < CHUNK_DRAIN_MAX_PER_SWEEP) {
|
|
2785
|
-
const embedded = await embedCompartmentChunkBatch(db, projectIdentity, CHUNK_DRAIN_BATCH_SIZE);
|
|
2906
|
+
while (!leaseLost && !drainAbort.signal.aborted && !isEmbeddingHostBusy2() && Date.now() < deadline && total < CHUNK_DRAIN_MAX_PER_SWEEP) {
|
|
2907
|
+
const embedded = await embedCompartmentChunkBatch(db, projectIdentity, CHUNK_DRAIN_BATCH_SIZE, drainAbort.signal);
|
|
2786
2908
|
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
2787
2909
|
leaseLost = true;
|
|
2788
2910
|
if (leaseLost || embedded === 0)
|
|
@@ -2792,6 +2914,7 @@ async function drainCompartmentChunkBacklogForProject(db, projectIdentity, deadl
|
|
|
2792
2914
|
break;
|
|
2793
2915
|
}
|
|
2794
2916
|
} finally {
|
|
2917
|
+
clearTimeout(deadlineTimer);
|
|
2795
2918
|
clearInterval(renewal);
|
|
2796
2919
|
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
2797
2920
|
}
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
sanitizeDiagnosticText,
|
|
15
15
|
log2,
|
|
16
16
|
sessionLog2
|
|
17
|
-
} from "./index-
|
|
17
|
+
} from "./index-7bqrfyny.js";
|
|
18
18
|
import {
|
|
19
19
|
resolveWorkspaceShareCategories,
|
|
20
20
|
resolveWorkspaceIdentitySet,
|
|
@@ -22,17 +22,17 @@ import {
|
|
|
22
22
|
resolveStoredPathWorkspaceIdentity,
|
|
23
23
|
sourceNameForMemory,
|
|
24
24
|
managedAuthorityNoteRow
|
|
25
|
-
} from "./index-
|
|
25
|
+
} from "./index-3cnarwf8.js";
|
|
26
26
|
import {
|
|
27
27
|
isKnownAutocommit
|
|
28
|
-
} from "./index-
|
|
28
|
+
} from "./index-jrap43gc.js";
|
|
29
29
|
import {
|
|
30
30
|
SubcClient
|
|
31
|
-
} from "./index-
|
|
31
|
+
} from "./index-6dz2ed1v.js";
|
|
32
32
|
import {
|
|
33
33
|
buildCanonicalChunkTextFromFts,
|
|
34
34
|
loadCompartmentChunkEmbeddingsForSearch
|
|
35
|
-
} from "./index-
|
|
35
|
+
} from "./index-af9nb5db.js";
|
|
36
36
|
import {
|
|
37
37
|
recordEmbeddingMeasurement,
|
|
38
38
|
saveCommitEmbedding,
|
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
getProjectEmbeddingSnapshot2,
|
|
51
51
|
embedTextForProject2,
|
|
52
52
|
embedItemsForProject2
|
|
53
|
-
} from "./index-
|
|
53
|
+
} from "./index-f6e8jsrh.js";
|
|
54
54
|
import {
|
|
55
55
|
saveEmbeddingIfHashMatches,
|
|
56
56
|
getProjectEmbeddings2
|
|
@@ -60,7 +60,7 @@ import {
|
|
|
60
60
|
isEmbeddingEnabled,
|
|
61
61
|
embedText,
|
|
62
62
|
ensureMemoryEmbeddings2
|
|
63
|
-
} from "./index-
|
|
63
|
+
} from "./index-xpsht877.js";
|
|
64
64
|
import {
|
|
65
65
|
getMemorySelectColumns2,
|
|
66
66
|
isMemoryRow2,
|
|
@@ -77,7 +77,7 @@ import {
|
|
|
77
77
|
} from "./index-hy44m6jc.js";
|
|
78
78
|
import {
|
|
79
79
|
getIndexedMessageCorpusSize2
|
|
80
|
-
} from "./index-
|
|
80
|
+
} from "./index-6hex51s9.js";
|
|
81
81
|
import {
|
|
82
82
|
computeNormalizedHash
|
|
83
83
|
} from "./index-9x1f96r8.js";
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
} from "./index-399p688g.js";
|
|
9
9
|
import {
|
|
10
10
|
log2
|
|
11
|
-
} from "./index-
|
|
11
|
+
} from "./index-7bqrfyny.js";
|
|
12
12
|
|
|
13
13
|
// ../plugin/src/hooks/magic-context/read-session-raw.ts
|
|
14
14
|
import { isMainThread } from "node:worker_threads";
|
|
@@ -91,6 +91,8 @@ function runSessionWarmupWorker(input, options) {
|
|
|
91
91
|
finish({ inserted: reply.inserted, measurements: reply.measurements });
|
|
92
92
|
else if (reply.kind === "orphan-sweep")
|
|
93
93
|
finish({ orphanSweep: reply.result });
|
|
94
|
+
else if (reply.kind === "exact-ordinal-ids")
|
|
95
|
+
finish({ exactOrdinalIds: reply.ids });
|
|
94
96
|
else
|
|
95
97
|
finish({ failure: "worker-error", detail: reply.error });
|
|
96
98
|
});
|
|
@@ -316,7 +318,7 @@ function readRawMessagePageRows(db, sessionId, afterOrdinal, limit, finalWaterma
|
|
|
316
318
|
const skip = anchor === after && after ? 0 : ordinal - (floor?.ordinal ?? 0);
|
|
317
319
|
if (skip > ORDINAL_LOOKUP_LIMIT) {
|
|
318
320
|
prewarmRawSessionOrdinalsForDb(db, sessionId);
|
|
319
|
-
throw new
|
|
321
|
+
throw new RawOrdinalsNotReady2;
|
|
320
322
|
}
|
|
321
323
|
const rawLimit = skip + pageSize + ORDINAL_LOOKUP_LIMIT;
|
|
322
324
|
const rows = db.prepare(`SELECT id, data, time_created, time_updated, ${EXCLUDED_ROW_SQL} AS excluded
|
|
@@ -329,7 +331,7 @@ function readRawMessagePageRows(db, sessionId, afterOrdinal, limit, finalWaterma
|
|
|
329
331
|
const eligible = rows.filter((row) => row.excluded !== 1);
|
|
330
332
|
if (eligible.length <= skip && rows.length === rawLimit) {
|
|
331
333
|
prewarmRawSessionOrdinalsForDb(db, sessionId);
|
|
332
|
-
throw new
|
|
334
|
+
throw new RawOrdinalsNotReady2;
|
|
333
335
|
}
|
|
334
336
|
return eligible.slice(skip, skip + pageSize).filter(isRawMessageRow).map((row, index) => ({
|
|
335
337
|
...row,
|
|
@@ -417,11 +419,13 @@ function countRawSessionMessageOrdinalsFromDb2(db, sessionId) {
|
|
|
417
419
|
)`).get(sessionId);
|
|
418
420
|
return typeof row?.count === "number" ? row.count : 0;
|
|
419
421
|
}
|
|
422
|
+
var exactOrdinalScansOnThisThread = 0;
|
|
420
423
|
function readRawSessionMessageIdOrdinalsForRangeFromDb2(db, sessionId, fromOrdinal, toOrdinal) {
|
|
421
424
|
const from = Math.max(1, Math.floor(fromOrdinal));
|
|
422
425
|
const to = Math.floor(toOrdinal);
|
|
423
426
|
if (to < from)
|
|
424
427
|
return new Map;
|
|
428
|
+
exactOrdinalScansOnThisThread += 1;
|
|
425
429
|
const rows = db.prepare(`
|
|
426
430
|
SELECT id, data FROM message WHERE session_id = ?
|
|
427
431
|
AND CASE WHEN json_valid(data) THEN NOT (
|
|
@@ -437,6 +441,12 @@ function readRawSessionMessageIdOrdinalsForRangeFromDb2(db, sessionId, fromOrdin
|
|
|
437
441
|
}
|
|
438
442
|
return result;
|
|
439
443
|
}
|
|
444
|
+
function readRawSessionMessageIdsAtOrdinalsFromDb2(db, sessionId, ordinals) {
|
|
445
|
+
return withReadSnapshot(db, () => ordinals.map((ordinal) => [
|
|
446
|
+
ordinal,
|
|
447
|
+
readRawSessionMessageIdOrdinalsForRangeFromDb2(db, sessionId, ordinal, ordinal).keys().next().value ?? null
|
|
448
|
+
]));
|
|
449
|
+
}
|
|
440
450
|
function readRawSessionMessageOrdinalPageFromDb2(db, sessionId, after, limit) {
|
|
441
451
|
const pageSize = Math.max(1, Math.floor(limit));
|
|
442
452
|
const rows = (after ? db.prepare(`SELECT id, data, time_created
|
|
@@ -733,7 +743,7 @@ function extendOrdinalCheckpoints(db, sessionId, scan, limit) {
|
|
|
733
743
|
return true;
|
|
734
744
|
}
|
|
735
745
|
|
|
736
|
-
class
|
|
746
|
+
class RawOrdinalsNotReady2 extends Error {
|
|
737
747
|
constructor() {
|
|
738
748
|
super("canonical ordinals are warming off-thread; retry after preparation");
|
|
739
749
|
}
|
|
@@ -782,7 +792,7 @@ function boundedOrdinalPrefix(db, sessionId, through) {
|
|
|
782
792
|
ordinalJsonRowsRead += rows.length;
|
|
783
793
|
if (rows.length > ORDINAL_LOOKUP_LIMIT) {
|
|
784
794
|
prewarmRawSessionOrdinalsForDb(db, sessionId);
|
|
785
|
-
throw new
|
|
795
|
+
throw new RawOrdinalsNotReady2;
|
|
786
796
|
}
|
|
787
797
|
return (floor?.ordinal ?? 0) + rows.reduce((sum, row) => sum + (row.excluded === 1 ? 0 : 1), 0);
|
|
788
798
|
}
|
|
@@ -1245,4 +1255,4 @@ function readRawSessionBoundaryEndpointsFromDb2(db, sessionId, ids, blockIds = [
|
|
|
1245
1255
|
}
|
|
1246
1256
|
}
|
|
1247
1257
|
|
|
1248
|
-
export { runSessionWarmupWorker, prewarmRawSessionOrdinalsForDb, isStrictGapHealingMessage2, readRawSessionMessagesFromDb2, readRawSessionMessagePageFromDb2, RAW_SUMMARY_TEXT_MAX_CHARS2, readRawSessionMessageSummaryPageFromDb2, countRawSessionMessageOrdinalsFromDb2, readRawSessionMessageIdOrdinalsForRangeFromDb2, readRawSessionMessageOrdinalPageFromDb2, countStoredRawSessionMessagesFromDb2, readRawSessionTailFromDb2, buildInMemoryTailRawMessages2, readRawSessionMessagePartsByIdFromDb2, forgetRawSessionOrdinalWatermark2, forgetRawSessionSummaryRows2, noteRawSessionSummaryRowWritten2, rawOrdinalSummarySeed2, scanRawSessionSummaryRows2, getRawSessionSummaryEpoch2, installRawSessionSummaryScan2, isRawSessionSummaryWarm2, readRawSessionMessageOrdinalByIdFromDb2, readRawSessionMessageByIdFromDb2, hasRawSessionMessageByIdFromDb2, readRawSeedTailFromDb2, readRawSessionBoundaryEndpointsFromDb2 };
|
|
1258
|
+
export { runSessionWarmupWorker, prewarmRawSessionOrdinalsForDb, isStrictGapHealingMessage2, readRawSessionMessagesFromDb2, readRawSessionMessagePageFromDb2, RAW_SUMMARY_TEXT_MAX_CHARS2, readRawSessionMessageSummaryPageFromDb2, countRawSessionMessageOrdinalsFromDb2, readRawSessionMessageIdOrdinalsForRangeFromDb2, readRawSessionMessageIdsAtOrdinalsFromDb2, readRawSessionMessageOrdinalPageFromDb2, countStoredRawSessionMessagesFromDb2, readRawSessionTailFromDb2, buildInMemoryTailRawMessages2, readRawSessionMessagePartsByIdFromDb2, forgetRawSessionOrdinalWatermark2, forgetRawSessionSummaryRows2, noteRawSessionSummaryRowWritten2, RawOrdinalsNotReady2, rawOrdinalSummarySeed2, scanRawSessionSummaryRows2, getRawSessionSummaryEpoch2, installRawSessionSummaryScan2, isRawSessionSummaryWarm2, readRawSessionMessageOrdinalByIdFromDb2, readRawSessionMessageByIdFromDb2, hasRawSessionMessageByIdFromDb2, readRawSeedTailFromDb2, readRawSessionBoundaryEndpointsFromDb2 };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
3
|
-
} from "./index-
|
|
3
|
+
} from "./index-1k729e4f.js";
|
|
4
4
|
import {
|
|
5
5
|
getSynapseLaneIdentity
|
|
6
|
-
} from "./index-
|
|
6
|
+
} from "./index-6dz2ed1v.js";
|
|
7
7
|
import {
|
|
8
8
|
getHarness2
|
|
9
9
|
} from "./index-0h2mkab2.js";
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
getMagicContextStorageDir,
|
|
12
12
|
importPluginModule,
|
|
13
13
|
log2
|
|
14
|
-
} from "./index-
|
|
14
|
+
} from "./index-7bqrfyny.js";
|
|
15
15
|
import {
|
|
16
16
|
inRootAsyncContext2
|
|
17
17
|
} from "./index-asrxgdt4.js";
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
3
|
-
} from "./index-
|
|
3
|
+
} from "./index-1k729e4f.js";
|
|
4
4
|
import {
|
|
5
5
|
log2
|
|
6
|
-
} from "./index-
|
|
6
|
+
} from "./index-7bqrfyny.js";
|
|
7
7
|
import {
|
|
8
8
|
isSynapseEmbeddingTruncated,
|
|
9
9
|
SynapseEmbeddingProvider
|
|
10
|
-
} from "./index-
|
|
10
|
+
} from "./index-6dz2ed1v.js";
|
|
11
11
|
import {
|
|
12
12
|
LocalEmbeddingProvider
|
|
13
|
-
} from "./index-
|
|
13
|
+
} from "./index-j24916mq.js";
|
|
14
14
|
import {
|
|
15
15
|
OpenAICompatibleEmbeddingProvider,
|
|
16
16
|
getProjectEmbeddingSnapshot2,
|
|
17
17
|
embedBatchForProject2
|
|
18
|
-
} from "./index-
|
|
18
|
+
} from "./index-f6e8jsrh.js";
|
|
19
19
|
import {
|
|
20
20
|
isEmbeddingHostBusy2
|
|
21
21
|
} from "./index-9p40dgkm.js";
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
importPluginModule,
|
|
6
6
|
writeForwardedLogLine2,
|
|
7
7
|
log2
|
|
8
|
-
} from "./index-
|
|
8
|
+
} from "./index-7bqrfyny.js";
|
|
9
9
|
import {
|
|
10
10
|
inRootAsyncContext2,
|
|
11
11
|
registerSlowWriteReporter2,
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import {
|
|
21
21
|
bootQuietRemainingMs,
|
|
22
22
|
scheduleAfterBootQuiet
|
|
23
|
-
} from "./index-
|
|
23
|
+
} from "./index-6ex901g3.js";
|
|
24
24
|
import {
|
|
25
25
|
getHarness2,
|
|
26
26
|
harnessOwnsOpenCodeStore2
|
|
@@ -35,13 +35,13 @@ import {
|
|
|
35
35
|
parseRpcPortFile,
|
|
36
36
|
inspectProcessesAsync,
|
|
37
37
|
inspectWindowsProcessesSync
|
|
38
|
-
} from "./index-
|
|
38
|
+
} from "./index-apgwdby5.js";
|
|
39
39
|
import {
|
|
40
40
|
configureContextDatabasePragmas2
|
|
41
41
|
} from "./index-fapenpch.js";
|
|
42
42
|
import {
|
|
43
43
|
startMessageFtsRowidMapBackfill
|
|
44
|
-
} from "./index-
|
|
44
|
+
} from "./index-djxc8w8c.js";
|
|
45
45
|
import {
|
|
46
46
|
LKG_SLOT_CHUNKS_DDL,
|
|
47
47
|
SESSION_REPLAY_DECISIONS_DDL,
|
|
@@ -51,13 +51,13 @@ import {
|
|
|
51
51
|
hasPendingMigrations,
|
|
52
52
|
getMainThreadMigrationBodyCount,
|
|
53
53
|
runMigrations
|
|
54
|
-
} from "./index-
|
|
54
|
+
} from "./index-3cnarwf8.js";
|
|
55
55
|
import {
|
|
56
56
|
closeQuietly,
|
|
57
57
|
ensureColumn,
|
|
58
58
|
installV95PerfSchema,
|
|
59
59
|
installTagIdentityRevisionTrigger
|
|
60
|
-
} from "./index-
|
|
60
|
+
} from "./index-jrap43gc.js";
|
|
61
61
|
import {
|
|
62
62
|
resolveOpenCodeDbPath,
|
|
63
63
|
hasV1MessageTables,
|
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
} from "./index-8976mezv.js";
|
|
67
67
|
import {
|
|
68
68
|
logSlowWriteTransaction
|
|
69
|
-
} from "./index-
|
|
69
|
+
} from "./index-d4dt19md.js";
|
|
70
70
|
import {
|
|
71
71
|
shouldEnforcePrivateStoragePermissions,
|
|
72
72
|
ensureStorageDirectorySync,
|
|
@@ -1298,8 +1298,8 @@ function finishDatabaseOpen(db, dbPath, explicitDbPath, latestSupportedVersion)
|
|
|
1298
1298
|
startMessageFtsRowidMapBackfill(db).then(async () => {
|
|
1299
1299
|
if (!readsOpenCodeStore)
|
|
1300
1300
|
return;
|
|
1301
|
-
const { readRawSessionMessagePage, readRawSessionMessages } = await importPluginModule(() => import("./read-session-chunk-
|
|
1302
|
-
const { startMessageTimeBackfill } = await importPluginModule(() => import("./message-time-backfill-
|
|
1301
|
+
const { readRawSessionMessagePage, readRawSessionMessages } = await importPluginModule(() => import("./read-session-chunk-83864b37.js"));
|
|
1302
|
+
const { startMessageTimeBackfill } = await importPluginModule(() => import("./message-time-backfill-r2ar1qvk.js"));
|
|
1303
1303
|
await startMessageTimeBackfill(db, Object.assign(readRawSessionMessages, {
|
|
1304
1304
|
readPage: readRawSessionMessagePage
|
|
1305
1305
|
}));
|