@cortexkit/pi-magic-context 0.49.15 → 0.49.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +76 -63
- package/dist/chunk-embedding-write-worker.js +11 -9
- package/dist/{compaction-marker-candidates-k4tqg4jb.js → compaction-marker-candidates-wjzqbspm.js} +2 -2
- package/dist/compaction-marker-candidates-worker.js +2 -2
- package/dist/embedding-backfill-vwag34t0.js +23 -0
- package/dist/embedding-worker-process.js +1 -1
- package/dist/embedding-worker.js +5 -5
- package/dist/historian-calibration-extension.js +4 -1
- package/dist/index-1cvyaw9a.js +58 -0
- package/dist/{index-g4jf772p.js → index-1kcfwk9c.js} +22 -2
- package/dist/{index-9jvktm7k.js → index-2mmy81qt.js} +3 -3
- package/dist/{index-ccsscaap.js → index-469ttgzv.js} +38 -17
- package/dist/index-4knyh1kt.js +1387 -0
- package/dist/{index-75ah9413.js → index-5k2mshvq.js} +10 -10
- package/dist/{index-a4pk5yv9.js → index-8r2t8fv2.js} +853 -84
- package/dist/{index-0awvkeb5.js → index-9j809q6f.js} +32 -72
- package/dist/{index-1k729e4f.js → index-bxmf4dhr.js} +6 -6
- package/dist/{index-6dz2ed1v.js → index-d8gc0112.js} +2 -2
- package/dist/{index-1vw4f2pe.js → index-dvbjsh98.js} +6 -8
- package/dist/{index-hy44m6jc.js → index-e1rza8gp.js} +1 -1
- package/dist/{index-j0n09wtp.js → index-ej4eak9d.js} +9 -7
- package/dist/{index-hshd96pt.js → index-es03azap.js} +127 -2093
- package/dist/{index-ajfpyk0x.js → index-fa485mqs.js} +59 -1
- package/dist/{index-6ex901g3.js → index-g7p75667.js} +33 -4
- package/dist/{index-bet4sd3e.js → index-gdjv1nnd.js} +2 -2
- package/dist/{index-46q4e104.js → index-qpq6w67c.js} +1 -1
- package/dist/{index-asrxgdt4.js → index-rrwv9y2k.js} +8 -0
- package/dist/{index-1mrvw9en.js → index-svgkv4g2.js} +1 -1
- package/dist/{index-he9s4qsj.js → index-vktahpdj.js} +23 -3
- package/dist/{index-nwbhe6qv.js → index-vmcgew7x.js} +11 -8
- package/dist/index-wrr9rp9a.js +16 -0
- package/dist/{index-7qc73yk5.js → index-xnch62s6.js} +14 -8
- package/dist/{index-djxc8w8c.js → index-yygsh681.js} +25 -26
- package/dist/{index-7wagp5jp.js → index-z2rkv0mb.js} +4 -4
- package/dist/index-z7kbck29.js +62 -0
- package/dist/index.js +688 -339
- package/dist/message-fts-rowid-map-cvpy2dmb.js +24 -0
- package/dist/{message-index-dw4z9efj.js → message-index-2g23b7fe.js} +13 -13
- package/dist/{message-index-bounded-z4d3x9ty.js → message-index-bounded-d6zj2tn5.js} +15 -15
- package/dist/{message-time-backfill-r2ar1qvk.js → message-time-backfill-ssshbdy8.js} +5 -5
- package/dist/migration-worker.js +8 -6
- package/dist/{project-embedding-registry-52zwmtbn.js → project-embedding-registry-vf1ncptn.js} +12 -10
- package/dist/raw-ordinal-warmup-worker.js +40 -10
- package/dist/{read-session-chunk-0fsgg2k1.js → read-session-chunk-2ky7mbqp.js} +7 -7
- package/dist/{read-session-raw-azkqjgzg.js → read-session-raw-9s3xn7fc.js} +2 -2
- package/dist/search-archive-9akqcr90.js +20 -0
- package/dist/{search-ddbz6v60.js → search-zjyc65g1.js} +20 -19
- package/dist/{sqlite-adjpy399.js → sqlite-fsp0sf3r.js} +1 -1
- package/dist/{storage-memory-axnsk5t1.js → storage-memory-yxxc5v4n.js} +2 -2
- package/dist/{storage-tags-3kb5tvgg.js → storage-tags-cx6ccsv1.js} +3 -3
- package/dist/subagent-entry.js +25 -22
- package/dist/tag-snapshot-worker-pi.js +17 -20
- package/package.json +1 -1
- package/dist/embedding-backfill-7kj16fh9.js +0 -21
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseTree
|
|
3
|
+
} from "./index-4knyh1kt.js";
|
|
4
|
+
|
|
1
5
|
// ../plugin/src/agents/permissions.ts
|
|
2
6
|
var DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS = [
|
|
3
7
|
"read",
|
|
@@ -9,6 +13,37 @@ var DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS = [
|
|
|
9
13
|
"ctx_search"
|
|
10
14
|
];
|
|
11
15
|
|
|
16
|
+
// ../plugin/src/shared/historian-cache.ts
|
|
17
|
+
function withoutHistorianTailCache(body) {
|
|
18
|
+
const value = JSON.parse(body);
|
|
19
|
+
if (!value || typeof value.model !== "string" || typeof value.max_tokens !== "number" || !Array.isArray(value.system) || value.system.length === 0 || !value.system.every((part) => part?.type === "text" && typeof part.text === "string") || !value.system.some((part) => part.cache_control) || !Array.isArray(value.messages) || value.cache_control !== undefined)
|
|
20
|
+
return body;
|
|
21
|
+
const root = parseTree(body);
|
|
22
|
+
const property = (node, key) => node?.children?.find((child) => child.children?.[0]?.value === key)?.children?.[1];
|
|
23
|
+
const removals = [];
|
|
24
|
+
for (const message of property(root, "messages")?.children ?? []) {
|
|
25
|
+
for (const block of property(message, "content")?.children ?? []) {
|
|
26
|
+
if (block.type !== "object")
|
|
27
|
+
continue;
|
|
28
|
+
const fields = block.children ?? [];
|
|
29
|
+
fields.forEach((field, index) => {
|
|
30
|
+
if (field.children?.[0]?.value !== "cache_control")
|
|
31
|
+
return;
|
|
32
|
+
const next = fields[index + 1];
|
|
33
|
+
const previous = fields[index - 1];
|
|
34
|
+
removals.push(next ? [field.offset, next.offset] : [
|
|
35
|
+
previous ? previous.offset + previous.length : field.offset,
|
|
36
|
+
field.offset + field.length
|
|
37
|
+
]);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
for (const [start, end] of removals.sort((a, b) => b[0] - a[0])) {
|
|
42
|
+
body = body.slice(0, start) + body.slice(end);
|
|
43
|
+
}
|
|
44
|
+
return body;
|
|
45
|
+
}
|
|
46
|
+
|
|
12
47
|
// ../plugin/src/agents/hidden-agent-registrations.ts
|
|
13
48
|
var HIDDEN_AGENT_DESCRIPTION = "Internal Magic Context maintenance agent. Not for general tasks — do not select for user work.";
|
|
14
49
|
function buildHiddenAgentRegistrations(args) {
|
|
@@ -371,6 +406,7 @@ class SubagentTelemetry {
|
|
|
371
406
|
|
|
372
407
|
// src/historian-calibration-extension.ts
|
|
373
408
|
var HISTORIAN_TEMPERATURE_ENV2 = "MAGIC_CONTEXT_HISTORIAN_TEMPERATURE";
|
|
409
|
+
var HISTORIAN_CACHE_TAIL_ENV2 = "MAGIC_CONTEXT_HISTORIAN_CACHE_TAIL";
|
|
374
410
|
var HISTORIAN_MAX_OUTPUT_TOKENS_ENV2 = "MAGIC_CONTEXT_HISTORIAN_MAX_OUTPUT_TOKENS";
|
|
375
411
|
var SUBAGENT_OUTPUT_CEILING_ENV2 = "MAGIC_CONTEXT_SUBAGENT_OUTPUT_CEILING";
|
|
376
412
|
function finiteNumber(value) {
|
|
@@ -481,6 +517,28 @@ function withoutProviderTemperature2(payload) {
|
|
|
481
517
|
return clean;
|
|
482
518
|
}
|
|
483
519
|
function historianCalibrationExtension(pi) {
|
|
520
|
+
if (process.env[HISTORIAN_CACHE_TAIL_ENV2] === "skip") {
|
|
521
|
+
let requests = 0;
|
|
522
|
+
let warned = false;
|
|
523
|
+
pi.on("before_provider_request", (event) => {
|
|
524
|
+
if (requests++ > 0)
|
|
525
|
+
return;
|
|
526
|
+
try {
|
|
527
|
+
const original = JSON.stringify(event.payload);
|
|
528
|
+
const body = withoutHistorianTailCache(original);
|
|
529
|
+
return body === original ? undefined : JSON.parse(body);
|
|
530
|
+
} catch (error) {
|
|
531
|
+
if (!warned) {
|
|
532
|
+
warned = true;
|
|
533
|
+
try {
|
|
534
|
+
process.stderr.write(`Historian cache optimization skipped: ${String(error)}
|
|
535
|
+
`);
|
|
536
|
+
} catch {}
|
|
537
|
+
}
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
}
|
|
484
542
|
const intendedPath = process.env.MAGIC_CONTEXT_SUBAGENT_PROMPT_FILE;
|
|
485
543
|
if (intendedPath) {
|
|
486
544
|
const intended = readFileSync2(intendedPath, "utf8");
|
|
@@ -519,4 +577,4 @@ function historianCalibrationExtension(pi) {
|
|
|
519
577
|
});
|
|
520
578
|
}
|
|
521
579
|
|
|
522
|
-
export { DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS, buildHiddenAgentRegistrations, logTemperatureDecision, readHostSystemPrompt, subagentStepCap, SubagentTelemetry, HISTORIAN_TEMPERATURE_ENV2, HISTORIAN_MAX_OUTPUT_TOKENS_ENV2, SUBAGENT_OUTPUT_CEILING_ENV2, calibrateHistorianProviderPayload2, ceilProviderOutputTokens2, withoutProviderTemperature2, historianCalibrationExtension };
|
|
580
|
+
export { DREAMER_PRIMER_INVESTIGATOR_ALLOWED_TOOLS, buildHiddenAgentRegistrations, logTemperatureDecision, readHostSystemPrompt, subagentStepCap, SubagentTelemetry, HISTORIAN_TEMPERATURE_ENV2, HISTORIAN_CACHE_TAIL_ENV2, HISTORIAN_MAX_OUTPUT_TOKENS_ENV2, SUBAGENT_OUTPUT_CEILING_ENV2, calibrateHistorianProviderPayload2, ceilProviderOutputTokens2, withoutProviderTemperature2, historianCalibrationExtension };
|
|
@@ -4,10 +4,10 @@ import {
|
|
|
4
4
|
} from "./index-7bqrfyny.js";
|
|
5
5
|
import {
|
|
6
6
|
inRootAsyncContext2
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-rrwv9y2k.js";
|
|
8
8
|
import {
|
|
9
9
|
projectDirectoryKey
|
|
10
|
-
} from "./index-
|
|
10
|
+
} from "./index-wrr9rp9a.js";
|
|
11
11
|
import {
|
|
12
12
|
writeStorageFileAtomicSync
|
|
13
13
|
} from "./index-81nthk8j.js";
|
|
@@ -21,7 +21,7 @@ import path3 from "node:path";
|
|
|
21
21
|
|
|
22
22
|
// ../plugin/src/features/magic-context/memory/project-identity-cache.ts
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
|
-
import { readFileSync } from "node:fs";
|
|
24
|
+
import { promises as fs, readFileSync } from "node:fs";
|
|
25
25
|
import path from "node:path";
|
|
26
26
|
function cachePath(directory, storageDir = getMagicContextStorageDir()) {
|
|
27
27
|
const hash = createHash("sha256").update(projectDirectoryKey(directory)).digest("hex");
|
|
@@ -41,6 +41,35 @@ function readRememberedGitIdentity(directory, storageDir) {
|
|
|
41
41
|
} catch {}
|
|
42
42
|
return;
|
|
43
43
|
}
|
|
44
|
+
async function pruneMissingProjectIdentities(storageDir = getMagicContextStorageDir()) {
|
|
45
|
+
const directory = path.join(storageDir, "project-identities");
|
|
46
|
+
let removed = 0;
|
|
47
|
+
let entries;
|
|
48
|
+
try {
|
|
49
|
+
entries = await fs.readdir(directory);
|
|
50
|
+
} catch {
|
|
51
|
+
return 0;
|
|
52
|
+
}
|
|
53
|
+
for (const entry of entries) {
|
|
54
|
+
if (!/^[0-9a-f]{64}\.json$/.test(entry))
|
|
55
|
+
continue;
|
|
56
|
+
const file = path.join(directory, entry);
|
|
57
|
+
try {
|
|
58
|
+
const record = JSON.parse(await fs.readFile(file, "utf8"));
|
|
59
|
+
if (typeof record.directory !== "string" || !path.isAbsolute(record.directory) || cachePath(record.directory, storageDir) !== file || !/^git:[0-9a-f]{7,64}$/.test(record.identity))
|
|
60
|
+
continue;
|
|
61
|
+
try {
|
|
62
|
+
await fs.stat(record.directory);
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (error.code !== "ENOENT")
|
|
65
|
+
continue;
|
|
66
|
+
await fs.unlink(file);
|
|
67
|
+
removed++;
|
|
68
|
+
}
|
|
69
|
+
} catch {}
|
|
70
|
+
}
|
|
71
|
+
return removed;
|
|
72
|
+
}
|
|
44
73
|
|
|
45
74
|
// ../plugin/src/features/magic-context/memory/unborn-head.ts
|
|
46
75
|
import { readFileSync as readFileSync2, realpathSync, statSync } from "node:fs";
|
|
@@ -667,4 +696,4 @@ function setBootQuietPeriodForTests(startAtMs) {
|
|
|
667
696
|
bootQuietUntilMs = startAtMs === null ? 0 : startAtMs + BOOT_QUIET_MS;
|
|
668
697
|
}
|
|
669
698
|
|
|
670
|
-
export { setHomeProjectPermission, ProjectIdentityError, resolveProjectIdentityStrict, isUserHomeDirectory, resolveProjectIdentity, directoryHasGitMetadata, isUsableProjectIdentity, resolveProjectIdentityForSession, normalizeStoredProjectPath, storedPathBelongsToIdentity, beginBootQuietPeriod,
|
|
699
|
+
export { pruneMissingProjectIdentities, setHomeProjectPermission, ProjectIdentityError, resolveProjectIdentityStrict, isUserHomeDirectory, resolveProjectIdentity, directoryHasGitMetadata, isUsableProjectIdentity, resolveProjectIdentityForSession, normalizeStoredProjectPath, storedPathBelongsToIdentity, beginBootQuietPeriod, scheduleAfterBootQuiet, setBootQuietPeriodForTests };
|
|
@@ -16,11 +16,11 @@ import {
|
|
|
16
16
|
} from "./index-8976mezv.js";
|
|
17
17
|
import {
|
|
18
18
|
Database2
|
|
19
|
-
} from "./index-
|
|
19
|
+
} from "./index-rrwv9y2k.js";
|
|
20
20
|
import {
|
|
21
21
|
normalizeStoredProjectPath,
|
|
22
22
|
storedPathBelongsToIdentity
|
|
23
|
-
} from "./index-
|
|
23
|
+
} from "./index-g7p75667.js";
|
|
24
24
|
import {
|
|
25
25
|
logSlowWriteTransaction
|
|
26
26
|
} from "./index-d4dt19md.js";
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
import {
|
|
8
8
|
inRootAsyncContext2,
|
|
9
9
|
getSqliteDatabasePath2
|
|
10
|
-
} from "./index-
|
|
10
|
+
} from "./index-rrwv9y2k.js";
|
|
11
11
|
|
|
12
12
|
// ../plugin/src/features/magic-context/compaction-marker-candidates.ts
|
|
13
13
|
import { existsSync, realpathSync } from "node:fs";
|
|
@@ -28,6 +28,14 @@ function detectSqliteRuntime2() {
|
|
|
28
28
|
var bunSpec = "bun:" + "sqlite";
|
|
29
29
|
var nodeSpec = "node:" + "sqlite";
|
|
30
30
|
async function importSqliteModule(specifier) {
|
|
31
|
+
if (specifier === nodeSpec && typeof process.getBuiltinModule === "function") {
|
|
32
|
+
const loaded = process.getBuiltinModule(specifier);
|
|
33
|
+
if (loaded)
|
|
34
|
+
return loaded;
|
|
35
|
+
throw Object.assign(new Error(`No such built-in module: ${specifier}`), {
|
|
36
|
+
code: "ERR_UNKNOWN_BUILTIN_MODULE"
|
|
37
|
+
});
|
|
38
|
+
}
|
|
31
39
|
return await import(specifier);
|
|
32
40
|
}
|
|
33
41
|
function isModuleNotFoundError(error, specifier) {
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
inRootAsyncContext2,
|
|
3
3
|
getSqliteDatabasePath2,
|
|
4
4
|
releaseIndexOwner2
|
|
5
|
-
} from "./index-
|
|
5
|
+
} from "./index-rrwv9y2k.js";
|
|
6
6
|
import {
|
|
7
7
|
registerExitAbort
|
|
8
8
|
} from "./index-399p688g.js";
|
|
@@ -43,6 +43,7 @@ function ensureExitHook() {
|
|
|
43
43
|
function runSessionWarmupWorker(input, options) {
|
|
44
44
|
ensureExitHook();
|
|
45
45
|
return new Promise((resolve) => {
|
|
46
|
+
const openCancellation = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
46
47
|
let worker;
|
|
47
48
|
let deadline;
|
|
48
49
|
let settled = false;
|
|
@@ -50,7 +51,9 @@ function runSessionWarmupWorker(input, options) {
|
|
|
50
51
|
if (settled)
|
|
51
52
|
return;
|
|
52
53
|
settled = true;
|
|
54
|
+
Atomics.store(openCancellation, 0, 1);
|
|
53
55
|
clearTimeout(deadline);
|
|
56
|
+
options.signal?.removeEventListener("abort", abort);
|
|
54
57
|
if (worker) {
|
|
55
58
|
activeWorkers.delete(worker);
|
|
56
59
|
const stopped = worker.terminate();
|
|
@@ -69,8 +72,16 @@ function runSessionWarmupWorker(input, options) {
|
|
|
69
72
|
deadline = setTimeout(() => finish({ failure, detail: `no reply within ${ms} ms` }), ms);
|
|
70
73
|
deadline.unref?.();
|
|
71
74
|
};
|
|
75
|
+
const abort = () => finish({ failure: "stopped", detail: "stopped" });
|
|
76
|
+
if (options.signal?.aborted) {
|
|
77
|
+
abort();
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
72
81
|
try {
|
|
73
|
-
worker = inRootAsyncContext2(() => new Worker(options.entry ?? defaultEntry(), {
|
|
82
|
+
worker = inRootAsyncContext2(() => new Worker(options.entry ?? defaultEntry(), {
|
|
83
|
+
workerData: { ...input, openCancellation: openCancellation.buffer }
|
|
84
|
+
}));
|
|
74
85
|
} catch (error) {
|
|
75
86
|
finish({ failure: "start-failed", detail: String(error) });
|
|
76
87
|
return;
|
|
@@ -93,8 +104,17 @@ function runSessionWarmupWorker(input, options) {
|
|
|
93
104
|
finish({ orphanSweep: reply.result });
|
|
94
105
|
else if (reply.kind === "exact-ordinal-ids")
|
|
95
106
|
finish({ exactOrdinalIds: reply.ids });
|
|
107
|
+
else if (reply.kind === "search-archive")
|
|
108
|
+
finish({ searchArchive: reply.text });
|
|
109
|
+
else if (reply.kind === "fts-rowid-backfill")
|
|
110
|
+
finish({ ftsRowidBackfill: true });
|
|
96
111
|
else
|
|
97
|
-
finish({
|
|
112
|
+
finish({
|
|
113
|
+
failure: "worker-error",
|
|
114
|
+
detail: reply.error,
|
|
115
|
+
code: reply.code,
|
|
116
|
+
errcode: reply.errcode
|
|
117
|
+
});
|
|
98
118
|
});
|
|
99
119
|
worker.on("error", (error) => finish({ failure: "worker-error", detail: String(error) }));
|
|
100
120
|
worker.on("exit", (code) => finish({ failure: "exited", detail: `exit code ${code}` }));
|
|
@@ -4,10 +4,10 @@ import {
|
|
|
4
4
|
withSqliteBackgroundWriter2,
|
|
5
5
|
withoutSqliteTransformPass2,
|
|
6
6
|
isTransientSqliteError2
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-rrwv9y2k.js";
|
|
8
8
|
import {
|
|
9
9
|
estimateTokens
|
|
10
|
-
} from "./index-
|
|
10
|
+
} from "./index-1kcfwk9c.js";
|
|
11
11
|
import {
|
|
12
12
|
getHarness2
|
|
13
13
|
} from "./index-0h2mkab2.js";
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
isSynapseEmbeddingTruncated,
|
|
21
21
|
getSynapseBatchRequestKey,
|
|
22
22
|
SynapseEmbeddingProvider
|
|
23
|
-
} from "./index-
|
|
23
|
+
} from "./index-d8gc0112.js";
|
|
24
24
|
import {
|
|
25
25
|
DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS,
|
|
26
26
|
CHUNK_WINDOW_SAFETY_RATIO,
|
|
@@ -38,20 +38,20 @@ import {
|
|
|
38
38
|
countSessionCompartmentEmbedCoverage,
|
|
39
39
|
countSessionCompartmentEmbedCoveragePolite,
|
|
40
40
|
countUnembeddedSessionCompartmentsPolite
|
|
41
|
-
} from "./index-
|
|
41
|
+
} from "./index-xnch62s6.js";
|
|
42
42
|
import {
|
|
43
43
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
44
|
-
} from "./index-
|
|
44
|
+
} from "./index-bxmf4dhr.js";
|
|
45
45
|
import {
|
|
46
46
|
embeddingModelsMatch,
|
|
47
47
|
resolveEmbeddingTextPrefixes,
|
|
48
48
|
getEmbeddingProviderIdentity,
|
|
49
49
|
dominantEmbeddingFailure,
|
|
50
50
|
LocalEmbeddingProvider
|
|
51
|
-
} from "./index-
|
|
51
|
+
} from "./index-2mmy81qt.js";
|
|
52
52
|
import {
|
|
53
53
|
setBootQuietPeriodForTests
|
|
54
|
-
} from "./index-
|
|
54
|
+
} from "./index-g7p75667.js";
|
|
55
55
|
import {
|
|
56
56
|
isEmbeddingHostBusy2
|
|
57
57
|
} from "./index-9p40dgkm.js";
|
|
@@ -62,7 +62,7 @@ import {
|
|
|
62
62
|
acquireGitSweepLease,
|
|
63
63
|
renewGitSweepLease,
|
|
64
64
|
releaseGitSweepLease
|
|
65
|
-
} from "./index-
|
|
65
|
+
} from "./index-svgkv4g2.js";
|
|
66
66
|
import {
|
|
67
67
|
saveEmbeddingIfHashMatches,
|
|
68
68
|
getMemoryEmbedCoverage,
|
|
@@ -428,6 +428,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
428
428
|
return Array.from({ length: texts.length }, () => null);
|
|
429
429
|
}
|
|
430
430
|
const isQuery = purpose === "query";
|
|
431
|
+
let rateLimitedResponses = 0;
|
|
431
432
|
for (;; ) {
|
|
432
433
|
if (Date.now() < this.rateLimit.until) {
|
|
433
434
|
if (isQuery)
|
|
@@ -441,6 +442,8 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
441
442
|
return outcome;
|
|
442
443
|
if (isQuery)
|
|
443
444
|
return Array.from({ length: texts.length }, () => null);
|
|
445
|
+
if (++rateLimitedResponses >= 8)
|
|
446
|
+
return Array.from({ length: texts.length }, () => null);
|
|
444
447
|
}
|
|
445
448
|
}
|
|
446
449
|
async requestOnce(texts, requestTexts, signal, purpose) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// ../plugin/src/shared/project-directory-key.ts
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
function projectDirectoryKey(directory) {
|
|
4
|
+
const slashed = directory.replaceAll("\\", "/").replace(/^\/\/\?\/UNC\//i, "//").replace(/^\/\/\?\//, "");
|
|
5
|
+
if (/^[a-z]:\//i.test(slashed) || slashed.startsWith("//")) {
|
|
6
|
+
return path.win32.normalize(slashed).replaceAll("\\", "/").replace(/\/$/, "").toLowerCase();
|
|
7
|
+
}
|
|
8
|
+
return path.resolve(directory);
|
|
9
|
+
}
|
|
10
|
+
function sameProjectDirectory(a, b) {
|
|
11
|
+
if (a.length === 0 || b.length === 0)
|
|
12
|
+
return false;
|
|
13
|
+
return projectDirectoryKey(a) === projectDirectoryKey(b);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { projectDirectoryKey, sameProjectDirectory };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getTokenEstimatorFingerprint,
|
|
3
3
|
estimateTokens
|
|
4
|
-
} from "./index-
|
|
4
|
+
} from "./index-1kcfwk9c.js";
|
|
5
5
|
import {
|
|
6
6
|
getHarness2
|
|
7
7
|
} from "./index-0h2mkab2.js";
|
|
@@ -10,10 +10,10 @@ import {
|
|
|
10
10
|
} from "./index-7bqrfyny.js";
|
|
11
11
|
import {
|
|
12
12
|
isSynapseEmbeddingTruncated
|
|
13
|
-
} from "./index-
|
|
13
|
+
} from "./index-d8gc0112.js";
|
|
14
14
|
import {
|
|
15
|
-
|
|
16
|
-
} from "./index-
|
|
15
|
+
messageFtsOrdinalRangeIsMapped2
|
|
16
|
+
} from "./index-yygsh681.js";
|
|
17
17
|
|
|
18
18
|
// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
|
|
19
19
|
import { createHash } from "node:crypto";
|
|
@@ -514,7 +514,7 @@ function toFloat32Array(blob) {
|
|
|
514
514
|
function buildCanonicalChunkTextFromFts(db, sessionId, startOrdinal, endOrdinal) {
|
|
515
515
|
if (endOrdinal < startOrdinal)
|
|
516
516
|
return "";
|
|
517
|
-
if (!
|
|
517
|
+
if (!messageFtsOrdinalRangeIsMapped2(db, sessionId, startOrdinal, endOrdinal))
|
|
518
518
|
return null;
|
|
519
519
|
const rows = getLoadFtsRowsStatement(db).all(sessionId, startOrdinal, endOrdinal).map((row) => row);
|
|
520
520
|
const lines = [];
|
|
@@ -900,12 +900,18 @@ var BACKOFF_PREFIX = "chunk_embed_backoff:";
|
|
|
900
900
|
function backoffKey(compartmentId) {
|
|
901
901
|
return `${BACKOFF_PREFIX}${compartmentId}`;
|
|
902
902
|
}
|
|
903
|
-
function deleteChunkEmbedBackoffForSession(db, sessionId) {
|
|
904
|
-
|
|
903
|
+
function deleteChunkEmbedBackoffForSession(db, sessionId, limit) {
|
|
904
|
+
if (limit !== undefined)
|
|
905
|
+
return db.prepare(`DELETE FROM schema_migrations_meta WHERE key IN (
|
|
906
|
+
SELECT key FROM schema_migrations_meta JOIN compartments
|
|
907
|
+
ON key = ? || compartments.id OR key = ? || compartments.id
|
|
908
|
+
WHERE compartments.session_id = ? LIMIT ?
|
|
909
|
+
)`).run(BACKOFF_PREFIX, WINDOW_SOURCE_PREFIX, sessionId, limit).changes;
|
|
910
|
+
return db.prepare(`DELETE FROM schema_migrations_meta WHERE key IN (
|
|
905
911
|
SELECT ? || id FROM compartments WHERE session_id = ?
|
|
906
912
|
UNION ALL
|
|
907
913
|
SELECT ? || id FROM compartments WHERE session_id = ?
|
|
908
|
-
)`).run(BACKOFF_PREFIX, sessionId, WINDOW_SOURCE_PREFIX, sessionId);
|
|
914
|
+
)`).run(BACKOFF_PREFIX, sessionId, WINDOW_SOURCE_PREFIX, sessionId).changes;
|
|
909
915
|
}
|
|
910
916
|
function recordChunkEmbedBackoff(db, candidate, projectPath, modelId) {
|
|
911
917
|
const mapped = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage);
|
|
@@ -1,26 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
+
getSqliteDatabasePath2,
|
|
2
3
|
withSqliteBackgroundWriter2,
|
|
3
4
|
withoutSqliteTransformPass2
|
|
4
|
-
} from "./index-
|
|
5
|
+
} from "./index-rrwv9y2k.js";
|
|
6
|
+
import {
|
|
7
|
+
runSessionWarmupWorker
|
|
8
|
+
} from "./index-vktahpdj.js";
|
|
5
9
|
import {
|
|
6
10
|
logSlowWriteTransaction
|
|
7
11
|
} from "./index-d4dt19md.js";
|
|
8
12
|
|
|
9
|
-
// ../plugin/src/shared/project-directory-key.ts
|
|
10
|
-
import path from "node:path";
|
|
11
|
-
function projectDirectoryKey(directory) {
|
|
12
|
-
const slashed = directory.replaceAll("\\", "/").replace(/^\/\/\?\/UNC\//i, "//").replace(/^\/\/\?\//, "");
|
|
13
|
-
if (/^[a-z]:\//i.test(slashed) || slashed.startsWith("//")) {
|
|
14
|
-
return path.win32.normalize(slashed).replaceAll("\\", "/").replace(/\/$/, "").toLowerCase();
|
|
15
|
-
}
|
|
16
|
-
return path.resolve(directory);
|
|
17
|
-
}
|
|
18
|
-
function sameProjectDirectory(a, b) {
|
|
19
|
-
if (a.length === 0 || b.length === 0)
|
|
20
|
-
return false;
|
|
21
|
-
return projectDirectoryKey(a) === projectDirectoryKey(b);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
13
|
// ../plugin/src/features/magic-context/message-fts-rowid-map.ts
|
|
25
14
|
import { createHash } from "node:crypto";
|
|
26
15
|
var MESSAGE_FTS_ROWID_MAP_BACKFILL_BATCH_SIZE = 100;
|
|
@@ -56,7 +45,7 @@ function getBackfillState(db) {
|
|
|
56
45
|
completed: row?.completed === 1
|
|
57
46
|
};
|
|
58
47
|
}
|
|
59
|
-
function
|
|
48
|
+
function recordMessageFtsRowid2(db, sessionId, messageOrdinal, ftsRowid, messageTimeMs = null) {
|
|
60
49
|
const numericRowid = Number(ftsRowid);
|
|
61
50
|
if (!Number.isSafeInteger(numericRowid) || numericRowid <= 0) {
|
|
62
51
|
throw new Error(`invalid message FTS rowid: ${String(ftsRowid)}`);
|
|
@@ -64,7 +53,7 @@ function recordMessageFtsRowid(db, sessionId, messageOrdinal, ftsRowid, messageT
|
|
|
64
53
|
const normalizedTime = typeof messageTimeMs === "number" && Number.isSafeInteger(messageTimeMs) && messageTimeMs >= 0 ? messageTimeMs : null;
|
|
65
54
|
getUpsertMapStatement(db).run(sessionId, messageOrdinal, numericRowid, normalizedTime);
|
|
66
55
|
}
|
|
67
|
-
function
|
|
56
|
+
function recordIndexedMessageTime2(db, sessionId, messageOrdinal, messageTimeMs) {
|
|
68
57
|
if (typeof messageTimeMs !== "number" || !Number.isSafeInteger(messageTimeMs) || messageTimeMs < 0) {
|
|
69
58
|
return;
|
|
70
59
|
}
|
|
@@ -106,7 +95,7 @@ function backfillMessageFtsRowidMapBatch(db, batchSize = MESSAGE_FTS_ROWID_MAP_B
|
|
|
106
95
|
watermarkRowid = ftsRowid;
|
|
107
96
|
}
|
|
108
97
|
if (typeof row.sessionId === "string" && Number.isSafeInteger(messageOrdinal) && messageOrdinal >= 0 && Number.isSafeInteger(ftsRowid) && ftsRowid > 0) {
|
|
109
|
-
|
|
98
|
+
recordMessageFtsRowid2(db, row.sessionId, messageOrdinal, ftsRowid);
|
|
110
99
|
}
|
|
111
100
|
}
|
|
112
101
|
const completed = rows.length < boundedBatchSize;
|
|
@@ -122,26 +111,33 @@ function backfillMessageFtsRowidMapBatch(db, batchSize = MESSAGE_FTS_ROWID_MAP_B
|
|
|
122
111
|
logSlowWriteTransaction("message_fts_rowid_backfill", transactionStartedAt);
|
|
123
112
|
return progress;
|
|
124
113
|
}
|
|
125
|
-
async function
|
|
114
|
+
async function runMessageFtsRowidMapBackfill2(db, signal) {
|
|
126
115
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
127
|
-
|
|
116
|
+
while (!signal?.aborted) {
|
|
128
117
|
const progress = backfillMessageFtsRowidMapBatch(db);
|
|
129
118
|
if (progress.completed)
|
|
130
119
|
return;
|
|
131
120
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
132
121
|
}
|
|
133
122
|
}
|
|
134
|
-
function
|
|
123
|
+
function startMessageFtsRowidMapBackfill2(db, signal) {
|
|
135
124
|
const active = activeBackfills.get(db);
|
|
136
125
|
if (active)
|
|
137
126
|
return active;
|
|
138
|
-
const
|
|
127
|
+
const path = getSqliteDatabasePath2(db);
|
|
128
|
+
const run = (path ? runSessionWarmupWorker({ path, sessionId: "", ftsRowidBackfill: true }, { signal }).then((result) => {
|
|
129
|
+
if (signal?.aborted)
|
|
130
|
+
return;
|
|
131
|
+
if (!("ftsRowidBackfill" in result)) {
|
|
132
|
+
throw Object.assign(new Error(`FTS rowid backfill unavailable: ${"failure" in result ? result.detail : "unexpected reply"}`), "failure" in result ? { code: result.code, errcode: result.errcode } : {});
|
|
133
|
+
}
|
|
134
|
+
}) : withoutSqliteTransformPass2(() => withSqliteBackgroundWriter2(() => runMessageFtsRowidMapBackfill2(db, signal)))).finally(() => {
|
|
139
135
|
activeBackfills.delete(db);
|
|
140
136
|
});
|
|
141
137
|
activeBackfills.set(db, run);
|
|
142
138
|
return run;
|
|
143
139
|
}
|
|
144
|
-
function
|
|
140
|
+
function messageFtsOrdinalRangeIsMapped2(db, sessionId, startOrdinal, endOrdinal) {
|
|
145
141
|
if (endOrdinal < startOrdinal)
|
|
146
142
|
return true;
|
|
147
143
|
if (getBackfillState(db).completed)
|
|
@@ -172,7 +168,10 @@ function messageFtsOrdinalRangeIsMapped(db, sessionId, startOrdinal, endOrdinal)
|
|
|
172
168
|
const ordinalCount = endOrdinal - startOrdinal + 1;
|
|
173
169
|
return row?.sourceOrdinalCount === ordinalCount && row.expectedMapCount === row.mappedCount;
|
|
174
170
|
}
|
|
175
|
-
function
|
|
171
|
+
function getMessageFtsRowidMapBackfillProgress2(db) {
|
|
172
|
+
return getBackfillState(db);
|
|
173
|
+
}
|
|
174
|
+
function deleteUnmappedMessageFtsRows2(db, sessionIds) {
|
|
176
175
|
if (sessionIds.length === 0)
|
|
177
176
|
return;
|
|
178
177
|
const state = getBackfillState(db);
|
|
@@ -183,4 +182,4 @@ function deleteUnmappedMessageFtsRows(db, sessionIds) {
|
|
|
183
182
|
WHERE rowid > ? AND session_id IN (${placeholders})`).run(state.watermarkRowid, ...sessionIds);
|
|
184
183
|
}
|
|
185
184
|
|
|
186
|
-
export {
|
|
185
|
+
export { recordMessageFtsRowid2, recordIndexedMessageTime2, runMessageFtsRowidMapBackfill2, startMessageFtsRowidMapBackfill2, messageFtsOrdinalRangeIsMapped2, getMessageFtsRowidMapBackfillProgress2, deleteUnmappedMessageFtsRows2 };
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_LOCAL_EMBEDDING_MODEL
|
|
3
|
-
} from "./index-
|
|
3
|
+
} from "./index-bxmf4dhr.js";
|
|
4
4
|
import {
|
|
5
5
|
log2
|
|
6
6
|
} from "./index-7bqrfyny.js";
|
|
7
7
|
import {
|
|
8
8
|
isSynapseEmbeddingTruncated,
|
|
9
9
|
SynapseEmbeddingProvider
|
|
10
|
-
} from "./index-
|
|
10
|
+
} from "./index-d8gc0112.js";
|
|
11
11
|
import {
|
|
12
12
|
LocalEmbeddingProvider
|
|
13
|
-
} from "./index-
|
|
13
|
+
} from "./index-2mmy81qt.js";
|
|
14
14
|
import {
|
|
15
15
|
OpenAICompatibleEmbeddingProvider,
|
|
16
16
|
getProjectEmbeddingSnapshot2,
|
|
17
17
|
embedBatchForProject2
|
|
18
|
-
} from "./index-
|
|
18
|
+
} from "./index-vmcgew7x.js";
|
|
19
19
|
import {
|
|
20
20
|
isEmbeddingHostBusy2
|
|
21
21
|
} from "./index-9p40dgkm.js";
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import {
|
|
2
|
+
withMessageFtsSessionFilter
|
|
3
|
+
} from "./index-1cvyaw9a.js";
|
|
4
|
+
import {
|
|
5
|
+
estimateTokens
|
|
6
|
+
} from "./index-1kcfwk9c.js";
|
|
7
|
+
import {
|
|
8
|
+
readRawSessionMessageIdOrdinalsForRange2
|
|
9
|
+
} from "./index-5k2mshvq.js";
|
|
10
|
+
|
|
11
|
+
// ../plugin/src/tools/ctx-expand/search-archive.ts
|
|
12
|
+
var archiveReadsOnThisThread = 0;
|
|
13
|
+
function renderSearchArchive2(db, sessionId, start, end, tokenBudget, archiveOnly = false, readRawIds = () => readRawSessionMessageIdOrdinalsForRange2(sessionId, start, end)) {
|
|
14
|
+
archiveReadsOnThisThread += 1;
|
|
15
|
+
return withMessageFtsSessionFilter(db, sessionId, (sessionFirst) => {
|
|
16
|
+
let rawIds;
|
|
17
|
+
const rows = db.prepare(sessionFirst ? `SELECT f.message_ordinal AS ordinal, f.message_id AS id, f.role, f.content
|
|
18
|
+
FROM message_fts_rowid_map AS m
|
|
19
|
+
JOIN message_history_fts AS f ON f.rowid = m.fts_rowid
|
|
20
|
+
WHERE m.session_id = ?1 AND m.message_ordinal BETWEEN ?2 AND ?3
|
|
21
|
+
AND f.session_id = ?1 AND f.message_ordinal BETWEEN ?2 AND ?3
|
|
22
|
+
ORDER BY m.message_ordinal` : `SELECT message_ordinal AS ordinal, message_id AS id, role, content
|
|
23
|
+
FROM message_history_fts
|
|
24
|
+
WHERE session_id = ?1 AND message_ordinal BETWEEN ?2 AND ?3
|
|
25
|
+
ORDER BY CAST(message_ordinal AS INTEGER)`);
|
|
26
|
+
const blocks = [];
|
|
27
|
+
let tokens = 0;
|
|
28
|
+
let lastOrdinal = start - 1;
|
|
29
|
+
let truncated = false;
|
|
30
|
+
for (const row of rows.iterate(sessionId, start, end)) {
|
|
31
|
+
if (!row.content)
|
|
32
|
+
continue;
|
|
33
|
+
if (!archiveOnly) {
|
|
34
|
+
rawIds ??= readRawIds();
|
|
35
|
+
if (rawIds.get(row.id) === row.ordinal)
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const block = `[${row.ordinal}] ${row.role === "user" ? "U" : "A"}: ${row.content}`;
|
|
39
|
+
const cost = estimateTokens(block);
|
|
40
|
+
if (blocks.length > 0 && tokens + cost > tokenBudget) {
|
|
41
|
+
truncated = true;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
blocks.push(block);
|
|
45
|
+
tokens += cost;
|
|
46
|
+
lastOrdinal = row.ordinal;
|
|
47
|
+
}
|
|
48
|
+
if (blocks.length === 0)
|
|
49
|
+
return null;
|
|
50
|
+
const text = [
|
|
51
|
+
"From the search archive: indexed text only; no surrounding messages or tool output.",
|
|
52
|
+
"",
|
|
53
|
+
...blocks
|
|
54
|
+
];
|
|
55
|
+
if (truncated) {
|
|
56
|
+
text.push("", `Truncated at message ${lastOrdinal} (budget: ~${tokenBudget} tokens). Call ctx_expand(start=${lastOrdinal + 1}, end=${end}) for more archived text.`);
|
|
57
|
+
}
|
|
58
|
+
return text.join(`
|
|
59
|
+
`);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
export { renderSearchArchive2 };
|