@cortexkit/pi-magic-context 0.24.1 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1006 -350
- package/dist/subagent-entry.js +559 -175
- package/package.json +1 -1
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -143218,6 +143226,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143226
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143227
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143228
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143229
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143230
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143231
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143232
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -159135,6 +159144,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159135
159144
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159136
159145
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159137
159146
|
}
|
|
159147
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159148
|
+
const row = db.prepare(`SELECT
|
|
159149
|
+
COUNT(*) AS total,
|
|
159150
|
+
SUM(CASE WHEN EXISTS (
|
|
159151
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159152
|
+
WHERE e.compartment_id = c.id
|
|
159153
|
+
AND e.project_path = ?
|
|
159154
|
+
AND e.model_id = ?
|
|
159155
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159156
|
+
FROM compartments c
|
|
159157
|
+
JOIN session_projects sp
|
|
159158
|
+
ON sp.session_id = c.session_id
|
|
159159
|
+
AND sp.harness = c.harness
|
|
159160
|
+
AND sp.project_path = ?
|
|
159161
|
+
WHERE c.session_id = ?
|
|
159162
|
+
AND c.start_message IS NOT NULL
|
|
159163
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159164
|
+
return {
|
|
159165
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159166
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159167
|
+
};
|
|
159168
|
+
}
|
|
159138
159169
|
|
|
159139
159170
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159140
159171
|
function cosineSimilarity(a, b) {
|
|
@@ -159287,6 +159318,19 @@ async function withQuietConsole(fn) {
|
|
|
159287
159318
|
console.error = origError;
|
|
159288
159319
|
}
|
|
159289
159320
|
}
|
|
159321
|
+
var nativeRuntimeMissing = false;
|
|
159322
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159323
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159324
|
+
const lower = message.toLowerCase();
|
|
159325
|
+
const code = error51?.code;
|
|
159326
|
+
const name2 = error51?.name;
|
|
159327
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159328
|
+
return true;
|
|
159329
|
+
}
|
|
159330
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159331
|
+
return false;
|
|
159332
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159333
|
+
}
|
|
159290
159334
|
function isTransientLoadError(error51) {
|
|
159291
159335
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159292
159336
|
if (!message)
|
|
@@ -159351,6 +159395,9 @@ class LocalEmbeddingProvider {
|
|
|
159351
159395
|
if (this.pipeline) {
|
|
159352
159396
|
return true;
|
|
159353
159397
|
}
|
|
159398
|
+
if (nativeRuntimeMissing) {
|
|
159399
|
+
return false;
|
|
159400
|
+
}
|
|
159354
159401
|
if (this.initPromise) {
|
|
159355
159402
|
await this.initPromise;
|
|
159356
159403
|
return this.pipeline !== null;
|
|
@@ -159417,7 +159464,12 @@ class LocalEmbeddingProvider {
|
|
|
159417
159464
|
await releaseLock();
|
|
159418
159465
|
}
|
|
159419
159466
|
} catch (error51) {
|
|
159420
|
-
|
|
159467
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159468
|
+
nativeRuntimeMissing = true;
|
|
159469
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @cortexkit/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159470
|
+
} else {
|
|
159471
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159472
|
+
}
|
|
159421
159473
|
this.pipeline = null;
|
|
159422
159474
|
} finally {
|
|
159423
159475
|
this.initPromise = null;
|
|
@@ -159903,6 +159955,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159903
159955
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159904
159956
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159905
159957
|
}
|
|
159958
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
159959
|
+
const row = db.prepare(`SELECT
|
|
159960
|
+
COUNT(*) AS total,
|
|
159961
|
+
SUM(CASE WHEN EXISTS (
|
|
159962
|
+
SELECT 1 FROM memory_embeddings e
|
|
159963
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
159964
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159965
|
+
FROM memories m
|
|
159966
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
159967
|
+
return {
|
|
159968
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159969
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159970
|
+
};
|
|
159971
|
+
}
|
|
159906
159972
|
|
|
159907
159973
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159908
159974
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -160009,6 +160075,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
160009
160075
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
160010
160076
|
}
|
|
160011
160077
|
|
|
160078
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160079
|
+
init_logger();
|
|
160080
|
+
var insertStatements = new WeakMap;
|
|
160081
|
+
var existingShasStatements = new WeakMap;
|
|
160082
|
+
var projectCountStatements = new WeakMap;
|
|
160083
|
+
var evictStatements = new WeakMap;
|
|
160084
|
+
var evictOverflowStatements = new WeakMap;
|
|
160085
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160086
|
+
function getInsertStatement(db) {
|
|
160087
|
+
let stmt = insertStatements.get(db);
|
|
160088
|
+
if (!stmt) {
|
|
160089
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160090
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160091
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160092
|
+
project_path = excluded.project_path,
|
|
160093
|
+
short_sha = excluded.short_sha,
|
|
160094
|
+
message = excluded.message,
|
|
160095
|
+
author = excluded.author,
|
|
160096
|
+
committed_at = excluded.committed_at,
|
|
160097
|
+
indexed_at = excluded.indexed_at
|
|
160098
|
+
WHERE git_commits.message != excluded.message`);
|
|
160099
|
+
insertStatements.set(db, stmt);
|
|
160100
|
+
}
|
|
160101
|
+
return stmt;
|
|
160102
|
+
}
|
|
160103
|
+
function getExistingShasStatement(db) {
|
|
160104
|
+
let stmt = existingShasStatements.get(db);
|
|
160105
|
+
if (!stmt) {
|
|
160106
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160107
|
+
existingShasStatements.set(db, stmt);
|
|
160108
|
+
}
|
|
160109
|
+
return stmt;
|
|
160110
|
+
}
|
|
160111
|
+
function getProjectCountStatement(db) {
|
|
160112
|
+
let stmt = projectCountStatements.get(db);
|
|
160113
|
+
if (!stmt) {
|
|
160114
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160115
|
+
projectCountStatements.set(db, stmt);
|
|
160116
|
+
}
|
|
160117
|
+
return stmt;
|
|
160118
|
+
}
|
|
160119
|
+
function getLatestCommitTimeStatement(db) {
|
|
160120
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160121
|
+
if (!stmt) {
|
|
160122
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160123
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160124
|
+
}
|
|
160125
|
+
return stmt;
|
|
160126
|
+
}
|
|
160127
|
+
function getEvictOverflowStatement(db) {
|
|
160128
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160129
|
+
if (!stmt) {
|
|
160130
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160131
|
+
WHERE rowid IN (
|
|
160132
|
+
SELECT rowid FROM git_commits
|
|
160133
|
+
WHERE project_path = ?
|
|
160134
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160135
|
+
LIMIT -1 OFFSET ?
|
|
160136
|
+
)`);
|
|
160137
|
+
evictOverflowStatements.set(db, stmt);
|
|
160138
|
+
}
|
|
160139
|
+
return stmt;
|
|
160140
|
+
}
|
|
160141
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160142
|
+
if (commits.length === 0)
|
|
160143
|
+
return { inserted: 0, updated: 0 };
|
|
160144
|
+
const existing = new Set;
|
|
160145
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160146
|
+
existing.add(row.sha);
|
|
160147
|
+
}
|
|
160148
|
+
let inserted = 0;
|
|
160149
|
+
let updated = 0;
|
|
160150
|
+
const now = Date.now();
|
|
160151
|
+
const insertStmt = getInsertStatement(db);
|
|
160152
|
+
db.transaction(() => {
|
|
160153
|
+
for (const commit of commits) {
|
|
160154
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160155
|
+
if (result.changes > 0) {
|
|
160156
|
+
if (existing.has(commit.sha)) {
|
|
160157
|
+
updated++;
|
|
160158
|
+
} else {
|
|
160159
|
+
inserted++;
|
|
160160
|
+
existing.add(commit.sha);
|
|
160161
|
+
}
|
|
160162
|
+
}
|
|
160163
|
+
}
|
|
160164
|
+
})();
|
|
160165
|
+
return { inserted, updated };
|
|
160166
|
+
}
|
|
160167
|
+
function getCommitCount(db, projectPath) {
|
|
160168
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160169
|
+
return row?.count ?? 0;
|
|
160170
|
+
}
|
|
160171
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160172
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160173
|
+
return row?.latest ?? null;
|
|
160174
|
+
}
|
|
160175
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160176
|
+
if (maxCommits <= 0)
|
|
160177
|
+
return 0;
|
|
160178
|
+
const count = getCommitCount(db, projectPath);
|
|
160179
|
+
if (count <= maxCommits)
|
|
160180
|
+
return 0;
|
|
160181
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160182
|
+
const after = getCommitCount(db, projectPath);
|
|
160183
|
+
const evicted = Math.max(0, count - after);
|
|
160184
|
+
if (evicted > 0) {
|
|
160185
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160186
|
+
}
|
|
160187
|
+
return evicted;
|
|
160188
|
+
}
|
|
160189
|
+
|
|
160012
160190
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
160013
160191
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
160014
160192
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160241,8 +160419,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160241
160419
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160242
160420
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160243
160421
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160244
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160422
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160245
160423
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160424
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160425
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160426
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160427
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160246
160428
|
var projectRegistrations = new Map;
|
|
160247
160429
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160248
160430
|
var globalRegistrationGeneration = 0;
|
|
@@ -160342,7 +160524,9 @@ function snapshotFor(registration) {
|
|
|
160342
160524
|
enabled,
|
|
160343
160525
|
gitCommitEnabled,
|
|
160344
160526
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160345
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160527
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160528
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160529
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160346
160530
|
};
|
|
160347
160531
|
}
|
|
160348
160532
|
function disposeProvider(provider) {
|
|
@@ -160553,8 +160737,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160553
160737
|
}
|
|
160554
160738
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160555
160739
|
const noWork = [];
|
|
160740
|
+
const failed = [];
|
|
160556
160741
|
if (candidates.length === 0)
|
|
160557
|
-
return { embedded: 0, noWork };
|
|
160742
|
+
return { embedded: 0, noWork, failed };
|
|
160558
160743
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160559
160744
|
const prepared = [];
|
|
160560
160745
|
for (const candidate of candidates) {
|
|
@@ -160571,7 +160756,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160571
160756
|
prepared.push({ candidate, windows });
|
|
160572
160757
|
}
|
|
160573
160758
|
if (prepared.length === 0)
|
|
160574
|
-
return { embedded: 0, noWork };
|
|
160759
|
+
return { embedded: 0, noWork, failed };
|
|
160575
160760
|
let embedded = 0;
|
|
160576
160761
|
let i = 0;
|
|
160577
160762
|
while (i < prepared.length) {
|
|
@@ -160588,35 +160773,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160588
160773
|
const texts = [];
|
|
160589
160774
|
for (const item of slice)
|
|
160590
160775
|
texts.push(...item.windows.map((w) => w.text));
|
|
160591
|
-
|
|
160592
|
-
|
|
160593
|
-
if (!result)
|
|
160594
|
-
continue;
|
|
160776
|
+
const persistedIds = new Set;
|
|
160777
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160595
160778
|
if (signal?.aborted)
|
|
160596
160779
|
break;
|
|
160597
|
-
let
|
|
160598
|
-
|
|
160599
|
-
|
|
160600
|
-
|
|
160601
|
-
|
|
160602
|
-
|
|
160780
|
+
let result = null;
|
|
160781
|
+
const attemptStart = Date.now();
|
|
160782
|
+
try {
|
|
160783
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160784
|
+
} catch (error51) {
|
|
160785
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160786
|
+
}
|
|
160787
|
+
if (signal?.aborted)
|
|
160788
|
+
break;
|
|
160789
|
+
if (result) {
|
|
160790
|
+
let offset = 0;
|
|
160791
|
+
for (const item of slice) {
|
|
160792
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160793
|
+
offset += item.windows.length;
|
|
160794
|
+
if (persistedIds.has(item.candidate.id))
|
|
160795
|
+
continue;
|
|
160796
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160797
|
+
continue;
|
|
160798
|
+
}
|
|
160799
|
+
const rows = item.windows.map((window, index) => ({
|
|
160800
|
+
compartmentId: item.candidate.id,
|
|
160801
|
+
sessionId: item.candidate.sessionId,
|
|
160802
|
+
projectPath: projectIdentity,
|
|
160803
|
+
window,
|
|
160804
|
+
modelId,
|
|
160805
|
+
vector: vectors[index]
|
|
160806
|
+
}));
|
|
160807
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160808
|
+
persistedIds.add(item.candidate.id);
|
|
160603
160809
|
}
|
|
160604
|
-
const rows = item.windows.map((window, index) => ({
|
|
160605
|
-
compartmentId: item.candidate.id,
|
|
160606
|
-
sessionId: item.candidate.sessionId,
|
|
160607
|
-
projectPath: projectIdentity,
|
|
160608
|
-
window,
|
|
160609
|
-
modelId,
|
|
160610
|
-
vector: vectors[index]
|
|
160611
|
-
}));
|
|
160612
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160613
|
-
embedded += 1;
|
|
160614
160810
|
}
|
|
160615
|
-
|
|
160616
|
-
|
|
160811
|
+
if (persistedIds.size === slice.length)
|
|
160812
|
+
break;
|
|
160813
|
+
if (persistedIds.size > 0)
|
|
160814
|
+
break;
|
|
160815
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160816
|
+
break;
|
|
160817
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160818
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160819
|
+
}
|
|
160820
|
+
}
|
|
160821
|
+
embedded += persistedIds.size;
|
|
160822
|
+
if (!signal?.aborted) {
|
|
160823
|
+
for (const item of slice) {
|
|
160824
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160825
|
+
failed.push(item.candidate.id);
|
|
160826
|
+
}
|
|
160617
160827
|
}
|
|
160618
160828
|
}
|
|
160619
|
-
return { embedded, noWork };
|
|
160829
|
+
return { embedded, noWork, failed };
|
|
160620
160830
|
}
|
|
160621
160831
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160622
160832
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160639,9 +160849,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160639
160849
|
renewal.unref?.();
|
|
160640
160850
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160641
160851
|
const skipIds = [];
|
|
160852
|
+
const failedIds = [];
|
|
160642
160853
|
let embedded = 0;
|
|
160643
160854
|
let aborted2 = false;
|
|
160644
|
-
let
|
|
160855
|
+
let providerDown = false;
|
|
160856
|
+
let consecutiveFailedBatches = 0;
|
|
160645
160857
|
try {
|
|
160646
160858
|
options?.onProgress?.({ embedded, total });
|
|
160647
160859
|
for (;; ) {
|
|
@@ -160649,15 +160861,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160649
160861
|
aborted2 = true;
|
|
160650
160862
|
break;
|
|
160651
160863
|
}
|
|
160652
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160864
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160653
160865
|
if (candidates.length === 0)
|
|
160654
160866
|
break;
|
|
160655
|
-
const {
|
|
160867
|
+
const {
|
|
160868
|
+
embedded: n,
|
|
160869
|
+
noWork,
|
|
160870
|
+
failed
|
|
160871
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160656
160872
|
for (const id of noWork)
|
|
160657
160873
|
skipIds.push(id);
|
|
160874
|
+
for (const id of failed)
|
|
160875
|
+
failedIds.push(id);
|
|
160658
160876
|
if (n === 0 && noWork.length === 0) {
|
|
160659
|
-
|
|
160660
|
-
|
|
160877
|
+
consecutiveFailedBatches += 1;
|
|
160878
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160879
|
+
providerDown = true;
|
|
160880
|
+
break;
|
|
160881
|
+
}
|
|
160882
|
+
} else {
|
|
160883
|
+
consecutiveFailedBatches = 0;
|
|
160661
160884
|
}
|
|
160662
160885
|
embedded += n;
|
|
160663
160886
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160665,16 +160888,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160665
160888
|
}
|
|
160666
160889
|
} finally {
|
|
160667
160890
|
clearInterval(renewal);
|
|
160668
|
-
|
|
160891
|
+
try {
|
|
160892
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
160893
|
+
} catch (error51) {
|
|
160894
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
160895
|
+
}
|
|
160669
160896
|
}
|
|
160670
160897
|
if (aborted2)
|
|
160671
|
-
return { status: "aborted", embedded, total };
|
|
160672
|
-
if (
|
|
160898
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
160899
|
+
if (providerDown || failedIds.length > 0) {
|
|
160673
160900
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160674
|
-
if (remaining > 0)
|
|
160675
|
-
return { status: "stalled", embedded, total, remaining };
|
|
160901
|
+
if (remaining > 0) {
|
|
160902
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
160903
|
+
}
|
|
160676
160904
|
}
|
|
160677
|
-
return { status: "done", embedded, total };
|
|
160905
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
160906
|
+
}
|
|
160907
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
160908
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
160909
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
160910
|
+
return {
|
|
160911
|
+
enabled: false,
|
|
160912
|
+
model: snapshot?.model ?? "off",
|
|
160913
|
+
provider: snapshot?.provider ?? "off",
|
|
160914
|
+
session: { embedded: 0, total: 0 },
|
|
160915
|
+
memories: { embedded: 0, total: 0 },
|
|
160916
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
160917
|
+
};
|
|
160918
|
+
}
|
|
160919
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
160920
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
160921
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
160922
|
+
const commits = gitEnabled ? {
|
|
160923
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
160924
|
+
total: getCommitCount(db, projectIdentity),
|
|
160925
|
+
gitEnabled: true
|
|
160926
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
160927
|
+
return {
|
|
160928
|
+
enabled: true,
|
|
160929
|
+
model: snapshot.model,
|
|
160930
|
+
provider: snapshot.provider,
|
|
160931
|
+
session,
|
|
160932
|
+
memories,
|
|
160933
|
+
commits
|
|
160934
|
+
};
|
|
160678
160935
|
}
|
|
160679
160936
|
|
|
160680
160937
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160903,6 +161160,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160903
161160
|
"conversation_tokens",
|
|
160904
161161
|
"tool_call_tokens",
|
|
160905
161162
|
"cleared_reasoning_through_tag",
|
|
161163
|
+
"tool_reclaim_watermark",
|
|
160906
161164
|
"last_todo_state",
|
|
160907
161165
|
"cached_m0_bytes",
|
|
160908
161166
|
"cached_m1_bytes",
|
|
@@ -160951,6 +161209,7 @@ var META_COLUMNS = {
|
|
|
160951
161209
|
conversationTokens: "conversation_tokens",
|
|
160952
161210
|
toolCallTokens: "tool_call_tokens",
|
|
160953
161211
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161212
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160954
161213
|
lastTodoState: "last_todo_state",
|
|
160955
161214
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160956
161215
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -161019,7 +161278,7 @@ function isSessionMetaRow(row) {
|
|
|
161019
161278
|
if (row === null || typeof row !== "object")
|
|
161020
161279
|
return false;
|
|
161021
161280
|
const r = row;
|
|
161022
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161281
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161023
161282
|
}
|
|
161024
161283
|
function getDefaultSessionMeta(sessionId) {
|
|
161025
161284
|
return {
|
|
@@ -161042,6 +161301,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161042
161301
|
conversationTokens: 0,
|
|
161043
161302
|
toolCallTokens: 0,
|
|
161044
161303
|
clearedReasoningThroughTag: 0,
|
|
161304
|
+
toolReclaimWatermark: 0,
|
|
161045
161305
|
lastTodoState: "",
|
|
161046
161306
|
cachedM0Bytes: null,
|
|
161047
161307
|
cachedM1Bytes: null,
|
|
@@ -161105,6 +161365,7 @@ function toSessionMeta(row) {
|
|
|
161105
161365
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161106
161366
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161107
161367
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161368
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161108
161369
|
lastTodoState: lastTodoStateRaw,
|
|
161109
161370
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161110
161371
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161531,6 +161792,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161531
161792
|
var encoder = new TextEncoder;
|
|
161532
161793
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161533
161794
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161795
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161796
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161534
161797
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161535
161798
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161536
161799
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161543,6 +161806,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161543
161806
|
function stripMalformedTagNotationGlobally(value) {
|
|
161544
161807
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161545
161808
|
}
|
|
161809
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161810
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161811
|
+
}
|
|
161546
161812
|
function stripTagSectionCharacters(value) {
|
|
161547
161813
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161548
161814
|
}
|
|
@@ -161550,6 +161816,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161550
161816
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161551
161817
|
text = stripCompleteTagPairsGlobally(text);
|
|
161552
161818
|
text = stripMalformedTagNotationGlobally(text);
|
|
161819
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161553
161820
|
text = stripTagSectionCharacters(text);
|
|
161554
161821
|
return text.trim();
|
|
161555
161822
|
}
|
|
@@ -161562,6 +161829,7 @@ function stripTagPrefix(value) {
|
|
|
161562
161829
|
const prev = stripped;
|
|
161563
161830
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161564
161831
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161832
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161565
161833
|
if (stripped === prev)
|
|
161566
161834
|
break;
|
|
161567
161835
|
}
|
|
@@ -161977,9 +162245,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161977
162245
|
|
|
161978
162246
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161979
162247
|
|
|
161980
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162248
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162249
|
+
|
|
162250
|
+
Two recovery modes for finer detail:
|
|
162251
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162252
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161981
162253
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161982
162254
|
|
|
162255
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162256
|
+
function isRecord2(value) {
|
|
162257
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162258
|
+
}
|
|
162259
|
+
function roleLabel(role) {
|
|
162260
|
+
if (role === "assistant")
|
|
162261
|
+
return "A (assistant)";
|
|
162262
|
+
if (role === "user")
|
|
162263
|
+
return "U (user)";
|
|
162264
|
+
return role;
|
|
162265
|
+
}
|
|
162266
|
+
function truncate(value, max) {
|
|
162267
|
+
const t = value.trim();
|
|
162268
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162269
|
+
}
|
|
162270
|
+
function keyArg(input) {
|
|
162271
|
+
if (!input)
|
|
162272
|
+
return "";
|
|
162273
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162274
|
+
const v = input[k];
|
|
162275
|
+
if (typeof v === "string" && v.length > 0)
|
|
162276
|
+
return truncate(v, 60);
|
|
162277
|
+
}
|
|
162278
|
+
if (typeof input.description === "string")
|
|
162279
|
+
return truncate(input.description, 60);
|
|
162280
|
+
return "";
|
|
162281
|
+
}
|
|
162282
|
+
function asToolPart(part) {
|
|
162283
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162284
|
+
if (type === "tool") {
|
|
162285
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162286
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162287
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162288
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162289
|
+
return {
|
|
162290
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162291
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162292
|
+
title,
|
|
162293
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162294
|
+
output
|
|
162295
|
+
};
|
|
162296
|
+
}
|
|
162297
|
+
if (type === "tool_use") {
|
|
162298
|
+
return {
|
|
162299
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162300
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162301
|
+
title: null,
|
|
162302
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162303
|
+
output: null
|
|
162304
|
+
};
|
|
162305
|
+
}
|
|
162306
|
+
if (type === "tool_result") {
|
|
162307
|
+
const content = part.content;
|
|
162308
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162309
|
+
return {
|
|
162310
|
+
name: "tool_result",
|
|
162311
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162312
|
+
title: null,
|
|
162313
|
+
input: null,
|
|
162314
|
+
output
|
|
162315
|
+
};
|
|
162316
|
+
}
|
|
162317
|
+
return null;
|
|
162318
|
+
}
|
|
162319
|
+
function textOf(part) {
|
|
162320
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162321
|
+
return part.text;
|
|
162322
|
+
return null;
|
|
162323
|
+
}
|
|
162324
|
+
function reasoningOf(part) {
|
|
162325
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162326
|
+
return part.text;
|
|
162327
|
+
}
|
|
162328
|
+
return null;
|
|
162329
|
+
}
|
|
162330
|
+
function renderPartPreview(part) {
|
|
162331
|
+
if (!isRecord2(part))
|
|
162332
|
+
return null;
|
|
162333
|
+
const text = textOf(part);
|
|
162334
|
+
if (text !== null) {
|
|
162335
|
+
const t = truncate(text, 200);
|
|
162336
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162337
|
+
}
|
|
162338
|
+
const tool = asToolPart(part);
|
|
162339
|
+
if (tool) {
|
|
162340
|
+
const arg = keyArg(tool.input);
|
|
162341
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162342
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162343
|
+
}
|
|
162344
|
+
const reasoning = reasoningOf(part);
|
|
162345
|
+
if (reasoning !== null)
|
|
162346
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162347
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162348
|
+
if (type === "file")
|
|
162349
|
+
return " • [file]";
|
|
162350
|
+
if (type === "step-start" || type === "step-finish")
|
|
162351
|
+
return null;
|
|
162352
|
+
return ` • [${type}]`;
|
|
162353
|
+
}
|
|
162354
|
+
function renderPartFull(part) {
|
|
162355
|
+
if (!isRecord2(part))
|
|
162356
|
+
return null;
|
|
162357
|
+
const text = textOf(part);
|
|
162358
|
+
if (text !== null) {
|
|
162359
|
+
return text.trim().length > 0 ? ` [text]
|
|
162360
|
+
${text}` : null;
|
|
162361
|
+
}
|
|
162362
|
+
const tool = asToolPart(part);
|
|
162363
|
+
if (tool) {
|
|
162364
|
+
const lines = [];
|
|
162365
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162366
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162367
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162368
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162369
|
+
}
|
|
162370
|
+
if (tool.input)
|
|
162371
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162372
|
+
if (tool.output !== null)
|
|
162373
|
+
lines.push(` output:
|
|
162374
|
+
${tool.output}`);
|
|
162375
|
+
return lines.join(`
|
|
162376
|
+
`);
|
|
162377
|
+
}
|
|
162378
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162379
|
+
if (type === "file") {
|
|
162380
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162381
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162382
|
+
}
|
|
162383
|
+
return null;
|
|
162384
|
+
}
|
|
162385
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162386
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162387
|
+
if (!msg) {
|
|
162388
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162389
|
+
}
|
|
162390
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162391
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162392
|
+
if (rendered.length === 0) {
|
|
162393
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162394
|
+
} else {
|
|
162395
|
+
lines.push(...rendered);
|
|
162396
|
+
}
|
|
162397
|
+
return lines.join(`
|
|
162398
|
+
`);
|
|
162399
|
+
}
|
|
162400
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162401
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162402
|
+
const out = [];
|
|
162403
|
+
let usedTokens = 0;
|
|
162404
|
+
let lastOrdinal = start - 1;
|
|
162405
|
+
let truncated = false;
|
|
162406
|
+
for (const msg of messages) {
|
|
162407
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162408
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162409
|
+
const block = partLines.length > 0 ? `${header}
|
|
162410
|
+
${partLines.join(`
|
|
162411
|
+
`)}` : header;
|
|
162412
|
+
const blockTokens = estimateTokens(block);
|
|
162413
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162414
|
+
truncated = true;
|
|
162415
|
+
break;
|
|
162416
|
+
}
|
|
162417
|
+
out.push(block);
|
|
162418
|
+
usedTokens += blockTokens;
|
|
162419
|
+
lastOrdinal = msg.ordinal;
|
|
162420
|
+
}
|
|
162421
|
+
return { text: out.join(`
|
|
162422
|
+
|
|
162423
|
+
`), lastOrdinal, truncated };
|
|
162424
|
+
}
|
|
162425
|
+
|
|
161983
162426
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161984
162427
|
var exports_memory = {};
|
|
161985
162428
|
__export(exports_memory, {
|
|
@@ -166336,12 +166779,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166336
166779
|
|
|
166337
166780
|
// src/tools/ctx-expand.ts
|
|
166338
166781
|
var ParamsSchema = exports_typebox.Object({
|
|
166339
|
-
start: exports_typebox.Number({
|
|
166782
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166340
166783
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166341
|
-
}),
|
|
166342
|
-
end: exports_typebox.Number({
|
|
166784
|
+
})),
|
|
166785
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166343
166786
|
description: "End message ordinal (from compartment end attribute)"
|
|
166344
|
-
})
|
|
166787
|
+
})),
|
|
166788
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166789
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166790
|
+
})),
|
|
166791
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166792
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166793
|
+
}))
|
|
166345
166794
|
});
|
|
166346
166795
|
function ok(text) {
|
|
166347
166796
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166360,22 +166809,41 @@ function createCtxExpandTool(deps) {
|
|
|
166360
166809
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166361
166810
|
parameters: ParamsSchema,
|
|
166362
166811
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166363
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166364
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166365
|
-
}
|
|
166366
166812
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166367
166813
|
if (!sessionId) {
|
|
166368
166814
|
return err("Error: no active Pi session.");
|
|
166369
166815
|
}
|
|
166370
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166371
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166372
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166373
|
-
}
|
|
166374
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166375
166816
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166376
166817
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166377
166818
|
});
|
|
166378
166819
|
try {
|
|
166820
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166821
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166822
|
+
}
|
|
166823
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166824
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166825
|
+
}
|
|
166826
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166827
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166828
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166829
|
+
}
|
|
166830
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166831
|
+
if (params.verbose === true) {
|
|
166832
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166833
|
+
if (!v.text) {
|
|
166834
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166835
|
+
}
|
|
166836
|
+
const out = [
|
|
166837
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166838
|
+
"",
|
|
166839
|
+
v.text
|
|
166840
|
+
];
|
|
166841
|
+
if (v.truncated) {
|
|
166842
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166843
|
+
}
|
|
166844
|
+
return ok(out.join(`
|
|
166845
|
+
`));
|
|
166846
|
+
}
|
|
166379
166847
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166380
166848
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166381
166849
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -168235,6 +168703,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168235
168703
|
last_transform_error: "'' AS last_transform_error",
|
|
168236
168704
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168237
168705
|
last_todo_state: "'' AS last_todo_state",
|
|
168706
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168238
168707
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168239
168708
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168240
168709
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168312,6 +168781,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168312
168781
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168313
168782
|
})();
|
|
168314
168783
|
}
|
|
168784
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168785
|
+
if (maxTagNumber <= 0)
|
|
168786
|
+
return;
|
|
168787
|
+
db.transaction(() => {
|
|
168788
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168789
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168790
|
+
})();
|
|
168791
|
+
}
|
|
168315
168792
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168316
168793
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168317
168794
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168709,6 +169186,26 @@ function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
|
168709
169186
|
nullCount: row?.null_count ?? 0
|
|
168710
169187
|
};
|
|
168711
169188
|
}
|
|
169189
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169190
|
+
if (limit <= 0)
|
|
169191
|
+
return [];
|
|
169192
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169193
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169194
|
+
SELECT tag_number FROM tags
|
|
169195
|
+
WHERE session_id = ? AND status = 'active'
|
|
169196
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169197
|
+
)` : "";
|
|
169198
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169199
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169200
|
+
FROM tags
|
|
169201
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169202
|
+
ORDER BY tag_number ASC, id ASC
|
|
169203
|
+
LIMIT ?`).all(...params);
|
|
169204
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169205
|
+
tagNumber: row.tag_number,
|
|
169206
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169207
|
+
}));
|
|
169208
|
+
}
|
|
168712
169209
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
168713
169210
|
const row = db.prepare(`SELECT
|
|
168714
169211
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -169658,15 +170155,16 @@ function parseInteger(str) {
|
|
|
169658
170155
|
}
|
|
169659
170156
|
|
|
169660
170157
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169661
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169662
|
-
|
|
170158
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170159
|
+
|
|
170160
|
+
How it works:
|
|
170161
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170162
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170163
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
169663
170164
|
|
|
169664
|
-
|
|
169665
|
-
|
|
169666
|
-
-
|
|
169667
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
169668
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
169669
|
-
- Dropped content is gone forever.`;
|
|
170165
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170166
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170167
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169670
170168
|
|
|
169671
170169
|
// src/tools/ctx-reduce.ts
|
|
169672
170170
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169851,120 +170349,6 @@ ${body}` : subject;
|
|
|
169851
170349
|
}
|
|
169852
170350
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169853
170351
|
init_logger();
|
|
169854
|
-
|
|
169855
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169856
|
-
init_logger();
|
|
169857
|
-
var insertStatements = new WeakMap;
|
|
169858
|
-
var existingShasStatements = new WeakMap;
|
|
169859
|
-
var projectCountStatements = new WeakMap;
|
|
169860
|
-
var evictStatements = new WeakMap;
|
|
169861
|
-
var evictOverflowStatements = new WeakMap;
|
|
169862
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169863
|
-
function getInsertStatement(db) {
|
|
169864
|
-
let stmt = insertStatements.get(db);
|
|
169865
|
-
if (!stmt) {
|
|
169866
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169867
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169868
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169869
|
-
project_path = excluded.project_path,
|
|
169870
|
-
short_sha = excluded.short_sha,
|
|
169871
|
-
message = excluded.message,
|
|
169872
|
-
author = excluded.author,
|
|
169873
|
-
committed_at = excluded.committed_at,
|
|
169874
|
-
indexed_at = excluded.indexed_at
|
|
169875
|
-
WHERE git_commits.message != excluded.message`);
|
|
169876
|
-
insertStatements.set(db, stmt);
|
|
169877
|
-
}
|
|
169878
|
-
return stmt;
|
|
169879
|
-
}
|
|
169880
|
-
function getExistingShasStatement(db) {
|
|
169881
|
-
let stmt = existingShasStatements.get(db);
|
|
169882
|
-
if (!stmt) {
|
|
169883
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169884
|
-
existingShasStatements.set(db, stmt);
|
|
169885
|
-
}
|
|
169886
|
-
return stmt;
|
|
169887
|
-
}
|
|
169888
|
-
function getProjectCountStatement(db) {
|
|
169889
|
-
let stmt = projectCountStatements.get(db);
|
|
169890
|
-
if (!stmt) {
|
|
169891
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169892
|
-
projectCountStatements.set(db, stmt);
|
|
169893
|
-
}
|
|
169894
|
-
return stmt;
|
|
169895
|
-
}
|
|
169896
|
-
function getLatestCommitTimeStatement(db) {
|
|
169897
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169898
|
-
if (!stmt) {
|
|
169899
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169900
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169901
|
-
}
|
|
169902
|
-
return stmt;
|
|
169903
|
-
}
|
|
169904
|
-
function getEvictOverflowStatement(db) {
|
|
169905
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169906
|
-
if (!stmt) {
|
|
169907
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169908
|
-
WHERE rowid IN (
|
|
169909
|
-
SELECT rowid FROM git_commits
|
|
169910
|
-
WHERE project_path = ?
|
|
169911
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169912
|
-
LIMIT -1 OFFSET ?
|
|
169913
|
-
)`);
|
|
169914
|
-
evictOverflowStatements.set(db, stmt);
|
|
169915
|
-
}
|
|
169916
|
-
return stmt;
|
|
169917
|
-
}
|
|
169918
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169919
|
-
if (commits.length === 0)
|
|
169920
|
-
return { inserted: 0, updated: 0 };
|
|
169921
|
-
const existing = new Set;
|
|
169922
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169923
|
-
existing.add(row.sha);
|
|
169924
|
-
}
|
|
169925
|
-
let inserted = 0;
|
|
169926
|
-
let updated = 0;
|
|
169927
|
-
const now = Date.now();
|
|
169928
|
-
const insertStmt = getInsertStatement(db);
|
|
169929
|
-
db.transaction(() => {
|
|
169930
|
-
for (const commit of commits) {
|
|
169931
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169932
|
-
if (result.changes > 0) {
|
|
169933
|
-
if (existing.has(commit.sha)) {
|
|
169934
|
-
updated++;
|
|
169935
|
-
} else {
|
|
169936
|
-
inserted++;
|
|
169937
|
-
existing.add(commit.sha);
|
|
169938
|
-
}
|
|
169939
|
-
}
|
|
169940
|
-
}
|
|
169941
|
-
})();
|
|
169942
|
-
return { inserted, updated };
|
|
169943
|
-
}
|
|
169944
|
-
function getCommitCount(db, projectPath) {
|
|
169945
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169946
|
-
return row?.count ?? 0;
|
|
169947
|
-
}
|
|
169948
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169949
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169950
|
-
return row?.latest ?? null;
|
|
169951
|
-
}
|
|
169952
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169953
|
-
if (maxCommits <= 0)
|
|
169954
|
-
return 0;
|
|
169955
|
-
const count = getCommitCount(db, projectPath);
|
|
169956
|
-
if (count <= maxCommits)
|
|
169957
|
-
return 0;
|
|
169958
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169959
|
-
const after = getCommitCount(db, projectPath);
|
|
169960
|
-
const evicted = Math.max(0, count - after);
|
|
169961
|
-
if (evicted > 0) {
|
|
169962
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169963
|
-
}
|
|
169964
|
-
return evicted;
|
|
169965
|
-
}
|
|
169966
|
-
|
|
169967
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169968
170352
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169969
170353
|
var EMBED_BATCH_SIZE = 16;
|
|
169970
170354
|
var EMBED_MAX_PER_SWEEP = 500;
|