@wolfx/pi-magic-context 0.24.1 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1008 -352
- package/dist/subagent-entry.js +559 -175
- package/package.json +3 -3
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -143218,6 +143226,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143226
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143227
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143228
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143229
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143230
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143231
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143232
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -159142,6 +159151,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159142
159151
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159143
159152
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159144
159153
|
}
|
|
159154
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159155
|
+
const row = db.prepare(`SELECT
|
|
159156
|
+
COUNT(*) AS total,
|
|
159157
|
+
SUM(CASE WHEN EXISTS (
|
|
159158
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159159
|
+
WHERE e.compartment_id = c.id
|
|
159160
|
+
AND e.project_path = ?
|
|
159161
|
+
AND e.model_id = ?
|
|
159162
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159163
|
+
FROM compartments c
|
|
159164
|
+
JOIN session_projects sp
|
|
159165
|
+
ON sp.session_id = c.session_id
|
|
159166
|
+
AND sp.harness = c.harness
|
|
159167
|
+
AND sp.project_path = ?
|
|
159168
|
+
WHERE c.session_id = ?
|
|
159169
|
+
AND c.start_message IS NOT NULL
|
|
159170
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159171
|
+
return {
|
|
159172
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159173
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159174
|
+
};
|
|
159175
|
+
}
|
|
159145
159176
|
|
|
159146
159177
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159147
159178
|
function cosineSimilarity(a, b) {
|
|
@@ -159294,6 +159325,19 @@ async function withQuietConsole(fn) {
|
|
|
159294
159325
|
console.error = origError;
|
|
159295
159326
|
}
|
|
159296
159327
|
}
|
|
159328
|
+
var nativeRuntimeMissing = false;
|
|
159329
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159330
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159331
|
+
const lower = message.toLowerCase();
|
|
159332
|
+
const code = error51?.code;
|
|
159333
|
+
const name2 = error51?.name;
|
|
159334
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159335
|
+
return true;
|
|
159336
|
+
}
|
|
159337
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159338
|
+
return false;
|
|
159339
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159340
|
+
}
|
|
159297
159341
|
function isTransientLoadError(error51) {
|
|
159298
159342
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159299
159343
|
if (!message)
|
|
@@ -159358,6 +159402,9 @@ class LocalEmbeddingProvider {
|
|
|
159358
159402
|
if (this.pipeline) {
|
|
159359
159403
|
return true;
|
|
159360
159404
|
}
|
|
159405
|
+
if (nativeRuntimeMissing) {
|
|
159406
|
+
return false;
|
|
159407
|
+
}
|
|
159361
159408
|
if (this.initPromise) {
|
|
159362
159409
|
await this.initPromise;
|
|
159363
159410
|
return this.pipeline !== null;
|
|
@@ -159424,7 +159471,12 @@ class LocalEmbeddingProvider {
|
|
|
159424
159471
|
await releaseLock();
|
|
159425
159472
|
}
|
|
159426
159473
|
} catch (error51) {
|
|
159427
|
-
|
|
159474
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159475
|
+
nativeRuntimeMissing = true;
|
|
159476
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @wolfx/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159477
|
+
} else {
|
|
159478
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159479
|
+
}
|
|
159428
159480
|
this.pipeline = null;
|
|
159429
159481
|
} finally {
|
|
159430
159482
|
this.initPromise = null;
|
|
@@ -159910,6 +159962,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159910
159962
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159911
159963
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159912
159964
|
}
|
|
159965
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
159966
|
+
const row = db.prepare(`SELECT
|
|
159967
|
+
COUNT(*) AS total,
|
|
159968
|
+
SUM(CASE WHEN EXISTS (
|
|
159969
|
+
SELECT 1 FROM memory_embeddings e
|
|
159970
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
159971
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159972
|
+
FROM memories m
|
|
159973
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
159974
|
+
return {
|
|
159975
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159976
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159977
|
+
};
|
|
159978
|
+
}
|
|
159913
159979
|
|
|
159914
159980
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159915
159981
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -160016,6 +160082,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
160016
160082
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
160017
160083
|
}
|
|
160018
160084
|
|
|
160085
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160086
|
+
init_logger();
|
|
160087
|
+
var insertStatements = new WeakMap;
|
|
160088
|
+
var existingShasStatements = new WeakMap;
|
|
160089
|
+
var projectCountStatements = new WeakMap;
|
|
160090
|
+
var evictStatements = new WeakMap;
|
|
160091
|
+
var evictOverflowStatements = new WeakMap;
|
|
160092
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160093
|
+
function getInsertStatement(db) {
|
|
160094
|
+
let stmt = insertStatements.get(db);
|
|
160095
|
+
if (!stmt) {
|
|
160096
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160097
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160098
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160099
|
+
project_path = excluded.project_path,
|
|
160100
|
+
short_sha = excluded.short_sha,
|
|
160101
|
+
message = excluded.message,
|
|
160102
|
+
author = excluded.author,
|
|
160103
|
+
committed_at = excluded.committed_at,
|
|
160104
|
+
indexed_at = excluded.indexed_at
|
|
160105
|
+
WHERE git_commits.message != excluded.message`);
|
|
160106
|
+
insertStatements.set(db, stmt);
|
|
160107
|
+
}
|
|
160108
|
+
return stmt;
|
|
160109
|
+
}
|
|
160110
|
+
function getExistingShasStatement(db) {
|
|
160111
|
+
let stmt = existingShasStatements.get(db);
|
|
160112
|
+
if (!stmt) {
|
|
160113
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160114
|
+
existingShasStatements.set(db, stmt);
|
|
160115
|
+
}
|
|
160116
|
+
return stmt;
|
|
160117
|
+
}
|
|
160118
|
+
function getProjectCountStatement(db) {
|
|
160119
|
+
let stmt = projectCountStatements.get(db);
|
|
160120
|
+
if (!stmt) {
|
|
160121
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160122
|
+
projectCountStatements.set(db, stmt);
|
|
160123
|
+
}
|
|
160124
|
+
return stmt;
|
|
160125
|
+
}
|
|
160126
|
+
function getLatestCommitTimeStatement(db) {
|
|
160127
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160128
|
+
if (!stmt) {
|
|
160129
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160130
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160131
|
+
}
|
|
160132
|
+
return stmt;
|
|
160133
|
+
}
|
|
160134
|
+
function getEvictOverflowStatement(db) {
|
|
160135
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160136
|
+
if (!stmt) {
|
|
160137
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160138
|
+
WHERE rowid IN (
|
|
160139
|
+
SELECT rowid FROM git_commits
|
|
160140
|
+
WHERE project_path = ?
|
|
160141
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160142
|
+
LIMIT -1 OFFSET ?
|
|
160143
|
+
)`);
|
|
160144
|
+
evictOverflowStatements.set(db, stmt);
|
|
160145
|
+
}
|
|
160146
|
+
return stmt;
|
|
160147
|
+
}
|
|
160148
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160149
|
+
if (commits.length === 0)
|
|
160150
|
+
return { inserted: 0, updated: 0 };
|
|
160151
|
+
const existing = new Set;
|
|
160152
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160153
|
+
existing.add(row.sha);
|
|
160154
|
+
}
|
|
160155
|
+
let inserted = 0;
|
|
160156
|
+
let updated = 0;
|
|
160157
|
+
const now = Date.now();
|
|
160158
|
+
const insertStmt = getInsertStatement(db);
|
|
160159
|
+
db.transaction(() => {
|
|
160160
|
+
for (const commit of commits) {
|
|
160161
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160162
|
+
if (result.changes > 0) {
|
|
160163
|
+
if (existing.has(commit.sha)) {
|
|
160164
|
+
updated++;
|
|
160165
|
+
} else {
|
|
160166
|
+
inserted++;
|
|
160167
|
+
existing.add(commit.sha);
|
|
160168
|
+
}
|
|
160169
|
+
}
|
|
160170
|
+
}
|
|
160171
|
+
})();
|
|
160172
|
+
return { inserted, updated };
|
|
160173
|
+
}
|
|
160174
|
+
function getCommitCount(db, projectPath) {
|
|
160175
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160176
|
+
return row?.count ?? 0;
|
|
160177
|
+
}
|
|
160178
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160179
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160180
|
+
return row?.latest ?? null;
|
|
160181
|
+
}
|
|
160182
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160183
|
+
if (maxCommits <= 0)
|
|
160184
|
+
return 0;
|
|
160185
|
+
const count = getCommitCount(db, projectPath);
|
|
160186
|
+
if (count <= maxCommits)
|
|
160187
|
+
return 0;
|
|
160188
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160189
|
+
const after = getCommitCount(db, projectPath);
|
|
160190
|
+
const evicted = Math.max(0, count - after);
|
|
160191
|
+
if (evicted > 0) {
|
|
160192
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160193
|
+
}
|
|
160194
|
+
return evicted;
|
|
160195
|
+
}
|
|
160196
|
+
|
|
160019
160197
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
160020
160198
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
160021
160199
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160248,8 +160426,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160248
160426
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160249
160427
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160250
160428
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160251
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160429
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160252
160430
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160431
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160432
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160433
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160434
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160253
160435
|
var projectRegistrations = new Map;
|
|
160254
160436
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160255
160437
|
var globalRegistrationGeneration = 0;
|
|
@@ -160349,7 +160531,9 @@ function snapshotFor(registration) {
|
|
|
160349
160531
|
enabled,
|
|
160350
160532
|
gitCommitEnabled,
|
|
160351
160533
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160352
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160534
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160535
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160536
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160353
160537
|
};
|
|
160354
160538
|
}
|
|
160355
160539
|
function disposeProvider(provider) {
|
|
@@ -160560,8 +160744,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160560
160744
|
}
|
|
160561
160745
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160562
160746
|
const noWork = [];
|
|
160747
|
+
const failed = [];
|
|
160563
160748
|
if (candidates.length === 0)
|
|
160564
|
-
return { embedded: 0, noWork };
|
|
160749
|
+
return { embedded: 0, noWork, failed };
|
|
160565
160750
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160566
160751
|
const prepared = [];
|
|
160567
160752
|
for (const candidate of candidates) {
|
|
@@ -160578,7 +160763,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160578
160763
|
prepared.push({ candidate, windows });
|
|
160579
160764
|
}
|
|
160580
160765
|
if (prepared.length === 0)
|
|
160581
|
-
return { embedded: 0, noWork };
|
|
160766
|
+
return { embedded: 0, noWork, failed };
|
|
160582
160767
|
let embedded = 0;
|
|
160583
160768
|
let i = 0;
|
|
160584
160769
|
while (i < prepared.length) {
|
|
@@ -160595,35 +160780,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160595
160780
|
const texts = [];
|
|
160596
160781
|
for (const item of slice)
|
|
160597
160782
|
texts.push(...item.windows.map((w) => w.text));
|
|
160598
|
-
|
|
160599
|
-
|
|
160600
|
-
if (!result)
|
|
160601
|
-
continue;
|
|
160783
|
+
const persistedIds = new Set;
|
|
160784
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160602
160785
|
if (signal?.aborted)
|
|
160603
160786
|
break;
|
|
160604
|
-
let
|
|
160605
|
-
|
|
160606
|
-
|
|
160607
|
-
|
|
160608
|
-
|
|
160609
|
-
|
|
160787
|
+
let result = null;
|
|
160788
|
+
const attemptStart = Date.now();
|
|
160789
|
+
try {
|
|
160790
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160791
|
+
} catch (error51) {
|
|
160792
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160793
|
+
}
|
|
160794
|
+
if (signal?.aborted)
|
|
160795
|
+
break;
|
|
160796
|
+
if (result) {
|
|
160797
|
+
let offset = 0;
|
|
160798
|
+
for (const item of slice) {
|
|
160799
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160800
|
+
offset += item.windows.length;
|
|
160801
|
+
if (persistedIds.has(item.candidate.id))
|
|
160802
|
+
continue;
|
|
160803
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160804
|
+
continue;
|
|
160805
|
+
}
|
|
160806
|
+
const rows = item.windows.map((window, index) => ({
|
|
160807
|
+
compartmentId: item.candidate.id,
|
|
160808
|
+
sessionId: item.candidate.sessionId,
|
|
160809
|
+
projectPath: projectIdentity,
|
|
160810
|
+
window,
|
|
160811
|
+
modelId,
|
|
160812
|
+
vector: vectors[index]
|
|
160813
|
+
}));
|
|
160814
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160815
|
+
persistedIds.add(item.candidate.id);
|
|
160610
160816
|
}
|
|
160611
|
-
const rows = item.windows.map((window, index) => ({
|
|
160612
|
-
compartmentId: item.candidate.id,
|
|
160613
|
-
sessionId: item.candidate.sessionId,
|
|
160614
|
-
projectPath: projectIdentity,
|
|
160615
|
-
window,
|
|
160616
|
-
modelId,
|
|
160617
|
-
vector: vectors[index]
|
|
160618
|
-
}));
|
|
160619
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160620
|
-
embedded += 1;
|
|
160621
160817
|
}
|
|
160622
|
-
|
|
160623
|
-
|
|
160818
|
+
if (persistedIds.size === slice.length)
|
|
160819
|
+
break;
|
|
160820
|
+
if (persistedIds.size > 0)
|
|
160821
|
+
break;
|
|
160822
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160823
|
+
break;
|
|
160824
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160825
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160826
|
+
}
|
|
160827
|
+
}
|
|
160828
|
+
embedded += persistedIds.size;
|
|
160829
|
+
if (!signal?.aborted) {
|
|
160830
|
+
for (const item of slice) {
|
|
160831
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160832
|
+
failed.push(item.candidate.id);
|
|
160833
|
+
}
|
|
160624
160834
|
}
|
|
160625
160835
|
}
|
|
160626
|
-
return { embedded, noWork };
|
|
160836
|
+
return { embedded, noWork, failed };
|
|
160627
160837
|
}
|
|
160628
160838
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160629
160839
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160646,9 +160856,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160646
160856
|
renewal.unref?.();
|
|
160647
160857
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160648
160858
|
const skipIds = [];
|
|
160859
|
+
const failedIds = [];
|
|
160649
160860
|
let embedded = 0;
|
|
160650
160861
|
let aborted2 = false;
|
|
160651
|
-
let
|
|
160862
|
+
let providerDown = false;
|
|
160863
|
+
let consecutiveFailedBatches = 0;
|
|
160652
160864
|
try {
|
|
160653
160865
|
options?.onProgress?.({ embedded, total });
|
|
160654
160866
|
for (;; ) {
|
|
@@ -160656,15 +160868,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160656
160868
|
aborted2 = true;
|
|
160657
160869
|
break;
|
|
160658
160870
|
}
|
|
160659
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160871
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160660
160872
|
if (candidates.length === 0)
|
|
160661
160873
|
break;
|
|
160662
|
-
const {
|
|
160874
|
+
const {
|
|
160875
|
+
embedded: n,
|
|
160876
|
+
noWork,
|
|
160877
|
+
failed
|
|
160878
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160663
160879
|
for (const id of noWork)
|
|
160664
160880
|
skipIds.push(id);
|
|
160881
|
+
for (const id of failed)
|
|
160882
|
+
failedIds.push(id);
|
|
160665
160883
|
if (n === 0 && noWork.length === 0) {
|
|
160666
|
-
|
|
160667
|
-
|
|
160884
|
+
consecutiveFailedBatches += 1;
|
|
160885
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160886
|
+
providerDown = true;
|
|
160887
|
+
break;
|
|
160888
|
+
}
|
|
160889
|
+
} else {
|
|
160890
|
+
consecutiveFailedBatches = 0;
|
|
160668
160891
|
}
|
|
160669
160892
|
embedded += n;
|
|
160670
160893
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160672,16 +160895,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160672
160895
|
}
|
|
160673
160896
|
} finally {
|
|
160674
160897
|
clearInterval(renewal);
|
|
160675
|
-
|
|
160898
|
+
try {
|
|
160899
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
160900
|
+
} catch (error51) {
|
|
160901
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
160902
|
+
}
|
|
160676
160903
|
}
|
|
160677
160904
|
if (aborted2)
|
|
160678
|
-
return { status: "aborted", embedded, total };
|
|
160679
|
-
if (
|
|
160905
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
160906
|
+
if (providerDown || failedIds.length > 0) {
|
|
160680
160907
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160681
|
-
if (remaining > 0)
|
|
160682
|
-
return { status: "stalled", embedded, total, remaining };
|
|
160908
|
+
if (remaining > 0) {
|
|
160909
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
160910
|
+
}
|
|
160683
160911
|
}
|
|
160684
|
-
return { status: "done", embedded, total };
|
|
160912
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
160913
|
+
}
|
|
160914
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
160915
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
160916
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
160917
|
+
return {
|
|
160918
|
+
enabled: false,
|
|
160919
|
+
model: snapshot?.model ?? "off",
|
|
160920
|
+
provider: snapshot?.provider ?? "off",
|
|
160921
|
+
session: { embedded: 0, total: 0 },
|
|
160922
|
+
memories: { embedded: 0, total: 0 },
|
|
160923
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
160924
|
+
};
|
|
160925
|
+
}
|
|
160926
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
160927
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
160928
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
160929
|
+
const commits = gitEnabled ? {
|
|
160930
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
160931
|
+
total: getCommitCount(db, projectIdentity),
|
|
160932
|
+
gitEnabled: true
|
|
160933
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
160934
|
+
return {
|
|
160935
|
+
enabled: true,
|
|
160936
|
+
model: snapshot.model,
|
|
160937
|
+
provider: snapshot.provider,
|
|
160938
|
+
session,
|
|
160939
|
+
memories,
|
|
160940
|
+
commits
|
|
160941
|
+
};
|
|
160685
160942
|
}
|
|
160686
160943
|
|
|
160687
160944
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160910,6 +161167,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160910
161167
|
"conversation_tokens",
|
|
160911
161168
|
"tool_call_tokens",
|
|
160912
161169
|
"cleared_reasoning_through_tag",
|
|
161170
|
+
"tool_reclaim_watermark",
|
|
160913
161171
|
"last_todo_state",
|
|
160914
161172
|
"cached_m0_bytes",
|
|
160915
161173
|
"cached_m1_bytes",
|
|
@@ -160958,6 +161216,7 @@ var META_COLUMNS = {
|
|
|
160958
161216
|
conversationTokens: "conversation_tokens",
|
|
160959
161217
|
toolCallTokens: "tool_call_tokens",
|
|
160960
161218
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161219
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160961
161220
|
lastTodoState: "last_todo_state",
|
|
160962
161221
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160963
161222
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -161026,7 +161285,7 @@ function isSessionMetaRow(row) {
|
|
|
161026
161285
|
if (row === null || typeof row !== "object")
|
|
161027
161286
|
return false;
|
|
161028
161287
|
const r = row;
|
|
161029
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161288
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161030
161289
|
}
|
|
161031
161290
|
function getDefaultSessionMeta(sessionId) {
|
|
161032
161291
|
return {
|
|
@@ -161049,6 +161308,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161049
161308
|
conversationTokens: 0,
|
|
161050
161309
|
toolCallTokens: 0,
|
|
161051
161310
|
clearedReasoningThroughTag: 0,
|
|
161311
|
+
toolReclaimWatermark: 0,
|
|
161052
161312
|
lastTodoState: "",
|
|
161053
161313
|
cachedM0Bytes: null,
|
|
161054
161314
|
cachedM1Bytes: null,
|
|
@@ -161112,6 +161372,7 @@ function toSessionMeta(row) {
|
|
|
161112
161372
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161113
161373
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161114
161374
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161375
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161115
161376
|
lastTodoState: lastTodoStateRaw,
|
|
161116
161377
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161117
161378
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161538,6 +161799,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161538
161799
|
var encoder = new TextEncoder;
|
|
161539
161800
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161540
161801
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161802
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161803
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161541
161804
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161542
161805
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161543
161806
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161550,6 +161813,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161550
161813
|
function stripMalformedTagNotationGlobally(value) {
|
|
161551
161814
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161552
161815
|
}
|
|
161816
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161817
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161818
|
+
}
|
|
161553
161819
|
function stripTagSectionCharacters(value) {
|
|
161554
161820
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161555
161821
|
}
|
|
@@ -161557,6 +161823,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161557
161823
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161558
161824
|
text = stripCompleteTagPairsGlobally(text);
|
|
161559
161825
|
text = stripMalformedTagNotationGlobally(text);
|
|
161826
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161560
161827
|
text = stripTagSectionCharacters(text);
|
|
161561
161828
|
return text.trim();
|
|
161562
161829
|
}
|
|
@@ -161569,6 +161836,7 @@ function stripTagPrefix(value) {
|
|
|
161569
161836
|
const prev = stripped;
|
|
161570
161837
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161571
161838
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161839
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161572
161840
|
if (stripped === prev)
|
|
161573
161841
|
break;
|
|
161574
161842
|
}
|
|
@@ -161984,9 +162252,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161984
162252
|
|
|
161985
162253
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161986
162254
|
|
|
161987
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162255
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162256
|
+
|
|
162257
|
+
Two recovery modes for finer detail:
|
|
162258
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162259
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161988
162260
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161989
162261
|
|
|
162262
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162263
|
+
function isRecord2(value) {
|
|
162264
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162265
|
+
}
|
|
162266
|
+
function roleLabel(role) {
|
|
162267
|
+
if (role === "assistant")
|
|
162268
|
+
return "A (assistant)";
|
|
162269
|
+
if (role === "user")
|
|
162270
|
+
return "U (user)";
|
|
162271
|
+
return role;
|
|
162272
|
+
}
|
|
162273
|
+
function truncate(value, max) {
|
|
162274
|
+
const t = value.trim();
|
|
162275
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162276
|
+
}
|
|
162277
|
+
function keyArg(input) {
|
|
162278
|
+
if (!input)
|
|
162279
|
+
return "";
|
|
162280
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162281
|
+
const v = input[k];
|
|
162282
|
+
if (typeof v === "string" && v.length > 0)
|
|
162283
|
+
return truncate(v, 60);
|
|
162284
|
+
}
|
|
162285
|
+
if (typeof input.description === "string")
|
|
162286
|
+
return truncate(input.description, 60);
|
|
162287
|
+
return "";
|
|
162288
|
+
}
|
|
162289
|
+
function asToolPart(part) {
|
|
162290
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162291
|
+
if (type === "tool") {
|
|
162292
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162293
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162294
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162295
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162296
|
+
return {
|
|
162297
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162298
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162299
|
+
title,
|
|
162300
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162301
|
+
output
|
|
162302
|
+
};
|
|
162303
|
+
}
|
|
162304
|
+
if (type === "tool_use") {
|
|
162305
|
+
return {
|
|
162306
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162307
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162308
|
+
title: null,
|
|
162309
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162310
|
+
output: null
|
|
162311
|
+
};
|
|
162312
|
+
}
|
|
162313
|
+
if (type === "tool_result") {
|
|
162314
|
+
const content = part.content;
|
|
162315
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162316
|
+
return {
|
|
162317
|
+
name: "tool_result",
|
|
162318
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162319
|
+
title: null,
|
|
162320
|
+
input: null,
|
|
162321
|
+
output
|
|
162322
|
+
};
|
|
162323
|
+
}
|
|
162324
|
+
return null;
|
|
162325
|
+
}
|
|
162326
|
+
function textOf(part) {
|
|
162327
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162328
|
+
return part.text;
|
|
162329
|
+
return null;
|
|
162330
|
+
}
|
|
162331
|
+
function reasoningOf(part) {
|
|
162332
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162333
|
+
return part.text;
|
|
162334
|
+
}
|
|
162335
|
+
return null;
|
|
162336
|
+
}
|
|
162337
|
+
function renderPartPreview(part) {
|
|
162338
|
+
if (!isRecord2(part))
|
|
162339
|
+
return null;
|
|
162340
|
+
const text = textOf(part);
|
|
162341
|
+
if (text !== null) {
|
|
162342
|
+
const t = truncate(text, 200);
|
|
162343
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162344
|
+
}
|
|
162345
|
+
const tool = asToolPart(part);
|
|
162346
|
+
if (tool) {
|
|
162347
|
+
const arg = keyArg(tool.input);
|
|
162348
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162349
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162350
|
+
}
|
|
162351
|
+
const reasoning = reasoningOf(part);
|
|
162352
|
+
if (reasoning !== null)
|
|
162353
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162354
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162355
|
+
if (type === "file")
|
|
162356
|
+
return " • [file]";
|
|
162357
|
+
if (type === "step-start" || type === "step-finish")
|
|
162358
|
+
return null;
|
|
162359
|
+
return ` • [${type}]`;
|
|
162360
|
+
}
|
|
162361
|
+
function renderPartFull(part) {
|
|
162362
|
+
if (!isRecord2(part))
|
|
162363
|
+
return null;
|
|
162364
|
+
const text = textOf(part);
|
|
162365
|
+
if (text !== null) {
|
|
162366
|
+
return text.trim().length > 0 ? ` [text]
|
|
162367
|
+
${text}` : null;
|
|
162368
|
+
}
|
|
162369
|
+
const tool = asToolPart(part);
|
|
162370
|
+
if (tool) {
|
|
162371
|
+
const lines = [];
|
|
162372
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162373
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162374
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162375
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162376
|
+
}
|
|
162377
|
+
if (tool.input)
|
|
162378
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162379
|
+
if (tool.output !== null)
|
|
162380
|
+
lines.push(` output:
|
|
162381
|
+
${tool.output}`);
|
|
162382
|
+
return lines.join(`
|
|
162383
|
+
`);
|
|
162384
|
+
}
|
|
162385
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162386
|
+
if (type === "file") {
|
|
162387
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162388
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162389
|
+
}
|
|
162390
|
+
return null;
|
|
162391
|
+
}
|
|
162392
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162393
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162394
|
+
if (!msg) {
|
|
162395
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162396
|
+
}
|
|
162397
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162398
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162399
|
+
if (rendered.length === 0) {
|
|
162400
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162401
|
+
} else {
|
|
162402
|
+
lines.push(...rendered);
|
|
162403
|
+
}
|
|
162404
|
+
return lines.join(`
|
|
162405
|
+
`);
|
|
162406
|
+
}
|
|
162407
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162408
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162409
|
+
const out = [];
|
|
162410
|
+
let usedTokens = 0;
|
|
162411
|
+
let lastOrdinal = start - 1;
|
|
162412
|
+
let truncated = false;
|
|
162413
|
+
for (const msg of messages) {
|
|
162414
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162415
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162416
|
+
const block = partLines.length > 0 ? `${header}
|
|
162417
|
+
${partLines.join(`
|
|
162418
|
+
`)}` : header;
|
|
162419
|
+
const blockTokens = estimateTokens(block);
|
|
162420
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162421
|
+
truncated = true;
|
|
162422
|
+
break;
|
|
162423
|
+
}
|
|
162424
|
+
out.push(block);
|
|
162425
|
+
usedTokens += blockTokens;
|
|
162426
|
+
lastOrdinal = msg.ordinal;
|
|
162427
|
+
}
|
|
162428
|
+
return { text: out.join(`
|
|
162429
|
+
|
|
162430
|
+
`), lastOrdinal, truncated };
|
|
162431
|
+
}
|
|
162432
|
+
|
|
161990
162433
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161991
162434
|
var exports_memory = {};
|
|
161992
162435
|
__export(exports_memory, {
|
|
@@ -166343,12 +166786,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166343
166786
|
|
|
166344
166787
|
// src/tools/ctx-expand.ts
|
|
166345
166788
|
var ParamsSchema = exports_typebox.Object({
|
|
166346
|
-
start: exports_typebox.Number({
|
|
166789
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166347
166790
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166348
|
-
}),
|
|
166349
|
-
end: exports_typebox.Number({
|
|
166791
|
+
})),
|
|
166792
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166350
166793
|
description: "End message ordinal (from compartment end attribute)"
|
|
166351
|
-
})
|
|
166794
|
+
})),
|
|
166795
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166796
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166797
|
+
})),
|
|
166798
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166799
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166800
|
+
}))
|
|
166352
166801
|
});
|
|
166353
166802
|
function ok(text) {
|
|
166354
166803
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166367,22 +166816,41 @@ function createCtxExpandTool(deps) {
|
|
|
166367
166816
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166368
166817
|
parameters: ParamsSchema,
|
|
166369
166818
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166370
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166371
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166372
|
-
}
|
|
166373
166819
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166374
166820
|
if (!sessionId) {
|
|
166375
166821
|
return err("Error: no active Pi session.");
|
|
166376
166822
|
}
|
|
166377
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166378
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166379
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166380
|
-
}
|
|
166381
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166382
166823
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166383
166824
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166384
166825
|
});
|
|
166385
166826
|
try {
|
|
166827
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166828
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166829
|
+
}
|
|
166830
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166831
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166832
|
+
}
|
|
166833
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166834
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166835
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166836
|
+
}
|
|
166837
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166838
|
+
if (params.verbose === true) {
|
|
166839
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166840
|
+
if (!v.text) {
|
|
166841
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166842
|
+
}
|
|
166843
|
+
const out = [
|
|
166844
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166845
|
+
"",
|
|
166846
|
+
v.text
|
|
166847
|
+
];
|
|
166848
|
+
if (v.truncated) {
|
|
166849
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166850
|
+
}
|
|
166851
|
+
return ok(out.join(`
|
|
166852
|
+
`));
|
|
166853
|
+
}
|
|
166386
166854
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166387
166855
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166388
166856
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -168242,6 +168710,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168242
168710
|
last_transform_error: "'' AS last_transform_error",
|
|
168243
168711
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168244
168712
|
last_todo_state: "'' AS last_todo_state",
|
|
168713
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168245
168714
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168246
168715
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168247
168716
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168319,6 +168788,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168319
168788
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168320
168789
|
})();
|
|
168321
168790
|
}
|
|
168791
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168792
|
+
if (maxTagNumber <= 0)
|
|
168793
|
+
return;
|
|
168794
|
+
db.transaction(() => {
|
|
168795
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168796
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168797
|
+
})();
|
|
168798
|
+
}
|
|
168322
168799
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168323
168800
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168324
168801
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168716,6 +169193,26 @@ function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
|
168716
169193
|
nullCount: row?.null_count ?? 0
|
|
168717
169194
|
};
|
|
168718
169195
|
}
|
|
169196
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169197
|
+
if (limit <= 0)
|
|
169198
|
+
return [];
|
|
169199
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169200
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169201
|
+
SELECT tag_number FROM tags
|
|
169202
|
+
WHERE session_id = ? AND status = 'active'
|
|
169203
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169204
|
+
)` : "";
|
|
169205
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169206
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169207
|
+
FROM tags
|
|
169208
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169209
|
+
ORDER BY tag_number ASC, id ASC
|
|
169210
|
+
LIMIT ?`).all(...params);
|
|
169211
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169212
|
+
tagNumber: row.tag_number,
|
|
169213
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169214
|
+
}));
|
|
169215
|
+
}
|
|
168719
169216
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
168720
169217
|
const row = db.prepare(`SELECT
|
|
168721
169218
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -169665,15 +170162,16 @@ function parseInteger(str) {
|
|
|
169665
170162
|
}
|
|
169666
170163
|
|
|
169667
170164
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169668
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169669
|
-
|
|
170165
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170166
|
+
|
|
170167
|
+
How it works:
|
|
170168
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170169
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170170
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
169670
170171
|
|
|
169671
|
-
|
|
169672
|
-
|
|
169673
|
-
-
|
|
169674
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
169675
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
169676
|
-
- Dropped content is gone forever.`;
|
|
170172
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170173
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170174
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169677
170175
|
|
|
169678
170176
|
// src/tools/ctx-reduce.ts
|
|
169679
170177
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169858,120 +170356,6 @@ ${body}` : subject;
|
|
|
169858
170356
|
}
|
|
169859
170357
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169860
170358
|
init_logger();
|
|
169861
|
-
|
|
169862
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169863
|
-
init_logger();
|
|
169864
|
-
var insertStatements = new WeakMap;
|
|
169865
|
-
var existingShasStatements = new WeakMap;
|
|
169866
|
-
var projectCountStatements = new WeakMap;
|
|
169867
|
-
var evictStatements = new WeakMap;
|
|
169868
|
-
var evictOverflowStatements = new WeakMap;
|
|
169869
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169870
|
-
function getInsertStatement(db) {
|
|
169871
|
-
let stmt = insertStatements.get(db);
|
|
169872
|
-
if (!stmt) {
|
|
169873
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169874
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169875
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169876
|
-
project_path = excluded.project_path,
|
|
169877
|
-
short_sha = excluded.short_sha,
|
|
169878
|
-
message = excluded.message,
|
|
169879
|
-
author = excluded.author,
|
|
169880
|
-
committed_at = excluded.committed_at,
|
|
169881
|
-
indexed_at = excluded.indexed_at
|
|
169882
|
-
WHERE git_commits.message != excluded.message`);
|
|
169883
|
-
insertStatements.set(db, stmt);
|
|
169884
|
-
}
|
|
169885
|
-
return stmt;
|
|
169886
|
-
}
|
|
169887
|
-
function getExistingShasStatement(db) {
|
|
169888
|
-
let stmt = existingShasStatements.get(db);
|
|
169889
|
-
if (!stmt) {
|
|
169890
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169891
|
-
existingShasStatements.set(db, stmt);
|
|
169892
|
-
}
|
|
169893
|
-
return stmt;
|
|
169894
|
-
}
|
|
169895
|
-
function getProjectCountStatement(db) {
|
|
169896
|
-
let stmt = projectCountStatements.get(db);
|
|
169897
|
-
if (!stmt) {
|
|
169898
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169899
|
-
projectCountStatements.set(db, stmt);
|
|
169900
|
-
}
|
|
169901
|
-
return stmt;
|
|
169902
|
-
}
|
|
169903
|
-
function getLatestCommitTimeStatement(db) {
|
|
169904
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169905
|
-
if (!stmt) {
|
|
169906
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169907
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169908
|
-
}
|
|
169909
|
-
return stmt;
|
|
169910
|
-
}
|
|
169911
|
-
function getEvictOverflowStatement(db) {
|
|
169912
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169913
|
-
if (!stmt) {
|
|
169914
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169915
|
-
WHERE rowid IN (
|
|
169916
|
-
SELECT rowid FROM git_commits
|
|
169917
|
-
WHERE project_path = ?
|
|
169918
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169919
|
-
LIMIT -1 OFFSET ?
|
|
169920
|
-
)`);
|
|
169921
|
-
evictOverflowStatements.set(db, stmt);
|
|
169922
|
-
}
|
|
169923
|
-
return stmt;
|
|
169924
|
-
}
|
|
169925
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169926
|
-
if (commits.length === 0)
|
|
169927
|
-
return { inserted: 0, updated: 0 };
|
|
169928
|
-
const existing = new Set;
|
|
169929
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169930
|
-
existing.add(row.sha);
|
|
169931
|
-
}
|
|
169932
|
-
let inserted = 0;
|
|
169933
|
-
let updated = 0;
|
|
169934
|
-
const now = Date.now();
|
|
169935
|
-
const insertStmt = getInsertStatement(db);
|
|
169936
|
-
db.transaction(() => {
|
|
169937
|
-
for (const commit of commits) {
|
|
169938
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169939
|
-
if (result.changes > 0) {
|
|
169940
|
-
if (existing.has(commit.sha)) {
|
|
169941
|
-
updated++;
|
|
169942
|
-
} else {
|
|
169943
|
-
inserted++;
|
|
169944
|
-
existing.add(commit.sha);
|
|
169945
|
-
}
|
|
169946
|
-
}
|
|
169947
|
-
}
|
|
169948
|
-
})();
|
|
169949
|
-
return { inserted, updated };
|
|
169950
|
-
}
|
|
169951
|
-
function getCommitCount(db, projectPath) {
|
|
169952
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169953
|
-
return row?.count ?? 0;
|
|
169954
|
-
}
|
|
169955
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169956
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169957
|
-
return row?.latest ?? null;
|
|
169958
|
-
}
|
|
169959
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169960
|
-
if (maxCommits <= 0)
|
|
169961
|
-
return 0;
|
|
169962
|
-
const count = getCommitCount(db, projectPath);
|
|
169963
|
-
if (count <= maxCommits)
|
|
169964
|
-
return 0;
|
|
169965
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169966
|
-
const after = getCommitCount(db, projectPath);
|
|
169967
|
-
const evicted = Math.max(0, count - after);
|
|
169968
|
-
if (evicted > 0) {
|
|
169969
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169970
|
-
}
|
|
169971
|
-
return evicted;
|
|
169972
|
-
}
|
|
169973
|
-
|
|
169974
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169975
170359
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169976
170360
|
var EMBED_BATCH_SIZE = 16;
|
|
169977
170361
|
var EMBED_MAX_PER_SWEEP = 500;
|