@wolfx/pi-magic-context 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1021 -320
- package/dist/subagent-entry.js +592 -182
- package/package.json +3 -3
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -143218,6 +143226,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143226
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143227
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143228
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143229
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143230
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143231
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143232
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -158709,6 +158718,7 @@ init_logger();
|
|
|
158709
158718
|
// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
|
|
158710
158719
|
import { createHash as createHash4 } from "node:crypto";
|
|
158711
158720
|
var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
|
|
158721
|
+
var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
|
|
158712
158722
|
var loadFtsRowsStatements = new WeakMap;
|
|
158713
158723
|
var existingHashStatements = new WeakMap;
|
|
158714
158724
|
var existingHashByProjectStatements = new WeakMap;
|
|
@@ -158944,9 +158954,10 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
158944
158954
|
if (lines.length === 0 || endOrdinal < startOrdinal)
|
|
158945
158955
|
return [];
|
|
158946
158956
|
const normalizedMax = normalizeCompartmentChunkMaxInputTokens(maxInputTokens);
|
|
158957
|
+
const effectiveMax = Math.max(1, Math.floor(normalizedMax * CHUNK_WINDOW_SAFETY_RATIO));
|
|
158947
158958
|
const fullText = lines.join(`
|
|
158948
158959
|
`);
|
|
158949
|
-
if (estimateTokens(fullText) <=
|
|
158960
|
+
if (estimateTokens(fullText) <= effectiveMax) {
|
|
158950
158961
|
return [
|
|
158951
158962
|
{
|
|
158952
158963
|
windowIndex: 0,
|
|
@@ -158984,7 +158995,7 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
158984
158995
|
const lineStart = range?.start ?? startOrdinal;
|
|
158985
158996
|
const lineEnd = range?.end ?? lineStart;
|
|
158986
158997
|
const lineTokens = estimateTokens(line);
|
|
158987
|
-
if (currentLines.length > 0 && currentTokens + lineTokens >
|
|
158998
|
+
if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
|
|
158988
158999
|
flush2();
|
|
158989
159000
|
}
|
|
158990
159001
|
if (currentLines.length === 0) {
|
|
@@ -159140,6 +159151,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159140
159151
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159141
159152
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159142
159153
|
}
|
|
159154
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159155
|
+
const row = db.prepare(`SELECT
|
|
159156
|
+
COUNT(*) AS total,
|
|
159157
|
+
SUM(CASE WHEN EXISTS (
|
|
159158
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159159
|
+
WHERE e.compartment_id = c.id
|
|
159160
|
+
AND e.project_path = ?
|
|
159161
|
+
AND e.model_id = ?
|
|
159162
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159163
|
+
FROM compartments c
|
|
159164
|
+
JOIN session_projects sp
|
|
159165
|
+
ON sp.session_id = c.session_id
|
|
159166
|
+
AND sp.harness = c.harness
|
|
159167
|
+
AND sp.project_path = ?
|
|
159168
|
+
WHERE c.session_id = ?
|
|
159169
|
+
AND c.start_message IS NOT NULL
|
|
159170
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159171
|
+
return {
|
|
159172
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159173
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159174
|
+
};
|
|
159175
|
+
}
|
|
159143
159176
|
|
|
159144
159177
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159145
159178
|
function cosineSimilarity(a, b) {
|
|
@@ -159292,6 +159325,19 @@ async function withQuietConsole(fn) {
|
|
|
159292
159325
|
console.error = origError;
|
|
159293
159326
|
}
|
|
159294
159327
|
}
|
|
159328
|
+
var nativeRuntimeMissing = false;
|
|
159329
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159330
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159331
|
+
const lower = message.toLowerCase();
|
|
159332
|
+
const code = error51?.code;
|
|
159333
|
+
const name2 = error51?.name;
|
|
159334
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159335
|
+
return true;
|
|
159336
|
+
}
|
|
159337
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159338
|
+
return false;
|
|
159339
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159340
|
+
}
|
|
159295
159341
|
function isTransientLoadError(error51) {
|
|
159296
159342
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159297
159343
|
if (!message)
|
|
@@ -159356,6 +159402,9 @@ class LocalEmbeddingProvider {
|
|
|
159356
159402
|
if (this.pipeline) {
|
|
159357
159403
|
return true;
|
|
159358
159404
|
}
|
|
159405
|
+
if (nativeRuntimeMissing) {
|
|
159406
|
+
return false;
|
|
159407
|
+
}
|
|
159359
159408
|
if (this.initPromise) {
|
|
159360
159409
|
await this.initPromise;
|
|
159361
159410
|
return this.pipeline !== null;
|
|
@@ -159422,7 +159471,12 @@ class LocalEmbeddingProvider {
|
|
|
159422
159471
|
await releaseLock();
|
|
159423
159472
|
}
|
|
159424
159473
|
} catch (error51) {
|
|
159425
|
-
|
|
159474
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159475
|
+
nativeRuntimeMissing = true;
|
|
159476
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @wolfx/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159477
|
+
} else {
|
|
159478
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159479
|
+
}
|
|
159426
159480
|
this.pipeline = null;
|
|
159427
159481
|
} finally {
|
|
159428
159482
|
this.initPromise = null;
|
|
@@ -159594,6 +159648,13 @@ function blockedEmbeddingEndpointReason(endpoint) {
|
|
|
159594
159648
|
function normalizeEndpoint3(endpoint) {
|
|
159595
159649
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
159596
159650
|
}
|
|
159651
|
+
function embeddingModelsMatch(served, requested) {
|
|
159652
|
+
const a = served.trim().toLowerCase();
|
|
159653
|
+
const b = requested.trim().toLowerCase();
|
|
159654
|
+
if (a.length === 0 || b.length === 0)
|
|
159655
|
+
return true;
|
|
159656
|
+
return a === b || a.includes(b) || b.includes(a);
|
|
159657
|
+
}
|
|
159597
159658
|
var FAILURE_THRESHOLD = 3;
|
|
159598
159659
|
var FAILURE_WINDOW_MS = 60000;
|
|
159599
159660
|
var OPEN_DURATION_MS = 5 * 60000;
|
|
@@ -159611,6 +159672,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159611
159672
|
failureTimes = [];
|
|
159612
159673
|
circuitOpenUntil = 0;
|
|
159613
159674
|
openLogged = false;
|
|
159675
|
+
modelMismatchLogged = false;
|
|
159614
159676
|
halfOpenProbeInFlight = false;
|
|
159615
159677
|
constructor(options) {
|
|
159616
159678
|
this.endpoint = normalizeEndpoint3(options.endpoint);
|
|
@@ -159709,6 +159771,15 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159709
159771
|
this.recordFailure(isProbe);
|
|
159710
159772
|
return Array.from({ length: texts.length }, () => null);
|
|
159711
159773
|
}
|
|
159774
|
+
const servedModel = typeof body.model === "string" ? body.model : "";
|
|
159775
|
+
if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
|
|
159776
|
+
if (!this.modelMismatchLogged) {
|
|
159777
|
+
log(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${this.model}" served="${servedModel}". The endpoint likely substituted a loaded model; load/select "${this.model}" on the endpoint, or set embedding.model to the served model.`);
|
|
159778
|
+
this.modelMismatchLogged = true;
|
|
159779
|
+
}
|
|
159780
|
+
this.recordFailure(isProbe);
|
|
159781
|
+
return Array.from({ length: texts.length }, () => null);
|
|
159782
|
+
}
|
|
159712
159783
|
const items = Array.isArray(body.data) ? body.data : [];
|
|
159713
159784
|
const results = Array.from({ length: texts.length }, (_, index) => {
|
|
159714
159785
|
const embedding = items[index]?.embedding;
|
|
@@ -159891,6 +159962,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159891
159962
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159892
159963
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159893
159964
|
}
|
|
159965
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
159966
|
+
const row = db.prepare(`SELECT
|
|
159967
|
+
COUNT(*) AS total,
|
|
159968
|
+
SUM(CASE WHEN EXISTS (
|
|
159969
|
+
SELECT 1 FROM memory_embeddings e
|
|
159970
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
159971
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159972
|
+
FROM memories m
|
|
159973
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
159974
|
+
return {
|
|
159975
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159976
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159977
|
+
};
|
|
159978
|
+
}
|
|
159894
159979
|
|
|
159895
159980
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159896
159981
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -159997,6 +160082,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
159997
160082
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159998
160083
|
}
|
|
159999
160084
|
|
|
160085
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160086
|
+
init_logger();
|
|
160087
|
+
var insertStatements = new WeakMap;
|
|
160088
|
+
var existingShasStatements = new WeakMap;
|
|
160089
|
+
var projectCountStatements = new WeakMap;
|
|
160090
|
+
var evictStatements = new WeakMap;
|
|
160091
|
+
var evictOverflowStatements = new WeakMap;
|
|
160092
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160093
|
+
function getInsertStatement(db) {
|
|
160094
|
+
let stmt = insertStatements.get(db);
|
|
160095
|
+
if (!stmt) {
|
|
160096
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160097
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160098
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160099
|
+
project_path = excluded.project_path,
|
|
160100
|
+
short_sha = excluded.short_sha,
|
|
160101
|
+
message = excluded.message,
|
|
160102
|
+
author = excluded.author,
|
|
160103
|
+
committed_at = excluded.committed_at,
|
|
160104
|
+
indexed_at = excluded.indexed_at
|
|
160105
|
+
WHERE git_commits.message != excluded.message`);
|
|
160106
|
+
insertStatements.set(db, stmt);
|
|
160107
|
+
}
|
|
160108
|
+
return stmt;
|
|
160109
|
+
}
|
|
160110
|
+
function getExistingShasStatement(db) {
|
|
160111
|
+
let stmt = existingShasStatements.get(db);
|
|
160112
|
+
if (!stmt) {
|
|
160113
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160114
|
+
existingShasStatements.set(db, stmt);
|
|
160115
|
+
}
|
|
160116
|
+
return stmt;
|
|
160117
|
+
}
|
|
160118
|
+
function getProjectCountStatement(db) {
|
|
160119
|
+
let stmt = projectCountStatements.get(db);
|
|
160120
|
+
if (!stmt) {
|
|
160121
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160122
|
+
projectCountStatements.set(db, stmt);
|
|
160123
|
+
}
|
|
160124
|
+
return stmt;
|
|
160125
|
+
}
|
|
160126
|
+
function getLatestCommitTimeStatement(db) {
|
|
160127
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160128
|
+
if (!stmt) {
|
|
160129
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160130
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160131
|
+
}
|
|
160132
|
+
return stmt;
|
|
160133
|
+
}
|
|
160134
|
+
function getEvictOverflowStatement(db) {
|
|
160135
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160136
|
+
if (!stmt) {
|
|
160137
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160138
|
+
WHERE rowid IN (
|
|
160139
|
+
SELECT rowid FROM git_commits
|
|
160140
|
+
WHERE project_path = ?
|
|
160141
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160142
|
+
LIMIT -1 OFFSET ?
|
|
160143
|
+
)`);
|
|
160144
|
+
evictOverflowStatements.set(db, stmt);
|
|
160145
|
+
}
|
|
160146
|
+
return stmt;
|
|
160147
|
+
}
|
|
160148
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160149
|
+
if (commits.length === 0)
|
|
160150
|
+
return { inserted: 0, updated: 0 };
|
|
160151
|
+
const existing = new Set;
|
|
160152
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160153
|
+
existing.add(row.sha);
|
|
160154
|
+
}
|
|
160155
|
+
let inserted = 0;
|
|
160156
|
+
let updated = 0;
|
|
160157
|
+
const now = Date.now();
|
|
160158
|
+
const insertStmt = getInsertStatement(db);
|
|
160159
|
+
db.transaction(() => {
|
|
160160
|
+
for (const commit of commits) {
|
|
160161
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160162
|
+
if (result.changes > 0) {
|
|
160163
|
+
if (existing.has(commit.sha)) {
|
|
160164
|
+
updated++;
|
|
160165
|
+
} else {
|
|
160166
|
+
inserted++;
|
|
160167
|
+
existing.add(commit.sha);
|
|
160168
|
+
}
|
|
160169
|
+
}
|
|
160170
|
+
}
|
|
160171
|
+
})();
|
|
160172
|
+
return { inserted, updated };
|
|
160173
|
+
}
|
|
160174
|
+
function getCommitCount(db, projectPath) {
|
|
160175
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160176
|
+
return row?.count ?? 0;
|
|
160177
|
+
}
|
|
160178
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160179
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160180
|
+
return row?.latest ?? null;
|
|
160181
|
+
}
|
|
160182
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160183
|
+
if (maxCommits <= 0)
|
|
160184
|
+
return 0;
|
|
160185
|
+
const count = getCommitCount(db, projectPath);
|
|
160186
|
+
if (count <= maxCommits)
|
|
160187
|
+
return 0;
|
|
160188
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160189
|
+
const after = getCommitCount(db, projectPath);
|
|
160190
|
+
const evicted = Math.max(0, count - after);
|
|
160191
|
+
if (evicted > 0) {
|
|
160192
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160193
|
+
}
|
|
160194
|
+
return evicted;
|
|
160195
|
+
}
|
|
160196
|
+
|
|
160000
160197
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
160001
160198
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
160002
160199
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160229,8 +160426,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160229
160426
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160230
160427
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160231
160428
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160232
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160429
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160233
160430
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160431
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160432
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160433
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160434
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160234
160435
|
var projectRegistrations = new Map;
|
|
160235
160436
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160236
160437
|
var globalRegistrationGeneration = 0;
|
|
@@ -160307,7 +160508,7 @@ function getChunkEmbeddingModelId(config2, providerIdentity) {
|
|
|
160307
160508
|
}
|
|
160308
160509
|
const chunkIdentity = {
|
|
160309
160510
|
providerIdentity,
|
|
160310
|
-
chunkerVersion:
|
|
160511
|
+
chunkerVersion: 2,
|
|
160311
160512
|
maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config2 ? config2.max_input_tokens : undefined),
|
|
160312
160513
|
truncate: config2.provider === "openai-compatible" ? config2.truncate ?? "" : ""
|
|
160313
160514
|
};
|
|
@@ -160330,7 +160531,9 @@ function snapshotFor(registration) {
|
|
|
160330
160531
|
enabled,
|
|
160331
160532
|
gitCommitEnabled,
|
|
160332
160533
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160333
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160534
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160535
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160536
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160334
160537
|
};
|
|
160335
160538
|
}
|
|
160336
160539
|
function disposeProvider(provider) {
|
|
@@ -160541,8 +160744,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160541
160744
|
}
|
|
160542
160745
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160543
160746
|
const noWork = [];
|
|
160747
|
+
const failed = [];
|
|
160544
160748
|
if (candidates.length === 0)
|
|
160545
|
-
return { embedded: 0, noWork };
|
|
160749
|
+
return { embedded: 0, noWork, failed };
|
|
160546
160750
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160547
160751
|
const prepared = [];
|
|
160548
160752
|
for (const candidate of candidates) {
|
|
@@ -160559,7 +160763,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160559
160763
|
prepared.push({ candidate, windows });
|
|
160560
160764
|
}
|
|
160561
160765
|
if (prepared.length === 0)
|
|
160562
|
-
return { embedded: 0, noWork };
|
|
160766
|
+
return { embedded: 0, noWork, failed };
|
|
160563
160767
|
let embedded = 0;
|
|
160564
160768
|
let i = 0;
|
|
160565
160769
|
while (i < prepared.length) {
|
|
@@ -160576,35 +160780,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160576
160780
|
const texts = [];
|
|
160577
160781
|
for (const item of slice)
|
|
160578
160782
|
texts.push(...item.windows.map((w) => w.text));
|
|
160579
|
-
|
|
160580
|
-
|
|
160581
|
-
if (!result)
|
|
160582
|
-
continue;
|
|
160783
|
+
const persistedIds = new Set;
|
|
160784
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160583
160785
|
if (signal?.aborted)
|
|
160584
160786
|
break;
|
|
160585
|
-
let
|
|
160586
|
-
|
|
160587
|
-
|
|
160588
|
-
|
|
160589
|
-
|
|
160590
|
-
|
|
160787
|
+
let result = null;
|
|
160788
|
+
const attemptStart = Date.now();
|
|
160789
|
+
try {
|
|
160790
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160791
|
+
} catch (error51) {
|
|
160792
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160793
|
+
}
|
|
160794
|
+
if (signal?.aborted)
|
|
160795
|
+
break;
|
|
160796
|
+
if (result) {
|
|
160797
|
+
let offset = 0;
|
|
160798
|
+
for (const item of slice) {
|
|
160799
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160800
|
+
offset += item.windows.length;
|
|
160801
|
+
if (persistedIds.has(item.candidate.id))
|
|
160802
|
+
continue;
|
|
160803
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160804
|
+
continue;
|
|
160805
|
+
}
|
|
160806
|
+
const rows = item.windows.map((window, index) => ({
|
|
160807
|
+
compartmentId: item.candidate.id,
|
|
160808
|
+
sessionId: item.candidate.sessionId,
|
|
160809
|
+
projectPath: projectIdentity,
|
|
160810
|
+
window,
|
|
160811
|
+
modelId,
|
|
160812
|
+
vector: vectors[index]
|
|
160813
|
+
}));
|
|
160814
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160815
|
+
persistedIds.add(item.candidate.id);
|
|
160591
160816
|
}
|
|
160592
|
-
const rows = item.windows.map((window, index) => ({
|
|
160593
|
-
compartmentId: item.candidate.id,
|
|
160594
|
-
sessionId: item.candidate.sessionId,
|
|
160595
|
-
projectPath: projectIdentity,
|
|
160596
|
-
window,
|
|
160597
|
-
modelId,
|
|
160598
|
-
vector: vectors[index]
|
|
160599
|
-
}));
|
|
160600
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160601
|
-
embedded += 1;
|
|
160602
160817
|
}
|
|
160603
|
-
|
|
160604
|
-
|
|
160818
|
+
if (persistedIds.size === slice.length)
|
|
160819
|
+
break;
|
|
160820
|
+
if (persistedIds.size > 0)
|
|
160821
|
+
break;
|
|
160822
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160823
|
+
break;
|
|
160824
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160825
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160826
|
+
}
|
|
160827
|
+
}
|
|
160828
|
+
embedded += persistedIds.size;
|
|
160829
|
+
if (!signal?.aborted) {
|
|
160830
|
+
for (const item of slice) {
|
|
160831
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160832
|
+
failed.push(item.candidate.id);
|
|
160833
|
+
}
|
|
160605
160834
|
}
|
|
160606
160835
|
}
|
|
160607
|
-
return { embedded, noWork };
|
|
160836
|
+
return { embedded, noWork, failed };
|
|
160608
160837
|
}
|
|
160609
160838
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160610
160839
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160627,9 +160856,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160627
160856
|
renewal.unref?.();
|
|
160628
160857
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160629
160858
|
const skipIds = [];
|
|
160859
|
+
const failedIds = [];
|
|
160630
160860
|
let embedded = 0;
|
|
160631
160861
|
let aborted2 = false;
|
|
160632
|
-
let
|
|
160862
|
+
let providerDown = false;
|
|
160863
|
+
let consecutiveFailedBatches = 0;
|
|
160633
160864
|
try {
|
|
160634
160865
|
options?.onProgress?.({ embedded, total });
|
|
160635
160866
|
for (;; ) {
|
|
@@ -160637,15 +160868,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160637
160868
|
aborted2 = true;
|
|
160638
160869
|
break;
|
|
160639
160870
|
}
|
|
160640
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160871
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160641
160872
|
if (candidates.length === 0)
|
|
160642
160873
|
break;
|
|
160643
|
-
const {
|
|
160874
|
+
const {
|
|
160875
|
+
embedded: n,
|
|
160876
|
+
noWork,
|
|
160877
|
+
failed
|
|
160878
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160644
160879
|
for (const id of noWork)
|
|
160645
160880
|
skipIds.push(id);
|
|
160881
|
+
for (const id of failed)
|
|
160882
|
+
failedIds.push(id);
|
|
160646
160883
|
if (n === 0 && noWork.length === 0) {
|
|
160647
|
-
|
|
160648
|
-
|
|
160884
|
+
consecutiveFailedBatches += 1;
|
|
160885
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160886
|
+
providerDown = true;
|
|
160887
|
+
break;
|
|
160888
|
+
}
|
|
160889
|
+
} else {
|
|
160890
|
+
consecutiveFailedBatches = 0;
|
|
160649
160891
|
}
|
|
160650
160892
|
embedded += n;
|
|
160651
160893
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160653,16 +160895,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160653
160895
|
}
|
|
160654
160896
|
} finally {
|
|
160655
160897
|
clearInterval(renewal);
|
|
160656
|
-
|
|
160898
|
+
try {
|
|
160899
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
160900
|
+
} catch (error51) {
|
|
160901
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
160902
|
+
}
|
|
160657
160903
|
}
|
|
160658
160904
|
if (aborted2)
|
|
160659
|
-
return { status: "aborted", embedded, total };
|
|
160660
|
-
if (
|
|
160905
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
160906
|
+
if (providerDown || failedIds.length > 0) {
|
|
160661
160907
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160662
|
-
if (remaining > 0)
|
|
160663
|
-
return { status: "stalled", embedded, total, remaining };
|
|
160908
|
+
if (remaining > 0) {
|
|
160909
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
160910
|
+
}
|
|
160911
|
+
}
|
|
160912
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
160913
|
+
}
|
|
160914
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
160915
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
160916
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
160917
|
+
return {
|
|
160918
|
+
enabled: false,
|
|
160919
|
+
model: snapshot?.model ?? "off",
|
|
160920
|
+
provider: snapshot?.provider ?? "off",
|
|
160921
|
+
session: { embedded: 0, total: 0 },
|
|
160922
|
+
memories: { embedded: 0, total: 0 },
|
|
160923
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
160924
|
+
};
|
|
160664
160925
|
}
|
|
160665
|
-
|
|
160926
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
160927
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
160928
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
160929
|
+
const commits = gitEnabled ? {
|
|
160930
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
160931
|
+
total: getCommitCount(db, projectIdentity),
|
|
160932
|
+
gitEnabled: true
|
|
160933
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
160934
|
+
return {
|
|
160935
|
+
enabled: true,
|
|
160936
|
+
model: snapshot.model,
|
|
160937
|
+
provider: snapshot.provider,
|
|
160938
|
+
session,
|
|
160939
|
+
memories,
|
|
160940
|
+
commits
|
|
160941
|
+
};
|
|
160666
160942
|
}
|
|
160667
160943
|
|
|
160668
160944
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160891,6 +161167,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160891
161167
|
"conversation_tokens",
|
|
160892
161168
|
"tool_call_tokens",
|
|
160893
161169
|
"cleared_reasoning_through_tag",
|
|
161170
|
+
"tool_reclaim_watermark",
|
|
160894
161171
|
"last_todo_state",
|
|
160895
161172
|
"cached_m0_bytes",
|
|
160896
161173
|
"cached_m1_bytes",
|
|
@@ -160939,6 +161216,7 @@ var META_COLUMNS = {
|
|
|
160939
161216
|
conversationTokens: "conversation_tokens",
|
|
160940
161217
|
toolCallTokens: "tool_call_tokens",
|
|
160941
161218
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161219
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160942
161220
|
lastTodoState: "last_todo_state",
|
|
160943
161221
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160944
161222
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -161007,7 +161285,7 @@ function isSessionMetaRow(row) {
|
|
|
161007
161285
|
if (row === null || typeof row !== "object")
|
|
161008
161286
|
return false;
|
|
161009
161287
|
const r = row;
|
|
161010
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161288
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161011
161289
|
}
|
|
161012
161290
|
function getDefaultSessionMeta(sessionId) {
|
|
161013
161291
|
return {
|
|
@@ -161030,6 +161308,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161030
161308
|
conversationTokens: 0,
|
|
161031
161309
|
toolCallTokens: 0,
|
|
161032
161310
|
clearedReasoningThroughTag: 0,
|
|
161311
|
+
toolReclaimWatermark: 0,
|
|
161033
161312
|
lastTodoState: "",
|
|
161034
161313
|
cachedM0Bytes: null,
|
|
161035
161314
|
cachedM1Bytes: null,
|
|
@@ -161093,6 +161372,7 @@ function toSessionMeta(row) {
|
|
|
161093
161372
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161094
161373
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161095
161374
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161375
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161096
161376
|
lastTodoState: lastTodoStateRaw,
|
|
161097
161377
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161098
161378
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161519,6 +161799,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161519
161799
|
var encoder = new TextEncoder;
|
|
161520
161800
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161521
161801
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161802
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161803
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161522
161804
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161523
161805
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161524
161806
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161531,6 +161813,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161531
161813
|
function stripMalformedTagNotationGlobally(value) {
|
|
161532
161814
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161533
161815
|
}
|
|
161816
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161817
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161818
|
+
}
|
|
161534
161819
|
function stripTagSectionCharacters(value) {
|
|
161535
161820
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161536
161821
|
}
|
|
@@ -161538,6 +161823,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161538
161823
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161539
161824
|
text = stripCompleteTagPairsGlobally(text);
|
|
161540
161825
|
text = stripMalformedTagNotationGlobally(text);
|
|
161826
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161541
161827
|
text = stripTagSectionCharacters(text);
|
|
161542
161828
|
return text.trim();
|
|
161543
161829
|
}
|
|
@@ -161550,6 +161836,7 @@ function stripTagPrefix(value) {
|
|
|
161550
161836
|
const prev = stripped;
|
|
161551
161837
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161552
161838
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161839
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161553
161840
|
if (stripped === prev)
|
|
161554
161841
|
break;
|
|
161555
161842
|
}
|
|
@@ -161965,9 +162252,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161965
162252
|
|
|
161966
162253
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161967
162254
|
|
|
161968
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162255
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162256
|
+
|
|
162257
|
+
Two recovery modes for finer detail:
|
|
162258
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162259
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161969
162260
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161970
162261
|
|
|
162262
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162263
|
+
function isRecord2(value) {
|
|
162264
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162265
|
+
}
|
|
162266
|
+
function roleLabel(role) {
|
|
162267
|
+
if (role === "assistant")
|
|
162268
|
+
return "A (assistant)";
|
|
162269
|
+
if (role === "user")
|
|
162270
|
+
return "U (user)";
|
|
162271
|
+
return role;
|
|
162272
|
+
}
|
|
162273
|
+
function truncate(value, max) {
|
|
162274
|
+
const t = value.trim();
|
|
162275
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162276
|
+
}
|
|
162277
|
+
function keyArg(input) {
|
|
162278
|
+
if (!input)
|
|
162279
|
+
return "";
|
|
162280
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162281
|
+
const v = input[k];
|
|
162282
|
+
if (typeof v === "string" && v.length > 0)
|
|
162283
|
+
return truncate(v, 60);
|
|
162284
|
+
}
|
|
162285
|
+
if (typeof input.description === "string")
|
|
162286
|
+
return truncate(input.description, 60);
|
|
162287
|
+
return "";
|
|
162288
|
+
}
|
|
162289
|
+
function asToolPart(part) {
|
|
162290
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162291
|
+
if (type === "tool") {
|
|
162292
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162293
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162294
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162295
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162296
|
+
return {
|
|
162297
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162298
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162299
|
+
title,
|
|
162300
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162301
|
+
output
|
|
162302
|
+
};
|
|
162303
|
+
}
|
|
162304
|
+
if (type === "tool_use") {
|
|
162305
|
+
return {
|
|
162306
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162307
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162308
|
+
title: null,
|
|
162309
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162310
|
+
output: null
|
|
162311
|
+
};
|
|
162312
|
+
}
|
|
162313
|
+
if (type === "tool_result") {
|
|
162314
|
+
const content = part.content;
|
|
162315
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162316
|
+
return {
|
|
162317
|
+
name: "tool_result",
|
|
162318
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162319
|
+
title: null,
|
|
162320
|
+
input: null,
|
|
162321
|
+
output
|
|
162322
|
+
};
|
|
162323
|
+
}
|
|
162324
|
+
return null;
|
|
162325
|
+
}
|
|
162326
|
+
function textOf(part) {
|
|
162327
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162328
|
+
return part.text;
|
|
162329
|
+
return null;
|
|
162330
|
+
}
|
|
162331
|
+
function reasoningOf(part) {
|
|
162332
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162333
|
+
return part.text;
|
|
162334
|
+
}
|
|
162335
|
+
return null;
|
|
162336
|
+
}
|
|
162337
|
+
function renderPartPreview(part) {
|
|
162338
|
+
if (!isRecord2(part))
|
|
162339
|
+
return null;
|
|
162340
|
+
const text = textOf(part);
|
|
162341
|
+
if (text !== null) {
|
|
162342
|
+
const t = truncate(text, 200);
|
|
162343
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162344
|
+
}
|
|
162345
|
+
const tool = asToolPart(part);
|
|
162346
|
+
if (tool) {
|
|
162347
|
+
const arg = keyArg(tool.input);
|
|
162348
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162349
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162350
|
+
}
|
|
162351
|
+
const reasoning = reasoningOf(part);
|
|
162352
|
+
if (reasoning !== null)
|
|
162353
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162354
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162355
|
+
if (type === "file")
|
|
162356
|
+
return " • [file]";
|
|
162357
|
+
if (type === "step-start" || type === "step-finish")
|
|
162358
|
+
return null;
|
|
162359
|
+
return ` • [${type}]`;
|
|
162360
|
+
}
|
|
162361
|
+
function renderPartFull(part) {
|
|
162362
|
+
if (!isRecord2(part))
|
|
162363
|
+
return null;
|
|
162364
|
+
const text = textOf(part);
|
|
162365
|
+
if (text !== null) {
|
|
162366
|
+
return text.trim().length > 0 ? ` [text]
|
|
162367
|
+
${text}` : null;
|
|
162368
|
+
}
|
|
162369
|
+
const tool = asToolPart(part);
|
|
162370
|
+
if (tool) {
|
|
162371
|
+
const lines = [];
|
|
162372
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162373
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162374
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162375
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162376
|
+
}
|
|
162377
|
+
if (tool.input)
|
|
162378
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162379
|
+
if (tool.output !== null)
|
|
162380
|
+
lines.push(` output:
|
|
162381
|
+
${tool.output}`);
|
|
162382
|
+
return lines.join(`
|
|
162383
|
+
`);
|
|
162384
|
+
}
|
|
162385
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162386
|
+
if (type === "file") {
|
|
162387
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162388
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162389
|
+
}
|
|
162390
|
+
return null;
|
|
162391
|
+
}
|
|
162392
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162393
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162394
|
+
if (!msg) {
|
|
162395
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162396
|
+
}
|
|
162397
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162398
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162399
|
+
if (rendered.length === 0) {
|
|
162400
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162401
|
+
} else {
|
|
162402
|
+
lines.push(...rendered);
|
|
162403
|
+
}
|
|
162404
|
+
return lines.join(`
|
|
162405
|
+
`);
|
|
162406
|
+
}
|
|
162407
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162408
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162409
|
+
const out = [];
|
|
162410
|
+
let usedTokens = 0;
|
|
162411
|
+
let lastOrdinal = start - 1;
|
|
162412
|
+
let truncated = false;
|
|
162413
|
+
for (const msg of messages) {
|
|
162414
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162415
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162416
|
+
const block = partLines.length > 0 ? `${header}
|
|
162417
|
+
${partLines.join(`
|
|
162418
|
+
`)}` : header;
|
|
162419
|
+
const blockTokens = estimateTokens(block);
|
|
162420
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162421
|
+
truncated = true;
|
|
162422
|
+
break;
|
|
162423
|
+
}
|
|
162424
|
+
out.push(block);
|
|
162425
|
+
usedTokens += blockTokens;
|
|
162426
|
+
lastOrdinal = msg.ordinal;
|
|
162427
|
+
}
|
|
162428
|
+
return { text: out.join(`
|
|
162429
|
+
|
|
162430
|
+
`), lastOrdinal, truncated };
|
|
162431
|
+
}
|
|
162432
|
+
|
|
161971
162433
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161972
162434
|
var exports_memory = {};
|
|
161973
162435
|
__export(exports_memory, {
|
|
@@ -166324,12 +166786,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166324
166786
|
|
|
166325
166787
|
// src/tools/ctx-expand.ts
|
|
166326
166788
|
var ParamsSchema = exports_typebox.Object({
|
|
166327
|
-
start: exports_typebox.Number({
|
|
166789
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166328
166790
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166329
|
-
}),
|
|
166330
|
-
end: exports_typebox.Number({
|
|
166791
|
+
})),
|
|
166792
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166331
166793
|
description: "End message ordinal (from compartment end attribute)"
|
|
166332
|
-
})
|
|
166794
|
+
})),
|
|
166795
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166796
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166797
|
+
})),
|
|
166798
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166799
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166800
|
+
}))
|
|
166333
166801
|
});
|
|
166334
166802
|
function ok(text) {
|
|
166335
166803
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166348,22 +166816,41 @@ function createCtxExpandTool(deps) {
|
|
|
166348
166816
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166349
166817
|
parameters: ParamsSchema,
|
|
166350
166818
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166351
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166352
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166353
|
-
}
|
|
166354
166819
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166355
166820
|
if (!sessionId) {
|
|
166356
166821
|
return err("Error: no active Pi session.");
|
|
166357
166822
|
}
|
|
166358
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166359
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166360
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166361
|
-
}
|
|
166362
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166363
166823
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166364
166824
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166365
166825
|
});
|
|
166366
166826
|
try {
|
|
166827
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166828
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166829
|
+
}
|
|
166830
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166831
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166832
|
+
}
|
|
166833
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166834
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166835
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166836
|
+
}
|
|
166837
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166838
|
+
if (params.verbose === true) {
|
|
166839
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166840
|
+
if (!v.text) {
|
|
166841
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166842
|
+
}
|
|
166843
|
+
const out = [
|
|
166844
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166845
|
+
"",
|
|
166846
|
+
v.text
|
|
166847
|
+
];
|
|
166848
|
+
if (v.truncated) {
|
|
166849
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166850
|
+
}
|
|
166851
|
+
return ok(out.join(`
|
|
166852
|
+
`));
|
|
166853
|
+
}
|
|
166367
166854
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166368
166855
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166369
166856
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -168223,6 +168710,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168223
168710
|
last_transform_error: "'' AS last_transform_error",
|
|
168224
168711
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168225
168712
|
last_todo_state: "'' AS last_todo_state",
|
|
168713
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168226
168714
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168227
168715
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168228
168716
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168300,6 +168788,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168300
168788
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168301
168789
|
})();
|
|
168302
168790
|
}
|
|
168791
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168792
|
+
if (maxTagNumber <= 0)
|
|
168793
|
+
return;
|
|
168794
|
+
db.transaction(() => {
|
|
168795
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168796
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168797
|
+
})();
|
|
168798
|
+
}
|
|
168303
168799
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168304
168800
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168305
168801
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168674,15 +169170,22 @@ function ownerMessageIdForTagRow(row) {
|
|
|
168674
169170
|
}
|
|
168675
169171
|
return row.message_id.replace(CONTENT_ID_SUFFIX, "");
|
|
168676
169172
|
}
|
|
168677
|
-
function getActiveTagTokenAggregate(db, sessionId) {
|
|
168678
|
-
const
|
|
169173
|
+
function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
169174
|
+
const toolOutputExpr = protectedTags > 0 ? `COALESCE(SUM(CASE WHEN type = 'tool' AND tag_number < (
|
|
169175
|
+
SELECT tag_number FROM tags
|
|
169176
|
+
WHERE session_id = ? AND status = 'active'
|
|
169177
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169178
|
+
) THEN COALESCE(token_count, 0) ELSE 0 END), 0)` : `COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)`;
|
|
169179
|
+
const sql = `SELECT
|
|
168679
169180
|
COALESCE(SUM(CASE WHEN type != 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)
|
|
168680
169181
|
+ COALESCE(SUM(COALESCE(reasoning_token_count, 0)), 0) AS conversation,
|
|
168681
169182
|
COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) + COALESCE(input_token_count, 0) ELSE 0 END), 0) AS tool_call,
|
|
168682
|
-
|
|
169183
|
+
${toolOutputExpr} AS tool_output,
|
|
168683
169184
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
168684
169185
|
FROM tags
|
|
168685
|
-
WHERE session_id = ? AND status = 'active'
|
|
169186
|
+
WHERE session_id = ? AND status = 'active'`;
|
|
169187
|
+
const params = protectedTags > 0 ? [sessionId, protectedTags - 1, sessionId] : [sessionId];
|
|
169188
|
+
const row = db.prepare(sql).get(...params);
|
|
168686
169189
|
return {
|
|
168687
169190
|
conversation: row?.conversation ?? 0,
|
|
168688
169191
|
toolCall: row?.tool_call ?? 0,
|
|
@@ -168690,6 +169193,26 @@ function getActiveTagTokenAggregate(db, sessionId) {
|
|
|
168690
169193
|
nullCount: row?.null_count ?? 0
|
|
168691
169194
|
};
|
|
168692
169195
|
}
|
|
169196
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169197
|
+
if (limit <= 0)
|
|
169198
|
+
return [];
|
|
169199
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169200
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169201
|
+
SELECT tag_number FROM tags
|
|
169202
|
+
WHERE session_id = ? AND status = 'active'
|
|
169203
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169204
|
+
)` : "";
|
|
169205
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169206
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169207
|
+
FROM tags
|
|
169208
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169209
|
+
ORDER BY tag_number ASC, id ASC
|
|
169210
|
+
LIMIT ?`).all(...params);
|
|
169211
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169212
|
+
tagNumber: row.tag_number,
|
|
169213
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169214
|
+
}));
|
|
169215
|
+
}
|
|
168693
169216
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
168694
169217
|
const row = db.prepare(`SELECT
|
|
168695
169218
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -169639,15 +170162,16 @@ function parseInteger(str) {
|
|
|
169639
170162
|
}
|
|
169640
170163
|
|
|
169641
170164
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169642
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169643
|
-
|
|
170165
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170166
|
+
|
|
170167
|
+
How it works:
|
|
170168
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170169
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170170
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
169644
170171
|
|
|
169645
|
-
|
|
169646
|
-
|
|
169647
|
-
-
|
|
169648
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
169649
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
169650
|
-
- Dropped content is gone forever.`;
|
|
170172
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170173
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170174
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169651
170175
|
|
|
169652
170176
|
// src/tools/ctx-reduce.ts
|
|
169653
170177
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169832,120 +170356,6 @@ ${body}` : subject;
|
|
|
169832
170356
|
}
|
|
169833
170357
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169834
170358
|
init_logger();
|
|
169835
|
-
|
|
169836
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169837
|
-
init_logger();
|
|
169838
|
-
var insertStatements = new WeakMap;
|
|
169839
|
-
var existingShasStatements = new WeakMap;
|
|
169840
|
-
var projectCountStatements = new WeakMap;
|
|
169841
|
-
var evictStatements = new WeakMap;
|
|
169842
|
-
var evictOverflowStatements = new WeakMap;
|
|
169843
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169844
|
-
function getInsertStatement(db) {
|
|
169845
|
-
let stmt = insertStatements.get(db);
|
|
169846
|
-
if (!stmt) {
|
|
169847
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169848
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169849
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169850
|
-
project_path = excluded.project_path,
|
|
169851
|
-
short_sha = excluded.short_sha,
|
|
169852
|
-
message = excluded.message,
|
|
169853
|
-
author = excluded.author,
|
|
169854
|
-
committed_at = excluded.committed_at,
|
|
169855
|
-
indexed_at = excluded.indexed_at
|
|
169856
|
-
WHERE git_commits.message != excluded.message`);
|
|
169857
|
-
insertStatements.set(db, stmt);
|
|
169858
|
-
}
|
|
169859
|
-
return stmt;
|
|
169860
|
-
}
|
|
169861
|
-
function getExistingShasStatement(db) {
|
|
169862
|
-
let stmt = existingShasStatements.get(db);
|
|
169863
|
-
if (!stmt) {
|
|
169864
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169865
|
-
existingShasStatements.set(db, stmt);
|
|
169866
|
-
}
|
|
169867
|
-
return stmt;
|
|
169868
|
-
}
|
|
169869
|
-
function getProjectCountStatement(db) {
|
|
169870
|
-
let stmt = projectCountStatements.get(db);
|
|
169871
|
-
if (!stmt) {
|
|
169872
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169873
|
-
projectCountStatements.set(db, stmt);
|
|
169874
|
-
}
|
|
169875
|
-
return stmt;
|
|
169876
|
-
}
|
|
169877
|
-
function getLatestCommitTimeStatement(db) {
|
|
169878
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169879
|
-
if (!stmt) {
|
|
169880
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169881
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169882
|
-
}
|
|
169883
|
-
return stmt;
|
|
169884
|
-
}
|
|
169885
|
-
function getEvictOverflowStatement(db) {
|
|
169886
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169887
|
-
if (!stmt) {
|
|
169888
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169889
|
-
WHERE rowid IN (
|
|
169890
|
-
SELECT rowid FROM git_commits
|
|
169891
|
-
WHERE project_path = ?
|
|
169892
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169893
|
-
LIMIT -1 OFFSET ?
|
|
169894
|
-
)`);
|
|
169895
|
-
evictOverflowStatements.set(db, stmt);
|
|
169896
|
-
}
|
|
169897
|
-
return stmt;
|
|
169898
|
-
}
|
|
169899
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169900
|
-
if (commits.length === 0)
|
|
169901
|
-
return { inserted: 0, updated: 0 };
|
|
169902
|
-
const existing = new Set;
|
|
169903
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169904
|
-
existing.add(row.sha);
|
|
169905
|
-
}
|
|
169906
|
-
let inserted = 0;
|
|
169907
|
-
let updated = 0;
|
|
169908
|
-
const now = Date.now();
|
|
169909
|
-
const insertStmt = getInsertStatement(db);
|
|
169910
|
-
db.transaction(() => {
|
|
169911
|
-
for (const commit of commits) {
|
|
169912
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169913
|
-
if (result.changes > 0) {
|
|
169914
|
-
if (existing.has(commit.sha)) {
|
|
169915
|
-
updated++;
|
|
169916
|
-
} else {
|
|
169917
|
-
inserted++;
|
|
169918
|
-
existing.add(commit.sha);
|
|
169919
|
-
}
|
|
169920
|
-
}
|
|
169921
|
-
}
|
|
169922
|
-
})();
|
|
169923
|
-
return { inserted, updated };
|
|
169924
|
-
}
|
|
169925
|
-
function getCommitCount(db, projectPath) {
|
|
169926
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169927
|
-
return row?.count ?? 0;
|
|
169928
|
-
}
|
|
169929
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169930
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169931
|
-
return row?.latest ?? null;
|
|
169932
|
-
}
|
|
169933
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169934
|
-
if (maxCommits <= 0)
|
|
169935
|
-
return 0;
|
|
169936
|
-
const count = getCommitCount(db, projectPath);
|
|
169937
|
-
if (count <= maxCommits)
|
|
169938
|
-
return 0;
|
|
169939
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169940
|
-
const after = getCommitCount(db, projectPath);
|
|
169941
|
-
const evicted = Math.max(0, count - after);
|
|
169942
|
-
if (evicted > 0) {
|
|
169943
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169944
|
-
}
|
|
169945
|
-
return evicted;
|
|
169946
|
-
}
|
|
169947
|
-
|
|
169948
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169949
170359
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169950
170360
|
var EMBED_BATCH_SIZE = 16;
|
|
169951
170361
|
var EMBED_MAX_PER_SWEEP = 500;
|