@cortexkit/pi-magic-context 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1019 -318
- package/dist/subagent-entry.js +592 -182
- package/package.json +1 -1
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -143218,6 +143226,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143226
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143227
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143228
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143229
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143230
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143231
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143232
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -158702,6 +158711,7 @@ init_logger();
|
|
|
158702
158711
|
// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
|
|
158703
158712
|
import { createHash as createHash4 } from "node:crypto";
|
|
158704
158713
|
var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
|
|
158714
|
+
var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
|
|
158705
158715
|
var loadFtsRowsStatements = new WeakMap;
|
|
158706
158716
|
var existingHashStatements = new WeakMap;
|
|
158707
158717
|
var existingHashByProjectStatements = new WeakMap;
|
|
@@ -158937,9 +158947,10 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
158937
158947
|
if (lines.length === 0 || endOrdinal < startOrdinal)
|
|
158938
158948
|
return [];
|
|
158939
158949
|
const normalizedMax = normalizeCompartmentChunkMaxInputTokens(maxInputTokens);
|
|
158950
|
+
const effectiveMax = Math.max(1, Math.floor(normalizedMax * CHUNK_WINDOW_SAFETY_RATIO));
|
|
158940
158951
|
const fullText = lines.join(`
|
|
158941
158952
|
`);
|
|
158942
|
-
if (estimateTokens(fullText) <=
|
|
158953
|
+
if (estimateTokens(fullText) <= effectiveMax) {
|
|
158943
158954
|
return [
|
|
158944
158955
|
{
|
|
158945
158956
|
windowIndex: 0,
|
|
@@ -158977,7 +158988,7 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
158977
158988
|
const lineStart = range?.start ?? startOrdinal;
|
|
158978
158989
|
const lineEnd = range?.end ?? lineStart;
|
|
158979
158990
|
const lineTokens = estimateTokens(line);
|
|
158980
|
-
if (currentLines.length > 0 && currentTokens + lineTokens >
|
|
158991
|
+
if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
|
|
158981
158992
|
flush2();
|
|
158982
158993
|
}
|
|
158983
158994
|
if (currentLines.length === 0) {
|
|
@@ -159133,6 +159144,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159133
159144
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159134
159145
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159135
159146
|
}
|
|
159147
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159148
|
+
const row = db.prepare(`SELECT
|
|
159149
|
+
COUNT(*) AS total,
|
|
159150
|
+
SUM(CASE WHEN EXISTS (
|
|
159151
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159152
|
+
WHERE e.compartment_id = c.id
|
|
159153
|
+
AND e.project_path = ?
|
|
159154
|
+
AND e.model_id = ?
|
|
159155
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159156
|
+
FROM compartments c
|
|
159157
|
+
JOIN session_projects sp
|
|
159158
|
+
ON sp.session_id = c.session_id
|
|
159159
|
+
AND sp.harness = c.harness
|
|
159160
|
+
AND sp.project_path = ?
|
|
159161
|
+
WHERE c.session_id = ?
|
|
159162
|
+
AND c.start_message IS NOT NULL
|
|
159163
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159164
|
+
return {
|
|
159165
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159166
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159167
|
+
};
|
|
159168
|
+
}
|
|
159136
159169
|
|
|
159137
159170
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159138
159171
|
function cosineSimilarity(a, b) {
|
|
@@ -159285,6 +159318,19 @@ async function withQuietConsole(fn) {
|
|
|
159285
159318
|
console.error = origError;
|
|
159286
159319
|
}
|
|
159287
159320
|
}
|
|
159321
|
+
var nativeRuntimeMissing = false;
|
|
159322
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159323
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159324
|
+
const lower = message.toLowerCase();
|
|
159325
|
+
const code = error51?.code;
|
|
159326
|
+
const name2 = error51?.name;
|
|
159327
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159328
|
+
return true;
|
|
159329
|
+
}
|
|
159330
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159331
|
+
return false;
|
|
159332
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159333
|
+
}
|
|
159288
159334
|
function isTransientLoadError(error51) {
|
|
159289
159335
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159290
159336
|
if (!message)
|
|
@@ -159349,6 +159395,9 @@ class LocalEmbeddingProvider {
|
|
|
159349
159395
|
if (this.pipeline) {
|
|
159350
159396
|
return true;
|
|
159351
159397
|
}
|
|
159398
|
+
if (nativeRuntimeMissing) {
|
|
159399
|
+
return false;
|
|
159400
|
+
}
|
|
159352
159401
|
if (this.initPromise) {
|
|
159353
159402
|
await this.initPromise;
|
|
159354
159403
|
return this.pipeline !== null;
|
|
@@ -159415,7 +159464,12 @@ class LocalEmbeddingProvider {
|
|
|
159415
159464
|
await releaseLock();
|
|
159416
159465
|
}
|
|
159417
159466
|
} catch (error51) {
|
|
159418
|
-
|
|
159467
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159468
|
+
nativeRuntimeMissing = true;
|
|
159469
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @cortexkit/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159470
|
+
} else {
|
|
159471
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159472
|
+
}
|
|
159419
159473
|
this.pipeline = null;
|
|
159420
159474
|
} finally {
|
|
159421
159475
|
this.initPromise = null;
|
|
@@ -159587,6 +159641,13 @@ function blockedEmbeddingEndpointReason(endpoint) {
|
|
|
159587
159641
|
function normalizeEndpoint3(endpoint) {
|
|
159588
159642
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
159589
159643
|
}
|
|
159644
|
+
function embeddingModelsMatch(served, requested) {
|
|
159645
|
+
const a = served.trim().toLowerCase();
|
|
159646
|
+
const b = requested.trim().toLowerCase();
|
|
159647
|
+
if (a.length === 0 || b.length === 0)
|
|
159648
|
+
return true;
|
|
159649
|
+
return a === b || a.includes(b) || b.includes(a);
|
|
159650
|
+
}
|
|
159590
159651
|
var FAILURE_THRESHOLD = 3;
|
|
159591
159652
|
var FAILURE_WINDOW_MS = 60000;
|
|
159592
159653
|
var OPEN_DURATION_MS = 5 * 60000;
|
|
@@ -159604,6 +159665,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159604
159665
|
failureTimes = [];
|
|
159605
159666
|
circuitOpenUntil = 0;
|
|
159606
159667
|
openLogged = false;
|
|
159668
|
+
modelMismatchLogged = false;
|
|
159607
159669
|
halfOpenProbeInFlight = false;
|
|
159608
159670
|
constructor(options) {
|
|
159609
159671
|
this.endpoint = normalizeEndpoint3(options.endpoint);
|
|
@@ -159702,6 +159764,15 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159702
159764
|
this.recordFailure(isProbe);
|
|
159703
159765
|
return Array.from({ length: texts.length }, () => null);
|
|
159704
159766
|
}
|
|
159767
|
+
const servedModel = typeof body.model === "string" ? body.model : "";
|
|
159768
|
+
if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
|
|
159769
|
+
if (!this.modelMismatchLogged) {
|
|
159770
|
+
log(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${this.model}" served="${servedModel}". The endpoint likely substituted a loaded model; load/select "${this.model}" on the endpoint, or set embedding.model to the served model.`);
|
|
159771
|
+
this.modelMismatchLogged = true;
|
|
159772
|
+
}
|
|
159773
|
+
this.recordFailure(isProbe);
|
|
159774
|
+
return Array.from({ length: texts.length }, () => null);
|
|
159775
|
+
}
|
|
159705
159776
|
const items = Array.isArray(body.data) ? body.data : [];
|
|
159706
159777
|
const results = Array.from({ length: texts.length }, (_, index) => {
|
|
159707
159778
|
const embedding = items[index]?.embedding;
|
|
@@ -159884,6 +159955,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159884
159955
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159885
159956
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159886
159957
|
}
|
|
159958
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
159959
|
+
const row = db.prepare(`SELECT
|
|
159960
|
+
COUNT(*) AS total,
|
|
159961
|
+
SUM(CASE WHEN EXISTS (
|
|
159962
|
+
SELECT 1 FROM memory_embeddings e
|
|
159963
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
159964
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159965
|
+
FROM memories m
|
|
159966
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
159967
|
+
return {
|
|
159968
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159969
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159970
|
+
};
|
|
159971
|
+
}
|
|
159887
159972
|
|
|
159888
159973
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159889
159974
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -159990,6 +160075,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
159990
160075
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159991
160076
|
}
|
|
159992
160077
|
|
|
160078
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160079
|
+
init_logger();
|
|
160080
|
+
var insertStatements = new WeakMap;
|
|
160081
|
+
var existingShasStatements = new WeakMap;
|
|
160082
|
+
var projectCountStatements = new WeakMap;
|
|
160083
|
+
var evictStatements = new WeakMap;
|
|
160084
|
+
var evictOverflowStatements = new WeakMap;
|
|
160085
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160086
|
+
function getInsertStatement(db) {
|
|
160087
|
+
let stmt = insertStatements.get(db);
|
|
160088
|
+
if (!stmt) {
|
|
160089
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160090
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160091
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160092
|
+
project_path = excluded.project_path,
|
|
160093
|
+
short_sha = excluded.short_sha,
|
|
160094
|
+
message = excluded.message,
|
|
160095
|
+
author = excluded.author,
|
|
160096
|
+
committed_at = excluded.committed_at,
|
|
160097
|
+
indexed_at = excluded.indexed_at
|
|
160098
|
+
WHERE git_commits.message != excluded.message`);
|
|
160099
|
+
insertStatements.set(db, stmt);
|
|
160100
|
+
}
|
|
160101
|
+
return stmt;
|
|
160102
|
+
}
|
|
160103
|
+
function getExistingShasStatement(db) {
|
|
160104
|
+
let stmt = existingShasStatements.get(db);
|
|
160105
|
+
if (!stmt) {
|
|
160106
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160107
|
+
existingShasStatements.set(db, stmt);
|
|
160108
|
+
}
|
|
160109
|
+
return stmt;
|
|
160110
|
+
}
|
|
160111
|
+
function getProjectCountStatement(db) {
|
|
160112
|
+
let stmt = projectCountStatements.get(db);
|
|
160113
|
+
if (!stmt) {
|
|
160114
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160115
|
+
projectCountStatements.set(db, stmt);
|
|
160116
|
+
}
|
|
160117
|
+
return stmt;
|
|
160118
|
+
}
|
|
160119
|
+
function getLatestCommitTimeStatement(db) {
|
|
160120
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160121
|
+
if (!stmt) {
|
|
160122
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160123
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160124
|
+
}
|
|
160125
|
+
return stmt;
|
|
160126
|
+
}
|
|
160127
|
+
function getEvictOverflowStatement(db) {
|
|
160128
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160129
|
+
if (!stmt) {
|
|
160130
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160131
|
+
WHERE rowid IN (
|
|
160132
|
+
SELECT rowid FROM git_commits
|
|
160133
|
+
WHERE project_path = ?
|
|
160134
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160135
|
+
LIMIT -1 OFFSET ?
|
|
160136
|
+
)`);
|
|
160137
|
+
evictOverflowStatements.set(db, stmt);
|
|
160138
|
+
}
|
|
160139
|
+
return stmt;
|
|
160140
|
+
}
|
|
160141
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160142
|
+
if (commits.length === 0)
|
|
160143
|
+
return { inserted: 0, updated: 0 };
|
|
160144
|
+
const existing = new Set;
|
|
160145
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160146
|
+
existing.add(row.sha);
|
|
160147
|
+
}
|
|
160148
|
+
let inserted = 0;
|
|
160149
|
+
let updated = 0;
|
|
160150
|
+
const now = Date.now();
|
|
160151
|
+
const insertStmt = getInsertStatement(db);
|
|
160152
|
+
db.transaction(() => {
|
|
160153
|
+
for (const commit of commits) {
|
|
160154
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160155
|
+
if (result.changes > 0) {
|
|
160156
|
+
if (existing.has(commit.sha)) {
|
|
160157
|
+
updated++;
|
|
160158
|
+
} else {
|
|
160159
|
+
inserted++;
|
|
160160
|
+
existing.add(commit.sha);
|
|
160161
|
+
}
|
|
160162
|
+
}
|
|
160163
|
+
}
|
|
160164
|
+
})();
|
|
160165
|
+
return { inserted, updated };
|
|
160166
|
+
}
|
|
160167
|
+
function getCommitCount(db, projectPath) {
|
|
160168
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160169
|
+
return row?.count ?? 0;
|
|
160170
|
+
}
|
|
160171
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160172
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160173
|
+
return row?.latest ?? null;
|
|
160174
|
+
}
|
|
160175
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160176
|
+
if (maxCommits <= 0)
|
|
160177
|
+
return 0;
|
|
160178
|
+
const count = getCommitCount(db, projectPath);
|
|
160179
|
+
if (count <= maxCommits)
|
|
160180
|
+
return 0;
|
|
160181
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160182
|
+
const after = getCommitCount(db, projectPath);
|
|
160183
|
+
const evicted = Math.max(0, count - after);
|
|
160184
|
+
if (evicted > 0) {
|
|
160185
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160186
|
+
}
|
|
160187
|
+
return evicted;
|
|
160188
|
+
}
|
|
160189
|
+
|
|
159993
160190
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
159994
160191
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
159995
160192
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160222,8 +160419,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160222
160419
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160223
160420
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160224
160421
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160225
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160422
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160226
160423
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160424
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160425
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160426
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160427
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160227
160428
|
var projectRegistrations = new Map;
|
|
160228
160429
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160229
160430
|
var globalRegistrationGeneration = 0;
|
|
@@ -160300,7 +160501,7 @@ function getChunkEmbeddingModelId(config2, providerIdentity) {
|
|
|
160300
160501
|
}
|
|
160301
160502
|
const chunkIdentity = {
|
|
160302
160503
|
providerIdentity,
|
|
160303
|
-
chunkerVersion:
|
|
160504
|
+
chunkerVersion: 2,
|
|
160304
160505
|
maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config2 ? config2.max_input_tokens : undefined),
|
|
160305
160506
|
truncate: config2.provider === "openai-compatible" ? config2.truncate ?? "" : ""
|
|
160306
160507
|
};
|
|
@@ -160323,7 +160524,9 @@ function snapshotFor(registration) {
|
|
|
160323
160524
|
enabled,
|
|
160324
160525
|
gitCommitEnabled,
|
|
160325
160526
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160326
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160527
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160528
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160529
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160327
160530
|
};
|
|
160328
160531
|
}
|
|
160329
160532
|
function disposeProvider(provider) {
|
|
@@ -160534,8 +160737,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160534
160737
|
}
|
|
160535
160738
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160536
160739
|
const noWork = [];
|
|
160740
|
+
const failed = [];
|
|
160537
160741
|
if (candidates.length === 0)
|
|
160538
|
-
return { embedded: 0, noWork };
|
|
160742
|
+
return { embedded: 0, noWork, failed };
|
|
160539
160743
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160540
160744
|
const prepared = [];
|
|
160541
160745
|
for (const candidate of candidates) {
|
|
@@ -160552,7 +160756,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160552
160756
|
prepared.push({ candidate, windows });
|
|
160553
160757
|
}
|
|
160554
160758
|
if (prepared.length === 0)
|
|
160555
|
-
return { embedded: 0, noWork };
|
|
160759
|
+
return { embedded: 0, noWork, failed };
|
|
160556
160760
|
let embedded = 0;
|
|
160557
160761
|
let i = 0;
|
|
160558
160762
|
while (i < prepared.length) {
|
|
@@ -160569,35 +160773,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160569
160773
|
const texts = [];
|
|
160570
160774
|
for (const item of slice)
|
|
160571
160775
|
texts.push(...item.windows.map((w) => w.text));
|
|
160572
|
-
|
|
160573
|
-
|
|
160574
|
-
if (!result)
|
|
160575
|
-
continue;
|
|
160776
|
+
const persistedIds = new Set;
|
|
160777
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160576
160778
|
if (signal?.aborted)
|
|
160577
160779
|
break;
|
|
160578
|
-
let
|
|
160579
|
-
|
|
160580
|
-
|
|
160581
|
-
|
|
160582
|
-
|
|
160583
|
-
|
|
160780
|
+
let result = null;
|
|
160781
|
+
const attemptStart = Date.now();
|
|
160782
|
+
try {
|
|
160783
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160784
|
+
} catch (error51) {
|
|
160785
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160786
|
+
}
|
|
160787
|
+
if (signal?.aborted)
|
|
160788
|
+
break;
|
|
160789
|
+
if (result) {
|
|
160790
|
+
let offset = 0;
|
|
160791
|
+
for (const item of slice) {
|
|
160792
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160793
|
+
offset += item.windows.length;
|
|
160794
|
+
if (persistedIds.has(item.candidate.id))
|
|
160795
|
+
continue;
|
|
160796
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160797
|
+
continue;
|
|
160798
|
+
}
|
|
160799
|
+
const rows = item.windows.map((window, index) => ({
|
|
160800
|
+
compartmentId: item.candidate.id,
|
|
160801
|
+
sessionId: item.candidate.sessionId,
|
|
160802
|
+
projectPath: projectIdentity,
|
|
160803
|
+
window,
|
|
160804
|
+
modelId,
|
|
160805
|
+
vector: vectors[index]
|
|
160806
|
+
}));
|
|
160807
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160808
|
+
persistedIds.add(item.candidate.id);
|
|
160584
160809
|
}
|
|
160585
|
-
const rows = item.windows.map((window, index) => ({
|
|
160586
|
-
compartmentId: item.candidate.id,
|
|
160587
|
-
sessionId: item.candidate.sessionId,
|
|
160588
|
-
projectPath: projectIdentity,
|
|
160589
|
-
window,
|
|
160590
|
-
modelId,
|
|
160591
|
-
vector: vectors[index]
|
|
160592
|
-
}));
|
|
160593
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160594
|
-
embedded += 1;
|
|
160595
160810
|
}
|
|
160596
|
-
|
|
160597
|
-
|
|
160811
|
+
if (persistedIds.size === slice.length)
|
|
160812
|
+
break;
|
|
160813
|
+
if (persistedIds.size > 0)
|
|
160814
|
+
break;
|
|
160815
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160816
|
+
break;
|
|
160817
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160818
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160819
|
+
}
|
|
160820
|
+
}
|
|
160821
|
+
embedded += persistedIds.size;
|
|
160822
|
+
if (!signal?.aborted) {
|
|
160823
|
+
for (const item of slice) {
|
|
160824
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160825
|
+
failed.push(item.candidate.id);
|
|
160826
|
+
}
|
|
160598
160827
|
}
|
|
160599
160828
|
}
|
|
160600
|
-
return { embedded, noWork };
|
|
160829
|
+
return { embedded, noWork, failed };
|
|
160601
160830
|
}
|
|
160602
160831
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160603
160832
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160620,9 +160849,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160620
160849
|
renewal.unref?.();
|
|
160621
160850
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160622
160851
|
const skipIds = [];
|
|
160852
|
+
const failedIds = [];
|
|
160623
160853
|
let embedded = 0;
|
|
160624
160854
|
let aborted2 = false;
|
|
160625
|
-
let
|
|
160855
|
+
let providerDown = false;
|
|
160856
|
+
let consecutiveFailedBatches = 0;
|
|
160626
160857
|
try {
|
|
160627
160858
|
options?.onProgress?.({ embedded, total });
|
|
160628
160859
|
for (;; ) {
|
|
@@ -160630,15 +160861,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160630
160861
|
aborted2 = true;
|
|
160631
160862
|
break;
|
|
160632
160863
|
}
|
|
160633
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160864
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160634
160865
|
if (candidates.length === 0)
|
|
160635
160866
|
break;
|
|
160636
|
-
const {
|
|
160867
|
+
const {
|
|
160868
|
+
embedded: n,
|
|
160869
|
+
noWork,
|
|
160870
|
+
failed
|
|
160871
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160637
160872
|
for (const id of noWork)
|
|
160638
160873
|
skipIds.push(id);
|
|
160874
|
+
for (const id of failed)
|
|
160875
|
+
failedIds.push(id);
|
|
160639
160876
|
if (n === 0 && noWork.length === 0) {
|
|
160640
|
-
|
|
160641
|
-
|
|
160877
|
+
consecutiveFailedBatches += 1;
|
|
160878
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160879
|
+
providerDown = true;
|
|
160880
|
+
break;
|
|
160881
|
+
}
|
|
160882
|
+
} else {
|
|
160883
|
+
consecutiveFailedBatches = 0;
|
|
160642
160884
|
}
|
|
160643
160885
|
embedded += n;
|
|
160644
160886
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160646,16 +160888,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160646
160888
|
}
|
|
160647
160889
|
} finally {
|
|
160648
160890
|
clearInterval(renewal);
|
|
160649
|
-
|
|
160891
|
+
try {
|
|
160892
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
160893
|
+
} catch (error51) {
|
|
160894
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
160895
|
+
}
|
|
160650
160896
|
}
|
|
160651
160897
|
if (aborted2)
|
|
160652
|
-
return { status: "aborted", embedded, total };
|
|
160653
|
-
if (
|
|
160898
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
160899
|
+
if (providerDown || failedIds.length > 0) {
|
|
160654
160900
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160655
|
-
if (remaining > 0)
|
|
160656
|
-
return { status: "stalled", embedded, total, remaining };
|
|
160901
|
+
if (remaining > 0) {
|
|
160902
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
160903
|
+
}
|
|
160904
|
+
}
|
|
160905
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
160906
|
+
}
|
|
160907
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
160908
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
160909
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
160910
|
+
return {
|
|
160911
|
+
enabled: false,
|
|
160912
|
+
model: snapshot?.model ?? "off",
|
|
160913
|
+
provider: snapshot?.provider ?? "off",
|
|
160914
|
+
session: { embedded: 0, total: 0 },
|
|
160915
|
+
memories: { embedded: 0, total: 0 },
|
|
160916
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
160917
|
+
};
|
|
160657
160918
|
}
|
|
160658
|
-
|
|
160919
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
160920
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
160921
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
160922
|
+
const commits = gitEnabled ? {
|
|
160923
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
160924
|
+
total: getCommitCount(db, projectIdentity),
|
|
160925
|
+
gitEnabled: true
|
|
160926
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
160927
|
+
return {
|
|
160928
|
+
enabled: true,
|
|
160929
|
+
model: snapshot.model,
|
|
160930
|
+
provider: snapshot.provider,
|
|
160931
|
+
session,
|
|
160932
|
+
memories,
|
|
160933
|
+
commits
|
|
160934
|
+
};
|
|
160659
160935
|
}
|
|
160660
160936
|
|
|
160661
160937
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160884,6 +161160,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160884
161160
|
"conversation_tokens",
|
|
160885
161161
|
"tool_call_tokens",
|
|
160886
161162
|
"cleared_reasoning_through_tag",
|
|
161163
|
+
"tool_reclaim_watermark",
|
|
160887
161164
|
"last_todo_state",
|
|
160888
161165
|
"cached_m0_bytes",
|
|
160889
161166
|
"cached_m1_bytes",
|
|
@@ -160932,6 +161209,7 @@ var META_COLUMNS = {
|
|
|
160932
161209
|
conversationTokens: "conversation_tokens",
|
|
160933
161210
|
toolCallTokens: "tool_call_tokens",
|
|
160934
161211
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161212
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160935
161213
|
lastTodoState: "last_todo_state",
|
|
160936
161214
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160937
161215
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -161000,7 +161278,7 @@ function isSessionMetaRow(row) {
|
|
|
161000
161278
|
if (row === null || typeof row !== "object")
|
|
161001
161279
|
return false;
|
|
161002
161280
|
const r = row;
|
|
161003
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161281
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161004
161282
|
}
|
|
161005
161283
|
function getDefaultSessionMeta(sessionId) {
|
|
161006
161284
|
return {
|
|
@@ -161023,6 +161301,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161023
161301
|
conversationTokens: 0,
|
|
161024
161302
|
toolCallTokens: 0,
|
|
161025
161303
|
clearedReasoningThroughTag: 0,
|
|
161304
|
+
toolReclaimWatermark: 0,
|
|
161026
161305
|
lastTodoState: "",
|
|
161027
161306
|
cachedM0Bytes: null,
|
|
161028
161307
|
cachedM1Bytes: null,
|
|
@@ -161086,6 +161365,7 @@ function toSessionMeta(row) {
|
|
|
161086
161365
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161087
161366
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161088
161367
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161368
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161089
161369
|
lastTodoState: lastTodoStateRaw,
|
|
161090
161370
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161091
161371
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161512,6 +161792,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161512
161792
|
var encoder = new TextEncoder;
|
|
161513
161793
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161514
161794
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161795
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161796
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161515
161797
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161516
161798
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161517
161799
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161524,6 +161806,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161524
161806
|
function stripMalformedTagNotationGlobally(value) {
|
|
161525
161807
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161526
161808
|
}
|
|
161809
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161810
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161811
|
+
}
|
|
161527
161812
|
function stripTagSectionCharacters(value) {
|
|
161528
161813
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161529
161814
|
}
|
|
@@ -161531,6 +161816,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161531
161816
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161532
161817
|
text = stripCompleteTagPairsGlobally(text);
|
|
161533
161818
|
text = stripMalformedTagNotationGlobally(text);
|
|
161819
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161534
161820
|
text = stripTagSectionCharacters(text);
|
|
161535
161821
|
return text.trim();
|
|
161536
161822
|
}
|
|
@@ -161543,6 +161829,7 @@ function stripTagPrefix(value) {
|
|
|
161543
161829
|
const prev = stripped;
|
|
161544
161830
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161545
161831
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161832
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161546
161833
|
if (stripped === prev)
|
|
161547
161834
|
break;
|
|
161548
161835
|
}
|
|
@@ -161958,9 +162245,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161958
162245
|
|
|
161959
162246
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161960
162247
|
|
|
161961
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162248
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162249
|
+
|
|
162250
|
+
Two recovery modes for finer detail:
|
|
162251
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162252
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161962
162253
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161963
162254
|
|
|
162255
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162256
|
+
function isRecord2(value) {
|
|
162257
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162258
|
+
}
|
|
162259
|
+
function roleLabel(role) {
|
|
162260
|
+
if (role === "assistant")
|
|
162261
|
+
return "A (assistant)";
|
|
162262
|
+
if (role === "user")
|
|
162263
|
+
return "U (user)";
|
|
162264
|
+
return role;
|
|
162265
|
+
}
|
|
162266
|
+
function truncate(value, max) {
|
|
162267
|
+
const t = value.trim();
|
|
162268
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162269
|
+
}
|
|
162270
|
+
function keyArg(input) {
|
|
162271
|
+
if (!input)
|
|
162272
|
+
return "";
|
|
162273
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162274
|
+
const v = input[k];
|
|
162275
|
+
if (typeof v === "string" && v.length > 0)
|
|
162276
|
+
return truncate(v, 60);
|
|
162277
|
+
}
|
|
162278
|
+
if (typeof input.description === "string")
|
|
162279
|
+
return truncate(input.description, 60);
|
|
162280
|
+
return "";
|
|
162281
|
+
}
|
|
162282
|
+
function asToolPart(part) {
|
|
162283
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162284
|
+
if (type === "tool") {
|
|
162285
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162286
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162287
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162288
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162289
|
+
return {
|
|
162290
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162291
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162292
|
+
title,
|
|
162293
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162294
|
+
output
|
|
162295
|
+
};
|
|
162296
|
+
}
|
|
162297
|
+
if (type === "tool_use") {
|
|
162298
|
+
return {
|
|
162299
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162300
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162301
|
+
title: null,
|
|
162302
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162303
|
+
output: null
|
|
162304
|
+
};
|
|
162305
|
+
}
|
|
162306
|
+
if (type === "tool_result") {
|
|
162307
|
+
const content = part.content;
|
|
162308
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162309
|
+
return {
|
|
162310
|
+
name: "tool_result",
|
|
162311
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162312
|
+
title: null,
|
|
162313
|
+
input: null,
|
|
162314
|
+
output
|
|
162315
|
+
};
|
|
162316
|
+
}
|
|
162317
|
+
return null;
|
|
162318
|
+
}
|
|
162319
|
+
function textOf(part) {
|
|
162320
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162321
|
+
return part.text;
|
|
162322
|
+
return null;
|
|
162323
|
+
}
|
|
162324
|
+
function reasoningOf(part) {
|
|
162325
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162326
|
+
return part.text;
|
|
162327
|
+
}
|
|
162328
|
+
return null;
|
|
162329
|
+
}
|
|
162330
|
+
function renderPartPreview(part) {
|
|
162331
|
+
if (!isRecord2(part))
|
|
162332
|
+
return null;
|
|
162333
|
+
const text = textOf(part);
|
|
162334
|
+
if (text !== null) {
|
|
162335
|
+
const t = truncate(text, 200);
|
|
162336
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162337
|
+
}
|
|
162338
|
+
const tool = asToolPart(part);
|
|
162339
|
+
if (tool) {
|
|
162340
|
+
const arg = keyArg(tool.input);
|
|
162341
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162342
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162343
|
+
}
|
|
162344
|
+
const reasoning = reasoningOf(part);
|
|
162345
|
+
if (reasoning !== null)
|
|
162346
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162347
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162348
|
+
if (type === "file")
|
|
162349
|
+
return " • [file]";
|
|
162350
|
+
if (type === "step-start" || type === "step-finish")
|
|
162351
|
+
return null;
|
|
162352
|
+
return ` • [${type}]`;
|
|
162353
|
+
}
|
|
162354
|
+
function renderPartFull(part) {
|
|
162355
|
+
if (!isRecord2(part))
|
|
162356
|
+
return null;
|
|
162357
|
+
const text = textOf(part);
|
|
162358
|
+
if (text !== null) {
|
|
162359
|
+
return text.trim().length > 0 ? ` [text]
|
|
162360
|
+
${text}` : null;
|
|
162361
|
+
}
|
|
162362
|
+
const tool = asToolPart(part);
|
|
162363
|
+
if (tool) {
|
|
162364
|
+
const lines = [];
|
|
162365
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162366
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162367
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162368
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162369
|
+
}
|
|
162370
|
+
if (tool.input)
|
|
162371
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162372
|
+
if (tool.output !== null)
|
|
162373
|
+
lines.push(` output:
|
|
162374
|
+
${tool.output}`);
|
|
162375
|
+
return lines.join(`
|
|
162376
|
+
`);
|
|
162377
|
+
}
|
|
162378
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162379
|
+
if (type === "file") {
|
|
162380
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162381
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162382
|
+
}
|
|
162383
|
+
return null;
|
|
162384
|
+
}
|
|
162385
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162386
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162387
|
+
if (!msg) {
|
|
162388
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162389
|
+
}
|
|
162390
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162391
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162392
|
+
if (rendered.length === 0) {
|
|
162393
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162394
|
+
} else {
|
|
162395
|
+
lines.push(...rendered);
|
|
162396
|
+
}
|
|
162397
|
+
return lines.join(`
|
|
162398
|
+
`);
|
|
162399
|
+
}
|
|
162400
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162401
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162402
|
+
const out = [];
|
|
162403
|
+
let usedTokens = 0;
|
|
162404
|
+
let lastOrdinal = start - 1;
|
|
162405
|
+
let truncated = false;
|
|
162406
|
+
for (const msg of messages) {
|
|
162407
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162408
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162409
|
+
const block = partLines.length > 0 ? `${header}
|
|
162410
|
+
${partLines.join(`
|
|
162411
|
+
`)}` : header;
|
|
162412
|
+
const blockTokens = estimateTokens(block);
|
|
162413
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162414
|
+
truncated = true;
|
|
162415
|
+
break;
|
|
162416
|
+
}
|
|
162417
|
+
out.push(block);
|
|
162418
|
+
usedTokens += blockTokens;
|
|
162419
|
+
lastOrdinal = msg.ordinal;
|
|
162420
|
+
}
|
|
162421
|
+
return { text: out.join(`
|
|
162422
|
+
|
|
162423
|
+
`), lastOrdinal, truncated };
|
|
162424
|
+
}
|
|
162425
|
+
|
|
161964
162426
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161965
162427
|
var exports_memory = {};
|
|
161966
162428
|
__export(exports_memory, {
|
|
@@ -166317,12 +166779,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166317
166779
|
|
|
166318
166780
|
// src/tools/ctx-expand.ts
|
|
166319
166781
|
var ParamsSchema = exports_typebox.Object({
|
|
166320
|
-
start: exports_typebox.Number({
|
|
166782
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166321
166783
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166322
|
-
}),
|
|
166323
|
-
end: exports_typebox.Number({
|
|
166784
|
+
})),
|
|
166785
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166324
166786
|
description: "End message ordinal (from compartment end attribute)"
|
|
166325
|
-
})
|
|
166787
|
+
})),
|
|
166788
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166789
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166790
|
+
})),
|
|
166791
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166792
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166793
|
+
}))
|
|
166326
166794
|
});
|
|
166327
166795
|
function ok(text) {
|
|
166328
166796
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166341,22 +166809,41 @@ function createCtxExpandTool(deps) {
|
|
|
166341
166809
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166342
166810
|
parameters: ParamsSchema,
|
|
166343
166811
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166344
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166345
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166346
|
-
}
|
|
166347
166812
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166348
166813
|
if (!sessionId) {
|
|
166349
166814
|
return err("Error: no active Pi session.");
|
|
166350
166815
|
}
|
|
166351
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166352
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166353
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166354
|
-
}
|
|
166355
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166356
166816
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166357
166817
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166358
166818
|
});
|
|
166359
166819
|
try {
|
|
166820
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166821
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166822
|
+
}
|
|
166823
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166824
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166825
|
+
}
|
|
166826
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166827
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166828
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166829
|
+
}
|
|
166830
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166831
|
+
if (params.verbose === true) {
|
|
166832
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166833
|
+
if (!v.text) {
|
|
166834
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166835
|
+
}
|
|
166836
|
+
const out = [
|
|
166837
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166838
|
+
"",
|
|
166839
|
+
v.text
|
|
166840
|
+
];
|
|
166841
|
+
if (v.truncated) {
|
|
166842
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166843
|
+
}
|
|
166844
|
+
return ok(out.join(`
|
|
166845
|
+
`));
|
|
166846
|
+
}
|
|
166360
166847
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166361
166848
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166362
166849
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -168216,6 +168703,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168216
168703
|
last_transform_error: "'' AS last_transform_error",
|
|
168217
168704
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168218
168705
|
last_todo_state: "'' AS last_todo_state",
|
|
168706
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168219
168707
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168220
168708
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168221
168709
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168293,6 +168781,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168293
168781
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168294
168782
|
})();
|
|
168295
168783
|
}
|
|
168784
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168785
|
+
if (maxTagNumber <= 0)
|
|
168786
|
+
return;
|
|
168787
|
+
db.transaction(() => {
|
|
168788
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168789
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168790
|
+
})();
|
|
168791
|
+
}
|
|
168296
168792
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168297
168793
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168298
168794
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168667,15 +169163,22 @@ function ownerMessageIdForTagRow(row) {
|
|
|
168667
169163
|
}
|
|
168668
169164
|
return row.message_id.replace(CONTENT_ID_SUFFIX, "");
|
|
168669
169165
|
}
|
|
168670
|
-
function getActiveTagTokenAggregate(db, sessionId) {
|
|
168671
|
-
const
|
|
169166
|
+
function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
169167
|
+
const toolOutputExpr = protectedTags > 0 ? `COALESCE(SUM(CASE WHEN type = 'tool' AND tag_number < (
|
|
169168
|
+
SELECT tag_number FROM tags
|
|
169169
|
+
WHERE session_id = ? AND status = 'active'
|
|
169170
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169171
|
+
) THEN COALESCE(token_count, 0) ELSE 0 END), 0)` : `COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)`;
|
|
169172
|
+
const sql = `SELECT
|
|
168672
169173
|
COALESCE(SUM(CASE WHEN type != 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)
|
|
168673
169174
|
+ COALESCE(SUM(COALESCE(reasoning_token_count, 0)), 0) AS conversation,
|
|
168674
169175
|
COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) + COALESCE(input_token_count, 0) ELSE 0 END), 0) AS tool_call,
|
|
168675
|
-
|
|
169176
|
+
${toolOutputExpr} AS tool_output,
|
|
168676
169177
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
168677
169178
|
FROM tags
|
|
168678
|
-
WHERE session_id = ? AND status = 'active'
|
|
169179
|
+
WHERE session_id = ? AND status = 'active'`;
|
|
169180
|
+
const params = protectedTags > 0 ? [sessionId, protectedTags - 1, sessionId] : [sessionId];
|
|
169181
|
+
const row = db.prepare(sql).get(...params);
|
|
168679
169182
|
return {
|
|
168680
169183
|
conversation: row?.conversation ?? 0,
|
|
168681
169184
|
toolCall: row?.tool_call ?? 0,
|
|
@@ -168683,6 +169186,26 @@ function getActiveTagTokenAggregate(db, sessionId) {
|
|
|
168683
169186
|
nullCount: row?.null_count ?? 0
|
|
168684
169187
|
};
|
|
168685
169188
|
}
|
|
169189
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169190
|
+
if (limit <= 0)
|
|
169191
|
+
return [];
|
|
169192
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169193
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169194
|
+
SELECT tag_number FROM tags
|
|
169195
|
+
WHERE session_id = ? AND status = 'active'
|
|
169196
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169197
|
+
)` : "";
|
|
169198
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169199
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169200
|
+
FROM tags
|
|
169201
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169202
|
+
ORDER BY tag_number ASC, id ASC
|
|
169203
|
+
LIMIT ?`).all(...params);
|
|
169204
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169205
|
+
tagNumber: row.tag_number,
|
|
169206
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169207
|
+
}));
|
|
169208
|
+
}
|
|
168686
169209
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
168687
169210
|
const row = db.prepare(`SELECT
|
|
168688
169211
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -169632,15 +170155,16 @@ function parseInteger(str) {
|
|
|
169632
170155
|
}
|
|
169633
170156
|
|
|
169634
170157
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169635
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169636
|
-
|
|
170158
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170159
|
+
|
|
170160
|
+
How it works:
|
|
170161
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170162
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170163
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
169637
170164
|
|
|
169638
|
-
|
|
169639
|
-
|
|
169640
|
-
-
|
|
169641
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
169642
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
169643
|
-
- Dropped content is gone forever.`;
|
|
170165
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170166
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170167
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169644
170168
|
|
|
169645
170169
|
// src/tools/ctx-reduce.ts
|
|
169646
170170
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169825,120 +170349,6 @@ ${body}` : subject;
|
|
|
169825
170349
|
}
|
|
169826
170350
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169827
170351
|
init_logger();
|
|
169828
|
-
|
|
169829
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169830
|
-
init_logger();
|
|
169831
|
-
var insertStatements = new WeakMap;
|
|
169832
|
-
var existingShasStatements = new WeakMap;
|
|
169833
|
-
var projectCountStatements = new WeakMap;
|
|
169834
|
-
var evictStatements = new WeakMap;
|
|
169835
|
-
var evictOverflowStatements = new WeakMap;
|
|
169836
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169837
|
-
function getInsertStatement(db) {
|
|
169838
|
-
let stmt = insertStatements.get(db);
|
|
169839
|
-
if (!stmt) {
|
|
169840
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169841
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169842
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169843
|
-
project_path = excluded.project_path,
|
|
169844
|
-
short_sha = excluded.short_sha,
|
|
169845
|
-
message = excluded.message,
|
|
169846
|
-
author = excluded.author,
|
|
169847
|
-
committed_at = excluded.committed_at,
|
|
169848
|
-
indexed_at = excluded.indexed_at
|
|
169849
|
-
WHERE git_commits.message != excluded.message`);
|
|
169850
|
-
insertStatements.set(db, stmt);
|
|
169851
|
-
}
|
|
169852
|
-
return stmt;
|
|
169853
|
-
}
|
|
169854
|
-
function getExistingShasStatement(db) {
|
|
169855
|
-
let stmt = existingShasStatements.get(db);
|
|
169856
|
-
if (!stmt) {
|
|
169857
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169858
|
-
existingShasStatements.set(db, stmt);
|
|
169859
|
-
}
|
|
169860
|
-
return stmt;
|
|
169861
|
-
}
|
|
169862
|
-
function getProjectCountStatement(db) {
|
|
169863
|
-
let stmt = projectCountStatements.get(db);
|
|
169864
|
-
if (!stmt) {
|
|
169865
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169866
|
-
projectCountStatements.set(db, stmt);
|
|
169867
|
-
}
|
|
169868
|
-
return stmt;
|
|
169869
|
-
}
|
|
169870
|
-
function getLatestCommitTimeStatement(db) {
|
|
169871
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169872
|
-
if (!stmt) {
|
|
169873
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169874
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169875
|
-
}
|
|
169876
|
-
return stmt;
|
|
169877
|
-
}
|
|
169878
|
-
function getEvictOverflowStatement(db) {
|
|
169879
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169880
|
-
if (!stmt) {
|
|
169881
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169882
|
-
WHERE rowid IN (
|
|
169883
|
-
SELECT rowid FROM git_commits
|
|
169884
|
-
WHERE project_path = ?
|
|
169885
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169886
|
-
LIMIT -1 OFFSET ?
|
|
169887
|
-
)`);
|
|
169888
|
-
evictOverflowStatements.set(db, stmt);
|
|
169889
|
-
}
|
|
169890
|
-
return stmt;
|
|
169891
|
-
}
|
|
169892
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169893
|
-
if (commits.length === 0)
|
|
169894
|
-
return { inserted: 0, updated: 0 };
|
|
169895
|
-
const existing = new Set;
|
|
169896
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169897
|
-
existing.add(row.sha);
|
|
169898
|
-
}
|
|
169899
|
-
let inserted = 0;
|
|
169900
|
-
let updated = 0;
|
|
169901
|
-
const now = Date.now();
|
|
169902
|
-
const insertStmt = getInsertStatement(db);
|
|
169903
|
-
db.transaction(() => {
|
|
169904
|
-
for (const commit of commits) {
|
|
169905
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169906
|
-
if (result.changes > 0) {
|
|
169907
|
-
if (existing.has(commit.sha)) {
|
|
169908
|
-
updated++;
|
|
169909
|
-
} else {
|
|
169910
|
-
inserted++;
|
|
169911
|
-
existing.add(commit.sha);
|
|
169912
|
-
}
|
|
169913
|
-
}
|
|
169914
|
-
}
|
|
169915
|
-
})();
|
|
169916
|
-
return { inserted, updated };
|
|
169917
|
-
}
|
|
169918
|
-
function getCommitCount(db, projectPath) {
|
|
169919
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169920
|
-
return row?.count ?? 0;
|
|
169921
|
-
}
|
|
169922
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169923
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169924
|
-
return row?.latest ?? null;
|
|
169925
|
-
}
|
|
169926
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169927
|
-
if (maxCommits <= 0)
|
|
169928
|
-
return 0;
|
|
169929
|
-
const count = getCommitCount(db, projectPath);
|
|
169930
|
-
if (count <= maxCommits)
|
|
169931
|
-
return 0;
|
|
169932
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169933
|
-
const after = getCommitCount(db, projectPath);
|
|
169934
|
-
const evicted = Math.max(0, count - after);
|
|
169935
|
-
if (evicted > 0) {
|
|
169936
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169937
|
-
}
|
|
169938
|
-
return evicted;
|
|
169939
|
-
}
|
|
169940
|
-
|
|
169941
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169942
170352
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169943
170353
|
var EMBED_BATCH_SIZE = 16;
|
|
169944
170354
|
var EMBED_MAX_PER_SWEEP = 500;
|