@wolfx/pi-magic-context 0.24.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1570 -515
- package/dist/subagent-entry.js +821 -212
- package/package.json +4 -5
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -9574,6 +9582,41 @@ var MIGRATIONS = [
|
|
|
9574
9582
|
`);
|
|
9575
9583
|
}
|
|
9576
9584
|
}
|
|
9585
|
+
},
|
|
9586
|
+
{
|
|
9587
|
+
version: 37,
|
|
9588
|
+
description: "emergency drain catch-up latch + historian drain failure backoff",
|
|
9589
|
+
up: (db) => {
|
|
9590
|
+
const hasSessionMeta = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='session_meta'").get();
|
|
9591
|
+
if (!hasSessionMeta)
|
|
9592
|
+
return;
|
|
9593
|
+
ensureColumn(db, "session_meta", "emergency_drain_active", "INTEGER NOT NULL DEFAULT 0");
|
|
9594
|
+
ensureColumn(db, "session_meta", "historian_drain_failure_at", "INTEGER NOT NULL DEFAULT 0");
|
|
9595
|
+
}
|
|
9596
|
+
},
|
|
9597
|
+
{
|
|
9598
|
+
version: 38,
|
|
9599
|
+
description: "durable transform decisions for cache-event cause attribution",
|
|
9600
|
+
up: (db) => {
|
|
9601
|
+
db.exec(`
|
|
9602
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
9603
|
+
session_id TEXT NOT NULL,
|
|
9604
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
9605
|
+
message_id TEXT NOT NULL,
|
|
9606
|
+
ts_ms INTEGER NOT NULL,
|
|
9607
|
+
decision TEXT NOT NULL,
|
|
9608
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
9609
|
+
materialize_reason TEXT,
|
|
9610
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
9611
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
9612
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
9613
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
9614
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
9615
|
+
);
|
|
9616
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
9617
|
+
ON transform_decisions(session_id, harness);
|
|
9618
|
+
`);
|
|
9619
|
+
}
|
|
9577
9620
|
}
|
|
9578
9621
|
];
|
|
9579
9622
|
var LATEST_MIGRATION_VERSION = MIGRATIONS.reduce((max, m) => Math.max(max, m.version), 0);
|
|
@@ -142550,8 +142593,9 @@ function backfillToolOwnersInChunks(db, result) {
|
|
|
142550
142593
|
var databases = new Map;
|
|
142551
142594
|
var persistenceByDatabase = new WeakMap;
|
|
142552
142595
|
var persistenceErrorByDatabase = new WeakMap;
|
|
142596
|
+
var pathByDatabase = new WeakMap;
|
|
142553
142597
|
var lastSchemaFenceRejection = null;
|
|
142554
|
-
var LATEST_SUPPORTED_VERSION =
|
|
142598
|
+
var LATEST_SUPPORTED_VERSION = 38;
|
|
142555
142599
|
function resolveDatabasePath(dbPathOverride) {
|
|
142556
142600
|
if (dbPathOverride) {
|
|
142557
142601
|
return { dbDir: dirname2(dbPathOverride), dbPath: dbPathOverride };
|
|
@@ -142564,6 +142608,9 @@ function resolveDatabasePath(dbPathOverride) {
|
|
|
142564
142608
|
const dbDir = getMagicContextStorageDir();
|
|
142565
142609
|
return { dbDir, dbPath: join4(dbDir, "context.db") };
|
|
142566
142610
|
}
|
|
142611
|
+
function getDatabasePath(db) {
|
|
142612
|
+
return pathByDatabase.get(db) ?? null;
|
|
142613
|
+
}
|
|
142567
142614
|
function migrateLegacyStorageIfNeeded(targetDbPath, targetDbDir) {
|
|
142568
142615
|
if (existsSync3(targetDbPath))
|
|
142569
142616
|
return;
|
|
@@ -143078,6 +143125,8 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143078
143125
|
recovery_no_eligible_head_count INTEGER NOT NULL DEFAULT 0,
|
|
143079
143126
|
force_emergency_bypass_window_start INTEGER NOT NULL DEFAULT 0,
|
|
143080
143127
|
force_emergency_bypass_used INTEGER NOT NULL DEFAULT 0,
|
|
143128
|
+
emergency_drain_active INTEGER NOT NULL DEFAULT 0,
|
|
143129
|
+
historian_drain_failure_at INTEGER NOT NULL DEFAULT 0,
|
|
143081
143130
|
cached_m0_materialized_at INTEGER,
|
|
143082
143131
|
cached_m0_session_facts_version INTEGER,
|
|
143083
143132
|
cached_m0_upgrade_state TEXT,
|
|
@@ -143141,6 +143190,23 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143141
143190
|
CREATE INDEX IF NOT EXISTS idx_historian_runs_status
|
|
143142
143191
|
ON historian_runs(status, created_at DESC);
|
|
143143
143192
|
|
|
143193
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
143194
|
+
session_id TEXT NOT NULL,
|
|
143195
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
143196
|
+
message_id TEXT NOT NULL,
|
|
143197
|
+
ts_ms INTEGER NOT NULL,
|
|
143198
|
+
decision TEXT NOT NULL,
|
|
143199
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
143200
|
+
materialize_reason TEXT,
|
|
143201
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
143202
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143203
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
143204
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143205
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
143206
|
+
);
|
|
143207
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
143208
|
+
ON transform_decisions(session_id, harness);
|
|
143209
|
+
|
|
143144
143210
|
CREATE INDEX IF NOT EXISTS idx_tags_session_tag_number ON tags(session_id, tag_number);
|
|
143145
143211
|
CREATE INDEX IF NOT EXISTS idx_tags_session_message_id ON tags(session_id, message_id);
|
|
143146
143212
|
CREATE INDEX IF NOT EXISTS idx_pending_ops_session ON pending_ops(session_id);
|
|
@@ -143218,6 +143284,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143284
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143285
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143286
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143287
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143288
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143289
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143290
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -143291,6 +143358,8 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143291
143358
|
ensureColumn(db, "session_meta", "recovery_no_eligible_head_count", "INTEGER NOT NULL DEFAULT 0");
|
|
143292
143359
|
ensureColumn(db, "session_meta", "force_emergency_bypass_window_start", "INTEGER NOT NULL DEFAULT 0");
|
|
143293
143360
|
ensureColumn(db, "session_meta", "force_emergency_bypass_used", "INTEGER NOT NULL DEFAULT 0");
|
|
143361
|
+
ensureColumn(db, "session_meta", "emergency_drain_active", "INTEGER NOT NULL DEFAULT 0");
|
|
143362
|
+
ensureColumn(db, "session_meta", "historian_drain_failure_at", "INTEGER NOT NULL DEFAULT 0");
|
|
143294
143363
|
ensureColumn(db, "session_meta", "cached_m0_materialized_at", "INTEGER");
|
|
143295
143364
|
ensureColumn(db, "session_meta", "cached_m0_session_facts_version", "INTEGER");
|
|
143296
143365
|
ensureColumn(db, "session_meta", "cached_m0_upgrade_state", "TEXT");
|
|
@@ -143369,6 +143438,22 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143369
143438
|
failed_at INTEGER NOT NULL,
|
|
143370
143439
|
UNIQUE(table_name, row_id)
|
|
143371
143440
|
);
|
|
143441
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
143442
|
+
session_id TEXT NOT NULL,
|
|
143443
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
143444
|
+
message_id TEXT NOT NULL,
|
|
143445
|
+
ts_ms INTEGER NOT NULL,
|
|
143446
|
+
decision TEXT NOT NULL,
|
|
143447
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
143448
|
+
materialize_reason TEXT,
|
|
143449
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
143450
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143451
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
143452
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143453
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
143454
|
+
);
|
|
143455
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
143456
|
+
ON transform_decisions(session_id, harness);
|
|
143372
143457
|
`);
|
|
143373
143458
|
ensureColumn(db, "tags", "harness", "TEXT NOT NULL DEFAULT 'opencode'");
|
|
143374
143459
|
ensureColumn(db, "pending_ops", "harness", "TEXT NOT NULL DEFAULT 'opencode'");
|
|
@@ -143455,7 +143540,9 @@ function healNullIntegerColumns(db) {
|
|
|
143455
143540
|
["protected_tail_drain_tokens", 0],
|
|
143456
143541
|
["recovery_no_eligible_head_count", 0],
|
|
143457
143542
|
["force_emergency_bypass_window_start", 0],
|
|
143458
|
-
["force_emergency_bypass_used", 0]
|
|
143543
|
+
["force_emergency_bypass_used", 0],
|
|
143544
|
+
["emergency_drain_active", 0],
|
|
143545
|
+
["historian_drain_failure_at", 0]
|
|
143459
143546
|
];
|
|
143460
143547
|
for (const [column, fallback] of columns) {
|
|
143461
143548
|
try {
|
|
@@ -143531,6 +143618,7 @@ function openDatabase(dbPathOrOptions) {
|
|
|
143531
143618
|
setDatabase(db);
|
|
143532
143619
|
loadToolDefinitionMeasurements(db);
|
|
143533
143620
|
databases.set(dbPath, db);
|
|
143621
|
+
pathByDatabase.set(db, dbPath);
|
|
143534
143622
|
persistenceByDatabase.set(db, true);
|
|
143535
143623
|
persistenceErrorByDatabase.delete(db);
|
|
143536
143624
|
return db;
|
|
@@ -158132,14 +158220,16 @@ var SidekickConfigSchema = AgentOverrideConfigSchema.extend({
|
|
|
158132
158220
|
}).optional();
|
|
158133
158221
|
var HistorianConfigSchema = AgentOverrideConfigSchema.extend({
|
|
158134
158222
|
two_pass: exports_external.boolean().default(false).describe("Run a second editor pass over historian output to clean low-signal U: lines and cross-compartment duplicates. Adds ~1 extra API call and ~1.3x cost per historian run. Useful for models without extended thinking support. (default: false)"),
|
|
158135
|
-
thinking_level: PiThinkingLevelSchema.describe("Pi only: explicit thinking level passed as --thinking <level> to Pi historian subagent invocations. Required when using reasoning models (e.g. github-copilot/gpt-5.4) because Pi's default thinking-level resolution can pick a value the provider rejects. OpenCode users set variant instead. Valid: off | minimal | low | medium | high | xhigh")
|
|
158223
|
+
thinking_level: PiThinkingLevelSchema.describe("Pi only: explicit thinking level passed as --thinking <level> to Pi historian subagent invocations. Required when using reasoning models (e.g. github-copilot/gpt-5.4) because Pi's default thinking-level resolution can pick a value the provider rejects. OpenCode users set variant instead. Valid: off | minimal | low | medium | high | xhigh"),
|
|
158224
|
+
disallowed_tools: exports_external.array(exports_external.enum(["*", "read", "aft_outline", "aft_zoom", "aft_search"])).default([]).describe(`OpenCode only. Tools to REMOVE from the historian's default allow-list [read, aft_outline, aft_zoom, aft_search]. Applies to both historian and historian-editor agents. Use ["*"] to strip all tool definitions from the model request — this prevents weak instruction-following models (e.g. mistral-small-latest) from entering tool-calling loops. Individual tool names remove just that tool. Note: a user-supplied historian.permission override can re-allow a tool that disallowed_tools removed — disallowed_tools sets the baseline, permission overrides take precedence. (default: [])`)
|
|
158136
158225
|
}).optional();
|
|
158137
158226
|
var BaseEmbeddingConfigSchema = exports_external.object({
|
|
158138
158227
|
provider: exports_external.enum(["local", "openai-compatible", "off"]).default("local").describe("Embedding provider. 'local' uses Xenova/all-MiniLM-L6-v2, 'openai-compatible' requires endpoint and model, 'off' disables embeddings."),
|
|
158139
158228
|
model: exports_external.string().optional().describe("Embedding model name. Required for openai-compatible, ignored for local."),
|
|
158140
158229
|
endpoint: exports_external.string().optional().describe("API endpoint URL. Required when provider is openai-compatible."),
|
|
158141
158230
|
api_key: exports_external.string().optional().describe("API key for remote embedding provider (optional)"),
|
|
158142
|
-
input_type: exports_external.string().optional().describe("
|
|
158231
|
+
input_type: exports_external.string().optional().describe("Default input_type for stored/indexed (passage) embeddings in the request body. Required by some openai-compatible providers (e.g. NVIDIA NIM). Omitted from the request when unset."),
|
|
158232
|
+
query_input_type: exports_external.string().optional().describe("Optional input_type for query (search) embeddings on asymmetric models (e.g. NVIDIA NIM 'query'). When unset, query embeddings use embedding.input_type. Passage/stored content always uses embedding.input_type."),
|
|
158143
158233
|
truncate: exports_external.string().optional().describe("Optional truncate mode sent in the embedding request body (e.g. NVIDIA NIM accepts 'NONE' | 'START' | 'END'). Omitted from the request when unset."),
|
|
158144
158234
|
max_input_tokens: exports_external.number().int().positive().optional().describe("Optional maximum input tokens for chunk embeddings. Defaults conservatively to 512 when omitted.")
|
|
158145
158235
|
}).superRefine((data, ctx) => {
|
|
@@ -158169,6 +158259,7 @@ var EmbeddingConfigSchema = BaseEmbeddingConfigSchema.transform((data) => {
|
|
|
158169
158259
|
if (data.provider === "openai-compatible") {
|
|
158170
158260
|
const apiKey = data.api_key?.trim();
|
|
158171
158261
|
const inputType = data.input_type?.trim();
|
|
158262
|
+
const queryInputType = data.query_input_type?.trim();
|
|
158172
158263
|
const truncate = data.truncate?.trim();
|
|
158173
158264
|
return {
|
|
158174
158265
|
provider: "openai-compatible",
|
|
@@ -158176,6 +158267,7 @@ var EmbeddingConfigSchema = BaseEmbeddingConfigSchema.transform((data) => {
|
|
|
158176
158267
|
endpoint: data.endpoint?.trim() ?? "",
|
|
158177
158268
|
...apiKey ? { api_key: apiKey } : {},
|
|
158178
158269
|
...inputType ? { input_type: inputType } : {},
|
|
158270
|
+
...queryInputType ? { query_input_type: queryInputType } : {},
|
|
158179
158271
|
...truncate ? { truncate } : {},
|
|
158180
158272
|
...data.max_input_tokens ? { max_input_tokens: data.max_input_tokens } : {}
|
|
158181
158273
|
};
|
|
@@ -158899,6 +158991,16 @@ function buildCanonicalChunkTextFromFts(db, sessionId, startOrdinal, endOrdinal)
|
|
|
158899
158991
|
return lines.join(`
|
|
158900
158992
|
`);
|
|
158901
158993
|
}
|
|
158994
|
+
function buildCompartmentSummaryFallbackText(db, compartmentId) {
|
|
158995
|
+
const row = db.prepare("SELECT title, p1, content FROM compartments WHERE id = ?").get(compartmentId);
|
|
158996
|
+
if (!row)
|
|
158997
|
+
return "";
|
|
158998
|
+
const title = typeof row.title === "string" ? row.title.trim() : "";
|
|
158999
|
+
const p1 = typeof row.p1 === "string" ? row.p1.trim() : "";
|
|
159000
|
+
const body = p1.length > 0 ? p1 : typeof row.content === "string" ? row.content.trim() : "";
|
|
159001
|
+
return [title, body].filter((s) => s.length > 0).join(`
|
|
159002
|
+
`);
|
|
159003
|
+
}
|
|
158902
159004
|
function canonicalizeInMemoryChunkTextForEmbedding(chunkText, startOrdinal, endOrdinal) {
|
|
158903
159005
|
const lines = [];
|
|
158904
159006
|
for (const rawLine of chunkText.split(/\r?\n/)) {
|
|
@@ -159142,6 +159244,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159142
159244
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159143
159245
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159144
159246
|
}
|
|
159247
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159248
|
+
const row = db.prepare(`SELECT
|
|
159249
|
+
COUNT(*) AS total,
|
|
159250
|
+
SUM(CASE WHEN EXISTS (
|
|
159251
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159252
|
+
WHERE e.compartment_id = c.id
|
|
159253
|
+
AND e.project_path = ?
|
|
159254
|
+
AND e.model_id = ?
|
|
159255
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159256
|
+
FROM compartments c
|
|
159257
|
+
JOIN session_projects sp
|
|
159258
|
+
ON sp.session_id = c.session_id
|
|
159259
|
+
AND sp.harness = c.harness
|
|
159260
|
+
AND sp.project_path = ?
|
|
159261
|
+
WHERE c.session_id = ?
|
|
159262
|
+
AND c.start_message IS NOT NULL
|
|
159263
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159264
|
+
return {
|
|
159265
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159266
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159267
|
+
};
|
|
159268
|
+
}
|
|
159145
159269
|
|
|
159146
159270
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159147
159271
|
function cosineSimilarity(a, b) {
|
|
@@ -159294,6 +159418,19 @@ async function withQuietConsole(fn) {
|
|
|
159294
159418
|
console.error = origError;
|
|
159295
159419
|
}
|
|
159296
159420
|
}
|
|
159421
|
+
var nativeRuntimeMissing = false;
|
|
159422
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159423
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159424
|
+
const lower = message.toLowerCase();
|
|
159425
|
+
const code = error51?.code;
|
|
159426
|
+
const name2 = error51?.name;
|
|
159427
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159428
|
+
return true;
|
|
159429
|
+
}
|
|
159430
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159431
|
+
return false;
|
|
159432
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159433
|
+
}
|
|
159297
159434
|
function isTransientLoadError(error51) {
|
|
159298
159435
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159299
159436
|
if (!message)
|
|
@@ -159358,6 +159495,9 @@ class LocalEmbeddingProvider {
|
|
|
159358
159495
|
if (this.pipeline) {
|
|
159359
159496
|
return true;
|
|
159360
159497
|
}
|
|
159498
|
+
if (nativeRuntimeMissing) {
|
|
159499
|
+
return false;
|
|
159500
|
+
}
|
|
159361
159501
|
if (this.initPromise) {
|
|
159362
159502
|
await this.initPromise;
|
|
159363
159503
|
return this.pipeline !== null;
|
|
@@ -159424,7 +159564,12 @@ class LocalEmbeddingProvider {
|
|
|
159424
159564
|
await releaseLock();
|
|
159425
159565
|
}
|
|
159426
159566
|
} catch (error51) {
|
|
159427
|
-
|
|
159567
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159568
|
+
nativeRuntimeMissing = true;
|
|
159569
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @wolfx/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159570
|
+
} else {
|
|
159571
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159572
|
+
}
|
|
159428
159573
|
this.pipeline = null;
|
|
159429
159574
|
} finally {
|
|
159430
159575
|
this.initPromise = null;
|
|
@@ -159450,7 +159595,7 @@ class LocalEmbeddingProvider {
|
|
|
159450
159595
|
waiter();
|
|
159451
159596
|
}
|
|
159452
159597
|
}
|
|
159453
|
-
async embed(text, signal) {
|
|
159598
|
+
async embed(text, signal, _purpose) {
|
|
159454
159599
|
if (signal?.aborted)
|
|
159455
159600
|
return null;
|
|
159456
159601
|
if (this.disposing)
|
|
@@ -159476,7 +159621,7 @@ class LocalEmbeddingProvider {
|
|
|
159476
159621
|
this.finishInFlight();
|
|
159477
159622
|
}
|
|
159478
159623
|
}
|
|
159479
|
-
async embedBatch(texts, signal) {
|
|
159624
|
+
async embedBatch(texts, signal, _purpose) {
|
|
159480
159625
|
if (texts.length === 0) {
|
|
159481
159626
|
return [];
|
|
159482
159627
|
}
|
|
@@ -159615,6 +159760,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159615
159760
|
model;
|
|
159616
159761
|
apiKey;
|
|
159617
159762
|
inputType;
|
|
159763
|
+
queryInputType;
|
|
159618
159764
|
truncate;
|
|
159619
159765
|
initialized = false;
|
|
159620
159766
|
failureTimes = [];
|
|
@@ -159627,6 +159773,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159627
159773
|
this.model = options.model?.trim() ?? "";
|
|
159628
159774
|
this.apiKey = options.apiKey?.trim() ?? "";
|
|
159629
159775
|
this.inputType = options.inputType?.trim() ?? "";
|
|
159776
|
+
this.queryInputType = options.queryInputType?.trim() ?? "";
|
|
159630
159777
|
this.truncate = options.truncate?.trim() ?? "";
|
|
159631
159778
|
this.maxInputTokens = typeof options.maxInputTokens === "number" && Number.isFinite(options.maxInputTokens) ? Math.max(1, Math.floor(options.maxInputTokens)) : 512;
|
|
159632
159779
|
this.modelId = getEmbeddingProviderIdentity({
|
|
@@ -159654,11 +159801,17 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159654
159801
|
this.initialized = true;
|
|
159655
159802
|
return true;
|
|
159656
159803
|
}
|
|
159657
|
-
|
|
159658
|
-
|
|
159804
|
+
resolveInputTypeForPurpose(purpose = "passage") {
|
|
159805
|
+
if (purpose === "query") {
|
|
159806
|
+
return this.queryInputType || this.inputType;
|
|
159807
|
+
}
|
|
159808
|
+
return this.inputType;
|
|
159809
|
+
}
|
|
159810
|
+
async embed(text, signal, purpose) {
|
|
159811
|
+
const [embedding] = await this.embedBatch([text], signal, purpose);
|
|
159659
159812
|
return embedding ?? null;
|
|
159660
159813
|
}
|
|
159661
|
-
async embedBatch(texts, signal) {
|
|
159814
|
+
async embedBatch(texts, signal, purpose) {
|
|
159662
159815
|
if (texts.length === 0) {
|
|
159663
159816
|
return [];
|
|
159664
159817
|
}
|
|
@@ -159684,6 +159837,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159684
159837
|
if (signal) {
|
|
159685
159838
|
signal.addEventListener("abort", onOuterAbort, { once: true });
|
|
159686
159839
|
}
|
|
159840
|
+
const inputTypeForRequest = this.resolveInputTypeForPurpose(purpose);
|
|
159687
159841
|
const response = await fetch(`${this.endpoint}/embeddings`, {
|
|
159688
159842
|
method: "POST",
|
|
159689
159843
|
headers: {
|
|
@@ -159693,7 +159847,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159693
159847
|
body: JSON.stringify({
|
|
159694
159848
|
model: this.model,
|
|
159695
159849
|
input: texts,
|
|
159696
|
-
...
|
|
159850
|
+
...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
|
|
159697
159851
|
...this.truncate ? { truncate: this.truncate } : {}
|
|
159698
159852
|
}),
|
|
159699
159853
|
redirect: "error",
|
|
@@ -159910,6 +160064,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159910
160064
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159911
160065
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159912
160066
|
}
|
|
160067
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
160068
|
+
const row = db.prepare(`SELECT
|
|
160069
|
+
COUNT(*) AS total,
|
|
160070
|
+
SUM(CASE WHEN EXISTS (
|
|
160071
|
+
SELECT 1 FROM memory_embeddings e
|
|
160072
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
160073
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
160074
|
+
FROM memories m
|
|
160075
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
160076
|
+
return {
|
|
160077
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
160078
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
160079
|
+
};
|
|
160080
|
+
}
|
|
159913
160081
|
|
|
159914
160082
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159915
160083
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -160016,6 +160184,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
160016
160184
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
160017
160185
|
}
|
|
160018
160186
|
|
|
160187
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160188
|
+
init_logger();
|
|
160189
|
+
var insertStatements = new WeakMap;
|
|
160190
|
+
var existingShasStatements = new WeakMap;
|
|
160191
|
+
var projectCountStatements = new WeakMap;
|
|
160192
|
+
var evictStatements = new WeakMap;
|
|
160193
|
+
var evictOverflowStatements = new WeakMap;
|
|
160194
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160195
|
+
function getInsertStatement(db) {
|
|
160196
|
+
let stmt = insertStatements.get(db);
|
|
160197
|
+
if (!stmt) {
|
|
160198
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160199
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160200
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160201
|
+
project_path = excluded.project_path,
|
|
160202
|
+
short_sha = excluded.short_sha,
|
|
160203
|
+
message = excluded.message,
|
|
160204
|
+
author = excluded.author,
|
|
160205
|
+
committed_at = excluded.committed_at,
|
|
160206
|
+
indexed_at = excluded.indexed_at
|
|
160207
|
+
WHERE git_commits.message != excluded.message`);
|
|
160208
|
+
insertStatements.set(db, stmt);
|
|
160209
|
+
}
|
|
160210
|
+
return stmt;
|
|
160211
|
+
}
|
|
160212
|
+
function getExistingShasStatement(db) {
|
|
160213
|
+
let stmt = existingShasStatements.get(db);
|
|
160214
|
+
if (!stmt) {
|
|
160215
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160216
|
+
existingShasStatements.set(db, stmt);
|
|
160217
|
+
}
|
|
160218
|
+
return stmt;
|
|
160219
|
+
}
|
|
160220
|
+
function getProjectCountStatement(db) {
|
|
160221
|
+
let stmt = projectCountStatements.get(db);
|
|
160222
|
+
if (!stmt) {
|
|
160223
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160224
|
+
projectCountStatements.set(db, stmt);
|
|
160225
|
+
}
|
|
160226
|
+
return stmt;
|
|
160227
|
+
}
|
|
160228
|
+
function getLatestCommitTimeStatement(db) {
|
|
160229
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160230
|
+
if (!stmt) {
|
|
160231
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160232
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160233
|
+
}
|
|
160234
|
+
return stmt;
|
|
160235
|
+
}
|
|
160236
|
+
function getEvictOverflowStatement(db) {
|
|
160237
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160238
|
+
if (!stmt) {
|
|
160239
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160240
|
+
WHERE rowid IN (
|
|
160241
|
+
SELECT rowid FROM git_commits
|
|
160242
|
+
WHERE project_path = ?
|
|
160243
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160244
|
+
LIMIT -1 OFFSET ?
|
|
160245
|
+
)`);
|
|
160246
|
+
evictOverflowStatements.set(db, stmt);
|
|
160247
|
+
}
|
|
160248
|
+
return stmt;
|
|
160249
|
+
}
|
|
160250
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160251
|
+
if (commits.length === 0)
|
|
160252
|
+
return { inserted: 0, updated: 0 };
|
|
160253
|
+
const existing = new Set;
|
|
160254
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160255
|
+
existing.add(row.sha);
|
|
160256
|
+
}
|
|
160257
|
+
let inserted = 0;
|
|
160258
|
+
let updated = 0;
|
|
160259
|
+
const now = Date.now();
|
|
160260
|
+
const insertStmt = getInsertStatement(db);
|
|
160261
|
+
db.transaction(() => {
|
|
160262
|
+
for (const commit of commits) {
|
|
160263
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160264
|
+
if (result.changes > 0) {
|
|
160265
|
+
if (existing.has(commit.sha)) {
|
|
160266
|
+
updated++;
|
|
160267
|
+
} else {
|
|
160268
|
+
inserted++;
|
|
160269
|
+
existing.add(commit.sha);
|
|
160270
|
+
}
|
|
160271
|
+
}
|
|
160272
|
+
}
|
|
160273
|
+
})();
|
|
160274
|
+
return { inserted, updated };
|
|
160275
|
+
}
|
|
160276
|
+
function getCommitCount(db, projectPath) {
|
|
160277
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160278
|
+
return row?.count ?? 0;
|
|
160279
|
+
}
|
|
160280
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160281
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160282
|
+
return row?.latest ?? null;
|
|
160283
|
+
}
|
|
160284
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160285
|
+
if (maxCommits <= 0)
|
|
160286
|
+
return 0;
|
|
160287
|
+
const count = getCommitCount(db, projectPath);
|
|
160288
|
+
if (count <= maxCommits)
|
|
160289
|
+
return 0;
|
|
160290
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160291
|
+
const after = getCommitCount(db, projectPath);
|
|
160292
|
+
const evicted = Math.max(0, count - after);
|
|
160293
|
+
if (evicted > 0) {
|
|
160294
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160295
|
+
}
|
|
160296
|
+
return evicted;
|
|
160297
|
+
}
|
|
160298
|
+
|
|
160019
160299
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
160020
160300
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
160021
160301
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160248,8 +160528,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160248
160528
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160249
160529
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160250
160530
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160251
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160531
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160252
160532
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160533
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160534
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160535
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160536
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160253
160537
|
var projectRegistrations = new Map;
|
|
160254
160538
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160255
160539
|
var globalRegistrationGeneration = 0;
|
|
@@ -160267,6 +160551,7 @@ function resolveEmbeddingConfig(config2) {
|
|
|
160267
160551
|
if (config2.provider === "openai-compatible") {
|
|
160268
160552
|
const apiKey = config2.api_key?.trim();
|
|
160269
160553
|
const inputType = config2.input_type?.trim();
|
|
160554
|
+
const queryInputType = config2.query_input_type?.trim();
|
|
160270
160555
|
const truncate = config2.truncate?.trim();
|
|
160271
160556
|
return {
|
|
160272
160557
|
provider: "openai-compatible",
|
|
@@ -160274,6 +160559,7 @@ function resolveEmbeddingConfig(config2) {
|
|
|
160274
160559
|
endpoint: config2.endpoint.trim(),
|
|
160275
160560
|
...apiKey ? { api_key: apiKey } : {},
|
|
160276
160561
|
...inputType ? { input_type: inputType } : {},
|
|
160562
|
+
...queryInputType ? { query_input_type: queryInputType } : {},
|
|
160277
160563
|
...truncate ? { truncate } : {},
|
|
160278
160564
|
...config2.max_input_tokens ? {
|
|
160279
160565
|
max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config2.max_input_tokens)
|
|
@@ -160295,6 +160581,7 @@ function createProvider(config2) {
|
|
|
160295
160581
|
model: config2.model,
|
|
160296
160582
|
apiKey: config2.api_key,
|
|
160297
160583
|
inputType: config2.input_type,
|
|
160584
|
+
queryInputType: config2.query_input_type,
|
|
160298
160585
|
truncate: config2.truncate,
|
|
160299
160586
|
maxInputTokens: config2.max_input_tokens
|
|
160300
160587
|
});
|
|
@@ -160349,7 +160636,9 @@ function snapshotFor(registration) {
|
|
|
160349
160636
|
enabled,
|
|
160350
160637
|
gitCommitEnabled,
|
|
160351
160638
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160352
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160639
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160640
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160641
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160353
160642
|
};
|
|
160354
160643
|
}
|
|
160355
160644
|
function disposeProvider(provider) {
|
|
@@ -160475,7 +160764,7 @@ function getOrCreateProjectProvider(registration) {
|
|
|
160475
160764
|
registration.provider = provider;
|
|
160476
160765
|
return provider;
|
|
160477
160766
|
}
|
|
160478
|
-
async function embedTextForProject(projectIdentity, text, signal) {
|
|
160767
|
+
async function embedTextForProject(projectIdentity, text, signal, purpose = "passage") {
|
|
160479
160768
|
const registration = projectRegistrations.get(projectIdentity);
|
|
160480
160769
|
if (!registration)
|
|
160481
160770
|
return null;
|
|
@@ -160484,7 +160773,7 @@ async function embedTextForProject(projectIdentity, text, signal) {
|
|
|
160484
160773
|
const provider = getOrCreateProjectProvider(registration);
|
|
160485
160774
|
if (!provider)
|
|
160486
160775
|
return null;
|
|
160487
|
-
const vector = await provider.embed(text, signal);
|
|
160776
|
+
const vector = await provider.embed(text, signal, purpose);
|
|
160488
160777
|
if (!vector)
|
|
160489
160778
|
return null;
|
|
160490
160779
|
const current = projectRegistrations.get(projectIdentity);
|
|
@@ -160493,7 +160782,7 @@ async function embedTextForProject(projectIdentity, text, signal) {
|
|
|
160493
160782
|
}
|
|
160494
160783
|
return { vector, modelId, generation };
|
|
160495
160784
|
}
|
|
160496
|
-
async function embedBatchForProject(projectIdentity, texts, signal) {
|
|
160785
|
+
async function embedBatchForProject(projectIdentity, texts, signal, purpose = "passage") {
|
|
160497
160786
|
if (texts.length === 0) {
|
|
160498
160787
|
const registration2 = projectRegistrations.get(projectIdentity);
|
|
160499
160788
|
if (!registration2 || registration2.observationMode)
|
|
@@ -160509,7 +160798,7 @@ async function embedBatchForProject(projectIdentity, texts, signal) {
|
|
|
160509
160798
|
const provider = getOrCreateProjectProvider(registration);
|
|
160510
160799
|
if (!provider)
|
|
160511
160800
|
return null;
|
|
160512
|
-
const vectors = await provider.embedBatch(texts, signal);
|
|
160801
|
+
const vectors = await provider.embedBatch(texts, signal, purpose);
|
|
160513
160802
|
const current = projectRegistrations.get(projectIdentity);
|
|
160514
160803
|
if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
|
|
160515
160804
|
return null;
|
|
@@ -160560,12 +160849,13 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160560
160849
|
}
|
|
160561
160850
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160562
160851
|
const noWork = [];
|
|
160852
|
+
const failed = [];
|
|
160563
160853
|
if (candidates.length === 0)
|
|
160564
|
-
return { embedded: 0, noWork };
|
|
160854
|
+
return { embedded: 0, noWork, failed };
|
|
160565
160855
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160566
160856
|
const prepared = [];
|
|
160567
160857
|
for (const candidate of candidates) {
|
|
160568
|
-
const canonicalText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage);
|
|
160858
|
+
const canonicalText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage) || buildCompartmentSummaryFallbackText(db, candidate.id);
|
|
160569
160859
|
if (canonicalText.length === 0) {
|
|
160570
160860
|
noWork.push(candidate.id);
|
|
160571
160861
|
continue;
|
|
@@ -160578,7 +160868,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160578
160868
|
prepared.push({ candidate, windows });
|
|
160579
160869
|
}
|
|
160580
160870
|
if (prepared.length === 0)
|
|
160581
|
-
return { embedded: 0, noWork };
|
|
160871
|
+
return { embedded: 0, noWork, failed };
|
|
160582
160872
|
let embedded = 0;
|
|
160583
160873
|
let i = 0;
|
|
160584
160874
|
while (i < prepared.length) {
|
|
@@ -160595,35 +160885,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160595
160885
|
const texts = [];
|
|
160596
160886
|
for (const item of slice)
|
|
160597
160887
|
texts.push(...item.windows.map((w) => w.text));
|
|
160598
|
-
|
|
160599
|
-
|
|
160600
|
-
if (!result)
|
|
160601
|
-
continue;
|
|
160888
|
+
const persistedIds = new Set;
|
|
160889
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160602
160890
|
if (signal?.aborted)
|
|
160603
160891
|
break;
|
|
160604
|
-
let
|
|
160605
|
-
|
|
160606
|
-
|
|
160607
|
-
|
|
160608
|
-
|
|
160609
|
-
|
|
160892
|
+
let result = null;
|
|
160893
|
+
const attemptStart = Date.now();
|
|
160894
|
+
try {
|
|
160895
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160896
|
+
} catch (error51) {
|
|
160897
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160898
|
+
}
|
|
160899
|
+
if (signal?.aborted)
|
|
160900
|
+
break;
|
|
160901
|
+
if (result) {
|
|
160902
|
+
let offset = 0;
|
|
160903
|
+
for (const item of slice) {
|
|
160904
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160905
|
+
offset += item.windows.length;
|
|
160906
|
+
if (persistedIds.has(item.candidate.id))
|
|
160907
|
+
continue;
|
|
160908
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160909
|
+
continue;
|
|
160910
|
+
}
|
|
160911
|
+
const rows = item.windows.map((window, index) => ({
|
|
160912
|
+
compartmentId: item.candidate.id,
|
|
160913
|
+
sessionId: item.candidate.sessionId,
|
|
160914
|
+
projectPath: projectIdentity,
|
|
160915
|
+
window,
|
|
160916
|
+
modelId,
|
|
160917
|
+
vector: vectors[index]
|
|
160918
|
+
}));
|
|
160919
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160920
|
+
persistedIds.add(item.candidate.id);
|
|
160610
160921
|
}
|
|
160611
|
-
const rows = item.windows.map((window, index) => ({
|
|
160612
|
-
compartmentId: item.candidate.id,
|
|
160613
|
-
sessionId: item.candidate.sessionId,
|
|
160614
|
-
projectPath: projectIdentity,
|
|
160615
|
-
window,
|
|
160616
|
-
modelId,
|
|
160617
|
-
vector: vectors[index]
|
|
160618
|
-
}));
|
|
160619
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160620
|
-
embedded += 1;
|
|
160621
160922
|
}
|
|
160622
|
-
|
|
160623
|
-
|
|
160923
|
+
if (persistedIds.size === slice.length)
|
|
160924
|
+
break;
|
|
160925
|
+
if (persistedIds.size > 0)
|
|
160926
|
+
break;
|
|
160927
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160928
|
+
break;
|
|
160929
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160930
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160931
|
+
}
|
|
160932
|
+
}
|
|
160933
|
+
embedded += persistedIds.size;
|
|
160934
|
+
if (!signal?.aborted) {
|
|
160935
|
+
for (const item of slice) {
|
|
160936
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160937
|
+
failed.push(item.candidate.id);
|
|
160938
|
+
}
|
|
160624
160939
|
}
|
|
160625
160940
|
}
|
|
160626
|
-
return { embedded, noWork };
|
|
160941
|
+
return { embedded, noWork, failed };
|
|
160627
160942
|
}
|
|
160628
160943
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160629
160944
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160646,9 +160961,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160646
160961
|
renewal.unref?.();
|
|
160647
160962
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160648
160963
|
const skipIds = [];
|
|
160964
|
+
const failedIds = [];
|
|
160649
160965
|
let embedded = 0;
|
|
160650
160966
|
let aborted2 = false;
|
|
160651
|
-
let
|
|
160967
|
+
let providerDown = false;
|
|
160968
|
+
let consecutiveFailedBatches = 0;
|
|
160652
160969
|
try {
|
|
160653
160970
|
options?.onProgress?.({ embedded, total });
|
|
160654
160971
|
for (;; ) {
|
|
@@ -160656,15 +160973,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160656
160973
|
aborted2 = true;
|
|
160657
160974
|
break;
|
|
160658
160975
|
}
|
|
160659
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160976
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160660
160977
|
if (candidates.length === 0)
|
|
160661
160978
|
break;
|
|
160662
|
-
const {
|
|
160979
|
+
const {
|
|
160980
|
+
embedded: n,
|
|
160981
|
+
noWork,
|
|
160982
|
+
failed
|
|
160983
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160663
160984
|
for (const id of noWork)
|
|
160664
160985
|
skipIds.push(id);
|
|
160986
|
+
for (const id of failed)
|
|
160987
|
+
failedIds.push(id);
|
|
160665
160988
|
if (n === 0 && noWork.length === 0) {
|
|
160666
|
-
|
|
160667
|
-
|
|
160989
|
+
consecutiveFailedBatches += 1;
|
|
160990
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160991
|
+
providerDown = true;
|
|
160992
|
+
break;
|
|
160993
|
+
}
|
|
160994
|
+
} else {
|
|
160995
|
+
consecutiveFailedBatches = 0;
|
|
160668
160996
|
}
|
|
160669
160997
|
embedded += n;
|
|
160670
160998
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160672,16 +161000,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160672
161000
|
}
|
|
160673
161001
|
} finally {
|
|
160674
161002
|
clearInterval(renewal);
|
|
160675
|
-
|
|
161003
|
+
try {
|
|
161004
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
161005
|
+
} catch (error51) {
|
|
161006
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
161007
|
+
}
|
|
160676
161008
|
}
|
|
160677
161009
|
if (aborted2)
|
|
160678
|
-
return { status: "aborted", embedded, total };
|
|
160679
|
-
if (
|
|
161010
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
161011
|
+
if (providerDown || failedIds.length > 0) {
|
|
160680
161012
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160681
|
-
if (remaining > 0)
|
|
160682
|
-
return { status: "stalled", embedded, total, remaining };
|
|
161013
|
+
if (remaining > 0) {
|
|
161014
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
161015
|
+
}
|
|
161016
|
+
}
|
|
161017
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
161018
|
+
}
|
|
161019
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
161020
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
161021
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
161022
|
+
return {
|
|
161023
|
+
enabled: false,
|
|
161024
|
+
model: snapshot?.model ?? "off",
|
|
161025
|
+
provider: snapshot?.provider ?? "off",
|
|
161026
|
+
session: { embedded: 0, total: 0 },
|
|
161027
|
+
memories: { embedded: 0, total: 0 },
|
|
161028
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
161029
|
+
};
|
|
160683
161030
|
}
|
|
160684
|
-
|
|
161031
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
161032
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
161033
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
161034
|
+
const commits = gitEnabled ? {
|
|
161035
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
161036
|
+
total: getCommitCount(db, projectIdentity),
|
|
161037
|
+
gitEnabled: true
|
|
161038
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
161039
|
+
return {
|
|
161040
|
+
enabled: true,
|
|
161041
|
+
model: snapshot.model,
|
|
161042
|
+
provider: snapshot.provider,
|
|
161043
|
+
session,
|
|
161044
|
+
memories,
|
|
161045
|
+
commits
|
|
161046
|
+
};
|
|
160685
161047
|
}
|
|
160686
161048
|
|
|
160687
161049
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160702,6 +161064,7 @@ function createProvider2(config2) {
|
|
|
160702
161064
|
model: config2.model,
|
|
160703
161065
|
apiKey: config2.api_key,
|
|
160704
161066
|
inputType: config2.input_type,
|
|
161067
|
+
queryInputType: config2.query_input_type,
|
|
160705
161068
|
truncate: config2.truncate,
|
|
160706
161069
|
maxInputTokens: config2.max_input_tokens
|
|
160707
161070
|
});
|
|
@@ -160910,6 +161273,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160910
161273
|
"conversation_tokens",
|
|
160911
161274
|
"tool_call_tokens",
|
|
160912
161275
|
"cleared_reasoning_through_tag",
|
|
161276
|
+
"tool_reclaim_watermark",
|
|
160913
161277
|
"last_todo_state",
|
|
160914
161278
|
"cached_m0_bytes",
|
|
160915
161279
|
"cached_m1_bytes",
|
|
@@ -160936,6 +161300,8 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160936
161300
|
"recovery_no_eligible_head_count",
|
|
160937
161301
|
"force_emergency_bypass_window_start",
|
|
160938
161302
|
"force_emergency_bypass_used",
|
|
161303
|
+
"emergency_drain_active",
|
|
161304
|
+
"historian_drain_failure_at",
|
|
160939
161305
|
"upgrade_reminded_at",
|
|
160940
161306
|
"pi_stable_id_scheme"
|
|
160941
161307
|
];
|
|
@@ -160958,6 +161324,7 @@ var META_COLUMNS = {
|
|
|
160958
161324
|
conversationTokens: "conversation_tokens",
|
|
160959
161325
|
toolCallTokens: "tool_call_tokens",
|
|
160960
161326
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161327
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160961
161328
|
lastTodoState: "last_todo_state",
|
|
160962
161329
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160963
161330
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -160984,6 +161351,8 @@ var META_COLUMNS = {
|
|
|
160984
161351
|
recoveryNoEligibleHeadCount: "recovery_no_eligible_head_count",
|
|
160985
161352
|
forceEmergencyBypassWindowStart: "force_emergency_bypass_window_start",
|
|
160986
161353
|
forceEmergencyBypassUsed: "force_emergency_bypass_used",
|
|
161354
|
+
emergencyDrainActive: "emergency_drain_active",
|
|
161355
|
+
historianDrainFailureAt: "historian_drain_failure_at",
|
|
160987
161356
|
upgradeRemindedAt: "upgrade_reminded_at",
|
|
160988
161357
|
piStableIdScheme: "pi_stable_id_scheme"
|
|
160989
161358
|
};
|
|
@@ -161026,7 +161395,7 @@ function isSessionMetaRow(row) {
|
|
|
161026
161395
|
if (row === null || typeof row !== "object")
|
|
161027
161396
|
return false;
|
|
161028
161397
|
const r = row;
|
|
161029
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161398
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161030
161399
|
}
|
|
161031
161400
|
function getDefaultSessionMeta(sessionId) {
|
|
161032
161401
|
return {
|
|
@@ -161049,6 +161418,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161049
161418
|
conversationTokens: 0,
|
|
161050
161419
|
toolCallTokens: 0,
|
|
161051
161420
|
clearedReasoningThroughTag: 0,
|
|
161421
|
+
toolReclaimWatermark: 0,
|
|
161052
161422
|
lastTodoState: "",
|
|
161053
161423
|
cachedM0Bytes: null,
|
|
161054
161424
|
cachedM1Bytes: null,
|
|
@@ -161112,6 +161482,7 @@ function toSessionMeta(row) {
|
|
|
161112
161482
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161113
161483
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161114
161484
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161485
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161115
161486
|
lastTodoState: lastTodoStateRaw,
|
|
161116
161487
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161117
161488
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161538,6 +161909,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161538
161909
|
var encoder = new TextEncoder;
|
|
161539
161910
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161540
161911
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161912
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161913
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161541
161914
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161542
161915
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161543
161916
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161550,6 +161923,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161550
161923
|
function stripMalformedTagNotationGlobally(value) {
|
|
161551
161924
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161552
161925
|
}
|
|
161926
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161927
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161928
|
+
}
|
|
161553
161929
|
function stripTagSectionCharacters(value) {
|
|
161554
161930
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161555
161931
|
}
|
|
@@ -161557,6 +161933,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161557
161933
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161558
161934
|
text = stripCompleteTagPairsGlobally(text);
|
|
161559
161935
|
text = stripMalformedTagNotationGlobally(text);
|
|
161936
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161560
161937
|
text = stripTagSectionCharacters(text);
|
|
161561
161938
|
return text.trim();
|
|
161562
161939
|
}
|
|
@@ -161569,6 +161946,7 @@ function stripTagPrefix(value) {
|
|
|
161569
161946
|
const prev = stripped;
|
|
161570
161947
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161571
161948
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161949
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161572
161950
|
if (stripped === prev)
|
|
161573
161951
|
break;
|
|
161574
161952
|
}
|
|
@@ -161676,6 +162054,24 @@ class ToolMutationBatch {
|
|
|
161676
162054
|
}
|
|
161677
162055
|
|
|
161678
162056
|
// ../plugin/src/hooks/magic-context/read-session-chunk.ts
|
|
162057
|
+
var BLOCK_TOKEN_MEMO_MAX = 2048;
|
|
162058
|
+
var blockTokenMemo = new Map;
|
|
162059
|
+
function estimateBlockTokens(blockText) {
|
|
162060
|
+
const cached2 = blockTokenMemo.get(blockText);
|
|
162061
|
+
if (cached2 !== undefined) {
|
|
162062
|
+
blockTokenMemo.delete(blockText);
|
|
162063
|
+
blockTokenMemo.set(blockText, cached2);
|
|
162064
|
+
return cached2;
|
|
162065
|
+
}
|
|
162066
|
+
const count = estimateTokens(blockText);
|
|
162067
|
+
if (blockTokenMemo.size >= BLOCK_TOKEN_MEMO_MAX) {
|
|
162068
|
+
const oldest = blockTokenMemo.keys().next().value;
|
|
162069
|
+
if (oldest !== undefined)
|
|
162070
|
+
blockTokenMemo.delete(oldest);
|
|
162071
|
+
}
|
|
162072
|
+
blockTokenMemo.set(blockText, count);
|
|
162073
|
+
return count;
|
|
162074
|
+
}
|
|
161679
162075
|
var activeRawMessageCache = null;
|
|
161680
162076
|
var activeAbsoluteCountCache = null;
|
|
161681
162077
|
var sessionProviders = new Map;
|
|
@@ -161858,7 +162254,7 @@ function readSessionChunk(sessionId, tokenBudget, offset = 1, eligibleEndOrdinal
|
|
|
161858
162254
|
if (!currentBlock)
|
|
161859
162255
|
return true;
|
|
161860
162256
|
const blockText = formatBlock(currentBlock);
|
|
161861
|
-
const blockTokens =
|
|
162257
|
+
const blockTokens = estimateBlockTokens(blockText);
|
|
161862
162258
|
if (totalTokens + blockTokens > tokenBudget && totalTokens > 0) {
|
|
161863
162259
|
return false;
|
|
161864
162260
|
}
|
|
@@ -161984,9 +162380,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161984
162380
|
|
|
161985
162381
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161986
162382
|
|
|
161987
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162383
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162384
|
+
|
|
162385
|
+
Two recovery modes for finer detail:
|
|
162386
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162387
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161988
162388
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161989
162389
|
|
|
162390
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162391
|
+
function isRecord2(value) {
|
|
162392
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162393
|
+
}
|
|
162394
|
+
function roleLabel(role) {
|
|
162395
|
+
if (role === "assistant")
|
|
162396
|
+
return "A (assistant)";
|
|
162397
|
+
if (role === "user")
|
|
162398
|
+
return "U (user)";
|
|
162399
|
+
return role;
|
|
162400
|
+
}
|
|
162401
|
+
function truncate(value, max) {
|
|
162402
|
+
const t = value.trim();
|
|
162403
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162404
|
+
}
|
|
162405
|
+
function keyArg(input) {
|
|
162406
|
+
if (!input)
|
|
162407
|
+
return "";
|
|
162408
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162409
|
+
const v = input[k];
|
|
162410
|
+
if (typeof v === "string" && v.length > 0)
|
|
162411
|
+
return truncate(v, 60);
|
|
162412
|
+
}
|
|
162413
|
+
if (typeof input.description === "string")
|
|
162414
|
+
return truncate(input.description, 60);
|
|
162415
|
+
return "";
|
|
162416
|
+
}
|
|
162417
|
+
function asToolPart(part) {
|
|
162418
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162419
|
+
if (type === "tool") {
|
|
162420
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162421
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162422
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162423
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162424
|
+
return {
|
|
162425
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162426
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162427
|
+
title,
|
|
162428
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162429
|
+
output
|
|
162430
|
+
};
|
|
162431
|
+
}
|
|
162432
|
+
if (type === "tool_use") {
|
|
162433
|
+
return {
|
|
162434
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162435
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162436
|
+
title: null,
|
|
162437
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162438
|
+
output: null
|
|
162439
|
+
};
|
|
162440
|
+
}
|
|
162441
|
+
if (type === "tool_result") {
|
|
162442
|
+
const content = part.content;
|
|
162443
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162444
|
+
return {
|
|
162445
|
+
name: "tool_result",
|
|
162446
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162447
|
+
title: null,
|
|
162448
|
+
input: null,
|
|
162449
|
+
output
|
|
162450
|
+
};
|
|
162451
|
+
}
|
|
162452
|
+
return null;
|
|
162453
|
+
}
|
|
162454
|
+
function textOf(part) {
|
|
162455
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162456
|
+
return part.text;
|
|
162457
|
+
return null;
|
|
162458
|
+
}
|
|
162459
|
+
function reasoningOf(part) {
|
|
162460
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162461
|
+
return part.text;
|
|
162462
|
+
}
|
|
162463
|
+
return null;
|
|
162464
|
+
}
|
|
162465
|
+
function renderPartPreview(part) {
|
|
162466
|
+
if (!isRecord2(part))
|
|
162467
|
+
return null;
|
|
162468
|
+
const text = textOf(part);
|
|
162469
|
+
if (text !== null) {
|
|
162470
|
+
const t = truncate(text, 200);
|
|
162471
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162472
|
+
}
|
|
162473
|
+
const tool = asToolPart(part);
|
|
162474
|
+
if (tool) {
|
|
162475
|
+
const arg = keyArg(tool.input);
|
|
162476
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162477
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162478
|
+
}
|
|
162479
|
+
const reasoning = reasoningOf(part);
|
|
162480
|
+
if (reasoning !== null)
|
|
162481
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162482
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162483
|
+
if (type === "file")
|
|
162484
|
+
return " • [file]";
|
|
162485
|
+
if (type === "step-start" || type === "step-finish")
|
|
162486
|
+
return null;
|
|
162487
|
+
return ` • [${type}]`;
|
|
162488
|
+
}
|
|
162489
|
+
function renderPartFull(part) {
|
|
162490
|
+
if (!isRecord2(part))
|
|
162491
|
+
return null;
|
|
162492
|
+
const text = textOf(part);
|
|
162493
|
+
if (text !== null) {
|
|
162494
|
+
return text.trim().length > 0 ? ` [text]
|
|
162495
|
+
${text}` : null;
|
|
162496
|
+
}
|
|
162497
|
+
const tool = asToolPart(part);
|
|
162498
|
+
if (tool) {
|
|
162499
|
+
const lines = [];
|
|
162500
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162501
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162502
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162503
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162504
|
+
}
|
|
162505
|
+
if (tool.input)
|
|
162506
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162507
|
+
if (tool.output !== null)
|
|
162508
|
+
lines.push(` output:
|
|
162509
|
+
${tool.output}`);
|
|
162510
|
+
return lines.join(`
|
|
162511
|
+
`);
|
|
162512
|
+
}
|
|
162513
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162514
|
+
if (type === "file") {
|
|
162515
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162516
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162517
|
+
}
|
|
162518
|
+
return null;
|
|
162519
|
+
}
|
|
162520
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162521
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162522
|
+
if (!msg) {
|
|
162523
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162524
|
+
}
|
|
162525
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162526
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162527
|
+
if (rendered.length === 0) {
|
|
162528
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162529
|
+
} else {
|
|
162530
|
+
lines.push(...rendered);
|
|
162531
|
+
}
|
|
162532
|
+
return lines.join(`
|
|
162533
|
+
`);
|
|
162534
|
+
}
|
|
162535
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162536
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162537
|
+
const out = [];
|
|
162538
|
+
let usedTokens = 0;
|
|
162539
|
+
let lastOrdinal = start - 1;
|
|
162540
|
+
let truncated = false;
|
|
162541
|
+
for (const msg of messages) {
|
|
162542
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162543
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162544
|
+
const block = partLines.length > 0 ? `${header}
|
|
162545
|
+
${partLines.join(`
|
|
162546
|
+
`)}` : header;
|
|
162547
|
+
const blockTokens = estimateTokens(block);
|
|
162548
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162549
|
+
truncated = true;
|
|
162550
|
+
break;
|
|
162551
|
+
}
|
|
162552
|
+
out.push(block);
|
|
162553
|
+
usedTokens += blockTokens;
|
|
162554
|
+
lastOrdinal = msg.ordinal;
|
|
162555
|
+
}
|
|
162556
|
+
return { text: out.join(`
|
|
162557
|
+
|
|
162558
|
+
`), lastOrdinal, truncated };
|
|
162559
|
+
}
|
|
162560
|
+
|
|
161990
162561
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161991
162562
|
var exports_memory = {};
|
|
161992
162563
|
__export(exports_memory, {
|
|
@@ -166343,12 +166914,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166343
166914
|
|
|
166344
166915
|
// src/tools/ctx-expand.ts
|
|
166345
166916
|
var ParamsSchema = exports_typebox.Object({
|
|
166346
|
-
start: exports_typebox.Number({
|
|
166917
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166347
166918
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166348
|
-
}),
|
|
166349
|
-
end: exports_typebox.Number({
|
|
166919
|
+
})),
|
|
166920
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166350
166921
|
description: "End message ordinal (from compartment end attribute)"
|
|
166351
|
-
})
|
|
166922
|
+
})),
|
|
166923
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166924
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166925
|
+
})),
|
|
166926
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166927
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166928
|
+
}))
|
|
166352
166929
|
});
|
|
166353
166930
|
function ok(text) {
|
|
166354
166931
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166367,22 +166944,41 @@ function createCtxExpandTool(deps) {
|
|
|
166367
166944
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166368
166945
|
parameters: ParamsSchema,
|
|
166369
166946
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166370
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166371
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166372
|
-
}
|
|
166373
166947
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166374
166948
|
if (!sessionId) {
|
|
166375
166949
|
return err("Error: no active Pi session.");
|
|
166376
166950
|
}
|
|
166377
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166378
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166379
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166380
|
-
}
|
|
166381
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166382
166951
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166383
166952
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166384
166953
|
});
|
|
166385
166954
|
try {
|
|
166955
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166956
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166957
|
+
}
|
|
166958
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166959
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166960
|
+
}
|
|
166961
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166962
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166963
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166964
|
+
}
|
|
166965
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166966
|
+
if (params.verbose === true) {
|
|
166967
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166968
|
+
if (!v.text) {
|
|
166969
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166970
|
+
}
|
|
166971
|
+
const out = [
|
|
166972
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166973
|
+
"",
|
|
166974
|
+
v.text
|
|
166975
|
+
];
|
|
166976
|
+
if (v.truncated) {
|
|
166977
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166978
|
+
}
|
|
166979
|
+
return ok(out.join(`
|
|
166980
|
+
`));
|
|
166981
|
+
}
|
|
166386
166982
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166387
166983
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166388
166984
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -167608,7 +168204,9 @@ var DEFAULT_PROTECTED_TAIL_META = {
|
|
|
167608
168204
|
protectedTailDrainTokens: 0,
|
|
167609
168205
|
recoveryNoEligibleHeadCount: 0,
|
|
167610
168206
|
forceEmergencyBypassWindowStart: 0,
|
|
167611
|
-
forceEmergencyBypassUsed: 0
|
|
168207
|
+
forceEmergencyBypassUsed: 0,
|
|
168208
|
+
emergencyDrainActive: 0,
|
|
168209
|
+
historianDrainFailureAt: 0
|
|
167612
168210
|
};
|
|
167613
168211
|
function toProtectedTailMeta(row) {
|
|
167614
168212
|
if (row === null || typeof row !== "object")
|
|
@@ -167622,7 +168220,9 @@ function toProtectedTailMeta(row) {
|
|
|
167622
168220
|
protectedTailDrainTokens: numberOr(r.protected_tail_drain_tokens, 0),
|
|
167623
168221
|
recoveryNoEligibleHeadCount: numberOr(r.recovery_no_eligible_head_count, 0),
|
|
167624
168222
|
forceEmergencyBypassWindowStart: numberOr(r.force_emergency_bypass_window_start, 0),
|
|
167625
|
-
forceEmergencyBypassUsed: numberOr(r.force_emergency_bypass_used, 0)
|
|
168223
|
+
forceEmergencyBypassUsed: numberOr(r.force_emergency_bypass_used, 0),
|
|
168224
|
+
emergencyDrainActive: numberOr(r.emergency_drain_active, 0),
|
|
168225
|
+
historianDrainFailureAt: numberOr(r.historian_drain_failure_at, 0)
|
|
167626
168226
|
};
|
|
167627
168227
|
}
|
|
167628
168228
|
function loadProtectedTailMeta(db, sessionId) {
|
|
@@ -167630,7 +168230,7 @@ function loadProtectedTailMeta(db, sessionId) {
|
|
|
167630
168230
|
const row = db.prepare(`SELECT prior_boundary_ordinal, protected_tail_policy_version,
|
|
167631
168231
|
protected_tail_drain_window_started_at, protected_tail_drain_tokens,
|
|
167632
168232
|
recovery_no_eligible_head_count, force_emergency_bypass_window_start,
|
|
167633
|
-
force_emergency_bypass_used
|
|
168233
|
+
force_emergency_bypass_used, emergency_drain_active, historian_drain_failure_at
|
|
167634
168234
|
FROM session_meta WHERE session_id = ?`).get(sessionId);
|
|
167635
168235
|
return toProtectedTailMeta(row);
|
|
167636
168236
|
}
|
|
@@ -167674,6 +168274,17 @@ function protectedTailWindowBudget(usagePercentage, usable, perRunCap) {
|
|
|
167674
168274
|
return Math.min(750000, Math.max(3 * perRunCap, Math.round(0.35 * usable)));
|
|
167675
168275
|
return Math.min(500000, Math.max(perRunCap, Math.round(0.2 * usable)));
|
|
167676
168276
|
}
|
|
168277
|
+
var EMERGENCY_DRAIN_ENTER_PERCENTAGE = 95;
|
|
168278
|
+
var EMERGENCY_DRAIN_EXIT_MARGIN = 10;
|
|
168279
|
+
var EMERGENCY_DRAIN_FALLBACK_EXIT_PERCENTAGE = 55;
|
|
168280
|
+
var EMERGENCY_DRAIN_FAILURE_BACKOFF_MS = 60000;
|
|
168281
|
+
var EMERGENCY_DRAIN_MAX_LATCH_MS = 30 * 60 * 1000;
|
|
168282
|
+
function emergencyDrainExitThreshold(executeThresholdPercentage) {
|
|
168283
|
+
if (!Number.isFinite(executeThresholdPercentage) || executeThresholdPercentage <= 0) {
|
|
168284
|
+
return EMERGENCY_DRAIN_FALLBACK_EXIT_PERCENTAGE;
|
|
168285
|
+
}
|
|
168286
|
+
return Math.max(0, executeThresholdPercentage - EMERGENCY_DRAIN_EXIT_MARGIN);
|
|
168287
|
+
}
|
|
167677
168288
|
function reserveProtectedTailDrainTokens(args) {
|
|
167678
168289
|
const now = args.now ?? Date.now();
|
|
167679
168290
|
const requested = Math.max(0, Math.floor(args.trueRawTokens));
|
|
@@ -167692,18 +168303,30 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167692
168303
|
let meta3 = loadProtectedTailMeta(args.db, args.sessionId);
|
|
167693
168304
|
if (now - meta3.protectedTailDrainWindowStartedAt > DRAIN_WINDOW_MS) {
|
|
167694
168305
|
args.db.prepare(`UPDATE session_meta
|
|
167695
|
-
SET protected_tail_drain_window_started_at = ?, protected_tail_drain_tokens = 0
|
|
167696
|
-
|
|
167697
|
-
WHERE session_id = ?`).run(now, now, args.sessionId);
|
|
168306
|
+
SET protected_tail_drain_window_started_at = ?, protected_tail_drain_tokens = 0
|
|
168307
|
+
WHERE session_id = ?`).run(now, args.sessionId);
|
|
167698
168308
|
meta3 = loadProtectedTailMeta(args.db, args.sessionId);
|
|
167699
168309
|
}
|
|
168310
|
+
const exitThreshold = emergencyDrainExitThreshold(args.executeThresholdPercentage);
|
|
168311
|
+
let latchActiveSince = meta3.emergencyDrainActive;
|
|
168312
|
+
if (args.usagePercentage >= EMERGENCY_DRAIN_ENTER_PERCENTAGE) {
|
|
168313
|
+
if (latchActiveSince <= 0)
|
|
168314
|
+
latchActiveSince = now;
|
|
168315
|
+
} else if (latchActiveSince > 0) {
|
|
168316
|
+
const expired = now - latchActiveSince > EMERGENCY_DRAIN_MAX_LATCH_MS;
|
|
168317
|
+
if (args.usagePercentage < exitThreshold || expired)
|
|
168318
|
+
latchActiveSince = 0;
|
|
168319
|
+
}
|
|
168320
|
+
if (latchActiveSince !== meta3.emergencyDrainActive) {
|
|
168321
|
+
args.db.prepare("UPDATE session_meta SET emergency_drain_active = ? WHERE session_id = ?").run(latchActiveSince, args.sessionId);
|
|
168322
|
+
}
|
|
168323
|
+
const latchActive = latchActiveSince > 0;
|
|
167700
168324
|
const budget = protectedTailWindowBudget(args.usagePercentage, args.usable, args.perRunCap);
|
|
167701
168325
|
const remaining = Math.max(0, budget - meta3.protectedTailDrainTokens);
|
|
167702
168326
|
let reserved = Math.min(requested, args.perRunCap, remaining);
|
|
167703
168327
|
let bypass = false;
|
|
167704
|
-
const
|
|
167705
|
-
|
|
167706
|
-
if (reserved <= 0 && args.usagePercentage >= 95 && bypassUsed === 0) {
|
|
168328
|
+
const inFailureBackoff = meta3.historianDrainFailureAt > 0 && now - meta3.historianDrainFailureAt < EMERGENCY_DRAIN_FAILURE_BACKOFF_MS;
|
|
168329
|
+
if (reserved <= 0 && latchActive && !inFailureBackoff) {
|
|
167707
168330
|
reserved = Math.min(requested, args.perRunCap);
|
|
167708
168331
|
bypass = true;
|
|
167709
168332
|
}
|
|
@@ -167711,10 +168334,8 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167711
168334
|
return;
|
|
167712
168335
|
args.db.prepare(`UPDATE session_meta
|
|
167713
168336
|
SET protected_tail_drain_window_started_at = CASE WHEN protected_tail_drain_window_started_at = 0 THEN ? ELSE protected_tail_drain_window_started_at END,
|
|
167714
|
-
protected_tail_drain_tokens = COALESCE(protected_tail_drain_tokens, 0) +
|
|
167715
|
-
|
|
167716
|
-
force_emergency_bypass_used = CASE WHEN ? THEN 1 ELSE force_emergency_bypass_used END
|
|
167717
|
-
WHERE session_id = ?`).run(now, reserved, bypass ? 1 : 0, now, bypass ? 1 : 0, args.sessionId);
|
|
168337
|
+
protected_tail_drain_tokens = COALESCE(protected_tail_drain_tokens, 0) + ?
|
|
168338
|
+
WHERE session_id = ?`).run(now, reserved, args.sessionId);
|
|
167718
168339
|
result = {
|
|
167719
168340
|
ok: true,
|
|
167720
168341
|
reservedTokens: reserved,
|
|
@@ -167724,6 +168345,25 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167724
168345
|
})();
|
|
167725
168346
|
return result;
|
|
167726
168347
|
}
|
|
168348
|
+
function clearEmergencyDrainLatch(db, sessionId) {
|
|
168349
|
+
db.transaction(() => {
|
|
168350
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168351
|
+
db.prepare("UPDATE session_meta SET emergency_drain_active = 0 WHERE session_id = ?").run(sessionId);
|
|
168352
|
+
})();
|
|
168353
|
+
}
|
|
168354
|
+
function recordHistorianDrainFailure(db, sessionId, now) {
|
|
168355
|
+
const ts = now ?? Date.now();
|
|
168356
|
+
db.transaction(() => {
|
|
168357
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168358
|
+
db.prepare("UPDATE session_meta SET historian_drain_failure_at = ? WHERE session_id = ?").run(ts, sessionId);
|
|
168359
|
+
})();
|
|
168360
|
+
}
|
|
168361
|
+
function clearHistorianDrainFailure(db, sessionId) {
|
|
168362
|
+
db.transaction(() => {
|
|
168363
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168364
|
+
db.prepare("UPDATE session_meta SET historian_drain_failure_at = 0 WHERE session_id = ?").run(sessionId);
|
|
168365
|
+
})();
|
|
168366
|
+
}
|
|
167727
168367
|
function rollbackProtectedTailDrainReservation(db, reservation) {
|
|
167728
168368
|
if (!reservation || reservation.tokens <= 0)
|
|
167729
168369
|
return;
|
|
@@ -168242,6 +168882,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168242
168882
|
last_transform_error: "'' AS last_transform_error",
|
|
168243
168883
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168244
168884
|
last_todo_state: "'' AS last_todo_state",
|
|
168885
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168245
168886
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168246
168887
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168247
168888
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168319,6 +168960,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168319
168960
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168320
168961
|
})();
|
|
168321
168962
|
}
|
|
168963
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168964
|
+
if (maxTagNumber <= 0)
|
|
168965
|
+
return;
|
|
168966
|
+
db.transaction(() => {
|
|
168967
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168968
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168969
|
+
})();
|
|
168970
|
+
}
|
|
168322
168971
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168323
168972
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168324
168973
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168716,12 +169365,37 @@ function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
|
168716
169365
|
nullCount: row?.null_count ?? 0
|
|
168717
169366
|
};
|
|
168718
169367
|
}
|
|
168719
|
-
function
|
|
168720
|
-
|
|
169368
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169369
|
+
if (limit <= 0)
|
|
169370
|
+
return [];
|
|
169371
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169372
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169373
|
+
SELECT tag_number FROM tags
|
|
169374
|
+
WHERE session_id = ? AND status = 'active'
|
|
169375
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169376
|
+
)` : "";
|
|
169377
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169378
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169379
|
+
FROM tags
|
|
169380
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169381
|
+
ORDER BY tag_number ASC, id ASC
|
|
169382
|
+
LIMIT ?`).all(...params);
|
|
169383
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169384
|
+
tagNumber: row.tag_number,
|
|
169385
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169386
|
+
}));
|
|
169387
|
+
}
|
|
169388
|
+
function getTriggerTagTokenUpperBound(db, sessionId, floor = 0) {
|
|
169389
|
+
const sql = floor > 0 ? `SELECT
|
|
169390
|
+
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
169391
|
+
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
169392
|
+
FROM tags
|
|
169393
|
+
WHERE session_id = ? AND status IN ('active', 'dropped') AND tag_number >= ?` : `SELECT
|
|
168721
169394
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
168722
169395
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
168723
169396
|
FROM tags
|
|
168724
|
-
WHERE session_id = ? AND status IN ('active', 'dropped')
|
|
169397
|
+
WHERE session_id = ? AND status IN ('active', 'dropped')`;
|
|
169398
|
+
const row = floor > 0 ? db.prepare(sql).get(sessionId, floor) : db.prepare(sql).get(sessionId);
|
|
168725
169399
|
return { bound: row?.bound ?? 0, nullCount: row?.null_count ?? 0 };
|
|
168726
169400
|
}
|
|
168727
169401
|
function updateTagInputByteSize(db, sessionId, tagNumber, newInputByteSize) {
|
|
@@ -168748,10 +169422,12 @@ function getUpdateTagInputTokenCountStatement(db) {
|
|
|
168748
169422
|
function updateTagTokenCount(db, sessionId, tagNumber, newTokenCount) {
|
|
168749
169423
|
getUpdateTagTokenCountStatement(db).run(newTokenCount, sessionId, tagNumber);
|
|
168750
169424
|
}
|
|
168751
|
-
function getAllStatusTagTokenTotalsFlat(db, sessionId) {
|
|
168752
|
-
const rows = db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
168753
|
-
|
|
168754
|
-
|
|
169425
|
+
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0) {
|
|
169426
|
+
const rows = floor > 0 ? db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
169427
|
+
FROM tags
|
|
169428
|
+
WHERE session_id = ? AND tag_number >= ?`).all(sessionId, floor) : db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
169429
|
+
FROM tags
|
|
169430
|
+
WHERE session_id = ?`).all(sessionId);
|
|
168755
169431
|
const totals = new Map;
|
|
168756
169432
|
const nullMessageIds = new Set;
|
|
168757
169433
|
for (const row of rows) {
|
|
@@ -168870,6 +169546,52 @@ function getTagNumberByMessageId(db, sessionId, messageId) {
|
|
|
168870
169546
|
const row = getTagNumberByMessageIdStatement(db).get(sessionId, messageId);
|
|
168871
169547
|
return isTagNumberRow(row) ? row.tag_number : null;
|
|
168872
169548
|
}
|
|
169549
|
+
var getMinMessageTagNumberForRawIdStatements = new WeakMap;
|
|
169550
|
+
function isMinTagNumberRow(row) {
|
|
169551
|
+
return row !== null && typeof row === "object" && "m" in row;
|
|
169552
|
+
}
|
|
169553
|
+
function getMinMessageTagNumberForRawId(db, sessionId, rawId) {
|
|
169554
|
+
if (rawId.includes(":"))
|
|
169555
|
+
return null;
|
|
169556
|
+
let stmt = getMinMessageTagNumberForRawIdStatements.get(db);
|
|
169557
|
+
if (!stmt) {
|
|
169558
|
+
stmt = db.prepare("SELECT MIN(tag_number) AS m FROM tags WHERE session_id = ? AND message_id >= ? AND message_id < ?");
|
|
169559
|
+
getMinMessageTagNumberForRawIdStatements.set(db, stmt);
|
|
169560
|
+
}
|
|
169561
|
+
const row = stmt.get(sessionId, `${rawId}:`, `${rawId};`);
|
|
169562
|
+
return isMinTagNumberRow(row) && typeof row.m === "number" ? row.m : null;
|
|
169563
|
+
}
|
|
169564
|
+
var TAGGER_FLOOR_SCAN_MESSAGES = 8;
|
|
169565
|
+
var TAGGER_FLOOR_MAX_PROBES = 64;
|
|
169566
|
+
var TAGGER_FLOOR_SAFETY_MARGIN = 256;
|
|
169567
|
+
var TAGGER_FLOOR_PER_SKIP_MARGIN = 64;
|
|
169568
|
+
function deriveTagLoadFloor(db, sessionId, rawIds) {
|
|
169569
|
+
let min = Number.POSITIVE_INFINITY;
|
|
169570
|
+
let probes = 0;
|
|
169571
|
+
let hits = 0;
|
|
169572
|
+
let skippedBeforeFirstHit = 0;
|
|
169573
|
+
for (const rawId of rawIds) {
|
|
169574
|
+
if (typeof rawId !== "string" || rawId.length === 0)
|
|
169575
|
+
continue;
|
|
169576
|
+
if (probes >= TAGGER_FLOOR_MAX_PROBES)
|
|
169577
|
+
break;
|
|
169578
|
+
probes++;
|
|
169579
|
+
const m = getMinMessageTagNumberForRawId(db, sessionId, rawId);
|
|
169580
|
+
if (m === null) {
|
|
169581
|
+
if (hits === 0)
|
|
169582
|
+
skippedBeforeFirstHit++;
|
|
169583
|
+
continue;
|
|
169584
|
+
}
|
|
169585
|
+
if (m < min)
|
|
169586
|
+
min = m;
|
|
169587
|
+
if (++hits >= TAGGER_FLOOR_SCAN_MESSAGES)
|
|
169588
|
+
break;
|
|
169589
|
+
}
|
|
169590
|
+
if (!Number.isFinite(min))
|
|
169591
|
+
return 0;
|
|
169592
|
+
const margin = TAGGER_FLOOR_SAFETY_MARGIN + skippedBeforeFirstHit * TAGGER_FLOOR_PER_SKIP_MARGIN;
|
|
169593
|
+
return Math.max(0, min - margin);
|
|
169594
|
+
}
|
|
168873
169595
|
var TAG_SELECT_COLUMNS = "id, message_id, type, status, drop_mode, tool_name, input_byte_size, byte_size, reasoning_byte_size, session_id, tag_number, caveman_depth, tool_owner_message_id";
|
|
168874
169596
|
function getTagsBySession(db, sessionId) {
|
|
168875
169597
|
const rows = db.prepare(`SELECT ${TAG_SELECT_COLUMNS} FROM tags WHERE session_id = ? ORDER BY tag_number ASC, id ASC`).all(sessionId).filter(isTagRow);
|
|
@@ -169665,15 +170387,16 @@ function parseInteger(str) {
|
|
|
169665
170387
|
}
|
|
169666
170388
|
|
|
169667
170389
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169668
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169669
|
-
Use §N§ identifiers visible in conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170390
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
169670
170391
|
|
|
169671
|
-
|
|
169672
|
-
-
|
|
169673
|
-
-
|
|
169674
|
-
-
|
|
169675
|
-
|
|
169676
|
-
|
|
170392
|
+
How it works:
|
|
170393
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170394
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170395
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
170396
|
+
|
|
170397
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170398
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170399
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169677
170400
|
|
|
169678
170401
|
// src/tools/ctx-reduce.ts
|
|
169679
170402
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169858,120 +170581,6 @@ ${body}` : subject;
|
|
|
169858
170581
|
}
|
|
169859
170582
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169860
170583
|
init_logger();
|
|
169861
|
-
|
|
169862
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169863
|
-
init_logger();
|
|
169864
|
-
var insertStatements = new WeakMap;
|
|
169865
|
-
var existingShasStatements = new WeakMap;
|
|
169866
|
-
var projectCountStatements = new WeakMap;
|
|
169867
|
-
var evictStatements = new WeakMap;
|
|
169868
|
-
var evictOverflowStatements = new WeakMap;
|
|
169869
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169870
|
-
function getInsertStatement(db) {
|
|
169871
|
-
let stmt = insertStatements.get(db);
|
|
169872
|
-
if (!stmt) {
|
|
169873
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169874
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169875
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169876
|
-
project_path = excluded.project_path,
|
|
169877
|
-
short_sha = excluded.short_sha,
|
|
169878
|
-
message = excluded.message,
|
|
169879
|
-
author = excluded.author,
|
|
169880
|
-
committed_at = excluded.committed_at,
|
|
169881
|
-
indexed_at = excluded.indexed_at
|
|
169882
|
-
WHERE git_commits.message != excluded.message`);
|
|
169883
|
-
insertStatements.set(db, stmt);
|
|
169884
|
-
}
|
|
169885
|
-
return stmt;
|
|
169886
|
-
}
|
|
169887
|
-
function getExistingShasStatement(db) {
|
|
169888
|
-
let stmt = existingShasStatements.get(db);
|
|
169889
|
-
if (!stmt) {
|
|
169890
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169891
|
-
existingShasStatements.set(db, stmt);
|
|
169892
|
-
}
|
|
169893
|
-
return stmt;
|
|
169894
|
-
}
|
|
169895
|
-
function getProjectCountStatement(db) {
|
|
169896
|
-
let stmt = projectCountStatements.get(db);
|
|
169897
|
-
if (!stmt) {
|
|
169898
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169899
|
-
projectCountStatements.set(db, stmt);
|
|
169900
|
-
}
|
|
169901
|
-
return stmt;
|
|
169902
|
-
}
|
|
169903
|
-
function getLatestCommitTimeStatement(db) {
|
|
169904
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169905
|
-
if (!stmt) {
|
|
169906
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169907
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169908
|
-
}
|
|
169909
|
-
return stmt;
|
|
169910
|
-
}
|
|
169911
|
-
function getEvictOverflowStatement(db) {
|
|
169912
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169913
|
-
if (!stmt) {
|
|
169914
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169915
|
-
WHERE rowid IN (
|
|
169916
|
-
SELECT rowid FROM git_commits
|
|
169917
|
-
WHERE project_path = ?
|
|
169918
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169919
|
-
LIMIT -1 OFFSET ?
|
|
169920
|
-
)`);
|
|
169921
|
-
evictOverflowStatements.set(db, stmt);
|
|
169922
|
-
}
|
|
169923
|
-
return stmt;
|
|
169924
|
-
}
|
|
169925
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169926
|
-
if (commits.length === 0)
|
|
169927
|
-
return { inserted: 0, updated: 0 };
|
|
169928
|
-
const existing = new Set;
|
|
169929
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169930
|
-
existing.add(row.sha);
|
|
169931
|
-
}
|
|
169932
|
-
let inserted = 0;
|
|
169933
|
-
let updated = 0;
|
|
169934
|
-
const now = Date.now();
|
|
169935
|
-
const insertStmt = getInsertStatement(db);
|
|
169936
|
-
db.transaction(() => {
|
|
169937
|
-
for (const commit of commits) {
|
|
169938
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169939
|
-
if (result.changes > 0) {
|
|
169940
|
-
if (existing.has(commit.sha)) {
|
|
169941
|
-
updated++;
|
|
169942
|
-
} else {
|
|
169943
|
-
inserted++;
|
|
169944
|
-
existing.add(commit.sha);
|
|
169945
|
-
}
|
|
169946
|
-
}
|
|
169947
|
-
}
|
|
169948
|
-
})();
|
|
169949
|
-
return { inserted, updated };
|
|
169950
|
-
}
|
|
169951
|
-
function getCommitCount(db, projectPath) {
|
|
169952
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169953
|
-
return row?.count ?? 0;
|
|
169954
|
-
}
|
|
169955
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169956
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169957
|
-
return row?.latest ?? null;
|
|
169958
|
-
}
|
|
169959
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169960
|
-
if (maxCommits <= 0)
|
|
169961
|
-
return 0;
|
|
169962
|
-
const count = getCommitCount(db, projectPath);
|
|
169963
|
-
if (count <= maxCommits)
|
|
169964
|
-
return 0;
|
|
169965
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169966
|
-
const after = getCommitCount(db, projectPath);
|
|
169967
|
-
const evicted = Math.max(0, count - after);
|
|
169968
|
-
if (evicted > 0) {
|
|
169969
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169970
|
-
}
|
|
169971
|
-
return evicted;
|
|
169972
|
-
}
|
|
169973
|
-
|
|
169974
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169975
170584
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169976
170585
|
var EMBED_BATCH_SIZE = 16;
|
|
169977
170586
|
var EMBED_MAX_PER_SWEEP = 500;
|
|
@@ -171721,7 +172330,7 @@ function createCtxSearchTool(deps) {
|
|
|
171721
172330
|
memoryEnabled,
|
|
171722
172331
|
embeddingEnabled,
|
|
171723
172332
|
embedQuery: async (text, signal) => {
|
|
171724
|
-
const result = await embedTextForProject(projectIdentity, text, signal);
|
|
172333
|
+
const result = await embedTextForProject(projectIdentity, text, signal, "query");
|
|
171725
172334
|
return result?.vector ?? null;
|
|
171726
172335
|
},
|
|
171727
172336
|
isEmbeddingRuntimeEnabled: () => embeddingEnabled === true,
|