@cortexkit/pi-magic-context 0.24.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1567 -511
- package/dist/subagent-entry.js +821 -212
- package/package.json +1 -1
package/dist/subagent-entry.js
CHANGED
|
@@ -8010,6 +8010,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8010
8010
|
}
|
|
8011
8011
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8012
8012
|
}
|
|
8013
|
+
prepare(sql) {
|
|
8014
|
+
const stmt = super.prepare(sql);
|
|
8015
|
+
for (const method of ["run", "get", "all"]) {
|
|
8016
|
+
const original = stmt[method].bind(stmt);
|
|
8017
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8018
|
+
}
|
|
8019
|
+
return stmt;
|
|
8020
|
+
}
|
|
8013
8021
|
transaction(fn) {
|
|
8014
8022
|
const self = this;
|
|
8015
8023
|
const wrapped = function(...args) {
|
|
@@ -9574,6 +9582,41 @@ var MIGRATIONS = [
|
|
|
9574
9582
|
`);
|
|
9575
9583
|
}
|
|
9576
9584
|
}
|
|
9585
|
+
},
|
|
9586
|
+
{
|
|
9587
|
+
version: 37,
|
|
9588
|
+
description: "emergency drain catch-up latch + historian drain failure backoff",
|
|
9589
|
+
up: (db) => {
|
|
9590
|
+
const hasSessionMeta = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='session_meta'").get();
|
|
9591
|
+
if (!hasSessionMeta)
|
|
9592
|
+
return;
|
|
9593
|
+
ensureColumn(db, "session_meta", "emergency_drain_active", "INTEGER NOT NULL DEFAULT 0");
|
|
9594
|
+
ensureColumn(db, "session_meta", "historian_drain_failure_at", "INTEGER NOT NULL DEFAULT 0");
|
|
9595
|
+
}
|
|
9596
|
+
},
|
|
9597
|
+
{
|
|
9598
|
+
version: 38,
|
|
9599
|
+
description: "durable transform decisions for cache-event cause attribution",
|
|
9600
|
+
up: (db) => {
|
|
9601
|
+
db.exec(`
|
|
9602
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
9603
|
+
session_id TEXT NOT NULL,
|
|
9604
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
9605
|
+
message_id TEXT NOT NULL,
|
|
9606
|
+
ts_ms INTEGER NOT NULL,
|
|
9607
|
+
decision TEXT NOT NULL,
|
|
9608
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
9609
|
+
materialize_reason TEXT,
|
|
9610
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
9611
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
9612
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
9613
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
9614
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
9615
|
+
);
|
|
9616
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
9617
|
+
ON transform_decisions(session_id, harness);
|
|
9618
|
+
`);
|
|
9619
|
+
}
|
|
9577
9620
|
}
|
|
9578
9621
|
];
|
|
9579
9622
|
var LATEST_MIGRATION_VERSION = MIGRATIONS.reduce((max, m) => Math.max(max, m.version), 0);
|
|
@@ -142550,8 +142593,9 @@ function backfillToolOwnersInChunks(db, result) {
|
|
|
142550
142593
|
var databases = new Map;
|
|
142551
142594
|
var persistenceByDatabase = new WeakMap;
|
|
142552
142595
|
var persistenceErrorByDatabase = new WeakMap;
|
|
142596
|
+
var pathByDatabase = new WeakMap;
|
|
142553
142597
|
var lastSchemaFenceRejection = null;
|
|
142554
|
-
var LATEST_SUPPORTED_VERSION =
|
|
142598
|
+
var LATEST_SUPPORTED_VERSION = 38;
|
|
142555
142599
|
function resolveDatabasePath(dbPathOverride) {
|
|
142556
142600
|
if (dbPathOverride) {
|
|
142557
142601
|
return { dbDir: dirname2(dbPathOverride), dbPath: dbPathOverride };
|
|
@@ -142564,6 +142608,9 @@ function resolveDatabasePath(dbPathOverride) {
|
|
|
142564
142608
|
const dbDir = getMagicContextStorageDir();
|
|
142565
142609
|
return { dbDir, dbPath: join4(dbDir, "context.db") };
|
|
142566
142610
|
}
|
|
142611
|
+
function getDatabasePath(db) {
|
|
142612
|
+
return pathByDatabase.get(db) ?? null;
|
|
142613
|
+
}
|
|
142567
142614
|
function migrateLegacyStorageIfNeeded(targetDbPath, targetDbDir) {
|
|
142568
142615
|
if (existsSync3(targetDbPath))
|
|
142569
142616
|
return;
|
|
@@ -143078,6 +143125,8 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143078
143125
|
recovery_no_eligible_head_count INTEGER NOT NULL DEFAULT 0,
|
|
143079
143126
|
force_emergency_bypass_window_start INTEGER NOT NULL DEFAULT 0,
|
|
143080
143127
|
force_emergency_bypass_used INTEGER NOT NULL DEFAULT 0,
|
|
143128
|
+
emergency_drain_active INTEGER NOT NULL DEFAULT 0,
|
|
143129
|
+
historian_drain_failure_at INTEGER NOT NULL DEFAULT 0,
|
|
143081
143130
|
cached_m0_materialized_at INTEGER,
|
|
143082
143131
|
cached_m0_session_facts_version INTEGER,
|
|
143083
143132
|
cached_m0_upgrade_state TEXT,
|
|
@@ -143141,6 +143190,23 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143141
143190
|
CREATE INDEX IF NOT EXISTS idx_historian_runs_status
|
|
143142
143191
|
ON historian_runs(status, created_at DESC);
|
|
143143
143192
|
|
|
143193
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
143194
|
+
session_id TEXT NOT NULL,
|
|
143195
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
143196
|
+
message_id TEXT NOT NULL,
|
|
143197
|
+
ts_ms INTEGER NOT NULL,
|
|
143198
|
+
decision TEXT NOT NULL,
|
|
143199
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
143200
|
+
materialize_reason TEXT,
|
|
143201
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
143202
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143203
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
143204
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143205
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
143206
|
+
);
|
|
143207
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
143208
|
+
ON transform_decisions(session_id, harness);
|
|
143209
|
+
|
|
143144
143210
|
CREATE INDEX IF NOT EXISTS idx_tags_session_tag_number ON tags(session_id, tag_number);
|
|
143145
143211
|
CREATE INDEX IF NOT EXISTS idx_tags_session_message_id ON tags(session_id, message_id);
|
|
143146
143212
|
CREATE INDEX IF NOT EXISTS idx_pending_ops_session ON pending_ops(session_id);
|
|
@@ -143218,6 +143284,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143218
143284
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143219
143285
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143220
143286
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143287
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143221
143288
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143222
143289
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143223
143290
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -143291,6 +143358,8 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143291
143358
|
ensureColumn(db, "session_meta", "recovery_no_eligible_head_count", "INTEGER NOT NULL DEFAULT 0");
|
|
143292
143359
|
ensureColumn(db, "session_meta", "force_emergency_bypass_window_start", "INTEGER NOT NULL DEFAULT 0");
|
|
143293
143360
|
ensureColumn(db, "session_meta", "force_emergency_bypass_used", "INTEGER NOT NULL DEFAULT 0");
|
|
143361
|
+
ensureColumn(db, "session_meta", "emergency_drain_active", "INTEGER NOT NULL DEFAULT 0");
|
|
143362
|
+
ensureColumn(db, "session_meta", "historian_drain_failure_at", "INTEGER NOT NULL DEFAULT 0");
|
|
143294
143363
|
ensureColumn(db, "session_meta", "cached_m0_materialized_at", "INTEGER");
|
|
143295
143364
|
ensureColumn(db, "session_meta", "cached_m0_session_facts_version", "INTEGER");
|
|
143296
143365
|
ensureColumn(db, "session_meta", "cached_m0_upgrade_state", "TEXT");
|
|
@@ -143369,6 +143438,22 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143369
143438
|
failed_at INTEGER NOT NULL,
|
|
143370
143439
|
UNIQUE(table_name, row_id)
|
|
143371
143440
|
);
|
|
143441
|
+
CREATE TABLE IF NOT EXISTS transform_decisions (
|
|
143442
|
+
session_id TEXT NOT NULL,
|
|
143443
|
+
harness TEXT NOT NULL DEFAULT 'opencode',
|
|
143444
|
+
message_id TEXT NOT NULL,
|
|
143445
|
+
ts_ms INTEGER NOT NULL,
|
|
143446
|
+
decision TEXT NOT NULL,
|
|
143447
|
+
materialized INTEGER NOT NULL DEFAULT 0,
|
|
143448
|
+
materialize_reason TEXT,
|
|
143449
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
143450
|
+
dropped_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143451
|
+
dropped_count INTEGER NOT NULL DEFAULT 0,
|
|
143452
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
143453
|
+
PRIMARY KEY (session_id, harness, message_id)
|
|
143454
|
+
);
|
|
143455
|
+
CREATE INDEX IF NOT EXISTS idx_transform_decisions_session_harness
|
|
143456
|
+
ON transform_decisions(session_id, harness);
|
|
143372
143457
|
`);
|
|
143373
143458
|
ensureColumn(db, "tags", "harness", "TEXT NOT NULL DEFAULT 'opencode'");
|
|
143374
143459
|
ensureColumn(db, "pending_ops", "harness", "TEXT NOT NULL DEFAULT 'opencode'");
|
|
@@ -143455,7 +143540,9 @@ function healNullIntegerColumns(db) {
|
|
|
143455
143540
|
["protected_tail_drain_tokens", 0],
|
|
143456
143541
|
["recovery_no_eligible_head_count", 0],
|
|
143457
143542
|
["force_emergency_bypass_window_start", 0],
|
|
143458
|
-
["force_emergency_bypass_used", 0]
|
|
143543
|
+
["force_emergency_bypass_used", 0],
|
|
143544
|
+
["emergency_drain_active", 0],
|
|
143545
|
+
["historian_drain_failure_at", 0]
|
|
143459
143546
|
];
|
|
143460
143547
|
for (const [column, fallback] of columns) {
|
|
143461
143548
|
try {
|
|
@@ -143531,6 +143618,7 @@ function openDatabase(dbPathOrOptions) {
|
|
|
143531
143618
|
setDatabase(db);
|
|
143532
143619
|
loadToolDefinitionMeasurements(db);
|
|
143533
143620
|
databases.set(dbPath, db);
|
|
143621
|
+
pathByDatabase.set(db, dbPath);
|
|
143534
143622
|
persistenceByDatabase.set(db, true);
|
|
143535
143623
|
persistenceErrorByDatabase.delete(db);
|
|
143536
143624
|
return db;
|
|
@@ -158128,14 +158216,16 @@ var SidekickConfigSchema = AgentOverrideConfigSchema.extend({
|
|
|
158128
158216
|
}).optional();
|
|
158129
158217
|
var HistorianConfigSchema = AgentOverrideConfigSchema.extend({
|
|
158130
158218
|
two_pass: exports_external.boolean().default(false).describe("Run a second editor pass over historian output to clean low-signal U: lines and cross-compartment duplicates. Adds ~1 extra API call and ~1.3x cost per historian run. Useful for models without extended thinking support. (default: false)"),
|
|
158131
|
-
thinking_level: PiThinkingLevelSchema.describe("Pi only: explicit thinking level passed as --thinking <level> to Pi historian subagent invocations. Required when using reasoning models (e.g. github-copilot/gpt-5.4) because Pi's default thinking-level resolution can pick a value the provider rejects. OpenCode users set variant instead. Valid: off | minimal | low | medium | high | xhigh")
|
|
158219
|
+
thinking_level: PiThinkingLevelSchema.describe("Pi only: explicit thinking level passed as --thinking <level> to Pi historian subagent invocations. Required when using reasoning models (e.g. github-copilot/gpt-5.4) because Pi's default thinking-level resolution can pick a value the provider rejects. OpenCode users set variant instead. Valid: off | minimal | low | medium | high | xhigh"),
|
|
158220
|
+
disallowed_tools: exports_external.array(exports_external.enum(["*", "read", "aft_outline", "aft_zoom", "aft_search"])).default([]).describe(`OpenCode only. Tools to REMOVE from the historian's default allow-list [read, aft_outline, aft_zoom, aft_search]. Applies to both historian and historian-editor agents. Use ["*"] to strip all tool definitions from the model request — this prevents weak instruction-following models (e.g. mistral-small-latest) from entering tool-calling loops. Individual tool names remove just that tool. Note: a user-supplied historian.permission override can re-allow a tool that disallowed_tools removed — disallowed_tools sets the baseline, permission overrides take precedence. (default: [])`)
|
|
158132
158221
|
}).optional();
|
|
158133
158222
|
var BaseEmbeddingConfigSchema = exports_external.object({
|
|
158134
158223
|
provider: exports_external.enum(["local", "openai-compatible", "off"]).default("local").describe("Embedding provider. 'local' uses Xenova/all-MiniLM-L6-v2, 'openai-compatible' requires endpoint and model, 'off' disables embeddings."),
|
|
158135
158224
|
model: exports_external.string().optional().describe("Embedding model name. Required for openai-compatible, ignored for local."),
|
|
158136
158225
|
endpoint: exports_external.string().optional().describe("API endpoint URL. Required when provider is openai-compatible."),
|
|
158137
158226
|
api_key: exports_external.string().optional().describe("API key for remote embedding provider (optional)"),
|
|
158138
|
-
input_type: exports_external.string().optional().describe("
|
|
158227
|
+
input_type: exports_external.string().optional().describe("Default input_type for stored/indexed (passage) embeddings in the request body. Required by some openai-compatible providers (e.g. NVIDIA NIM). Omitted from the request when unset."),
|
|
158228
|
+
query_input_type: exports_external.string().optional().describe("Optional input_type for query (search) embeddings on asymmetric models (e.g. NVIDIA NIM 'query'). When unset, query embeddings use embedding.input_type. Passage/stored content always uses embedding.input_type."),
|
|
158139
158229
|
truncate: exports_external.string().optional().describe("Optional truncate mode sent in the embedding request body (e.g. NVIDIA NIM accepts 'NONE' | 'START' | 'END'). Omitted from the request when unset."),
|
|
158140
158230
|
max_input_tokens: exports_external.number().int().positive().optional().describe("Optional maximum input tokens for chunk embeddings. Defaults conservatively to 512 when omitted.")
|
|
158141
158231
|
}).superRefine((data, ctx) => {
|
|
@@ -158165,6 +158255,7 @@ var EmbeddingConfigSchema = BaseEmbeddingConfigSchema.transform((data) => {
|
|
|
158165
158255
|
if (data.provider === "openai-compatible") {
|
|
158166
158256
|
const apiKey = data.api_key?.trim();
|
|
158167
158257
|
const inputType = data.input_type?.trim();
|
|
158258
|
+
const queryInputType = data.query_input_type?.trim();
|
|
158168
158259
|
const truncate = data.truncate?.trim();
|
|
158169
158260
|
return {
|
|
158170
158261
|
provider: "openai-compatible",
|
|
@@ -158172,6 +158263,7 @@ var EmbeddingConfigSchema = BaseEmbeddingConfigSchema.transform((data) => {
|
|
|
158172
158263
|
endpoint: data.endpoint?.trim() ?? "",
|
|
158173
158264
|
...apiKey ? { api_key: apiKey } : {},
|
|
158174
158265
|
...inputType ? { input_type: inputType } : {},
|
|
158266
|
+
...queryInputType ? { query_input_type: queryInputType } : {},
|
|
158175
158267
|
...truncate ? { truncate } : {},
|
|
158176
158268
|
...data.max_input_tokens ? { max_input_tokens: data.max_input_tokens } : {}
|
|
158177
158269
|
};
|
|
@@ -158892,6 +158984,16 @@ function buildCanonicalChunkTextFromFts(db, sessionId, startOrdinal, endOrdinal)
|
|
|
158892
158984
|
return lines.join(`
|
|
158893
158985
|
`);
|
|
158894
158986
|
}
|
|
158987
|
+
function buildCompartmentSummaryFallbackText(db, compartmentId) {
|
|
158988
|
+
const row = db.prepare("SELECT title, p1, content FROM compartments WHERE id = ?").get(compartmentId);
|
|
158989
|
+
if (!row)
|
|
158990
|
+
return "";
|
|
158991
|
+
const title = typeof row.title === "string" ? row.title.trim() : "";
|
|
158992
|
+
const p1 = typeof row.p1 === "string" ? row.p1.trim() : "";
|
|
158993
|
+
const body = p1.length > 0 ? p1 : typeof row.content === "string" ? row.content.trim() : "";
|
|
158994
|
+
return [title, body].filter((s) => s.length > 0).join(`
|
|
158995
|
+
`);
|
|
158996
|
+
}
|
|
158895
158997
|
function canonicalizeInMemoryChunkTextForEmbedding(chunkText, startOrdinal, endOrdinal) {
|
|
158896
158998
|
const lines = [];
|
|
158897
158999
|
for (const rawLine of chunkText.split(/\r?\n/)) {
|
|
@@ -159135,6 +159237,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
159135
159237
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
159136
159238
|
return typeof row?.n === "number" ? row.n : 0;
|
|
159137
159239
|
}
|
|
159240
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
159241
|
+
const row = db.prepare(`SELECT
|
|
159242
|
+
COUNT(*) AS total,
|
|
159243
|
+
SUM(CASE WHEN EXISTS (
|
|
159244
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
159245
|
+
WHERE e.compartment_id = c.id
|
|
159246
|
+
AND e.project_path = ?
|
|
159247
|
+
AND e.model_id = ?
|
|
159248
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
159249
|
+
FROM compartments c
|
|
159250
|
+
JOIN session_projects sp
|
|
159251
|
+
ON sp.session_id = c.session_id
|
|
159252
|
+
AND sp.harness = c.harness
|
|
159253
|
+
AND sp.project_path = ?
|
|
159254
|
+
WHERE c.session_id = ?
|
|
159255
|
+
AND c.start_message IS NOT NULL
|
|
159256
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
159257
|
+
return {
|
|
159258
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
159259
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
159260
|
+
};
|
|
159261
|
+
}
|
|
159138
159262
|
|
|
159139
159263
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
159140
159264
|
function cosineSimilarity(a, b) {
|
|
@@ -159287,6 +159411,19 @@ async function withQuietConsole(fn) {
|
|
|
159287
159411
|
console.error = origError;
|
|
159288
159412
|
}
|
|
159289
159413
|
}
|
|
159414
|
+
var nativeRuntimeMissing = false;
|
|
159415
|
+
function isNativeRuntimeMissingError(error51) {
|
|
159416
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159417
|
+
const lower = message.toLowerCase();
|
|
159418
|
+
const code = error51?.code;
|
|
159419
|
+
const name2 = error51?.name;
|
|
159420
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
159421
|
+
return true;
|
|
159422
|
+
}
|
|
159423
|
+
if (!lower.includes("onnxruntime-node"))
|
|
159424
|
+
return false;
|
|
159425
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
159426
|
+
}
|
|
159290
159427
|
function isTransientLoadError(error51) {
|
|
159291
159428
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
159292
159429
|
if (!message)
|
|
@@ -159351,6 +159488,9 @@ class LocalEmbeddingProvider {
|
|
|
159351
159488
|
if (this.pipeline) {
|
|
159352
159489
|
return true;
|
|
159353
159490
|
}
|
|
159491
|
+
if (nativeRuntimeMissing) {
|
|
159492
|
+
return false;
|
|
159493
|
+
}
|
|
159354
159494
|
if (this.initPromise) {
|
|
159355
159495
|
await this.initPromise;
|
|
159356
159496
|
return this.pipeline !== null;
|
|
@@ -159417,7 +159557,12 @@ class LocalEmbeddingProvider {
|
|
|
159417
159557
|
await releaseLock();
|
|
159418
159558
|
}
|
|
159419
159559
|
} catch (error51) {
|
|
159420
|
-
|
|
159560
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
159561
|
+
nativeRuntimeMissing = true;
|
|
159562
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @cortexkit/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
159563
|
+
} else {
|
|
159564
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
159565
|
+
}
|
|
159421
159566
|
this.pipeline = null;
|
|
159422
159567
|
} finally {
|
|
159423
159568
|
this.initPromise = null;
|
|
@@ -159443,7 +159588,7 @@ class LocalEmbeddingProvider {
|
|
|
159443
159588
|
waiter();
|
|
159444
159589
|
}
|
|
159445
159590
|
}
|
|
159446
|
-
async embed(text, signal) {
|
|
159591
|
+
async embed(text, signal, _purpose) {
|
|
159447
159592
|
if (signal?.aborted)
|
|
159448
159593
|
return null;
|
|
159449
159594
|
if (this.disposing)
|
|
@@ -159469,7 +159614,7 @@ class LocalEmbeddingProvider {
|
|
|
159469
159614
|
this.finishInFlight();
|
|
159470
159615
|
}
|
|
159471
159616
|
}
|
|
159472
|
-
async embedBatch(texts, signal) {
|
|
159617
|
+
async embedBatch(texts, signal, _purpose) {
|
|
159473
159618
|
if (texts.length === 0) {
|
|
159474
159619
|
return [];
|
|
159475
159620
|
}
|
|
@@ -159608,6 +159753,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159608
159753
|
model;
|
|
159609
159754
|
apiKey;
|
|
159610
159755
|
inputType;
|
|
159756
|
+
queryInputType;
|
|
159611
159757
|
truncate;
|
|
159612
159758
|
initialized = false;
|
|
159613
159759
|
failureTimes = [];
|
|
@@ -159620,6 +159766,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159620
159766
|
this.model = options.model?.trim() ?? "";
|
|
159621
159767
|
this.apiKey = options.apiKey?.trim() ?? "";
|
|
159622
159768
|
this.inputType = options.inputType?.trim() ?? "";
|
|
159769
|
+
this.queryInputType = options.queryInputType?.trim() ?? "";
|
|
159623
159770
|
this.truncate = options.truncate?.trim() ?? "";
|
|
159624
159771
|
this.maxInputTokens = typeof options.maxInputTokens === "number" && Number.isFinite(options.maxInputTokens) ? Math.max(1, Math.floor(options.maxInputTokens)) : 512;
|
|
159625
159772
|
this.modelId = getEmbeddingProviderIdentity({
|
|
@@ -159647,11 +159794,17 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159647
159794
|
this.initialized = true;
|
|
159648
159795
|
return true;
|
|
159649
159796
|
}
|
|
159650
|
-
|
|
159651
|
-
|
|
159797
|
+
resolveInputTypeForPurpose(purpose = "passage") {
|
|
159798
|
+
if (purpose === "query") {
|
|
159799
|
+
return this.queryInputType || this.inputType;
|
|
159800
|
+
}
|
|
159801
|
+
return this.inputType;
|
|
159802
|
+
}
|
|
159803
|
+
async embed(text, signal, purpose) {
|
|
159804
|
+
const [embedding] = await this.embedBatch([text], signal, purpose);
|
|
159652
159805
|
return embedding ?? null;
|
|
159653
159806
|
}
|
|
159654
|
-
async embedBatch(texts, signal) {
|
|
159807
|
+
async embedBatch(texts, signal, purpose) {
|
|
159655
159808
|
if (texts.length === 0) {
|
|
159656
159809
|
return [];
|
|
159657
159810
|
}
|
|
@@ -159677,6 +159830,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159677
159830
|
if (signal) {
|
|
159678
159831
|
signal.addEventListener("abort", onOuterAbort, { once: true });
|
|
159679
159832
|
}
|
|
159833
|
+
const inputTypeForRequest = this.resolveInputTypeForPurpose(purpose);
|
|
159680
159834
|
const response = await fetch(`${this.endpoint}/embeddings`, {
|
|
159681
159835
|
method: "POST",
|
|
159682
159836
|
headers: {
|
|
@@ -159686,7 +159840,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
159686
159840
|
body: JSON.stringify({
|
|
159687
159841
|
model: this.model,
|
|
159688
159842
|
input: texts,
|
|
159689
|
-
...
|
|
159843
|
+
...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
|
|
159690
159844
|
...this.truncate ? { truncate: this.truncate } : {}
|
|
159691
159845
|
}),
|
|
159692
159846
|
redirect: "error",
|
|
@@ -159903,6 +160057,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
159903
160057
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
159904
160058
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
159905
160059
|
}
|
|
160060
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
160061
|
+
const row = db.prepare(`SELECT
|
|
160062
|
+
COUNT(*) AS total,
|
|
160063
|
+
SUM(CASE WHEN EXISTS (
|
|
160064
|
+
SELECT 1 FROM memory_embeddings e
|
|
160065
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
160066
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
160067
|
+
FROM memories m
|
|
160068
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
160069
|
+
return {
|
|
160070
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
160071
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
160072
|
+
};
|
|
160073
|
+
}
|
|
159906
160074
|
|
|
159907
160075
|
// ../plugin/src/features/magic-context/project-embedding-registry.ts
|
|
159908
160076
|
import { createHash as createHash6, randomUUID } from "node:crypto";
|
|
@@ -160009,6 +160177,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
160009
160177
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
160010
160178
|
}
|
|
160011
160179
|
|
|
160180
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
160181
|
+
init_logger();
|
|
160182
|
+
var insertStatements = new WeakMap;
|
|
160183
|
+
var existingShasStatements = new WeakMap;
|
|
160184
|
+
var projectCountStatements = new WeakMap;
|
|
160185
|
+
var evictStatements = new WeakMap;
|
|
160186
|
+
var evictOverflowStatements = new WeakMap;
|
|
160187
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
160188
|
+
function getInsertStatement(db) {
|
|
160189
|
+
let stmt = insertStatements.get(db);
|
|
160190
|
+
if (!stmt) {
|
|
160191
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
160192
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
160193
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
160194
|
+
project_path = excluded.project_path,
|
|
160195
|
+
short_sha = excluded.short_sha,
|
|
160196
|
+
message = excluded.message,
|
|
160197
|
+
author = excluded.author,
|
|
160198
|
+
committed_at = excluded.committed_at,
|
|
160199
|
+
indexed_at = excluded.indexed_at
|
|
160200
|
+
WHERE git_commits.message != excluded.message`);
|
|
160201
|
+
insertStatements.set(db, stmt);
|
|
160202
|
+
}
|
|
160203
|
+
return stmt;
|
|
160204
|
+
}
|
|
160205
|
+
function getExistingShasStatement(db) {
|
|
160206
|
+
let stmt = existingShasStatements.get(db);
|
|
160207
|
+
if (!stmt) {
|
|
160208
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
160209
|
+
existingShasStatements.set(db, stmt);
|
|
160210
|
+
}
|
|
160211
|
+
return stmt;
|
|
160212
|
+
}
|
|
160213
|
+
function getProjectCountStatement(db) {
|
|
160214
|
+
let stmt = projectCountStatements.get(db);
|
|
160215
|
+
if (!stmt) {
|
|
160216
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
160217
|
+
projectCountStatements.set(db, stmt);
|
|
160218
|
+
}
|
|
160219
|
+
return stmt;
|
|
160220
|
+
}
|
|
160221
|
+
function getLatestCommitTimeStatement(db) {
|
|
160222
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
160223
|
+
if (!stmt) {
|
|
160224
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
160225
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
160226
|
+
}
|
|
160227
|
+
return stmt;
|
|
160228
|
+
}
|
|
160229
|
+
function getEvictOverflowStatement(db) {
|
|
160230
|
+
let stmt = evictOverflowStatements.get(db);
|
|
160231
|
+
if (!stmt) {
|
|
160232
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
160233
|
+
WHERE rowid IN (
|
|
160234
|
+
SELECT rowid FROM git_commits
|
|
160235
|
+
WHERE project_path = ?
|
|
160236
|
+
ORDER BY committed_at DESC, sha DESC
|
|
160237
|
+
LIMIT -1 OFFSET ?
|
|
160238
|
+
)`);
|
|
160239
|
+
evictOverflowStatements.set(db, stmt);
|
|
160240
|
+
}
|
|
160241
|
+
return stmt;
|
|
160242
|
+
}
|
|
160243
|
+
function upsertCommits(db, projectPath, commits) {
|
|
160244
|
+
if (commits.length === 0)
|
|
160245
|
+
return { inserted: 0, updated: 0 };
|
|
160246
|
+
const existing = new Set;
|
|
160247
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
160248
|
+
existing.add(row.sha);
|
|
160249
|
+
}
|
|
160250
|
+
let inserted = 0;
|
|
160251
|
+
let updated = 0;
|
|
160252
|
+
const now = Date.now();
|
|
160253
|
+
const insertStmt = getInsertStatement(db);
|
|
160254
|
+
db.transaction(() => {
|
|
160255
|
+
for (const commit of commits) {
|
|
160256
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
160257
|
+
if (result.changes > 0) {
|
|
160258
|
+
if (existing.has(commit.sha)) {
|
|
160259
|
+
updated++;
|
|
160260
|
+
} else {
|
|
160261
|
+
inserted++;
|
|
160262
|
+
existing.add(commit.sha);
|
|
160263
|
+
}
|
|
160264
|
+
}
|
|
160265
|
+
}
|
|
160266
|
+
})();
|
|
160267
|
+
return { inserted, updated };
|
|
160268
|
+
}
|
|
160269
|
+
function getCommitCount(db, projectPath) {
|
|
160270
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
160271
|
+
return row?.count ?? 0;
|
|
160272
|
+
}
|
|
160273
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
160274
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
160275
|
+
return row?.latest ?? null;
|
|
160276
|
+
}
|
|
160277
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
160278
|
+
if (maxCommits <= 0)
|
|
160279
|
+
return 0;
|
|
160280
|
+
const count = getCommitCount(db, projectPath);
|
|
160281
|
+
if (count <= maxCommits)
|
|
160282
|
+
return 0;
|
|
160283
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
160284
|
+
const after = getCommitCount(db, projectPath);
|
|
160285
|
+
const evicted = Math.max(0, count - after);
|
|
160286
|
+
if (evicted > 0) {
|
|
160287
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
160288
|
+
}
|
|
160289
|
+
return evicted;
|
|
160290
|
+
}
|
|
160291
|
+
|
|
160012
160292
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
160013
160293
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
160014
160294
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -160241,8 +160521,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
160241
160521
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
160242
160522
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
160243
160523
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
160244
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
160524
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
160245
160525
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
160526
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
160527
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
160528
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
160529
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
160246
160530
|
var projectRegistrations = new Map;
|
|
160247
160531
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
160248
160532
|
var globalRegistrationGeneration = 0;
|
|
@@ -160260,6 +160544,7 @@ function resolveEmbeddingConfig(config2) {
|
|
|
160260
160544
|
if (config2.provider === "openai-compatible") {
|
|
160261
160545
|
const apiKey = config2.api_key?.trim();
|
|
160262
160546
|
const inputType = config2.input_type?.trim();
|
|
160547
|
+
const queryInputType = config2.query_input_type?.trim();
|
|
160263
160548
|
const truncate = config2.truncate?.trim();
|
|
160264
160549
|
return {
|
|
160265
160550
|
provider: "openai-compatible",
|
|
@@ -160267,6 +160552,7 @@ function resolveEmbeddingConfig(config2) {
|
|
|
160267
160552
|
endpoint: config2.endpoint.trim(),
|
|
160268
160553
|
...apiKey ? { api_key: apiKey } : {},
|
|
160269
160554
|
...inputType ? { input_type: inputType } : {},
|
|
160555
|
+
...queryInputType ? { query_input_type: queryInputType } : {},
|
|
160270
160556
|
...truncate ? { truncate } : {},
|
|
160271
160557
|
...config2.max_input_tokens ? {
|
|
160272
160558
|
max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config2.max_input_tokens)
|
|
@@ -160288,6 +160574,7 @@ function createProvider(config2) {
|
|
|
160288
160574
|
model: config2.model,
|
|
160289
160575
|
apiKey: config2.api_key,
|
|
160290
160576
|
inputType: config2.input_type,
|
|
160577
|
+
queryInputType: config2.query_input_type,
|
|
160291
160578
|
truncate: config2.truncate,
|
|
160292
160579
|
maxInputTokens: config2.max_input_tokens
|
|
160293
160580
|
});
|
|
@@ -160342,7 +160629,9 @@ function snapshotFor(registration) {
|
|
|
160342
160629
|
enabled,
|
|
160343
160630
|
gitCommitEnabled,
|
|
160344
160631
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
160345
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
160632
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
160633
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
160634
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
160346
160635
|
};
|
|
160347
160636
|
}
|
|
160348
160637
|
function disposeProvider(provider) {
|
|
@@ -160468,7 +160757,7 @@ function getOrCreateProjectProvider(registration) {
|
|
|
160468
160757
|
registration.provider = provider;
|
|
160469
160758
|
return provider;
|
|
160470
160759
|
}
|
|
160471
|
-
async function embedTextForProject(projectIdentity, text, signal) {
|
|
160760
|
+
async function embedTextForProject(projectIdentity, text, signal, purpose = "passage") {
|
|
160472
160761
|
const registration = projectRegistrations.get(projectIdentity);
|
|
160473
160762
|
if (!registration)
|
|
160474
160763
|
return null;
|
|
@@ -160477,7 +160766,7 @@ async function embedTextForProject(projectIdentity, text, signal) {
|
|
|
160477
160766
|
const provider = getOrCreateProjectProvider(registration);
|
|
160478
160767
|
if (!provider)
|
|
160479
160768
|
return null;
|
|
160480
|
-
const vector = await provider.embed(text, signal);
|
|
160769
|
+
const vector = await provider.embed(text, signal, purpose);
|
|
160481
160770
|
if (!vector)
|
|
160482
160771
|
return null;
|
|
160483
160772
|
const current = projectRegistrations.get(projectIdentity);
|
|
@@ -160486,7 +160775,7 @@ async function embedTextForProject(projectIdentity, text, signal) {
|
|
|
160486
160775
|
}
|
|
160487
160776
|
return { vector, modelId, generation };
|
|
160488
160777
|
}
|
|
160489
|
-
async function embedBatchForProject(projectIdentity, texts, signal) {
|
|
160778
|
+
async function embedBatchForProject(projectIdentity, texts, signal, purpose = "passage") {
|
|
160490
160779
|
if (texts.length === 0) {
|
|
160491
160780
|
const registration2 = projectRegistrations.get(projectIdentity);
|
|
160492
160781
|
if (!registration2 || registration2.observationMode)
|
|
@@ -160502,7 +160791,7 @@ async function embedBatchForProject(projectIdentity, texts, signal) {
|
|
|
160502
160791
|
const provider = getOrCreateProjectProvider(registration);
|
|
160503
160792
|
if (!provider)
|
|
160504
160793
|
return null;
|
|
160505
|
-
const vectors = await provider.embedBatch(texts, signal);
|
|
160794
|
+
const vectors = await provider.embedBatch(texts, signal, purpose);
|
|
160506
160795
|
const current = projectRegistrations.get(projectIdentity);
|
|
160507
160796
|
if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
|
|
160508
160797
|
return null;
|
|
@@ -160553,12 +160842,13 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
160553
160842
|
}
|
|
160554
160843
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
160555
160844
|
const noWork = [];
|
|
160845
|
+
const failed = [];
|
|
160556
160846
|
if (candidates.length === 0)
|
|
160557
|
-
return { embedded: 0, noWork };
|
|
160847
|
+
return { embedded: 0, noWork, failed };
|
|
160558
160848
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
160559
160849
|
const prepared = [];
|
|
160560
160850
|
for (const candidate of candidates) {
|
|
160561
|
-
const canonicalText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage);
|
|
160851
|
+
const canonicalText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage) || buildCompartmentSummaryFallbackText(db, candidate.id);
|
|
160562
160852
|
if (canonicalText.length === 0) {
|
|
160563
160853
|
noWork.push(candidate.id);
|
|
160564
160854
|
continue;
|
|
@@ -160571,7 +160861,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160571
160861
|
prepared.push({ candidate, windows });
|
|
160572
160862
|
}
|
|
160573
160863
|
if (prepared.length === 0)
|
|
160574
|
-
return { embedded: 0, noWork };
|
|
160864
|
+
return { embedded: 0, noWork, failed };
|
|
160575
160865
|
let embedded = 0;
|
|
160576
160866
|
let i = 0;
|
|
160577
160867
|
while (i < prepared.length) {
|
|
@@ -160588,35 +160878,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
160588
160878
|
const texts = [];
|
|
160589
160879
|
for (const item of slice)
|
|
160590
160880
|
texts.push(...item.windows.map((w) => w.text));
|
|
160591
|
-
|
|
160592
|
-
|
|
160593
|
-
if (!result)
|
|
160594
|
-
continue;
|
|
160881
|
+
const persistedIds = new Set;
|
|
160882
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
160595
160883
|
if (signal?.aborted)
|
|
160596
160884
|
break;
|
|
160597
|
-
let
|
|
160598
|
-
|
|
160599
|
-
|
|
160600
|
-
|
|
160601
|
-
|
|
160602
|
-
|
|
160885
|
+
let result = null;
|
|
160886
|
+
const attemptStart = Date.now();
|
|
160887
|
+
try {
|
|
160888
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
160889
|
+
} catch (error51) {
|
|
160890
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
160891
|
+
}
|
|
160892
|
+
if (signal?.aborted)
|
|
160893
|
+
break;
|
|
160894
|
+
if (result) {
|
|
160895
|
+
let offset = 0;
|
|
160896
|
+
for (const item of slice) {
|
|
160897
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
160898
|
+
offset += item.windows.length;
|
|
160899
|
+
if (persistedIds.has(item.candidate.id))
|
|
160900
|
+
continue;
|
|
160901
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
160902
|
+
continue;
|
|
160903
|
+
}
|
|
160904
|
+
const rows = item.windows.map((window, index) => ({
|
|
160905
|
+
compartmentId: item.candidate.id,
|
|
160906
|
+
sessionId: item.candidate.sessionId,
|
|
160907
|
+
projectPath: projectIdentity,
|
|
160908
|
+
window,
|
|
160909
|
+
modelId,
|
|
160910
|
+
vector: vectors[index]
|
|
160911
|
+
}));
|
|
160912
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160913
|
+
persistedIds.add(item.candidate.id);
|
|
160603
160914
|
}
|
|
160604
|
-
const rows = item.windows.map((window, index) => ({
|
|
160605
|
-
compartmentId: item.candidate.id,
|
|
160606
|
-
sessionId: item.candidate.sessionId,
|
|
160607
|
-
projectPath: projectIdentity,
|
|
160608
|
-
window,
|
|
160609
|
-
modelId,
|
|
160610
|
-
vector: vectors[index]
|
|
160611
|
-
}));
|
|
160612
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
160613
|
-
embedded += 1;
|
|
160614
160915
|
}
|
|
160615
|
-
|
|
160616
|
-
|
|
160916
|
+
if (persistedIds.size === slice.length)
|
|
160917
|
+
break;
|
|
160918
|
+
if (persistedIds.size > 0)
|
|
160919
|
+
break;
|
|
160920
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
160921
|
+
break;
|
|
160922
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
160923
|
+
await new Promise((resolve3) => setTimeout(resolve3, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
160924
|
+
}
|
|
160925
|
+
}
|
|
160926
|
+
embedded += persistedIds.size;
|
|
160927
|
+
if (!signal?.aborted) {
|
|
160928
|
+
for (const item of slice) {
|
|
160929
|
+
if (!persistedIds.has(item.candidate.id))
|
|
160930
|
+
failed.push(item.candidate.id);
|
|
160931
|
+
}
|
|
160617
160932
|
}
|
|
160618
160933
|
}
|
|
160619
|
-
return { embedded, noWork };
|
|
160934
|
+
return { embedded, noWork, failed };
|
|
160620
160935
|
}
|
|
160621
160936
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
160622
160937
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -160639,9 +160954,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160639
160954
|
renewal.unref?.();
|
|
160640
160955
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
160641
160956
|
const skipIds = [];
|
|
160957
|
+
const failedIds = [];
|
|
160642
160958
|
let embedded = 0;
|
|
160643
160959
|
let aborted2 = false;
|
|
160644
|
-
let
|
|
160960
|
+
let providerDown = false;
|
|
160961
|
+
let consecutiveFailedBatches = 0;
|
|
160645
160962
|
try {
|
|
160646
160963
|
options?.onProgress?.({ embedded, total });
|
|
160647
160964
|
for (;; ) {
|
|
@@ -160649,15 +160966,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160649
160966
|
aborted2 = true;
|
|
160650
160967
|
break;
|
|
160651
160968
|
}
|
|
160652
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
160969
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
160653
160970
|
if (candidates.length === 0)
|
|
160654
160971
|
break;
|
|
160655
|
-
const {
|
|
160972
|
+
const {
|
|
160973
|
+
embedded: n,
|
|
160974
|
+
noWork,
|
|
160975
|
+
failed
|
|
160976
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
160656
160977
|
for (const id of noWork)
|
|
160657
160978
|
skipIds.push(id);
|
|
160979
|
+
for (const id of failed)
|
|
160980
|
+
failedIds.push(id);
|
|
160658
160981
|
if (n === 0 && noWork.length === 0) {
|
|
160659
|
-
|
|
160660
|
-
|
|
160982
|
+
consecutiveFailedBatches += 1;
|
|
160983
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
160984
|
+
providerDown = true;
|
|
160985
|
+
break;
|
|
160986
|
+
}
|
|
160987
|
+
} else {
|
|
160988
|
+
consecutiveFailedBatches = 0;
|
|
160661
160989
|
}
|
|
160662
160990
|
embedded += n;
|
|
160663
160991
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -160665,16 +160993,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
160665
160993
|
}
|
|
160666
160994
|
} finally {
|
|
160667
160995
|
clearInterval(renewal);
|
|
160668
|
-
|
|
160996
|
+
try {
|
|
160997
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
160998
|
+
} catch (error51) {
|
|
160999
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
161000
|
+
}
|
|
160669
161001
|
}
|
|
160670
161002
|
if (aborted2)
|
|
160671
|
-
return { status: "aborted", embedded, total };
|
|
160672
|
-
if (
|
|
161003
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
161004
|
+
if (providerDown || failedIds.length > 0) {
|
|
160673
161005
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
160674
|
-
if (remaining > 0)
|
|
160675
|
-
return { status: "stalled", embedded, total, remaining };
|
|
161006
|
+
if (remaining > 0) {
|
|
161007
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
161008
|
+
}
|
|
161009
|
+
}
|
|
161010
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
161011
|
+
}
|
|
161012
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
161013
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
161014
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
161015
|
+
return {
|
|
161016
|
+
enabled: false,
|
|
161017
|
+
model: snapshot?.model ?? "off",
|
|
161018
|
+
provider: snapshot?.provider ?? "off",
|
|
161019
|
+
session: { embedded: 0, total: 0 },
|
|
161020
|
+
memories: { embedded: 0, total: 0 },
|
|
161021
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
161022
|
+
};
|
|
160676
161023
|
}
|
|
160677
|
-
|
|
161024
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
161025
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
161026
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
161027
|
+
const commits = gitEnabled ? {
|
|
161028
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
161029
|
+
total: getCommitCount(db, projectIdentity),
|
|
161030
|
+
gitEnabled: true
|
|
161031
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
161032
|
+
return {
|
|
161033
|
+
enabled: true,
|
|
161034
|
+
model: snapshot.model,
|
|
161035
|
+
provider: snapshot.provider,
|
|
161036
|
+
session,
|
|
161037
|
+
memories,
|
|
161038
|
+
commits
|
|
161039
|
+
};
|
|
160678
161040
|
}
|
|
160679
161041
|
|
|
160680
161042
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -160695,6 +161057,7 @@ function createProvider2(config2) {
|
|
|
160695
161057
|
model: config2.model,
|
|
160696
161058
|
apiKey: config2.api_key,
|
|
160697
161059
|
inputType: config2.input_type,
|
|
161060
|
+
queryInputType: config2.query_input_type,
|
|
160698
161061
|
truncate: config2.truncate,
|
|
160699
161062
|
maxInputTokens: config2.max_input_tokens
|
|
160700
161063
|
});
|
|
@@ -160903,6 +161266,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160903
161266
|
"conversation_tokens",
|
|
160904
161267
|
"tool_call_tokens",
|
|
160905
161268
|
"cleared_reasoning_through_tag",
|
|
161269
|
+
"tool_reclaim_watermark",
|
|
160906
161270
|
"last_todo_state",
|
|
160907
161271
|
"cached_m0_bytes",
|
|
160908
161272
|
"cached_m1_bytes",
|
|
@@ -160929,6 +161293,8 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
160929
161293
|
"recovery_no_eligible_head_count",
|
|
160930
161294
|
"force_emergency_bypass_window_start",
|
|
160931
161295
|
"force_emergency_bypass_used",
|
|
161296
|
+
"emergency_drain_active",
|
|
161297
|
+
"historian_drain_failure_at",
|
|
160932
161298
|
"upgrade_reminded_at",
|
|
160933
161299
|
"pi_stable_id_scheme"
|
|
160934
161300
|
];
|
|
@@ -160951,6 +161317,7 @@ var META_COLUMNS = {
|
|
|
160951
161317
|
conversationTokens: "conversation_tokens",
|
|
160952
161318
|
toolCallTokens: "tool_call_tokens",
|
|
160953
161319
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
161320
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
160954
161321
|
lastTodoState: "last_todo_state",
|
|
160955
161322
|
cachedM0Bytes: "cached_m0_bytes",
|
|
160956
161323
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -160977,6 +161344,8 @@ var META_COLUMNS = {
|
|
|
160977
161344
|
recoveryNoEligibleHeadCount: "recovery_no_eligible_head_count",
|
|
160978
161345
|
forceEmergencyBypassWindowStart: "force_emergency_bypass_window_start",
|
|
160979
161346
|
forceEmergencyBypassUsed: "force_emergency_bypass_used",
|
|
161347
|
+
emergencyDrainActive: "emergency_drain_active",
|
|
161348
|
+
historianDrainFailureAt: "historian_drain_failure_at",
|
|
160980
161349
|
upgradeRemindedAt: "upgrade_reminded_at",
|
|
160981
161350
|
piStableIdScheme: "pi_stable_id_scheme"
|
|
160982
161351
|
};
|
|
@@ -161019,7 +161388,7 @@ function isSessionMetaRow(row) {
|
|
|
161019
161388
|
if (row === null || typeof row !== "object")
|
|
161020
161389
|
return false;
|
|
161021
161390
|
const r = row;
|
|
161022
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
161391
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
161023
161392
|
}
|
|
161024
161393
|
function getDefaultSessionMeta(sessionId) {
|
|
161025
161394
|
return {
|
|
@@ -161042,6 +161411,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
161042
161411
|
conversationTokens: 0,
|
|
161043
161412
|
toolCallTokens: 0,
|
|
161044
161413
|
clearedReasoningThroughTag: 0,
|
|
161414
|
+
toolReclaimWatermark: 0,
|
|
161045
161415
|
lastTodoState: "",
|
|
161046
161416
|
cachedM0Bytes: null,
|
|
161047
161417
|
cachedM1Bytes: null,
|
|
@@ -161105,6 +161475,7 @@ function toSessionMeta(row) {
|
|
|
161105
161475
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
161106
161476
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
161107
161477
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
161478
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
161108
161479
|
lastTodoState: lastTodoStateRaw,
|
|
161109
161480
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
161110
161481
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -161531,6 +161902,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
161531
161902
|
var encoder = new TextEncoder;
|
|
161532
161903
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
161533
161904
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
161905
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
161906
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
161534
161907
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
161535
161908
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
161536
161909
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -161543,6 +161916,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
161543
161916
|
function stripMalformedTagNotationGlobally(value) {
|
|
161544
161917
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
161545
161918
|
}
|
|
161919
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
161920
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
161921
|
+
}
|
|
161546
161922
|
function stripTagSectionCharacters(value) {
|
|
161547
161923
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
161548
161924
|
}
|
|
@@ -161550,6 +161926,7 @@ function stripPersistedAssistantText(value) {
|
|
|
161550
161926
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
161551
161927
|
text = stripCompleteTagPairsGlobally(text);
|
|
161552
161928
|
text = stripMalformedTagNotationGlobally(text);
|
|
161929
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
161553
161930
|
text = stripTagSectionCharacters(text);
|
|
161554
161931
|
return text.trim();
|
|
161555
161932
|
}
|
|
@@ -161562,6 +161939,7 @@ function stripTagPrefix(value) {
|
|
|
161562
161939
|
const prev = stripped;
|
|
161563
161940
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
161564
161941
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
161942
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
161565
161943
|
if (stripped === prev)
|
|
161566
161944
|
break;
|
|
161567
161945
|
}
|
|
@@ -161669,6 +162047,24 @@ class ToolMutationBatch {
|
|
|
161669
162047
|
}
|
|
161670
162048
|
|
|
161671
162049
|
// ../plugin/src/hooks/magic-context/read-session-chunk.ts
|
|
162050
|
+
var BLOCK_TOKEN_MEMO_MAX = 2048;
|
|
162051
|
+
var blockTokenMemo = new Map;
|
|
162052
|
+
function estimateBlockTokens(blockText) {
|
|
162053
|
+
const cached2 = blockTokenMemo.get(blockText);
|
|
162054
|
+
if (cached2 !== undefined) {
|
|
162055
|
+
blockTokenMemo.delete(blockText);
|
|
162056
|
+
blockTokenMemo.set(blockText, cached2);
|
|
162057
|
+
return cached2;
|
|
162058
|
+
}
|
|
162059
|
+
const count = estimateTokens(blockText);
|
|
162060
|
+
if (blockTokenMemo.size >= BLOCK_TOKEN_MEMO_MAX) {
|
|
162061
|
+
const oldest = blockTokenMemo.keys().next().value;
|
|
162062
|
+
if (oldest !== undefined)
|
|
162063
|
+
blockTokenMemo.delete(oldest);
|
|
162064
|
+
}
|
|
162065
|
+
blockTokenMemo.set(blockText, count);
|
|
162066
|
+
return count;
|
|
162067
|
+
}
|
|
161672
162068
|
var activeRawMessageCache = null;
|
|
161673
162069
|
var activeAbsoluteCountCache = null;
|
|
161674
162070
|
var sessionProviders = new Map;
|
|
@@ -161851,7 +162247,7 @@ function readSessionChunk(sessionId, tokenBudget, offset = 1, eligibleEndOrdinal
|
|
|
161851
162247
|
if (!currentBlock)
|
|
161852
162248
|
return true;
|
|
161853
162249
|
const blockText = formatBlock(currentBlock);
|
|
161854
|
-
const blockTokens =
|
|
162250
|
+
const blockTokens = estimateBlockTokens(blockText);
|
|
161855
162251
|
if (totalTokens + blockTokens > tokenBudget && totalTokens > 0) {
|
|
161856
162252
|
return false;
|
|
161857
162253
|
}
|
|
@@ -161977,9 +162373,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
161977
162373
|
|
|
161978
162374
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
161979
162375
|
|
|
161980
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
162376
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
162377
|
+
|
|
162378
|
+
Two recovery modes for finer detail:
|
|
162379
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
162380
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
161981
162381
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
161982
162382
|
|
|
162383
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
162384
|
+
function isRecord2(value) {
|
|
162385
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162386
|
+
}
|
|
162387
|
+
function roleLabel(role) {
|
|
162388
|
+
if (role === "assistant")
|
|
162389
|
+
return "A (assistant)";
|
|
162390
|
+
if (role === "user")
|
|
162391
|
+
return "U (user)";
|
|
162392
|
+
return role;
|
|
162393
|
+
}
|
|
162394
|
+
function truncate(value, max) {
|
|
162395
|
+
const t = value.trim();
|
|
162396
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
162397
|
+
}
|
|
162398
|
+
function keyArg(input) {
|
|
162399
|
+
if (!input)
|
|
162400
|
+
return "";
|
|
162401
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
162402
|
+
const v = input[k];
|
|
162403
|
+
if (typeof v === "string" && v.length > 0)
|
|
162404
|
+
return truncate(v, 60);
|
|
162405
|
+
}
|
|
162406
|
+
if (typeof input.description === "string")
|
|
162407
|
+
return truncate(input.description, 60);
|
|
162408
|
+
return "";
|
|
162409
|
+
}
|
|
162410
|
+
function asToolPart(part) {
|
|
162411
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
162412
|
+
if (type === "tool") {
|
|
162413
|
+
const state = isRecord2(part.state) ? part.state : null;
|
|
162414
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
162415
|
+
const metadata = state && isRecord2(state.metadata) ? state.metadata : null;
|
|
162416
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
162417
|
+
return {
|
|
162418
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
162419
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
162420
|
+
title,
|
|
162421
|
+
input: state && isRecord2(state.input) ? state.input : null,
|
|
162422
|
+
output
|
|
162423
|
+
};
|
|
162424
|
+
}
|
|
162425
|
+
if (type === "tool_use") {
|
|
162426
|
+
return {
|
|
162427
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
162428
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
162429
|
+
title: null,
|
|
162430
|
+
input: isRecord2(part.input) ? part.input : null,
|
|
162431
|
+
output: null
|
|
162432
|
+
};
|
|
162433
|
+
}
|
|
162434
|
+
if (type === "tool_result") {
|
|
162435
|
+
const content = part.content;
|
|
162436
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
162437
|
+
return {
|
|
162438
|
+
name: "tool_result",
|
|
162439
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
162440
|
+
title: null,
|
|
162441
|
+
input: null,
|
|
162442
|
+
output
|
|
162443
|
+
};
|
|
162444
|
+
}
|
|
162445
|
+
return null;
|
|
162446
|
+
}
|
|
162447
|
+
function textOf(part) {
|
|
162448
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
162449
|
+
return part.text;
|
|
162450
|
+
return null;
|
|
162451
|
+
}
|
|
162452
|
+
function reasoningOf(part) {
|
|
162453
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
162454
|
+
return part.text;
|
|
162455
|
+
}
|
|
162456
|
+
return null;
|
|
162457
|
+
}
|
|
162458
|
+
function renderPartPreview(part) {
|
|
162459
|
+
if (!isRecord2(part))
|
|
162460
|
+
return null;
|
|
162461
|
+
const text = textOf(part);
|
|
162462
|
+
if (text !== null) {
|
|
162463
|
+
const t = truncate(text, 200);
|
|
162464
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
162465
|
+
}
|
|
162466
|
+
const tool = asToolPart(part);
|
|
162467
|
+
if (tool) {
|
|
162468
|
+
const arg = keyArg(tool.input);
|
|
162469
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
162470
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
162471
|
+
}
|
|
162472
|
+
const reasoning = reasoningOf(part);
|
|
162473
|
+
if (reasoning !== null)
|
|
162474
|
+
return ` • [reasoning] ${truncate(reasoning, 120)}`;
|
|
162475
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162476
|
+
if (type === "file")
|
|
162477
|
+
return " • [file]";
|
|
162478
|
+
if (type === "step-start" || type === "step-finish")
|
|
162479
|
+
return null;
|
|
162480
|
+
return ` • [${type}]`;
|
|
162481
|
+
}
|
|
162482
|
+
function renderPartFull(part) {
|
|
162483
|
+
if (!isRecord2(part))
|
|
162484
|
+
return null;
|
|
162485
|
+
const text = textOf(part);
|
|
162486
|
+
if (text !== null) {
|
|
162487
|
+
return text.trim().length > 0 ? ` [text]
|
|
162488
|
+
${text}` : null;
|
|
162489
|
+
}
|
|
162490
|
+
const tool = asToolPart(part);
|
|
162491
|
+
if (tool) {
|
|
162492
|
+
const lines = [];
|
|
162493
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
162494
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
162495
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
162496
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
162497
|
+
}
|
|
162498
|
+
if (tool.input)
|
|
162499
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
162500
|
+
if (tool.output !== null)
|
|
162501
|
+
lines.push(` output:
|
|
162502
|
+
${tool.output}`);
|
|
162503
|
+
return lines.join(`
|
|
162504
|
+
`);
|
|
162505
|
+
}
|
|
162506
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
162507
|
+
if (type === "file") {
|
|
162508
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
162509
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
162510
|
+
}
|
|
162511
|
+
return null;
|
|
162512
|
+
}
|
|
162513
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
162514
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
162515
|
+
if (!msg) {
|
|
162516
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
162517
|
+
}
|
|
162518
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
162519
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
162520
|
+
if (rendered.length === 0) {
|
|
162521
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
162522
|
+
} else {
|
|
162523
|
+
lines.push(...rendered);
|
|
162524
|
+
}
|
|
162525
|
+
return lines.join(`
|
|
162526
|
+
`);
|
|
162527
|
+
}
|
|
162528
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
162529
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
162530
|
+
const out = [];
|
|
162531
|
+
let usedTokens = 0;
|
|
162532
|
+
let lastOrdinal = start - 1;
|
|
162533
|
+
let truncated = false;
|
|
162534
|
+
for (const msg of messages) {
|
|
162535
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
162536
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
162537
|
+
const block = partLines.length > 0 ? `${header}
|
|
162538
|
+
${partLines.join(`
|
|
162539
|
+
`)}` : header;
|
|
162540
|
+
const blockTokens = estimateTokens(block);
|
|
162541
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
162542
|
+
truncated = true;
|
|
162543
|
+
break;
|
|
162544
|
+
}
|
|
162545
|
+
out.push(block);
|
|
162546
|
+
usedTokens += blockTokens;
|
|
162547
|
+
lastOrdinal = msg.ordinal;
|
|
162548
|
+
}
|
|
162549
|
+
return { text: out.join(`
|
|
162550
|
+
|
|
162551
|
+
`), lastOrdinal, truncated };
|
|
162552
|
+
}
|
|
162553
|
+
|
|
161983
162554
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
161984
162555
|
var exports_memory = {};
|
|
161985
162556
|
__export(exports_memory, {
|
|
@@ -166336,12 +166907,18 @@ function synthesizeToolResultParts(msg) {
|
|
|
166336
166907
|
|
|
166337
166908
|
// src/tools/ctx-expand.ts
|
|
166338
166909
|
var ParamsSchema = exports_typebox.Object({
|
|
166339
|
-
start: exports_typebox.Number({
|
|
166910
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
166340
166911
|
description: "Start message ordinal (from compartment start attribute)"
|
|
166341
|
-
}),
|
|
166342
|
-
end: exports_typebox.Number({
|
|
166912
|
+
})),
|
|
166913
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
166343
166914
|
description: "End message ordinal (from compartment end attribute)"
|
|
166344
|
-
})
|
|
166915
|
+
})),
|
|
166916
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
166917
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
166918
|
+
})),
|
|
166919
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
166920
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
166921
|
+
}))
|
|
166345
166922
|
});
|
|
166346
166923
|
function ok(text) {
|
|
166347
166924
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -166360,22 +166937,41 @@ function createCtxExpandTool(deps) {
|
|
|
166360
166937
|
description: CTX_EXPAND_DESCRIPTION,
|
|
166361
166938
|
parameters: ParamsSchema,
|
|
166362
166939
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
166363
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166364
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
166365
|
-
}
|
|
166366
166940
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
166367
166941
|
if (!sessionId) {
|
|
166368
166942
|
return err("Error: no active Pi session.");
|
|
166369
166943
|
}
|
|
166370
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166371
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166372
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166373
|
-
}
|
|
166374
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166375
166944
|
const unregister = setRawMessageProvider(sessionId, {
|
|
166376
166945
|
readMessages: () => readPiSessionMessages(ctx)
|
|
166377
166946
|
});
|
|
166378
166947
|
try {
|
|
166948
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
166949
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
166950
|
+
}
|
|
166951
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
166952
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
166953
|
+
}
|
|
166954
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
166955
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
166956
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
166957
|
+
}
|
|
166958
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
166959
|
+
if (params.verbose === true) {
|
|
166960
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
166961
|
+
if (!v.text) {
|
|
166962
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
166963
|
+
}
|
|
166964
|
+
const out = [
|
|
166965
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
166966
|
+
"",
|
|
166967
|
+
v.text
|
|
166968
|
+
];
|
|
166969
|
+
if (v.truncated) {
|
|
166970
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
166971
|
+
}
|
|
166972
|
+
return ok(out.join(`
|
|
166973
|
+
`));
|
|
166974
|
+
}
|
|
166379
166975
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
166380
166976
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
166381
166977
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -167601,7 +168197,9 @@ var DEFAULT_PROTECTED_TAIL_META = {
|
|
|
167601
168197
|
protectedTailDrainTokens: 0,
|
|
167602
168198
|
recoveryNoEligibleHeadCount: 0,
|
|
167603
168199
|
forceEmergencyBypassWindowStart: 0,
|
|
167604
|
-
forceEmergencyBypassUsed: 0
|
|
168200
|
+
forceEmergencyBypassUsed: 0,
|
|
168201
|
+
emergencyDrainActive: 0,
|
|
168202
|
+
historianDrainFailureAt: 0
|
|
167605
168203
|
};
|
|
167606
168204
|
function toProtectedTailMeta(row) {
|
|
167607
168205
|
if (row === null || typeof row !== "object")
|
|
@@ -167615,7 +168213,9 @@ function toProtectedTailMeta(row) {
|
|
|
167615
168213
|
protectedTailDrainTokens: numberOr(r.protected_tail_drain_tokens, 0),
|
|
167616
168214
|
recoveryNoEligibleHeadCount: numberOr(r.recovery_no_eligible_head_count, 0),
|
|
167617
168215
|
forceEmergencyBypassWindowStart: numberOr(r.force_emergency_bypass_window_start, 0),
|
|
167618
|
-
forceEmergencyBypassUsed: numberOr(r.force_emergency_bypass_used, 0)
|
|
168216
|
+
forceEmergencyBypassUsed: numberOr(r.force_emergency_bypass_used, 0),
|
|
168217
|
+
emergencyDrainActive: numberOr(r.emergency_drain_active, 0),
|
|
168218
|
+
historianDrainFailureAt: numberOr(r.historian_drain_failure_at, 0)
|
|
167619
168219
|
};
|
|
167620
168220
|
}
|
|
167621
168221
|
function loadProtectedTailMeta(db, sessionId) {
|
|
@@ -167623,7 +168223,7 @@ function loadProtectedTailMeta(db, sessionId) {
|
|
|
167623
168223
|
const row = db.prepare(`SELECT prior_boundary_ordinal, protected_tail_policy_version,
|
|
167624
168224
|
protected_tail_drain_window_started_at, protected_tail_drain_tokens,
|
|
167625
168225
|
recovery_no_eligible_head_count, force_emergency_bypass_window_start,
|
|
167626
|
-
force_emergency_bypass_used
|
|
168226
|
+
force_emergency_bypass_used, emergency_drain_active, historian_drain_failure_at
|
|
167627
168227
|
FROM session_meta WHERE session_id = ?`).get(sessionId);
|
|
167628
168228
|
return toProtectedTailMeta(row);
|
|
167629
168229
|
}
|
|
@@ -167667,6 +168267,17 @@ function protectedTailWindowBudget(usagePercentage, usable, perRunCap) {
|
|
|
167667
168267
|
return Math.min(750000, Math.max(3 * perRunCap, Math.round(0.35 * usable)));
|
|
167668
168268
|
return Math.min(500000, Math.max(perRunCap, Math.round(0.2 * usable)));
|
|
167669
168269
|
}
|
|
168270
|
+
var EMERGENCY_DRAIN_ENTER_PERCENTAGE = 95;
|
|
168271
|
+
var EMERGENCY_DRAIN_EXIT_MARGIN = 10;
|
|
168272
|
+
var EMERGENCY_DRAIN_FALLBACK_EXIT_PERCENTAGE = 55;
|
|
168273
|
+
var EMERGENCY_DRAIN_FAILURE_BACKOFF_MS = 60000;
|
|
168274
|
+
var EMERGENCY_DRAIN_MAX_LATCH_MS = 30 * 60 * 1000;
|
|
168275
|
+
function emergencyDrainExitThreshold(executeThresholdPercentage) {
|
|
168276
|
+
if (!Number.isFinite(executeThresholdPercentage) || executeThresholdPercentage <= 0) {
|
|
168277
|
+
return EMERGENCY_DRAIN_FALLBACK_EXIT_PERCENTAGE;
|
|
168278
|
+
}
|
|
168279
|
+
return Math.max(0, executeThresholdPercentage - EMERGENCY_DRAIN_EXIT_MARGIN);
|
|
168280
|
+
}
|
|
167670
168281
|
function reserveProtectedTailDrainTokens(args) {
|
|
167671
168282
|
const now = args.now ?? Date.now();
|
|
167672
168283
|
const requested = Math.max(0, Math.floor(args.trueRawTokens));
|
|
@@ -167685,18 +168296,30 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167685
168296
|
let meta3 = loadProtectedTailMeta(args.db, args.sessionId);
|
|
167686
168297
|
if (now - meta3.protectedTailDrainWindowStartedAt > DRAIN_WINDOW_MS) {
|
|
167687
168298
|
args.db.prepare(`UPDATE session_meta
|
|
167688
|
-
SET protected_tail_drain_window_started_at = ?, protected_tail_drain_tokens = 0
|
|
167689
|
-
|
|
167690
|
-
WHERE session_id = ?`).run(now, now, args.sessionId);
|
|
168299
|
+
SET protected_tail_drain_window_started_at = ?, protected_tail_drain_tokens = 0
|
|
168300
|
+
WHERE session_id = ?`).run(now, args.sessionId);
|
|
167691
168301
|
meta3 = loadProtectedTailMeta(args.db, args.sessionId);
|
|
167692
168302
|
}
|
|
168303
|
+
const exitThreshold = emergencyDrainExitThreshold(args.executeThresholdPercentage);
|
|
168304
|
+
let latchActiveSince = meta3.emergencyDrainActive;
|
|
168305
|
+
if (args.usagePercentage >= EMERGENCY_DRAIN_ENTER_PERCENTAGE) {
|
|
168306
|
+
if (latchActiveSince <= 0)
|
|
168307
|
+
latchActiveSince = now;
|
|
168308
|
+
} else if (latchActiveSince > 0) {
|
|
168309
|
+
const expired = now - latchActiveSince > EMERGENCY_DRAIN_MAX_LATCH_MS;
|
|
168310
|
+
if (args.usagePercentage < exitThreshold || expired)
|
|
168311
|
+
latchActiveSince = 0;
|
|
168312
|
+
}
|
|
168313
|
+
if (latchActiveSince !== meta3.emergencyDrainActive) {
|
|
168314
|
+
args.db.prepare("UPDATE session_meta SET emergency_drain_active = ? WHERE session_id = ?").run(latchActiveSince, args.sessionId);
|
|
168315
|
+
}
|
|
168316
|
+
const latchActive = latchActiveSince > 0;
|
|
167693
168317
|
const budget = protectedTailWindowBudget(args.usagePercentage, args.usable, args.perRunCap);
|
|
167694
168318
|
const remaining = Math.max(0, budget - meta3.protectedTailDrainTokens);
|
|
167695
168319
|
let reserved = Math.min(requested, args.perRunCap, remaining);
|
|
167696
168320
|
let bypass = false;
|
|
167697
|
-
const
|
|
167698
|
-
|
|
167699
|
-
if (reserved <= 0 && args.usagePercentage >= 95 && bypassUsed === 0) {
|
|
168321
|
+
const inFailureBackoff = meta3.historianDrainFailureAt > 0 && now - meta3.historianDrainFailureAt < EMERGENCY_DRAIN_FAILURE_BACKOFF_MS;
|
|
168322
|
+
if (reserved <= 0 && latchActive && !inFailureBackoff) {
|
|
167700
168323
|
reserved = Math.min(requested, args.perRunCap);
|
|
167701
168324
|
bypass = true;
|
|
167702
168325
|
}
|
|
@@ -167704,10 +168327,8 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167704
168327
|
return;
|
|
167705
168328
|
args.db.prepare(`UPDATE session_meta
|
|
167706
168329
|
SET protected_tail_drain_window_started_at = CASE WHEN protected_tail_drain_window_started_at = 0 THEN ? ELSE protected_tail_drain_window_started_at END,
|
|
167707
|
-
protected_tail_drain_tokens = COALESCE(protected_tail_drain_tokens, 0) +
|
|
167708
|
-
|
|
167709
|
-
force_emergency_bypass_used = CASE WHEN ? THEN 1 ELSE force_emergency_bypass_used END
|
|
167710
|
-
WHERE session_id = ?`).run(now, reserved, bypass ? 1 : 0, now, bypass ? 1 : 0, args.sessionId);
|
|
168330
|
+
protected_tail_drain_tokens = COALESCE(protected_tail_drain_tokens, 0) + ?
|
|
168331
|
+
WHERE session_id = ?`).run(now, reserved, args.sessionId);
|
|
167711
168332
|
result = {
|
|
167712
168333
|
ok: true,
|
|
167713
168334
|
reservedTokens: reserved,
|
|
@@ -167717,6 +168338,25 @@ function reserveProtectedTailDrainTokens(args) {
|
|
|
167717
168338
|
})();
|
|
167718
168339
|
return result;
|
|
167719
168340
|
}
|
|
168341
|
+
function clearEmergencyDrainLatch(db, sessionId) {
|
|
168342
|
+
db.transaction(() => {
|
|
168343
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168344
|
+
db.prepare("UPDATE session_meta SET emergency_drain_active = 0 WHERE session_id = ?").run(sessionId);
|
|
168345
|
+
})();
|
|
168346
|
+
}
|
|
168347
|
+
function recordHistorianDrainFailure(db, sessionId, now) {
|
|
168348
|
+
const ts = now ?? Date.now();
|
|
168349
|
+
db.transaction(() => {
|
|
168350
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168351
|
+
db.prepare("UPDATE session_meta SET historian_drain_failure_at = ? WHERE session_id = ?").run(ts, sessionId);
|
|
168352
|
+
})();
|
|
168353
|
+
}
|
|
168354
|
+
function clearHistorianDrainFailure(db, sessionId) {
|
|
168355
|
+
db.transaction(() => {
|
|
168356
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168357
|
+
db.prepare("UPDATE session_meta SET historian_drain_failure_at = 0 WHERE session_id = ?").run(sessionId);
|
|
168358
|
+
})();
|
|
168359
|
+
}
|
|
167720
168360
|
function rollbackProtectedTailDrainReservation(db, reservation) {
|
|
167721
168361
|
if (!reservation || reservation.tokens <= 0)
|
|
167722
168362
|
return;
|
|
@@ -168235,6 +168875,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
168235
168875
|
last_transform_error: "'' AS last_transform_error",
|
|
168236
168876
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
168237
168877
|
last_todo_state: "'' AS last_todo_state",
|
|
168878
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
168238
168879
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
168239
168880
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
168240
168881
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -168312,6 +168953,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
168312
168953
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
168313
168954
|
})();
|
|
168314
168955
|
}
|
|
168956
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
168957
|
+
if (maxTagNumber <= 0)
|
|
168958
|
+
return;
|
|
168959
|
+
db.transaction(() => {
|
|
168960
|
+
ensureSessionMetaRow(db, sessionId);
|
|
168961
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
168962
|
+
})();
|
|
168963
|
+
}
|
|
168315
168964
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
168316
168965
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
168317
168966
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -168709,12 +169358,37 @@ function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
|
168709
169358
|
nullCount: row?.null_count ?? 0
|
|
168710
169359
|
};
|
|
168711
169360
|
}
|
|
168712
|
-
function
|
|
168713
|
-
|
|
169361
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
169362
|
+
if (limit <= 0)
|
|
169363
|
+
return [];
|
|
169364
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
169365
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
169366
|
+
SELECT tag_number FROM tags
|
|
169367
|
+
WHERE session_id = ? AND status = 'active'
|
|
169368
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
169369
|
+
)` : "";
|
|
169370
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
169371
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
169372
|
+
FROM tags
|
|
169373
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
169374
|
+
ORDER BY tag_number ASC, id ASC
|
|
169375
|
+
LIMIT ?`).all(...params);
|
|
169376
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
169377
|
+
tagNumber: row.tag_number,
|
|
169378
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
169379
|
+
}));
|
|
169380
|
+
}
|
|
169381
|
+
function getTriggerTagTokenUpperBound(db, sessionId, floor = 0) {
|
|
169382
|
+
const sql = floor > 0 ? `SELECT
|
|
169383
|
+
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
169384
|
+
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
169385
|
+
FROM tags
|
|
169386
|
+
WHERE session_id = ? AND status IN ('active', 'dropped') AND tag_number >= ?` : `SELECT
|
|
168714
169387
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
168715
169388
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
168716
169389
|
FROM tags
|
|
168717
|
-
WHERE session_id = ? AND status IN ('active', 'dropped')
|
|
169390
|
+
WHERE session_id = ? AND status IN ('active', 'dropped')`;
|
|
169391
|
+
const row = floor > 0 ? db.prepare(sql).get(sessionId, floor) : db.prepare(sql).get(sessionId);
|
|
168718
169392
|
return { bound: row?.bound ?? 0, nullCount: row?.null_count ?? 0 };
|
|
168719
169393
|
}
|
|
168720
169394
|
function updateTagInputByteSize(db, sessionId, tagNumber, newInputByteSize) {
|
|
@@ -168741,10 +169415,12 @@ function getUpdateTagInputTokenCountStatement(db) {
|
|
|
168741
169415
|
function updateTagTokenCount(db, sessionId, tagNumber, newTokenCount) {
|
|
168742
169416
|
getUpdateTagTokenCountStatement(db).run(newTokenCount, sessionId, tagNumber);
|
|
168743
169417
|
}
|
|
168744
|
-
function getAllStatusTagTokenTotalsFlat(db, sessionId) {
|
|
168745
|
-
const rows = db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
168746
|
-
|
|
168747
|
-
|
|
169418
|
+
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0) {
|
|
169419
|
+
const rows = floor > 0 ? db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
169420
|
+
FROM tags
|
|
169421
|
+
WHERE session_id = ? AND tag_number >= ?`).all(sessionId, floor) : db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
169422
|
+
FROM tags
|
|
169423
|
+
WHERE session_id = ?`).all(sessionId);
|
|
168748
169424
|
const totals = new Map;
|
|
168749
169425
|
const nullMessageIds = new Set;
|
|
168750
169426
|
for (const row of rows) {
|
|
@@ -168863,6 +169539,52 @@ function getTagNumberByMessageId(db, sessionId, messageId) {
|
|
|
168863
169539
|
const row = getTagNumberByMessageIdStatement(db).get(sessionId, messageId);
|
|
168864
169540
|
return isTagNumberRow(row) ? row.tag_number : null;
|
|
168865
169541
|
}
|
|
169542
|
+
var getMinMessageTagNumberForRawIdStatements = new WeakMap;
|
|
169543
|
+
function isMinTagNumberRow(row) {
|
|
169544
|
+
return row !== null && typeof row === "object" && "m" in row;
|
|
169545
|
+
}
|
|
169546
|
+
function getMinMessageTagNumberForRawId(db, sessionId, rawId) {
|
|
169547
|
+
if (rawId.includes(":"))
|
|
169548
|
+
return null;
|
|
169549
|
+
let stmt = getMinMessageTagNumberForRawIdStatements.get(db);
|
|
169550
|
+
if (!stmt) {
|
|
169551
|
+
stmt = db.prepare("SELECT MIN(tag_number) AS m FROM tags WHERE session_id = ? AND message_id >= ? AND message_id < ?");
|
|
169552
|
+
getMinMessageTagNumberForRawIdStatements.set(db, stmt);
|
|
169553
|
+
}
|
|
169554
|
+
const row = stmt.get(sessionId, `${rawId}:`, `${rawId};`);
|
|
169555
|
+
return isMinTagNumberRow(row) && typeof row.m === "number" ? row.m : null;
|
|
169556
|
+
}
|
|
169557
|
+
var TAGGER_FLOOR_SCAN_MESSAGES = 8;
|
|
169558
|
+
var TAGGER_FLOOR_MAX_PROBES = 64;
|
|
169559
|
+
var TAGGER_FLOOR_SAFETY_MARGIN = 256;
|
|
169560
|
+
var TAGGER_FLOOR_PER_SKIP_MARGIN = 64;
|
|
169561
|
+
function deriveTagLoadFloor(db, sessionId, rawIds) {
|
|
169562
|
+
let min = Number.POSITIVE_INFINITY;
|
|
169563
|
+
let probes = 0;
|
|
169564
|
+
let hits = 0;
|
|
169565
|
+
let skippedBeforeFirstHit = 0;
|
|
169566
|
+
for (const rawId of rawIds) {
|
|
169567
|
+
if (typeof rawId !== "string" || rawId.length === 0)
|
|
169568
|
+
continue;
|
|
169569
|
+
if (probes >= TAGGER_FLOOR_MAX_PROBES)
|
|
169570
|
+
break;
|
|
169571
|
+
probes++;
|
|
169572
|
+
const m = getMinMessageTagNumberForRawId(db, sessionId, rawId);
|
|
169573
|
+
if (m === null) {
|
|
169574
|
+
if (hits === 0)
|
|
169575
|
+
skippedBeforeFirstHit++;
|
|
169576
|
+
continue;
|
|
169577
|
+
}
|
|
169578
|
+
if (m < min)
|
|
169579
|
+
min = m;
|
|
169580
|
+
if (++hits >= TAGGER_FLOOR_SCAN_MESSAGES)
|
|
169581
|
+
break;
|
|
169582
|
+
}
|
|
169583
|
+
if (!Number.isFinite(min))
|
|
169584
|
+
return 0;
|
|
169585
|
+
const margin = TAGGER_FLOOR_SAFETY_MARGIN + skippedBeforeFirstHit * TAGGER_FLOOR_PER_SKIP_MARGIN;
|
|
169586
|
+
return Math.max(0, min - margin);
|
|
169587
|
+
}
|
|
168866
169588
|
var TAG_SELECT_COLUMNS = "id, message_id, type, status, drop_mode, tool_name, input_byte_size, byte_size, reasoning_byte_size, session_id, tag_number, caveman_depth, tool_owner_message_id";
|
|
168867
169589
|
function getTagsBySession(db, sessionId) {
|
|
168868
169590
|
const rows = db.prepare(`SELECT ${TAG_SELECT_COLUMNS} FROM tags WHERE session_id = ? ORDER BY tag_number ASC, id ASC`).all(sessionId).filter(isTagRow);
|
|
@@ -169658,15 +170380,16 @@ function parseInteger(str) {
|
|
|
169658
170380
|
}
|
|
169659
170381
|
|
|
169660
170382
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
169661
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
169662
|
-
Use §N§ identifiers visible in conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
170383
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
169663
170384
|
|
|
169664
|
-
|
|
169665
|
-
-
|
|
169666
|
-
-
|
|
169667
|
-
-
|
|
169668
|
-
|
|
169669
|
-
|
|
170385
|
+
How it works:
|
|
170386
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
170387
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
170388
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
170389
|
+
|
|
170390
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
170391
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
170392
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
169670
170393
|
|
|
169671
170394
|
// src/tools/ctx-reduce.ts
|
|
169672
170395
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -169851,120 +170574,6 @@ ${body}` : subject;
|
|
|
169851
170574
|
}
|
|
169852
170575
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169853
170576
|
init_logger();
|
|
169854
|
-
|
|
169855
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
169856
|
-
init_logger();
|
|
169857
|
-
var insertStatements = new WeakMap;
|
|
169858
|
-
var existingShasStatements = new WeakMap;
|
|
169859
|
-
var projectCountStatements = new WeakMap;
|
|
169860
|
-
var evictStatements = new WeakMap;
|
|
169861
|
-
var evictOverflowStatements = new WeakMap;
|
|
169862
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
169863
|
-
function getInsertStatement(db) {
|
|
169864
|
-
let stmt = insertStatements.get(db);
|
|
169865
|
-
if (!stmt) {
|
|
169866
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
169867
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
169868
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
169869
|
-
project_path = excluded.project_path,
|
|
169870
|
-
short_sha = excluded.short_sha,
|
|
169871
|
-
message = excluded.message,
|
|
169872
|
-
author = excluded.author,
|
|
169873
|
-
committed_at = excluded.committed_at,
|
|
169874
|
-
indexed_at = excluded.indexed_at
|
|
169875
|
-
WHERE git_commits.message != excluded.message`);
|
|
169876
|
-
insertStatements.set(db, stmt);
|
|
169877
|
-
}
|
|
169878
|
-
return stmt;
|
|
169879
|
-
}
|
|
169880
|
-
function getExistingShasStatement(db) {
|
|
169881
|
-
let stmt = existingShasStatements.get(db);
|
|
169882
|
-
if (!stmt) {
|
|
169883
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
169884
|
-
existingShasStatements.set(db, stmt);
|
|
169885
|
-
}
|
|
169886
|
-
return stmt;
|
|
169887
|
-
}
|
|
169888
|
-
function getProjectCountStatement(db) {
|
|
169889
|
-
let stmt = projectCountStatements.get(db);
|
|
169890
|
-
if (!stmt) {
|
|
169891
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
169892
|
-
projectCountStatements.set(db, stmt);
|
|
169893
|
-
}
|
|
169894
|
-
return stmt;
|
|
169895
|
-
}
|
|
169896
|
-
function getLatestCommitTimeStatement(db) {
|
|
169897
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
169898
|
-
if (!stmt) {
|
|
169899
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
169900
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
169901
|
-
}
|
|
169902
|
-
return stmt;
|
|
169903
|
-
}
|
|
169904
|
-
function getEvictOverflowStatement(db) {
|
|
169905
|
-
let stmt = evictOverflowStatements.get(db);
|
|
169906
|
-
if (!stmt) {
|
|
169907
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
169908
|
-
WHERE rowid IN (
|
|
169909
|
-
SELECT rowid FROM git_commits
|
|
169910
|
-
WHERE project_path = ?
|
|
169911
|
-
ORDER BY committed_at DESC, sha DESC
|
|
169912
|
-
LIMIT -1 OFFSET ?
|
|
169913
|
-
)`);
|
|
169914
|
-
evictOverflowStatements.set(db, stmt);
|
|
169915
|
-
}
|
|
169916
|
-
return stmt;
|
|
169917
|
-
}
|
|
169918
|
-
function upsertCommits(db, projectPath, commits) {
|
|
169919
|
-
if (commits.length === 0)
|
|
169920
|
-
return { inserted: 0, updated: 0 };
|
|
169921
|
-
const existing = new Set;
|
|
169922
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
169923
|
-
existing.add(row.sha);
|
|
169924
|
-
}
|
|
169925
|
-
let inserted = 0;
|
|
169926
|
-
let updated = 0;
|
|
169927
|
-
const now = Date.now();
|
|
169928
|
-
const insertStmt = getInsertStatement(db);
|
|
169929
|
-
db.transaction(() => {
|
|
169930
|
-
for (const commit of commits) {
|
|
169931
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
169932
|
-
if (result.changes > 0) {
|
|
169933
|
-
if (existing.has(commit.sha)) {
|
|
169934
|
-
updated++;
|
|
169935
|
-
} else {
|
|
169936
|
-
inserted++;
|
|
169937
|
-
existing.add(commit.sha);
|
|
169938
|
-
}
|
|
169939
|
-
}
|
|
169940
|
-
}
|
|
169941
|
-
})();
|
|
169942
|
-
return { inserted, updated };
|
|
169943
|
-
}
|
|
169944
|
-
function getCommitCount(db, projectPath) {
|
|
169945
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
169946
|
-
return row?.count ?? 0;
|
|
169947
|
-
}
|
|
169948
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
169949
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
169950
|
-
return row?.latest ?? null;
|
|
169951
|
-
}
|
|
169952
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
169953
|
-
if (maxCommits <= 0)
|
|
169954
|
-
return 0;
|
|
169955
|
-
const count = getCommitCount(db, projectPath);
|
|
169956
|
-
if (count <= maxCommits)
|
|
169957
|
-
return 0;
|
|
169958
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
169959
|
-
const after = getCommitCount(db, projectPath);
|
|
169960
|
-
const evicted = Math.max(0, count - after);
|
|
169961
|
-
if (evicted > 0) {
|
|
169962
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
169963
|
-
}
|
|
169964
|
-
return evicted;
|
|
169965
|
-
}
|
|
169966
|
-
|
|
169967
|
-
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
169968
170577
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
169969
170578
|
var EMBED_BATCH_SIZE = 16;
|
|
169970
170579
|
var EMBED_MAX_PER_SWEEP = 500;
|
|
@@ -171714,7 +172323,7 @@ function createCtxSearchTool(deps) {
|
|
|
171714
172323
|
memoryEnabled,
|
|
171715
172324
|
embeddingEnabled,
|
|
171716
172325
|
embedQuery: async (text, signal) => {
|
|
171717
|
-
const result = await embedTextForProject(projectIdentity, text, signal);
|
|
172326
|
+
const result = await embedTextForProject(projectIdentity, text, signal, "query");
|
|
171718
172327
|
return result?.vector ?? null;
|
|
171719
172328
|
},
|
|
171720
172329
|
isEmbeddingRuntimeEnabled: () => embeddingEnabled === true,
|