@cortexkit/pi-magic-context 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1019 -318
- package/dist/subagent-entry.js +592 -182
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -8275,6 +8275,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8275
8275
|
}
|
|
8276
8276
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8277
8277
|
}
|
|
8278
|
+
prepare(sql) {
|
|
8279
|
+
const stmt = super.prepare(sql);
|
|
8280
|
+
for (const method of ["run", "get", "all"]) {
|
|
8281
|
+
const original = stmt[method].bind(stmt);
|
|
8282
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8283
|
+
}
|
|
8284
|
+
return stmt;
|
|
8285
|
+
}
|
|
8278
8286
|
transaction(fn) {
|
|
8279
8287
|
const self = this;
|
|
8280
8288
|
const wrapped = function(...args) {
|
|
@@ -141059,6 +141067,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
141059
141067
|
var encoder = new TextEncoder;
|
|
141060
141068
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
141061
141069
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
141070
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
141071
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
141062
141072
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
141063
141073
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
141064
141074
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -141071,6 +141081,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
141071
141081
|
function stripMalformedTagNotationGlobally(value) {
|
|
141072
141082
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
141073
141083
|
}
|
|
141084
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
141085
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
141086
|
+
}
|
|
141074
141087
|
function stripTagSectionCharacters(value) {
|
|
141075
141088
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
141076
141089
|
}
|
|
@@ -141078,6 +141091,7 @@ function stripPersistedAssistantText(value) {
|
|
|
141078
141091
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
141079
141092
|
text = stripCompleteTagPairsGlobally(text);
|
|
141080
141093
|
text = stripMalformedTagNotationGlobally(text);
|
|
141094
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
141081
141095
|
text = stripTagSectionCharacters(text);
|
|
141082
141096
|
return text.trim();
|
|
141083
141097
|
}
|
|
@@ -141090,6 +141104,7 @@ function stripTagPrefix(value) {
|
|
|
141090
141104
|
const prev = stripped;
|
|
141091
141105
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
141092
141106
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
141107
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
141093
141108
|
if (stripped === prev)
|
|
141094
141109
|
break;
|
|
141095
141110
|
}
|
|
@@ -141957,6 +141972,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
141957
141972
|
"conversation_tokens",
|
|
141958
141973
|
"tool_call_tokens",
|
|
141959
141974
|
"cleared_reasoning_through_tag",
|
|
141975
|
+
"tool_reclaim_watermark",
|
|
141960
141976
|
"last_todo_state",
|
|
141961
141977
|
"cached_m0_bytes",
|
|
141962
141978
|
"cached_m1_bytes",
|
|
@@ -142005,6 +142021,7 @@ var META_COLUMNS = {
|
|
|
142005
142021
|
conversationTokens: "conversation_tokens",
|
|
142006
142022
|
toolCallTokens: "tool_call_tokens",
|
|
142007
142023
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
142024
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
142008
142025
|
lastTodoState: "last_todo_state",
|
|
142009
142026
|
cachedM0Bytes: "cached_m0_bytes",
|
|
142010
142027
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -142073,7 +142090,7 @@ function isSessionMetaRow(row) {
|
|
|
142073
142090
|
if (row === null || typeof row !== "object")
|
|
142074
142091
|
return false;
|
|
142075
142092
|
const r = row;
|
|
142076
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
142093
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
142077
142094
|
}
|
|
142078
142095
|
function getDefaultSessionMeta(sessionId) {
|
|
142079
142096
|
return {
|
|
@@ -142096,6 +142113,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
142096
142113
|
conversationTokens: 0,
|
|
142097
142114
|
toolCallTokens: 0,
|
|
142098
142115
|
clearedReasoningThroughTag: 0,
|
|
142116
|
+
toolReclaimWatermark: 0,
|
|
142099
142117
|
lastTodoState: "",
|
|
142100
142118
|
cachedM0Bytes: null,
|
|
142101
142119
|
cachedM1Bytes: null,
|
|
@@ -142159,6 +142177,7 @@ function toSessionMeta(row) {
|
|
|
142159
142177
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
142160
142178
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
142161
142179
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
142180
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
142162
142181
|
lastTodoState: lastTodoStateRaw,
|
|
142163
142182
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
142164
142183
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -143598,6 +143617,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143598
143617
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143599
143618
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143600
143619
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143620
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143601
143621
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143602
143622
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143603
143623
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -146217,6 +146237,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
146217
146237
|
last_transform_error: "'' AS last_transform_error",
|
|
146218
146238
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
146219
146239
|
last_todo_state: "'' AS last_todo_state",
|
|
146240
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
146220
146241
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
146221
146242
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
146222
146243
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -146294,6 +146315,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
146294
146315
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
146295
146316
|
})();
|
|
146296
146317
|
}
|
|
146318
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
146319
|
+
if (maxTagNumber <= 0)
|
|
146320
|
+
return;
|
|
146321
|
+
db.transaction(() => {
|
|
146322
|
+
ensureSessionMetaRow(db, sessionId);
|
|
146323
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
146324
|
+
})();
|
|
146325
|
+
}
|
|
146297
146326
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
146298
146327
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
146299
146328
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -146668,15 +146697,22 @@ function ownerMessageIdForTagRow(row) {
|
|
|
146668
146697
|
}
|
|
146669
146698
|
return row.message_id.replace(CONTENT_ID_SUFFIX, "");
|
|
146670
146699
|
}
|
|
146671
|
-
function getActiveTagTokenAggregate(db, sessionId) {
|
|
146672
|
-
const
|
|
146700
|
+
function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
146701
|
+
const toolOutputExpr = protectedTags > 0 ? `COALESCE(SUM(CASE WHEN type = 'tool' AND tag_number < (
|
|
146702
|
+
SELECT tag_number FROM tags
|
|
146703
|
+
WHERE session_id = ? AND status = 'active'
|
|
146704
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
146705
|
+
) THEN COALESCE(token_count, 0) ELSE 0 END), 0)` : `COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)`;
|
|
146706
|
+
const sql = `SELECT
|
|
146673
146707
|
COALESCE(SUM(CASE WHEN type != 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)
|
|
146674
146708
|
+ COALESCE(SUM(COALESCE(reasoning_token_count, 0)), 0) AS conversation,
|
|
146675
146709
|
COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) + COALESCE(input_token_count, 0) ELSE 0 END), 0) AS tool_call,
|
|
146676
|
-
|
|
146710
|
+
${toolOutputExpr} AS tool_output,
|
|
146677
146711
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
146678
146712
|
FROM tags
|
|
146679
|
-
WHERE session_id = ? AND status = 'active'
|
|
146713
|
+
WHERE session_id = ? AND status = 'active'`;
|
|
146714
|
+
const params = protectedTags > 0 ? [sessionId, protectedTags - 1, sessionId] : [sessionId];
|
|
146715
|
+
const row = db.prepare(sql).get(...params);
|
|
146680
146716
|
return {
|
|
146681
146717
|
conversation: row?.conversation ?? 0,
|
|
146682
146718
|
toolCall: row?.tool_call ?? 0,
|
|
@@ -146684,6 +146720,26 @@ function getActiveTagTokenAggregate(db, sessionId) {
|
|
|
146684
146720
|
nullCount: row?.null_count ?? 0
|
|
146685
146721
|
};
|
|
146686
146722
|
}
|
|
146723
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
146724
|
+
if (limit <= 0)
|
|
146725
|
+
return [];
|
|
146726
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
146727
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
146728
|
+
SELECT tag_number FROM tags
|
|
146729
|
+
WHERE session_id = ? AND status = 'active'
|
|
146730
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
146731
|
+
)` : "";
|
|
146732
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
146733
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
146734
|
+
FROM tags
|
|
146735
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
146736
|
+
ORDER BY tag_number ASC, id ASC
|
|
146737
|
+
LIMIT ?`).all(...params);
|
|
146738
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
146739
|
+
tagNumber: row.tag_number,
|
|
146740
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
146741
|
+
}));
|
|
146742
|
+
}
|
|
146687
146743
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
146688
146744
|
const row = db.prepare(`SELECT
|
|
146689
146745
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -147736,13 +147792,13 @@ async function maybeSendUpgradeReminder(deps, sessionId) {
|
|
|
147736
147792
|
init_data_path();
|
|
147737
147793
|
import * as fs2 from "node:fs";
|
|
147738
147794
|
import * as path6 from "node:path";
|
|
147739
|
-
var ANNOUNCEMENT_VERSION = "0.
|
|
147795
|
+
var ANNOUNCEMENT_VERSION = "0.25.0";
|
|
147740
147796
|
var ANNOUNCEMENT_FEATURES = [
|
|
147741
|
-
"
|
|
147742
|
-
"
|
|
147743
|
-
"
|
|
147744
|
-
"
|
|
147745
|
-
"
|
|
147797
|
+
"Old tool output is now reclaimed automatically: once a file read / search / command output has gone a full execute cycle unused, it's dropped on the next one — no need to call ctx_reduce for stale results.",
|
|
147798
|
+
"Recover anything that was dropped: ctx_expand({ message: N }) returns a dropped message's full content (every tool call's input + output) from storage. ctx_expand({ start, end, verbose: true }) lists a range message-by-message to find it.",
|
|
147799
|
+
"Searchable history made reliable: /ctx-embed shows embedding coverage and runs a resilient backfill (retries transient failures, no longer bails on the first hiccup); the active session now auto-embeds in the background. ctx_reduce guidance also reframed as deferred + recoverable so models trim spent output earlier.",
|
|
147800
|
+
"Pi: fixed /ctx-dream (was failing with 'Unknown named parameter') and local-embedding load failures on Windows/Desktop (#151, #128).",
|
|
147801
|
+
"Runaway background agents on weak/local models are now capped and force-stopped (#154, #152). Plus several prompt-cache busts removed."
|
|
147746
147802
|
];
|
|
147747
147803
|
var ANNOUNCEMENT_FOOTER = "Join us on Discord: https://discord.gg/F2uWxjGnU";
|
|
147748
147804
|
var STATE_FILENAME = "last_announced_version";
|
|
@@ -148686,7 +148742,7 @@ function enqueueDream(db, projectIdentity, reason, force = false) {
|
|
|
148686
148742
|
return db.transaction(() => {
|
|
148687
148743
|
if (!hasActiveDreamLease(db)) {
|
|
148688
148744
|
const staleThresholdMs = force ? 2 * 60 * 1000 : 120 * 60 * 1000;
|
|
148689
|
-
db.prepare("DELETE FROM dream_queue WHERE project_path = ? AND started_at IS NOT NULL AND started_at < ?").run(
|
|
148745
|
+
db.prepare("DELETE FROM dream_queue WHERE project_path = ? AND started_at IS NOT NULL AND started_at < ?").run(projectIdentity, now - staleThresholdMs);
|
|
148690
148746
|
}
|
|
148691
148747
|
const existing = db.prepare("SELECT id FROM dream_queue WHERE project_path = ?").get(projectIdentity);
|
|
148692
148748
|
if (existing) {
|
|
@@ -148821,9 +148877,11 @@ async function promptWithTimeout(client, args, timeoutMs, signal) {
|
|
|
148821
148877
|
});
|
|
148822
148878
|
} catch (error) {
|
|
148823
148879
|
if (signal?.aborted) {
|
|
148880
|
+
await abortChildRun(client, args.path.id);
|
|
148824
148881
|
throw new Error("prompt aborted by external signal");
|
|
148825
148882
|
}
|
|
148826
148883
|
if (controller.signal.aborted) {
|
|
148884
|
+
await abortChildRun(client, args.path.id);
|
|
148827
148885
|
throw new Error(`prompt timed out after ${timeoutMs}ms`);
|
|
148828
148886
|
}
|
|
148829
148887
|
throw error;
|
|
@@ -148832,6 +148890,13 @@ async function promptWithTimeout(client, args, timeoutMs, signal) {
|
|
|
148832
148890
|
signal?.removeEventListener("abort", onExternalAbort);
|
|
148833
148891
|
}
|
|
148834
148892
|
}
|
|
148893
|
+
async function abortChildRun(client, sessionId) {
|
|
148894
|
+
try {
|
|
148895
|
+
await client.session.abort({ path: { id: sessionId } });
|
|
148896
|
+
} catch (error) {
|
|
148897
|
+
log(`[model-retry] child session abort failed for ${sessionId}: ${String(error)}`);
|
|
148898
|
+
}
|
|
148899
|
+
}
|
|
148835
148900
|
function isNonRetryable(error, externalSignal) {
|
|
148836
148901
|
if (externalSignal?.aborted)
|
|
148837
148902
|
return true;
|
|
@@ -149688,6 +149753,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
149688
149753
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
149689
149754
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
149690
149755
|
}
|
|
149756
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
149757
|
+
const row = db.prepare(`SELECT
|
|
149758
|
+
COUNT(*) AS total,
|
|
149759
|
+
SUM(CASE WHEN EXISTS (
|
|
149760
|
+
SELECT 1 FROM memory_embeddings e
|
|
149761
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
149762
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
149763
|
+
FROM memories m
|
|
149764
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
149765
|
+
return {
|
|
149766
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
149767
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
149768
|
+
};
|
|
149769
|
+
}
|
|
149691
149770
|
|
|
149692
149771
|
// ../plugin/src/features/magic-context/memory/embedding-cache.ts
|
|
149693
149772
|
var DEFAULT_EMBEDDING_CACHE_TTL_MS = 60000;
|
|
@@ -166293,6 +166372,7 @@ init_logger();
|
|
|
166293
166372
|
// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
|
|
166294
166373
|
import { createHash as createHash8 } from "node:crypto";
|
|
166295
166374
|
var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
|
|
166375
|
+
var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
|
|
166296
166376
|
var loadFtsRowsStatements = new WeakMap;
|
|
166297
166377
|
var existingHashStatements = new WeakMap;
|
|
166298
166378
|
var existingHashByProjectStatements = new WeakMap;
|
|
@@ -166528,9 +166608,10 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
166528
166608
|
if (lines.length === 0 || endOrdinal < startOrdinal)
|
|
166529
166609
|
return [];
|
|
166530
166610
|
const normalizedMax = normalizeCompartmentChunkMaxInputTokens(maxInputTokens);
|
|
166611
|
+
const effectiveMax = Math.max(1, Math.floor(normalizedMax * CHUNK_WINDOW_SAFETY_RATIO));
|
|
166531
166612
|
const fullText = lines.join(`
|
|
166532
166613
|
`);
|
|
166533
|
-
if (estimateTokens(fullText) <=
|
|
166614
|
+
if (estimateTokens(fullText) <= effectiveMax) {
|
|
166534
166615
|
return [
|
|
166535
166616
|
{
|
|
166536
166617
|
windowIndex: 0,
|
|
@@ -166568,7 +166649,7 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
166568
166649
|
const lineStart = range?.start ?? startOrdinal;
|
|
166569
166650
|
const lineEnd = range?.end ?? lineStart;
|
|
166570
166651
|
const lineTokens = estimateTokens(line);
|
|
166571
|
-
if (currentLines.length > 0 && currentTokens + lineTokens >
|
|
166652
|
+
if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
|
|
166572
166653
|
flush2();
|
|
166573
166654
|
}
|
|
166574
166655
|
if (currentLines.length === 0) {
|
|
@@ -166724,6 +166805,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
166724
166805
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
166725
166806
|
return typeof row?.n === "number" ? row.n : 0;
|
|
166726
166807
|
}
|
|
166808
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
166809
|
+
const row = db.prepare(`SELECT
|
|
166810
|
+
COUNT(*) AS total,
|
|
166811
|
+
SUM(CASE WHEN EXISTS (
|
|
166812
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
166813
|
+
WHERE e.compartment_id = c.id
|
|
166814
|
+
AND e.project_path = ?
|
|
166815
|
+
AND e.model_id = ?
|
|
166816
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
166817
|
+
FROM compartments c
|
|
166818
|
+
JOIN session_projects sp
|
|
166819
|
+
ON sp.session_id = c.session_id
|
|
166820
|
+
AND sp.harness = c.harness
|
|
166821
|
+
AND sp.project_path = ?
|
|
166822
|
+
WHERE c.session_id = ?
|
|
166823
|
+
AND c.start_message IS NOT NULL
|
|
166824
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
166825
|
+
return {
|
|
166826
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
166827
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
166828
|
+
};
|
|
166829
|
+
}
|
|
166727
166830
|
|
|
166728
166831
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
166729
166832
|
function cosineSimilarity(a, b) {
|
|
@@ -166866,6 +166969,19 @@ async function withQuietConsole(fn) {
|
|
|
166866
166969
|
console.error = origError;
|
|
166867
166970
|
}
|
|
166868
166971
|
}
|
|
166972
|
+
var nativeRuntimeMissing = false;
|
|
166973
|
+
function isNativeRuntimeMissingError(error51) {
|
|
166974
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
166975
|
+
const lower = message.toLowerCase();
|
|
166976
|
+
const code = error51?.code;
|
|
166977
|
+
const name2 = error51?.name;
|
|
166978
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
166979
|
+
return true;
|
|
166980
|
+
}
|
|
166981
|
+
if (!lower.includes("onnxruntime-node"))
|
|
166982
|
+
return false;
|
|
166983
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
166984
|
+
}
|
|
166869
166985
|
function isTransientLoadError(error51) {
|
|
166870
166986
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
166871
166987
|
if (!message)
|
|
@@ -166930,6 +167046,9 @@ class LocalEmbeddingProvider {
|
|
|
166930
167046
|
if (this.pipeline) {
|
|
166931
167047
|
return true;
|
|
166932
167048
|
}
|
|
167049
|
+
if (nativeRuntimeMissing) {
|
|
167050
|
+
return false;
|
|
167051
|
+
}
|
|
166933
167052
|
if (this.initPromise) {
|
|
166934
167053
|
await this.initPromise;
|
|
166935
167054
|
return this.pipeline !== null;
|
|
@@ -166996,7 +167115,12 @@ class LocalEmbeddingProvider {
|
|
|
166996
167115
|
await releaseLock();
|
|
166997
167116
|
}
|
|
166998
167117
|
} catch (error51) {
|
|
166999
|
-
|
|
167118
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
167119
|
+
nativeRuntimeMissing = true;
|
|
167120
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @cortexkit/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
167121
|
+
} else {
|
|
167122
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
167123
|
+
}
|
|
167000
167124
|
this.pipeline = null;
|
|
167001
167125
|
} finally {
|
|
167002
167126
|
this.initPromise = null;
|
|
@@ -167168,6 +167292,13 @@ function blockedEmbeddingEndpointReason(endpoint) {
|
|
|
167168
167292
|
function normalizeEndpoint2(endpoint) {
|
|
167169
167293
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
167170
167294
|
}
|
|
167295
|
+
function embeddingModelsMatch(served, requested) {
|
|
167296
|
+
const a = served.trim().toLowerCase();
|
|
167297
|
+
const b = requested.trim().toLowerCase();
|
|
167298
|
+
if (a.length === 0 || b.length === 0)
|
|
167299
|
+
return true;
|
|
167300
|
+
return a === b || a.includes(b) || b.includes(a);
|
|
167301
|
+
}
|
|
167171
167302
|
var FAILURE_THRESHOLD = 3;
|
|
167172
167303
|
var FAILURE_WINDOW_MS = 60000;
|
|
167173
167304
|
var OPEN_DURATION_MS = 5 * 60000;
|
|
@@ -167185,6 +167316,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
167185
167316
|
failureTimes = [];
|
|
167186
167317
|
circuitOpenUntil = 0;
|
|
167187
167318
|
openLogged = false;
|
|
167319
|
+
modelMismatchLogged = false;
|
|
167188
167320
|
halfOpenProbeInFlight = false;
|
|
167189
167321
|
constructor(options) {
|
|
167190
167322
|
this.endpoint = normalizeEndpoint2(options.endpoint);
|
|
@@ -167283,6 +167415,15 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
167283
167415
|
this.recordFailure(isProbe);
|
|
167284
167416
|
return Array.from({ length: texts.length }, () => null);
|
|
167285
167417
|
}
|
|
167418
|
+
const servedModel = typeof body.model === "string" ? body.model : "";
|
|
167419
|
+
if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
|
|
167420
|
+
if (!this.modelMismatchLogged) {
|
|
167421
|
+
log(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${this.model}" served="${servedModel}". The endpoint likely substituted a loaded model; load/select "${this.model}" on the endpoint, or set embedding.model to the served model.`);
|
|
167422
|
+
this.modelMismatchLogged = true;
|
|
167423
|
+
}
|
|
167424
|
+
this.recordFailure(isProbe);
|
|
167425
|
+
return Array.from({ length: texts.length }, () => null);
|
|
167426
|
+
}
|
|
167286
167427
|
const items = Array.isArray(body.data) ? body.data : [];
|
|
167287
167428
|
const results = Array.from({ length: texts.length }, (_, index) => {
|
|
167288
167429
|
const embedding = items[index]?.embedding;
|
|
@@ -167493,6 +167634,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
167493
167634
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
167494
167635
|
}
|
|
167495
167636
|
|
|
167637
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
167638
|
+
init_logger();
|
|
167639
|
+
var insertStatements = new WeakMap;
|
|
167640
|
+
var existingShasStatements = new WeakMap;
|
|
167641
|
+
var projectCountStatements = new WeakMap;
|
|
167642
|
+
var evictStatements = new WeakMap;
|
|
167643
|
+
var evictOverflowStatements = new WeakMap;
|
|
167644
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
167645
|
+
function getInsertStatement(db) {
|
|
167646
|
+
let stmt = insertStatements.get(db);
|
|
167647
|
+
if (!stmt) {
|
|
167648
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
167649
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
167650
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
167651
|
+
project_path = excluded.project_path,
|
|
167652
|
+
short_sha = excluded.short_sha,
|
|
167653
|
+
message = excluded.message,
|
|
167654
|
+
author = excluded.author,
|
|
167655
|
+
committed_at = excluded.committed_at,
|
|
167656
|
+
indexed_at = excluded.indexed_at
|
|
167657
|
+
WHERE git_commits.message != excluded.message`);
|
|
167658
|
+
insertStatements.set(db, stmt);
|
|
167659
|
+
}
|
|
167660
|
+
return stmt;
|
|
167661
|
+
}
|
|
167662
|
+
function getExistingShasStatement(db) {
|
|
167663
|
+
let stmt = existingShasStatements.get(db);
|
|
167664
|
+
if (!stmt) {
|
|
167665
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
167666
|
+
existingShasStatements.set(db, stmt);
|
|
167667
|
+
}
|
|
167668
|
+
return stmt;
|
|
167669
|
+
}
|
|
167670
|
+
function getProjectCountStatement(db) {
|
|
167671
|
+
let stmt = projectCountStatements.get(db);
|
|
167672
|
+
if (!stmt) {
|
|
167673
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
167674
|
+
projectCountStatements.set(db, stmt);
|
|
167675
|
+
}
|
|
167676
|
+
return stmt;
|
|
167677
|
+
}
|
|
167678
|
+
function getLatestCommitTimeStatement(db) {
|
|
167679
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
167680
|
+
if (!stmt) {
|
|
167681
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
167682
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
167683
|
+
}
|
|
167684
|
+
return stmt;
|
|
167685
|
+
}
|
|
167686
|
+
function getEvictOverflowStatement(db) {
|
|
167687
|
+
let stmt = evictOverflowStatements.get(db);
|
|
167688
|
+
if (!stmt) {
|
|
167689
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
167690
|
+
WHERE rowid IN (
|
|
167691
|
+
SELECT rowid FROM git_commits
|
|
167692
|
+
WHERE project_path = ?
|
|
167693
|
+
ORDER BY committed_at DESC, sha DESC
|
|
167694
|
+
LIMIT -1 OFFSET ?
|
|
167695
|
+
)`);
|
|
167696
|
+
evictOverflowStatements.set(db, stmt);
|
|
167697
|
+
}
|
|
167698
|
+
return stmt;
|
|
167699
|
+
}
|
|
167700
|
+
function upsertCommits(db, projectPath, commits) {
|
|
167701
|
+
if (commits.length === 0)
|
|
167702
|
+
return { inserted: 0, updated: 0 };
|
|
167703
|
+
const existing = new Set;
|
|
167704
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
167705
|
+
existing.add(row.sha);
|
|
167706
|
+
}
|
|
167707
|
+
let inserted = 0;
|
|
167708
|
+
let updated = 0;
|
|
167709
|
+
const now = Date.now();
|
|
167710
|
+
const insertStmt = getInsertStatement(db);
|
|
167711
|
+
db.transaction(() => {
|
|
167712
|
+
for (const commit of commits) {
|
|
167713
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
167714
|
+
if (result.changes > 0) {
|
|
167715
|
+
if (existing.has(commit.sha)) {
|
|
167716
|
+
updated++;
|
|
167717
|
+
} else {
|
|
167718
|
+
inserted++;
|
|
167719
|
+
existing.add(commit.sha);
|
|
167720
|
+
}
|
|
167721
|
+
}
|
|
167722
|
+
}
|
|
167723
|
+
})();
|
|
167724
|
+
return { inserted, updated };
|
|
167725
|
+
}
|
|
167726
|
+
function getCommitCount(db, projectPath) {
|
|
167727
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
167728
|
+
return row?.count ?? 0;
|
|
167729
|
+
}
|
|
167730
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
167731
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
167732
|
+
return row?.latest ?? null;
|
|
167733
|
+
}
|
|
167734
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
167735
|
+
if (maxCommits <= 0)
|
|
167736
|
+
return 0;
|
|
167737
|
+
const count = getCommitCount(db, projectPath);
|
|
167738
|
+
if (count <= maxCommits)
|
|
167739
|
+
return 0;
|
|
167740
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
167741
|
+
const after = getCommitCount(db, projectPath);
|
|
167742
|
+
const evicted = Math.max(0, count - after);
|
|
167743
|
+
if (evicted > 0) {
|
|
167744
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
167745
|
+
}
|
|
167746
|
+
return evicted;
|
|
167747
|
+
}
|
|
167748
|
+
|
|
167496
167749
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
167497
167750
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
167498
167751
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -167687,8 +167940,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
167687
167940
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
167688
167941
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
167689
167942
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
167690
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
167943
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
167691
167944
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
167945
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
167946
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
167947
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
167948
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
167692
167949
|
var projectRegistrations = new Map;
|
|
167693
167950
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
167694
167951
|
var globalRegistrationGeneration = 0;
|
|
@@ -167765,7 +168022,7 @@ function getChunkEmbeddingModelId(config2, providerIdentity) {
|
|
|
167765
168022
|
}
|
|
167766
168023
|
const chunkIdentity = {
|
|
167767
168024
|
providerIdentity,
|
|
167768
|
-
chunkerVersion:
|
|
168025
|
+
chunkerVersion: 2,
|
|
167769
168026
|
maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config2 ? config2.max_input_tokens : undefined),
|
|
167770
168027
|
truncate: config2.provider === "openai-compatible" ? config2.truncate ?? "" : ""
|
|
167771
168028
|
};
|
|
@@ -167788,7 +168045,9 @@ function snapshotFor(registration) {
|
|
|
167788
168045
|
enabled,
|
|
167789
168046
|
gitCommitEnabled,
|
|
167790
168047
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
167791
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
168048
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
168049
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
168050
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
167792
168051
|
};
|
|
167793
168052
|
}
|
|
167794
168053
|
function disposeProvider(provider) {
|
|
@@ -167999,8 +168258,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
167999
168258
|
}
|
|
168000
168259
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
168001
168260
|
const noWork = [];
|
|
168261
|
+
const failed = [];
|
|
168002
168262
|
if (candidates.length === 0)
|
|
168003
|
-
return { embedded: 0, noWork };
|
|
168263
|
+
return { embedded: 0, noWork, failed };
|
|
168004
168264
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
168005
168265
|
const prepared = [];
|
|
168006
168266
|
for (const candidate of candidates) {
|
|
@@ -168017,7 +168277,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
168017
168277
|
prepared.push({ candidate, windows });
|
|
168018
168278
|
}
|
|
168019
168279
|
if (prepared.length === 0)
|
|
168020
|
-
return { embedded: 0, noWork };
|
|
168280
|
+
return { embedded: 0, noWork, failed };
|
|
168021
168281
|
let embedded = 0;
|
|
168022
168282
|
let i = 0;
|
|
168023
168283
|
while (i < prepared.length) {
|
|
@@ -168034,35 +168294,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
168034
168294
|
const texts = [];
|
|
168035
168295
|
for (const item of slice)
|
|
168036
168296
|
texts.push(...item.windows.map((w) => w.text));
|
|
168037
|
-
|
|
168038
|
-
|
|
168039
|
-
if (!result)
|
|
168040
|
-
continue;
|
|
168297
|
+
const persistedIds = new Set;
|
|
168298
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
168041
168299
|
if (signal?.aborted)
|
|
168042
168300
|
break;
|
|
168043
|
-
let
|
|
168044
|
-
|
|
168045
|
-
|
|
168046
|
-
|
|
168047
|
-
|
|
168048
|
-
|
|
168301
|
+
let result = null;
|
|
168302
|
+
const attemptStart = Date.now();
|
|
168303
|
+
try {
|
|
168304
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
168305
|
+
} catch (error51) {
|
|
168306
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
168307
|
+
}
|
|
168308
|
+
if (signal?.aborted)
|
|
168309
|
+
break;
|
|
168310
|
+
if (result) {
|
|
168311
|
+
let offset = 0;
|
|
168312
|
+
for (const item of slice) {
|
|
168313
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
168314
|
+
offset += item.windows.length;
|
|
168315
|
+
if (persistedIds.has(item.candidate.id))
|
|
168316
|
+
continue;
|
|
168317
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
168318
|
+
continue;
|
|
168319
|
+
}
|
|
168320
|
+
const rows = item.windows.map((window, index) => ({
|
|
168321
|
+
compartmentId: item.candidate.id,
|
|
168322
|
+
sessionId: item.candidate.sessionId,
|
|
168323
|
+
projectPath: projectIdentity,
|
|
168324
|
+
window,
|
|
168325
|
+
modelId,
|
|
168326
|
+
vector: vectors[index]
|
|
168327
|
+
}));
|
|
168328
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
168329
|
+
persistedIds.add(item.candidate.id);
|
|
168049
168330
|
}
|
|
168050
|
-
const rows = item.windows.map((window, index) => ({
|
|
168051
|
-
compartmentId: item.candidate.id,
|
|
168052
|
-
sessionId: item.candidate.sessionId,
|
|
168053
|
-
projectPath: projectIdentity,
|
|
168054
|
-
window,
|
|
168055
|
-
modelId,
|
|
168056
|
-
vector: vectors[index]
|
|
168057
|
-
}));
|
|
168058
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
168059
|
-
embedded += 1;
|
|
168060
168331
|
}
|
|
168061
|
-
|
|
168062
|
-
|
|
168332
|
+
if (persistedIds.size === slice.length)
|
|
168333
|
+
break;
|
|
168334
|
+
if (persistedIds.size > 0)
|
|
168335
|
+
break;
|
|
168336
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
168337
|
+
break;
|
|
168338
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
168339
|
+
await new Promise((resolve4) => setTimeout(resolve4, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
168340
|
+
}
|
|
168341
|
+
}
|
|
168342
|
+
embedded += persistedIds.size;
|
|
168343
|
+
if (!signal?.aborted) {
|
|
168344
|
+
for (const item of slice) {
|
|
168345
|
+
if (!persistedIds.has(item.candidate.id))
|
|
168346
|
+
failed.push(item.candidate.id);
|
|
168347
|
+
}
|
|
168063
168348
|
}
|
|
168064
168349
|
}
|
|
168065
|
-
return { embedded, noWork };
|
|
168350
|
+
return { embedded, noWork, failed };
|
|
168066
168351
|
}
|
|
168067
168352
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
168068
168353
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -168085,9 +168370,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168085
168370
|
renewal.unref?.();
|
|
168086
168371
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
168087
168372
|
const skipIds = [];
|
|
168373
|
+
const failedIds = [];
|
|
168088
168374
|
let embedded = 0;
|
|
168089
168375
|
let aborted2 = false;
|
|
168090
|
-
let
|
|
168376
|
+
let providerDown = false;
|
|
168377
|
+
let consecutiveFailedBatches = 0;
|
|
168091
168378
|
try {
|
|
168092
168379
|
options?.onProgress?.({ embedded, total });
|
|
168093
168380
|
for (;; ) {
|
|
@@ -168095,15 +168382,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168095
168382
|
aborted2 = true;
|
|
168096
168383
|
break;
|
|
168097
168384
|
}
|
|
168098
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
168385
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
168099
168386
|
if (candidates.length === 0)
|
|
168100
168387
|
break;
|
|
168101
|
-
const {
|
|
168388
|
+
const {
|
|
168389
|
+
embedded: n,
|
|
168390
|
+
noWork,
|
|
168391
|
+
failed
|
|
168392
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
168102
168393
|
for (const id of noWork)
|
|
168103
168394
|
skipIds.push(id);
|
|
168395
|
+
for (const id of failed)
|
|
168396
|
+
failedIds.push(id);
|
|
168104
168397
|
if (n === 0 && noWork.length === 0) {
|
|
168105
|
-
|
|
168106
|
-
|
|
168398
|
+
consecutiveFailedBatches += 1;
|
|
168399
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
168400
|
+
providerDown = true;
|
|
168401
|
+
break;
|
|
168402
|
+
}
|
|
168403
|
+
} else {
|
|
168404
|
+
consecutiveFailedBatches = 0;
|
|
168107
168405
|
}
|
|
168108
168406
|
embedded += n;
|
|
168109
168407
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -168111,16 +168409,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168111
168409
|
}
|
|
168112
168410
|
} finally {
|
|
168113
168411
|
clearInterval(renewal);
|
|
168114
|
-
|
|
168412
|
+
try {
|
|
168413
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
168414
|
+
} catch (error51) {
|
|
168415
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
168416
|
+
}
|
|
168115
168417
|
}
|
|
168116
168418
|
if (aborted2)
|
|
168117
|
-
return { status: "aborted", embedded, total };
|
|
168118
|
-
if (
|
|
168419
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
168420
|
+
if (providerDown || failedIds.length > 0) {
|
|
168119
168421
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
168120
|
-
if (remaining > 0)
|
|
168121
|
-
return { status: "stalled", embedded, total, remaining };
|
|
168422
|
+
if (remaining > 0) {
|
|
168423
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
168424
|
+
}
|
|
168122
168425
|
}
|
|
168123
|
-
return { status: "done", embedded, total };
|
|
168426
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
168427
|
+
}
|
|
168428
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
168429
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
168430
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
168431
|
+
return {
|
|
168432
|
+
enabled: false,
|
|
168433
|
+
model: snapshot?.model ?? "off",
|
|
168434
|
+
provider: snapshot?.provider ?? "off",
|
|
168435
|
+
session: { embedded: 0, total: 0 },
|
|
168436
|
+
memories: { embedded: 0, total: 0 },
|
|
168437
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
168438
|
+
};
|
|
168439
|
+
}
|
|
168440
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
168441
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
168442
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
168443
|
+
const commits = gitEnabled ? {
|
|
168444
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
168445
|
+
total: getCommitCount(db, projectIdentity),
|
|
168446
|
+
gitEnabled: true
|
|
168447
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
168448
|
+
return {
|
|
168449
|
+
enabled: true,
|
|
168450
|
+
model: snapshot.model,
|
|
168451
|
+
provider: snapshot.provider,
|
|
168452
|
+
session,
|
|
168453
|
+
memories,
|
|
168454
|
+
commits
|
|
168455
|
+
};
|
|
168124
168456
|
}
|
|
168125
168457
|
|
|
168126
168458
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -168169,118 +168501,6 @@ async function embedText(text, signal) {
|
|
|
168169
168501
|
}
|
|
168170
168502
|
var SWEEP_MAX_WALL_CLOCK_MS2 = 10 * 60 * 1000;
|
|
168171
168503
|
|
|
168172
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
168173
|
-
init_logger();
|
|
168174
|
-
var insertStatements = new WeakMap;
|
|
168175
|
-
var existingShasStatements = new WeakMap;
|
|
168176
|
-
var projectCountStatements = new WeakMap;
|
|
168177
|
-
var evictStatements = new WeakMap;
|
|
168178
|
-
var evictOverflowStatements = new WeakMap;
|
|
168179
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
168180
|
-
function getInsertStatement(db) {
|
|
168181
|
-
let stmt = insertStatements.get(db);
|
|
168182
|
-
if (!stmt) {
|
|
168183
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
168184
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
168185
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
168186
|
-
project_path = excluded.project_path,
|
|
168187
|
-
short_sha = excluded.short_sha,
|
|
168188
|
-
message = excluded.message,
|
|
168189
|
-
author = excluded.author,
|
|
168190
|
-
committed_at = excluded.committed_at,
|
|
168191
|
-
indexed_at = excluded.indexed_at
|
|
168192
|
-
WHERE git_commits.message != excluded.message`);
|
|
168193
|
-
insertStatements.set(db, stmt);
|
|
168194
|
-
}
|
|
168195
|
-
return stmt;
|
|
168196
|
-
}
|
|
168197
|
-
function getExistingShasStatement(db) {
|
|
168198
|
-
let stmt = existingShasStatements.get(db);
|
|
168199
|
-
if (!stmt) {
|
|
168200
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
168201
|
-
existingShasStatements.set(db, stmt);
|
|
168202
|
-
}
|
|
168203
|
-
return stmt;
|
|
168204
|
-
}
|
|
168205
|
-
function getProjectCountStatement(db) {
|
|
168206
|
-
let stmt = projectCountStatements.get(db);
|
|
168207
|
-
if (!stmt) {
|
|
168208
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
168209
|
-
projectCountStatements.set(db, stmt);
|
|
168210
|
-
}
|
|
168211
|
-
return stmt;
|
|
168212
|
-
}
|
|
168213
|
-
function getLatestCommitTimeStatement(db) {
|
|
168214
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
168215
|
-
if (!stmt) {
|
|
168216
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
168217
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
168218
|
-
}
|
|
168219
|
-
return stmt;
|
|
168220
|
-
}
|
|
168221
|
-
function getEvictOverflowStatement(db) {
|
|
168222
|
-
let stmt = evictOverflowStatements.get(db);
|
|
168223
|
-
if (!stmt) {
|
|
168224
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
168225
|
-
WHERE rowid IN (
|
|
168226
|
-
SELECT rowid FROM git_commits
|
|
168227
|
-
WHERE project_path = ?
|
|
168228
|
-
ORDER BY committed_at DESC, sha DESC
|
|
168229
|
-
LIMIT -1 OFFSET ?
|
|
168230
|
-
)`);
|
|
168231
|
-
evictOverflowStatements.set(db, stmt);
|
|
168232
|
-
}
|
|
168233
|
-
return stmt;
|
|
168234
|
-
}
|
|
168235
|
-
function upsertCommits(db, projectPath, commits) {
|
|
168236
|
-
if (commits.length === 0)
|
|
168237
|
-
return { inserted: 0, updated: 0 };
|
|
168238
|
-
const existing = new Set;
|
|
168239
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
168240
|
-
existing.add(row.sha);
|
|
168241
|
-
}
|
|
168242
|
-
let inserted = 0;
|
|
168243
|
-
let updated = 0;
|
|
168244
|
-
const now = Date.now();
|
|
168245
|
-
const insertStmt = getInsertStatement(db);
|
|
168246
|
-
db.transaction(() => {
|
|
168247
|
-
for (const commit of commits) {
|
|
168248
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
168249
|
-
if (result.changes > 0) {
|
|
168250
|
-
if (existing.has(commit.sha)) {
|
|
168251
|
-
updated++;
|
|
168252
|
-
} else {
|
|
168253
|
-
inserted++;
|
|
168254
|
-
existing.add(commit.sha);
|
|
168255
|
-
}
|
|
168256
|
-
}
|
|
168257
|
-
}
|
|
168258
|
-
})();
|
|
168259
|
-
return { inserted, updated };
|
|
168260
|
-
}
|
|
168261
|
-
function getCommitCount(db, projectPath) {
|
|
168262
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
168263
|
-
return row?.count ?? 0;
|
|
168264
|
-
}
|
|
168265
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
168266
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
168267
|
-
return row?.latest ?? null;
|
|
168268
|
-
}
|
|
168269
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
168270
|
-
if (maxCommits <= 0)
|
|
168271
|
-
return 0;
|
|
168272
|
-
const count = getCommitCount(db, projectPath);
|
|
168273
|
-
if (count <= maxCommits)
|
|
168274
|
-
return 0;
|
|
168275
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
168276
|
-
const after = getCommitCount(db, projectPath);
|
|
168277
|
-
const evicted = Math.max(0, count - after);
|
|
168278
|
-
if (evicted > 0) {
|
|
168279
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
168280
|
-
}
|
|
168281
|
-
return evicted;
|
|
168282
|
-
}
|
|
168283
|
-
|
|
168284
168504
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
168285
168505
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
168286
168506
|
var EMBED_BATCH_SIZE = 16;
|
|
@@ -169810,87 +170030,229 @@ function registerCtxDreamCommand(pi, deps) {
|
|
|
169810
170030
|
});
|
|
169811
170031
|
}
|
|
169812
170032
|
|
|
169813
|
-
// src/
|
|
169814
|
-
|
|
169815
|
-
|
|
169816
|
-
|
|
169817
|
-
|
|
170033
|
+
// ../plugin/src/hooks/magic-context/embed-session-state.ts
|
|
170034
|
+
var embedPauseBySession = new Set;
|
|
170035
|
+
var embedRunStateBySession = new Map;
|
|
170036
|
+
var autoEmbedAttemptedBySession = new Set;
|
|
170037
|
+
|
|
170038
|
+
// ../plugin/src/hooks/magic-context/format-embed-status.ts
|
|
170039
|
+
function formatEmbedStatusText(coverage, drain) {
|
|
170040
|
+
if (!coverage.enabled) {
|
|
170041
|
+
return "Embedding is off (no provider configured).";
|
|
170042
|
+
}
|
|
170043
|
+
const lines = [];
|
|
170044
|
+
lines.push(`Embedding — model: ${coverage.model} (${coverage.provider})`);
|
|
170045
|
+
lines.push(`This session: ${coverage.session.embedded} / ${coverage.session.total} compartments embedded`);
|
|
170046
|
+
lines.push(`Project memories: ${coverage.memories.embedded} / ${coverage.memories.total} embedded`);
|
|
170047
|
+
if (coverage.commits.gitEnabled) {
|
|
170048
|
+
lines.push(`Git commits: ${coverage.commits.embedded} / ${coverage.commits.total}`);
|
|
170049
|
+
} else {
|
|
170050
|
+
lines.push("Git commits: 0 / 0 (git indexing off)");
|
|
170051
|
+
}
|
|
170052
|
+
let drainLine = "Drain: idle";
|
|
170053
|
+
switch (drain.status) {
|
|
170054
|
+
case "running": {
|
|
170055
|
+
const e = drain.embedded ?? coverage.session.embedded;
|
|
170056
|
+
const t = drain.total ?? coverage.session.total;
|
|
170057
|
+
const failedSuffix = drain.failed && drain.failed > 0 ? ` (${drain.failed} failed)` : "";
|
|
170058
|
+
drainLine = `Drain: running ${e}/${t}${failedSuffix}`;
|
|
170059
|
+
break;
|
|
170060
|
+
}
|
|
170061
|
+
case "paused": {
|
|
170062
|
+
const e = drain.embedded ?? coverage.session.embedded;
|
|
170063
|
+
const t = drain.total ?? coverage.session.total;
|
|
170064
|
+
drainLine = `Drain: paused ${e}/${t}`;
|
|
170065
|
+
break;
|
|
170066
|
+
}
|
|
170067
|
+
case "stopped":
|
|
170068
|
+
drainLine = "Drain: stopped (provider down)";
|
|
170069
|
+
break;
|
|
170070
|
+
default:
|
|
170071
|
+
drainLine = "Drain: idle";
|
|
170072
|
+
}
|
|
170073
|
+
lines.push(drainLine);
|
|
170074
|
+
return lines.join(`
|
|
170075
|
+
`);
|
|
170076
|
+
}
|
|
170077
|
+
|
|
170078
|
+
// src/commands/ctx-embed.ts
|
|
170079
|
+
function clearPiEmbedSessionState(sessionId) {
|
|
170080
|
+
embedPauseBySession.delete(sessionId);
|
|
170081
|
+
const ctrl = embedRunStateBySession.get(sessionId);
|
|
170082
|
+
if (ctrl) {
|
|
170083
|
+
ctrl.abort();
|
|
170084
|
+
embedRunStateBySession.delete(sessionId);
|
|
170085
|
+
}
|
|
170086
|
+
autoEmbedAttemptedBySession.delete(sessionId);
|
|
170087
|
+
}
|
|
170088
|
+
async function runEmbedDrain(db, projectIdentity, sessionId) {
|
|
170089
|
+
const activeCtrl = embedRunStateBySession.get(sessionId);
|
|
170090
|
+
if (activeCtrl && !activeCtrl.signal.aborted) {
|
|
170091
|
+
return {
|
|
170092
|
+
text: `## /ctx-embed
|
|
170093
|
+
|
|
170094
|
+
Embedding is already running for this session.`,
|
|
170095
|
+
level: "info"
|
|
170096
|
+
};
|
|
170097
|
+
}
|
|
170098
|
+
embedPauseBySession.delete(sessionId);
|
|
170099
|
+
const prior = embedRunStateBySession.get(sessionId);
|
|
170100
|
+
if (prior)
|
|
170101
|
+
prior.abort();
|
|
170102
|
+
const controller = new AbortController;
|
|
170103
|
+
embedRunStateBySession.set(sessionId, controller);
|
|
170104
|
+
let outcome;
|
|
170105
|
+
try {
|
|
170106
|
+
outcome = await embedSessionCompartmentChunks(db, projectIdentity, sessionId, {
|
|
170107
|
+
signal: controller.signal
|
|
170108
|
+
});
|
|
170109
|
+
} finally {
|
|
170110
|
+
if (embedRunStateBySession.get(sessionId) === controller) {
|
|
170111
|
+
embedRunStateBySession.delete(sessionId);
|
|
170112
|
+
}
|
|
170113
|
+
}
|
|
170114
|
+
switch (outcome.status) {
|
|
170115
|
+
case "nothing":
|
|
170116
|
+
return {
|
|
170117
|
+
text: `## /ctx-embed
|
|
170118
|
+
|
|
170119
|
+
All of this session's history is already embedded.`,
|
|
170120
|
+
level: "info"
|
|
170121
|
+
};
|
|
170122
|
+
case "disabled":
|
|
170123
|
+
return {
|
|
170124
|
+
text: `## /ctx-embed
|
|
170125
|
+
|
|
170126
|
+
No embedding provider is configured, so there is nothing to embed.`,
|
|
170127
|
+
level: "info"
|
|
170128
|
+
};
|
|
170129
|
+
case "busy":
|
|
170130
|
+
return {
|
|
170131
|
+
text: `## /ctx-embed
|
|
170132
|
+
|
|
170133
|
+
Embedding is already running for this project. Try again shortly.`,
|
|
170134
|
+
level: "info"
|
|
170135
|
+
};
|
|
170136
|
+
case "aborted": {
|
|
170137
|
+
const cov = getEmbeddingCoverageStatus(db, projectIdentity, sessionId);
|
|
170138
|
+
return {
|
|
170139
|
+
text: `## /ctx-embed
|
|
170140
|
+
|
|
170141
|
+
Paused at ${cov.session.embedded}/${cov.session.total} compartments embedded.`,
|
|
170142
|
+
level: "info"
|
|
170143
|
+
};
|
|
170144
|
+
}
|
|
170145
|
+
case "stalled":
|
|
170146
|
+
return {
|
|
170147
|
+
text: `## /ctx-embed
|
|
170148
|
+
|
|
170149
|
+
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"}; ${outcome.remaining} could not be embedded (the provider returned no result). Run /ctx-embed start again to retry them.`,
|
|
170150
|
+
level: "info"
|
|
170151
|
+
};
|
|
170152
|
+
default:
|
|
170153
|
+
return {
|
|
170154
|
+
text: `## /ctx-embed
|
|
170155
|
+
|
|
170156
|
+
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"} of history for semantic search.`,
|
|
170157
|
+
level: "success"
|
|
170158
|
+
};
|
|
170159
|
+
}
|
|
170160
|
+
}
|
|
170161
|
+
function registerCtxEmbedCommand(pi, deps) {
|
|
170162
|
+
pi.registerCommand("ctx-embed", {
|
|
170163
|
+
description: "Embedding status, or start/pause history compartment embedding (start | pause)",
|
|
170164
|
+
handler: async (args, ctx) => {
|
|
169818
170165
|
const sessionId = resolveSessionId(ctx);
|
|
169819
170166
|
if (!sessionId) {
|
|
169820
170167
|
sendCtxStatusMessage(pi, {
|
|
169821
|
-
title: "/ctx-embed
|
|
169822
|
-
text: `## /ctx-embed
|
|
170168
|
+
title: "/ctx-embed",
|
|
170169
|
+
text: `## /ctx-embed
|
|
169823
170170
|
|
|
169824
170171
|
No active Pi session is available.`,
|
|
169825
170172
|
level: "error"
|
|
169826
170173
|
});
|
|
169827
170174
|
return;
|
|
169828
170175
|
}
|
|
170176
|
+
const project = deps.resolveProject?.(ctx) ?? {
|
|
170177
|
+
projectDir: deps.projectDir,
|
|
170178
|
+
projectIdentity: deps.projectIdentity
|
|
170179
|
+
};
|
|
170180
|
+
const sub = args.trim().toLowerCase();
|
|
170181
|
+
if (sub === "pause") {
|
|
170182
|
+
embedPauseBySession.add(sessionId);
|
|
170183
|
+
const ctrl = embedRunStateBySession.get(sessionId);
|
|
170184
|
+
if (ctrl)
|
|
170185
|
+
ctrl.abort();
|
|
170186
|
+
const cov = getEmbeddingCoverageStatus(deps.db, project.projectIdentity, sessionId);
|
|
170187
|
+
sendCtxStatusMessage(pi, {
|
|
170188
|
+
title: "/ctx-embed",
|
|
170189
|
+
text: `## /ctx-embed
|
|
170190
|
+
|
|
170191
|
+
Paused at ${cov.session.embedded}/${cov.session.total} compartments embedded.`,
|
|
170192
|
+
level: "info"
|
|
170193
|
+
});
|
|
170194
|
+
return;
|
|
170195
|
+
}
|
|
169829
170196
|
if (deps.memoryEnabled === false) {
|
|
169830
170197
|
sendCtxStatusMessage(pi, {
|
|
169831
|
-
title: "/ctx-embed
|
|
169832
|
-
text: `## /ctx-embed
|
|
170198
|
+
title: "/ctx-embed",
|
|
170199
|
+
text: `## /ctx-embed
|
|
169833
170200
|
|
|
169834
170201
|
Memory is disabled for this project, so there is no semantic embedding to backfill.`,
|
|
169835
170202
|
level: "info"
|
|
169836
170203
|
});
|
|
169837
170204
|
return;
|
|
169838
170205
|
}
|
|
169839
|
-
const project = deps.resolveProject?.(ctx) ?? {
|
|
169840
|
-
projectDir: deps.projectDir,
|
|
169841
|
-
projectIdentity: deps.projectIdentity
|
|
169842
|
-
};
|
|
169843
170206
|
await ensureProjectRegisteredFromPiDirectory(project.projectDir, deps.db);
|
|
169844
|
-
|
|
169845
|
-
|
|
169846
|
-
|
|
169847
|
-
|
|
169848
|
-
|
|
169849
|
-
|
|
169850
|
-
|
|
169851
|
-
|
|
169852
|
-
|
|
169853
|
-
|
|
169854
|
-
|
|
169855
|
-
|
|
169856
|
-
|
|
169857
|
-
|
|
169858
|
-
|
|
169859
|
-
|
|
169860
|
-
|
|
169861
|
-
|
|
169862
|
-
return {
|
|
169863
|
-
text: `## /ctx-embed-history
|
|
169864
|
-
|
|
169865
|
-
Embedding is already running for this project — ${outcome.total} compartment${outcome.total === 1 ? "" : "s"} still pending. Try again shortly.`,
|
|
169866
|
-
level: "info"
|
|
169867
|
-
};
|
|
169868
|
-
case "stalled":
|
|
169869
|
-
return {
|
|
169870
|
-
text: `## /ctx-embed-history
|
|
169871
|
-
|
|
169872
|
-
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"}; ${outcome.remaining} could not be embedded (the provider returned no result). Run /ctx-embed-history again to retry them.`,
|
|
169873
|
-
level: "info"
|
|
169874
|
-
};
|
|
169875
|
-
default:
|
|
169876
|
-
return {
|
|
169877
|
-
text: `## /ctx-embed-history
|
|
170207
|
+
if (sub === "start") {
|
|
170208
|
+
const { text, level } = await runEmbedDrain(deps.db, project.projectIdentity, sessionId);
|
|
170209
|
+
sendCtxStatusMessage(pi, { title: "/ctx-embed", text, level });
|
|
170210
|
+
return;
|
|
170211
|
+
}
|
|
170212
|
+
if (sub !== "") {
|
|
170213
|
+
sendCtxStatusMessage(pi, {
|
|
170214
|
+
title: "/ctx-embed",
|
|
170215
|
+
text: "## /ctx-embed\n\nUsage: `/ctx-embed` (status), `/ctx-embed start`, or `/ctx-embed pause`.",
|
|
170216
|
+
level: "info"
|
|
170217
|
+
});
|
|
170218
|
+
return;
|
|
170219
|
+
}
|
|
170220
|
+
const coverage = getEmbeddingCoverageStatus(deps.db, project.projectIdentity, sessionId);
|
|
170221
|
+
const statusText = formatEmbedStatusText(coverage, { status: "idle" });
|
|
170222
|
+
sendCtxStatusMessage(pi, {
|
|
170223
|
+
title: "/ctx-embed",
|
|
170224
|
+
text: `## Embedding Status
|
|
169878
170225
|
|
|
169879
|
-
|
|
169880
|
-
|
|
169881
|
-
};
|
|
169882
|
-
}
|
|
169883
|
-
})();
|
|
169884
|
-
sendCtxStatusMessage(pi, { title: "/ctx-embed-history", text, level }, {
|
|
169885
|
-
sessionId,
|
|
169886
|
-
projectIdentity: project.projectIdentity,
|
|
169887
|
-
status: outcome.status,
|
|
169888
|
-
embedded: outcome.embedded,
|
|
169889
|
-
total: outcome.total
|
|
170226
|
+
${statusText}`,
|
|
170227
|
+
level: "info"
|
|
169890
170228
|
});
|
|
169891
170229
|
}
|
|
169892
170230
|
});
|
|
169893
170231
|
}
|
|
170232
|
+
function maybeAutoEmbedPiSession(deps, sessionId, projectDir, projectIdentity, notify) {
|
|
170233
|
+
if (autoEmbedAttemptedBySession.has(sessionId))
|
|
170234
|
+
return;
|
|
170235
|
+
if (embedPauseBySession.has(sessionId))
|
|
170236
|
+
return;
|
|
170237
|
+
if (deps.memoryEnabled === false)
|
|
170238
|
+
return;
|
|
170239
|
+
autoEmbedAttemptedBySession.add(sessionId);
|
|
170240
|
+
(async () => {
|
|
170241
|
+
try {
|
|
170242
|
+
await new Promise((resolve5) => setTimeout(resolve5, 0));
|
|
170243
|
+
await ensureProjectRegisteredFromPiDirectory(projectDir, deps.db);
|
|
170244
|
+
const coverage = getEmbeddingCoverageStatus(deps.db, projectIdentity, sessionId);
|
|
170245
|
+
if (!coverage.enabled)
|
|
170246
|
+
return;
|
|
170247
|
+
const remaining = coverage.session.total - coverage.session.embedded;
|
|
170248
|
+
if (remaining <= 0)
|
|
170249
|
+
return;
|
|
170250
|
+
notify(`Embedding ${remaining} compartment${remaining === 1 ? "" : "s"} of history in the background…`);
|
|
170251
|
+
const { text } = await runEmbedDrain(deps.db, projectIdentity, sessionId);
|
|
170252
|
+
notify(text.replace(/^## \/ctx-embed\n\n/, ""));
|
|
170253
|
+
} catch {}
|
|
170254
|
+
})();
|
|
170255
|
+
}
|
|
169894
170256
|
|
|
169895
170257
|
// ../plugin/src/hooks/magic-context/execute-flush.ts
|
|
169896
170258
|
init_logger();
|
|
@@ -170306,26 +170668,11 @@ function computePiWorkMetrics(sessionEntries) {
|
|
|
170306
170668
|
}
|
|
170307
170669
|
|
|
170308
170670
|
// ../plugin/src/hooks/magic-context/apply-operations.ts
|
|
170309
|
-
var USER_DROP_PREVIEW_CHARS = 250;
|
|
170310
170671
|
var RECENT_TOOL_SKELETON_WINDOW = 20;
|
|
170311
|
-
function buildReplacementContent(tagId
|
|
170312
|
-
|
|
170313
|
-
|
|
170314
|
-
|
|
170315
|
-
}
|
|
170316
|
-
const currentContent = target.getContent?.() ?? "";
|
|
170317
|
-
const originalText = stripTagPrefix(currentContent);
|
|
170318
|
-
if (originalText.length <= USER_DROP_PREVIEW_CHARS) {
|
|
170319
|
-
return `[truncated §${tagId}§]
|
|
170320
|
-
${originalText}`;
|
|
170321
|
-
}
|
|
170322
|
-
const hardCut = originalText.slice(0, USER_DROP_PREVIEW_CHARS);
|
|
170323
|
-
const softCutIndex = hardCut.search(/\s\S*$/);
|
|
170324
|
-
const preview = softCutIndex > USER_DROP_PREVIEW_CHARS - 30 ? hardCut.slice(0, softCutIndex) : hardCut;
|
|
170325
|
-
return `[truncated §${tagId}§]
|
|
170326
|
-
${preview}…`;
|
|
170327
|
-
}
|
|
170328
|
-
function applyPendingOperations(sessionId, db, targets, protectedTags = 0, preloadedTags, preloadedPendingOps) {
|
|
170672
|
+
function buildReplacementContent(tagId) {
|
|
170673
|
+
return `[dropped §${tagId}§]`;
|
|
170674
|
+
}
|
|
170675
|
+
function applyPendingOperations(sessionId, db, targets, protectedTags = 0, preloadedTags, preloadedPendingOps, syntheticPendingOps = []) {
|
|
170329
170676
|
let didMutateMessage = false;
|
|
170330
170677
|
db.transaction(() => {
|
|
170331
170678
|
const tags = preloadedTags ?? getTagsBySession(db, sessionId);
|
|
@@ -170333,11 +170680,16 @@ function applyPendingOperations(sessionId, db, targets, protectedTags = 0, prelo
|
|
|
170333
170680
|
const tagTypeById = new Map(tags.map((tag) => [tag.tagNumber, tag.type]));
|
|
170334
170681
|
const protectedTagIds = protectedTags > 0 ? new Set(tags.filter((tag) => tag.status === "active").map((tag) => tag.tagNumber).sort((left, right) => right - left).slice(0, protectedTags)) : new Set;
|
|
170335
170682
|
const pendingOps = preloadedPendingOps ?? getPendingOps(db, sessionId);
|
|
170683
|
+
const opsToApply = [
|
|
170684
|
+
...pendingOps.map((op) => ({ op, synthetic: false })),
|
|
170685
|
+
...syntheticPendingOps.map((op) => ({ op, synthetic: true }))
|
|
170686
|
+
];
|
|
170336
170687
|
const skeletonWindow = new Set(tags.filter((tag) => tag.type === "tool").map((tag) => tag.tagNumber).sort((left, right) => right - left).slice(0, RECENT_TOOL_SKELETON_WINDOW));
|
|
170337
|
-
for (const pendingOp of
|
|
170688
|
+
for (const { op: pendingOp, synthetic } of opsToApply) {
|
|
170338
170689
|
const tagStatus = tagStatusById.get(pendingOp.tagId);
|
|
170339
170690
|
if (tagStatus === "compacted" || tagStatus === "dropped") {
|
|
170340
|
-
|
|
170691
|
+
if (!synthetic)
|
|
170692
|
+
removePendingOp(db, sessionId, pendingOp.tagId);
|
|
170341
170693
|
continue;
|
|
170342
170694
|
}
|
|
170343
170695
|
if (protectedTagIds.has(pendingOp.tagId)) {
|
|
@@ -170345,33 +170697,46 @@ function applyPendingOperations(sessionId, db, targets, protectedTags = 0, prelo
|
|
|
170345
170697
|
}
|
|
170346
170698
|
const target = targets.get(pendingOp.tagId);
|
|
170347
170699
|
const isToolTag = tagTypeById.get(pendingOp.tagId) === "tool";
|
|
170700
|
+
if (synthetic) {
|
|
170701
|
+
if (!isToolTag || target?.canDrop?.() !== true)
|
|
170702
|
+
continue;
|
|
170703
|
+
}
|
|
170704
|
+
let shouldPersistDrop = false;
|
|
170348
170705
|
if (isToolTag) {
|
|
170349
170706
|
if (skeletonWindow.has(pendingOp.tagId)) {
|
|
170350
170707
|
const truncResult = target?.truncate?.() ?? "absent";
|
|
170351
|
-
if (truncResult === "incomplete") {
|
|
170708
|
+
if (truncResult === "incomplete" || synthetic && truncResult !== "truncated") {
|
|
170352
170709
|
continue;
|
|
170353
170710
|
}
|
|
170354
170711
|
if (truncResult === "truncated") {
|
|
170355
170712
|
didMutateMessage = true;
|
|
170356
170713
|
}
|
|
170357
170714
|
updateTagDropMode(db, sessionId, pendingOp.tagId, "truncated");
|
|
170715
|
+
shouldPersistDrop = true;
|
|
170358
170716
|
} else {
|
|
170359
170717
|
const dropResult = target?.drop?.() ?? "absent";
|
|
170360
|
-
if (dropResult === "incomplete") {
|
|
170718
|
+
if (dropResult === "incomplete" || synthetic && dropResult !== "removed") {
|
|
170361
170719
|
continue;
|
|
170362
170720
|
}
|
|
170363
170721
|
if (dropResult === "removed") {
|
|
170364
170722
|
didMutateMessage = true;
|
|
170365
170723
|
}
|
|
170366
170724
|
updateTagDropMode(db, sessionId, pendingOp.tagId, "full");
|
|
170725
|
+
shouldPersistDrop = true;
|
|
170367
170726
|
}
|
|
170368
170727
|
} else if (target) {
|
|
170369
|
-
const changed = target.setContent(buildReplacementContent(pendingOp.tagId
|
|
170728
|
+
const changed = target.setContent(buildReplacementContent(pendingOp.tagId));
|
|
170370
170729
|
if (changed)
|
|
170371
170730
|
didMutateMessage = true;
|
|
170731
|
+
shouldPersistDrop = true;
|
|
170732
|
+
} else if (!synthetic) {
|
|
170733
|
+
shouldPersistDrop = true;
|
|
170372
170734
|
}
|
|
170735
|
+
if (!shouldPersistDrop)
|
|
170736
|
+
continue;
|
|
170373
170737
|
updateTagStatus(db, sessionId, pendingOp.tagId, "dropped");
|
|
170374
|
-
|
|
170738
|
+
if (!synthetic)
|
|
170739
|
+
removePendingOp(db, sessionId, pendingOp.tagId);
|
|
170375
170740
|
}
|
|
170376
170741
|
})();
|
|
170377
170742
|
return didMutateMessage;
|
|
@@ -170395,7 +170760,7 @@ function applyFlushedStatuses(sessionId, db, targets, preloadedTags) {
|
|
|
170395
170760
|
}
|
|
170396
170761
|
}
|
|
170397
170762
|
} else if (target) {
|
|
170398
|
-
const changed = target.setContent(buildReplacementContent(tag.tagNumber
|
|
170763
|
+
const changed = target.setContent(buildReplacementContent(tag.tagNumber));
|
|
170399
170764
|
if (changed)
|
|
170400
170765
|
didMutateMessage = true;
|
|
170401
170766
|
}
|
|
@@ -170683,7 +171048,8 @@ function applyCavemanCleanup(sessionId, db, targets, tags, config2) {
|
|
|
170683
171048
|
const result = {
|
|
170684
171049
|
compressedToLite: 0,
|
|
170685
171050
|
compressedToFull: 0,
|
|
170686
|
-
compressedToUltra: 0
|
|
171051
|
+
compressedToUltra: 0,
|
|
171052
|
+
mutatedTextTags: 0
|
|
170687
171053
|
};
|
|
170688
171054
|
if (!config2.enabled)
|
|
170689
171055
|
return result;
|
|
@@ -170724,7 +171090,9 @@ function applyCavemanCleanup(sessionId, db, targets, tags, config2) {
|
|
|
170724
171090
|
const target = targets.get(tag.tagNumber);
|
|
170725
171091
|
if (!target)
|
|
170726
171092
|
continue;
|
|
170727
|
-
target.setContent(compressed);
|
|
171093
|
+
const didMutate = target.setContent(compressed);
|
|
171094
|
+
if (didMutate)
|
|
171095
|
+
result.mutatedTextTags += 1;
|
|
170728
171096
|
updateCavemanDepth(db, sessionId, tag.tagNumber, targetDepth);
|
|
170729
171097
|
if (targetDepth === DEPTH_LITE)
|
|
170730
171098
|
result.compressedToLite += 1;
|
|
@@ -171102,7 +171470,7 @@ function buildToolArcs(messages) {
|
|
|
171102
171470
|
}
|
|
171103
171471
|
return arcs.sort((a, b) => a.invOrdinal - b.invOrdinal || (a.resOrdinal ?? Number.MAX_SAFE_INTEGER) - (b.resOrdinal ?? Number.MAX_SAFE_INTEGER));
|
|
171104
171472
|
}
|
|
171105
|
-
function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal) {
|
|
171473
|
+
function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal, recentOpenArcCutoff) {
|
|
171106
171474
|
let boundary = candidate;
|
|
171107
171475
|
for (const arc of arcs) {
|
|
171108
171476
|
if (arc.resOrdinal !== null) {
|
|
@@ -171111,6 +171479,8 @@ function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal) {
|
|
|
171111
171479
|
}
|
|
171112
171480
|
continue;
|
|
171113
171481
|
}
|
|
171482
|
+
if (arc.invOrdinal < recentOpenArcCutoff)
|
|
171483
|
+
continue;
|
|
171114
171484
|
if (arc.invOrdinal >= lastCompartmentEndOrdinal + 1 && arc.invOrdinal < boundary) {
|
|
171115
171485
|
return arc.invOrdinal;
|
|
171116
171486
|
}
|
|
@@ -171348,7 +171718,7 @@ function semanticSnapBoundary(args) {
|
|
|
171348
171718
|
return snapped;
|
|
171349
171719
|
}
|
|
171350
171720
|
function applyHeadCap(args) {
|
|
171351
|
-
const { index, protectedTailStart, offset, arcs, capTokens } = args;
|
|
171721
|
+
const { index, protectedTailStart, offset, arcs, capTokens, recentOpenArcCutoff } = args;
|
|
171352
171722
|
if (offset >= protectedTailStart)
|
|
171353
171723
|
return { eligibleEndOrdinal: offset, oversizeAtomicUnit: false };
|
|
171354
171724
|
let end = index.findHeadEndForCap(offset, protectedTailStart, capTokens);
|
|
@@ -171356,7 +171726,7 @@ function applyHeadCap(args) {
|
|
|
171356
171726
|
for (const arc of arcs) {
|
|
171357
171727
|
const resOrdinal = arc.resOrdinal;
|
|
171358
171728
|
if (resOrdinal === null) {
|
|
171359
|
-
if (arc.invOrdinal >= offset && arc.invOrdinal < end) {
|
|
171729
|
+
if (arc.invOrdinal >= recentOpenArcCutoff && arc.invOrdinal >= offset && arc.invOrdinal < end) {
|
|
171360
171730
|
end = Math.min(end, arc.invOrdinal);
|
|
171361
171731
|
}
|
|
171362
171732
|
continue;
|
|
@@ -171423,7 +171793,14 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171423
171793
|
}
|
|
171424
171794
|
if (ctx.mode === "manual-full-recomp") {
|
|
171425
171795
|
const arcs2 = buildToolArcs(messages);
|
|
171426
|
-
const
|
|
171796
|
+
const recompTarget = deriveProtectedTailTokenTarget({
|
|
171797
|
+
contextLimit: ctx.contextLimit,
|
|
171798
|
+
executeThresholdPercentage: ctx.executeThresholdPercentage,
|
|
171799
|
+
usagePercentage: 0,
|
|
171800
|
+
triggerBudget: ctx.triggerBudget
|
|
171801
|
+
});
|
|
171802
|
+
const recentOpenArcCutoff2 = index.findSuffixStartForTokens(recompTarget.N);
|
|
171803
|
+
const firstOpenArc = arcs2.find((arc) => arc.resOrdinal === null && arc.invOrdinal >= offset && arc.invOrdinal >= recentOpenArcCutoff2);
|
|
171427
171804
|
const protectedTailStart2 = firstOpenArc?.invOrdinal ?? rawMessageCount + 1;
|
|
171428
171805
|
const rawRangeFingerprint2 = computeRawRangeFingerprint(messages, offset, protectedTailStart2);
|
|
171429
171806
|
return {
|
|
@@ -171465,13 +171842,14 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171465
171842
|
const scaledN = ctx.emergencyTailScale ? Math.max(1, Math.floor(target.N * ctx.emergencyTailScale)) : target.N;
|
|
171466
171843
|
const arcs = buildToolArcs(messages);
|
|
171467
171844
|
let boundary = index.findSuffixStartForTokens(scaledN);
|
|
171845
|
+
const recentOpenArcCutoff = boundary;
|
|
171468
171846
|
let boundaryReason = boundary === 1 ? "whole-session-smaller-than-tail" : "size-walk";
|
|
171469
171847
|
const tokenAtBoundary = index.tokenForOrdinal(boundary);
|
|
171470
171848
|
if (boundary <= rawMessageCount && tokenAtBoundary > Math.max(2 * scaledN, 64000) && boundary < rawMessageCount) {
|
|
171471
171849
|
boundary += 1;
|
|
171472
171850
|
boundaryReason = "huge-message-exception";
|
|
171473
171851
|
}
|
|
171474
|
-
boundary = fenceBoundaryForToolArcs(boundary, arcs, ctx.lastCompartmentEndOrdinal);
|
|
171852
|
+
boundary = fenceBoundaryForToolArcs(boundary, arcs, ctx.lastCompartmentEndOrdinal, recentOpenArcCutoff);
|
|
171475
171853
|
const snapped = semanticSnapBoundary({
|
|
171476
171854
|
messages,
|
|
171477
171855
|
index,
|
|
@@ -171481,7 +171859,7 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171481
171859
|
});
|
|
171482
171860
|
if (snapped !== boundary)
|
|
171483
171861
|
boundaryReason = "semantic-snap";
|
|
171484
|
-
boundary = fenceBoundaryForToolArcs(snapped, arcs, ctx.lastCompartmentEndOrdinal);
|
|
171862
|
+
boundary = fenceBoundaryForToolArcs(snapped, arcs, ctx.lastCompartmentEndOrdinal, recentOpenArcCutoff);
|
|
171485
171863
|
let runtimeFloor = offset;
|
|
171486
171864
|
if (ctx.migrationFloorActive)
|
|
171487
171865
|
runtimeFloor = Math.max(runtimeFloor, ctx.priorBoundaryOrdinal);
|
|
@@ -171517,7 +171895,8 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171517
171895
|
offset,
|
|
171518
171896
|
arcs,
|
|
171519
171897
|
lastCompartmentEndOrdinal: ctx.lastCompartmentEndOrdinal,
|
|
171520
|
-
capTokens: perRunCap
|
|
171898
|
+
capTokens: perRunCap,
|
|
171899
|
+
recentOpenArcCutoff
|
|
171521
171900
|
});
|
|
171522
171901
|
const rawRangeFingerprint = computeRawRangeFingerprint(messages, offset, head.eligibleEndOrdinal);
|
|
171523
171902
|
return {
|
|
@@ -171922,8 +172301,8 @@ var CHANNEL1_SENTINEL = "<system-reminder>";
|
|
|
171922
172301
|
var TOKENS_PER_BYTE = 0.25;
|
|
171923
172302
|
var CHANNEL1_FLOOR_TOKENS = 1e4;
|
|
171924
172303
|
var CHANNEL1_REFIRE_FLOOR_TOKENS = 1e4;
|
|
171925
|
-
function channel1RefireTokens(
|
|
171926
|
-
const scaled = Math.round(0.05 * Math.max(0,
|
|
172304
|
+
function channel1RefireTokens(workingWindowTokens) {
|
|
172305
|
+
const scaled = Math.round(0.05 * Math.max(0, workingWindowTokens));
|
|
171927
172306
|
return Math.max(CHANNEL1_REFIRE_FLOOR_TOKENS, scaled);
|
|
171928
172307
|
}
|
|
171929
172308
|
var S_GENTLE = 0.2;
|
|
@@ -171939,7 +172318,7 @@ function toolOutputTokens(output) {
|
|
|
171939
172318
|
return Math.round(byteSize(output) * TOKENS_PER_BYTE);
|
|
171940
172319
|
}
|
|
171941
172320
|
function decideChannel1(input) {
|
|
171942
|
-
const { undroppedTokens, pressure,
|
|
172321
|
+
const { undroppedTokens, pressure, workingWindowTokens, hasRecentReduce } = input;
|
|
171943
172322
|
const resetCycle = hasRecentReduce || undroppedTokens < input.lastNudgeUndropped;
|
|
171944
172323
|
const lastNudge = resetCycle ? 0 : input.lastNudgeUndropped;
|
|
171945
172324
|
const lastLevel = resetCycle ? "" : input.lastNudgeLevel;
|
|
@@ -171954,7 +172333,7 @@ function decideChannel1(input) {
|
|
|
171954
172333
|
return quiet();
|
|
171955
172334
|
if (undroppedTokens < CHANNEL1_FLOOR_TOKENS)
|
|
171956
172335
|
return quiet();
|
|
171957
|
-
const budget =
|
|
172336
|
+
const budget = workingWindowTokens > 0 ? workingWindowTokens : undroppedTokens || 1;
|
|
171958
172337
|
const severity = undroppedTokens / budget * pressure;
|
|
171959
172338
|
if (severity < S_GENTLE)
|
|
171960
172339
|
return quiet();
|
|
@@ -171966,7 +172345,7 @@ function decideChannel1(input) {
|
|
|
171966
172345
|
else
|
|
171967
172346
|
level = "gentle";
|
|
171968
172347
|
if (lastLevel === "") {
|
|
171969
|
-
if (undroppedTokens < lastNudge + channel1RefireTokens(
|
|
172348
|
+
if (undroppedTokens < lastNudge + channel1RefireTokens(workingWindowTokens)) {
|
|
171970
172349
|
return quiet();
|
|
171971
172350
|
}
|
|
171972
172351
|
} else if (LEVEL_RANK[level] <= LEVEL_RANK[lastLevel]) {
|
|
@@ -171991,6 +172370,13 @@ function computePressure(input) {
|
|
|
171991
172370
|
function approxThousands(tokens) {
|
|
171992
172371
|
return `${Math.round(tokens / 1000)}k`;
|
|
171993
172372
|
}
|
|
172373
|
+
function formatOldestReclaimableHint(hint) {
|
|
172374
|
+
if (!hint || hint.length === 0)
|
|
172375
|
+
return "";
|
|
172376
|
+
const rendered = hint.slice(0, 4).map((tag) => `§${tag.tagNumber}§ ${tag.toolName ?? "tool"}`).join(" · ");
|
|
172377
|
+
return rendered.length > 0 ? `
|
|
172378
|
+
oldest reclaimable: ${rendered}.` : "";
|
|
172379
|
+
}
|
|
171994
172380
|
var CHANNEL2_USABLE_FRACTION = 1 / 3;
|
|
171995
172381
|
var CHANNEL2_MIN_RECLAIMABLE = 1e4;
|
|
171996
172382
|
function shouldTriggerChannel2(input) {
|
|
@@ -172000,30 +172386,32 @@ function shouldTriggerChannel2(input) {
|
|
|
172000
172386
|
return true;
|
|
172001
172387
|
return input.reclaimableTokens >= input.usableTokens * CHANNEL2_USABLE_FRACTION;
|
|
172002
172388
|
}
|
|
172003
|
-
function buildChannel2Reminder(undroppedTokens) {
|
|
172389
|
+
function buildChannel2Reminder(undroppedTokens, hint) {
|
|
172004
172390
|
const amount = approxThousands(undroppedTokens);
|
|
172391
|
+
const hintText = formatOldestReclaimableHint(hint);
|
|
172005
172392
|
return `<system-reminder>
|
|
172006
|
-
` + `Routine context housekeeping is near: a large span of this session will be comparted soon, ` + `and ~${amount} tokens of tool output remain unreduced. Drop spent outputs with ctx_reduce ` + `first so the archived span is the part that matters
|
|
172393
|
+
` + `Routine context housekeeping is near: a large span of this session will be comparted soon, ` + `and ~${amount} tokens of tool output remain unreduced. Drop spent outputs with ctx_reduce ` + `first so the archived span is the part that matters.${hintText}
|
|
172007
172394
|
` + `</system-reminder>`;
|
|
172008
172395
|
}
|
|
172009
|
-
function buildChannel1Reminder(level, undroppedTokens) {
|
|
172396
|
+
function buildChannel1Reminder(level, undroppedTokens, hint) {
|
|
172010
172397
|
const amount = approxThousands(undroppedTokens);
|
|
172398
|
+
const hintText = formatOldestReclaimableHint(hint);
|
|
172011
172399
|
let body;
|
|
172012
172400
|
switch (level) {
|
|
172013
172401
|
case "gentle":
|
|
172014
|
-
body = `You have ~${amount} tokens of tool output you have not reduced. ` + `
|
|
172402
|
+
body = `You have ~${amount} tokens of tool output you have not reduced. ` + `When you are done with earlier outputs, dropping them with ctx_reduce keeps context lean.`;
|
|
172015
172403
|
break;
|
|
172016
172404
|
case "firm":
|
|
172017
|
-
body = `~${amount} tokens of unreduced tool output
|
|
172405
|
+
body = `~${amount} tokens of unreduced tool output has built up. ` + `At your next natural stopping point, consider dropping what you have already processed with ctx_reduce.`;
|
|
172018
172406
|
break;
|
|
172019
172407
|
case "urgent":
|
|
172020
|
-
body = `~${amount} tokens of unreduced tool output remain
|
|
172408
|
+
body = `~${amount} tokens of unreduced tool output remain, and a large span of this session will be comparted before long. ` + `Consider dropping spent outputs with ctx_reduce so the archived span is the part that matters.`;
|
|
172021
172409
|
break;
|
|
172022
172410
|
}
|
|
172023
172411
|
return `
|
|
172024
172412
|
|
|
172025
172413
|
<system-reminder>
|
|
172026
|
-
${body}
|
|
172414
|
+
${body}${hintText}
|
|
172027
172415
|
</system-reminder>`;
|
|
172028
172416
|
}
|
|
172029
172417
|
|
|
@@ -172571,6 +172959,41 @@ function renderMemoryBlockV2(memories, wrapper = "project-memory", renderOptions
|
|
|
172571
172959
|
`);
|
|
172572
172960
|
}
|
|
172573
172961
|
|
|
172962
|
+
// ../plugin/src/hooks/magic-context/tool-reclaim.ts
|
|
172963
|
+
function buildSyntheticToolReclaimOps(input) {
|
|
172964
|
+
const watermark = Math.max(0, input.watermark);
|
|
172965
|
+
if (watermark <= 0)
|
|
172966
|
+
return [];
|
|
172967
|
+
const realPendingTagIds = new Set((input.pendingOps ?? []).map((op) => op.tagId));
|
|
172968
|
+
const tags = getActiveTagsBySession(input.db, input.sessionId);
|
|
172969
|
+
const synthetic = [];
|
|
172970
|
+
for (const tag of tags) {
|
|
172971
|
+
if (tag.type !== "tool")
|
|
172972
|
+
continue;
|
|
172973
|
+
if (tag.status !== "active")
|
|
172974
|
+
continue;
|
|
172975
|
+
if (tag.tagNumber > watermark)
|
|
172976
|
+
continue;
|
|
172977
|
+
if (realPendingTagIds.has(tag.tagNumber))
|
|
172978
|
+
continue;
|
|
172979
|
+
if (input.targets.get(tag.tagNumber)?.canDrop?.() !== true)
|
|
172980
|
+
continue;
|
|
172981
|
+
synthetic.push({
|
|
172982
|
+
id: 0,
|
|
172983
|
+
sessionId: input.sessionId,
|
|
172984
|
+
tagId: tag.tagNumber,
|
|
172985
|
+
operation: "drop",
|
|
172986
|
+
queuedAt: 0
|
|
172987
|
+
});
|
|
172988
|
+
}
|
|
172989
|
+
return synthetic;
|
|
172990
|
+
}
|
|
172991
|
+
function advanceToolReclaimWatermarkToCurrentMax(db, sessionId) {
|
|
172992
|
+
const maxTagNumber = getMaxTagNumberBySession(db, sessionId);
|
|
172993
|
+
advanceToolReclaimWatermark(db, sessionId, maxTagNumber);
|
|
172994
|
+
return maxTagNumber;
|
|
172995
|
+
}
|
|
172996
|
+
|
|
172574
172997
|
// src/context-handler.ts
|
|
172575
172998
|
init_logger();
|
|
172576
172999
|
|
|
@@ -172981,7 +173404,7 @@ function tagToolPart(args) {
|
|
|
172981
173404
|
const tagged = prependTag(tagId, text);
|
|
172982
173405
|
args.part.setText(tagged);
|
|
172983
173406
|
}
|
|
172984
|
-
args.targets.set(tagId, buildToolTarget(args.part, args.message));
|
|
173407
|
+
args.targets.set(tagId, buildToolTarget(args.part, args.message, tagId));
|
|
172985
173408
|
}
|
|
172986
173409
|
function setToolContentOrText(part, content) {
|
|
172987
173410
|
try {
|
|
@@ -173021,7 +173444,7 @@ function buildAggregateTarget(tagId, occurrences) {
|
|
|
173021
173444
|
return any2 ? "removed" : "absent";
|
|
173022
173445
|
},
|
|
173023
173446
|
truncate() {
|
|
173024
|
-
const sentinel =
|
|
173447
|
+
const sentinel = `[dropped §${tagId}§]`;
|
|
173025
173448
|
let any2 = false;
|
|
173026
173449
|
for (const occ of occurrences) {
|
|
173027
173450
|
if (setToolContentOrText(occ.part, sentinel)) {
|
|
@@ -173053,7 +173476,7 @@ function buildTextTarget(part, message) {
|
|
|
173053
173476
|
}
|
|
173054
173477
|
};
|
|
173055
173478
|
}
|
|
173056
|
-
function buildToolTarget(part, message) {
|
|
173479
|
+
function buildToolTarget(part, message, tagId) {
|
|
173057
173480
|
return {
|
|
173058
173481
|
setContent(content) {
|
|
173059
173482
|
return setToolContentOrText(part, content);
|
|
@@ -173062,11 +173485,11 @@ function buildToolTarget(part, message) {
|
|
|
173062
173485
|
return part.getText() ?? null;
|
|
173063
173486
|
},
|
|
173064
173487
|
drop() {
|
|
173065
|
-
const replaced = part.replaceWithSentinel(`[dropped §${
|
|
173488
|
+
const replaced = part.replaceWithSentinel(`[dropped §${tagId}§]`);
|
|
173066
173489
|
return replaced ? "removed" : "absent";
|
|
173067
173490
|
},
|
|
173068
173491
|
truncate() {
|
|
173069
|
-
const ok = setToolContentOrText(part,
|
|
173492
|
+
const ok = setToolContentOrText(part, `[dropped §${tagId}§]`);
|
|
173070
173493
|
return ok ? "truncated" : "absent";
|
|
173071
173494
|
},
|
|
173072
173495
|
message: {
|
|
@@ -174336,10 +174759,11 @@ function maybeChannel1ReminderForToolResult(args) {
|
|
|
174336
174759
|
contextLimit: state.contextLimit,
|
|
174337
174760
|
executeThresholdPercentage: state.executeThresholdPercentage
|
|
174338
174761
|
});
|
|
174762
|
+
const workingWindowTokens = Math.round(state.contextLimit * state.executeThresholdPercentage / 100);
|
|
174339
174763
|
const decision = decideChannel1({
|
|
174340
174764
|
undroppedTokens,
|
|
174341
174765
|
pressure,
|
|
174342
|
-
|
|
174766
|
+
workingWindowTokens,
|
|
174343
174767
|
lastNudgeUndropped: getLastNudgeUndropped(db, sessionId),
|
|
174344
174768
|
lastNudgeLevel: getLastNudgeLevel(db, sessionId),
|
|
174345
174769
|
hasRecentReduce: false
|
|
@@ -174350,9 +174774,10 @@ function maybeChannel1ReminderForToolResult(args) {
|
|
|
174350
174774
|
return null;
|
|
174351
174775
|
return {
|
|
174352
174776
|
type: "text",
|
|
174353
|
-
text: buildChannel1Reminder(decision.level, decision.undroppedTokens)
|
|
174777
|
+
text: buildChannel1Reminder(decision.level, decision.undroppedTokens, state.oldestReclaimableToolTags)
|
|
174354
174778
|
};
|
|
174355
174779
|
}
|
|
174780
|
+
var CHANNEL2_NUDGE_CUSTOM_TYPE = "magic-context:ceiling-nudge";
|
|
174356
174781
|
function maybeDeliverChannel2Pi(pi, db, sessionId, deliverAs = "followUp") {
|
|
174357
174782
|
let state;
|
|
174358
174783
|
try {
|
|
@@ -174380,9 +174805,12 @@ function maybeDeliverChannel2Pi(pi, db, sessionId, deliverAs = "followUp") {
|
|
|
174380
174805
|
if (!casChannel2NudgeState(db, sessionId, "pending", "claimed"))
|
|
174381
174806
|
return false;
|
|
174382
174807
|
try {
|
|
174383
|
-
pi.
|
|
174384
|
-
|
|
174385
|
-
|
|
174808
|
+
pi.sendMessage({
|
|
174809
|
+
customType: CHANNEL2_NUDGE_CUSTOM_TYPE,
|
|
174810
|
+
content: buildChannel2Reminder(undropped, baseline.oldestReclaimableToolTags),
|
|
174811
|
+
display: false,
|
|
174812
|
+
details: { kind: "channel-2-ceiling-nudge" }
|
|
174813
|
+
}, { deliverAs });
|
|
174386
174814
|
} catch (error51) {
|
|
174387
174815
|
try {
|
|
174388
174816
|
casChannel2NudgeState(db, sessionId, "claimed", "pending");
|
|
@@ -174777,10 +175205,14 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174777
175205
|
if (!matched)
|
|
174778
175206
|
continue;
|
|
174779
175207
|
const target = targets.get(tag.tagNumber);
|
|
174780
|
-
target?.drop?.();
|
|
175208
|
+
const result = target?.drop?.() ?? "absent";
|
|
175209
|
+
if (result === "incomplete")
|
|
175210
|
+
continue;
|
|
174781
175211
|
updateTagDropMode(db, sessionId, tag.tagNumber, "full");
|
|
174782
175212
|
updateTagStatus(db, sessionId, tag.tagNumber, "dropped");
|
|
174783
|
-
|
|
175213
|
+
if (result === "removed" || result === "truncated") {
|
|
175214
|
+
droppedStaleReduceCalls++;
|
|
175215
|
+
}
|
|
174784
175216
|
}
|
|
174785
175217
|
})();
|
|
174786
175218
|
}
|
|
@@ -174852,7 +175284,9 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174852
175284
|
continue;
|
|
174853
175285
|
updateTagDropMode(db, sessionId, tag.tagNumber, "full");
|
|
174854
175286
|
updateTagStatus(db, sessionId, tag.tagNumber, "dropped");
|
|
174855
|
-
|
|
175287
|
+
if (result === "removed" || result === "truncated") {
|
|
175288
|
+
deduplicatedTools++;
|
|
175289
|
+
}
|
|
174856
175290
|
}
|
|
174857
175291
|
}
|
|
174858
175292
|
})();
|
|
@@ -174861,6 +175295,7 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174861
175295
|
sessionLog(sessionId, `heuristic cleanup: dropped ${droppedTools} tool tags, stale ctx_reduce=${droppedStaleReduceCalls}, deduplicated ${deduplicatedTools} tool calls, dropped ${droppedInjections} system injections`);
|
|
174862
175296
|
}
|
|
174863
175297
|
let compressedTextTags = 0;
|
|
175298
|
+
let mutatedTextTags = 0;
|
|
174864
175299
|
if (config2.caveman?.enabled) {
|
|
174865
175300
|
const cavemanResult = applyCavemanCleanup(sessionId, db, targets, tags, {
|
|
174866
175301
|
enabled: true,
|
|
@@ -174868,13 +175303,15 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174868
175303
|
protectedTags: config2.protectedTags
|
|
174869
175304
|
});
|
|
174870
175305
|
compressedTextTags = cavemanResult.compressedToLite + cavemanResult.compressedToFull + cavemanResult.compressedToUltra;
|
|
175306
|
+
mutatedTextTags = cavemanResult.mutatedTextTags;
|
|
174871
175307
|
}
|
|
174872
175308
|
return {
|
|
174873
175309
|
droppedTools,
|
|
174874
175310
|
deduplicatedTools,
|
|
174875
175311
|
droppedInjections,
|
|
174876
175312
|
droppedStaleReduceCalls,
|
|
174877
|
-
compressedTextTags
|
|
175313
|
+
compressedTextTags,
|
|
175314
|
+
mutatedTextTags
|
|
174878
175315
|
};
|
|
174879
175316
|
}
|
|
174880
175317
|
function buildMessageIdToMaxTagFromTargets(targets) {
|
|
@@ -180251,9 +180688,7 @@ Context is managed for you entirely automatically — there's nothing to prune a
|
|
|
180251
180688
|
var CTX_NOTE_GUIDANCE = `Use \`ctx_note\` ONLY for genuinely future concerns — something to revisit much later, not work coming up in the next few turns (that's already in your active context) and not active multi-step work (use todos for that). Magic Context preserves your full context across both compaction and restarts, so an upcoming restart or "let's come back to this later" is never a reason to take a note — nothing is lost either way. Notes you do take survive compression and resurface at natural work boundaries (after commits, historian runs, todo completion).`;
|
|
180252
180689
|
var TOOL_HISTORY_GUIDANCE = `Compressed history intentionally omits tool calls and their outputs — summaries like "I edited file X" are historian records, not patterns to replicate. In the live conversation, older tool calls and their results are cleaned up to save context — you may see your own past messages referencing actions without the corresponding tool call or result visible. This is normal context management. ALWAYS use real tool calls; never simulate, fabricate, or inline tool outputs in your text. If there is no tool result message, the action did not happen. NEVER simulate, hallucinate or claim tool calls, command output, search results, file edits, or diffs in plain text as if they actually occurred.`;
|
|
180253
180690
|
var BASE_INTRO = (protectedTags) => `Messages and tool outputs are tagged with §N§ identifiers (e.g., §1§, §42§).
|
|
180254
|
-
Use \`ctx_reduce\` to
|
|
180255
|
-
- \`drop\`: Remove entirely (best for tool outputs you already acted on).
|
|
180256
|
-
Syntax: "3-5", "1,2,9", or "1-5,8,12-15". Last ${protectedTags} tags are protected.
|
|
180691
|
+
Use \`ctx_reduce\` to mark spent tagged content as discardable and reclaim space. Marking is NOT an immediate delete — it queues the content, which stays fully visible until space is actually needed (as soon as the next turn if you're already under pressure, much later if not), so mark a tool output as soon as you're done with it rather than hoarding the call for the end of the turn. The last ${protectedTags} tags are protected (marking one just queues it until it ages out). Syntax: "3-5", "1,2,9", or "1-5,8,12-15".
|
|
180257
180692
|
Do not announce or narrate \`ctx_reduce\` drops — just call the tool silently. Saying "I'll drop these outputs" wastes tokens the user does not care about.
|
|
180258
180693
|
${CTX_NOTE_GUIDANCE}
|
|
180259
180694
|
Use \`ctx_memory\` for durable project knowledge: write what future sessions must know, update/archive/merge the memories you see in \`<project-memory>\` when they drift. Memories persist across sessions and every new session starts with them.
|
|
@@ -180272,7 +180707,7 @@ Use \`ctx_expand\` to recover the raw conversation behind a \`<compartment>\` su
|
|
|
180272
180707
|
\`ctx_search\` returns ranked results from memories, git commits, and raw message history. Use message ordinals from results with \`ctx_expand\` to retrieve surrounding conversation context.
|
|
180273
180708
|
${TOOL_HISTORY_GUIDANCE}
|
|
180274
180709
|
NEVER drop large ranges blindly (e.g., "1-50"). Review each tag before deciding.
|
|
180275
|
-
|
|
180710
|
+
Keep your user's instructions and intent — never drop a user message for its directive, even an old one. But a large block of pasted content inside a user message (logs, data dumps, long code, attachments) is fair to mark discardable once you've extracted what you need — it stays searchable via \`ctx_search\`.
|
|
180276
180711
|
NEVER drop assistant text messages unless they are exceptionally large. Your conversation messages are lightweight; only large tool outputs are worth dropping.
|
|
180277
180712
|
Before your turn finishes, consider using \`ctx_reduce\` to drop large tool outputs you no longer need.`;
|
|
180278
180713
|
var BASE_INTRO_NO_REDUCE = () => `${CTX_NOTE_GUIDANCE}
|
|
@@ -181738,7 +182173,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181738
182173
|
let tailToolTokens;
|
|
181739
182174
|
let liveTailTokens;
|
|
181740
182175
|
try {
|
|
181741
|
-
const agg = getActiveTagTokenAggregate(options.db, sessionId);
|
|
182176
|
+
const agg = getActiveTagTokenAggregate(options.db, sessionId, options.protectedTags ?? 20);
|
|
181742
182177
|
tailToolTokens = agg.toolOutput;
|
|
181743
182178
|
liveTailTokens = agg.conversation + agg.toolCall;
|
|
181744
182179
|
} catch {
|
|
@@ -181749,6 +182184,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181749
182184
|
const executeThresholdTokensPi = Math.round((usageContextLimit ?? 0) * resolvedExecuteThresholdPct / 100);
|
|
181750
182185
|
const usableTokensPi = Math.max(0, executeThresholdTokensPi - usageInputTokens + liveTailTokens);
|
|
181751
182186
|
resetLastNudgeCycleIfTailShrank(options.db, sessionId, tailToolTokens);
|
|
182187
|
+
const oldestReclaimableToolTags = getOldestActiveUnprotectedToolTags(options.db, sessionId, options.protectedTags ?? 20);
|
|
181752
182188
|
setPiChannel1Baseline(sessionId, {
|
|
181753
182189
|
tailToolTokens,
|
|
181754
182190
|
historyBudgetTokens: historyBudgetTokens ?? 0,
|
|
@@ -181757,7 +182193,8 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181757
182193
|
lastInputTokens: usageInputTokens,
|
|
181758
182194
|
turnToolTokens: 0,
|
|
181759
182195
|
usableTokens: usableTokensPi,
|
|
181760
|
-
reducedSinceRefresh: false
|
|
182196
|
+
reducedSinceRefresh: false,
|
|
182197
|
+
oldestReclaimableToolTags
|
|
181761
182198
|
});
|
|
181762
182199
|
if (usageContextLimit && usageContextLimit > 0 && resolvedExecuteThresholdPct > 0) {
|
|
181763
182200
|
const channel2ShouldTrigger = shouldTriggerChannel2({
|
|
@@ -181785,6 +182222,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181785
182222
|
logTransformTiming(sessionId, "postTransformPhase", tPostTransform);
|
|
181786
182223
|
sessionLog(sessionId, `transform completed in ${(performance.now() - transformStartTime).toFixed(1)}ms (${outputMessages.length} messages, ${result.targetCount} targets, watermark: ${result.reasoningWatermark})`);
|
|
181787
182224
|
clearLastTransformErrorIfSet(options.db, sessionId);
|
|
182225
|
+
options.maybeAutoEmbedSession?.(sessionId, projectDirectory, projectIdentity);
|
|
181788
182226
|
return { messages: outputMessages };
|
|
181789
182227
|
} catch (err) {
|
|
181790
182228
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -182091,6 +182529,15 @@ Historian previously failed ${failureState.failureCount} time(s), so Magic Conte
|
|
|
182091
182529
|
const trigger = checkCompartmentTrigger(db, sessionId, sessionMeta, usage, 0, triggerInputs.executeThresholdPercentage, triggerInputs.triggerBudget, triggerInputs.clearReasoningAge, triggerInputs.commitClusterTrigger, args.activeTags, boundaryContextLimit);
|
|
182092
182530
|
if (!trigger.shouldFire) {
|
|
182093
182531
|
sessionLog(sessionId, `historian trigger eval: shouldFire=false (no trigger condition met)`);
|
|
182532
|
+
try {
|
|
182533
|
+
const overflowState = getOverflowState(db, sessionId);
|
|
182534
|
+
if (overflowState.needsEmergencyRecovery && usage.percentage < FORCE_MATERIALIZATION_PERCENTAGE && !inFlightHistorian.has(sessionId) && !hasRunnableCompartmentWindow(boundarySnapshot)) {
|
|
182535
|
+
clearEmergencyRecovery(db, sessionId);
|
|
182536
|
+
sessionLog(sessionId, `historian: disarming stale emergency recovery — real pressure ${usage.percentage.toFixed(1)}% with no runnable compartment window (would otherwise bump to 95% every pass)`);
|
|
182537
|
+
}
|
|
182538
|
+
} catch (err) {
|
|
182539
|
+
sessionLog(sessionId, `historian: emergency-recovery disarm check failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
182540
|
+
}
|
|
182094
182541
|
return;
|
|
182095
182542
|
}
|
|
182096
182543
|
triggered = true;
|
|
@@ -182122,6 +182569,8 @@ async function runPipeline(args) {
|
|
|
182122
182569
|
let historyWasConsumedThisPass = false;
|
|
182123
182570
|
let materializationSatisfiedThisPass = false;
|
|
182124
182571
|
let pendingOpsAppliedThisPass = false;
|
|
182572
|
+
let pendingOpsDidMutate = false;
|
|
182573
|
+
let heuristicOrReasoningDidMutate = false;
|
|
182125
182574
|
let suppressDeferredHistoryDrain = false;
|
|
182126
182575
|
let casLost = false;
|
|
182127
182576
|
const deferredHistoryWasPendingAtPassStart = deferredHistoryRefreshSessions.has(args.sessionId);
|
|
@@ -182207,7 +182656,7 @@ async function runPipeline(args) {
|
|
|
182207
182656
|
sessionLog(args.sessionId, `pending ops WILL APPLY — reason=${applyReason}, pendingOps=${pendingOps.length}, context=${args.contextUsage.percentage.toFixed(1)}%`);
|
|
182208
182657
|
try {
|
|
182209
182658
|
const tApplyPending = performance.now();
|
|
182210
|
-
applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, pendingOps);
|
|
182659
|
+
pendingOpsDidMutate = applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, pendingOps);
|
|
182211
182660
|
logTransformTiming(args.sessionId, "applyPendingOperations", tApplyPending);
|
|
182212
182661
|
executedWorkThisPass = true;
|
|
182213
182662
|
materializationSatisfiedThisPass = true;
|
|
@@ -182291,6 +182740,9 @@ async function runPipeline(args) {
|
|
|
182291
182740
|
} : undefined,
|
|
182292
182741
|
caveman: args.heuristics.caveman
|
|
182293
182742
|
}, activeTags, stableIdResolver);
|
|
182743
|
+
const heuristicMutationCount = heuristicsResult.droppedTools + heuristicsResult.deduplicatedTools + heuristicsResult.droppedInjections + heuristicsResult.droppedStaleReduceCalls + heuristicsResult.mutatedTextTags;
|
|
182744
|
+
if (heuristicMutationCount > 0)
|
|
182745
|
+
heuristicOrReasoningDidMutate = true;
|
|
182294
182746
|
heuristicsExecuted = true;
|
|
182295
182747
|
executedWorkThisPass = true;
|
|
182296
182748
|
if (hasPendingMaterializeSignal) {
|
|
@@ -182299,7 +182751,7 @@ async function runPipeline(args) {
|
|
|
182299
182751
|
if (currentTurnId !== null) {
|
|
182300
182752
|
lastHeuristicsTurnIdBySession.set(args.sessionId, currentTurnId);
|
|
182301
182753
|
}
|
|
182302
|
-
logTransformTiming(args.sessionId, "applyHeuristicCleanup", tHeuristic, `droppedTools=${heuristicsResult.droppedTools} deduplicatedTools=${heuristicsResult.deduplicatedTools} droppedInjections=${heuristicsResult.droppedInjections} compressedTextTags=${heuristicsResult.compressedTextTags}`);
|
|
182754
|
+
logTransformTiming(args.sessionId, "applyHeuristicCleanup", tHeuristic, `droppedTools=${heuristicsResult.droppedTools} deduplicatedTools=${heuristicsResult.deduplicatedTools} droppedInjections=${heuristicsResult.droppedInjections} staleReduce=${heuristicsResult.droppedStaleReduceCalls} compressedTextTags=${heuristicsResult.compressedTextTags} mutatedTextTags=${heuristicsResult.mutatedTextTags}`);
|
|
182303
182755
|
} catch (err) {
|
|
182304
182756
|
sessionLog(args.sessionId, `heuristic cleanup failed (continuing): ${err instanceof Error ? err.message : String(err)}`);
|
|
182305
182757
|
}
|
|
@@ -182336,6 +182788,9 @@ async function runPipeline(args) {
|
|
|
182336
182788
|
}
|
|
182337
182789
|
logTransformTiming(args.sessionId, "clearOldReasoning", tClearReasoning);
|
|
182338
182790
|
logTransformTiming(args.sessionId, "watermarkCleanup", tClearReasoning);
|
|
182791
|
+
if (clearOutcome.cleared > 0 || stripOutcome.stripped > 0) {
|
|
182792
|
+
heuristicOrReasoningDidMutate = true;
|
|
182793
|
+
}
|
|
182339
182794
|
if (combinedWatermark > prevWatermark || clearOutcome.cleared > 0 || stripOutcome.stripped > 0) {
|
|
182340
182795
|
executedWorkThisPass = true;
|
|
182341
182796
|
}
|
|
@@ -182343,7 +182798,32 @@ async function runPipeline(args) {
|
|
|
182343
182798
|
sessionLog(args.sessionId, `reasoning clearing failed (continuing): ${err instanceof Error ? err.message : String(err)}`);
|
|
182344
182799
|
}
|
|
182345
182800
|
}
|
|
182801
|
+
const toolReclaimExecutePass = args.schedulerDecision === "execute";
|
|
182802
|
+
const alreadyMutatingThisPass = pendingOpsDidMutate || heuristicOrReasoningDidMutate;
|
|
182803
|
+
const emergencyDropEligible = args.forceMaterialization === true || args.contextUsage.percentage >= FORCE_MATERIALIZATION_PERCENTAGE;
|
|
182804
|
+
let autoReclaimTargetCount = 0;
|
|
182805
|
+
let autoReclaimDidMutate = false;
|
|
182806
|
+
if (toolReclaimExecutePass && alreadyMutatingThisPass && !emergencyDropEligible) {
|
|
182807
|
+
const reclaimMeta = getOrCreateSessionMeta(args.db, args.sessionId);
|
|
182808
|
+
const syntheticPendingOps = buildSyntheticToolReclaimOps({
|
|
182809
|
+
db: args.db,
|
|
182810
|
+
sessionId: args.sessionId,
|
|
182811
|
+
targets,
|
|
182812
|
+
watermark: reclaimMeta.toolReclaimWatermark ?? 0,
|
|
182813
|
+
pendingOps
|
|
182814
|
+
});
|
|
182815
|
+
autoReclaimTargetCount = syntheticPendingOps.length;
|
|
182816
|
+
if (syntheticPendingOps.length > 0) {
|
|
182817
|
+
autoReclaimDidMutate = applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, [], syntheticPendingOps);
|
|
182818
|
+
}
|
|
182819
|
+
}
|
|
182346
182820
|
transcript.commit();
|
|
182821
|
+
if (toolReclaimExecutePass) {
|
|
182822
|
+
advanceToolReclaimWatermarkToCurrentMax(args.db, args.sessionId);
|
|
182823
|
+
}
|
|
182824
|
+
if (autoReclaimTargetCount > 0) {
|
|
182825
|
+
sessionLog(args.sessionId, `tool reclaim auto-drop: targets=${autoReclaimTargetCount} mutated=${autoReclaimDidMutate}`);
|
|
182826
|
+
}
|
|
182347
182827
|
const postCommitStableIdByRef = new Map;
|
|
182348
182828
|
const postCommitEntryIdByRef = new Map;
|
|
182349
182829
|
for (let i = 0;i < args.messages.length; i++) {
|
|
@@ -182627,6 +183107,7 @@ function clearContextHandlerSession(sessionId) {
|
|
|
182627
183107
|
rawMessageProviderUnregistersBySession.delete(sessionId);
|
|
182628
183108
|
}
|
|
182629
183109
|
clearSessionTracking(sessionId);
|
|
183110
|
+
clearPiEmbedSessionState(sessionId);
|
|
182630
183111
|
}
|
|
182631
183112
|
|
|
182632
183113
|
// src/commands/ctx-flush.ts
|
|
@@ -184471,6 +184952,11 @@ Historian recomp started. Rebuilding compartments and facts from raw Pi session
|
|
|
184471
184952
|
fallbackModelId: ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined
|
|
184472
184953
|
}, parsed.kind === "partial" ? { range: parsed.range } : {});
|
|
184473
184954
|
if (result.published) {
|
|
184955
|
+
try {
|
|
184956
|
+
clearEmergencyRecovery(deps.db, sessionId);
|
|
184957
|
+
} catch (recoveryError) {
|
|
184958
|
+
sessionLog(sessionId, `/ctx-recomp: clearEmergencyRecovery failed (continuing): ${describeError(recoveryError).brief}`);
|
|
184959
|
+
}
|
|
184474
184960
|
try {
|
|
184475
184961
|
stagePiRecompMarker({ db: deps.db, sessionId, ctx });
|
|
184476
184962
|
} catch (markerError) {
|
|
@@ -185124,7 +185610,7 @@ function formatThresholdPercent(value) {
|
|
|
185124
185610
|
// package.json
|
|
185125
185611
|
var package_default = {
|
|
185126
185612
|
name: "@cortexkit/pi-magic-context",
|
|
185127
|
-
version: "0.
|
|
185613
|
+
version: "0.25.0",
|
|
185128
185614
|
type: "module",
|
|
185129
185615
|
description: "Pi coding agent extension for Magic Context — cross-session memory and context management",
|
|
185130
185616
|
main: "dist/index.js",
|
|
@@ -185737,9 +186223,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
185737
186223
|
|
|
185738
186224
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
185739
186225
|
|
|
185740
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
186226
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
186227
|
+
|
|
186228
|
+
Two recovery modes for finer detail:
|
|
186229
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
186230
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
185741
186231
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
185742
186232
|
|
|
186233
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
186234
|
+
function isRecord3(value) {
|
|
186235
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
186236
|
+
}
|
|
186237
|
+
function roleLabel(role) {
|
|
186238
|
+
if (role === "assistant")
|
|
186239
|
+
return "A (assistant)";
|
|
186240
|
+
if (role === "user")
|
|
186241
|
+
return "U (user)";
|
|
186242
|
+
return role;
|
|
186243
|
+
}
|
|
186244
|
+
function truncate2(value, max) {
|
|
186245
|
+
const t = value.trim();
|
|
186246
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
186247
|
+
}
|
|
186248
|
+
function keyArg(input) {
|
|
186249
|
+
if (!input)
|
|
186250
|
+
return "";
|
|
186251
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
186252
|
+
const v = input[k];
|
|
186253
|
+
if (typeof v === "string" && v.length > 0)
|
|
186254
|
+
return truncate2(v, 60);
|
|
186255
|
+
}
|
|
186256
|
+
if (typeof input.description === "string")
|
|
186257
|
+
return truncate2(input.description, 60);
|
|
186258
|
+
return "";
|
|
186259
|
+
}
|
|
186260
|
+
function asToolPart(part) {
|
|
186261
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
186262
|
+
if (type === "tool") {
|
|
186263
|
+
const state = isRecord3(part.state) ? part.state : null;
|
|
186264
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
186265
|
+
const metadata = state && isRecord3(state.metadata) ? state.metadata : null;
|
|
186266
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
186267
|
+
return {
|
|
186268
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
186269
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
186270
|
+
title,
|
|
186271
|
+
input: state && isRecord3(state.input) ? state.input : null,
|
|
186272
|
+
output
|
|
186273
|
+
};
|
|
186274
|
+
}
|
|
186275
|
+
if (type === "tool_use") {
|
|
186276
|
+
return {
|
|
186277
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
186278
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
186279
|
+
title: null,
|
|
186280
|
+
input: isRecord3(part.input) ? part.input : null,
|
|
186281
|
+
output: null
|
|
186282
|
+
};
|
|
186283
|
+
}
|
|
186284
|
+
if (type === "tool_result") {
|
|
186285
|
+
const content = part.content;
|
|
186286
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
186287
|
+
return {
|
|
186288
|
+
name: "tool_result",
|
|
186289
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
186290
|
+
title: null,
|
|
186291
|
+
input: null,
|
|
186292
|
+
output
|
|
186293
|
+
};
|
|
186294
|
+
}
|
|
186295
|
+
return null;
|
|
186296
|
+
}
|
|
186297
|
+
function textOf(part) {
|
|
186298
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
186299
|
+
return part.text;
|
|
186300
|
+
return null;
|
|
186301
|
+
}
|
|
186302
|
+
function reasoningOf(part) {
|
|
186303
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
186304
|
+
return part.text;
|
|
186305
|
+
}
|
|
186306
|
+
return null;
|
|
186307
|
+
}
|
|
186308
|
+
function renderPartPreview(part) {
|
|
186309
|
+
if (!isRecord3(part))
|
|
186310
|
+
return null;
|
|
186311
|
+
const text = textOf(part);
|
|
186312
|
+
if (text !== null) {
|
|
186313
|
+
const t = truncate2(text, 200);
|
|
186314
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
186315
|
+
}
|
|
186316
|
+
const tool = asToolPart(part);
|
|
186317
|
+
if (tool) {
|
|
186318
|
+
const arg = keyArg(tool.input);
|
|
186319
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
186320
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
186321
|
+
}
|
|
186322
|
+
const reasoning = reasoningOf(part);
|
|
186323
|
+
if (reasoning !== null)
|
|
186324
|
+
return ` • [reasoning] ${truncate2(reasoning, 120)}`;
|
|
186325
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
186326
|
+
if (type === "file")
|
|
186327
|
+
return " • [file]";
|
|
186328
|
+
if (type === "step-start" || type === "step-finish")
|
|
186329
|
+
return null;
|
|
186330
|
+
return ` • [${type}]`;
|
|
186331
|
+
}
|
|
186332
|
+
function renderPartFull(part) {
|
|
186333
|
+
if (!isRecord3(part))
|
|
186334
|
+
return null;
|
|
186335
|
+
const text = textOf(part);
|
|
186336
|
+
if (text !== null) {
|
|
186337
|
+
return text.trim().length > 0 ? ` [text]
|
|
186338
|
+
${text}` : null;
|
|
186339
|
+
}
|
|
186340
|
+
const tool = asToolPart(part);
|
|
186341
|
+
if (tool) {
|
|
186342
|
+
const lines = [];
|
|
186343
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
186344
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
186345
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
186346
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
186347
|
+
}
|
|
186348
|
+
if (tool.input)
|
|
186349
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
186350
|
+
if (tool.output !== null)
|
|
186351
|
+
lines.push(` output:
|
|
186352
|
+
${tool.output}`);
|
|
186353
|
+
return lines.join(`
|
|
186354
|
+
`);
|
|
186355
|
+
}
|
|
186356
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
186357
|
+
if (type === "file") {
|
|
186358
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
186359
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
186360
|
+
}
|
|
186361
|
+
return null;
|
|
186362
|
+
}
|
|
186363
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
186364
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
186365
|
+
if (!msg) {
|
|
186366
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
186367
|
+
}
|
|
186368
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
186369
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
186370
|
+
if (rendered.length === 0) {
|
|
186371
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
186372
|
+
} else {
|
|
186373
|
+
lines.push(...rendered);
|
|
186374
|
+
}
|
|
186375
|
+
return lines.join(`
|
|
186376
|
+
`);
|
|
186377
|
+
}
|
|
186378
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
186379
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
186380
|
+
const out = [];
|
|
186381
|
+
let usedTokens = 0;
|
|
186382
|
+
let lastOrdinal = start - 1;
|
|
186383
|
+
let truncated = false;
|
|
186384
|
+
for (const msg of messages) {
|
|
186385
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
186386
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
186387
|
+
const block = partLines.length > 0 ? `${header}
|
|
186388
|
+
${partLines.join(`
|
|
186389
|
+
`)}` : header;
|
|
186390
|
+
const blockTokens = estimateTokens(block);
|
|
186391
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
186392
|
+
truncated = true;
|
|
186393
|
+
break;
|
|
186394
|
+
}
|
|
186395
|
+
out.push(block);
|
|
186396
|
+
usedTokens += blockTokens;
|
|
186397
|
+
lastOrdinal = msg.ordinal;
|
|
186398
|
+
}
|
|
186399
|
+
return { text: out.join(`
|
|
186400
|
+
|
|
186401
|
+
`), lastOrdinal, truncated };
|
|
186402
|
+
}
|
|
186403
|
+
|
|
185743
186404
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
185744
186405
|
var exports_memory = {};
|
|
185745
186406
|
__export(exports_memory, {
|
|
@@ -189805,12 +190466,18 @@ __export(exports_typebox, {
|
|
|
189805
190466
|
});
|
|
189806
190467
|
// src/tools/ctx-expand.ts
|
|
189807
190468
|
var ParamsSchema = exports_typebox.Object({
|
|
189808
|
-
start: exports_typebox.Number({
|
|
190469
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
189809
190470
|
description: "Start message ordinal (from compartment start attribute)"
|
|
189810
|
-
}),
|
|
189811
|
-
end: exports_typebox.Number({
|
|
190471
|
+
})),
|
|
190472
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
189812
190473
|
description: "End message ordinal (from compartment end attribute)"
|
|
189813
|
-
})
|
|
190474
|
+
})),
|
|
190475
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
190476
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
190477
|
+
})),
|
|
190478
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
190479
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
190480
|
+
}))
|
|
189814
190481
|
});
|
|
189815
190482
|
function ok(text) {
|
|
189816
190483
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -189829,22 +190496,41 @@ function createCtxExpandTool(deps) {
|
|
|
189829
190496
|
description: CTX_EXPAND_DESCRIPTION,
|
|
189830
190497
|
parameters: ParamsSchema,
|
|
189831
190498
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
189832
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
189833
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
189834
|
-
}
|
|
189835
190499
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
189836
190500
|
if (!sessionId) {
|
|
189837
190501
|
return err("Error: no active Pi session.");
|
|
189838
190502
|
}
|
|
189839
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
189840
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
189841
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
189842
|
-
}
|
|
189843
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
189844
190503
|
const unregister = setRawMessageProvider(sessionId, {
|
|
189845
190504
|
readMessages: () => readPiSessionMessages(ctx)
|
|
189846
190505
|
});
|
|
189847
190506
|
try {
|
|
190507
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
190508
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
190509
|
+
}
|
|
190510
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
190511
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
190512
|
+
}
|
|
190513
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
190514
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
190515
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
190516
|
+
}
|
|
190517
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
190518
|
+
if (params.verbose === true) {
|
|
190519
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
190520
|
+
if (!v.text) {
|
|
190521
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
190522
|
+
}
|
|
190523
|
+
const out = [
|
|
190524
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
190525
|
+
"",
|
|
190526
|
+
v.text
|
|
190527
|
+
];
|
|
190528
|
+
if (v.truncated) {
|
|
190529
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
190530
|
+
}
|
|
190531
|
+
return ok(out.join(`
|
|
190532
|
+
`));
|
|
190533
|
+
}
|
|
189848
190534
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
189849
190535
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
189850
190536
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -190563,15 +191249,16 @@ function parseInteger(str) {
|
|
|
190563
191249
|
}
|
|
190564
191250
|
|
|
190565
191251
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
190566
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
190567
|
-
|
|
191252
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
191253
|
+
|
|
191254
|
+
How it works:
|
|
191255
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
191256
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
191257
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
190568
191258
|
|
|
190569
|
-
|
|
190570
|
-
|
|
190571
|
-
-
|
|
190572
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
190573
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
190574
|
-
- Dropped content is gone forever.`;
|
|
191259
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
191260
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
191261
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
190575
191262
|
|
|
190576
191263
|
// src/tools/ctx-reduce.ts
|
|
190577
191264
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -191172,7 +191859,21 @@ async function src_default2(pi) {
|
|
|
191172
191859
|
},
|
|
191173
191860
|
historian: hist,
|
|
191174
191861
|
autoSearch: auto,
|
|
191175
|
-
resolveForProject: resolveContextOptionsForProject
|
|
191862
|
+
resolveForProject: resolveContextOptionsForProject,
|
|
191863
|
+
maybeAutoEmbedSession: (sessionId, dir, identity) => {
|
|
191864
|
+
maybeAutoEmbedPiSession({
|
|
191865
|
+
db: database,
|
|
191866
|
+
projectDir: dir,
|
|
191867
|
+
projectIdentity: identity,
|
|
191868
|
+
memoryEnabled: cfg.memory.enabled
|
|
191869
|
+
}, sessionId, dir, identity, (text) => {
|
|
191870
|
+
pi.sendMessage({
|
|
191871
|
+
customType: "ctx-status",
|
|
191872
|
+
content: text,
|
|
191873
|
+
display: true
|
|
191874
|
+
}, { triggerTurn: false });
|
|
191875
|
+
});
|
|
191876
|
+
}
|
|
191176
191877
|
});
|
|
191177
191878
|
function resolveContextOptionsForProject(dir) {
|
|
191178
191879
|
const cached2 = contextOptionsByDir.get(dir);
|
|
@@ -191254,14 +191955,14 @@ async function src_default2(pi) {
|
|
|
191254
191955
|
onProjectSeen: (identity) => seenDreamerProjectIdentities.add(identity)
|
|
191255
191956
|
});
|
|
191256
191957
|
info("registered /ctx-dream");
|
|
191257
|
-
|
|
191958
|
+
registerCtxEmbedCommand(pi, {
|
|
191258
191959
|
db,
|
|
191259
191960
|
projectDir,
|
|
191260
191961
|
projectIdentity,
|
|
191261
191962
|
memoryEnabled: config2.memory.enabled,
|
|
191262
191963
|
resolveProject: resolveCurrentProject
|
|
191263
191964
|
});
|
|
191264
|
-
info("registered /ctx-embed
|
|
191965
|
+
info("registered /ctx-embed");
|
|
191265
191966
|
const dreamerConfig = resolveDreamerFromConfig(config2);
|
|
191266
191967
|
if (dreamerConfig) {
|
|
191267
191968
|
registerPiDreamerProject({
|