@wolfx/pi-magic-context 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1021 -320
- package/dist/subagent-entry.js +592 -182
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -8275,6 +8275,14 @@ function buildNodeSqliteDatabaseClass(DatabaseSync) {
|
|
|
8275
8275
|
}
|
|
8276
8276
|
super(typeof filename === "string" ? filename : ":memory:", translated);
|
|
8277
8277
|
}
|
|
8278
|
+
prepare(sql) {
|
|
8279
|
+
const stmt = super.prepare(sql);
|
|
8280
|
+
for (const method of ["run", "get", "all"]) {
|
|
8281
|
+
const original = stmt[method].bind(stmt);
|
|
8282
|
+
stmt[method] = (...args) => args.length === 1 && Array.isArray(args[0]) ? original(...args[0]) : original(...args);
|
|
8283
|
+
}
|
|
8284
|
+
return stmt;
|
|
8285
|
+
}
|
|
8278
8286
|
transaction(fn) {
|
|
8279
8287
|
const self = this;
|
|
8280
8288
|
const wrapped = function(...args) {
|
|
@@ -141059,6 +141067,8 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
141059
141067
|
var encoder = new TextEncoder;
|
|
141060
141068
|
var TAG_PREFIX_REGEX = /^(?:§\d+§\s*)+/;
|
|
141061
141069
|
var MALFORMED_TAG_PREFIX_REGEX = /^(?:§\d+">§(?:\d+§)?\s*)+/;
|
|
141070
|
+
var DANGLING_TAG_GLOBAL_REGEX = /\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?/g;
|
|
141071
|
+
var DANGLING_TAG_PREFIX_REGEX = /^(?:\u00a7\d+(?!\.\d)[^\s\u00a7\w.]?\s*)+/;
|
|
141062
141072
|
var COMPLETE_TAG_PAIR_GLOBAL_REGEX = /\u00a7\d+\u00a7/g;
|
|
141063
141073
|
var MALFORMED_TAG_GLOBAL_REGEX = /\u00a7\d+">(?:\u00a7(?:\d+\u00a7)?)?/g;
|
|
141064
141074
|
var STRAY_SECTION_CHAR_REGEX = /\u00a7/g;
|
|
@@ -141071,6 +141081,9 @@ function stripCompleteTagPairsGlobally(value) {
|
|
|
141071
141081
|
function stripMalformedTagNotationGlobally(value) {
|
|
141072
141082
|
return value.replace(MALFORMED_TAG_GLOBAL_REGEX, "");
|
|
141073
141083
|
}
|
|
141084
|
+
function stripDanglingTagNotationGlobally(value) {
|
|
141085
|
+
return value.replace(DANGLING_TAG_GLOBAL_REGEX, "");
|
|
141086
|
+
}
|
|
141074
141087
|
function stripTagSectionCharacters(value) {
|
|
141075
141088
|
return value.replace(STRAY_SECTION_CHAR_REGEX, "");
|
|
141076
141089
|
}
|
|
@@ -141078,6 +141091,7 @@ function stripPersistedAssistantText(value) {
|
|
|
141078
141091
|
let text = stripWellFormedLeadingTagPrefix(value);
|
|
141079
141092
|
text = stripCompleteTagPairsGlobally(text);
|
|
141080
141093
|
text = stripMalformedTagNotationGlobally(text);
|
|
141094
|
+
text = stripDanglingTagNotationGlobally(text);
|
|
141081
141095
|
text = stripTagSectionCharacters(text);
|
|
141082
141096
|
return text.trim();
|
|
141083
141097
|
}
|
|
@@ -141090,6 +141104,7 @@ function stripTagPrefix(value) {
|
|
|
141090
141104
|
const prev = stripped;
|
|
141091
141105
|
stripped = stripped.replace(MALFORMED_TAG_PREFIX_REGEX, "");
|
|
141092
141106
|
stripped = stripped.replace(TAG_PREFIX_REGEX, "");
|
|
141107
|
+
stripped = stripped.replace(DANGLING_TAG_PREFIX_REGEX, "");
|
|
141093
141108
|
if (stripped === prev)
|
|
141094
141109
|
break;
|
|
141095
141110
|
}
|
|
@@ -141957,6 +141972,7 @@ var SESSION_META_SELECT_COLUMNS = [
|
|
|
141957
141972
|
"conversation_tokens",
|
|
141958
141973
|
"tool_call_tokens",
|
|
141959
141974
|
"cleared_reasoning_through_tag",
|
|
141975
|
+
"tool_reclaim_watermark",
|
|
141960
141976
|
"last_todo_state",
|
|
141961
141977
|
"cached_m0_bytes",
|
|
141962
141978
|
"cached_m1_bytes",
|
|
@@ -142005,6 +142021,7 @@ var META_COLUMNS = {
|
|
|
142005
142021
|
conversationTokens: "conversation_tokens",
|
|
142006
142022
|
toolCallTokens: "tool_call_tokens",
|
|
142007
142023
|
clearedReasoningThroughTag: "cleared_reasoning_through_tag",
|
|
142024
|
+
toolReclaimWatermark: "tool_reclaim_watermark",
|
|
142008
142025
|
lastTodoState: "last_todo_state",
|
|
142009
142026
|
cachedM0Bytes: "cached_m0_bytes",
|
|
142010
142027
|
cachedM1Bytes: "cached_m1_bytes",
|
|
@@ -142073,7 +142090,7 @@ function isSessionMetaRow(row) {
|
|
|
142073
142090
|
if (row === null || typeof row !== "object")
|
|
142074
142091
|
return false;
|
|
142075
142092
|
const r = row;
|
|
142076
|
-
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme);
|
|
142093
|
+
return typeof r.session_id === "string" && typeof r.last_response_time === "number" && isStringOrNull(r.cache_ttl) && typeof r.counter === "number" && typeof r.last_nudge_tokens === "number" && isStringOrNull(r.last_nudge_band) && isStringOrNull(r.last_transform_error) && typeof r.is_subagent === "number" && typeof r.last_context_percentage === "number" && typeof r.last_input_tokens === "number" && isNumberOrNull(r.observed_safe_input_tokens) && isNumberOrNull(r.cache_alert_sent) && isNumberOrNull(r.times_execute_threshold_reached) && isNumberOrNull(r.compartment_in_progress) && (r.system_prompt_hash === null || typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") && isNumberOrNull(r.system_prompt_tokens) && isNumberOrNull(r.conversation_tokens) && isNumberOrNull(r.tool_call_tokens) && isNumberOrNull(r.cleared_reasoning_through_tag) && isStringOrNull(r.last_todo_state) && isBlobOrNull(r.cached_m0_bytes) && isBlobOrNull(r.cached_m1_bytes) && isNumberOrNull(r.cached_m0_project_memory_epoch) && isStringOrNull(r.cached_m0_workspace_fingerprint) && isNumberOrNull(r.cached_m0_project_user_profile_version) && isNumberOrNull(r.cached_m0_max_compartment_seq) && isNumberOrNull(r.cached_m0_max_memory_id) && isNumberOrNull(r.cached_m0_max_mutation_id) && isNumberOrNull(r.cached_m0_max_memory_mutation_id) && isStringOrNull(r.cached_m0_project_docs_hash) && isNumberOrNull(r.cached_m0_materialized_at) && isNumberOrNull(r.cached_m0_session_facts_version) && isStringOrNull(r.cached_m0_upgrade_state) && isStringOrNull(r.cached_m0_system_hash) && isStringOrNull(r.cached_m0_tool_set_hash) && isStringOrNull(r.cached_m0_model_key) && isStringOrNull(r.last_observed_model_key) && isNumberOrNull(r.last_usage_context_limit) && isNumberOrNull(r.prior_boundary_ordinal) && isNumberOrNull(r.protected_tail_policy_version) && isNumberOrNull(r.protected_tail_drain_window_started_at) && isNumberOrNull(r.protected_tail_drain_tokens) && isNumberOrNull(r.recovery_no_eligible_head_count) && isNumberOrNull(r.force_emergency_bypass_window_start) && isNumberOrNull(r.force_emergency_bypass_used) && isNumberOrNull(r.upgrade_reminded_at) && isNumberOrNull(r.pi_stable_id_scheme) && isNumberOrNull(r.tool_reclaim_watermark);
|
|
142077
142094
|
}
|
|
142078
142095
|
function getDefaultSessionMeta(sessionId) {
|
|
142079
142096
|
return {
|
|
@@ -142096,6 +142113,7 @@ function getDefaultSessionMeta(sessionId) {
|
|
|
142096
142113
|
conversationTokens: 0,
|
|
142097
142114
|
toolCallTokens: 0,
|
|
142098
142115
|
clearedReasoningThroughTag: 0,
|
|
142116
|
+
toolReclaimWatermark: 0,
|
|
142099
142117
|
lastTodoState: "",
|
|
142100
142118
|
cachedM0Bytes: null,
|
|
142101
142119
|
cachedM1Bytes: null,
|
|
@@ -142159,6 +142177,7 @@ function toSessionMeta(row) {
|
|
|
142159
142177
|
conversationTokens: numOrZero(row.conversation_tokens),
|
|
142160
142178
|
toolCallTokens: numOrZero(row.tool_call_tokens),
|
|
142161
142179
|
clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
|
|
142180
|
+
toolReclaimWatermark: numOrZero(row.tool_reclaim_watermark),
|
|
142162
142181
|
lastTodoState: lastTodoStateRaw,
|
|
142163
142182
|
cachedM0Bytes: toBufferOrNull(row.cached_m0_bytes),
|
|
142164
142183
|
cachedM1Bytes: toBufferOrNull(row.cached_m1_bytes),
|
|
@@ -143598,6 +143617,7 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
|
|
|
143598
143617
|
ensureColumn(db, "session_meta", "historian_last_failure_at", "INTEGER DEFAULT NULL");
|
|
143599
143618
|
ensureColumn(db, "session_meta", "system_prompt_hash", "TEXT DEFAULT ''");
|
|
143600
143619
|
ensureColumn(db, "session_meta", "cleared_reasoning_through_tag", "INTEGER DEFAULT 0");
|
|
143620
|
+
ensureColumn(db, "session_meta", "tool_reclaim_watermark", "INTEGER DEFAULT 0");
|
|
143601
143621
|
ensureColumn(db, "session_meta", "stripped_placeholder_ids", "TEXT DEFAULT ''");
|
|
143602
143622
|
ensureColumn(db, "session_meta", "stale_reduce_stripped_ids", "TEXT DEFAULT ''");
|
|
143603
143623
|
ensureColumn(db, "session_meta", "processed_image_stripped_ids", "TEXT DEFAULT ''");
|
|
@@ -146217,6 +146237,7 @@ var SESSION_META_FALLBACK_SELECTS = {
|
|
|
146217
146237
|
last_transform_error: "'' AS last_transform_error",
|
|
146218
146238
|
system_prompt_hash: "'' AS system_prompt_hash",
|
|
146219
146239
|
last_todo_state: "'' AS last_todo_state",
|
|
146240
|
+
tool_reclaim_watermark: "0 AS tool_reclaim_watermark",
|
|
146220
146241
|
cached_m0_bytes: "NULL AS cached_m0_bytes",
|
|
146221
146242
|
cached_m1_bytes: "NULL AS cached_m1_bytes",
|
|
146222
146243
|
cached_m0_project_memory_epoch: "NULL AS cached_m0_project_memory_epoch",
|
|
@@ -146294,6 +146315,14 @@ function updateSessionMeta(db, sessionId, updates) {
|
|
|
146294
146315
|
db.prepare(`UPDATE session_meta SET ${setClauses.join(", ")} WHERE session_id = ?`).run(...values, sessionId);
|
|
146295
146316
|
})();
|
|
146296
146317
|
}
|
|
146318
|
+
function advanceToolReclaimWatermark(db, sessionId, maxTagNumber) {
|
|
146319
|
+
if (maxTagNumber <= 0)
|
|
146320
|
+
return;
|
|
146321
|
+
db.transaction(() => {
|
|
146322
|
+
ensureSessionMetaRow(db, sessionId);
|
|
146323
|
+
db.prepare("UPDATE session_meta SET tool_reclaim_watermark = MAX(COALESCE(tool_reclaim_watermark, 0), ?) WHERE session_id = ?").run(maxTagNumber, sessionId);
|
|
146324
|
+
})();
|
|
146325
|
+
}
|
|
146297
146326
|
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
146298
146327
|
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
146299
146328
|
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
@@ -146668,15 +146697,22 @@ function ownerMessageIdForTagRow(row) {
|
|
|
146668
146697
|
}
|
|
146669
146698
|
return row.message_id.replace(CONTENT_ID_SUFFIX, "");
|
|
146670
146699
|
}
|
|
146671
|
-
function getActiveTagTokenAggregate(db, sessionId) {
|
|
146672
|
-
const
|
|
146700
|
+
function getActiveTagTokenAggregate(db, sessionId, protectedTags = 0) {
|
|
146701
|
+
const toolOutputExpr = protectedTags > 0 ? `COALESCE(SUM(CASE WHEN type = 'tool' AND tag_number < (
|
|
146702
|
+
SELECT tag_number FROM tags
|
|
146703
|
+
WHERE session_id = ? AND status = 'active'
|
|
146704
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
146705
|
+
) THEN COALESCE(token_count, 0) ELSE 0 END), 0)` : `COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)`;
|
|
146706
|
+
const sql = `SELECT
|
|
146673
146707
|
COALESCE(SUM(CASE WHEN type != 'tool' THEN COALESCE(token_count, 0) ELSE 0 END), 0)
|
|
146674
146708
|
+ COALESCE(SUM(COALESCE(reasoning_token_count, 0)), 0) AS conversation,
|
|
146675
146709
|
COALESCE(SUM(CASE WHEN type = 'tool' THEN COALESCE(token_count, 0) + COALESCE(input_token_count, 0) ELSE 0 END), 0) AS tool_call,
|
|
146676
|
-
|
|
146710
|
+
${toolOutputExpr} AS tool_output,
|
|
146677
146711
|
COALESCE(SUM(CASE WHEN token_count IS NULL THEN 1 ELSE 0 END), 0) AS null_count
|
|
146678
146712
|
FROM tags
|
|
146679
|
-
WHERE session_id = ? AND status = 'active'
|
|
146713
|
+
WHERE session_id = ? AND status = 'active'`;
|
|
146714
|
+
const params = protectedTags > 0 ? [sessionId, protectedTags - 1, sessionId] : [sessionId];
|
|
146715
|
+
const row = db.prepare(sql).get(...params);
|
|
146680
146716
|
return {
|
|
146681
146717
|
conversation: row?.conversation ?? 0,
|
|
146682
146718
|
toolCall: row?.tool_call ?? 0,
|
|
@@ -146684,6 +146720,26 @@ function getActiveTagTokenAggregate(db, sessionId) {
|
|
|
146684
146720
|
nullCount: row?.null_count ?? 0
|
|
146685
146721
|
};
|
|
146686
146722
|
}
|
|
146723
|
+
function getOldestActiveUnprotectedToolTags(db, sessionId, protectedTags = 0, limit = 4) {
|
|
146724
|
+
if (limit <= 0)
|
|
146725
|
+
return [];
|
|
146726
|
+
const boundedLimit = Math.max(1, Math.min(10, Math.floor(limit)));
|
|
146727
|
+
const whereProtected = protectedTags > 0 ? `AND tag_number < (
|
|
146728
|
+
SELECT tag_number FROM tags
|
|
146729
|
+
WHERE session_id = ? AND status = 'active'
|
|
146730
|
+
ORDER BY tag_number DESC LIMIT 1 OFFSET ?
|
|
146731
|
+
)` : "";
|
|
146732
|
+
const params = protectedTags > 0 ? [sessionId, sessionId, protectedTags - 1, boundedLimit] : [sessionId, boundedLimit];
|
|
146733
|
+
const rows = db.prepare(`SELECT tag_number, tool_name
|
|
146734
|
+
FROM tags
|
|
146735
|
+
WHERE session_id = ? AND status = 'active' AND type = 'tool' ${whereProtected}
|
|
146736
|
+
ORDER BY tag_number ASC, id ASC
|
|
146737
|
+
LIMIT ?`).all(...params);
|
|
146738
|
+
return rows.filter((row) => typeof row.tag_number === "number").map((row) => ({
|
|
146739
|
+
tagNumber: row.tag_number,
|
|
146740
|
+
toolName: typeof row.tool_name === "string" ? row.tool_name : null
|
|
146741
|
+
}));
|
|
146742
|
+
}
|
|
146687
146743
|
function getTriggerTagTokenUpperBound(db, sessionId) {
|
|
146688
146744
|
const row = db.prepare(`SELECT
|
|
146689
146745
|
COALESCE(SUM(COALESCE(token_count, 0) + COALESCE(input_token_count, 0) + COALESCE(reasoning_token_count, 0)), 0) AS bound,
|
|
@@ -147736,13 +147792,13 @@ async function maybeSendUpgradeReminder(deps, sessionId) {
|
|
|
147736
147792
|
init_data_path();
|
|
147737
147793
|
import * as fs2 from "node:fs";
|
|
147738
147794
|
import * as path6 from "node:path";
|
|
147739
|
-
var ANNOUNCEMENT_VERSION = "0.
|
|
147795
|
+
var ANNOUNCEMENT_VERSION = "0.25.0";
|
|
147740
147796
|
var ANNOUNCEMENT_FEATURES = [
|
|
147741
|
-
"
|
|
147742
|
-
"
|
|
147743
|
-
"
|
|
147744
|
-
"
|
|
147745
|
-
"
|
|
147797
|
+
"Old tool output is now reclaimed automatically: once a file read / search / command output has gone a full execute cycle unused, it's dropped on the next one — no need to call ctx_reduce for stale results.",
|
|
147798
|
+
"Recover anything that was dropped: ctx_expand({ message: N }) returns a dropped message's full content (every tool call's input + output) from storage. ctx_expand({ start, end, verbose: true }) lists a range message-by-message to find it.",
|
|
147799
|
+
"Searchable history made reliable: /ctx-embed shows embedding coverage and runs a resilient backfill (retries transient failures, no longer bails on the first hiccup); the active session now auto-embeds in the background. ctx_reduce guidance also reframed as deferred + recoverable so models trim spent output earlier.",
|
|
147800
|
+
"Pi: fixed /ctx-dream (was failing with 'Unknown named parameter') and local-embedding load failures on Windows/Desktop (#151, #128).",
|
|
147801
|
+
"Runaway background agents on weak/local models are now capped and force-stopped (#154, #152). Plus several prompt-cache busts removed."
|
|
147746
147802
|
];
|
|
147747
147803
|
var ANNOUNCEMENT_FOOTER = "Join us on Discord: https://discord.gg/F2uWxjGnU";
|
|
147748
147804
|
var STATE_FILENAME = "last_announced_version";
|
|
@@ -148686,7 +148742,7 @@ function enqueueDream(db, projectIdentity, reason, force = false) {
|
|
|
148686
148742
|
return db.transaction(() => {
|
|
148687
148743
|
if (!hasActiveDreamLease(db)) {
|
|
148688
148744
|
const staleThresholdMs = force ? 2 * 60 * 1000 : 120 * 60 * 1000;
|
|
148689
|
-
db.prepare("DELETE FROM dream_queue WHERE project_path = ? AND started_at IS NOT NULL AND started_at < ?").run(
|
|
148745
|
+
db.prepare("DELETE FROM dream_queue WHERE project_path = ? AND started_at IS NOT NULL AND started_at < ?").run(projectIdentity, now - staleThresholdMs);
|
|
148690
148746
|
}
|
|
148691
148747
|
const existing = db.prepare("SELECT id FROM dream_queue WHERE project_path = ?").get(projectIdentity);
|
|
148692
148748
|
if (existing) {
|
|
@@ -148821,9 +148877,11 @@ async function promptWithTimeout(client, args, timeoutMs, signal) {
|
|
|
148821
148877
|
});
|
|
148822
148878
|
} catch (error) {
|
|
148823
148879
|
if (signal?.aborted) {
|
|
148880
|
+
await abortChildRun(client, args.path.id);
|
|
148824
148881
|
throw new Error("prompt aborted by external signal");
|
|
148825
148882
|
}
|
|
148826
148883
|
if (controller.signal.aborted) {
|
|
148884
|
+
await abortChildRun(client, args.path.id);
|
|
148827
148885
|
throw new Error(`prompt timed out after ${timeoutMs}ms`);
|
|
148828
148886
|
}
|
|
148829
148887
|
throw error;
|
|
@@ -148832,6 +148890,13 @@ async function promptWithTimeout(client, args, timeoutMs, signal) {
|
|
|
148832
148890
|
signal?.removeEventListener("abort", onExternalAbort);
|
|
148833
148891
|
}
|
|
148834
148892
|
}
|
|
148893
|
+
async function abortChildRun(client, sessionId) {
|
|
148894
|
+
try {
|
|
148895
|
+
await client.session.abort({ path: { id: sessionId } });
|
|
148896
|
+
} catch (error) {
|
|
148897
|
+
log(`[model-retry] child session abort failed for ${sessionId}: ${String(error)}`);
|
|
148898
|
+
}
|
|
148899
|
+
}
|
|
148835
148900
|
function isNonRetryable(error, externalSignal) {
|
|
148836
148901
|
if (externalSignal?.aborted)
|
|
148837
148902
|
return true;
|
|
@@ -149688,6 +149753,20 @@ function getDistinctStoredModelIds(db, projectPath) {
|
|
|
149688
149753
|
const rows = getDistinctStoredModelIdsStatement(db).all(projectPath);
|
|
149689
149754
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
149690
149755
|
}
|
|
149756
|
+
function getMemoryEmbedCoverage(db, projectPath, modelId) {
|
|
149757
|
+
const row = db.prepare(`SELECT
|
|
149758
|
+
COUNT(*) AS total,
|
|
149759
|
+
SUM(CASE WHEN EXISTS (
|
|
149760
|
+
SELECT 1 FROM memory_embeddings e
|
|
149761
|
+
WHERE e.memory_id = m.id AND e.model_id = ?
|
|
149762
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
149763
|
+
FROM memories m
|
|
149764
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(modelId, projectPath);
|
|
149765
|
+
return {
|
|
149766
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
149767
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
149768
|
+
};
|
|
149769
|
+
}
|
|
149691
149770
|
|
|
149692
149771
|
// ../plugin/src/features/magic-context/memory/embedding-cache.ts
|
|
149693
149772
|
var DEFAULT_EMBEDDING_CACHE_TTL_MS = 60000;
|
|
@@ -166296,6 +166375,7 @@ init_logger();
|
|
|
166296
166375
|
// ../plugin/src/features/magic-context/compartment-chunk-embedding.ts
|
|
166297
166376
|
import { createHash as createHash8 } from "node:crypto";
|
|
166298
166377
|
var DEFAULT_COMPARTMENT_CHUNK_MAX_INPUT_TOKENS = 512;
|
|
166378
|
+
var CHUNK_WINDOW_SAFETY_RATIO = 0.9;
|
|
166299
166379
|
var loadFtsRowsStatements = new WeakMap;
|
|
166300
166380
|
var existingHashStatements = new WeakMap;
|
|
166301
166381
|
var existingHashByProjectStatements = new WeakMap;
|
|
@@ -166531,9 +166611,10 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
166531
166611
|
if (lines.length === 0 || endOrdinal < startOrdinal)
|
|
166532
166612
|
return [];
|
|
166533
166613
|
const normalizedMax = normalizeCompartmentChunkMaxInputTokens(maxInputTokens);
|
|
166614
|
+
const effectiveMax = Math.max(1, Math.floor(normalizedMax * CHUNK_WINDOW_SAFETY_RATIO));
|
|
166534
166615
|
const fullText = lines.join(`
|
|
166535
166616
|
`);
|
|
166536
|
-
if (estimateTokens(fullText) <=
|
|
166617
|
+
if (estimateTokens(fullText) <= effectiveMax) {
|
|
166537
166618
|
return [
|
|
166538
166619
|
{
|
|
166539
166620
|
windowIndex: 0,
|
|
@@ -166571,7 +166652,7 @@ function chunkCanonicalText(canonicalText, startOrdinal, endOrdinal, maxInputTok
|
|
|
166571
166652
|
const lineStart = range?.start ?? startOrdinal;
|
|
166572
166653
|
const lineEnd = range?.end ?? lineStart;
|
|
166573
166654
|
const lineTokens = estimateTokens(line);
|
|
166574
|
-
if (currentLines.length > 0 && currentTokens + lineTokens >
|
|
166655
|
+
if (currentLines.length > 0 && currentTokens + lineTokens > effectiveMax) {
|
|
166575
166656
|
flush2();
|
|
166576
166657
|
}
|
|
166577
166658
|
if (currentLines.length === 0) {
|
|
@@ -166727,6 +166808,28 @@ function countUnembeddedSessionCompartments(db, projectPath, sessionId, modelId)
|
|
|
166727
166808
|
)`).get(projectPath, sessionId, projectPath, modelId);
|
|
166728
166809
|
return typeof row?.n === "number" ? row.n : 0;
|
|
166729
166810
|
}
|
|
166811
|
+
function countSessionCompartmentEmbedCoverage(db, projectPath, sessionId, modelId) {
|
|
166812
|
+
const row = db.prepare(`SELECT
|
|
166813
|
+
COUNT(*) AS total,
|
|
166814
|
+
SUM(CASE WHEN EXISTS (
|
|
166815
|
+
SELECT 1 FROM compartment_chunk_embeddings e
|
|
166816
|
+
WHERE e.compartment_id = c.id
|
|
166817
|
+
AND e.project_path = ?
|
|
166818
|
+
AND e.model_id = ?
|
|
166819
|
+
) THEN 1 ELSE 0 END) AS embedded
|
|
166820
|
+
FROM compartments c
|
|
166821
|
+
JOIN session_projects sp
|
|
166822
|
+
ON sp.session_id = c.session_id
|
|
166823
|
+
AND sp.harness = c.harness
|
|
166824
|
+
AND sp.project_path = ?
|
|
166825
|
+
WHERE c.session_id = ?
|
|
166826
|
+
AND c.start_message IS NOT NULL
|
|
166827
|
+
AND c.end_message IS NOT NULL`).get(projectPath, modelId, projectPath, sessionId);
|
|
166828
|
+
return {
|
|
166829
|
+
total: typeof row?.total === "number" ? row.total : 0,
|
|
166830
|
+
embedded: typeof row?.embedded === "number" ? row.embedded : 0
|
|
166831
|
+
};
|
|
166832
|
+
}
|
|
166730
166833
|
|
|
166731
166834
|
// ../plugin/src/features/magic-context/memory/cosine-similarity.ts
|
|
166732
166835
|
function cosineSimilarity(a, b) {
|
|
@@ -166869,6 +166972,19 @@ async function withQuietConsole(fn) {
|
|
|
166869
166972
|
console.error = origError;
|
|
166870
166973
|
}
|
|
166871
166974
|
}
|
|
166975
|
+
var nativeRuntimeMissing = false;
|
|
166976
|
+
function isNativeRuntimeMissingError(error51) {
|
|
166977
|
+
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
166978
|
+
const lower = message.toLowerCase();
|
|
166979
|
+
const code = error51?.code;
|
|
166980
|
+
const name2 = error51?.name;
|
|
166981
|
+
if (code === "ERR_DLOPEN_FAILED" && lower.includes("onnxruntime")) {
|
|
166982
|
+
return true;
|
|
166983
|
+
}
|
|
166984
|
+
if (!lower.includes("onnxruntime-node"))
|
|
166985
|
+
return false;
|
|
166986
|
+
return code === "ERR_MODULE_NOT_FOUND" || name2 === "ResolveMessage" || lower.includes("cannot find package") || lower.includes("cannot find module") || lower.includes("err_module_not_found");
|
|
166987
|
+
}
|
|
166872
166988
|
function isTransientLoadError(error51) {
|
|
166873
166989
|
const message = error51 instanceof Error ? error51.message : String(error51 ?? "");
|
|
166874
166990
|
if (!message)
|
|
@@ -166933,6 +167049,9 @@ class LocalEmbeddingProvider {
|
|
|
166933
167049
|
if (this.pipeline) {
|
|
166934
167050
|
return true;
|
|
166935
167051
|
}
|
|
167052
|
+
if (nativeRuntimeMissing) {
|
|
167053
|
+
return false;
|
|
167054
|
+
}
|
|
166936
167055
|
if (this.initPromise) {
|
|
166937
167056
|
await this.initPromise;
|
|
166938
167057
|
return this.pipeline !== null;
|
|
@@ -166999,7 +167118,12 @@ class LocalEmbeddingProvider {
|
|
|
166999
167118
|
await releaseLock();
|
|
167000
167119
|
}
|
|
167001
167120
|
} catch (error51) {
|
|
167002
|
-
|
|
167121
|
+
if (isNativeRuntimeMissingError(error51)) {
|
|
167122
|
+
nativeRuntimeMissing = true;
|
|
167123
|
+
log("[magic-context] local embedding runtime is not installed (onnxruntime-node missing from this install). Local embeddings are disabled. Fix: reinstall the plugin (run `npx @wolfx/magic-context@latest doctor --force`), or configure an `openai-compatible`/`ollama` embedding endpoint instead. Existing memories are unaffected.");
|
|
167124
|
+
} else {
|
|
167125
|
+
log("[magic-context] embedding model failed to load:", error51);
|
|
167126
|
+
}
|
|
167003
167127
|
this.pipeline = null;
|
|
167004
167128
|
} finally {
|
|
167005
167129
|
this.initPromise = null;
|
|
@@ -167171,6 +167295,13 @@ function blockedEmbeddingEndpointReason(endpoint) {
|
|
|
167171
167295
|
function normalizeEndpoint2(endpoint) {
|
|
167172
167296
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
167173
167297
|
}
|
|
167298
|
+
function embeddingModelsMatch(served, requested) {
|
|
167299
|
+
const a = served.trim().toLowerCase();
|
|
167300
|
+
const b = requested.trim().toLowerCase();
|
|
167301
|
+
if (a.length === 0 || b.length === 0)
|
|
167302
|
+
return true;
|
|
167303
|
+
return a === b || a.includes(b) || b.includes(a);
|
|
167304
|
+
}
|
|
167174
167305
|
var FAILURE_THRESHOLD = 3;
|
|
167175
167306
|
var FAILURE_WINDOW_MS = 60000;
|
|
167176
167307
|
var OPEN_DURATION_MS = 5 * 60000;
|
|
@@ -167188,6 +167319,7 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
167188
167319
|
failureTimes = [];
|
|
167189
167320
|
circuitOpenUntil = 0;
|
|
167190
167321
|
openLogged = false;
|
|
167322
|
+
modelMismatchLogged = false;
|
|
167191
167323
|
halfOpenProbeInFlight = false;
|
|
167192
167324
|
constructor(options) {
|
|
167193
167325
|
this.endpoint = normalizeEndpoint2(options.endpoint);
|
|
@@ -167286,6 +167418,15 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
167286
167418
|
this.recordFailure(isProbe);
|
|
167287
167419
|
return Array.from({ length: texts.length }, () => null);
|
|
167288
167420
|
}
|
|
167421
|
+
const servedModel = typeof body.model === "string" ? body.model : "";
|
|
167422
|
+
if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
|
|
167423
|
+
if (!this.modelMismatchLogged) {
|
|
167424
|
+
log(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${this.model}" served="${servedModel}". The endpoint likely substituted a loaded model; load/select "${this.model}" on the endpoint, or set embedding.model to the served model.`);
|
|
167425
|
+
this.modelMismatchLogged = true;
|
|
167426
|
+
}
|
|
167427
|
+
this.recordFailure(isProbe);
|
|
167428
|
+
return Array.from({ length: texts.length }, () => null);
|
|
167429
|
+
}
|
|
167289
167430
|
const items = Array.isArray(body.data) ? body.data : [];
|
|
167290
167431
|
const results = Array.from({ length: texts.length }, (_, index) => {
|
|
167291
167432
|
const embedding = items[index]?.embedding;
|
|
@@ -167496,6 +167637,118 @@ function getDistinctCommitEmbeddingModelIds(db, projectPath) {
|
|
|
167496
167637
|
return new Set(rows.map((row) => typeof row.modelId === "string" ? row.modelId : null));
|
|
167497
167638
|
}
|
|
167498
167639
|
|
|
167640
|
+
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
167641
|
+
init_logger();
|
|
167642
|
+
var insertStatements = new WeakMap;
|
|
167643
|
+
var existingShasStatements = new WeakMap;
|
|
167644
|
+
var projectCountStatements = new WeakMap;
|
|
167645
|
+
var evictStatements = new WeakMap;
|
|
167646
|
+
var evictOverflowStatements = new WeakMap;
|
|
167647
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
167648
|
+
function getInsertStatement(db) {
|
|
167649
|
+
let stmt = insertStatements.get(db);
|
|
167650
|
+
if (!stmt) {
|
|
167651
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
167652
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
167653
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
167654
|
+
project_path = excluded.project_path,
|
|
167655
|
+
short_sha = excluded.short_sha,
|
|
167656
|
+
message = excluded.message,
|
|
167657
|
+
author = excluded.author,
|
|
167658
|
+
committed_at = excluded.committed_at,
|
|
167659
|
+
indexed_at = excluded.indexed_at
|
|
167660
|
+
WHERE git_commits.message != excluded.message`);
|
|
167661
|
+
insertStatements.set(db, stmt);
|
|
167662
|
+
}
|
|
167663
|
+
return stmt;
|
|
167664
|
+
}
|
|
167665
|
+
function getExistingShasStatement(db) {
|
|
167666
|
+
let stmt = existingShasStatements.get(db);
|
|
167667
|
+
if (!stmt) {
|
|
167668
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
167669
|
+
existingShasStatements.set(db, stmt);
|
|
167670
|
+
}
|
|
167671
|
+
return stmt;
|
|
167672
|
+
}
|
|
167673
|
+
function getProjectCountStatement(db) {
|
|
167674
|
+
let stmt = projectCountStatements.get(db);
|
|
167675
|
+
if (!stmt) {
|
|
167676
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
167677
|
+
projectCountStatements.set(db, stmt);
|
|
167678
|
+
}
|
|
167679
|
+
return stmt;
|
|
167680
|
+
}
|
|
167681
|
+
function getLatestCommitTimeStatement(db) {
|
|
167682
|
+
let stmt = latestCommitTimeStatements.get(db);
|
|
167683
|
+
if (!stmt) {
|
|
167684
|
+
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
167685
|
+
latestCommitTimeStatements.set(db, stmt);
|
|
167686
|
+
}
|
|
167687
|
+
return stmt;
|
|
167688
|
+
}
|
|
167689
|
+
function getEvictOverflowStatement(db) {
|
|
167690
|
+
let stmt = evictOverflowStatements.get(db);
|
|
167691
|
+
if (!stmt) {
|
|
167692
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
167693
|
+
WHERE rowid IN (
|
|
167694
|
+
SELECT rowid FROM git_commits
|
|
167695
|
+
WHERE project_path = ?
|
|
167696
|
+
ORDER BY committed_at DESC, sha DESC
|
|
167697
|
+
LIMIT -1 OFFSET ?
|
|
167698
|
+
)`);
|
|
167699
|
+
evictOverflowStatements.set(db, stmt);
|
|
167700
|
+
}
|
|
167701
|
+
return stmt;
|
|
167702
|
+
}
|
|
167703
|
+
function upsertCommits(db, projectPath, commits) {
|
|
167704
|
+
if (commits.length === 0)
|
|
167705
|
+
return { inserted: 0, updated: 0 };
|
|
167706
|
+
const existing = new Set;
|
|
167707
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
167708
|
+
existing.add(row.sha);
|
|
167709
|
+
}
|
|
167710
|
+
let inserted = 0;
|
|
167711
|
+
let updated = 0;
|
|
167712
|
+
const now = Date.now();
|
|
167713
|
+
const insertStmt = getInsertStatement(db);
|
|
167714
|
+
db.transaction(() => {
|
|
167715
|
+
for (const commit of commits) {
|
|
167716
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
167717
|
+
if (result.changes > 0) {
|
|
167718
|
+
if (existing.has(commit.sha)) {
|
|
167719
|
+
updated++;
|
|
167720
|
+
} else {
|
|
167721
|
+
inserted++;
|
|
167722
|
+
existing.add(commit.sha);
|
|
167723
|
+
}
|
|
167724
|
+
}
|
|
167725
|
+
}
|
|
167726
|
+
})();
|
|
167727
|
+
return { inserted, updated };
|
|
167728
|
+
}
|
|
167729
|
+
function getCommitCount(db, projectPath) {
|
|
167730
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
167731
|
+
return row?.count ?? 0;
|
|
167732
|
+
}
|
|
167733
|
+
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
167734
|
+
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
167735
|
+
return row?.latest ?? null;
|
|
167736
|
+
}
|
|
167737
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
167738
|
+
if (maxCommits <= 0)
|
|
167739
|
+
return 0;
|
|
167740
|
+
const count = getCommitCount(db, projectPath);
|
|
167741
|
+
if (count <= maxCommits)
|
|
167742
|
+
return 0;
|
|
167743
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
167744
|
+
const after = getCommitCount(db, projectPath);
|
|
167745
|
+
const evicted = Math.max(0, count - after);
|
|
167746
|
+
if (evicted > 0) {
|
|
167747
|
+
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
167748
|
+
}
|
|
167749
|
+
return evicted;
|
|
167750
|
+
}
|
|
167751
|
+
|
|
167499
167752
|
// ../plugin/src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
167500
167753
|
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
167501
167754
|
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
@@ -167690,8 +167943,12 @@ function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath) {
|
|
|
167690
167943
|
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
167691
167944
|
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
167692
167945
|
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
167693
|
-
var MAX_WINDOWS_PER_EMBED_CALL =
|
|
167946
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
167694
167947
|
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
167948
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
167949
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
167950
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
167951
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
167695
167952
|
var projectRegistrations = new Map;
|
|
167696
167953
|
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
167697
167954
|
var globalRegistrationGeneration = 0;
|
|
@@ -167768,7 +168025,7 @@ function getChunkEmbeddingModelId(config2, providerIdentity) {
|
|
|
167768
168025
|
}
|
|
167769
168026
|
const chunkIdentity = {
|
|
167770
168027
|
providerIdentity,
|
|
167771
|
-
chunkerVersion:
|
|
168028
|
+
chunkerVersion: 2,
|
|
167772
168029
|
maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config2 ? config2.max_input_tokens : undefined),
|
|
167773
168030
|
truncate: config2.provider === "openai-compatible" ? config2.truncate ?? "" : ""
|
|
167774
168031
|
};
|
|
@@ -167791,7 +168048,9 @@ function snapshotFor(registration) {
|
|
|
167791
168048
|
enabled,
|
|
167792
168049
|
gitCommitEnabled,
|
|
167793
168050
|
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
167794
|
-
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId
|
|
168051
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
168052
|
+
model: registration.observationMode || !providerIsOn ? "off" : ("model" in registration.config) && registration.config.model.trim() ? registration.config.model.trim() : registration.modelId,
|
|
168053
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local"
|
|
167795
168054
|
};
|
|
167796
168055
|
}
|
|
167797
168056
|
function disposeProvider(provider) {
|
|
@@ -168002,8 +168261,9 @@ async function embedUnembeddedMemoriesForProject(db, projectIdentity, batchSize
|
|
|
168002
168261
|
}
|
|
168003
168262
|
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
168004
168263
|
const noWork = [];
|
|
168264
|
+
const failed = [];
|
|
168005
168265
|
if (candidates.length === 0)
|
|
168006
|
-
return { embedded: 0, noWork };
|
|
168266
|
+
return { embedded: 0, noWork, failed };
|
|
168007
168267
|
const maxInputTokens = getProjectEmbeddingMaxInputTokens(projectIdentity);
|
|
168008
168268
|
const prepared = [];
|
|
168009
168269
|
for (const candidate of candidates) {
|
|
@@ -168020,7 +168280,7 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
168020
168280
|
prepared.push({ candidate, windows });
|
|
168021
168281
|
}
|
|
168022
168282
|
if (prepared.length === 0)
|
|
168023
|
-
return { embedded: 0, noWork };
|
|
168283
|
+
return { embedded: 0, noWork, failed };
|
|
168024
168284
|
let embedded = 0;
|
|
168025
168285
|
let i = 0;
|
|
168026
168286
|
while (i < prepared.length) {
|
|
@@ -168037,35 +168297,60 @@ async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates
|
|
|
168037
168297
|
const texts = [];
|
|
168038
168298
|
for (const item of slice)
|
|
168039
168299
|
texts.push(...item.windows.map((w) => w.text));
|
|
168040
|
-
|
|
168041
|
-
|
|
168042
|
-
if (!result)
|
|
168043
|
-
continue;
|
|
168300
|
+
const persistedIds = new Set;
|
|
168301
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
168044
168302
|
if (signal?.aborted)
|
|
168045
168303
|
break;
|
|
168046
|
-
let
|
|
168047
|
-
|
|
168048
|
-
|
|
168049
|
-
|
|
168050
|
-
|
|
168051
|
-
|
|
168304
|
+
let result = null;
|
|
168305
|
+
const attemptStart = Date.now();
|
|
168306
|
+
try {
|
|
168307
|
+
result = await embedBatchForProject(projectIdentity, texts, signal);
|
|
168308
|
+
} catch (error51) {
|
|
168309
|
+
log("[magic-context] failed to proactively embed compartment chunks:", error51);
|
|
168310
|
+
}
|
|
168311
|
+
if (signal?.aborted)
|
|
168312
|
+
break;
|
|
168313
|
+
if (result) {
|
|
168314
|
+
let offset = 0;
|
|
168315
|
+
for (const item of slice) {
|
|
168316
|
+
const vectors = result.vectors.slice(offset, offset + item.windows.length);
|
|
168317
|
+
offset += item.windows.length;
|
|
168318
|
+
if (persistedIds.has(item.candidate.id))
|
|
168319
|
+
continue;
|
|
168320
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
168321
|
+
continue;
|
|
168322
|
+
}
|
|
168323
|
+
const rows = item.windows.map((window, index) => ({
|
|
168324
|
+
compartmentId: item.candidate.id,
|
|
168325
|
+
sessionId: item.candidate.sessionId,
|
|
168326
|
+
projectPath: projectIdentity,
|
|
168327
|
+
window,
|
|
168328
|
+
modelId,
|
|
168329
|
+
vector: vectors[index]
|
|
168330
|
+
}));
|
|
168331
|
+
replaceCompartmentChunkEmbeddings(db, rows);
|
|
168332
|
+
persistedIds.add(item.candidate.id);
|
|
168052
168333
|
}
|
|
168053
|
-
const rows = item.windows.map((window, index) => ({
|
|
168054
|
-
compartmentId: item.candidate.id,
|
|
168055
|
-
sessionId: item.candidate.sessionId,
|
|
168056
|
-
projectPath: projectIdentity,
|
|
168057
|
-
window,
|
|
168058
|
-
modelId,
|
|
168059
|
-
vector: vectors[index]
|
|
168060
|
-
}));
|
|
168061
|
-
replaceCompartmentChunkEmbeddings(db, rows);
|
|
168062
|
-
embedded += 1;
|
|
168063
168334
|
}
|
|
168064
|
-
|
|
168065
|
-
|
|
168335
|
+
if (persistedIds.size === slice.length)
|
|
168336
|
+
break;
|
|
168337
|
+
if (persistedIds.size > 0)
|
|
168338
|
+
break;
|
|
168339
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
168340
|
+
break;
|
|
168341
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
168342
|
+
await new Promise((resolve4) => setTimeout(resolve4, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
168343
|
+
}
|
|
168344
|
+
}
|
|
168345
|
+
embedded += persistedIds.size;
|
|
168346
|
+
if (!signal?.aborted) {
|
|
168347
|
+
for (const item of slice) {
|
|
168348
|
+
if (!persistedIds.has(item.candidate.id))
|
|
168349
|
+
failed.push(item.candidate.id);
|
|
168350
|
+
}
|
|
168066
168351
|
}
|
|
168067
168352
|
}
|
|
168068
|
-
return { embedded, noWork };
|
|
168353
|
+
return { embedded, noWork, failed };
|
|
168069
168354
|
}
|
|
168070
168355
|
async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, options) {
|
|
168071
168356
|
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
@@ -168088,9 +168373,11 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168088
168373
|
renewal.unref?.();
|
|
168089
168374
|
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
168090
168375
|
const skipIds = [];
|
|
168376
|
+
const failedIds = [];
|
|
168091
168377
|
let embedded = 0;
|
|
168092
168378
|
let aborted2 = false;
|
|
168093
|
-
let
|
|
168379
|
+
let providerDown = false;
|
|
168380
|
+
let consecutiveFailedBatches = 0;
|
|
168094
168381
|
try {
|
|
168095
168382
|
options?.onProgress?.({ embedded, total });
|
|
168096
168383
|
for (;; ) {
|
|
@@ -168098,15 +168385,26 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168098
168385
|
aborted2 = true;
|
|
168099
168386
|
break;
|
|
168100
168387
|
}
|
|
168101
|
-
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, skipIds);
|
|
168388
|
+
const candidates = loadUnembeddedSessionChunkCandidates(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds]);
|
|
168102
168389
|
if (candidates.length === 0)
|
|
168103
168390
|
break;
|
|
168104
|
-
const {
|
|
168391
|
+
const {
|
|
168392
|
+
embedded: n,
|
|
168393
|
+
noWork,
|
|
168394
|
+
failed
|
|
168395
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, options?.signal);
|
|
168105
168396
|
for (const id of noWork)
|
|
168106
168397
|
skipIds.push(id);
|
|
168398
|
+
for (const id of failed)
|
|
168399
|
+
failedIds.push(id);
|
|
168107
168400
|
if (n === 0 && noWork.length === 0) {
|
|
168108
|
-
|
|
168109
|
-
|
|
168401
|
+
consecutiveFailedBatches += 1;
|
|
168402
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
168403
|
+
providerDown = true;
|
|
168404
|
+
break;
|
|
168405
|
+
}
|
|
168406
|
+
} else {
|
|
168407
|
+
consecutiveFailedBatches = 0;
|
|
168110
168408
|
}
|
|
168111
168409
|
embedded += n;
|
|
168112
168410
|
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
@@ -168114,16 +168412,50 @@ async function embedSessionCompartmentChunks(db, projectIdentity, sessionId, opt
|
|
|
168114
168412
|
}
|
|
168115
168413
|
} finally {
|
|
168116
168414
|
clearInterval(renewal);
|
|
168117
|
-
|
|
168415
|
+
try {
|
|
168416
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
168417
|
+
} catch (error51) {
|
|
168418
|
+
log("[magic-context] embed drain: lease release failed (will TTL-expire):", error51);
|
|
168419
|
+
}
|
|
168118
168420
|
}
|
|
168119
168421
|
if (aborted2)
|
|
168120
|
-
return { status: "aborted", embedded, total };
|
|
168121
|
-
if (
|
|
168422
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
168423
|
+
if (providerDown || failedIds.length > 0) {
|
|
168122
168424
|
const remaining = Math.max(0, countUnembeddedSessionCompartments(db, projectIdentity, sessionId, snapshot.chunkModelId) - skipIds.length);
|
|
168123
|
-
if (remaining > 0)
|
|
168124
|
-
return { status: "stalled", embedded, total, remaining };
|
|
168425
|
+
if (remaining > 0) {
|
|
168426
|
+
return { status: "stalled", embedded, total, remaining, failed: failedIds.length };
|
|
168427
|
+
}
|
|
168125
168428
|
}
|
|
168126
|
-
return { status: "done", embedded, total };
|
|
168429
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
168430
|
+
}
|
|
168431
|
+
function getEmbeddingCoverageStatus(db, projectIdentity, sessionId) {
|
|
168432
|
+
const snapshot = getProjectEmbeddingSnapshot(projectIdentity);
|
|
168433
|
+
if (!snapshot?.enabled || snapshot.chunkModelId === "off") {
|
|
168434
|
+
return {
|
|
168435
|
+
enabled: false,
|
|
168436
|
+
model: snapshot?.model ?? "off",
|
|
168437
|
+
provider: snapshot?.provider ?? "off",
|
|
168438
|
+
session: { embedded: 0, total: 0 },
|
|
168439
|
+
memories: { embedded: 0, total: 0 },
|
|
168440
|
+
commits: { embedded: 0, total: 0, gitEnabled: false }
|
|
168441
|
+
};
|
|
168442
|
+
}
|
|
168443
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId);
|
|
168444
|
+
const memories = getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId);
|
|
168445
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
168446
|
+
const commits = gitEnabled ? {
|
|
168447
|
+
embedded: countEmbeddedCommits(db, projectIdentity),
|
|
168448
|
+
total: getCommitCount(db, projectIdentity),
|
|
168449
|
+
gitEnabled: true
|
|
168450
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
168451
|
+
return {
|
|
168452
|
+
enabled: true,
|
|
168453
|
+
model: snapshot.model,
|
|
168454
|
+
provider: snapshot.provider,
|
|
168455
|
+
session,
|
|
168456
|
+
memories,
|
|
168457
|
+
commits
|
|
168458
|
+
};
|
|
168127
168459
|
}
|
|
168128
168460
|
|
|
168129
168461
|
// ../plugin/src/features/magic-context/memory/embedding.ts
|
|
@@ -168172,118 +168504,6 @@ async function embedText(text, signal) {
|
|
|
168172
168504
|
}
|
|
168173
168505
|
var SWEEP_MAX_WALL_CLOCK_MS2 = 10 * 60 * 1000;
|
|
168174
168506
|
|
|
168175
|
-
// ../plugin/src/features/magic-context/git-commits/storage-git-commits.ts
|
|
168176
|
-
init_logger();
|
|
168177
|
-
var insertStatements = new WeakMap;
|
|
168178
|
-
var existingShasStatements = new WeakMap;
|
|
168179
|
-
var projectCountStatements = new WeakMap;
|
|
168180
|
-
var evictStatements = new WeakMap;
|
|
168181
|
-
var evictOverflowStatements = new WeakMap;
|
|
168182
|
-
var latestCommitTimeStatements = new WeakMap;
|
|
168183
|
-
function getInsertStatement(db) {
|
|
168184
|
-
let stmt = insertStatements.get(db);
|
|
168185
|
-
if (!stmt) {
|
|
168186
|
-
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
168187
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
168188
|
-
ON CONFLICT(sha) DO UPDATE SET
|
|
168189
|
-
project_path = excluded.project_path,
|
|
168190
|
-
short_sha = excluded.short_sha,
|
|
168191
|
-
message = excluded.message,
|
|
168192
|
-
author = excluded.author,
|
|
168193
|
-
committed_at = excluded.committed_at,
|
|
168194
|
-
indexed_at = excluded.indexed_at
|
|
168195
|
-
WHERE git_commits.message != excluded.message`);
|
|
168196
|
-
insertStatements.set(db, stmt);
|
|
168197
|
-
}
|
|
168198
|
-
return stmt;
|
|
168199
|
-
}
|
|
168200
|
-
function getExistingShasStatement(db) {
|
|
168201
|
-
let stmt = existingShasStatements.get(db);
|
|
168202
|
-
if (!stmt) {
|
|
168203
|
-
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
168204
|
-
existingShasStatements.set(db, stmt);
|
|
168205
|
-
}
|
|
168206
|
-
return stmt;
|
|
168207
|
-
}
|
|
168208
|
-
function getProjectCountStatement(db) {
|
|
168209
|
-
let stmt = projectCountStatements.get(db);
|
|
168210
|
-
if (!stmt) {
|
|
168211
|
-
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
168212
|
-
projectCountStatements.set(db, stmt);
|
|
168213
|
-
}
|
|
168214
|
-
return stmt;
|
|
168215
|
-
}
|
|
168216
|
-
function getLatestCommitTimeStatement(db) {
|
|
168217
|
-
let stmt = latestCommitTimeStatements.get(db);
|
|
168218
|
-
if (!stmt) {
|
|
168219
|
-
stmt = db.prepare("SELECT MAX(committed_at) AS latest FROM git_commits WHERE project_path = ?");
|
|
168220
|
-
latestCommitTimeStatements.set(db, stmt);
|
|
168221
|
-
}
|
|
168222
|
-
return stmt;
|
|
168223
|
-
}
|
|
168224
|
-
function getEvictOverflowStatement(db) {
|
|
168225
|
-
let stmt = evictOverflowStatements.get(db);
|
|
168226
|
-
if (!stmt) {
|
|
168227
|
-
stmt = db.prepare(`DELETE FROM git_commits
|
|
168228
|
-
WHERE rowid IN (
|
|
168229
|
-
SELECT rowid FROM git_commits
|
|
168230
|
-
WHERE project_path = ?
|
|
168231
|
-
ORDER BY committed_at DESC, sha DESC
|
|
168232
|
-
LIMIT -1 OFFSET ?
|
|
168233
|
-
)`);
|
|
168234
|
-
evictOverflowStatements.set(db, stmt);
|
|
168235
|
-
}
|
|
168236
|
-
return stmt;
|
|
168237
|
-
}
|
|
168238
|
-
function upsertCommits(db, projectPath, commits) {
|
|
168239
|
-
if (commits.length === 0)
|
|
168240
|
-
return { inserted: 0, updated: 0 };
|
|
168241
|
-
const existing = new Set;
|
|
168242
|
-
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
168243
|
-
existing.add(row.sha);
|
|
168244
|
-
}
|
|
168245
|
-
let inserted = 0;
|
|
168246
|
-
let updated = 0;
|
|
168247
|
-
const now = Date.now();
|
|
168248
|
-
const insertStmt = getInsertStatement(db);
|
|
168249
|
-
db.transaction(() => {
|
|
168250
|
-
for (const commit of commits) {
|
|
168251
|
-
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
168252
|
-
if (result.changes > 0) {
|
|
168253
|
-
if (existing.has(commit.sha)) {
|
|
168254
|
-
updated++;
|
|
168255
|
-
} else {
|
|
168256
|
-
inserted++;
|
|
168257
|
-
existing.add(commit.sha);
|
|
168258
|
-
}
|
|
168259
|
-
}
|
|
168260
|
-
}
|
|
168261
|
-
})();
|
|
168262
|
-
return { inserted, updated };
|
|
168263
|
-
}
|
|
168264
|
-
function getCommitCount(db, projectPath) {
|
|
168265
|
-
const row = getProjectCountStatement(db).get(projectPath);
|
|
168266
|
-
return row?.count ?? 0;
|
|
168267
|
-
}
|
|
168268
|
-
function getLatestIndexedCommitTimeMs(db, projectPath) {
|
|
168269
|
-
const row = getLatestCommitTimeStatement(db).get(projectPath);
|
|
168270
|
-
return row?.latest ?? null;
|
|
168271
|
-
}
|
|
168272
|
-
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
168273
|
-
if (maxCommits <= 0)
|
|
168274
|
-
return 0;
|
|
168275
|
-
const count = getCommitCount(db, projectPath);
|
|
168276
|
-
if (count <= maxCommits)
|
|
168277
|
-
return 0;
|
|
168278
|
-
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
168279
|
-
const after = getCommitCount(db, projectPath);
|
|
168280
|
-
const evicted = Math.max(0, count - after);
|
|
168281
|
-
if (evicted > 0) {
|
|
168282
|
-
log(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
168283
|
-
}
|
|
168284
|
-
return evicted;
|
|
168285
|
-
}
|
|
168286
|
-
|
|
168287
168507
|
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
168288
168508
|
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
168289
168509
|
var EMBED_BATCH_SIZE = 16;
|
|
@@ -169817,87 +170037,229 @@ function registerCtxDreamCommand(pi, deps) {
|
|
|
169817
170037
|
});
|
|
169818
170038
|
}
|
|
169819
170039
|
|
|
169820
|
-
// src/
|
|
169821
|
-
|
|
169822
|
-
|
|
169823
|
-
|
|
169824
|
-
|
|
170040
|
+
// ../plugin/src/hooks/magic-context/embed-session-state.ts
|
|
170041
|
+
var embedPauseBySession = new Set;
|
|
170042
|
+
var embedRunStateBySession = new Map;
|
|
170043
|
+
var autoEmbedAttemptedBySession = new Set;
|
|
170044
|
+
|
|
170045
|
+
// ../plugin/src/hooks/magic-context/format-embed-status.ts
|
|
170046
|
+
function formatEmbedStatusText(coverage, drain) {
|
|
170047
|
+
if (!coverage.enabled) {
|
|
170048
|
+
return "Embedding is off (no provider configured).";
|
|
170049
|
+
}
|
|
170050
|
+
const lines = [];
|
|
170051
|
+
lines.push(`Embedding — model: ${coverage.model} (${coverage.provider})`);
|
|
170052
|
+
lines.push(`This session: ${coverage.session.embedded} / ${coverage.session.total} compartments embedded`);
|
|
170053
|
+
lines.push(`Project memories: ${coverage.memories.embedded} / ${coverage.memories.total} embedded`);
|
|
170054
|
+
if (coverage.commits.gitEnabled) {
|
|
170055
|
+
lines.push(`Git commits: ${coverage.commits.embedded} / ${coverage.commits.total}`);
|
|
170056
|
+
} else {
|
|
170057
|
+
lines.push("Git commits: 0 / 0 (git indexing off)");
|
|
170058
|
+
}
|
|
170059
|
+
let drainLine = "Drain: idle";
|
|
170060
|
+
switch (drain.status) {
|
|
170061
|
+
case "running": {
|
|
170062
|
+
const e = drain.embedded ?? coverage.session.embedded;
|
|
170063
|
+
const t = drain.total ?? coverage.session.total;
|
|
170064
|
+
const failedSuffix = drain.failed && drain.failed > 0 ? ` (${drain.failed} failed)` : "";
|
|
170065
|
+
drainLine = `Drain: running ${e}/${t}${failedSuffix}`;
|
|
170066
|
+
break;
|
|
170067
|
+
}
|
|
170068
|
+
case "paused": {
|
|
170069
|
+
const e = drain.embedded ?? coverage.session.embedded;
|
|
170070
|
+
const t = drain.total ?? coverage.session.total;
|
|
170071
|
+
drainLine = `Drain: paused ${e}/${t}`;
|
|
170072
|
+
break;
|
|
170073
|
+
}
|
|
170074
|
+
case "stopped":
|
|
170075
|
+
drainLine = "Drain: stopped (provider down)";
|
|
170076
|
+
break;
|
|
170077
|
+
default:
|
|
170078
|
+
drainLine = "Drain: idle";
|
|
170079
|
+
}
|
|
170080
|
+
lines.push(drainLine);
|
|
170081
|
+
return lines.join(`
|
|
170082
|
+
`);
|
|
170083
|
+
}
|
|
170084
|
+
|
|
170085
|
+
// src/commands/ctx-embed.ts
|
|
170086
|
+
function clearPiEmbedSessionState(sessionId) {
|
|
170087
|
+
embedPauseBySession.delete(sessionId);
|
|
170088
|
+
const ctrl = embedRunStateBySession.get(sessionId);
|
|
170089
|
+
if (ctrl) {
|
|
170090
|
+
ctrl.abort();
|
|
170091
|
+
embedRunStateBySession.delete(sessionId);
|
|
170092
|
+
}
|
|
170093
|
+
autoEmbedAttemptedBySession.delete(sessionId);
|
|
170094
|
+
}
|
|
170095
|
+
async function runEmbedDrain(db, projectIdentity, sessionId) {
|
|
170096
|
+
const activeCtrl = embedRunStateBySession.get(sessionId);
|
|
170097
|
+
if (activeCtrl && !activeCtrl.signal.aborted) {
|
|
170098
|
+
return {
|
|
170099
|
+
text: `## /ctx-embed
|
|
170100
|
+
|
|
170101
|
+
Embedding is already running for this session.`,
|
|
170102
|
+
level: "info"
|
|
170103
|
+
};
|
|
170104
|
+
}
|
|
170105
|
+
embedPauseBySession.delete(sessionId);
|
|
170106
|
+
const prior = embedRunStateBySession.get(sessionId);
|
|
170107
|
+
if (prior)
|
|
170108
|
+
prior.abort();
|
|
170109
|
+
const controller = new AbortController;
|
|
170110
|
+
embedRunStateBySession.set(sessionId, controller);
|
|
170111
|
+
let outcome;
|
|
170112
|
+
try {
|
|
170113
|
+
outcome = await embedSessionCompartmentChunks(db, projectIdentity, sessionId, {
|
|
170114
|
+
signal: controller.signal
|
|
170115
|
+
});
|
|
170116
|
+
} finally {
|
|
170117
|
+
if (embedRunStateBySession.get(sessionId) === controller) {
|
|
170118
|
+
embedRunStateBySession.delete(sessionId);
|
|
170119
|
+
}
|
|
170120
|
+
}
|
|
170121
|
+
switch (outcome.status) {
|
|
170122
|
+
case "nothing":
|
|
170123
|
+
return {
|
|
170124
|
+
text: `## /ctx-embed
|
|
170125
|
+
|
|
170126
|
+
All of this session's history is already embedded.`,
|
|
170127
|
+
level: "info"
|
|
170128
|
+
};
|
|
170129
|
+
case "disabled":
|
|
170130
|
+
return {
|
|
170131
|
+
text: `## /ctx-embed
|
|
170132
|
+
|
|
170133
|
+
No embedding provider is configured, so there is nothing to embed.`,
|
|
170134
|
+
level: "info"
|
|
170135
|
+
};
|
|
170136
|
+
case "busy":
|
|
170137
|
+
return {
|
|
170138
|
+
text: `## /ctx-embed
|
|
170139
|
+
|
|
170140
|
+
Embedding is already running for this project. Try again shortly.`,
|
|
170141
|
+
level: "info"
|
|
170142
|
+
};
|
|
170143
|
+
case "aborted": {
|
|
170144
|
+
const cov = getEmbeddingCoverageStatus(db, projectIdentity, sessionId);
|
|
170145
|
+
return {
|
|
170146
|
+
text: `## /ctx-embed
|
|
170147
|
+
|
|
170148
|
+
Paused at ${cov.session.embedded}/${cov.session.total} compartments embedded.`,
|
|
170149
|
+
level: "info"
|
|
170150
|
+
};
|
|
170151
|
+
}
|
|
170152
|
+
case "stalled":
|
|
170153
|
+
return {
|
|
170154
|
+
text: `## /ctx-embed
|
|
170155
|
+
|
|
170156
|
+
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"}; ${outcome.remaining} could not be embedded (the provider returned no result). Run /ctx-embed start again to retry them.`,
|
|
170157
|
+
level: "info"
|
|
170158
|
+
};
|
|
170159
|
+
default:
|
|
170160
|
+
return {
|
|
170161
|
+
text: `## /ctx-embed
|
|
170162
|
+
|
|
170163
|
+
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"} of history for semantic search.`,
|
|
170164
|
+
level: "success"
|
|
170165
|
+
};
|
|
170166
|
+
}
|
|
170167
|
+
}
|
|
170168
|
+
function registerCtxEmbedCommand(pi, deps) {
|
|
170169
|
+
pi.registerCommand("ctx-embed", {
|
|
170170
|
+
description: "Embedding status, or start/pause history compartment embedding (start | pause)",
|
|
170171
|
+
handler: async (args, ctx) => {
|
|
169825
170172
|
const sessionId = resolveSessionId(ctx);
|
|
169826
170173
|
if (!sessionId) {
|
|
169827
170174
|
sendCtxStatusMessage(pi, {
|
|
169828
|
-
title: "/ctx-embed
|
|
169829
|
-
text: `## /ctx-embed
|
|
170175
|
+
title: "/ctx-embed",
|
|
170176
|
+
text: `## /ctx-embed
|
|
169830
170177
|
|
|
169831
170178
|
No active Pi session is available.`,
|
|
169832
170179
|
level: "error"
|
|
169833
170180
|
});
|
|
169834
170181
|
return;
|
|
169835
170182
|
}
|
|
170183
|
+
const project = deps.resolveProject?.(ctx) ?? {
|
|
170184
|
+
projectDir: deps.projectDir,
|
|
170185
|
+
projectIdentity: deps.projectIdentity
|
|
170186
|
+
};
|
|
170187
|
+
const sub = args.trim().toLowerCase();
|
|
170188
|
+
if (sub === "pause") {
|
|
170189
|
+
embedPauseBySession.add(sessionId);
|
|
170190
|
+
const ctrl = embedRunStateBySession.get(sessionId);
|
|
170191
|
+
if (ctrl)
|
|
170192
|
+
ctrl.abort();
|
|
170193
|
+
const cov = getEmbeddingCoverageStatus(deps.db, project.projectIdentity, sessionId);
|
|
170194
|
+
sendCtxStatusMessage(pi, {
|
|
170195
|
+
title: "/ctx-embed",
|
|
170196
|
+
text: `## /ctx-embed
|
|
170197
|
+
|
|
170198
|
+
Paused at ${cov.session.embedded}/${cov.session.total} compartments embedded.`,
|
|
170199
|
+
level: "info"
|
|
170200
|
+
});
|
|
170201
|
+
return;
|
|
170202
|
+
}
|
|
169836
170203
|
if (deps.memoryEnabled === false) {
|
|
169837
170204
|
sendCtxStatusMessage(pi, {
|
|
169838
|
-
title: "/ctx-embed
|
|
169839
|
-
text: `## /ctx-embed
|
|
170205
|
+
title: "/ctx-embed",
|
|
170206
|
+
text: `## /ctx-embed
|
|
169840
170207
|
|
|
169841
170208
|
Memory is disabled for this project, so there is no semantic embedding to backfill.`,
|
|
169842
170209
|
level: "info"
|
|
169843
170210
|
});
|
|
169844
170211
|
return;
|
|
169845
170212
|
}
|
|
169846
|
-
const project = deps.resolveProject?.(ctx) ?? {
|
|
169847
|
-
projectDir: deps.projectDir,
|
|
169848
|
-
projectIdentity: deps.projectIdentity
|
|
169849
|
-
};
|
|
169850
170213
|
await ensureProjectRegisteredFromPiDirectory(project.projectDir, deps.db);
|
|
169851
|
-
|
|
169852
|
-
|
|
169853
|
-
|
|
169854
|
-
|
|
169855
|
-
|
|
169856
|
-
|
|
169857
|
-
|
|
169858
|
-
|
|
169859
|
-
|
|
169860
|
-
|
|
169861
|
-
|
|
169862
|
-
|
|
169863
|
-
|
|
169864
|
-
|
|
169865
|
-
|
|
169866
|
-
|
|
169867
|
-
|
|
169868
|
-
|
|
169869
|
-
return {
|
|
169870
|
-
text: `## /ctx-embed-history
|
|
169871
|
-
|
|
169872
|
-
Embedding is already running for this project — ${outcome.total} compartment${outcome.total === 1 ? "" : "s"} still pending. Try again shortly.`,
|
|
169873
|
-
level: "info"
|
|
169874
|
-
};
|
|
169875
|
-
case "stalled":
|
|
169876
|
-
return {
|
|
169877
|
-
text: `## /ctx-embed-history
|
|
169878
|
-
|
|
169879
|
-
Embedded ${outcome.embedded} compartment${outcome.embedded === 1 ? "" : "s"}; ${outcome.remaining} could not be embedded (the provider returned no result). Run /ctx-embed-history again to retry them.`,
|
|
169880
|
-
level: "info"
|
|
169881
|
-
};
|
|
169882
|
-
default:
|
|
169883
|
-
return {
|
|
169884
|
-
text: `## /ctx-embed-history
|
|
170214
|
+
if (sub === "start") {
|
|
170215
|
+
const { text, level } = await runEmbedDrain(deps.db, project.projectIdentity, sessionId);
|
|
170216
|
+
sendCtxStatusMessage(pi, { title: "/ctx-embed", text, level });
|
|
170217
|
+
return;
|
|
170218
|
+
}
|
|
170219
|
+
if (sub !== "") {
|
|
170220
|
+
sendCtxStatusMessage(pi, {
|
|
170221
|
+
title: "/ctx-embed",
|
|
170222
|
+
text: "## /ctx-embed\n\nUsage: `/ctx-embed` (status), `/ctx-embed start`, or `/ctx-embed pause`.",
|
|
170223
|
+
level: "info"
|
|
170224
|
+
});
|
|
170225
|
+
return;
|
|
170226
|
+
}
|
|
170227
|
+
const coverage = getEmbeddingCoverageStatus(deps.db, project.projectIdentity, sessionId);
|
|
170228
|
+
const statusText = formatEmbedStatusText(coverage, { status: "idle" });
|
|
170229
|
+
sendCtxStatusMessage(pi, {
|
|
170230
|
+
title: "/ctx-embed",
|
|
170231
|
+
text: `## Embedding Status
|
|
169885
170232
|
|
|
169886
|
-
|
|
169887
|
-
|
|
169888
|
-
};
|
|
169889
|
-
}
|
|
169890
|
-
})();
|
|
169891
|
-
sendCtxStatusMessage(pi, { title: "/ctx-embed-history", text, level }, {
|
|
169892
|
-
sessionId,
|
|
169893
|
-
projectIdentity: project.projectIdentity,
|
|
169894
|
-
status: outcome.status,
|
|
169895
|
-
embedded: outcome.embedded,
|
|
169896
|
-
total: outcome.total
|
|
170233
|
+
${statusText}`,
|
|
170234
|
+
level: "info"
|
|
169897
170235
|
});
|
|
169898
170236
|
}
|
|
169899
170237
|
});
|
|
169900
170238
|
}
|
|
170239
|
+
function maybeAutoEmbedPiSession(deps, sessionId, projectDir, projectIdentity, notify) {
|
|
170240
|
+
if (autoEmbedAttemptedBySession.has(sessionId))
|
|
170241
|
+
return;
|
|
170242
|
+
if (embedPauseBySession.has(sessionId))
|
|
170243
|
+
return;
|
|
170244
|
+
if (deps.memoryEnabled === false)
|
|
170245
|
+
return;
|
|
170246
|
+
autoEmbedAttemptedBySession.add(sessionId);
|
|
170247
|
+
(async () => {
|
|
170248
|
+
try {
|
|
170249
|
+
await new Promise((resolve5) => setTimeout(resolve5, 0));
|
|
170250
|
+
await ensureProjectRegisteredFromPiDirectory(projectDir, deps.db);
|
|
170251
|
+
const coverage = getEmbeddingCoverageStatus(deps.db, projectIdentity, sessionId);
|
|
170252
|
+
if (!coverage.enabled)
|
|
170253
|
+
return;
|
|
170254
|
+
const remaining = coverage.session.total - coverage.session.embedded;
|
|
170255
|
+
if (remaining <= 0)
|
|
170256
|
+
return;
|
|
170257
|
+
notify(`Embedding ${remaining} compartment${remaining === 1 ? "" : "s"} of history in the background…`);
|
|
170258
|
+
const { text } = await runEmbedDrain(deps.db, projectIdentity, sessionId);
|
|
170259
|
+
notify(text.replace(/^## \/ctx-embed\n\n/, ""));
|
|
170260
|
+
} catch {}
|
|
170261
|
+
})();
|
|
170262
|
+
}
|
|
169901
170263
|
|
|
169902
170264
|
// ../plugin/src/hooks/magic-context/execute-flush.ts
|
|
169903
170265
|
init_logger();
|
|
@@ -170313,26 +170675,11 @@ function computePiWorkMetrics(sessionEntries) {
|
|
|
170313
170675
|
}
|
|
170314
170676
|
|
|
170315
170677
|
// ../plugin/src/hooks/magic-context/apply-operations.ts
|
|
170316
|
-
var USER_DROP_PREVIEW_CHARS = 250;
|
|
170317
170678
|
var RECENT_TOOL_SKELETON_WINDOW = 20;
|
|
170318
|
-
function buildReplacementContent(tagId
|
|
170319
|
-
|
|
170320
|
-
|
|
170321
|
-
|
|
170322
|
-
}
|
|
170323
|
-
const currentContent = target.getContent?.() ?? "";
|
|
170324
|
-
const originalText = stripTagPrefix(currentContent);
|
|
170325
|
-
if (originalText.length <= USER_DROP_PREVIEW_CHARS) {
|
|
170326
|
-
return `[truncated §${tagId}§]
|
|
170327
|
-
${originalText}`;
|
|
170328
|
-
}
|
|
170329
|
-
const hardCut = originalText.slice(0, USER_DROP_PREVIEW_CHARS);
|
|
170330
|
-
const softCutIndex = hardCut.search(/\s\S*$/);
|
|
170331
|
-
const preview = softCutIndex > USER_DROP_PREVIEW_CHARS - 30 ? hardCut.slice(0, softCutIndex) : hardCut;
|
|
170332
|
-
return `[truncated §${tagId}§]
|
|
170333
|
-
${preview}…`;
|
|
170334
|
-
}
|
|
170335
|
-
function applyPendingOperations(sessionId, db, targets, protectedTags = 0, preloadedTags, preloadedPendingOps) {
|
|
170679
|
+
function buildReplacementContent(tagId) {
|
|
170680
|
+
return `[dropped §${tagId}§]`;
|
|
170681
|
+
}
|
|
170682
|
+
function applyPendingOperations(sessionId, db, targets, protectedTags = 0, preloadedTags, preloadedPendingOps, syntheticPendingOps = []) {
|
|
170336
170683
|
let didMutateMessage = false;
|
|
170337
170684
|
db.transaction(() => {
|
|
170338
170685
|
const tags = preloadedTags ?? getTagsBySession(db, sessionId);
|
|
@@ -170340,11 +170687,16 @@ function applyPendingOperations(sessionId, db, targets, protectedTags = 0, prelo
|
|
|
170340
170687
|
const tagTypeById = new Map(tags.map((tag) => [tag.tagNumber, tag.type]));
|
|
170341
170688
|
const protectedTagIds = protectedTags > 0 ? new Set(tags.filter((tag) => tag.status === "active").map((tag) => tag.tagNumber).sort((left, right) => right - left).slice(0, protectedTags)) : new Set;
|
|
170342
170689
|
const pendingOps = preloadedPendingOps ?? getPendingOps(db, sessionId);
|
|
170690
|
+
const opsToApply = [
|
|
170691
|
+
...pendingOps.map((op) => ({ op, synthetic: false })),
|
|
170692
|
+
...syntheticPendingOps.map((op) => ({ op, synthetic: true }))
|
|
170693
|
+
];
|
|
170343
170694
|
const skeletonWindow = new Set(tags.filter((tag) => tag.type === "tool").map((tag) => tag.tagNumber).sort((left, right) => right - left).slice(0, RECENT_TOOL_SKELETON_WINDOW));
|
|
170344
|
-
for (const pendingOp of
|
|
170695
|
+
for (const { op: pendingOp, synthetic } of opsToApply) {
|
|
170345
170696
|
const tagStatus = tagStatusById.get(pendingOp.tagId);
|
|
170346
170697
|
if (tagStatus === "compacted" || tagStatus === "dropped") {
|
|
170347
|
-
|
|
170698
|
+
if (!synthetic)
|
|
170699
|
+
removePendingOp(db, sessionId, pendingOp.tagId);
|
|
170348
170700
|
continue;
|
|
170349
170701
|
}
|
|
170350
170702
|
if (protectedTagIds.has(pendingOp.tagId)) {
|
|
@@ -170352,33 +170704,46 @@ function applyPendingOperations(sessionId, db, targets, protectedTags = 0, prelo
|
|
|
170352
170704
|
}
|
|
170353
170705
|
const target = targets.get(pendingOp.tagId);
|
|
170354
170706
|
const isToolTag = tagTypeById.get(pendingOp.tagId) === "tool";
|
|
170707
|
+
if (synthetic) {
|
|
170708
|
+
if (!isToolTag || target?.canDrop?.() !== true)
|
|
170709
|
+
continue;
|
|
170710
|
+
}
|
|
170711
|
+
let shouldPersistDrop = false;
|
|
170355
170712
|
if (isToolTag) {
|
|
170356
170713
|
if (skeletonWindow.has(pendingOp.tagId)) {
|
|
170357
170714
|
const truncResult = target?.truncate?.() ?? "absent";
|
|
170358
|
-
if (truncResult === "incomplete") {
|
|
170715
|
+
if (truncResult === "incomplete" || synthetic && truncResult !== "truncated") {
|
|
170359
170716
|
continue;
|
|
170360
170717
|
}
|
|
170361
170718
|
if (truncResult === "truncated") {
|
|
170362
170719
|
didMutateMessage = true;
|
|
170363
170720
|
}
|
|
170364
170721
|
updateTagDropMode(db, sessionId, pendingOp.tagId, "truncated");
|
|
170722
|
+
shouldPersistDrop = true;
|
|
170365
170723
|
} else {
|
|
170366
170724
|
const dropResult = target?.drop?.() ?? "absent";
|
|
170367
|
-
if (dropResult === "incomplete") {
|
|
170725
|
+
if (dropResult === "incomplete" || synthetic && dropResult !== "removed") {
|
|
170368
170726
|
continue;
|
|
170369
170727
|
}
|
|
170370
170728
|
if (dropResult === "removed") {
|
|
170371
170729
|
didMutateMessage = true;
|
|
170372
170730
|
}
|
|
170373
170731
|
updateTagDropMode(db, sessionId, pendingOp.tagId, "full");
|
|
170732
|
+
shouldPersistDrop = true;
|
|
170374
170733
|
}
|
|
170375
170734
|
} else if (target) {
|
|
170376
|
-
const changed = target.setContent(buildReplacementContent(pendingOp.tagId
|
|
170735
|
+
const changed = target.setContent(buildReplacementContent(pendingOp.tagId));
|
|
170377
170736
|
if (changed)
|
|
170378
170737
|
didMutateMessage = true;
|
|
170738
|
+
shouldPersistDrop = true;
|
|
170739
|
+
} else if (!synthetic) {
|
|
170740
|
+
shouldPersistDrop = true;
|
|
170379
170741
|
}
|
|
170742
|
+
if (!shouldPersistDrop)
|
|
170743
|
+
continue;
|
|
170380
170744
|
updateTagStatus(db, sessionId, pendingOp.tagId, "dropped");
|
|
170381
|
-
|
|
170745
|
+
if (!synthetic)
|
|
170746
|
+
removePendingOp(db, sessionId, pendingOp.tagId);
|
|
170382
170747
|
}
|
|
170383
170748
|
})();
|
|
170384
170749
|
return didMutateMessage;
|
|
@@ -170402,7 +170767,7 @@ function applyFlushedStatuses(sessionId, db, targets, preloadedTags) {
|
|
|
170402
170767
|
}
|
|
170403
170768
|
}
|
|
170404
170769
|
} else if (target) {
|
|
170405
|
-
const changed = target.setContent(buildReplacementContent(tag.tagNumber
|
|
170770
|
+
const changed = target.setContent(buildReplacementContent(tag.tagNumber));
|
|
170406
170771
|
if (changed)
|
|
170407
170772
|
didMutateMessage = true;
|
|
170408
170773
|
}
|
|
@@ -170690,7 +171055,8 @@ function applyCavemanCleanup(sessionId, db, targets, tags, config2) {
|
|
|
170690
171055
|
const result = {
|
|
170691
171056
|
compressedToLite: 0,
|
|
170692
171057
|
compressedToFull: 0,
|
|
170693
|
-
compressedToUltra: 0
|
|
171058
|
+
compressedToUltra: 0,
|
|
171059
|
+
mutatedTextTags: 0
|
|
170694
171060
|
};
|
|
170695
171061
|
if (!config2.enabled)
|
|
170696
171062
|
return result;
|
|
@@ -170731,7 +171097,9 @@ function applyCavemanCleanup(sessionId, db, targets, tags, config2) {
|
|
|
170731
171097
|
const target = targets.get(tag.tagNumber);
|
|
170732
171098
|
if (!target)
|
|
170733
171099
|
continue;
|
|
170734
|
-
target.setContent(compressed);
|
|
171100
|
+
const didMutate = target.setContent(compressed);
|
|
171101
|
+
if (didMutate)
|
|
171102
|
+
result.mutatedTextTags += 1;
|
|
170735
171103
|
updateCavemanDepth(db, sessionId, tag.tagNumber, targetDepth);
|
|
170736
171104
|
if (targetDepth === DEPTH_LITE)
|
|
170737
171105
|
result.compressedToLite += 1;
|
|
@@ -171109,7 +171477,7 @@ function buildToolArcs(messages) {
|
|
|
171109
171477
|
}
|
|
171110
171478
|
return arcs.sort((a, b) => a.invOrdinal - b.invOrdinal || (a.resOrdinal ?? Number.MAX_SAFE_INTEGER) - (b.resOrdinal ?? Number.MAX_SAFE_INTEGER));
|
|
171111
171479
|
}
|
|
171112
|
-
function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal) {
|
|
171480
|
+
function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal, recentOpenArcCutoff) {
|
|
171113
171481
|
let boundary = candidate;
|
|
171114
171482
|
for (const arc of arcs) {
|
|
171115
171483
|
if (arc.resOrdinal !== null) {
|
|
@@ -171118,6 +171486,8 @@ function fenceBoundaryForToolArcs(candidate, arcs, lastCompartmentEndOrdinal) {
|
|
|
171118
171486
|
}
|
|
171119
171487
|
continue;
|
|
171120
171488
|
}
|
|
171489
|
+
if (arc.invOrdinal < recentOpenArcCutoff)
|
|
171490
|
+
continue;
|
|
171121
171491
|
if (arc.invOrdinal >= lastCompartmentEndOrdinal + 1 && arc.invOrdinal < boundary) {
|
|
171122
171492
|
return arc.invOrdinal;
|
|
171123
171493
|
}
|
|
@@ -171355,7 +171725,7 @@ function semanticSnapBoundary(args) {
|
|
|
171355
171725
|
return snapped;
|
|
171356
171726
|
}
|
|
171357
171727
|
function applyHeadCap(args) {
|
|
171358
|
-
const { index, protectedTailStart, offset, arcs, capTokens } = args;
|
|
171728
|
+
const { index, protectedTailStart, offset, arcs, capTokens, recentOpenArcCutoff } = args;
|
|
171359
171729
|
if (offset >= protectedTailStart)
|
|
171360
171730
|
return { eligibleEndOrdinal: offset, oversizeAtomicUnit: false };
|
|
171361
171731
|
let end = index.findHeadEndForCap(offset, protectedTailStart, capTokens);
|
|
@@ -171363,7 +171733,7 @@ function applyHeadCap(args) {
|
|
|
171363
171733
|
for (const arc of arcs) {
|
|
171364
171734
|
const resOrdinal = arc.resOrdinal;
|
|
171365
171735
|
if (resOrdinal === null) {
|
|
171366
|
-
if (arc.invOrdinal >= offset && arc.invOrdinal < end) {
|
|
171736
|
+
if (arc.invOrdinal >= recentOpenArcCutoff && arc.invOrdinal >= offset && arc.invOrdinal < end) {
|
|
171367
171737
|
end = Math.min(end, arc.invOrdinal);
|
|
171368
171738
|
}
|
|
171369
171739
|
continue;
|
|
@@ -171430,7 +171800,14 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171430
171800
|
}
|
|
171431
171801
|
if (ctx.mode === "manual-full-recomp") {
|
|
171432
171802
|
const arcs2 = buildToolArcs(messages);
|
|
171433
|
-
const
|
|
171803
|
+
const recompTarget = deriveProtectedTailTokenTarget({
|
|
171804
|
+
contextLimit: ctx.contextLimit,
|
|
171805
|
+
executeThresholdPercentage: ctx.executeThresholdPercentage,
|
|
171806
|
+
usagePercentage: 0,
|
|
171807
|
+
triggerBudget: ctx.triggerBudget
|
|
171808
|
+
});
|
|
171809
|
+
const recentOpenArcCutoff2 = index.findSuffixStartForTokens(recompTarget.N);
|
|
171810
|
+
const firstOpenArc = arcs2.find((arc) => arc.resOrdinal === null && arc.invOrdinal >= offset && arc.invOrdinal >= recentOpenArcCutoff2);
|
|
171434
171811
|
const protectedTailStart2 = firstOpenArc?.invOrdinal ?? rawMessageCount + 1;
|
|
171435
171812
|
const rawRangeFingerprint2 = computeRawRangeFingerprint(messages, offset, protectedTailStart2);
|
|
171436
171813
|
return {
|
|
@@ -171472,13 +171849,14 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171472
171849
|
const scaledN = ctx.emergencyTailScale ? Math.max(1, Math.floor(target.N * ctx.emergencyTailScale)) : target.N;
|
|
171473
171850
|
const arcs = buildToolArcs(messages);
|
|
171474
171851
|
let boundary = index.findSuffixStartForTokens(scaledN);
|
|
171852
|
+
const recentOpenArcCutoff = boundary;
|
|
171475
171853
|
let boundaryReason = boundary === 1 ? "whole-session-smaller-than-tail" : "size-walk";
|
|
171476
171854
|
const tokenAtBoundary = index.tokenForOrdinal(boundary);
|
|
171477
171855
|
if (boundary <= rawMessageCount && tokenAtBoundary > Math.max(2 * scaledN, 64000) && boundary < rawMessageCount) {
|
|
171478
171856
|
boundary += 1;
|
|
171479
171857
|
boundaryReason = "huge-message-exception";
|
|
171480
171858
|
}
|
|
171481
|
-
boundary = fenceBoundaryForToolArcs(boundary, arcs, ctx.lastCompartmentEndOrdinal);
|
|
171859
|
+
boundary = fenceBoundaryForToolArcs(boundary, arcs, ctx.lastCompartmentEndOrdinal, recentOpenArcCutoff);
|
|
171482
171860
|
const snapped = semanticSnapBoundary({
|
|
171483
171861
|
messages,
|
|
171484
171862
|
index,
|
|
@@ -171488,7 +171866,7 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171488
171866
|
});
|
|
171489
171867
|
if (snapped !== boundary)
|
|
171490
171868
|
boundaryReason = "semantic-snap";
|
|
171491
|
-
boundary = fenceBoundaryForToolArcs(snapped, arcs, ctx.lastCompartmentEndOrdinal);
|
|
171869
|
+
boundary = fenceBoundaryForToolArcs(snapped, arcs, ctx.lastCompartmentEndOrdinal, recentOpenArcCutoff);
|
|
171492
171870
|
let runtimeFloor = offset;
|
|
171493
171871
|
if (ctx.migrationFloorActive)
|
|
171494
171872
|
runtimeFloor = Math.max(runtimeFloor, ctx.priorBoundaryOrdinal);
|
|
@@ -171524,7 +171902,8 @@ function resolveProtectedTailBoundary(ctx) {
|
|
|
171524
171902
|
offset,
|
|
171525
171903
|
arcs,
|
|
171526
171904
|
lastCompartmentEndOrdinal: ctx.lastCompartmentEndOrdinal,
|
|
171527
|
-
capTokens: perRunCap
|
|
171905
|
+
capTokens: perRunCap,
|
|
171906
|
+
recentOpenArcCutoff
|
|
171528
171907
|
});
|
|
171529
171908
|
const rawRangeFingerprint = computeRawRangeFingerprint(messages, offset, head.eligibleEndOrdinal);
|
|
171530
171909
|
return {
|
|
@@ -171929,8 +172308,8 @@ var CHANNEL1_SENTINEL = "<system-reminder>";
|
|
|
171929
172308
|
var TOKENS_PER_BYTE = 0.25;
|
|
171930
172309
|
var CHANNEL1_FLOOR_TOKENS = 1e4;
|
|
171931
172310
|
var CHANNEL1_REFIRE_FLOOR_TOKENS = 1e4;
|
|
171932
|
-
function channel1RefireTokens(
|
|
171933
|
-
const scaled = Math.round(0.05 * Math.max(0,
|
|
172311
|
+
function channel1RefireTokens(workingWindowTokens) {
|
|
172312
|
+
const scaled = Math.round(0.05 * Math.max(0, workingWindowTokens));
|
|
171934
172313
|
return Math.max(CHANNEL1_REFIRE_FLOOR_TOKENS, scaled);
|
|
171935
172314
|
}
|
|
171936
172315
|
var S_GENTLE = 0.2;
|
|
@@ -171946,7 +172325,7 @@ function toolOutputTokens(output) {
|
|
|
171946
172325
|
return Math.round(byteSize(output) * TOKENS_PER_BYTE);
|
|
171947
172326
|
}
|
|
171948
172327
|
function decideChannel1(input) {
|
|
171949
|
-
const { undroppedTokens, pressure,
|
|
172328
|
+
const { undroppedTokens, pressure, workingWindowTokens, hasRecentReduce } = input;
|
|
171950
172329
|
const resetCycle = hasRecentReduce || undroppedTokens < input.lastNudgeUndropped;
|
|
171951
172330
|
const lastNudge = resetCycle ? 0 : input.lastNudgeUndropped;
|
|
171952
172331
|
const lastLevel = resetCycle ? "" : input.lastNudgeLevel;
|
|
@@ -171961,7 +172340,7 @@ function decideChannel1(input) {
|
|
|
171961
172340
|
return quiet();
|
|
171962
172341
|
if (undroppedTokens < CHANNEL1_FLOOR_TOKENS)
|
|
171963
172342
|
return quiet();
|
|
171964
|
-
const budget =
|
|
172343
|
+
const budget = workingWindowTokens > 0 ? workingWindowTokens : undroppedTokens || 1;
|
|
171965
172344
|
const severity = undroppedTokens / budget * pressure;
|
|
171966
172345
|
if (severity < S_GENTLE)
|
|
171967
172346
|
return quiet();
|
|
@@ -171973,7 +172352,7 @@ function decideChannel1(input) {
|
|
|
171973
172352
|
else
|
|
171974
172353
|
level = "gentle";
|
|
171975
172354
|
if (lastLevel === "") {
|
|
171976
|
-
if (undroppedTokens < lastNudge + channel1RefireTokens(
|
|
172355
|
+
if (undroppedTokens < lastNudge + channel1RefireTokens(workingWindowTokens)) {
|
|
171977
172356
|
return quiet();
|
|
171978
172357
|
}
|
|
171979
172358
|
} else if (LEVEL_RANK[level] <= LEVEL_RANK[lastLevel]) {
|
|
@@ -171998,6 +172377,13 @@ function computePressure(input) {
|
|
|
171998
172377
|
function approxThousands(tokens) {
|
|
171999
172378
|
return `${Math.round(tokens / 1000)}k`;
|
|
172000
172379
|
}
|
|
172380
|
+
function formatOldestReclaimableHint(hint) {
|
|
172381
|
+
if (!hint || hint.length === 0)
|
|
172382
|
+
return "";
|
|
172383
|
+
const rendered = hint.slice(0, 4).map((tag) => `§${tag.tagNumber}§ ${tag.toolName ?? "tool"}`).join(" · ");
|
|
172384
|
+
return rendered.length > 0 ? `
|
|
172385
|
+
oldest reclaimable: ${rendered}.` : "";
|
|
172386
|
+
}
|
|
172001
172387
|
var CHANNEL2_USABLE_FRACTION = 1 / 3;
|
|
172002
172388
|
var CHANNEL2_MIN_RECLAIMABLE = 1e4;
|
|
172003
172389
|
function shouldTriggerChannel2(input) {
|
|
@@ -172007,30 +172393,32 @@ function shouldTriggerChannel2(input) {
|
|
|
172007
172393
|
return true;
|
|
172008
172394
|
return input.reclaimableTokens >= input.usableTokens * CHANNEL2_USABLE_FRACTION;
|
|
172009
172395
|
}
|
|
172010
|
-
function buildChannel2Reminder(undroppedTokens) {
|
|
172396
|
+
function buildChannel2Reminder(undroppedTokens, hint) {
|
|
172011
172397
|
const amount = approxThousands(undroppedTokens);
|
|
172398
|
+
const hintText = formatOldestReclaimableHint(hint);
|
|
172012
172399
|
return `<system-reminder>
|
|
172013
|
-
` + `Routine context housekeeping is near: a large span of this session will be comparted soon, ` + `and ~${amount} tokens of tool output remain unreduced. Drop spent outputs with ctx_reduce ` + `first so the archived span is the part that matters
|
|
172400
|
+
` + `Routine context housekeeping is near: a large span of this session will be comparted soon, ` + `and ~${amount} tokens of tool output remain unreduced. Drop spent outputs with ctx_reduce ` + `first so the archived span is the part that matters.${hintText}
|
|
172014
172401
|
` + `</system-reminder>`;
|
|
172015
172402
|
}
|
|
172016
|
-
function buildChannel1Reminder(level, undroppedTokens) {
|
|
172403
|
+
function buildChannel1Reminder(level, undroppedTokens, hint) {
|
|
172017
172404
|
const amount = approxThousands(undroppedTokens);
|
|
172405
|
+
const hintText = formatOldestReclaimableHint(hint);
|
|
172018
172406
|
let body;
|
|
172019
172407
|
switch (level) {
|
|
172020
172408
|
case "gentle":
|
|
172021
|
-
body = `You have ~${amount} tokens of tool output you have not reduced. ` + `
|
|
172409
|
+
body = `You have ~${amount} tokens of tool output you have not reduced. ` + `When you are done with earlier outputs, dropping them with ctx_reduce keeps context lean.`;
|
|
172022
172410
|
break;
|
|
172023
172411
|
case "firm":
|
|
172024
|
-
body = `~${amount} tokens of unreduced tool output
|
|
172412
|
+
body = `~${amount} tokens of unreduced tool output has built up. ` + `At your next natural stopping point, consider dropping what you have already processed with ctx_reduce.`;
|
|
172025
172413
|
break;
|
|
172026
172414
|
case "urgent":
|
|
172027
|
-
body = `~${amount} tokens of unreduced tool output remain
|
|
172415
|
+
body = `~${amount} tokens of unreduced tool output remain, and a large span of this session will be comparted before long. ` + `Consider dropping spent outputs with ctx_reduce so the archived span is the part that matters.`;
|
|
172028
172416
|
break;
|
|
172029
172417
|
}
|
|
172030
172418
|
return `
|
|
172031
172419
|
|
|
172032
172420
|
<system-reminder>
|
|
172033
|
-
${body}
|
|
172421
|
+
${body}${hintText}
|
|
172034
172422
|
</system-reminder>`;
|
|
172035
172423
|
}
|
|
172036
172424
|
|
|
@@ -172578,6 +172966,41 @@ function renderMemoryBlockV2(memories, wrapper = "project-memory", renderOptions
|
|
|
172578
172966
|
`);
|
|
172579
172967
|
}
|
|
172580
172968
|
|
|
172969
|
+
// ../plugin/src/hooks/magic-context/tool-reclaim.ts
|
|
172970
|
+
function buildSyntheticToolReclaimOps(input) {
|
|
172971
|
+
const watermark = Math.max(0, input.watermark);
|
|
172972
|
+
if (watermark <= 0)
|
|
172973
|
+
return [];
|
|
172974
|
+
const realPendingTagIds = new Set((input.pendingOps ?? []).map((op) => op.tagId));
|
|
172975
|
+
const tags = getActiveTagsBySession(input.db, input.sessionId);
|
|
172976
|
+
const synthetic = [];
|
|
172977
|
+
for (const tag of tags) {
|
|
172978
|
+
if (tag.type !== "tool")
|
|
172979
|
+
continue;
|
|
172980
|
+
if (tag.status !== "active")
|
|
172981
|
+
continue;
|
|
172982
|
+
if (tag.tagNumber > watermark)
|
|
172983
|
+
continue;
|
|
172984
|
+
if (realPendingTagIds.has(tag.tagNumber))
|
|
172985
|
+
continue;
|
|
172986
|
+
if (input.targets.get(tag.tagNumber)?.canDrop?.() !== true)
|
|
172987
|
+
continue;
|
|
172988
|
+
synthetic.push({
|
|
172989
|
+
id: 0,
|
|
172990
|
+
sessionId: input.sessionId,
|
|
172991
|
+
tagId: tag.tagNumber,
|
|
172992
|
+
operation: "drop",
|
|
172993
|
+
queuedAt: 0
|
|
172994
|
+
});
|
|
172995
|
+
}
|
|
172996
|
+
return synthetic;
|
|
172997
|
+
}
|
|
172998
|
+
function advanceToolReclaimWatermarkToCurrentMax(db, sessionId) {
|
|
172999
|
+
const maxTagNumber = getMaxTagNumberBySession(db, sessionId);
|
|
173000
|
+
advanceToolReclaimWatermark(db, sessionId, maxTagNumber);
|
|
173001
|
+
return maxTagNumber;
|
|
173002
|
+
}
|
|
173003
|
+
|
|
172581
173004
|
// src/context-handler.ts
|
|
172582
173005
|
init_logger();
|
|
172583
173006
|
|
|
@@ -172988,7 +173411,7 @@ function tagToolPart(args) {
|
|
|
172988
173411
|
const tagged = prependTag(tagId, text);
|
|
172989
173412
|
args.part.setText(tagged);
|
|
172990
173413
|
}
|
|
172991
|
-
args.targets.set(tagId, buildToolTarget(args.part, args.message));
|
|
173414
|
+
args.targets.set(tagId, buildToolTarget(args.part, args.message, tagId));
|
|
172992
173415
|
}
|
|
172993
173416
|
function setToolContentOrText(part, content) {
|
|
172994
173417
|
try {
|
|
@@ -173028,7 +173451,7 @@ function buildAggregateTarget(tagId, occurrences) {
|
|
|
173028
173451
|
return any2 ? "removed" : "absent";
|
|
173029
173452
|
},
|
|
173030
173453
|
truncate() {
|
|
173031
|
-
const sentinel =
|
|
173454
|
+
const sentinel = `[dropped §${tagId}§]`;
|
|
173032
173455
|
let any2 = false;
|
|
173033
173456
|
for (const occ of occurrences) {
|
|
173034
173457
|
if (setToolContentOrText(occ.part, sentinel)) {
|
|
@@ -173060,7 +173483,7 @@ function buildTextTarget(part, message) {
|
|
|
173060
173483
|
}
|
|
173061
173484
|
};
|
|
173062
173485
|
}
|
|
173063
|
-
function buildToolTarget(part, message) {
|
|
173486
|
+
function buildToolTarget(part, message, tagId) {
|
|
173064
173487
|
return {
|
|
173065
173488
|
setContent(content) {
|
|
173066
173489
|
return setToolContentOrText(part, content);
|
|
@@ -173069,11 +173492,11 @@ function buildToolTarget(part, message) {
|
|
|
173069
173492
|
return part.getText() ?? null;
|
|
173070
173493
|
},
|
|
173071
173494
|
drop() {
|
|
173072
|
-
const replaced = part.replaceWithSentinel(`[dropped §${
|
|
173495
|
+
const replaced = part.replaceWithSentinel(`[dropped §${tagId}§]`);
|
|
173073
173496
|
return replaced ? "removed" : "absent";
|
|
173074
173497
|
},
|
|
173075
173498
|
truncate() {
|
|
173076
|
-
const ok = setToolContentOrText(part,
|
|
173499
|
+
const ok = setToolContentOrText(part, `[dropped §${tagId}§]`);
|
|
173077
173500
|
return ok ? "truncated" : "absent";
|
|
173078
173501
|
},
|
|
173079
173502
|
message: {
|
|
@@ -174343,10 +174766,11 @@ function maybeChannel1ReminderForToolResult(args) {
|
|
|
174343
174766
|
contextLimit: state.contextLimit,
|
|
174344
174767
|
executeThresholdPercentage: state.executeThresholdPercentage
|
|
174345
174768
|
});
|
|
174769
|
+
const workingWindowTokens = Math.round(state.contextLimit * state.executeThresholdPercentage / 100);
|
|
174346
174770
|
const decision = decideChannel1({
|
|
174347
174771
|
undroppedTokens,
|
|
174348
174772
|
pressure,
|
|
174349
|
-
|
|
174773
|
+
workingWindowTokens,
|
|
174350
174774
|
lastNudgeUndropped: getLastNudgeUndropped(db, sessionId),
|
|
174351
174775
|
lastNudgeLevel: getLastNudgeLevel(db, sessionId),
|
|
174352
174776
|
hasRecentReduce: false
|
|
@@ -174357,9 +174781,10 @@ function maybeChannel1ReminderForToolResult(args) {
|
|
|
174357
174781
|
return null;
|
|
174358
174782
|
return {
|
|
174359
174783
|
type: "text",
|
|
174360
|
-
text: buildChannel1Reminder(decision.level, decision.undroppedTokens)
|
|
174784
|
+
text: buildChannel1Reminder(decision.level, decision.undroppedTokens, state.oldestReclaimableToolTags)
|
|
174361
174785
|
};
|
|
174362
174786
|
}
|
|
174787
|
+
var CHANNEL2_NUDGE_CUSTOM_TYPE = "magic-context:ceiling-nudge";
|
|
174363
174788
|
function maybeDeliverChannel2Pi(pi, db, sessionId, deliverAs = "followUp") {
|
|
174364
174789
|
let state;
|
|
174365
174790
|
try {
|
|
@@ -174387,9 +174812,12 @@ function maybeDeliverChannel2Pi(pi, db, sessionId, deliverAs = "followUp") {
|
|
|
174387
174812
|
if (!casChannel2NudgeState(db, sessionId, "pending", "claimed"))
|
|
174388
174813
|
return false;
|
|
174389
174814
|
try {
|
|
174390
|
-
pi.
|
|
174391
|
-
|
|
174392
|
-
|
|
174815
|
+
pi.sendMessage({
|
|
174816
|
+
customType: CHANNEL2_NUDGE_CUSTOM_TYPE,
|
|
174817
|
+
content: buildChannel2Reminder(undropped, baseline.oldestReclaimableToolTags),
|
|
174818
|
+
display: false,
|
|
174819
|
+
details: { kind: "channel-2-ceiling-nudge" }
|
|
174820
|
+
}, { deliverAs });
|
|
174393
174821
|
} catch (error51) {
|
|
174394
174822
|
try {
|
|
174395
174823
|
casChannel2NudgeState(db, sessionId, "claimed", "pending");
|
|
@@ -174784,10 +175212,14 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174784
175212
|
if (!matched)
|
|
174785
175213
|
continue;
|
|
174786
175214
|
const target = targets.get(tag.tagNumber);
|
|
174787
|
-
target?.drop?.();
|
|
175215
|
+
const result = target?.drop?.() ?? "absent";
|
|
175216
|
+
if (result === "incomplete")
|
|
175217
|
+
continue;
|
|
174788
175218
|
updateTagDropMode(db, sessionId, tag.tagNumber, "full");
|
|
174789
175219
|
updateTagStatus(db, sessionId, tag.tagNumber, "dropped");
|
|
174790
|
-
|
|
175220
|
+
if (result === "removed" || result === "truncated") {
|
|
175221
|
+
droppedStaleReduceCalls++;
|
|
175222
|
+
}
|
|
174791
175223
|
}
|
|
174792
175224
|
})();
|
|
174793
175225
|
}
|
|
@@ -174859,7 +175291,9 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174859
175291
|
continue;
|
|
174860
175292
|
updateTagDropMode(db, sessionId, tag.tagNumber, "full");
|
|
174861
175293
|
updateTagStatus(db, sessionId, tag.tagNumber, "dropped");
|
|
174862
|
-
|
|
175294
|
+
if (result === "removed" || result === "truncated") {
|
|
175295
|
+
deduplicatedTools++;
|
|
175296
|
+
}
|
|
174863
175297
|
}
|
|
174864
175298
|
}
|
|
174865
175299
|
})();
|
|
@@ -174868,6 +175302,7 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174868
175302
|
sessionLog(sessionId, `heuristic cleanup: dropped ${droppedTools} tool tags, stale ctx_reduce=${droppedStaleReduceCalls}, deduplicated ${deduplicatedTools} tool calls, dropped ${droppedInjections} system injections`);
|
|
174869
175303
|
}
|
|
174870
175304
|
let compressedTextTags = 0;
|
|
175305
|
+
let mutatedTextTags = 0;
|
|
174871
175306
|
if (config2.caveman?.enabled) {
|
|
174872
175307
|
const cavemanResult = applyCavemanCleanup(sessionId, db, targets, tags, {
|
|
174873
175308
|
enabled: true,
|
|
@@ -174875,13 +175310,15 @@ function applyPiHeuristicCleanup(sessionId, db, targets, piMessages, config2, pr
|
|
|
174875
175310
|
protectedTags: config2.protectedTags
|
|
174876
175311
|
});
|
|
174877
175312
|
compressedTextTags = cavemanResult.compressedToLite + cavemanResult.compressedToFull + cavemanResult.compressedToUltra;
|
|
175313
|
+
mutatedTextTags = cavemanResult.mutatedTextTags;
|
|
174878
175314
|
}
|
|
174879
175315
|
return {
|
|
174880
175316
|
droppedTools,
|
|
174881
175317
|
deduplicatedTools,
|
|
174882
175318
|
droppedInjections,
|
|
174883
175319
|
droppedStaleReduceCalls,
|
|
174884
|
-
compressedTextTags
|
|
175320
|
+
compressedTextTags,
|
|
175321
|
+
mutatedTextTags
|
|
174885
175322
|
};
|
|
174886
175323
|
}
|
|
174887
175324
|
function buildMessageIdToMaxTagFromTargets(targets) {
|
|
@@ -180258,9 +180695,7 @@ Context is managed for you entirely automatically — there's nothing to prune a
|
|
|
180258
180695
|
var CTX_NOTE_GUIDANCE = `Use \`ctx_note\` ONLY for genuinely future concerns — something to revisit much later, not work coming up in the next few turns (that's already in your active context) and not active multi-step work (use todos for that). Magic Context preserves your full context across both compaction and restarts, so an upcoming restart or "let's come back to this later" is never a reason to take a note — nothing is lost either way. Notes you do take survive compression and resurface at natural work boundaries (after commits, historian runs, todo completion).`;
|
|
180259
180696
|
var TOOL_HISTORY_GUIDANCE = `Compressed history intentionally omits tool calls and their outputs — summaries like "I edited file X" are historian records, not patterns to replicate. In the live conversation, older tool calls and their results are cleaned up to save context — you may see your own past messages referencing actions without the corresponding tool call or result visible. This is normal context management. ALWAYS use real tool calls; never simulate, fabricate, or inline tool outputs in your text. If there is no tool result message, the action did not happen. NEVER simulate, hallucinate or claim tool calls, command output, search results, file edits, or diffs in plain text as if they actually occurred.`;
|
|
180260
180697
|
var BASE_INTRO = (protectedTags) => `Messages and tool outputs are tagged with §N§ identifiers (e.g., §1§, §42§).
|
|
180261
|
-
Use \`ctx_reduce\` to
|
|
180262
|
-
- \`drop\`: Remove entirely (best for tool outputs you already acted on).
|
|
180263
|
-
Syntax: "3-5", "1,2,9", or "1-5,8,12-15". Last ${protectedTags} tags are protected.
|
|
180698
|
+
Use \`ctx_reduce\` to mark spent tagged content as discardable and reclaim space. Marking is NOT an immediate delete — it queues the content, which stays fully visible until space is actually needed (as soon as the next turn if you're already under pressure, much later if not), so mark a tool output as soon as you're done with it rather than hoarding the call for the end of the turn. The last ${protectedTags} tags are protected (marking one just queues it until it ages out). Syntax: "3-5", "1,2,9", or "1-5,8,12-15".
|
|
180264
180699
|
Do not announce or narrate \`ctx_reduce\` drops — just call the tool silently. Saying "I'll drop these outputs" wastes tokens the user does not care about.
|
|
180265
180700
|
${CTX_NOTE_GUIDANCE}
|
|
180266
180701
|
Use \`ctx_memory\` for durable project knowledge: write what future sessions must know, update/archive/merge the memories you see in \`<project-memory>\` when they drift. Memories persist across sessions and every new session starts with them.
|
|
@@ -180279,7 +180714,7 @@ Use \`ctx_expand\` to recover the raw conversation behind a \`<compartment>\` su
|
|
|
180279
180714
|
\`ctx_search\` returns ranked results from memories, git commits, and raw message history. Use message ordinals from results with \`ctx_expand\` to retrieve surrounding conversation context.
|
|
180280
180715
|
${TOOL_HISTORY_GUIDANCE}
|
|
180281
180716
|
NEVER drop large ranges blindly (e.g., "1-50"). Review each tag before deciding.
|
|
180282
|
-
|
|
180717
|
+
Keep your user's instructions and intent — never drop a user message for its directive, even an old one. But a large block of pasted content inside a user message (logs, data dumps, long code, attachments) is fair to mark discardable once you've extracted what you need — it stays searchable via \`ctx_search\`.
|
|
180283
180718
|
NEVER drop assistant text messages unless they are exceptionally large. Your conversation messages are lightweight; only large tool outputs are worth dropping.
|
|
180284
180719
|
Before your turn finishes, consider using \`ctx_reduce\` to drop large tool outputs you no longer need.`;
|
|
180285
180720
|
var BASE_INTRO_NO_REDUCE = () => `${CTX_NOTE_GUIDANCE}
|
|
@@ -181745,7 +182180,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181745
182180
|
let tailToolTokens;
|
|
181746
182181
|
let liveTailTokens;
|
|
181747
182182
|
try {
|
|
181748
|
-
const agg = getActiveTagTokenAggregate(options.db, sessionId);
|
|
182183
|
+
const agg = getActiveTagTokenAggregate(options.db, sessionId, options.protectedTags ?? 20);
|
|
181749
182184
|
tailToolTokens = agg.toolOutput;
|
|
181750
182185
|
liveTailTokens = agg.conversation + agg.toolCall;
|
|
181751
182186
|
} catch {
|
|
@@ -181756,6 +182191,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181756
182191
|
const executeThresholdTokensPi = Math.round((usageContextLimit ?? 0) * resolvedExecuteThresholdPct / 100);
|
|
181757
182192
|
const usableTokensPi = Math.max(0, executeThresholdTokensPi - usageInputTokens + liveTailTokens);
|
|
181758
182193
|
resetLastNudgeCycleIfTailShrank(options.db, sessionId, tailToolTokens);
|
|
182194
|
+
const oldestReclaimableToolTags = getOldestActiveUnprotectedToolTags(options.db, sessionId, options.protectedTags ?? 20);
|
|
181759
182195
|
setPiChannel1Baseline(sessionId, {
|
|
181760
182196
|
tailToolTokens,
|
|
181761
182197
|
historyBudgetTokens: historyBudgetTokens ?? 0,
|
|
@@ -181764,7 +182200,8 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181764
182200
|
lastInputTokens: usageInputTokens,
|
|
181765
182201
|
turnToolTokens: 0,
|
|
181766
182202
|
usableTokens: usableTokensPi,
|
|
181767
|
-
reducedSinceRefresh: false
|
|
182203
|
+
reducedSinceRefresh: false,
|
|
182204
|
+
oldestReclaimableToolTags
|
|
181768
182205
|
});
|
|
181769
182206
|
if (usageContextLimit && usageContextLimit > 0 && resolvedExecuteThresholdPct > 0) {
|
|
181770
182207
|
const channel2ShouldTrigger = shouldTriggerChannel2({
|
|
@@ -181792,6 +182229,7 @@ function registerPiContextHandler(pi, baseOptions) {
|
|
|
181792
182229
|
logTransformTiming(sessionId, "postTransformPhase", tPostTransform);
|
|
181793
182230
|
sessionLog(sessionId, `transform completed in ${(performance.now() - transformStartTime).toFixed(1)}ms (${outputMessages.length} messages, ${result.targetCount} targets, watermark: ${result.reasoningWatermark})`);
|
|
181794
182231
|
clearLastTransformErrorIfSet(options.db, sessionId);
|
|
182232
|
+
options.maybeAutoEmbedSession?.(sessionId, projectDirectory, projectIdentity);
|
|
181795
182233
|
return { messages: outputMessages };
|
|
181796
182234
|
} catch (err) {
|
|
181797
182235
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -182098,6 +182536,15 @@ Historian previously failed ${failureState.failureCount} time(s), so Magic Conte
|
|
|
182098
182536
|
const trigger = checkCompartmentTrigger(db, sessionId, sessionMeta, usage, 0, triggerInputs.executeThresholdPercentage, triggerInputs.triggerBudget, triggerInputs.clearReasoningAge, triggerInputs.commitClusterTrigger, args.activeTags, boundaryContextLimit);
|
|
182099
182537
|
if (!trigger.shouldFire) {
|
|
182100
182538
|
sessionLog(sessionId, `historian trigger eval: shouldFire=false (no trigger condition met)`);
|
|
182539
|
+
try {
|
|
182540
|
+
const overflowState = getOverflowState(db, sessionId);
|
|
182541
|
+
if (overflowState.needsEmergencyRecovery && usage.percentage < FORCE_MATERIALIZATION_PERCENTAGE && !inFlightHistorian.has(sessionId) && !hasRunnableCompartmentWindow(boundarySnapshot)) {
|
|
182542
|
+
clearEmergencyRecovery(db, sessionId);
|
|
182543
|
+
sessionLog(sessionId, `historian: disarming stale emergency recovery — real pressure ${usage.percentage.toFixed(1)}% with no runnable compartment window (would otherwise bump to 95% every pass)`);
|
|
182544
|
+
}
|
|
182545
|
+
} catch (err) {
|
|
182546
|
+
sessionLog(sessionId, `historian: emergency-recovery disarm check failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
182547
|
+
}
|
|
182101
182548
|
return;
|
|
182102
182549
|
}
|
|
182103
182550
|
triggered = true;
|
|
@@ -182129,6 +182576,8 @@ async function runPipeline(args) {
|
|
|
182129
182576
|
let historyWasConsumedThisPass = false;
|
|
182130
182577
|
let materializationSatisfiedThisPass = false;
|
|
182131
182578
|
let pendingOpsAppliedThisPass = false;
|
|
182579
|
+
let pendingOpsDidMutate = false;
|
|
182580
|
+
let heuristicOrReasoningDidMutate = false;
|
|
182132
182581
|
let suppressDeferredHistoryDrain = false;
|
|
182133
182582
|
let casLost = false;
|
|
182134
182583
|
const deferredHistoryWasPendingAtPassStart = deferredHistoryRefreshSessions.has(args.sessionId);
|
|
@@ -182214,7 +182663,7 @@ async function runPipeline(args) {
|
|
|
182214
182663
|
sessionLog(args.sessionId, `pending ops WILL APPLY — reason=${applyReason}, pendingOps=${pendingOps.length}, context=${args.contextUsage.percentage.toFixed(1)}%`);
|
|
182215
182664
|
try {
|
|
182216
182665
|
const tApplyPending = performance.now();
|
|
182217
|
-
applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, pendingOps);
|
|
182666
|
+
pendingOpsDidMutate = applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, pendingOps);
|
|
182218
182667
|
logTransformTiming(args.sessionId, "applyPendingOperations", tApplyPending);
|
|
182219
182668
|
executedWorkThisPass = true;
|
|
182220
182669
|
materializationSatisfiedThisPass = true;
|
|
@@ -182298,6 +182747,9 @@ async function runPipeline(args) {
|
|
|
182298
182747
|
} : undefined,
|
|
182299
182748
|
caveman: args.heuristics.caveman
|
|
182300
182749
|
}, activeTags, stableIdResolver);
|
|
182750
|
+
const heuristicMutationCount = heuristicsResult.droppedTools + heuristicsResult.deduplicatedTools + heuristicsResult.droppedInjections + heuristicsResult.droppedStaleReduceCalls + heuristicsResult.mutatedTextTags;
|
|
182751
|
+
if (heuristicMutationCount > 0)
|
|
182752
|
+
heuristicOrReasoningDidMutate = true;
|
|
182301
182753
|
heuristicsExecuted = true;
|
|
182302
182754
|
executedWorkThisPass = true;
|
|
182303
182755
|
if (hasPendingMaterializeSignal) {
|
|
@@ -182306,7 +182758,7 @@ async function runPipeline(args) {
|
|
|
182306
182758
|
if (currentTurnId !== null) {
|
|
182307
182759
|
lastHeuristicsTurnIdBySession.set(args.sessionId, currentTurnId);
|
|
182308
182760
|
}
|
|
182309
|
-
logTransformTiming(args.sessionId, "applyHeuristicCleanup", tHeuristic, `droppedTools=${heuristicsResult.droppedTools} deduplicatedTools=${heuristicsResult.deduplicatedTools} droppedInjections=${heuristicsResult.droppedInjections} compressedTextTags=${heuristicsResult.compressedTextTags}`);
|
|
182761
|
+
logTransformTiming(args.sessionId, "applyHeuristicCleanup", tHeuristic, `droppedTools=${heuristicsResult.droppedTools} deduplicatedTools=${heuristicsResult.deduplicatedTools} droppedInjections=${heuristicsResult.droppedInjections} staleReduce=${heuristicsResult.droppedStaleReduceCalls} compressedTextTags=${heuristicsResult.compressedTextTags} mutatedTextTags=${heuristicsResult.mutatedTextTags}`);
|
|
182310
182762
|
} catch (err) {
|
|
182311
182763
|
sessionLog(args.sessionId, `heuristic cleanup failed (continuing): ${err instanceof Error ? err.message : String(err)}`);
|
|
182312
182764
|
}
|
|
@@ -182343,6 +182795,9 @@ async function runPipeline(args) {
|
|
|
182343
182795
|
}
|
|
182344
182796
|
logTransformTiming(args.sessionId, "clearOldReasoning", tClearReasoning);
|
|
182345
182797
|
logTransformTiming(args.sessionId, "watermarkCleanup", tClearReasoning);
|
|
182798
|
+
if (clearOutcome.cleared > 0 || stripOutcome.stripped > 0) {
|
|
182799
|
+
heuristicOrReasoningDidMutate = true;
|
|
182800
|
+
}
|
|
182346
182801
|
if (combinedWatermark > prevWatermark || clearOutcome.cleared > 0 || stripOutcome.stripped > 0) {
|
|
182347
182802
|
executedWorkThisPass = true;
|
|
182348
182803
|
}
|
|
@@ -182350,7 +182805,32 @@ async function runPipeline(args) {
|
|
|
182350
182805
|
sessionLog(args.sessionId, `reasoning clearing failed (continuing): ${err instanceof Error ? err.message : String(err)}`);
|
|
182351
182806
|
}
|
|
182352
182807
|
}
|
|
182808
|
+
const toolReclaimExecutePass = args.schedulerDecision === "execute";
|
|
182809
|
+
const alreadyMutatingThisPass = pendingOpsDidMutate || heuristicOrReasoningDidMutate;
|
|
182810
|
+
const emergencyDropEligible = args.forceMaterialization === true || args.contextUsage.percentage >= FORCE_MATERIALIZATION_PERCENTAGE;
|
|
182811
|
+
let autoReclaimTargetCount = 0;
|
|
182812
|
+
let autoReclaimDidMutate = false;
|
|
182813
|
+
if (toolReclaimExecutePass && alreadyMutatingThisPass && !emergencyDropEligible) {
|
|
182814
|
+
const reclaimMeta = getOrCreateSessionMeta(args.db, args.sessionId);
|
|
182815
|
+
const syntheticPendingOps = buildSyntheticToolReclaimOps({
|
|
182816
|
+
db: args.db,
|
|
182817
|
+
sessionId: args.sessionId,
|
|
182818
|
+
targets,
|
|
182819
|
+
watermark: reclaimMeta.toolReclaimWatermark ?? 0,
|
|
182820
|
+
pendingOps
|
|
182821
|
+
});
|
|
182822
|
+
autoReclaimTargetCount = syntheticPendingOps.length;
|
|
182823
|
+
if (syntheticPendingOps.length > 0) {
|
|
182824
|
+
autoReclaimDidMutate = applyPendingOperations(args.sessionId, args.db, targets, args.protectedTags, undefined, [], syntheticPendingOps);
|
|
182825
|
+
}
|
|
182826
|
+
}
|
|
182353
182827
|
transcript.commit();
|
|
182828
|
+
if (toolReclaimExecutePass) {
|
|
182829
|
+
advanceToolReclaimWatermarkToCurrentMax(args.db, args.sessionId);
|
|
182830
|
+
}
|
|
182831
|
+
if (autoReclaimTargetCount > 0) {
|
|
182832
|
+
sessionLog(args.sessionId, `tool reclaim auto-drop: targets=${autoReclaimTargetCount} mutated=${autoReclaimDidMutate}`);
|
|
182833
|
+
}
|
|
182354
182834
|
const postCommitStableIdByRef = new Map;
|
|
182355
182835
|
const postCommitEntryIdByRef = new Map;
|
|
182356
182836
|
for (let i = 0;i < args.messages.length; i++) {
|
|
@@ -182634,6 +183114,7 @@ function clearContextHandlerSession(sessionId) {
|
|
|
182634
183114
|
rawMessageProviderUnregistersBySession.delete(sessionId);
|
|
182635
183115
|
}
|
|
182636
183116
|
clearSessionTracking(sessionId);
|
|
183117
|
+
clearPiEmbedSessionState(sessionId);
|
|
182637
183118
|
}
|
|
182638
183119
|
|
|
182639
183120
|
// src/commands/ctx-flush.ts
|
|
@@ -184478,6 +184959,11 @@ Historian recomp started. Rebuilding compartments and facts from raw Pi session
|
|
|
184478
184959
|
fallbackModelId: ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined
|
|
184479
184960
|
}, parsed.kind === "partial" ? { range: parsed.range } : {});
|
|
184480
184961
|
if (result.published) {
|
|
184962
|
+
try {
|
|
184963
|
+
clearEmergencyRecovery(deps.db, sessionId);
|
|
184964
|
+
} catch (recoveryError) {
|
|
184965
|
+
sessionLog(sessionId, `/ctx-recomp: clearEmergencyRecovery failed (continuing): ${describeError(recoveryError).brief}`);
|
|
184966
|
+
}
|
|
184481
184967
|
try {
|
|
184482
184968
|
stagePiRecompMarker({ db: deps.db, sessionId, ctx });
|
|
184483
184969
|
} catch (markerError) {
|
|
@@ -185131,7 +185617,7 @@ function formatThresholdPercent(value) {
|
|
|
185131
185617
|
// package.json
|
|
185132
185618
|
var package_default = {
|
|
185133
185619
|
name: "@wolfx/pi-magic-context",
|
|
185134
|
-
version: "0.
|
|
185620
|
+
version: "0.25.0",
|
|
185135
185621
|
type: "module",
|
|
185136
185622
|
description: "Pi coding agent extension for Magic Context — cross-session memory and context management",
|
|
185137
185623
|
main: "dist/index.js",
|
|
@@ -185185,8 +185671,8 @@ var package_default = {
|
|
|
185185
185671
|
typescript: "^5.8.0"
|
|
185186
185672
|
},
|
|
185187
185673
|
peerDependencies: {
|
|
185188
|
-
"@earendil-works/pi-coding-agent": "
|
|
185189
|
-
"@earendil-works/pi-tui": "
|
|
185674
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
185675
|
+
"@earendil-works/pi-tui": "*"
|
|
185190
185676
|
},
|
|
185191
185677
|
exports: {
|
|
185192
185678
|
".": {
|
|
@@ -185771,9 +186257,184 @@ Older parts of this session are summarized into <compartment> blocks inside <ses
|
|
|
185771
186257
|
|
|
185772
186258
|
ctx_expand(start=120, end=245) ← the compartment's own start/end attributes
|
|
185773
186259
|
|
|
185774
|
-
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable
|
|
186260
|
+
Returns the raw transcript as [N] U:/A: lines, capped at ~15K tokens; an oversized range returns the head and tells you where to continue. Also works with ordinals from ctx_search message results — expand a window around a hit (e.g. start=N-10, end=N+5). Ranges after the last compartment are your live tail — already visible in context, not expandable.
|
|
186261
|
+
|
|
186262
|
+
Two recovery modes for finer detail:
|
|
186263
|
+
- ctx_expand(start=120, end=245, verbose=true) — lists each message SEPARATELY with its ordinal [N] and a per-part preview (each tool call shown with its output size). Use this to find the exact message or tool call you want, then recover it in full by ordinal.
|
|
186264
|
+
- ctx_expand(message=138) — returns the FULL untruncated content of the message at that ordinal: every text part, and every tool call's complete input + output, read from stored history. This is the cheap way to get back a tool output you dropped with ctx_reduce — the original is still in storage even though the wire shows [dropped §N§]. If the message was deleted from history (session prune/revert), it says so.`;
|
|
185775
186265
|
var CTX_EXPAND_TOKEN_BUDGET = 15000;
|
|
185776
186266
|
|
|
186267
|
+
// ../plugin/src/tools/ctx-expand/render.ts
|
|
186268
|
+
function isRecord3(value) {
|
|
186269
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
186270
|
+
}
|
|
186271
|
+
function roleLabel(role) {
|
|
186272
|
+
if (role === "assistant")
|
|
186273
|
+
return "A (assistant)";
|
|
186274
|
+
if (role === "user")
|
|
186275
|
+
return "U (user)";
|
|
186276
|
+
return role;
|
|
186277
|
+
}
|
|
186278
|
+
function truncate2(value, max) {
|
|
186279
|
+
const t = value.trim();
|
|
186280
|
+
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
186281
|
+
}
|
|
186282
|
+
function keyArg(input) {
|
|
186283
|
+
if (!input)
|
|
186284
|
+
return "";
|
|
186285
|
+
for (const k of ["filePath", "path", "pattern", "query", "symbol", "module", "action"]) {
|
|
186286
|
+
const v = input[k];
|
|
186287
|
+
if (typeof v === "string" && v.length > 0)
|
|
186288
|
+
return truncate2(v, 60);
|
|
186289
|
+
}
|
|
186290
|
+
if (typeof input.description === "string")
|
|
186291
|
+
return truncate2(input.description, 60);
|
|
186292
|
+
return "";
|
|
186293
|
+
}
|
|
186294
|
+
function asToolPart(part) {
|
|
186295
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
186296
|
+
if (type === "tool") {
|
|
186297
|
+
const state = isRecord3(part.state) ? part.state : null;
|
|
186298
|
+
const output = state && typeof state.output === "string" ? state.output : state && state.output != null ? JSON.stringify(state.output) : null;
|
|
186299
|
+
const metadata = state && isRecord3(state.metadata) ? state.metadata : null;
|
|
186300
|
+
const title = state && typeof state.title === "string" && state.title || metadata && typeof metadata.title === "string" && metadata.title || null;
|
|
186301
|
+
return {
|
|
186302
|
+
name: typeof part.tool === "string" ? part.tool : "tool",
|
|
186303
|
+
callId: typeof part.callID === "string" ? part.callID : "",
|
|
186304
|
+
title,
|
|
186305
|
+
input: state && isRecord3(state.input) ? state.input : null,
|
|
186306
|
+
output
|
|
186307
|
+
};
|
|
186308
|
+
}
|
|
186309
|
+
if (type === "tool_use") {
|
|
186310
|
+
return {
|
|
186311
|
+
name: typeof part.name === "string" ? part.name : "tool",
|
|
186312
|
+
callId: typeof part.id === "string" ? part.id : "",
|
|
186313
|
+
title: null,
|
|
186314
|
+
input: isRecord3(part.input) ? part.input : null,
|
|
186315
|
+
output: null
|
|
186316
|
+
};
|
|
186317
|
+
}
|
|
186318
|
+
if (type === "tool_result") {
|
|
186319
|
+
const content = part.content;
|
|
186320
|
+
const output = typeof content === "string" ? content : content != null ? JSON.stringify(content) : null;
|
|
186321
|
+
return {
|
|
186322
|
+
name: "tool_result",
|
|
186323
|
+
callId: typeof part.tool_use_id === "string" ? part.tool_use_id : "",
|
|
186324
|
+
title: null,
|
|
186325
|
+
input: null,
|
|
186326
|
+
output
|
|
186327
|
+
};
|
|
186328
|
+
}
|
|
186329
|
+
return null;
|
|
186330
|
+
}
|
|
186331
|
+
function textOf(part) {
|
|
186332
|
+
if (part.type === "text" && typeof part.text === "string")
|
|
186333
|
+
return part.text;
|
|
186334
|
+
return null;
|
|
186335
|
+
}
|
|
186336
|
+
function reasoningOf(part) {
|
|
186337
|
+
if ((part.type === "reasoning" || part.type === "thinking") && typeof part.text === "string") {
|
|
186338
|
+
return part.text;
|
|
186339
|
+
}
|
|
186340
|
+
return null;
|
|
186341
|
+
}
|
|
186342
|
+
function renderPartPreview(part) {
|
|
186343
|
+
if (!isRecord3(part))
|
|
186344
|
+
return null;
|
|
186345
|
+
const text = textOf(part);
|
|
186346
|
+
if (text !== null) {
|
|
186347
|
+
const t = truncate2(text, 200);
|
|
186348
|
+
return t.length > 0 ? ` • ${t}` : null;
|
|
186349
|
+
}
|
|
186350
|
+
const tool = asToolPart(part);
|
|
186351
|
+
if (tool) {
|
|
186352
|
+
const arg = keyArg(tool.input);
|
|
186353
|
+
const head = arg ? `${tool.name}(${arg})` : tool.name;
|
|
186354
|
+
return tool.output !== null ? ` • tool ${head} → output ~${estimateTokens(tool.output)} tok` : ` • tool ${head}`;
|
|
186355
|
+
}
|
|
186356
|
+
const reasoning = reasoningOf(part);
|
|
186357
|
+
if (reasoning !== null)
|
|
186358
|
+
return ` • [reasoning] ${truncate2(reasoning, 120)}`;
|
|
186359
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
186360
|
+
if (type === "file")
|
|
186361
|
+
return " • [file]";
|
|
186362
|
+
if (type === "step-start" || type === "step-finish")
|
|
186363
|
+
return null;
|
|
186364
|
+
return ` • [${type}]`;
|
|
186365
|
+
}
|
|
186366
|
+
function renderPartFull(part) {
|
|
186367
|
+
if (!isRecord3(part))
|
|
186368
|
+
return null;
|
|
186369
|
+
const text = textOf(part);
|
|
186370
|
+
if (text !== null) {
|
|
186371
|
+
return text.trim().length > 0 ? ` [text]
|
|
186372
|
+
${text}` : null;
|
|
186373
|
+
}
|
|
186374
|
+
const tool = asToolPart(part);
|
|
186375
|
+
if (tool) {
|
|
186376
|
+
const lines = [];
|
|
186377
|
+
const idSuffix = tool.callId ? ` #${tool.callId}` : "";
|
|
186378
|
+
lines.push(` [tool: ${tool.name}${idSuffix}]`);
|
|
186379
|
+
if (tool.title && tool.title.trim().length > 0) {
|
|
186380
|
+
lines.push(` description: ${tool.title.trim()}`);
|
|
186381
|
+
}
|
|
186382
|
+
if (tool.input)
|
|
186383
|
+
lines.push(` input: ${JSON.stringify(tool.input)}`);
|
|
186384
|
+
if (tool.output !== null)
|
|
186385
|
+
lines.push(` output:
|
|
186386
|
+
${tool.output}`);
|
|
186387
|
+
return lines.join(`
|
|
186388
|
+
`);
|
|
186389
|
+
}
|
|
186390
|
+
const type = typeof part.type === "string" ? part.type : "part";
|
|
186391
|
+
if (type === "file") {
|
|
186392
|
+
const name2 = typeof part.filename === "string" && part.filename || typeof part.url === "string" && part.url || "";
|
|
186393
|
+
return ` [file]${name2 ? ` ${name2}` : ""}`;
|
|
186394
|
+
}
|
|
186395
|
+
return null;
|
|
186396
|
+
}
|
|
186397
|
+
function renderMessageByOrdinal(sessionId, ordinal) {
|
|
186398
|
+
const msg = readRawSessionMessages(sessionId).find((m) => m.ordinal === ordinal);
|
|
186399
|
+
if (!msg) {
|
|
186400
|
+
return `No message at ordinal ${ordinal} in this session's stored history — it was deleted ` + `(session prune/revert) or the ordinal is wrong, so it can't be recovered. ` + `Re-run the tool if you still need the data.`;
|
|
186401
|
+
}
|
|
186402
|
+
const rendered = msg.parts.map(renderPartFull).filter((l) => l !== null);
|
|
186403
|
+
const lines = [`[${msg.ordinal}] ${roleLabel(msg.role)} — full recovery:`, ""];
|
|
186404
|
+
if (rendered.length === 0) {
|
|
186405
|
+
lines.push(" (no recoverable content — message had only structural/reasoning parts)");
|
|
186406
|
+
} else {
|
|
186407
|
+
lines.push(...rendered);
|
|
186408
|
+
}
|
|
186409
|
+
return lines.join(`
|
|
186410
|
+
`);
|
|
186411
|
+
}
|
|
186412
|
+
function renderVerboseRange(sessionId, start, end, tokenBudget) {
|
|
186413
|
+
const messages = readRawSessionMessages(sessionId).filter((m) => m.ordinal >= start && m.ordinal <= end);
|
|
186414
|
+
const out = [];
|
|
186415
|
+
let usedTokens = 0;
|
|
186416
|
+
let lastOrdinal = start - 1;
|
|
186417
|
+
let truncated = false;
|
|
186418
|
+
for (const msg of messages) {
|
|
186419
|
+
const header = `[${msg.ordinal}] ${roleLabel(msg.role)}`;
|
|
186420
|
+
const partLines = msg.parts.map(renderPartPreview).filter((l) => l !== null);
|
|
186421
|
+
const block = partLines.length > 0 ? `${header}
|
|
186422
|
+
${partLines.join(`
|
|
186423
|
+
`)}` : header;
|
|
186424
|
+
const blockTokens = estimateTokens(block);
|
|
186425
|
+
if (usedTokens + blockTokens > tokenBudget && out.length > 0) {
|
|
186426
|
+
truncated = true;
|
|
186427
|
+
break;
|
|
186428
|
+
}
|
|
186429
|
+
out.push(block);
|
|
186430
|
+
usedTokens += blockTokens;
|
|
186431
|
+
lastOrdinal = msg.ordinal;
|
|
186432
|
+
}
|
|
186433
|
+
return { text: out.join(`
|
|
186434
|
+
|
|
186435
|
+
`), lastOrdinal, truncated };
|
|
186436
|
+
}
|
|
186437
|
+
|
|
185777
186438
|
// ../../node_modules/.bun/typebox@1.1.38/node_modules/typebox/build/system/memory/memory.mjs
|
|
185778
186439
|
var exports_memory = {};
|
|
185779
186440
|
__export(exports_memory, {
|
|
@@ -189839,12 +190500,18 @@ __export(exports_typebox, {
|
|
|
189839
190500
|
});
|
|
189840
190501
|
// src/tools/ctx-expand.ts
|
|
189841
190502
|
var ParamsSchema = exports_typebox.Object({
|
|
189842
|
-
start: exports_typebox.Number({
|
|
190503
|
+
start: exports_typebox.Optional(exports_typebox.Number({
|
|
189843
190504
|
description: "Start message ordinal (from compartment start attribute)"
|
|
189844
|
-
}),
|
|
189845
|
-
end: exports_typebox.Number({
|
|
190505
|
+
})),
|
|
190506
|
+
end: exports_typebox.Optional(exports_typebox.Number({
|
|
189846
190507
|
description: "End message ordinal (from compartment end attribute)"
|
|
189847
|
-
})
|
|
190508
|
+
})),
|
|
190509
|
+
verbose: exports_typebox.Optional(exports_typebox.Boolean({
|
|
190510
|
+
description: "With start/end: list each message separately with its ordinal [N] and per-part preview, so you can recover one in full by ordinal."
|
|
190511
|
+
})),
|
|
190512
|
+
message: exports_typebox.Optional(exports_typebox.Number({
|
|
190513
|
+
description: "Full untruncated recovery of ONE message by its ordinal (text + every tool call's full input/output). Recovers a tool output you dropped with ctx_reduce."
|
|
190514
|
+
}))
|
|
189848
190515
|
});
|
|
189849
190516
|
function ok(text) {
|
|
189850
190517
|
return { content: [{ type: "text", text }], details: undefined };
|
|
@@ -189863,22 +190530,41 @@ function createCtxExpandTool(deps) {
|
|
|
189863
190530
|
description: CTX_EXPAND_DESCRIPTION,
|
|
189864
190531
|
parameters: ParamsSchema,
|
|
189865
190532
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
189866
|
-
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
189867
|
-
return err("Error: start and end must be positive integers with start <= end.");
|
|
189868
|
-
}
|
|
189869
190533
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
189870
190534
|
if (!sessionId) {
|
|
189871
190535
|
return err("Error: no active Pi session.");
|
|
189872
190536
|
}
|
|
189873
|
-
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
189874
|
-
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
189875
|
-
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
189876
|
-
}
|
|
189877
|
-
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
189878
190537
|
const unregister = setRawMessageProvider(sessionId, {
|
|
189879
190538
|
readMessages: () => readPiSessionMessages(ctx)
|
|
189880
190539
|
});
|
|
189881
190540
|
try {
|
|
190541
|
+
if (typeof params.message === "number" && params.message >= 1) {
|
|
190542
|
+
return ok(renderMessageByOrdinal(sessionId, params.message));
|
|
190543
|
+
}
|
|
190544
|
+
if (typeof params.start !== "number" || typeof params.end !== "number" || params.start < 1 || params.end < params.start) {
|
|
190545
|
+
return err("Error: provide either message=<ordinal>, or start and end (positive integers, start <= end).");
|
|
190546
|
+
}
|
|
190547
|
+
const lastCompartmentEnd = getLastCompartmentEndMessage(deps.db, sessionId);
|
|
190548
|
+
if (lastCompartmentEnd >= 0 && params.start > lastCompartmentEnd) {
|
|
190549
|
+
return ok(`Range ${params.start}-${params.end} is entirely within the live tail (after the last compacted message ${lastCompartmentEnd}); those messages are already visible in context.`);
|
|
190550
|
+
}
|
|
190551
|
+
const effectiveEnd = lastCompartmentEnd >= 0 ? Math.min(params.end, lastCompartmentEnd) : params.end;
|
|
190552
|
+
if (params.verbose === true) {
|
|
190553
|
+
const v = renderVerboseRange(sessionId, params.start, effectiveEnd, CTX_EXPAND_TOKEN_BUDGET);
|
|
190554
|
+
if (!v.text) {
|
|
190555
|
+
return ok(`No messages found in range ${params.start}-${effectiveEnd}. The range may be outside this session's history.`);
|
|
190556
|
+
}
|
|
190557
|
+
const out = [
|
|
190558
|
+
`Messages ${params.start}-${v.lastOrdinal} (verbose). Recover any one in full with ctx_expand(message=<ordinal>):`,
|
|
190559
|
+
"",
|
|
190560
|
+
v.text
|
|
190561
|
+
];
|
|
190562
|
+
if (v.truncated) {
|
|
190563
|
+
out.push("", `Truncated at message ${v.lastOrdinal} (budget: ~${CTX_EXPAND_TOKEN_BUDGET} tokens). Call again with start=${v.lastOrdinal + 1} end=${effectiveEnd} verbose=true for more.`);
|
|
190564
|
+
}
|
|
190565
|
+
return ok(out.join(`
|
|
190566
|
+
`));
|
|
190567
|
+
}
|
|
189882
190568
|
const chunk = readSessionChunk(sessionId, CTX_EXPAND_TOKEN_BUDGET, params.start, effectiveEnd + 1);
|
|
189883
190569
|
if (!chunk.text || chunk.messageCount === 0) {
|
|
189884
190570
|
return ok(`No messages found in range ${params.start}-${params.end}. The range may be outside this session's history.`);
|
|
@@ -190597,15 +191283,16 @@ function parseInteger(str) {
|
|
|
190597
191283
|
}
|
|
190598
191284
|
|
|
190599
191285
|
// ../plugin/src/tools/ctx-reduce/constants.ts
|
|
190600
|
-
var CTX_REDUCE_DESCRIPTION = `
|
|
190601
|
-
|
|
191286
|
+
var CTX_REDUCE_DESCRIPTION = `Mark spent tagged content as discardable to reclaim context space. This is NOT an immediate delete. Use §N§ identifiers visible in the conversation. The \`drop\` param accepts ranges: "3-5", "1,2,9", "1-5,8".
|
|
191287
|
+
|
|
191288
|
+
How it works:
|
|
191289
|
+
- Marking QUEUES content for release. It stays fully visible to you until context space is actually needed — which may be as soon as the next turn if you are already under pressure, or many turns later if not. So mark spent outputs as soon as you finish with them; don't hoard the call for the end of the turn.
|
|
191290
|
+
- The newest tags are protected: marking one just queues it until it ages out of the recent window, so marking recent output is harmless.
|
|
191291
|
+
- When content is finally released it becomes a short placeholder, and re-running the tool is the only way to get it back. So mark only what you are genuinely DONE with — the test is "have I extracted what I need from this?", not "is it safe / do I have time before it drops?".
|
|
190602
191292
|
|
|
190603
|
-
|
|
190604
|
-
|
|
190605
|
-
-
|
|
190606
|
-
- Protected tags are accepted but deferred until they leave the last protected range.
|
|
190607
|
-
- Keep recent context — only reduce OLD content that is no longer relevant to current work.
|
|
190608
|
-
- Dropped content is gone forever.`;
|
|
191293
|
+
Mark discardable once processed: large outputs you've summarized, repeated or redundant dumps, data written to disk, status/log output that only confirmed an expected state.
|
|
191294
|
+
Keep: user messages, unresolved errors, raw evidence you haven't extracted yet, and outputs whose exact wording may matter later.
|
|
191295
|
+
Never blanket-mark large ranges (e.g. "1-50") — review what each tag holds first.`;
|
|
190609
191296
|
|
|
190610
191297
|
// src/tools/ctx-reduce.ts
|
|
190611
191298
|
var ParamsSchema4 = exports_typebox.Object({
|
|
@@ -191206,7 +191893,21 @@ async function src_default2(pi) {
|
|
|
191206
191893
|
},
|
|
191207
191894
|
historian: hist,
|
|
191208
191895
|
autoSearch: auto,
|
|
191209
|
-
resolveForProject: resolveContextOptionsForProject
|
|
191896
|
+
resolveForProject: resolveContextOptionsForProject,
|
|
191897
|
+
maybeAutoEmbedSession: (sessionId, dir, identity) => {
|
|
191898
|
+
maybeAutoEmbedPiSession({
|
|
191899
|
+
db: database,
|
|
191900
|
+
projectDir: dir,
|
|
191901
|
+
projectIdentity: identity,
|
|
191902
|
+
memoryEnabled: cfg.memory.enabled
|
|
191903
|
+
}, sessionId, dir, identity, (text) => {
|
|
191904
|
+
pi.sendMessage({
|
|
191905
|
+
customType: "ctx-status",
|
|
191906
|
+
content: text,
|
|
191907
|
+
display: true
|
|
191908
|
+
}, { triggerTurn: false });
|
|
191909
|
+
});
|
|
191910
|
+
}
|
|
191210
191911
|
});
|
|
191211
191912
|
function resolveContextOptionsForProject(dir) {
|
|
191212
191913
|
const cached2 = contextOptionsByDir.get(dir);
|
|
@@ -191288,14 +191989,14 @@ async function src_default2(pi) {
|
|
|
191288
191989
|
onProjectSeen: (identity) => seenDreamerProjectIdentities.add(identity)
|
|
191289
191990
|
});
|
|
191290
191991
|
info("registered /ctx-dream");
|
|
191291
|
-
|
|
191992
|
+
registerCtxEmbedCommand(pi, {
|
|
191292
191993
|
db,
|
|
191293
191994
|
projectDir,
|
|
191294
191995
|
projectIdentity,
|
|
191295
191996
|
memoryEnabled: config2.memory.enabled,
|
|
191296
191997
|
resolveProject: resolveCurrentProject
|
|
191297
191998
|
});
|
|
191298
|
-
info("registered /ctx-embed
|
|
191999
|
+
info("registered /ctx-embed");
|
|
191299
192000
|
const dreamerConfig = resolveDreamerFromConfig(config2);
|
|
191300
192001
|
if (dreamerConfig) {
|
|
191301
192002
|
registerPiDreamerProject({
|