@dex-ai/context 0.7.16

package/src/formatter.ts

@@ -0,0 +1,127 @@
+/**
+ * Format context usage for terminal display.
+ */
+
+import type { ContextSnapshot } from "./types";
+
+const RESET = "\x1b[0m";
+const BOLD = "\x1b[1m";
+const DIM = "\x1b[2m";
+
+const BLOCK_USED = "\x1b[38;5;185m■" + RESET;
+const BLOCK_FREE = "\x1b[38;5;240m□" + RESET;
+
+const CAT_COLORS: Record<string, string> = {
+	"system-prompt": "\x1b[38;5;246m",
+	"system-tools": "\x1b[38;5;248m",
+	"tool-calls": "\x1b[38;5;179m",
+	"tool-results": "\x1b[38;5;185m",
+	messages: "\x1b[38;5;116m",
+	images: "\x1b[38;5;141m",
+	files: "\x1b[38;5;174m",
+	reasoning: "\x1b[38;5;110m",
+};
+
+const CAT_LABELS: Record<string, string> = {
+	"system-prompt": "System Prompt",
+	"system-tools": "System Tools",
+	"tool-calls": "Tool Calls",
+	"tool-results": "Tool Results",
+	messages: "Messages",
+	images: "Images",
+	files: "Files",
+	reasoning: "Reasoning",
+};
+
+export function formatContextUsage(snap: ContextSnapshot): string {
+	const lines: string[] = [];
+	lines.push(`  ${BOLD}Context Usage${RESET}`);
+	lines.push("");
+
+	// Grid: 10×5 = 50 cells
+	const COLS = 10,
+		ROWS = 5,
+		TOTAL = COLS * ROWS;
+	const usedCells = Math.round((snap.usagePercent / 100) * TOTAL);
+	const grid: string[][] = [];
+	for (let r = 0; r < ROWS; r++) {
+		const row: string[] = [];
+		for (let c = 0; c < COLS; c++) {
+			row.push(r * COLS + c < usedCells ? BLOCK_USED : BLOCK_FREE);
+		}
+		grid.push(row);
+	}
+
+	// Right-side stats
+	const right: string[] = [];
+	right.push(
+		`${BOLD}Total Usage${RESET}    ${fmtK(snap.totalTokens)} (${fmtPct(snap.usagePercent)})`,
+	);
+	right.push("");
+	for (const cat of snap.categories) {
+		const color = CAT_COLORS[cat.category] ?? "";
+		const label = CAT_LABELS[cat.category] ?? cat.category;
+		right.push(
+			`${color}■${RESET} ${color}${label}${RESET}${" ".repeat(Math.max(1, 15 - label.length))}${fmtK(cat.tokens).padStart(5)} (${fmtPct(cat.percent)})`,
+		);
+	}
+	right.push(
+		`${DIM}□${RESET} ${DIM}Available${RESET}      ${fmtK(snap.availableTokens).padStart(5)} (${fmtPct(100 - snap.usagePercent)})`,
+	);
+
+	if (snap.tokensSaved > 0) {
+		right.push("");
+		right.push(
+			`${DIM}Saved: ~${fmtK(snap.tokensSaved)} (${snap.compressions} compression${snap.compressions !== 1 ? "s" : ""})${RESET}`,
+		);
+	}
+
+	// Merge
+	const maxRows = Math.max(grid.length, right.length);
+	for (let i = 0; i < maxRows; i++) {
+		const left = i < grid.length ? "  " + grid[i].join(" ") : " ".repeat(21);
+		const r = right[i] ?? "";
+		lines.push(`${left}     ${r}`);
+	}
+
+	return lines.join("\n");
+}
+
+export function formatContextUsagePlain(snap: ContextSnapshot): string {
+	const lines: string[] = [];
+	lines.push("Context Usage");
+	lines.push("─".repeat(45));
+	lines.push(
+		`Total: ${fmtK(snap.totalTokens)} / ${fmtK(snap.maxTokens)} (${fmtPct(snap.usagePercent)})`,
+	);
+	lines.push("");
+	for (const cat of snap.categories) {
+		const label = CAT_LABELS[cat.category] ?? cat.category;
+		const bar =
+			"█".repeat(Math.round(cat.percent / 5)) +
+			"░".repeat(20 - Math.round(cat.percent / 5));
+		lines.push(
+			`  ${label.padEnd(15)} ${bar} ${fmtK(cat.tokens).padStart(5)} (${fmtPct(cat.percent)})`,
+		);
+	}
+	lines.push(
+		`  ${"Available".padEnd(15)} ${"░".repeat(20)} ${fmtK(snap.availableTokens).padStart(5)} (${fmtPct(100 - snap.usagePercent)})`,
+	);
+	if (snap.tokensSaved > 0) {
+		lines.push("");
+		lines.push(
+			`  Saved: ~${fmtK(snap.tokensSaved)} (${snap.compressions} compression${snap.compressions !== 1 ? "s" : ""})`,
+		);
+	}
+	return lines.join("\n");
+}
+
+function fmtK(n: number): string {
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+	if (n >= 1_000) return `${Math.round(n / 1_000)}k`;
+	return `${n}`;
+}
+
+function fmtPct(p: number): string {
+	return `${p.toFixed(1).padStart(5)}%`;
+}

package/src/index.ts

@@ -0,0 +1,45 @@
+/**
+ * @dex-ai/context — Index-and-pointer context management for Dex.
+ *
+ * Philosophy: NEVER LOSE INFORMATION.
+ *
+ *   1. INDEX — large tool results are stored in an FTS5 knowledge base
+ *   2. POINTER — context gets a compact reference, not the raw data
+ *   3. SEARCH — ctx_search retrieves specific chunks on demand
+ *   4. COMPRESS — graduated aging with lossless session resume
+ *
+ * The context window shrinks, but the knowledge base grows. The agent
+ * can always recover any previously-seen content via search.
+ */
+
+export { contextExtension } from "./extension";
+export type {
+	ContextExtensionOptions,
+	ContextSnapshot,
+	ContextCategory,
+	ContextBudget,
+	ContextEvent,
+	ContextEventType,
+} from "./types";
+export { estimateTokens } from "./tokenizer";
+export { formatContextUsage, formatContextUsagePlain } from "./formatter";
+export {
+	EventLog,
+	extractEvents,
+	type SessionEvent,
+	type ToolResultInput,
+} from "./event-log";
+export { buildSnapshot } from "./snapshot";
+export { resolveBudget, getSendCap, getContextBudget } from "./pressure";
+export { summarizeToolResults } from "./summarize";
+export { ContentStore, sanitizeQuery } from "./store";
+export type {
+	IndexResult,
+	SearchResult,
+	SourceInfo,
+	StoreStats,
+} from "./store";
+export { createSearchTool } from "./search-tool";
+
+import { contextExtension as _ctx } from "./extension";
+export default _ctx();

package/src/pressure.ts

@@ -0,0 +1,61 @@
+/**
+ * Context Budget Resolution.
+ *
+ * Fixed-cap budgets. Percentages don't work for large-context models
+ * (Gemini 1M, etc.) where even 50% = 500k tokens — far beyond what's
+ * useful for quality, cost, and caching.
+ *
+ * Two limits:
+ * - contextBudget: maximum total tokens tracked (history + system + tools)
+ * - sendCap: maximum tokens actually sent to the model per request
+ *
+ * The budget determines when compression tiers activate.
+ * The sendCap ensures we never waste tokens on oversized requests.
+ */
+
+import type { ContextBudget } from "./types";
+
+/**
+ * Maximum total context budget (what we track and manage).
+ * Fixed at 200k regardless of model window size.
+ */
+const CONTEXT_BUDGET = 200_000;
+
+/**
+ * Maximum tokens to actually send to the model per request.
+ * Keeps prompt focused and cost-efficient.
+ */
+const SEND_CAP = 80_000;
+
+/**
+ * Resolve the effective token budget for context management.
+ *
+ * All modes use a fixed 200k cap. The budget tier controls how
+ * aggressively compression is applied (via message-count thresholds
+ * in the extension), not the total budget size.
+ *
+ * The modelWindowTokens parameter is kept for safety — if a model
+ * has a window smaller than our cap (rare), we respect it.
+ */
+export function resolveBudget(
+	budget: ContextBudget,
+	modelWindowTokens: number,
+): number {
+	// Never exceed the model's actual window
+	return Math.min(CONTEXT_BUDGET, modelWindowTokens);
+}
+
+/**
+ * Get the send cap — maximum tokens to include in a single model request.
+ * This is independent of the budget tier.
+ */
+export function getSendCap(): number {
+	return SEND_CAP;
+}
+
+/**
+ * Get the raw context budget constant.
+ */
+export function getContextBudget(): number {
+	return CONTEXT_BUDGET;
+}

package/src/search-tool.ts

@@ -0,0 +1,230 @@
+/**
+ * Context Search Tool — on-demand retrieval from the FTS5 knowledge base.
+ *
+ * This tool is registered by the context extension and allows the agent to
+ * retrieve specific chunks from previously-indexed content. When large tool
+ * results are auto-indexed (instead of filling context), this tool is the
+ * mechanism for the agent to pull back exactly what it needs.
+ *
+ * Design principles:
+ * - Batch queries in a single call (["query1", "query2", ...])
+ * - Return relevant chunks with titles + content
+ * - Progressive throttling to prevent flooding
+ * - Source filtering for scoped searches
+ */
+
+import { z } from "zod";
+import type { Tool } from "@dex-ai/sdk";
+import type { ContentStore, SearchResult } from "./store";
+
+/* ── Types ─────────────────────────────────────────────── */
+
+interface SearchToolInput {
+	queries: string[];
+	limit?: number;
+	source?: string;
+}
+
+/* ── Throttle State ────────────────────────────────────── */
+
+const THROTTLE_WINDOW_MS = 60_000;
+const THROTTLE_WARN_AFTER = 5;
+const THROTTLE_BLOCK_AFTER = 10;
+
+let windowStart = Date.now();
+let callsInWindow = 0;
+
+function resetThrottleIfExpired(): void {
+	const now = Date.now();
+	if (now - windowStart > THROTTLE_WINDOW_MS) {
+		callsInWindow = 0;
+		windowStart = now;
+	}
+}
+
+/* ── Tool Factory ──────────────────────────────────────── */
+
+/**
+ * Create the ctx_search tool bound to a ContentStore instance.
+ * The store reference is captured in a closure so it can be swapped/rebuilt.
+ */
+export function createSearchTool(
+	getStore: () => ContentStore,
+): Tool<SearchToolInput, unknown> {
+	return {
+		name: "ctx_search",
+		displayName: "Context Search",
+		description:
+			"Search the session knowledge base. Content is auto-indexed from large tool results " +
+			"that were too big for the context window. Pass ALL questions as a queries array in ONE call.\n\n" +
+			"WHEN TO USE:\n" +
+			"- After seeing 'Indexed N sections from: <source>' messages\n" +
+			"- To recall file contents that were read earlier but compressed\n" +
+			"- To find specific code/text from large command outputs\n" +
+			"- When session_resume references work you need details on\n\n" +
+			"TIPS:\n" +
+			"- Use 2-4 specific terms per query (function names, error messages, file names)\n" +
+			"- Use 'source' to scope results (e.g. source: 'read(extension.ts)')\n" +
+			"- Batch queries: ctx_search(queries: ['query1', 'query2', 'query3'])",
+		parameters: z.object({
+			queries: z
+				.array(z.string())
+				.describe(
+					"Array of search queries. Batch ALL questions into one call.",
+				),
+			limit: z
+				.number()
+				.optional()
+				.default(3)
+				.describe("Max results per query (default: 3)"),
+			source: z
+				.string()
+				.optional()
+				.describe("Filter to a specific source label (partial match)."),
+		}) as any,
+		access: "read",
+
+		async execute(input: SearchToolInput) {
+			const store = getStore();
+
+			// Check if store is empty
+			if (store.isEmpty()) {
+				return {
+					type: "text" as const,
+					value:
+						"Knowledge base is empty — no content has been indexed yet.\n\n" +
+						"Content is auto-indexed when tool results exceed the context threshold. " +
+						"Use tools normally and large outputs will be indexed automatically.",
+				};
+			}
+
+			// Throttle check
+			resetThrottleIfExpired();
+			callsInWindow++;
+
+			if (callsInWindow > THROTTLE_BLOCK_AFTER) {
+				return {
+					type: "error-text" as const,
+					value:
+						`BLOCKED: ${callsInWindow} search calls in ${Math.round((Date.now() - windowStart) / 1000)}s. ` +
+						"You're flooding context. Consolidate your searches into fewer calls with more specific queries.",
+				};
+			}
+
+			const { queries, limit = 3, source } = input;
+			const effectiveLimit =
+				callsInWindow > THROTTLE_WARN_AFTER ? 1 : Math.min(limit, 5);
+
+			const sections: string[] = [];
+			let totalSize = 0;
+			const MAX_TOTAL = 40_000; // 40KB total cap
+
+			for (const query of queries) {
+				if (totalSize > MAX_TOTAL) {
+					sections.push(
+						`## ${query}\n(output cap reached — refine your search)`,
+					);
+					continue;
+				}
+
+				const results = store.search(query, effectiveLimit, source);
+
+				if (results.length === 0) {
+					sections.push(`## ${query}\nNo results found.`);
+					continue;
+				}
+
+				const formatted = results
+					.map((r: SearchResult) => {
+						const header = `--- [${r.source}] ---`;
+						const heading = `### ${r.title}`;
+						const snippet = extractSnippet(r.content, query, 1500);
+						return `${header}\n${heading}\n\n${snippet}`;
+					})
+					.join("\n\n");
+
+				sections.push(`## ${query}\n\n${formatted}`);
+				totalSize += formatted.length;
+			}
+
+			let output = sections.join("\n\n---\n\n");
+
+			// Throttle warning
+			if (callsInWindow >= THROTTLE_WARN_AFTER) {
+				output +=
+					`\n\n⚠ Search call #${callsInWindow}/${THROTTLE_BLOCK_AFTER} in this window. ` +
+					`Results limited to ${effectiveLimit}/query. Batch your queries.`;
+			}
+
+			return { type: "text" as const, value: output };
+		},
+	};
+}
+
+/* ── Helpers ───────────────────────────────────────────── */
+
+/**
+ * Extract a relevant snippet from content, centered around query term matches.
+ * Returns at most `maxChars` characters.
+ */
+function extractSnippet(
+	content: string,
+	query: string,
+	maxChars: number,
+): string {
+	if (content.length <= maxChars) return content;
+
+	// Find the best match position
+	const terms = query
+		.toLowerCase()
+		.split(/\s+/)
+		.filter((t) => t.length > 2);
+	const lowerContent = content.toLowerCase();
+
+	let bestPos = 0;
+	let bestScore = 0;
+
+	for (const term of terms) {
+		const pos = lowerContent.indexOf(term);
+		if (pos >= 0) {
+			// Score by how many other terms are nearby
+			let score = 1;
+			for (const other of terms) {
+				if (other === term) continue;
+				const nearby = lowerContent.indexOf(other, Math.max(0, pos - 200));
+				if (nearby >= 0 && nearby < pos + 200) score++;
+			}
+			if (score > bestScore) {
+				bestScore = score;
+				bestPos = pos;
+			}
+		}
+	}
+
+	// Center snippet around best position
+	const halfWindow = Math.floor(maxChars / 2);
+	const start = Math.max(0, bestPos - halfWindow);
+	const end = Math.min(content.length, start + maxChars);
+
+	let snippet = content.slice(start, end);
+
+	// Clean up: don't start/end mid-line
+	if (start > 0) {
+		const firstNewline = snippet.indexOf("\n");
+		if (firstNewline > 0 && firstNewline < 100) {
+			snippet = snippet.slice(firstNewline + 1);
+		} else {
+			snippet = "…" + snippet;
+		}
+	}
+	if (end < content.length) {
+		const lastNewline = snippet.lastIndexOf("\n");
+		if (lastNewline > snippet.length - 100) {
+			snippet = snippet.slice(0, lastNewline);
+		} else {
+			snippet = snippet + "…";
+		}
+	}
+
+	return snippet;
+}

package/src/snapshot.ts

@@ -0,0 +1,240 @@
+/**
+ * Session Snapshot — builds a compact XML resume from the event log.
+ *
+ * v5: Now includes RUNNABLE SEARCH QUERIES for each section so the agent
+ * can use ctx_search to recover full content from the knowledge base.
+ *
+ * The snapshot replaces old messages entirely — but because ALL content
+ * has been indexed into FTS5 before snapshot creation, this is truly
+ * lossless. The agent can always get the data back via search.
+ */
+
+import type { SessionEvent } from "./event-log";
+
+/* ── Types ─────────────────────────────────────────────── */
+
+export interface SnapshotOptions {
+	/** Maximum events to include per section */
+	maxPerSection?: number;
+	/** Include ctx_search hints in each section */
+	includeSearchHints?: boolean;
+}
+
+/* ── Main Builder ──────────────────────────────────────── */
+
+/**
+ * Build a session_resume XML snapshot from accumulated events.
+ * Each section includes a compact summary + ctx_search query for full details.
+ */
+export function buildSnapshot(
+	events: ReadonlyArray<SessionEvent>,
+	turnCount: number,
+	opts: SnapshotOptions = {},
+): string {
+	const maxPerSection = opts.maxPerSection ?? 10;
+	const includeSearchHints = opts.includeSearchHints ?? false;
+	const now = new Date().toISOString();
+
+	const sections: string[] = [];
+
+	// Search instruction (always present when hints are enabled)
+	if (includeSearchHints) {
+		sections.push(`  <how_to_recover>
+    All content from compressed turns is stored in the knowledge base.
+    Use ctx_search(queries: [...]) to retrieve FULL details for any section.
+    Do NOT ask the user to re-explain prior work. Search first.
+    Do NOT re-read files you already read. Search first.
+  </how_to_recover>`);
+	}
+
+	// Files section
+	const files = buildFilesSection(events, maxPerSection, includeSearchHints);
+	if (files) sections.push(files);
+
+	// Commands section
+	const commands = buildCommandsSection(events, maxPerSection, includeSearchHints);
+	if (commands) sections.push(commands);
+
+	// Errors section
+	const errors = buildErrorsSection(events, maxPerSection, includeSearchHints);
+	if (errors) sections.push(errors);
+
+	// Search section
+	const searches = buildSearchSection(events, maxPerSection);
+	if (searches) sections.push(searches);
+
+	if (sections.length === 0) {
+		return `<session_resume turns="${turnCount}" generated_at="${now}"/>\n`;
+	}
+
+	const body = sections.join("\n\n");
+	return [
+		`<session_resume turns="${turnCount}" generated_at="${now}">`,
+		body,
+		`</session_resume>`,
+	].join("\n");
+}
+
+/* ── Section Builders ──────────────────────────────────── */
+
+function buildFilesSection(
+	events: ReadonlyArray<SessionEvent>,
+	max: number,
+	includeSearchHints: boolean,
+): string {
+	const fileEvents = events.filter((e) => e.category === "file");
+	if (fileEvents.length === 0) return "";
+
+	// Group by path, count ops
+	const fileMap = new Map<string, Map<string, number>>();
+	for (const ev of fileEvents) {
+		let ops = fileMap.get(ev.data);
+		if (!ops) {
+			ops = new Map();
+			fileMap.set(ev.data, ops);
+		}
+		const op = ev.type.replace("file_", "");
+		ops.set(op, (ops.get(op) ?? 0) + 1);
+	}
+
+	// Take most recent files (last N)
+	const entries = [...fileMap.entries()].slice(-max);
+	const lines = entries.map(([path, ops]) => {
+		const opsStr = [...ops.entries()]
+			.map(([k, v]) => `${k}×${v}`)
+			.join(", ");
+		return `    ${basename(path)} (${opsStr})`;
+	});
+
+	// Build search queries from file paths
+	const searchHint = includeSearchHints
+		? buildSearchHint(entries.slice(-4).map(([path]) => `read(${path})`))
+		: "";
+
+	return [
+		`  <files count="${fileMap.size}">`,
+		...lines,
+		searchHint,
+		`  </files>`,
+	].filter(Boolean).join("\n");
+}
+
+function buildCommandsSection(
+	events: ReadonlyArray<SessionEvent>,
+	max: number,
+	includeSearchHints: boolean,
+): string {
+	const cmdEvents = events.filter((e) => e.category === "command");
+	if (cmdEvents.length === 0) return "";
+
+	// Take last N commands
+	const recent = cmdEvents.slice(-max);
+	const lines = recent.map((ev) => `    ${ev.data}`);
+
+	// Build search queries from command names
+	const searchHint = includeSearchHints
+		? buildSearchHint(recent.slice(-4).map((ev) => {
+			// Extract the core command for searchability
+			const cmd = ev.data.split("→")[0]!.trim().slice(0, 60);
+			return `bash(${cmd})`;
+		}))
+		: "";
+
+	return [
+		`  <commands count="${cmdEvents.length}">`,
+		...lines,
+		searchHint,
+		`  </commands>`,
+	].filter(Boolean).join("\n");
+}
+
+function buildErrorsSection(
+	events: ReadonlyArray<SessionEvent>,
+	max: number,
+	includeSearchHints: boolean,
+): string {
+	const errorEvents = events.filter((e) => e.category === "error");
+	if (errorEvents.length === 0) return "";
+
+	// Deduplicate errors
+	const seen = new Set<string>();
+	const unique: SessionEvent[] = [];
+	for (const ev of errorEvents) {
+		if (!seen.has(ev.data)) {
+			seen.add(ev.data);
+			unique.push(ev);
+		}
+	}
+
+	const recent = unique.slice(-max);
+	const lines = recent.map((ev) => `    ${ev.data}`);
+
+	// Error search hints use the error messages themselves
+	const searchHint = includeSearchHints
+		? buildSearchHint(recent.slice(-3).map((ev) => {
+			// Extract key error terms
+			const parts = ev.data.split(":");
+			return parts.length > 1 ? parts.slice(1).join(":").trim().slice(0, 60) : ev.data.slice(0, 60);
+		}))
+		: "";
+
+	return [
+		`  <errors count="${unique.length}">`,
+		...lines,
+		searchHint,
+		`  </errors>`,
+	].filter(Boolean).join("\n");
+}
+
+function buildSearchSection(
+	events: ReadonlyArray<SessionEvent>,
+	max: number,
+): string {
+	const searchEvents = events.filter((e) => e.category === "search");
+	if (searchEvents.length === 0) return "";
+
+	// Deduplicate
+	const seen = new Set<string>();
+	const unique: SessionEvent[] = [];
+	for (const ev of searchEvents) {
+		if (!seen.has(ev.data)) {
+			seen.add(ev.data);
+			unique.push(ev);
+		}
+	}
+
+	const recent = unique.slice(-max);
+	const lines = recent.map((ev) => `    ${ev.data}`);
+
+	return [
+		`  <searches count="${unique.length}">`,
+		...lines,
+		`  </searches>`,
+	].join("\n");
+}
+
+/* ── Helpers ───────────────────────────────────────────── */
+
+function basename(path: string): string {
+	const parts = path.split("/");
+	return parts[parts.length - 1] || path;
+}
+
+/**
+ * Build a ctx_search hint line for a section.
+ * Shows the agent exactly how to retrieve full details.
+ */
+function buildSearchHint(queries: string[]): string {
+	if (queries.length === 0) return "";
+	const escaped = queries.map((q) => `"${escapeXML(q)}"`).join(", ");
+	return `\n    For full details: ctx_search(queries: [${escaped}])`;
+}
+
+function escapeXML(str: string): string {
+	return str
+		.replace(/&/g, "&amp;")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&apos;");
+}