@dex-ai/context 0.7.16

package/README.md

@@ -0,0 +1,204 @@
+# @dex-ai/context
+
+Context management extension for Dex — tracks token usage, provides a visual `/context` command, enforces context budgets, and guides the LLM to avoid flooding the context window.
+
+## What It Does
+
+```
+Context Usage
+
+■ ■ ■ ■ ■ ■ ■ ■ ■ ■      Total Usage    102k ( 51.1%)
+■ ■ ■ ■ ■ ■ ■ ■ ■ ■
+■ ■ ■ ■ ■ ■ ■ ■ □ □      ■ System Prompt   1k (  0.7%)
+□ □ □ □ □ □ □ □ □ □      ■ System Tools    2k (  0.9%)
+□ □ □ □ □ □ □ □ □ □      ■ Tool Results   96k ( 48.1%)
+                           ■ Messages        3k (  1.5%)
+                           □ Available      98k ( 48.9%)
+```
+
+### Features
+
+1. **Real-time context tracking** — estimates token usage across categories (system prompt, tools, messages, tool calls/results, images, files, reasoning) using fast heuristics calibrated against cl100k_base.
+
+2. **`/context` command** — visual grid + percentage breakdown showing exactly where context is being consumed, with color-coded categories.
+
+3. **Threshold warnings** — automatically injects guidance to the LLM when context usage exceeds configurable thresholds (default: warn at 75%, critical at 90%).
+
+4. **Context-awareness skill** — teaches the LLM patterns for keeping outputs small (targeted reads, search-before-read, output truncation).
+
+5. **Tool output tracking** — records per-tool output sizes to identify the biggest context consumers.
+
+## Usage
+
+```typescript
+import { contextExtension } from "@dex-ai/context";
+import { Agent } from "@dex-ai/sdk";
+
+const agent = await Agent.create({
+  provider: "anthropic",
+  model: "claude-sonnet-4-20250514",
+  extensions: [
+    providerExt,
+    contextExtension(), // sensible defaults
+  ],
+});
+```
+
+### With Options
+
+```typescript
+contextExtension({
+  maxTokens: 128_000, // override context window (auto-detected from model)
+  warnAt: 70, // warn threshold (default: 75%)
+  criticalAt: 85, // critical threshold (default: 90%)
+  injectGuidance: true, // inject context-awareness skill (default: true)
+  trackToolOutputs: true, // track per-tool output sizes (default: true)
+  largeOutputThreshold: 4_000, // tokens above which an output is "large"
+});
+```
+
+## `/context` Command Integration
+
+The extension stores accessor functions in `AgentContext.state` that the host (CLI/TUI) can invoke:
+
+```typescript
+// In your host/CLI command handler:
+const getFormatted = agent.context.state.get(
+  "context:getFormatted",
+) as () => string;
+const getPlain = agent.context.state.get("context:getPlain") as () => string;
+const getSnapshot = agent.context.state.get(
+  "context:getSnapshot",
+) as () => ContextSnapshot;
+
+// For TUI rendering (ANSI color codes):
+console.log(getFormatted());
+
+// For plain text (no ANSI):
+console.log(getPlain());
+
+// For programmatic access:
+const snapshot = getSnapshot();
+console.log(
+  `${snapshot.usagePercent}% used, ${snapshot.availableTokens} remaining`,
+);
+```
+
+## How It Works
+
+### Token Estimation
+
+Uses character-based heuristics calibrated for modern tokenizers:
+
+- English text: ~4 chars per token
+- Code: ~3.5 chars per token
+- JSON/structured: ~3 chars per token
+- Images: Anthropic tile-based estimation
+
+This is intentionally approximate (±10%). Real token counts come from the provider's `usage` response and are used to calibrate the estimates.
+
+### Category Tracking
+
+| Category      | What's counted                                       |
+| ------------- | ---------------------------------------------------- |
+| System Prompt | System messages + injected skills                    |
+| System Tools  | Tool definitions (name + description + JSON Schema)  |
+| Tool Calls    | Assistant tool invocations (name + serialized input) |
+| Tool Results  | Tool outputs (text, JSON, rich content)              |
+| Messages      | User + assistant text messages                       |
+| Images        | Image content (resolution-based estimation)          |
+| Files         | File attachments (size-based estimation)             |
+| Reasoning     | Chain-of-thought / extended thinking                 |
+
+### Threshold Behavior
+
+When context usage exceeds a threshold, the extension injects a concise system message into the next model request:
+
+- **Warning (75%)**: `"⚠️ Context usage: 76%. Be concise. Avoid large reads/outputs."`
+- **Critical (90%)**: `"🚨 Context nearly full: 91% used. Complete the current task as concisely as possible."`
+
+These are injected once per threshold crossing (not on every model call).
+
+### Context-Awareness Skill
+
+When `injectGuidance: true` (default), a skill is added to the system prompt teaching the LLM:
+
+- Prefer targeted reads (line ranges) over full file reads
+- Use search before read to find relevant sections
+- Truncate bash output with `head`/`tail`/`grep`
+- Avoid redundant re-reads of files already in context
+- Be more concise when context is running low
+
+## Comparison with context-mode
+
+| Feature             | context-mode           | @dex-ai/context                                   |
+| ------------------- | ---------------------- | ------------------------------------------------- |
+| Token tracking      | ❌ (defers to host)    | ✅ Built-in estimation                            |
+| Visual display      | ❌                     | ✅ Grid + categories                              |
+| Hard-blocks tools   | ✅ (blocks curl/fetch) | ❌ (guidance-based)                               |
+| Sandboxed execution | ✅ (separate process)  | ❌ (not needed — Dex tools already handle this)   |
+| FTS5 knowledge base | ✅ (SQLite)            | ❌ (out of scope — use @dex-ai/knowledge)         |
+| Session persistence | ✅ (SessionDB)         | ❌ (out of scope — use @dex-ai/session-extension) |
+| Native dependencies | better-sqlite3         | None                                              |
+| Weight              | 3.6 MB                 | ~15 KB                                            |
+
+**Philosophy difference**: context-mode hard-blocks certain tool calls to prevent context flooding. @dex-ai/context takes a guidance approach — it teaches the LLM good patterns and warns when budgets are exceeded, but trusts the model to make the right choice. This works better with modern models that can follow instructions.
+
+## API
+
+### `contextExtension(opts?): Extension`
+
+Factory function — creates the extension.
+
+### `estimateTokens(text: string): number`
+
+Estimate token count for a string.
+
+### `formatContextUsage(snapshot: ContextSnapshot): string`
+
+Format a snapshot with ANSI colors for terminal display.
+
+### `formatContextUsagePlain(snapshot: ContextSnapshot): string`
+
+Format a snapshot as plain text (no colors).
+
+### Types
+
+```typescript
+interface ContextSnapshot {
+  timestamp: number;
+  totalTokens: number;
+  maxTokens: number;
+  usagePercent: number;
+  categories: CategoryUsage[];
+  availableTokens: number;
+}
+
+interface CategoryUsage {
+  category: ContextCategory;
+  tokens: number;
+  percent: number;
+}
+
+type ContextCategory =
+  | "system-prompt"
+  | "system-tools"
+  | "messages"
+  | "tool-calls"
+  | "tool-results"
+  | "images"
+  | "files"
+  | "reasoning";
+```
+
+## Development
+
+```bash
+bun install
+bun run typecheck
+bun test
+```
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@dex-ai/context",
+  "version": "0.7.16",
+  "description": "Index-and-pointer context management — indexes all tool outputs into FTS5 knowledge base for zero-loss compression. Provides ctx_search for on-demand retrieval. Zero config.",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "bun test",
+    "changeset": "changeset",
+    "version": "changeset version",
+    "release": "changeset publish"
+  },
+  "dependencies": {
+    "@dex-ai/sdk": "^0.1.22",
+    "zod": "^3.23.0"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.6.3",
+    "@changesets/cli": "^2.29.0"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  }
+}

package/src/event-log.ts

@@ -0,0 +1,246 @@
+/**
+ * Event Log — passive extraction of structured events from tool results.
+ *
+ * This module observes tool results without modifying them. It extracts
+ * structured session events (file ops, commands, errors) that are used
+ * to build resume snapshots when context pressure requires compression.
+ *
+ * Zero modification. Zero truncation. Pure observation.
+ */
+
+import { estimateTokens } from "./tokenizer";
+
+/* ── Types ─────────────────────────────────────────────── */
+
+export type EventCategory =
+	| "file"
+	| "command"
+	| "error"
+	| "search"
+	| "decision";
+
+export interface SessionEvent {
+	type: string;
+	category: EventCategory;
+	/** Compact data payload — file path, command, error message */
+	data: string;
+	/** Unix ms timestamp */
+	timestamp: number;
+}
+
+/* ── Event Log ─────────────────────────────────────────── */
+
+export class EventLog {
+	private events: SessionEvent[] = [];
+
+	append(event: SessionEvent): void {
+		this.events.push(event);
+	}
+
+	appendAll(events: SessionEvent[]): void {
+		this.events.push(...events);
+	}
+
+	getAll(): ReadonlyArray<SessionEvent> {
+		return this.events;
+	}
+
+	getByCategory(category: EventCategory): SessionEvent[] {
+		return this.events.filter((e) => e.category === category);
+	}
+
+	get length(): number {
+		return this.events.length;
+	}
+
+	clear(): void {
+		this.events = [];
+	}
+}
+
+/* ── Event Extraction ──────────────────────────────────── */
+
+export interface ToolResultInput {
+	toolName: string;
+	input: Record<string, unknown> | undefined;
+	outputText: string | null;
+	isError: boolean;
+}
+
+/**
+ * Extract structured events from a tool result.
+ * Returns 0 or more events depending on the tool type.
+ */
+export function extractEvents(result: ToolResultInput): SessionEvent[] {
+	const now = Date.now();
+	const events: SessionEvent[] = [];
+
+	switch (result.toolName) {
+		case "read": {
+			const path = str(result.input?.path);
+			if (path) {
+				events.push({
+					type: "file_read",
+					category: "file",
+					data: path,
+					timestamp: now,
+				});
+			}
+			break;
+		}
+
+		case "write": {
+			const path = str(result.input?.path);
+			if (path) {
+				events.push({
+					type: "file_write",
+					category: "file",
+					data: path,
+					timestamp: now,
+				});
+			}
+			break;
+		}
+
+		case "edit": {
+			const path = str(result.input?.path);
+			if (path) {
+				events.push({
+					type: "file_edit",
+					category: "file",
+					data: path,
+					timestamp: now,
+				});
+			}
+			break;
+		}
+
+		case "bash": {
+			const command = str(result.input?.command);
+			if (command) {
+				const summary = summarizeBashCommand(command, result.outputText);
+				events.push({
+					type: "bash_command",
+					category: "command",
+					data: summary,
+					timestamp: now,
+				});
+
+				// Also extract errors
+				if (result.isError || detectBashError(result.outputText)) {
+					const errorLine = extractErrorLine(result.outputText);
+					if (errorLine) {
+						events.push({
+							type: "error",
+							category: "error",
+							data: `${truncStr(command, 50)}: ${errorLine}`,
+							timestamp: now,
+						});
+					}
+				}
+			}
+			break;
+		}
+
+		case "search": {
+			const pattern = str(result.input?.pattern);
+			const mode = str(result.input?.mode) || "grep";
+			if (pattern) {
+				events.push({
+					type: "search",
+					category: "search",
+					data: `${mode} "${truncStr(pattern, 40)}"`,
+					timestamp: now,
+				});
+			}
+			break;
+		}
+
+		case "lsp_navigation": {
+			const op = str(result.input?.operation);
+			const file = str(result.input?.filePath);
+			if (op && file) {
+				events.push({
+					type: "lsp",
+					category: "search",
+					data: `lsp:${op} ${basename(file)}`,
+					timestamp: now,
+				});
+			}
+			break;
+		}
+	}
+
+	return events;
+}
+
+/* ── Helpers ───────────────────────────────────────────── */
+
+function str(v: unknown): string {
+	return typeof v === "string" ? v : "";
+}
+
+function truncStr(s: string, max: number): string {
+	const cleaned = s.replace(/\s+/g, " ").trim();
+	return cleaned.length > max ? cleaned.slice(0, max - 1) + "…" : cleaned;
+}
+
+function basename(path: string): string {
+	const parts = path.split("/");
+	return parts[parts.length - 1] || path;
+}
+
+function summarizeBashCommand(command: string, output: string | null): string {
+	const cmd = truncStr(command, 60);
+
+	if (!output) return cmd;
+
+	// Try to extract test results
+	const testMatch = output.match(/(\d+)\s*pass(?:ed|ing)?/i);
+	const failMatch = output.match(/(\d+)\s*fail(?:ed|ing|ure)?/i);
+	if (testMatch || failMatch) {
+		const parts: string[] = [];
+		if (testMatch) parts.push(`${testMatch[1]} pass`);
+		if (failMatch) parts.push(`${failMatch[1]} fail`);
+		return `${cmd} → ${parts.join(", ")}`;
+	}
+
+	// Check for publish/deploy output
+	const publishMatch = output.match(/\+\s*(@[\w/-]+@[\d.]+)/);
+	if (publishMatch) return `${cmd} → ${publishMatch[1]}`;
+
+	// Exit code
+	const exitMatch = output.match(/\[exit code: (\d+)\]/);
+	if (exitMatch && exitMatch[1] !== "0") {
+		return `${cmd} → exit=${exitMatch[1]}`;
+	}
+
+	// Short output — include directly
+	const lines = output.trim().split("\n");
+	if (lines.length <= 2 && output.trim().length < 80) {
+		return `${cmd} → ${output.trim()}`;
+	}
+
+	return `${cmd} (${lines.length} lines)`;
+}
+
+function detectBashError(output: string | null): boolean {
+	if (!output) return false;
+	// Non-zero exit code is a definitive error signal
+	if (/\[exit code: [1-9]\]/.test(output)) return true;
+	return false;
+}
+
+function extractErrorLine(output: string | null): string | null {
+	if (!output) return null;
+	const lines = output.split("\n");
+	// Search from end for error patterns
+	for (let i = lines.length - 1; i >= Math.max(0, lines.length - 20); i--) {
+		const line = lines[i]!.trim();
+		if (!line) continue;
+		if (/error|Error|ERR!|FAIL|panic|exception/i.test(line)) {
+			return truncStr(line, 100);
+		}
+	}
+	return null;
+}