@ricky-stevens/context-guardian 2.1.0 → 2.2.0

package/lib/estimate.mjs

@@ -1,254 +0,0 @@
-/**
- * Fast compaction savings estimator.
- *
- * Performs a single-pass byte categorisation of the transcript to estimate
- * how much context each compaction mode would save — WITHOUT running
- * the full extraction pipeline. Used by the statusline state and /cg:stats
- * to show estimated post-compaction percentages.
- *
- * @module estimate
- */
-
-import fs from "node:fs";
-import { estimateOverhead } from "./tokens.mjs";
-import { readTranscriptLines } from "./transcript.mjs";
-
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-
-/**
- * Estimated ratio of a tool_use summary vs the original input size.
- * Edit/Write keep ~80% (diffs preserved), Read/Grep/Glob keep ~5% (note only),
- * average across all tool types is roughly 15%.
- */
-const TOOL_USE_SUMMARY_RATIO = 0.15;
-
-// ---------------------------------------------------------------------------
-// Main estimator
-// ---------------------------------------------------------------------------
-
-/**
- * Estimate post-compaction token percentages for Smart Compact and Keep Recent.
- *
- * Does a fast single-pass categorisation of transcript bytes into
- * "keep" vs "remove" buckets, then projects the savings onto the
- * real token count.
- *
- * @param {string} transcriptPath - Path to the JSONL transcript
- * @param {number} currentTokens - Current real token count from API
- * @param {number} maxTokens - Model's max token limit
- * @returns {{ smartPct: number, recentPct: number }} Estimated post-compaction percentages (0-100)
- */
-export function estimateSavings(
-	transcriptPath,
-	currentTokens,
-	maxTokens,
-	baselineOverhead = 0,
-) {
-	if (!transcriptPath || !Number.isFinite(maxTokens) || maxTokens <= 0) {
-		return { smartPct: 0, recentPct: 0 };
-	}
-	try {
-		fs.statSync(transcriptPath);
-	} catch {
-		return { smartPct: 0, recentPct: 0 };
-	}
-
-	try {
-		const lines = readTranscriptLines(transcriptPath);
-		const scan = categoriseBytes(lines);
-
-		// Smart Compact: remove noise, keep content
-		const totalBytes = scan.keepBytes + scan.removeBytes;
-		if (totalBytes === 0) return { smartPct: 0, recentPct: 0 };
-
-		// Per-session overhead (system prompt, tools, memory, skills).
-		// Uses measured baseline from first response when available.
-		const overhead = estimateOverhead(
-			currentTokens,
-			transcriptPath,
-			baselineOverhead,
-		);
-		const conversationTokens = Math.max(0, currentTokens - overhead);
-
-		const smartKeepRatio = scan.keepBytes / totalBytes;
-		const smartTokens =
-			Math.round(conversationTokens * smartKeepRatio) + overhead;
-		const smartPct = Number(((smartTokens / maxTokens) * 100).toFixed(1));
-
-		// Keep Recent: last 10 user exchanges, with same noise removal
-		const recentRatio =
-			scan.userExchanges > 0 ? Math.min(1, 10 / scan.userExchanges) : 1;
-		const recentTokens =
-			Math.round(conversationTokens * smartKeepRatio * recentRatio) + overhead;
-		const recentPct = Number(((recentTokens / maxTokens) * 100).toFixed(1));
-
-		return { smartPct, recentPct };
-	} catch {
-		return { smartPct: 0, recentPct: 0 };
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Byte categorisation — single-pass scan
-// ---------------------------------------------------------------------------
-
-/**
- * Categorise transcript content bytes into "keep" vs "remove" buckets
- * and count user exchanges.
- *
- * @param {string[]} lines - JSONL transcript lines
- * @returns {{ keepBytes: number, removeBytes: number, userExchanges: number }}
- */
-function categoriseBytes(lines) {
-	let keepBytes = 0;
-	let removeBytes = 0;
-	let userExchanges = 0;
-
-	for (const line of lines) {
-		let obj;
-		try {
-			obj = JSON.parse(line);
-		} catch {
-			continue;
-		}
-
-		// System / progress — entirely removed
-		if (obj.type === "system" || obj.type === "progress") {
-			removeBytes += Buffer.byteLength(line, "utf8");
-			continue;
-		}
-
-		if (obj.type === "assistant" && obj.message?.content) {
-			const result = categoriseAssistantContent(obj.message.content);
-			keepBytes += result.keep;
-			removeBytes += result.remove;
-			continue;
-		}
-
-		if (obj.type === "user" && obj.message?.content) {
-			const result = categoriseUserContent(obj.message.content);
-			keepBytes += result.keep;
-			removeBytes += result.remove;
-			if (result.hasUserText) userExchanges++;
-		}
-	}
-
-	return { keepBytes, removeBytes, userExchanges };
-}
-
-// ---------------------------------------------------------------------------
-// Content block categorisation helpers
-// ---------------------------------------------------------------------------
-
-/**
- * Categorise bytes from a single assistant content block.
- * @param {object} block - A content block from an assistant message
- * @returns {{ keep: number, remove: number }}
- */
-function categoriseAssistantBlock(block) {
-	if (block.type === "text" && block.text) {
-		return { keep: Buffer.byteLength(block.text, "utf8"), remove: 0 };
-	}
-	if (block.type === "tool_use" && block.input) {
-		const inputBytes = Buffer.byteLength(JSON.stringify(block.input), "utf8");
-		return {
-			keep: Math.round(inputBytes * TOOL_USE_SUMMARY_RATIO),
-			remove: Math.round(inputBytes * (1 - TOOL_USE_SUMMARY_RATIO)),
-		};
-	}
-	if (block.type === "thinking" || block.type === "redacted_thinking") {
-		const thinkBytes = block.thinking
-			? Buffer.byteLength(block.thinking, "utf8")
-			: 100;
-		return { keep: 0, remove: thinkBytes };
-	}
-	return { keep: 0, remove: 0 };
-}
-
-/**
- * Categorise bytes from assistant message content (array or string).
- * @param {Array|string} content - The message.content field
- * @returns {{ keep: number, remove: number }}
- */
-function categoriseAssistantContent(content) {
-	if (typeof content === "string") {
-		return { keep: Buffer.byteLength(content, "utf8"), remove: 0 };
-	}
-	if (!Array.isArray(content)) {
-		return { keep: 0, remove: 0 };
-	}
-
-	let keep = 0;
-	let remove = 0;
-	for (const block of content) {
-		const result = categoriseAssistantBlock(block);
-		keep += result.keep;
-		remove += result.remove;
-	}
-	return { keep, remove };
-}
-
-/**
- * Categorise bytes from a single user content block.
- * @param {object} block - A content block from a user message
- * @returns {{ keep: number, remove: number, isText: boolean }}
- */
-function categoriseUserBlock(block) {
-	if (block.type === "text" && block.text) {
-		return {
-			keep: Buffer.byteLength(block.text, "utf8"),
-			remove: 0,
-			isText: true,
-		};
-	}
-	if (block.type === "tool_result") {
-		const resultContent = block.content;
-		let resultBytes;
-		if (resultContent) {
-			const raw =
-				typeof resultContent === "string"
-					? resultContent
-					: JSON.stringify(resultContent);
-			resultBytes = Buffer.byteLength(raw, "utf8");
-		} else {
-			resultBytes = 0;
-		}
-		return {
-			keep: Math.round(resultBytes * 0.1),
-			remove: Math.round(resultBytes * 0.9),
-			isText: false,
-		};
-	}
-	return { keep: 0, remove: 0, isText: false };
-}
-
-/**
- * Categorise bytes from user message content (array or string).
- * @param {Array|string} content - The message.content field
- * @returns {{ keep: number, remove: number, hasUserText: boolean }}
- */
-function categoriseUserContent(content) {
-	if (typeof content === "string") {
-		return {
-			keep: Buffer.byteLength(content, "utf8"),
-			remove: 0,
-			hasUserText: true,
-		};
-	}
-	if (!Array.isArray(content)) {
-		return { keep: 0, remove: 0, hasUserText: false };
-	}
-
-	let keep = 0;
-	let remove = 0;
-	let hasUserText = false;
-	for (const block of content) {
-		const result = categoriseUserBlock(block);
-		keep += result.keep;
-		remove += result.remove;
-		if (result.isText) hasUserText = true;
-	}
-	return { keep, remove, hasUserText };
-}

package/test/estimate.test.mjs

@@ -1,262 +0,0 @@
-import assert from "node:assert/strict";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import { estimateSavings } from "../lib/estimate.mjs";
-
-let tmpDir;
-
-beforeEach(() => {
-	tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "cg-estimate-test-"));
-});
-
-afterEach(() => {
-	fs.rmSync(tmpDir, { recursive: true, force: true });
-});
-
-/** Write a JSONL transcript file from an array of objects. */
-function writeTranscript(name, lines) {
-	const p = path.join(tmpDir, name);
-	fs.writeFileSync(p, `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`);
-	return p;
-}
-
-// ---------------------------------------------------------------------------
-// Helpers for building transcript lines
-// ---------------------------------------------------------------------------
-
-function userText(text) {
-	return { type: "user", message: { role: "user", content: text } };
-}
-
-function userBlocks(blocks) {
-	return { type: "user", message: { role: "user", content: blocks } };
-}
-
-function assistantText(text) {
-	return {
-		type: "assistant",
-		message: {
-			role: "assistant",
-			content: [{ type: "text", text }],
-		},
-	};
-}
-
-function systemMsg(text) {
-	return { type: "system", message: { content: text } };
-}
-
-function progressMsg(text) {
-	return { type: "progress", message: { content: text } };
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("estimateSavings", () => {
-	it("returns zeros when transcriptPath is null", () => {
-		const result = estimateSavings(null, 50000, 200000);
-		assert.deepStrictEqual(result, { smartPct: 0, recentPct: 0 });
-	});
-
-	it("returns zeros when transcriptPath does not exist", () => {
-		const result = estimateSavings("/nonexistent/path.jsonl", 50000, 200000);
-		assert.deepStrictEqual(result, { smartPct: 0, recentPct: 0 });
-	});
-
-	it("returns zeros when transcript is empty", () => {
-		const p = path.join(tmpDir, "empty.jsonl");
-		fs.writeFileSync(p, "");
-		const result = estimateSavings(p, 50000, 200000);
-		assert.deepStrictEqual(result, { smartPct: 0, recentPct: 0 });
-	});
-
-	it("system/progress messages are all removable — estimates lower than raw percentage", () => {
-		const p = writeTranscript("sys-progress.jsonl", [
-			systemMsg("System prompt content here with some length to it"),
-			progressMsg("Loading tools..."),
-			systemMsg("Another system message with extra content padding"),
-			progressMsg("Still loading more tools and resources..."),
-		]);
-
-		const rawPct = (50000 / 200000) * 100; // 25%
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct <= rawPct);
-		assert.ok(result.recentPct <= rawPct);
-		assert.equal(result.smartPct, result.recentPct);
-	});
-
-	it("user text + assistant text — both kept, estimates reflect kept content", () => {
-		const p = writeTranscript("text-only.jsonl", [
-			userText("Please help me refactor the auth module"),
-			assistantText(
-				"I will refactor the auth module by extracting the token validation into a separate function.",
-			),
-		]);
-
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct > 10);
-		assert.ok(result.recentPct > 0);
-	});
-
-	it("large tool_result content is mostly removed — low estimate", () => {
-		const bigContent = "x".repeat(50000);
-		const p = writeTranscript("tool-result.jsonl", [
-			userText("Read the config file"),
-			assistantText("I will read the config file for you."),
-			userBlocks([
-				{ type: "text", text: "Here is the result" },
-				{
-					type: "tool_result",
-					tool_use_id: "toolu_123",
-					content: bigContent,
-				},
-			]),
-		]);
-
-		const rawPct = (80000 / 200000) * 100; // 40%
-		const result = estimateSavings(p, 80000, 200000);
-		assert.ok(result.smartPct < rawPct);
-	});
-
-	it("baselineOverhead parameter affects the result", () => {
-		const bigContent = "y".repeat(20000);
-		const p = writeTranscript("overhead.jsonl", [
-			userText("Analyse the file"),
-			assistantText("I will read and analyse the file."),
-			userBlocks([
-				{ type: "text", text: "Result" },
-				{
-					type: "tool_result",
-					tool_use_id: "toolu_oh1",
-					content: bigContent,
-				},
-			]),
-			assistantText("The file looks good."),
-		]);
-
-		const withoutOverhead = estimateSavings(p, 80000, 200000, 0);
-		const withOverhead = estimateSavings(p, 80000, 200000, 40000);
-		assert.notEqual(withoutOverhead.smartPct, withOverhead.smartPct);
-	});
-
-	it("returns zeros when maxTokens is zero or NaN", () => {
-		const p = writeTranscript("zero-max.jsonl", [
-			userText("hello"),
-			assistantText("hi"),
-		]);
-		assert.deepStrictEqual(estimateSavings(p, 1000, 0), {
-			smartPct: 0,
-			recentPct: 0,
-		});
-		assert.deepStrictEqual(estimateSavings(p, 1000, NaN), {
-			smartPct: 0,
-			recentPct: 0,
-		});
-		assert.deepStrictEqual(estimateSavings(p, 1000, -100), {
-			smartPct: 0,
-			recentPct: 0,
-		});
-	});
-
-	it("thinking blocks are categorised as removable", () => {
-		const p = writeTranscript("thinking.jsonl", [
-			userText("solve this problem"),
-			{
-				type: "assistant",
-				message: {
-					role: "assistant",
-					content: [
-						{
-							type: "thinking",
-							thinking: "Let me think about this carefully... ".repeat(100),
-						},
-						{ type: "text", text: "The answer is 42." },
-					],
-				},
-			},
-		]);
-
-		const rawPct = (50000 / 200000) * 100;
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct < rawPct);
-	});
-
-	it("tool_use blocks are partially kept (summary ratio)", () => {
-		const p = writeTranscript("tool-use.jsonl", [
-			userText("read the file"),
-			{
-				type: "assistant",
-				message: {
-					role: "assistant",
-					content: [
-						{
-							type: "tool_use",
-							id: "t1",
-							name: "Read",
-							input: { file_path: "/some/very/long/path/to/file.js" },
-						},
-						{ type: "text", text: "Here is the file content." },
-					],
-				},
-			},
-		]);
-
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct > 0);
-	});
-
-	it("assistant string content (non-array) is categorised as kept", () => {
-		const p = writeTranscript("string-content.jsonl", [
-			userText("hello"),
-			{
-				type: "assistant",
-				message: { role: "assistant", content: "Simple string response" },
-			},
-		]);
-
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct > 0);
-	});
-
-	it("redacted_thinking blocks are categorised as removable", () => {
-		const p = writeTranscript("redacted.jsonl", [
-			userText("think about this"),
-			{
-				type: "assistant",
-				message: {
-					role: "assistant",
-					content: [
-						{ type: "redacted_thinking" },
-						{ type: "text", text: "Done thinking." },
-					],
-				},
-			},
-		]);
-
-		const result = estimateSavings(p, 50000, 200000);
-		assert.ok(result.smartPct > 0);
-	});
-
-	it("userExchanges counting — 20 exchanges makes recentPct less than smartPct", () => {
-		const padding = "A".repeat(5000);
-		const lines = [];
-		for (let i = 0; i < 20; i++) {
-			lines.push(userText(`User message ${i + 1}: ${padding}`));
-			lines.push(assistantText(`Response ${i + 1}: ${padding}`));
-		}
-		const p = writeTranscript("twenty-exchanges.jsonl", lines);
-
-		const result = estimateSavings(p, 60000, 200000);
-		assert.ok(result.smartPct > 0);
-		assert.ok(result.recentPct > 0);
-		assert.ok(result.recentPct < result.smartPct);
-
-		const ratio = result.recentPct / result.smartPct;
-		assert.ok(ratio < 0.85);
-		assert.ok(ratio > 0.4);
-	});
-});