@xynogen/pix-optimizer 1.0.0

package/src/json.ts

@@ -0,0 +1,270 @@
+/**
+ * json.ts — JSON token-optimization via jq + TOON.
+ *
+ * Two parts, mirroring rtk.ts/caveman.ts:
+ *   1. A small system-prompt nudge teaching the model to run JSON through
+ *      `jq` (query/reshape) and `toon` (compress for context), and to convert
+ *      back to JSON only when a strict contract requires it.
+ *   2. A bundled `toon-json` skill, surfaced via `resources_discover`, that
+ *      holds the full workflow + when-NOT-to-use guidance.
+ *
+ * TOON = Token-Oriented Object Notation (https://github.com/toon-format/spec).
+ * It shines on uniform/tabular arrays of objects (declare keys once, stream
+ * rows) and loses to compact JSON on deeply nested / non-uniform / array-of-
+ * arrays data. The prompt encodes that boundary so the model picks correctly.
+ *
+ * Pure helpers are exported for tests; json(pi) is the extension entry, called
+ * by index.ts alongside caveman(pi) and rtk(pi).
+ */
+
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+	ExtensionAPI,
+	ExtensionCommandContext,
+	ExtensionContext,
+} from "@earendil-works/pi-coding-agent";
+import type { OptimizerHandle, OptimizerStatus } from "./status.ts";
+
+// ── System prompt ───────────────────────────────────────────────────────────
+
+export const JSON_SYSTEM_PROMPT = `# JSON Handling — jq + TOON
+
+When working with information-dense JSON (LLM/OpenAPI schemas, API responses,
+config dumps, datasets), prefer this pipeline over dumping raw JSON into context:
+
+\`\`\`bash
+curl -s <url> | jq '<query>' | toon          # fetch → reshape → compress
+cat data.json   | jq '.items'  | toon --stats # local file, show token savings
+echo "$TOON"    | toon -d                      # convert TOON back to JSON
+\`\`\`
+
+## Why
+- **jq** queries/reshapes so you only carry the slice you need.
+- **toon** re-encodes JSON as TOON: uniform arrays of objects declare their
+  keys once (\`key[N]{a,b,c}:\`) then stream bare rows — large token savings on
+  tabular/dense data. \`toon\` auto-detects direction; \`-d\` decodes back.
+
+## When TOON helps (use it)
+- Uniform/tabular arrays of objects (TOON's sweet spot — savings scale with rows × fields)
+- Flat objects and primitive arrays
+- Shallow nesting
+
+## When to SKIP TOON (keep JSON)
+- API-level contracts / payloads you must send or store verbatim
+- Deeply nested or non-uniform structures (compact JSON can win)
+- Arrays of arrays (TOON is less efficient here)
+- Anything a downstream parser requires as strict JSON
+
+Rule of thumb: TOON for **reading** dense data into context; JSON for **contracts**.
+See the \`toon-json\` skill for the full workflow.`;
+
+// ── Prompt relevance gate ─────────────────────────────────────────────────────
+
+/**
+ * Tokens that signal the user prompt is about JSON / the jq+TOON workflow.
+ * Matched as whole words (case-insensitive) so "adjust" / "jsx" don't trip "js".
+ */
+const JSON_TRIGGERS = [
+	"json",
+	"jsonl",
+	"ndjson",
+	"js",
+	"jq",
+	"toon",
+	"openapi",
+	"swagger",
+] as const;
+
+const JSON_TRIGGER_RE = new RegExp(
+	`\\b(${JSON_TRIGGERS.join("|")})\\b`,
+	"i",
+);
+
+/**
+ * True when the user prompt mentions JSON (or a related token), so the
+ * jq+TOON guidance is only injected when actually relevant. Case-insensitive
+ * and word-bounded ("JSON", "Json", "json" all match; "adjust" does not).
+ */
+export function mentionsJson(prompt: string | undefined | null): boolean {
+	if (!prompt) return false;
+	return JSON_TRIGGER_RE.test(prompt);
+}
+
+// ── Pure decision helper ──────────────────────────────────────────────────────
+
+export interface ToonAdvice {
+	/** Whether TOON is likely to reduce tokens for this shape. */
+	useToon: boolean;
+	/** Short human-readable reason. */
+	reason: string;
+}
+
+/**
+ * Heuristic guidance on whether a parsed JSON value is a good TOON candidate.
+ *
+ * This is intentionally a *recommendation* surfaced to the model / user, not an
+ * enforcement. It encodes the efficiency boundary from the TOON spec:
+ *   - uniform arrays of objects → strongly yes (tabular sweet spot)
+ *   - flat objects / primitive arrays → yes
+ *   - arrays of arrays → no (TOON's one losing case)
+ *   - deeply nested / non-uniform → no (compact JSON can win)
+ *
+ * @param value parsed JSON (object/array/primitive)
+ * @param maxDepth nesting depth at which we stop recommending TOON (default 4)
+ */
+export function adviseToon(value: unknown, maxDepth = 4): ToonAdvice {
+	if (Array.isArray(value)) {
+		if (value.length === 0) {
+			return { useToon: false, reason: "empty array — nothing to compress" };
+		}
+		// Array of arrays: TOON's only structurally-worse case.
+		if (value.every((v) => Array.isArray(v))) {
+			return { useToon: false, reason: "array of arrays — JSON is more compact" };
+		}
+		// Uniform array of flat objects → tabular sweet spot.
+		if (isUniformObjectArray(value)) {
+			return {
+				useToon: true,
+				reason: `uniform array of ${value.length} objects — TOON tabular sweet spot`,
+			};
+		}
+		// Array of primitives.
+		if (value.every((v) => !isObjectLike(v))) {
+			return { useToon: true, reason: "primitive array — TOON omits quotes/braces" };
+		}
+		return { useToon: false, reason: "non-uniform array — savings uncertain, keep JSON" };
+	}
+
+	if (isObjectLike(value)) {
+		const depth = objectDepth(value);
+		if (depth > maxDepth) {
+			return {
+				useToon: false,
+				reason: `nesting depth ${depth} > ${maxDepth} — compact JSON may win`,
+			};
+		}
+		return { useToon: true, reason: "shallow object — TOON drops quotes/braces" };
+	}
+
+	return { useToon: false, reason: "primitive value — nothing to compress" };
+}
+
+/** True for plain objects/arrays (things with nested structure). */
+function isObjectLike(v: unknown): v is Record<string, unknown> | unknown[] {
+	return typeof v === "object" && v !== null;
+}
+
+/**
+ * A uniform array of objects: every element is a plain (non-array) object and
+ * they all share the same set of keys, each holding a primitive value. This is
+ * exactly the shape TOON encodes as a single header + bare rows.
+ */
+export function isUniformObjectArray(arr: unknown[]): boolean {
+	if (arr.length === 0) return false;
+	const first = arr[0];
+	if (!isPlainObject(first)) return false;
+	const keys = Object.keys(first).sort();
+	if (keys.length === 0) return false;
+
+	return arr.every((el) => {
+		if (!isPlainObject(el)) return false;
+		const elKeys = Object.keys(el).sort();
+		if (elKeys.length !== keys.length) return false;
+		for (let i = 0; i < keys.length; i++) {
+			if (elKeys[i] !== keys[i]) return false;
+			// values must be primitive for a clean tabular row
+			if (isObjectLike((el as Record<string, unknown>)[keys[i]!])) return false;
+		}
+		return true;
+	});
+}
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+	return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+/** Max nesting depth of a JSON value (primitives = 0). */
+export function objectDepth(value: unknown): number {
+	if (!isObjectLike(value)) return 0;
+	const children = Array.isArray(value) ? value : Object.values(value);
+	let max = 0;
+	for (const child of children) {
+		const d = objectDepth(child);
+		if (d > max) max = d;
+	}
+	return max + 1;
+}
+
+// ── Bundled skill path ────────────────────────────────────────────────────────
+
+/**
+ * Absolute path to the bundled `toon-json` skill directory, resolved relative
+ * to this module so it works regardless of the extension's install location.
+ */
+export function skillDir(): string {
+	const here = fileURLToPath(new URL(".", import.meta.url));
+	return join(here, "skills", "toon-json");
+}
+
+// ── Pi extension ──────────────────────────────────────────────────────────────
+
+export function json(pi: ExtensionAPI, status: OptimizerStatus): OptimizerHandle {
+	let enabled = true;
+
+	// Report into the shared optimizer indicator.
+	function syncStatus(ctx: Pick<ExtensionContext, "ui">) {
+		status.set("toon", enabled, ctx);
+	}
+
+	pi.on("session_start", async (_event, ctx) => { syncStatus(ctx); });
+	pi.on("agent_start", async (_event, ctx) => { syncStatus(ctx); });
+	pi.on("agent_end", async (_event, ctx) => { syncStatus(ctx); });
+
+	// Surface the bundled skill so the model can load the full workflow.
+	pi.on("resources_discover", async () => {
+		if (!enabled) return undefined;
+		return { skillPaths: [skillDir()] };
+	});
+
+	// Inject the JSON-handling nudge into the system prompt, but ONLY when the
+	// user prompt actually mentions JSON / a related token — otherwise it's dead
+	// weight in every turn.
+	pi.on("before_agent_start", async (event) => {
+		if (!enabled) return undefined;
+		if (!mentionsJson(event.prompt)) return undefined;
+		const existing = event.systemPrompt ?? "";
+		return { systemPrompt: JSON_SYSTEM_PROMPT + "\n\n" + existing };
+	});
+
+	// -- Subcommand handler (dispatched by the merged /opt router) --
+
+	function complete(prefix: string) {
+		const items = [
+			{ value: "on", label: "on", description: "Force JSON/TOON guidance on" },
+			{ value: "off", label: "off", description: "Force JSON/TOON guidance off" },
+		];
+		const n = prefix.trim().toLowerCase();
+		const filtered = items.filter((i) => i.value.startsWith(n));
+		return filtered.length > 0 ? filtered : null;
+	}
+
+	async function run(args: string, ctx: ExtensionCommandContext): Promise<void> {
+		{
+			const arg = args.trim().toLowerCase();
+			if (arg === "on") enabled = true;
+			else if (arg === "off") enabled = false;
+			else enabled = !enabled; // bare toggles
+
+			syncStatus(ctx);
+			ctx.ui.notify(`JSON/TOON guidance ${enabled ? "on" : "off"}.`, "info");
+		}
+	}
+
+	return {
+		name: "toon",
+		help: "toon [on|off] — jq+TOON guidance for dense JSON",
+		run,
+		complete,
+	};
+}

package/src/opt.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect } from "bun:test";
+import {
+	parseInvocation,
+	completeInvocation,
+	buildOptHelp,
+} from "./opt.ts";
+import type { OptimizerHandle, OptimizerTool } from "./status.ts";
+
+/** Build a handle set with spy-able run/complete. */
+function fakeHandles(): Record<OptimizerTool, OptimizerHandle> {
+	const mk = (name: OptimizerTool): OptimizerHandle => ({
+		name,
+		help: `${name} help`,
+		run: () => {},
+		complete: (prefix: string) =>
+			["on", "off"]
+				.filter((v) => v.startsWith(prefix.trim().toLowerCase()))
+				.map((v) => ({ value: v, label: v, description: v })),
+	});
+	return { caveman: mk("caveman"), rtk: mk("rtk"), toon: mk("toon") };
+}
+
+describe("parseInvocation", () => {
+	it("empty args → empty name", () => {
+		expect(parseInvocation("")).toEqual({ name: "", rest: "" });
+		expect(parseInvocation("   ")).toEqual({ name: "", rest: "" });
+	});
+
+	it("single token → name only", () => {
+		expect(parseInvocation("rtk")).toEqual({ name: "rtk", rest: "" });
+		expect(parseInvocation("  RTK  ")).toEqual({ name: "rtk", rest: "" });
+	});
+
+	it("splits name from rest", () => {
+		expect(parseInvocation("caveman ultra")).toEqual({
+			name: "caveman",
+			rest: "ultra",
+		});
+		expect(parseInvocation("rtk on extra")).toEqual({
+			name: "rtk",
+			rest: "on extra",
+		});
+	});
+});
+
+describe("completeInvocation", () => {
+	it("completes tool names before any space", () => {
+		const handles = fakeHandles();
+		const out = completeInvocation("r", handles);
+		expect(out?.map((i) => i.value)).toEqual(["rtk"]);
+	});
+
+	it("returns all tool names for empty prefix", () => {
+		const handles = fakeHandles();
+		const out = completeInvocation("", handles);
+		expect(out?.map((i) => i.value)).toEqual(["caveman", "rtk", "toon"]);
+	});
+
+	it("delegates to the tool completer after the name", () => {
+		const handles = fakeHandles();
+		const out = completeInvocation("rtk o", handles);
+		expect(out?.map((i) => i.value)).toEqual(["on", "off"]);
+	});
+
+	it("null for unknown tool", () => {
+		const handles = fakeHandles();
+		expect(completeInvocation("nope x", handles)).toBeNull();
+	});
+});
+
+describe("buildOptHelp", () => {
+	it("lists every tool's help line", () => {
+		const help = buildOptHelp(fakeHandles());
+		expect(help).toContain("/opt caveman help");
+		expect(help).toContain("/opt rtk help");
+		expect(help).toContain("/opt toon help");
+		expect(help).toContain("Usage: /opt <tool> [args]");
+	});
+});
+
+describe("dispatch via run", () => {
+	it("routes rest to the matching handle", () => {
+		const handles = fakeHandles();
+		let received: string | null = null;
+		handles.rtk.run = (args: string) => {
+			received = args;
+		};
+		const { name, rest } = parseInvocation("rtk off");
+		expect(name).toBe("rtk");
+		void handles[name as OptimizerTool].run(rest, {} as never);
+		expect(received).toBe("off");
+	});
+});

package/src/opt.ts

@@ -0,0 +1,96 @@
+/**
+ * opt.ts — the single `/opt` command that fronts every optimizer tool.
+ *
+ * caveman / rtk / json each register their own lifecycle hooks but expose an
+ * OptimizerHandle for command dispatch. This router wires them under one
+ * command so there's exactly one entry point:
+ *
+ *   /opt                  → status + help
+ *   /opt caveman <level>  → caveman subcommand
+ *   /opt rtk [on|off]     → rtk subcommand
+ *   /opt toon [on|off]    → toon subcommand
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { CompletionItem, OptimizerHandle, OptimizerTool } from "./status.ts";
+
+/** Split raw args into the subcommand name and the rest. */
+export function parseInvocation(args: string): { name: string; rest: string } {
+	const trimmed = args.trim();
+	if (!trimmed) return { name: "", rest: "" };
+	const idx = trimmed.search(/\s/);
+	if (idx === -1) return { name: trimmed.toLowerCase(), rest: "" };
+	return {
+		name: trimmed.slice(0, idx).toLowerCase(),
+		rest: trimmed.slice(idx + 1),
+	};
+}
+
+/**
+ * Compute completions for `/opt ...`:
+ *   - first token → tool names (filtered by prefix)
+ *   - after a known tool name → delegate to that tool's complete()
+ *
+ * Pure + exported for tests.
+ */
+export function completeInvocation(
+	args: string,
+	handles: Record<OptimizerTool, OptimizerHandle>,
+): CompletionItem[] | null {
+	// No space yet → still completing the tool name.
+	if (!/\s/.test(args.trimStart()) ) {
+		const prefix = args.trim().toLowerCase();
+		const names = (Object.keys(handles) as OptimizerTool[])
+			.filter((n) => n.startsWith(prefix))
+			.map((n) => ({
+				value: n,
+				label: n,
+				description: handles[n].help,
+			}));
+		return names.length > 0 ? names : null;
+	}
+
+	// Past the tool name → delegate to the tool's own completer.
+	const { name, rest } = parseInvocation(args);
+	const handle = handles[name as OptimizerTool];
+	if (!handle) return null;
+	return handle.complete(rest);
+}
+
+/** Build the `/opt` help text listing every tool. */
+export function buildOptHelp(handles: Record<OptimizerTool, OptimizerHandle>): string {
+	const lines = (Object.keys(handles) as OptimizerTool[]).map(
+		(n) => `  /opt ${handles[n].help}`,
+	);
+	return ["pix-optimizer — token tools", "", "Usage: /opt <tool> [args]", ...lines].join("\n");
+}
+
+export function registerOptCommand(
+	pi: ExtensionAPI,
+	handles: Record<OptimizerTool, OptimizerHandle>,
+): void {
+	pi.registerCommand("opt", {
+		description: "pix-optimizer: caveman / rtk / toon token tools",
+		getArgumentCompletions: (prefix: string) => completeInvocation(prefix, handles),
+		handler: async (args, ctx) => {
+			const { name, rest } = parseInvocation(args ?? "");
+
+			// No subcommand → show help.
+			if (!name) {
+				ctx.ui.notify(buildOptHelp(handles), "info");
+				return;
+			}
+
+			const handle = handles[name as OptimizerTool];
+			if (!handle) {
+				ctx.ui.notify(
+					`Unknown tool: "${name}". Try: ${Object.keys(handles).join(", ")}`,
+					"error",
+				);
+				return;
+			}
+
+			await handle.run(rest, ctx);
+		},
+	});
+}

package/src/rtk.test.ts

@@ -0,0 +1,173 @@
+import { describe, it, expect } from "bun:test";
+import {
+	splitChain,
+	rewriteChain,
+	applyRtkRewrite,
+	type BashCallEvent,
+} from "./rtk.ts";
+
+/** Build a fresh bash tool_call event for hook tests. */
+function bashEvent(command: string): BashCallEvent {
+	return { toolName: "bash", input: { command } };
+}
+
+describe("splitChain", () => {
+	it("returns single segment for plain command", () => {
+		expect(splitChain("git status")).toEqual(["git status"]);
+	});
+
+	it("splits on && keeping operator", () => {
+		expect(splitChain("git add . && git push")).toEqual([
+			"git add . ",
+			"&&",
+			" git push",
+		]);
+	});
+
+	it("splits on ||, ;, |", () => {
+		expect(splitChain("a || b")).toEqual(["a ", "||", " b"]);
+		expect(splitChain("a ; b")).toEqual(["a ", ";", " b"]);
+		expect(splitChain("a | b")).toEqual(["a ", "|", " b"]);
+	});
+
+	it("ignores operators inside double quotes", () => {
+		expect(splitChain('git commit -m "a && b"')).toEqual([
+			'git commit -m "a && b"',
+		]);
+	});
+
+	it("ignores operators inside single quotes", () => {
+		expect(splitChain("echo 'x | y'")).toEqual(["echo 'x | y'"]);
+	});
+
+	it("returns null on unbalanced quotes", () => {
+		expect(splitChain('git commit -m "oops')).toBeNull();
+	});
+});
+
+describe("rewriteChain", () => {
+	it("prefixes a single known command", () => {
+		expect(rewriteChain("git status")).toBe("rtk git status");
+	});
+
+	it("prefixes every segment in a chain", () => {
+		expect(rewriteChain("git add . && git commit -m x && git push")).toBe(
+			"rtk git add . && rtk git commit -m x && rtk git push",
+		);
+	});
+
+	it("prefixes mixed known commands", () => {
+		expect(rewriteChain("cargo build && npm test")).toBe(
+			"rtk cargo build && rtk npm test",
+		);
+	});
+
+	it("leaves unknown commands alone", () => {
+		expect(rewriteChain("echo hi && mkdir x")).toBe("echo hi && mkdir x");
+	});
+
+	it("only prefixes known segments in a mixed chain", () => {
+		expect(rewriteChain("cd /tmp && git status")).toBe(
+			"cd /tmp && rtk git status",
+		);
+	});
+
+	it("does not double-prefix already-rtk commands", () => {
+		expect(rewriteChain("rtk git status")).toBe("rtk git status");
+		expect(rewriteChain("rtk git add . && git push")).toBe(
+			"rtk git add . && rtk git push",
+		);
+	});
+
+	it("does not touch operators inside quotes", () => {
+		expect(rewriteChain('git commit -m "a && b"')).toBe(
+			'rtk git commit -m "a && b"',
+		);
+	});
+
+	it("returns original on unbalanced quotes", () => {
+		const cmd = 'git commit -m "oops';
+		expect(rewriteChain(cmd)).toBe(cmd);
+	});
+
+	it("prefixes known commands across a pipe (ls, wc)", () => {
+		expect(rewriteChain("ls -la | wc -l")).toBe("rtk ls -la | rtk wc -l");
+	});
+
+	it("truly leaves a chain of only-unknown commands untouched", () => {
+		expect(rewriteChain("cd /tmp | sort | uniq")).toBe("cd /tmp | sort | uniq");
+	});
+
+	it("handles pipes between known commands", () => {
+		expect(rewriteChain("git log | grep fix")).toBe(
+			"rtk git log | rtk grep fix",
+		);
+	});
+});
+
+// Integration tests for the `tool_call` hook step. These guard the bug that
+// silently disabled rewriting: wrong event name + wrong field + wrong patch
+// mechanism. They assert on the IN-PLACE mutation contract the SDK requires.
+describe("applyRtkRewrite (tool_call hook step)", () => {
+	it("mutates event.input.command in place for a known bash command", () => {
+		const event = bashEvent("git status");
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(changed).toBe(true);
+		expect(event.input.command).toBe("rtk git status");
+	});
+
+	it("rewrites every segment of a chain in place", () => {
+		const event = bashEvent("git add . && git push");
+		applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(event.input.command).toBe("rtk git add . && rtk git push");
+	});
+
+	it("does not mutate when disabled", () => {
+		const event = bashEvent("git status");
+		const changed = applyRtkRewrite(event, { enabled: false, rtkAvailable: true });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe("git status");
+	});
+
+	it("does not mutate when rtk binary is unavailable", () => {
+		const event = bashEvent("git status");
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: false });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe("git status");
+	});
+
+	it("ignores non-bash tools", () => {
+		const event: BashCallEvent = { toolName: "grep", input: { command: "git status" } };
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe("git status");
+	});
+
+	it("leaves unknown commands untouched", () => {
+		const event = bashEvent("mkdir build && cd build");
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe("mkdir build && cd build");
+	});
+
+	it("does not double-prefix an already-rtk command", () => {
+		const event = bashEvent("rtk git status");
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe("rtk git status");
+	});
+
+	it("handles missing / non-string command safely", () => {
+		const event: BashCallEvent = { toolName: "bash", input: {} };
+		expect(applyRtkRewrite(event, { enabled: true, rtkAvailable: true })).toBe(false);
+		const event2: BashCallEvent = { toolName: "bash", input: { command: 123 } };
+		expect(applyRtkRewrite(event2, { enabled: true, rtkAvailable: true })).toBe(false);
+	});
+
+	it("leaves command unchanged on unbalanced quotes", () => {
+		const event = bashEvent('git commit -m "oops');
+		const changed = applyRtkRewrite(event, { enabled: true, rtkAvailable: true });
+		expect(changed).toBe(false);
+		expect(event.input.command).toBe('git commit -m "oops');
+	});
+});