@iowarp/clio-coder 0.3.0 → 0.3.1

package/src/engine/strip-tokenizer-sentinels.ts

@@ -0,0 +1,125 @@
+/**
+ * Strips tokenizer special-token sentinels from assistant-generated text.
+ *
+ * Local inference servers (LM Studio, llama.cpp, Ollama) and routed providers
+ * sometimes leak the underlying tokenizer's end-of-text or chat-template
+ * sentinels into the streamed response. The leak happens when the model emits
+ * the sentinel as a regular generated token because the chat template is
+ * misconfigured or the server does not strip special tokens before returning
+ * them. Once leaked, strings like `<|endoftext|>` render verbatim in the TUI
+ * and persist into turn history, where they confuse the next turn.
+ *
+ * This helper is conservative: it strips only the exact union of known
+ * tokenizer sentinels listed in {@link KNOWN_SENTINELS}. Lookalike strings
+ * such as `<|something_random|>` are preserved so that a user pasting
+ * tokenizer documentation into Clio sees their text untouched. The helper
+ * is designed for assistant-generated text deltas only; never call it on
+ * user-typed input or tool result content.
+ *
+ * The sentinel union covers Qwen (`<|endoftext|>`, `<|im_start|>`,
+ * `<|im_end|>`, `<|fim_*|>`), Llama 3 (`<|eot_id|>`), Gemma (`<|end_of_turn|>`),
+ * older sentence-piece models (`<s>`, `</s>`), and CodeGemma's
+ * `<|file_separator|>`. Add new entries to {@link KNOWN_SENTINELS} when a
+ * model family with a different chat template starts leaking.
+ */
+
+const KNOWN_SENTINELS: ReadonlyArray<string> = [
+	"<|endoftext|>",
+	"<|im_start|>",
+	"<|im_end|>",
+	"<|eot_id|>",
+	"<|end_of_turn|>",
+	"<|fim_prefix|>",
+	"<|fim_middle|>",
+	"<|fim_suffix|>",
+	"<|file_separator|>",
+	"<s>",
+	"</s>",
+];
+
+/**
+ * Single compiled RegExp matching every known sentinel. Sourced from
+ * {@link KNOWN_SENTINELS} so the list stays the single source of truth.
+ */
+const SENTINEL_RE = new RegExp(KNOWN_SENTINELS.map((s) => s.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&")).join("|"), "g");
+
+/**
+ * Length of the longest known sentinel. Used by the streaming stripper to
+ * decide how many trailing characters of a chunk to buffer when those
+ * characters could be the prefix of a sentinel that completes in the next
+ * chunk.
+ */
+const MAX_SENTINEL_LEN = KNOWN_SENTINELS.reduce((max, s) => (s.length > max ? s.length : max), 0);
+
+/**
+ * Strip every occurrence of a known sentinel from `text`. Use this on
+ * non-streaming assistant text or to scrub a complete buffer in one shot.
+ */
+export function stripTokenizerSentinels(text: string): string {
+	if (text.length === 0) return text;
+	return text.replace(SENTINEL_RE, "");
+}
+
+/**
+ * Returns true when `tail` is a non-empty prefix of any known sentinel.
+ * The streaming stripper uses this to decide whether to hold a trailing
+ * fragment back until the next chunk arrives.
+ */
+function isSentinelPrefix(tail: string): boolean {
+	if (tail.length === 0) return false;
+	for (const sentinel of KNOWN_SENTINELS) {
+		if (sentinel.length > tail.length && sentinel.startsWith(tail)) return true;
+	}
+	return false;
+}
+
+/**
+ * Stateful stripper for streaming text deltas. A sentinel may straddle a
+ * chunk boundary (e.g. `<|endoftex` arrives in one delta and `t|>` in the
+ * next), so the stripper keeps a small trailing buffer when the tail of the
+ * incoming chunk could be the start of a sentinel and only emits that tail
+ * once the next chunk confirms or denies the match. Call {@link flush} when
+ * the upstream stream ends to release any buffered remainder; anything left
+ * at that point cannot be a sentinel and is emitted verbatim minus any
+ * complete sentinels still embedded in it.
+ */
+export interface SentinelStripper {
+	/** Sanitize the next streamed chunk, returning the safe-to-emit prefix. */
+	push(chunk: string): string;
+	/** Drain the trailing buffer at end-of-stream. */
+	flush(): string;
+}
+
+export function createSentinelStripper(): SentinelStripper {
+	let buffer = "";
+	return {
+		push(chunk: string): string {
+			if (chunk.length === 0) return "";
+			buffer += chunk;
+			// Strip every complete sentinel currently in the buffer.
+			const stripped = buffer.replace(SENTINEL_RE, "");
+			// Hold back at most MAX_SENTINEL_LEN - 1 trailing characters when
+			// they form a non-empty prefix of any known sentinel. Anything
+			// shorter than that cannot complete a sentinel without further
+			// input.
+			const holdMax = Math.min(stripped.length, MAX_SENTINEL_LEN - 1);
+			let hold = 0;
+			for (let take = holdMax; take > 0; take--) {
+				const tail = stripped.slice(stripped.length - take);
+				if (isSentinelPrefix(tail)) {
+					hold = take;
+					break;
+				}
+			}
+			const safe = hold === 0 ? stripped : stripped.slice(0, stripped.length - hold);
+			buffer = hold === 0 ? "" : stripped.slice(stripped.length - hold);
+			return safe;
+		},
+		flush(): string {
+			if (buffer.length === 0) return "";
+			const remainder = buffer.replace(SENTINEL_RE, "");
+			buffer = "";
+			return remainder;
+		},
+	};
+}

package/src/engine/tui.ts

@@ -0,0 +1,103 @@
+/**
+ * Re-export the Clio terminal-engine primitives the interactive layer consumes. Adding a
+ * new terminal-engine symbol to Clio happens here first, then the consuming file in
+ * src/interactive/ imports it from this module.
+ */
+
+export type {
+	AutocompleteItem,
+	AutocompleteProvider,
+	AutocompleteSuggestions,
+	Component,
+	EditorOptions,
+	EditorTheme,
+	ImageTheme,
+	Keybinding,
+	KeybindingConflict,
+	KeybindingDefinitions,
+	Keybindings,
+	KeybindingsConfig,
+	KeyId,
+	MarkdownTheme,
+	OverlayHandle,
+	OverlayOptions,
+	SelectItem,
+	SelectListLayoutOptions,
+	SettingItem,
+	SettingsListTheme,
+	SlashCommand,
+	Terminal,
+} from "@earendil-works/pi-tui";
+
+/**
+ * Structural projection of the terminal engine covering just the progress
+ * sink. The engine-boundary rules keep terminal-engine value imports inside this
+ * module; consumers accept this narrower shape so the helper is unit-testable
+ * without a real ProcessTerminal.
+ */
+export interface AgentProgressSink {
+	setProgress(active: boolean): void;
+}
+
+/**
+ * Toggle OSC 9;4 indeterminate progress around an agent run. The Clio terminal
+ * engine emits the sequence terminals like WezTerm, Ghostty,
+ * Konsole, and Windows Terminal render as a taskbar/tab progress badge.
+ *
+ * Start/stop are idempotent: repeated calls coalesce so multiple agent_start
+ * events in a row (or a stop with no active run) never emit stray sequences.
+ */
+export function createAgentProgress(terminal: AgentProgressSink): {
+	start(): void;
+	stop(): void;
+	isActive(): boolean;
+} {
+	let active = false;
+	return {
+		start(): void {
+			if (active) return;
+			active = true;
+			terminal.setProgress(true);
+		},
+		stop(): void {
+			if (!active) return;
+			active = false;
+			terminal.setProgress(false);
+		},
+		isActive(): boolean {
+			return active;
+		},
+	};
+}
+export {
+	Box,
+	CombinedAutocompleteProvider,
+	Container,
+	Editor,
+	fuzzyFilter,
+	fuzzyMatch,
+	getCapabilities,
+	getKeybindings,
+	Image,
+	Input,
+	isKeyRelease,
+	KeybindingsManager,
+	Loader,
+	Markdown,
+	matchesKey,
+	ProcessTerminal,
+	setKeybindings,
+	Text,
+	TUI,
+	TUI_KEYBINDINGS,
+	truncateToWidth,
+	visibleWidth,
+	wrapTextWithAnsi,
+} from "@earendil-works/pi-tui";
+export type { SelectListTheme } from "./select-list.js";
+/**
+ * SelectList ships from the engine as a thin subclass that honors a design
+ * cursor exposed through SelectListTheme, so raw pickers no longer inherit
+ * pi-tui's hardcoded arrow. See ./select-list.ts.
+ */
+export { SelectList } from "./select-list.js";

package/src/engine/types.ts

@@ -0,0 +1,66 @@
+/**
+ * Re-exports of Clio engine-boundary types consumed by the rest of the codebase.
+ *
+ * Importing engine package types from anywhere else in the codebase violates the engine boundary.
+ * Add new re-exports here when domains need additional engine types, and update the audit
+ * document in the same commit.
+ */
+
+export type {
+	AgentEvent,
+	AgentMessage,
+	AgentOptions,
+	AgentState,
+	AgentTool,
+	AgentToolResult,
+	StreamFn,
+} from "@earendil-works/pi-agent-core";
+export { Agent } from "@earendil-works/pi-agent-core";
+
+import type { AgentState as PiAgentState } from "@earendil-works/pi-agent-core";
+
+/**
+ * Writable view onto the engine AgentState. The public typings mark
+ * `errorMessage` as readonly because the agent itself is the canonical
+ * mutator, but the chat-loop legitimately needs to clear it before retrying
+ * after a recoverable failure (overflow recovery, transient retry chains).
+ * Pi's runtime assigns to `_state.errorMessage = undefined` in the same
+ * shape, so the field is writable in practice. Exposing the writable view
+ * here keeps the engine boundary clean and lets call sites avoid `as unknown`
+ * casts on the private surface.
+ */
+export type MutableAgentState = Omit<PiAgentState, "errorMessage"> & {
+	errorMessage?: string | undefined;
+};
+
+export type {
+	AnthropicMessagesCompat,
+	Api,
+	ImageContent,
+	KnownProvider,
+	Model,
+	OpenAICompletionsCompat,
+	Usage,
+} from "@earendil-works/pi-ai";
+
+import type { Api as PiApi, Model as PiModel } from "@earendil-works/pi-ai";
+
+/**
+ * The one Clio-facing model shape. Runtime descriptors synthesize
+ * `Model<Api>` values and every consumer (chat loop, orchestrator, worker
+ * runtime, payload patchers) holds this erased view; `Model<never>` and the
+ * `as unknown as` casts it forced are banned.
+ */
+export type EngineModel = PiModel<PiApi>;
+export type {
+	EditorOptions,
+	EditorTheme,
+	SelectItem,
+	SelectListLayoutOptions,
+	SettingItem,
+	SettingsListTheme,
+} from "@earendil-works/pi-tui";
+export { TUI } from "@earendil-works/pi-tui";
+// SelectListTheme carries the Clio design cursor as an engine augmentation; see
+// ./select-list.ts. Re-export the augmented type so every engine surface agrees.
+export type { SelectListTheme } from "./select-list.js";

package/src/engine/worker-events.ts

@@ -0,0 +1,100 @@
+/**
+ * Clio-specific events that ride the same NDJSON IPC channel as
+ * pi-agent-core's `AgentEvent`. The worker subprocess emits these alongside
+ * AgentEvents; the dispatch parent decodes the union and aggregates Clio
+ * events without disturbing pi-agent-core consumers.
+ */
+
+import type { RunOutcomeCode } from "../domains/dispatch/types.js";
+import type { ToolFinishEvent, ToolStartEvent } from "../tools/agent-tools.js";
+
+export interface ClioToolStartEvent {
+	type: "clio_tool_start";
+	payload: ToolStartEvent;
+}
+
+export interface ClioToolFinishEvent {
+	type: "clio_tool_finish";
+	payload: ToolFinishEvent;
+}
+
+/** Emitted when the worker's non-stall policy resolves a permission-requiring tool call. */
+export interface ClioPermissionResolvedEvent {
+	type: "clio_permission_resolved";
+	payload: {
+		tool: string;
+		actionClass: string;
+		mode: "deny" | "fail" | "escalate";
+		reason: string;
+		/**
+		 * Resolution provenance. Policy deny/fail uses "policy"; escalation
+		 * resolutions distinguish operator decisions from timeout fallbacks.
+		 * "remembered" answers an identical call with the operator's earlier
+		 * decision from this run, without a new escalation.
+		 */
+		source?: "operator" | "timeout" | "policy" | "remembered";
+		/** Approval request id this resolution answers. */
+		requestId?: string;
+		/** Resolved outcome for an escalation; escalate path only. */
+		decision?: "approved" | "denied";
+	};
+}
+
+/**
+ * Emitted when an escalate-posture worker parks a permission-requiring tool
+ * call and hands the decision up to the operator. The orchestrator republishes
+ * it as a PermissionRequested bus event; an operator permission_decision line
+ * on stdin (or the timeout fallback) resolves it.
+ */
+export interface ClioPermissionEscalatedEvent {
+	type: "clio_permission_escalated";
+	payload: {
+		requestId: string;
+		tool: string;
+		summary: string;
+		/**
+		 * Sanitized one-line preview of the call's object (command, path, or
+		 * compact args), so the operator's approval overlay can show what the
+		 * call will touch. The args themselves never cross the stdout seam.
+		 */
+		target?: string;
+		axis?: string;
+		decision: {
+			actionClass: string;
+			reasons: ReadonlyArray<string>;
+			reasonCode?: string;
+			ruleId?: string;
+			policySource?: string;
+		};
+		timeoutMs: number;
+	};
+}
+
+/**
+ * Emitted when the worker accepts an operator steer line from stdin and
+ * queues it on the agent's steering queue. The steer text itself reaches the
+ * transcript as a normal user message at the next loop boundary; this event
+ * is the delivery ack for operator surfaces.
+ */
+export interface ClioSteerReceivedEvent {
+	type: "clio_steer_received";
+	payload: {
+		chars: number;
+		/** Exact parent-side steering provenance entry this acceptance closes. */
+		sequence: number;
+	};
+}
+
+/** Machine-readable terminal classification, emitted where the condition is known. */
+export interface ClioRunOutcomeEvent {
+	type: "clio_run_outcome";
+	payload: { outcomeCode: RunOutcomeCode };
+}
+
+export type ClioWorkerEvent =
+	| ClioToolStartEvent
+	| ClioToolFinishEvent
+	| ClioPermissionResolvedEvent
+	| ClioPermissionEscalatedEvent
+	| ClioSteerReceivedEvent
+	| ClioRunOutcomeEvent;