@huseynsnmz/pi-suite 0.0.0-stage → 0.1.0

package/packages/frame/src/data/telemetry.ts

@@ -0,0 +1,337 @@
+import type { SessionEntry } from "@earendil-works/pi-coding-agent";
+
+import { STALL_MS } from "../render/timing.ts";
+import {
+	emptyTotals,
+	perSecond,
+	promptOf,
+	REQUEST_CHANGES,
+	type RequestRecord,
+	type Totals,
+	usageTotals,
+} from "./totals.ts";
+
+const MIN_SAMPLE_MS = 250;
+/** Measured over 16,889 assistant messages: 2.88 characters per output token. */
+const DEFAULT_CHARS_PER_TOKEN = 2.9;
+
+/** So the wait for the first thought can be told from the first word. */
+export type DeltaKind = "thinking" | "writing";
+
+export interface RequestView {
+	streaming: boolean;
+	waiting: boolean;
+	elapsedMs: number;
+	/** From the request leaving to the first token of any kind. */
+	waitMs: number | undefined;
+	/** The same wait while it runs. */
+	waitingMs: number;
+	/** To the response headers: the provider's own latency. */
+	serverMs: number | undefined;
+	/** From the headers to the first token: the model reading the prompt. */
+	prefillMs: number | undefined;
+	/** Unset when it wrote without thinking. */
+	thoughtMs: number | undefined;
+	decode: number | undefined;
+	prefill: number | undefined;
+	usage: Totals;
+	estimated: boolean;
+}
+
+/** One call to the model, for the line under its response. */
+export interface Timing {
+	/** When it left. */
+	at: number;
+	streaming: boolean;
+	waitMs: number | undefined;
+	waitingMs: number;
+	serverMs: number | undefined;
+	thoughtMs: number | undefined;
+	totalMs: number;
+	writingMs: number;
+	/** Pauses of `STALL_MS` or more while it wrote. */
+	stallMs: number;
+	output: number;
+	estimated: boolean;
+}
+
+export interface RunView {
+	running: boolean;
+	elapsedMs: number;
+	requests: number;
+	cost: number;
+}
+
+interface Request {
+	startedAt: number;
+	sentAt: number | undefined;
+	headersAt: number | undefined;
+	firstTokenAt: number | undefined;
+	firstWritingAt: number | undefined;
+	endedAt: number | undefined;
+	messageAt?: number;
+	usage: Totals;
+	characters: number;
+	lastDeltaAt: number | undefined;
+	stallMs: number;
+}
+
+interface Run {
+	startedAt: number;
+	endedAt: number | undefined;
+	requests: number;
+	cost: number;
+}
+
+interface UsageLike {
+	input: number;
+	output: number;
+	cacheRead: number;
+	cacheWrite: number;
+	cost: { total: number };
+}
+
+function span(from: number | undefined, to: number | undefined): number | undefined {
+	return from === undefined || to === undefined ? undefined : to - from;
+}
+
+export class Telemetry {
+	readonly #now: () => number;
+	#run: Run | undefined;
+	#request: Request | undefined;
+	#last: RequestView | undefined;
+	/** By the response's timestamp, which stays the same while it's written. */
+	#byMessage = new Map<number, Request>();
+	#restored = new Map<number, Timing>();
+	/** They arrive before the pi message they belong to. */
+	#sentAt: number | undefined;
+	#headersAt: number | undefined;
+	/** So the live estimate uses a real ratio. */
+	#sampled = { characters: 0, output: 0 };
+
+	constructor(now: () => number = Date.now) {
+		this.#now = now;
+	}
+
+	runStarted(): void {
+		this.#run = { startedAt: this.#now(), endedAt: undefined, requests: 0, cost: 0 };
+	}
+
+	runEnded(): void {
+		if (this.#run !== undefined && this.#run.endedAt === undefined) this.#run.endedAt = this.#now();
+	}
+
+	/** Before every call to the model: the start of the wait. */
+	providerRequestSent(): void {
+		this.#sentAt = this.#now();
+		this.#headersAt = undefined;
+		this.#attach();
+	}
+
+	/** Before the stream is read. */
+	providerResponseArrived(): void {
+		this.#headersAt = this.#now();
+		this.#attach();
+	}
+
+	requestStarted(): void {
+		this.#sentAt = undefined;
+		this.#headersAt = undefined;
+		this.#begin();
+		if (this.#run === undefined || this.#run.endedAt !== undefined) this.runStarted();
+		if (this.#run !== undefined) this.#run.requests++;
+	}
+
+	/** Later requests of a turn: the next call after a tool ran. */
+	answerStarted(messageAt?: number): void {
+		if (this.#request === undefined || this.#request.endedAt !== undefined) this.#begin();
+		if (messageAt !== undefined && this.#request !== undefined) {
+			this.#byMessage.set(messageAt, this.#request);
+			this.#request.messageAt = messageAt;
+		}
+	}
+
+	answerGrew(delta: string, usage: UsageLike, kind: DeltaKind): void {
+		const request = this.#request;
+		if (request === undefined || request.endedAt !== undefined) return;
+		request.usage = usageTotals(usage);
+		if (delta.length === 0) return;
+		const grewAt = this.#now();
+		const gap = grewAt - (request.lastDeltaAt ?? grewAt);
+		if (gap >= STALL_MS) request.stallMs += gap;
+		request.lastDeltaAt = grewAt;
+		request.characters += delta.length;
+		request.firstTokenAt ??= this.#now();
+		if (kind === "writing") request.firstWritingAt ??= this.#now();
+	}
+
+	answerEnded(usage: UsageLike): RequestRecord | undefined {
+		const request = this.#request;
+		if (request === undefined || request.endedAt !== undefined) return undefined;
+
+		const now = this.#now();
+		request.endedAt = now;
+		request.usage = usageTotals(usage);
+		if (this.#run !== undefined) this.#run.cost += usage.cost.total;
+		if (request.usage.output === 0 && request.firstTokenAt === undefined) return undefined;
+
+		if (request.characters > 0 && request.usage.output > 0) {
+			this.#sampled.characters += request.characters;
+			this.#sampled.output += request.usage.output;
+		}
+
+		const firstTokenAt = request.firstTokenAt ?? now;
+		const from = request.sentAt ?? request.startedAt;
+		this.#last = this.#view(request);
+		const timing = this.#timing(request);
+		return {
+			output: request.usage.output,
+			prompt: promptOf(request.usage),
+			firstTokenMs: firstTokenAt - from,
+			generationMs: now - firstTokenAt,
+			at: from,
+			messageAt: request.messageAt,
+			serverMs: span(request.sentAt, request.headersAt),
+			thoughtMs: timing.thoughtMs,
+			totalMs: now - from,
+			stallMs: request.stallMs,
+		};
+	}
+
+	timingOf(messageAt: number): Timing | undefined {
+		const request = this.#byMessage.get(messageAt);
+		return request === undefined ? this.#restored.get(messageAt) : this.#timing(request);
+	}
+
+	/** The calls an earlier run of this session recorded. */
+	restore(entries: readonly SessionEntry[]): void {
+		for (const entry of entries) {
+			if (entry.type !== "custom" || entry.customType !== REQUEST_CHANGES) continue;
+			const record = entry.data as Partial<RequestRecord> | undefined;
+			if (record?.messageAt === undefined || record.at === undefined) continue;
+			this.#restored.set(record.messageAt, {
+				at: record.at,
+				streaming: false,
+				waitMs: record.firstTokenMs,
+				waitingMs: record.firstTokenMs ?? 0,
+				serverMs: record.serverMs,
+				thoughtMs: record.thoughtMs,
+				totalMs: record.totalMs ?? 0,
+				writingMs: record.generationMs ?? 0,
+				stallMs: record.stallMs ?? 0,
+				output: record.output ?? 0,
+				estimated: false,
+			});
+		}
+	}
+
+	request(): RequestView | undefined {
+		const request = this.#request;
+		return request === undefined ? undefined : this.#view(request);
+	}
+
+	last(): RequestView | undefined {
+		return this.#last;
+	}
+
+	run(): RunView | undefined {
+		const run = this.#run;
+		if (run === undefined) return undefined;
+		const running = run.endedAt === undefined;
+		return {
+			running,
+			elapsedMs: (run.endedAt ?? this.#now()) - run.startedAt,
+			requests: run.requests,
+			cost: run.cost,
+		};
+	}
+
+	reset(): void {
+		this.#run = undefined;
+		this.#request = undefined;
+		this.#last = undefined;
+		this.#byMessage.clear();
+		this.#restored.clear();
+		this.#sentAt = undefined;
+		this.#headersAt = undefined;
+	}
+
+	#begin(): void {
+		this.#request = {
+			startedAt: this.#now(),
+			sentAt: this.#sentAt,
+			headersAt: this.#headersAt,
+			firstTokenAt: undefined,
+			firstWritingAt: undefined,
+			endedAt: undefined,
+			usage: emptyTotals(),
+			characters: 0,
+			lastDeltaAt: undefined,
+			stallMs: 0,
+		};
+	}
+
+	/** Pi sends before it opens the message, so the marks may arrive first. */
+	#attach(): void {
+		const request = this.#request;
+		if (request === undefined || request.endedAt !== undefined) return;
+		request.sentAt = this.#sentAt;
+		request.headersAt = this.#headersAt;
+	}
+
+	#timing(request: Request): Timing {
+		const view = this.#view(request);
+		const end = request.endedAt ?? this.#now();
+		const from = request.sentAt ?? request.startedAt;
+		// While it's paused, the pause so far counts too.
+		const pause = request.endedAt === undefined ? end - (request.lastDeltaAt ?? end) : 0;
+		return {
+			at: from,
+			streaming: view.streaming,
+			waitMs: view.waitMs,
+			waitingMs: view.waitingMs,
+			serverMs: view.serverMs,
+			thoughtMs: view.thoughtMs,
+			totalMs: end - from,
+			writingMs: request.firstTokenAt === undefined ? 0 : end - request.firstTokenAt,
+			stallMs: request.stallMs + (pause >= STALL_MS ? pause : 0),
+			output: view.usage.output,
+			estimated: view.estimated,
+		};
+	}
+
+	#view(request: Request): RequestView {
+		const now = this.#now();
+		const streaming = request.endedAt === undefined;
+		const from = request.sentAt ?? request.startedAt;
+		const waitMs = span(from, request.firstTokenAt);
+		const estimated = streaming && request.usage.output === 0;
+		const output = estimated
+			? Math.round(request.characters / this.#charsPerToken())
+			: request.usage.output;
+		const writingMs =
+			request.firstTokenAt === undefined ? 0 : (request.endedAt ?? now) - request.firstTokenAt;
+		const thoughtMs = span(request.firstTokenAt, request.firstWritingAt);
+
+		return {
+			streaming,
+			waiting: streaming && request.firstTokenAt === undefined,
+			elapsedMs: (request.endedAt ?? now) - request.startedAt,
+			waitMs,
+			waitingMs: (request.endedAt ?? now) - from,
+			serverMs: span(request.sentAt, request.headersAt),
+			prefillMs: span(request.headersAt ?? request.sentAt, request.firstTokenAt),
+			thoughtMs: thoughtMs !== undefined && thoughtMs > 0 ? thoughtMs : undefined,
+			decode: writingMs < MIN_SAMPLE_MS && streaming ? undefined : perSecond(output, writingMs),
+			prefill: waitMs === undefined ? undefined : perSecond(promptOf(request.usage), waitMs),
+			usage: { ...request.usage, output },
+			estimated,
+		};
+	}
+
+	/** This session's ratio, or the measured default. */
+	#charsPerToken(): number {
+		const { characters, output } = this.#sampled;
+		return output > 0 ? characters / output : DEFAULT_CHARS_PER_TOKEN;
+	}
+}

package/packages/frame/src/data/todos.ts

@@ -0,0 +1,162 @@
+// The agent's checklist. Pure: the tool, the panel and the session all go through these.
+import { plainText } from "../render/safe.ts";
+
+export const TODO_ENTRY = "pi-frame:todo";
+const MAX_ITEMS = 50;
+const MAX_TEXT = 200;
+
+const INVISIBLE = /[\u00ad\u034f\u061c\u180e\u200b-\u200f\u2060-\u206f\u3164\ufeff\ufff9-\ufffb\u{e0000}-\u{e007f}]/gu;
+
+export type TodoStatus = "open" | "active" | "done";
+
+export interface Todo {
+	id: number;
+	text: string;
+	status: TodoStatus;
+}
+
+export interface TodoList {
+	items: readonly Todo[];
+	nextId: number;
+}
+
+export interface TodoChange {
+	plan?: readonly string[];
+	add?: readonly string[];
+	drop?: readonly number[];
+	done?: readonly number[];
+	start?: number;
+}
+
+export const NO_TODOS: TodoList = { items: [], nextId: 1 };
+
+/** Model text, drawn on one row, so it can't drive the terminal or the layout. */
+function cleanText(text: string): string {
+	const clean = plainText(text).replace(INVISIBLE, "").replace(/\s+/g, " ").trim();
+	return clean.length > MAX_TEXT ? `${clean.slice(0, MAX_TEXT - 3)}...` : clean;
+}
+
+function added(texts: readonly string[], from: number): Todo[] {
+	return texts.map((text, index) => {
+		const clean = cleanText(text);
+		if (clean === "") throw new Error("A todo item needs text.");
+		return { id: from + index, text: clean, status: "open" };
+	});
+}
+
+function checkedIds(list: TodoList, ids: readonly number[]): void {
+	const known = new Set(list.items.map((item) => item.id));
+	const missing = ids.filter((id) => !known.has(id));
+	if (missing.length > 0) throw new Error(`No todo item ${missing.join(", ")}.`);
+}
+
+/** All or nothing: a bad id or text throws, with the list, which stays as it was. No arguments just reads it. */
+export function changedTodos(list: TodoList, change: TodoChange): TodoList {
+	try {
+		return applied(list, change);
+	} catch (error) {
+		throw new Error(`${(error as Error).message} The list has not changed.\n${todoText(list)}`);
+	}
+}
+
+function applied(list: TodoList, change: TodoChange): TodoList {
+	let items: readonly Todo[] = list.items;
+	let nextId = list.nextId;
+	if (change.plan !== undefined) {
+		items = added(change.plan, 1);
+		nextId = items.length + 1;
+	}
+	if (change.add !== undefined) {
+		items = [...items, ...added(change.add, nextId)];
+		nextId += change.add.length;
+	}
+	if (items.length > MAX_ITEMS) throw new Error(`A todo list holds at most ${MAX_ITEMS} items.`);
+
+	const next = (): TodoList => ({ items, nextId });
+	checkedIds(next(), change.drop ?? []);
+	items = items.filter((item) => !change.drop?.includes(item.id));
+	checkedIds(next(), change.done ?? []);
+	items = items.map((item) => (change.done?.includes(item.id) ? { ...item, status: "done" } : item));
+
+	const start = change.start;
+	if (start !== undefined) {
+		checkedIds(next(), [start]);
+		// Only one at a time, so starting one puts back whichever was active.
+		items = items.map((item) => {
+			if (item.id === start) return { ...item, status: "active" };
+			return item.status === "active" ? { ...item, status: "open" } : item;
+		});
+	}
+	return next();
+}
+
+export function hasOpenTodos(list: TodoList): boolean {
+	return list.items.some((item) => item.status !== "done");
+}
+
+export function doneCount(list: TodoList): number {
+	return list.items.filter((item) => item.status === "done").length;
+}
+
+const MARKS: Record<TodoStatus, string> = { open: "[ ]", active: "[>]", done: "[x]" };
+
+/** What the model reads. */
+export function todoText(list: TodoList): string {
+	if (list.items.length === 0) return "The todo list is empty.";
+	const rows = list.items.map((item) => `${MARKS[item.status]} ${item.id} ${item.text}`);
+	return [`${doneCount(list)}/${list.items.length} done`, ...rows].join("\n");
+}
+
+/** The transcript's one line: what's being worked on, or how far it got. */
+export function todoSummary(list: TodoList): string {
+	if (list.items.length === 0) return "list empty";
+	const active = list.items.find((item) => item.status === "active");
+	const count = `${doneCount(list)}/${list.items.length} done`;
+	return active === undefined ? count : `${count}, now: ${active.text}`;
+}
+
+function decoded(data: unknown): TodoList | undefined {
+	const list = data as { items?: unknown; nextId?: unknown } | null;
+	if (!Array.isArray(list?.items) || !Number.isSafeInteger(list.nextId)) return undefined;
+	if (list.items.length > MAX_ITEMS) return undefined;
+	const items: Todo[] = [];
+	const nextId = list.nextId as number;
+	const seen = new Set<number>();
+	for (const raw of list.items as Partial<Todo>[]) {
+		if (!Number.isSafeInteger(raw?.id) || typeof raw?.text !== "string") return undefined;
+		if (raw.status !== "open" && raw.status !== "active" && raw.status !== "done") return undefined;
+		const id = raw.id as number;
+		const text = cleanText(raw.text);
+		if (text === "" || id >= nextId || seen.has(id)) return undefined;
+		seen.add(id);
+		items.push({ id, text, status: raw.status });
+	}
+	return { items, nextId };
+}
+
+/** The last list saved on this branch. A session can come from anyone, so a bad entry is skipped. */
+export function todosOnBranch(entries: Iterable<unknown>): TodoList {
+	let list = NO_TODOS;
+	for (const entry of entries) {
+		const custom = entry as { type?: string; customType?: string; data?: unknown };
+		if (custom.type !== "custom" || custom.customType !== TODO_ENTRY) continue;
+		list = decoded(custom.data) ?? list;
+	}
+	return list;
+}
+
+/**
+ * Compaction can take the tool's last answer out of the model's context while the list goes on.
+ * Then the list comes first, where it doesn't move between calls, which keeps most of the prompt cache.
+ * A failed call may not show the list, so only an answer that worked counts.
+ */
+export function withTodoReminder<M extends { role: string; toolName?: string; isError?: boolean }>(
+	messages: readonly M[],
+	list: TodoList,
+	reminder: (text: string) => M,
+): M[] | undefined {
+	if (!hasOpenTodos(list)) return undefined;
+	const seen = (message: M) => message.role === "toolResult" && message.toolName === "todo" && message.isError !== true;
+	if (messages.some(seen)) return undefined;
+	return [reminder(`Your todo list, kept from earlier in this session:\n${todoText(list)}`), ...messages];
+}

package/packages/frame/src/data/totals.ts

@@ -0,0 +1,94 @@
+import { type UsageLike } from "@adeildo/pi-kit";
+
+export {
+	emptyTotals,
+	promptOf,
+	tokensOf,
+	totalsOf,
+	type Totals,
+	usageTotals,
+} from "@adeildo/pi-kit";
+import type { SessionEntry } from "@earendil-works/pi-coding-agent";
+
+export interface RequestRecord {
+	output: number;
+	prompt: number;
+	firstTokenMs: number;
+	generationMs: number;
+	/** For the line under the response. Older records don't have them. */
+	at?: number;
+	messageAt?: number;
+	serverMs?: number;
+	thoughtMs?: number;
+	totalMs?: number;
+	stallMs?: number;
+}
+
+export const REQUEST_CHANGES = "pi-frame:request";
+/** Entries from this package's old name. They have no `prompt`, so they count only for decode. */
+const LEGACY_CHANGES = "pi-statusline:turn";
+
+export interface Averages {
+	decode: number | undefined;
+	prefill: number | undefined;
+}
+
+function lastAssistantUsage(entries: readonly SessionEntry[]): UsageLike | undefined {
+	for (let index = entries.length - 1; index >= 0; index--) {
+		const entry = entries[index];
+		if (entry?.type !== "message" || entry.message.role !== "assistant") continue;
+		return entry.message.usage;
+	}
+	return undefined;
+}
+
+export function cacheHitPercent(entries: readonly SessionEntry[]): number | undefined {
+	const usage = lastAssistantUsage(entries);
+	if (usage === undefined) return undefined;
+	const prompt = usage.input + usage.cacheRead + usage.cacheWrite;
+	return prompt === 0 ? undefined : (usage.cacheRead / prompt) * 100;
+}
+
+export function cacheWarming(entries: readonly SessionEntry[]): boolean {
+	const usage = lastAssistantUsage(entries);
+	return usage !== undefined && usage.cacheRead === 0 && usage.cacheWrite > 0;
+}
+
+export function averagesOf(entries: readonly SessionEntry[]): Averages {
+	let output = 0;
+	let generationMs = 0;
+	let prompt = 0;
+	let waitingMs = 0;
+
+	for (const entry of entries) {
+		if (entry.type !== "custom") continue;
+		if (entry.customType !== REQUEST_CHANGES && entry.customType !== LEGACY_CHANGES) continue;
+		const record = entry.data;
+		if (!isRecord(record)) continue;
+
+		output += record.output;
+		generationMs += record.generationMs;
+		if (typeof record.prompt === "number" && record.prompt > 0) {
+			prompt += record.prompt;
+			waitingMs += record.firstTokenMs;
+		}
+	}
+
+	return { decode: perSecond(output, generationMs), prefill: perSecond(prompt, waitingMs) };
+}
+
+export function perSecond(tokens: number, milliseconds: number): number | undefined {
+	return tokens > 0 && milliseconds > 0 ? tokens / (milliseconds / 1000) : undefined;
+}
+
+function isRecord(
+	data: unknown,
+): data is Omit<RequestRecord, "prompt"> & { prompt?: number | undefined } {
+	if (typeof data !== "object" || data === null) return false;
+	const record = data as Partial<RequestRecord>;
+	return (
+		typeof record.output === "number" &&
+		typeof record.firstTokenMs === "number" &&
+		typeof record.generationMs === "number"
+	);
+}

package/packages/frame/src/data/visits.ts

@@ -0,0 +1,45 @@
+// The start screen says what changed since the last session in a folder, so it remembers when that was.
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { isObject } from "@adeildo/pi-kit";
+
+import type { LastVisit } from "./start-notes.ts";
+
+type Visits = Record<string, LastVisit>;
+
+/** Records this visit too. */
+export function visit(
+	file: string,
+	cwd: string,
+	version: string,
+	now: number,
+): LastVisit | undefined {
+	const visits = read(file);
+	const last = visits[cwd];
+	visits[cwd] = { at: now, version };
+	// Two sessions starting together must not read each other's half-written file. It lists your folders.
+	const temporary = `${file}.${process.pid}.tmp`;
+	try {
+		mkdirSync(dirname(file), { recursive: true });
+		writeFileSync(temporary, JSON.stringify(visits), { mode: 0o600 });
+		renameSync(temporary, file);
+	} catch {
+		// A start screen that cannot remember only loses "since when".
+		rmSync(temporary, { force: true });
+	}
+	return isVisit(last) ? last : undefined;
+}
+
+function isVisit(value: unknown): value is LastVisit {
+	return isObject(value) && Number.isFinite(value.at) && typeof value.version === "string";
+}
+
+function read(file: string): Visits {
+	try {
+		const value = JSON.parse(readFileSync(file, "utf8")) as unknown;
+		return isObject(value) ? (value as Visits) : {};
+	} catch {
+		return {};
+	}
+}