@nghyane/arcane 0.1.0

package/src/session/blob-store.ts

@@ -0,0 +1,103 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { isEnoent, logger } from "@nghyane/arcane-utils";
+
+const BLOB_PREFIX = "blob:sha256:";
+
+export interface BlobPutResult {
+	hash: string;
+	path: string;
+	get ref(): string;
+}
+
+/**
+ * Content-addressed blob store for externalizing large binary data (images) from session JSONL files.
+ *
+ * Files are stored at `<dir>/<sha256-hex>` with no extension. The SHA-256 hash is computed
+ * over the raw binary data (not base64). Content-addressing makes writes idempotent and
+ * provides automatic deduplication across sessions.
+ */
+export class BlobStore {
+	constructor(readonly dir: string) {}
+
+	/**
+	 * Write binary data to the blob store.
+	 * @returns SHA-256 hex hash of the data
+	 */
+	async put(data: Buffer): Promise<BlobPutResult> {
+		const hash = new Bun.CryptoHasher("sha256").update(data).digest("hex");
+		const blobPath = path.join(this.dir, hash);
+		const result = {
+			hash,
+			path: blobPath,
+			get ref() {
+				return `${BLOB_PREFIX}${hash}`;
+			},
+		};
+
+		await Bun.write(blobPath, data);
+		return result;
+	}
+
+	/** Read blob by hash, returns Buffer or null if not found. */
+	async get(hash: string): Promise<Buffer | null> {
+		const blobPath = path.join(this.dir, hash);
+		try {
+			const file = Bun.file(blobPath);
+			const ab = await file.arrayBuffer();
+			return Buffer.from(ab);
+		} catch (err) {
+			if (isEnoent(err)) return null;
+			throw err;
+		}
+	}
+
+	/** Check if a blob exists. */
+	async has(hash: string): Promise<boolean> {
+		try {
+			await fs.access(path.join(this.dir, hash));
+			return true;
+		} catch {
+			return false;
+		}
+	}
+}
+
+/** Check if a data string is a blob reference. */
+export function isBlobRef(data: string): boolean {
+	return data.startsWith(BLOB_PREFIX);
+}
+
+/** Extract the SHA-256 hash from a blob reference string. */
+export function parseBlobRef(data: string): string | null {
+	if (!data.startsWith(BLOB_PREFIX)) return null;
+	return data.slice(BLOB_PREFIX.length);
+}
+
+/**
+ * Externalize an image's base64 data to the blob store, returning a blob reference.
+ * If the data is already a blob reference, returns it unchanged.
+ */
+export async function externalizeImageData(blobStore: BlobStore, base64Data: string): Promise<string> {
+	if (isBlobRef(base64Data)) return base64Data;
+	const buffer = Buffer.from(base64Data, "base64");
+	const { ref } = await blobStore.put(buffer);
+	return ref;
+}
+
+/**
+ * Resolve a blob reference back to base64 data.
+ * If the data is not a blob reference, returns it unchanged.
+ * If the blob is missing, logs a warning and returns a placeholder.
+ */
+export async function resolveImageData(blobStore: BlobStore, data: string): Promise<string> {
+	const hash = parseBlobRef(data);
+	if (!hash) return data;
+
+	const buffer = await blobStore.get(hash);
+	if (!buffer) {
+		logger.warn("Blob not found for image reference", { hash });
+		return data; // Return the ref as-is; downstream will see invalid base64 but won't crash
+	}
+	return buffer.toString("base64");
+}

package/src/session/compaction/branch-summarization.ts

@@ -0,0 +1,315 @@
+/**
+ * Branch summarization for tree navigation.
+ *
+ * When navigating to a different point in the session tree, this generates
+ * a summary of the branch being left so context isn't lost.
+ */
+import type { AgentMessage } from "@nghyane/arcane-agent";
+import type { Model } from "@nghyane/arcane-ai";
+import { completeSimple } from "@nghyane/arcane-ai";
+import { renderPromptTemplate } from "../../config/prompt-templates";
+import branchSummaryPrompt from "../../prompts/compaction/branch-summary.md" with { type: "text" };
+import branchSummaryPreamble from "../../prompts/compaction/branch-summary-preamble.md" with { type: "text" };
+import {
+	convertToLlm,
+	createBranchSummaryMessage,
+	createCompactionSummaryMessage,
+	createCustomMessage,
+} from "../../session/messages";
+import type { ReadonlySessionManager, SessionEntry } from "../../session/session-manager";
+import { estimateTokens } from "./compaction";
+import {
+	computeFileLists,
+	createFileOps,
+	extractFileOpsFromMessage,
+	type FileOperations,
+	SUMMARIZATION_SYSTEM_PROMPT,
+	serializeConversation,
+	upsertFileOperations,
+} from "./utils";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface BranchSummaryResult {
+	summary?: string;
+	readFiles?: string[];
+	modifiedFiles?: string[];
+	aborted?: boolean;
+	error?: string;
+}
+
+/** Details stored in BranchSummaryEntry.details for file tracking */
+export interface BranchSummaryDetails {
+	readFiles: string[];
+	modifiedFiles: string[];
+}
+
+export type { FileOperations } from "./utils";
+
+export interface BranchPreparation {
+	/** Messages extracted for summarization, in chronological order */
+	messages: AgentMessage[];
+	/** File operations extracted from tool calls */
+	fileOps: FileOperations;
+	/** Total estimated tokens in messages */
+	totalTokens: number;
+}
+
+export interface CollectEntriesResult {
+	/** Entries to summarize, in chronological order */
+	entries: SessionEntry[];
+	/** Common ancestor between old and new position, if any */
+	commonAncestorId: string | null;
+}
+
+export interface GenerateBranchSummaryOptions {
+	/** Model to use for summarization */
+	model: Model;
+	/** API key for the model */
+	apiKey: string;
+	/** Abort signal for cancellation */
+	signal: AbortSignal;
+	/** Optional custom instructions for summarization */
+	customInstructions?: string;
+	/** Tokens reserved for prompt + LLM response (default 16384) */
+	reserveTokens?: number;
+}
+
+// ============================================================================
+// Entry Collection
+// ============================================================================
+
+/**
+ * Collect entries that should be summarized when navigating from one position to another.
+ *
+ * Walks from oldLeafId back to the common ancestor with targetId, collecting entries
+ * along the way. Does NOT stop at compaction boundaries - those are included and their
+ * summaries become context.
+ *
+ * @param session - Session manager (read-only access)
+ * @param oldLeafId - Current position (where we're navigating from)
+ * @param targetId - Target position (where we're navigating to)
+ * @returns Entries to summarize and the common ancestor
+ */
+export function collectEntriesForBranchSummary(
+	session: ReadonlySessionManager,
+	oldLeafId: string | null,
+	targetId: string,
+): CollectEntriesResult {
+	// If no old position, nothing to summarize
+	if (!oldLeafId) {
+		return { entries: [], commonAncestorId: null };
+	}
+
+	// Find common ancestor (deepest node that's on both paths)
+	const oldPath = new Set(session.getBranch(oldLeafId).map(e => e.id));
+	const targetPath = session.getBranch(targetId);
+
+	// targetPath is root-first, so iterate backwards to find deepest common ancestor
+	let commonAncestorId: string | null = null;
+	for (let i = targetPath.length - 1; i >= 0; i--) {
+		if (oldPath.has(targetPath[i].id)) {
+			commonAncestorId = targetPath[i].id;
+			break;
+		}
+	}
+
+	// Collect entries from old leaf back to common ancestor
+	const entries: SessionEntry[] = [];
+	let current: string | null = oldLeafId;
+
+	while (current && current !== commonAncestorId) {
+		const entry = session.getEntry(current);
+		if (!entry) break;
+		entries.push(entry);
+		current = entry.parentId;
+	}
+
+	// Reverse to get chronological order
+	entries.reverse();
+
+	return { entries, commonAncestorId };
+}
+
+// ============================================================================
+// Entry to Message Conversion
+// ============================================================================
+
+/**
+ * Extract AgentMessage from a session entry.
+ * Similar to getMessageFromEntry in compaction.ts but also handles compaction entries.
+ */
+function getMessageFromEntry(entry: SessionEntry): AgentMessage | undefined {
+	switch (entry.type) {
+		case "message":
+			// Skip tool results - context is in assistant's tool call
+			if (entry.message.role === "toolResult") return undefined;
+			return entry.message;
+
+		case "custom_message":
+			return createCustomMessage(entry.customType, entry.content, entry.display, entry.details, entry.timestamp);
+
+		case "branch_summary":
+			return createBranchSummaryMessage(entry.summary, entry.fromId, entry.timestamp);
+
+		case "compaction":
+			return createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp, entry.shortSummary);
+
+		// These don't contribute to conversation content
+		case "thinking_level_change":
+		case "model_change":
+		case "custom":
+		case "label":
+			return undefined;
+	}
+}
+
+/**
+ * Prepare entries for summarization with token budget.
+ *
+ * Walks entries from NEWEST to OLDEST, adding messages until we hit the token budget.
+ * This ensures we keep the most recent context when the branch is too long.
+ *
+ * Also collects file operations from:
+ * - Tool calls in assistant messages
+ * - Existing branch_summary entries' details (for cumulative tracking)
+ *
+ * @param entries - Entries in chronological order
+ * @param tokenBudget - Maximum tokens to include (0 = no limit)
+ */
+export function prepareBranchEntries(entries: SessionEntry[], tokenBudget: number = 0): BranchPreparation {
+	const messages: AgentMessage[] = [];
+	const fileOps = createFileOps();
+	let totalTokens = 0;
+
+	// First pass: collect file ops from ALL entries (even if they don't fit in token budget)
+	// This ensures we capture cumulative file tracking from nested branch summaries
+	// Only extract from pi-generated summaries (fromExtension !== true), not extension-generated ones
+	for (const entry of entries) {
+		if (entry.type === "branch_summary" && !entry.fromExtension && entry.details) {
+			const details = entry.details as BranchSummaryDetails;
+			if (Array.isArray(details.readFiles)) {
+				for (const f of details.readFiles) fileOps.read.add(f);
+			}
+			if (Array.isArray(details.modifiedFiles)) {
+				// Modified files go into both edited and written for proper deduplication
+				for (const f of details.modifiedFiles) {
+					fileOps.edited.add(f);
+				}
+			}
+		}
+	}
+
+	// Second pass: walk from newest to oldest, adding messages until token budget
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		const message = getMessageFromEntry(entry);
+		if (!message) continue;
+
+		// Extract file ops from assistant messages (tool calls)
+		extractFileOpsFromMessage(message, fileOps);
+
+		const tokens = estimateTokens(message);
+
+		// Check budget before adding
+		if (tokenBudget > 0 && totalTokens + tokens > tokenBudget) {
+			// If this is a summary entry, try to fit it anyway as it's important context
+			if (entry.type === "compaction" || entry.type === "branch_summary") {
+				if (totalTokens < tokenBudget * 0.9) {
+					messages.unshift(message);
+					totalTokens += tokens;
+				}
+			}
+			// Stop - we've hit the budget
+			break;
+		}
+
+		messages.unshift(message);
+		totalTokens += tokens;
+	}
+
+	return { messages, fileOps, totalTokens };
+}
+
+// ============================================================================
+// Summary Generation
+// ============================================================================
+
+const BRANCH_SUMMARY_PREAMBLE = renderPromptTemplate(branchSummaryPreamble);
+
+const BRANCH_SUMMARY_PROMPT = renderPromptTemplate(branchSummaryPrompt);
+
+/**
+ * Generate a summary of abandoned branch entries.
+ *
+ * @param entries - Session entries to summarize (chronological order)
+ * @param options - Generation options
+ */
+export async function generateBranchSummary(
+	entries: SessionEntry[],
+	options: GenerateBranchSummaryOptions,
+): Promise<BranchSummaryResult> {
+	const { model, apiKey, signal, customInstructions, reserveTokens = 16384 } = options;
+
+	// Token budget = context window minus reserved space for prompt + response
+	const contextWindow = model.contextWindow || 128000;
+	const tokenBudget = contextWindow - reserveTokens;
+
+	const { messages, fileOps } = prepareBranchEntries(entries, tokenBudget);
+
+	if (messages.length === 0) {
+		return { summary: "No content to summarize" };
+	}
+
+	// Transform to LLM-compatible messages, then serialize to text
+	// Serialization prevents the model from treating it as a conversation to continue
+	const llmMessages = convertToLlm(messages);
+	const conversationText = serializeConversation(llmMessages);
+
+	// Build prompt
+	const instructions = customInstructions || BRANCH_SUMMARY_PROMPT;
+	const promptText = `<conversation>\n${conversationText}\n</conversation>\n\n${instructions}`;
+
+	const summarizationMessages = [
+		{
+			role: "user" as const,
+			content: [{ type: "text" as const, text: promptText }],
+			timestamp: Date.now(),
+		},
+	];
+
+	// Call LLM for summarization
+	const response = await completeSimple(
+		model,
+		{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
+		{ apiKey, signal, maxTokens: 2048 },
+	);
+
+	// Check if aborted or errored
+	if (response.stopReason === "aborted") {
+		return { aborted: true };
+	}
+	if (response.stopReason === "error") {
+		return { error: response.errorMessage || "Summarization failed" };
+	}
+
+	let summary = response.content
+		.filter((c): c is { type: "text"; text: string } => c.type === "text")
+		.map(c => c.text)
+		.join("\n");
+
+	// Prepend preamble to provide context about the branch summary
+	summary = BRANCH_SUMMARY_PREAMBLE + summary;
+
+	// Compute file lists and append to summary
+	const { readFiles, modifiedFiles } = computeFileLists(fileOps);
+	summary = upsertFileOperations(summary, readFiles, modifiedFiles);
+
+	return {
+		summary: summary || "No summary generated",
+		readFiles,
+		modifiedFiles,
+	};
+}