@clinebot/core 0.0.37 → 0.0.38

package/src/extensions/context/compaction-shared.ts

@@ -1,459 +0,0 @@
-import type { ToolResultContent } from "@clinebot/llms";
-import type { MessageWithMetadata } from "@clinebot/shared";
-import type {
-	CoreCompactionContext,
-	CoreCompactionSummarizerConfig,
-} from "../../types/config";
-import type { ProviderConfig } from "../../types/provider-settings";
-
-export const DEFAULT_CONTEXT_WINDOW_TOKENS = 200_000;
-export const DEFAULT_THRESHOLD_RATIO = 0.95;
-export const DEFAULT_RESERVE_TOKENS = 16_384;
-export const DEFAULT_PRESERVE_RECENT_TOKENS = 20_000;
-export const DEFAULT_SUMMARY_MAX_OUTPUT_TOKENS = 1_024;
-export const TOOL_RESULT_CHAR_LIMIT = 2_000;
-export const FILE_CONTENT_CHAR_LIMIT = 2_000;
-export const MIN_TRUNCATED_MESSAGE_TOKENS = 8;
-
-export interface FileOperationSummary {
-	readFiles: string[];
-	modifiedFiles: string[];
-}
-
-export interface CompactionSummaryMetadata {
-	kind: "compaction_summary";
-	summary: string;
-	details: FileOperationSummary;
-	tokensBefore: number;
-	generatedAt: number;
-}
-
-export type EstimateMessageTokens = (message: MessageWithMetadata) => number;
-
-export function estimateTokens(text: string): number {
-	return Math.max(1, Math.ceil(text.length / 4));
-}
-
-export function truncateText(text: string, limit: number): string {
-	if (text.length <= limit) {
-		return text;
-	}
-	return `${text.slice(0, limit)}\n...[truncated ${text.length - limit} chars]`;
-}
-
-export function flattenToolResultContent(
-	content: ToolResultContent["content"],
-): string {
-	if (typeof content === "string") {
-		return truncateText(content, TOOL_RESULT_CHAR_LIMIT);
-	}
-	return content
-		.map((block) => {
-			switch (block.type) {
-				case "text":
-					return block.text;
-				case "file":
-					return `<file path="${block.path}">\n${truncateText(block.content, FILE_CONTENT_CHAR_LIMIT)}\n</file>`;
-				case "image":
-					return `[image:${block.mediaType}]`;
-				default:
-					return "";
-			}
-		})
-		.join("\n");
-}
-
-export function formatToolInput(input: Record<string, unknown>): string {
-	return Object.entries(input)
-		.map(([key, value]) => `${key}=${JSON.stringify(value)}`)
-		.join(", ");
-}
-
-export function serializeMessage(message: MessageWithMetadata): string {
-	if (typeof message.content === "string") {
-		return `[${message.role === "user" ? "User" : "Bot"}]: ${message.content}`;
-	}
-	const lines: string[] = [];
-	for (const block of message.content) {
-		switch (block.type) {
-			case "text":
-				lines.push(
-					`[${message.role === "user" ? "User" : "Bot"}]: ${block.text}`,
-				);
-				break;
-			case "thinking":
-				lines.push(`[Bot thinking]: ${truncateText(block.thinking, 2_000)}`);
-				break;
-			case "redacted_thinking":
-				lines.push("[Bot thinking]: [redacted]");
-				break;
-			case "tool_use":
-				lines.push(
-					`[Bot tool calls]: ${block.name}(${formatToolInput(block.input)})`,
-				);
-				break;
-			case "tool_result":
-				lines.push(`[Tool result]: ${flattenToolResultContent(block.content)}`);
-				break;
-			case "file":
-				lines.push(
-					`[${message.role === "user" ? "User" : "Bot"} file ${block.path}]: ${truncateText(block.content, FILE_CONTENT_CHAR_LIMIT)}`,
-				);
-				break;
-			case "image":
-				lines.push(
-					`[${message.role === "user" ? "User" : "Bot"} image]: ${block.mediaType}`,
-				);
-				break;
-		}
-	}
-	return lines.join("\n");
-}
-
-export function serializeConversation(messages: MessageWithMetadata[]): string {
-	return messages.map(serializeMessage).join("\n\n").trim();
-}
-
-export function createTokenEstimator(): EstimateMessageTokens {
-	const cache = new WeakMap<object, number>();
-	return (message) => {
-		const ref = message as unknown as object;
-		const cached = cache.get(ref);
-		if (typeof cached === "number") {
-			return cached;
-		}
-		const value = estimateTokens(serializeMessage(message));
-		cache.set(ref, value);
-		return value;
-	};
-}
-
-export function isCompactionSummaryMessage(
-	message: MessageWithMetadata,
-): boolean {
-	return (
-		(message.metadata as { kind?: string } | undefined)?.kind ===
-		"compaction_summary"
-	);
-}
-
-export function getCompactionSummaryMetadata(
-	message: MessageWithMetadata,
-): CompactionSummaryMetadata | undefined {
-	if (!isCompactionSummaryMessage(message)) {
-		return undefined;
-	}
-	const metadata = message.metadata as Record<string, unknown> | undefined;
-	if (!metadata) {
-		return undefined;
-	}
-	const details = metadata.details as Record<string, unknown> | undefined;
-	return {
-		kind: "compaction_summary",
-		summary: String(metadata.summary ?? ""),
-		details: {
-			readFiles: Array.isArray(details?.readFiles)
-				? details.readFiles
-						.filter((value): value is string => typeof value === "string")
-						.map((value) => value.trim())
-						.filter((value) => value.length > 0)
-				: [],
-			modifiedFiles: Array.isArray(details?.modifiedFiles)
-				? details.modifiedFiles
-						.filter((value): value is string => typeof value === "string")
-						.map((value) => value.trim())
-						.filter((value) => value.length > 0)
-				: [],
-		},
-		tokensBefore: Number(metadata.tokensBefore ?? 0),
-		generatedAt: Number(metadata.generatedAt ?? 0),
-	};
-}
-
-export function isToolResultOnlyUserMessage(
-	message: MessageWithMetadata,
-): boolean {
-	if (message.role !== "user" || !Array.isArray(message.content)) {
-		return false;
-	}
-	return (
-		message.content.length > 0 &&
-		message.content.every((block) => block.type === "tool_result")
-	);
-}
-
-export function isTurnStartMessage(message: MessageWithMetadata): boolean {
-	return (
-		message.role === "user" &&
-		!isToolResultOnlyUserMessage(message) &&
-		!isCompactionSummaryMessage(message)
-	);
-}
-
-export function findFirstUserMessageIndex(
-	messages: MessageWithMetadata[],
-): number {
-	for (let index = 0; index < messages.length; index += 1) {
-		if (isTurnStartMessage(messages[index])) {
-			return index;
-		}
-	}
-	return -1;
-}
-
-export function findLastTurnStartIndex(
-	messages: MessageWithMetadata[],
-): number {
-	for (let index = messages.length - 1; index >= 0; index -= 1) {
-		if (isTurnStartMessage(messages[index])) {
-			return index;
-		}
-	}
-	return 0;
-}
-
-export function findLastAssistantIndex(
-	messages: MessageWithMetadata[],
-): number {
-	for (let index = messages.length - 1; index >= 0; index -= 1) {
-		if (messages[index].role === "assistant") {
-			return index;
-		}
-	}
-	return -1;
-}
-
-export function findLatestSummaryIndex(
-	messages: MessageWithMetadata[],
-): number {
-	for (let index = messages.length - 1; index >= 0; index -= 1) {
-		if (isCompactionSummaryMessage(messages[index])) {
-			return index;
-		}
-	}
-	return -1;
-}
-
-export function findCutIndex(
-	messages: MessageWithMetadata[],
-	preserveRecentTokens: number,
-	estimateMessageTokens: EstimateMessageTokens,
-): number {
-	const lastTurnStartIndex = findLastTurnStartIndex(messages);
-	if (lastTurnStartIndex <= 0) {
-		return 0;
-	}
-
-	let total = 0;
-	let candidate = messages.length;
-	for (let index = messages.length - 1; index >= 0; index -= 1) {
-		total += estimateMessageTokens(messages[index]);
-		candidate = index;
-		if (total >= preserveRecentTokens) {
-			break;
-		}
-	}
-
-	if (candidate <= 0) {
-		return 0;
-	}
-
-	return Math.min(candidate, lastTurnStartIndex);
-}
-
-export function collectPaths(value: unknown): string[] {
-	if (typeof value === "string" && value.trim().length > 0) {
-		return [value];
-	}
-	if (Array.isArray(value)) {
-		return value.flatMap((item) => collectPaths(item));
-	}
-	if (value && typeof value === "object") {
-		const record = value as Record<string, unknown>;
-		const paths: string[] = [];
-		for (const key of [
-			"path",
-			"file_path",
-			"target_file",
-			"new_file_path",
-			"old_file_path",
-		]) {
-			paths.push(...collectPaths(record[key]));
-		}
-		if (Array.isArray(record.files)) {
-			for (const item of record.files) {
-				if (item && typeof item === "object") {
-					paths.push(...collectPaths((item as Record<string, unknown>).path));
-				}
-			}
-		}
-		if (Array.isArray(record.file_paths)) {
-			paths.push(...collectPaths(record.file_paths));
-		}
-		return paths;
-	}
-	return [];
-}
-
-export function mergeUnique(base: string[], next: Iterable<string>): string[] {
-	const seen = new Set(base);
-	for (const value of next) {
-		const trimmed = value.trim();
-		if (!trimmed) {
-			continue;
-		}
-		seen.add(trimmed);
-	}
-	return [...seen].sort((a, b) => a.localeCompare(b));
-}
-
-export function extractFileOps(
-	messages: MessageWithMetadata[],
-): FileOperationSummary {
-	let readFiles: string[] = [];
-	let modifiedFiles: string[] = [];
-	for (const message of messages) {
-		const summaryMetadata = getCompactionSummaryMetadata(message);
-		if (summaryMetadata) {
-			readFiles = mergeUnique(readFiles, summaryMetadata.details.readFiles);
-			modifiedFiles = mergeUnique(
-				modifiedFiles,
-				summaryMetadata.details.modifiedFiles,
-			);
-			continue;
-		}
-		if (!Array.isArray(message.content)) {
-			continue;
-		}
-		for (const block of message.content) {
-			if (block.type === "file") {
-				readFiles = mergeUnique(readFiles, [block.path]);
-				continue;
-			}
-			if (block.type !== "tool_use") {
-				continue;
-			}
-			const paths = collectPaths(block.input);
-			if (block.name === "read_files") {
-				readFiles = mergeUnique(readFiles, paths);
-				continue;
-			}
-			if (block.name === "editor" || block.name === "apply_patch") {
-				modifiedFiles = mergeUnique(modifiedFiles, paths);
-			}
-		}
-	}
-	return { readFiles, modifiedFiles };
-}
-
-export function renderFilesSection(fileOps: FileOperationSummary): string {
-	const readLines =
-		fileOps.readFiles.length > 0
-			? fileOps.readFiles.map((path) => `- ${path}`).join("\n")
-			: "- none";
-	const modifiedLines =
-		fileOps.modifiedFiles.length > 0
-			? fileOps.modifiedFiles.map((path) => `- ${path}`).join("\n")
-			: "- none";
-	return `## Files\nRead:\n${readLines}\nModified:\n${modifiedLines}`;
-}
-
-export function ensureFilesSection(
-	summary: string,
-	fileOps: FileOperationSummary,
-): string {
-	if (/^## Files$/im.test(summary)) {
-		return summary.trim();
-	}
-	return `${summary.trim()}\n\n${renderFilesSection(fileOps)}`.trim();
-}
-
-export function buildSummaryRequest(options: {
-	previousSummary?: string;
-	conversationText: string;
-	fileOps: FileOperationSummary;
-}): string {
-	const parts: string[] = [
-		`Summarize this session for continuation. Be concise and factual.
-
-## Goal
-One sentence: what is being built or fixed.
-
-## State
-- Done: completed steps
-- In Progress: current work
-- Blocked: blockers or open questions
-
-## Highlights
-Key technical choices or notable findings (omit if none).
-
-## Next
-Immediate next steps.
-
-## Files
-Read: ${options.fileOps.readFiles.join(", ") || "none"}
-Edited: ${options.fileOps.modifiedFiles.join(", ") || "none"}`,
-	];
-
-	if (options.previousSummary?.trim()) {
-		parts.push(`Previous summary:\n${options.previousSummary.trim()}`);
-	}
-
-	parts.push(`Conversation:\n${options.conversationText || "(empty)"}`);
-
-	return parts.join("\n\n");
-}
-
-export function resolveSummarizerConfig(options: {
-	activeProviderConfig: ProviderConfig;
-	summarizer?: CoreCompactionSummarizerConfig;
-}): ProviderConfig {
-	const summarizer = options.summarizer;
-	if (!summarizer) {
-		return {
-			...options.activeProviderConfig,
-			maxOutputTokens: DEFAULT_SUMMARY_MAX_OUTPUT_TOKENS,
-			thinking: false,
-		};
-	}
-	const baseProviderConfig =
-		summarizer.providerConfig?.providerId === summarizer.providerId
-			? summarizer.providerConfig
-			: undefined;
-	return {
-		...(baseProviderConfig ?? {}),
-		providerId: summarizer.providerId,
-		modelId: summarizer.modelId,
-		apiKey: summarizer.apiKey ?? baseProviderConfig?.apiKey,
-		baseUrl: summarizer.baseUrl ?? baseProviderConfig?.baseUrl,
-		headers: summarizer.headers ?? baseProviderConfig?.headers,
-		knownModels: summarizer.knownModels ?? baseProviderConfig?.knownModels,
-		maxOutputTokens:
-			summarizer.maxOutputTokens ?? DEFAULT_SUMMARY_MAX_OUTPUT_TOKENS,
-		thinking: false,
-	};
-}
-
-export function buildSummaryMessage(options: {
-	summary: string;
-	fileOps: FileOperationSummary;
-	tokensBefore: number;
-}): MessageWithMetadata {
-	return {
-		role: "user",
-		content: `Context summary:\n\n${options.summary}`,
-		metadata: {
-			kind: "compaction_summary",
-			summary: options.summary,
-			details: options.fileOps,
-			tokensBefore: options.tokensBefore,
-			generatedAt: Date.now(),
-		} satisfies CompactionSummaryMetadata,
-	};
-}
-
-export function getContextWindowTokens(
-	context: Pick<CoreCompactionContext, "contextWindowTokens">,
-): number {
-	return context.contextWindowTokens;
-}

package/src/extensions/context/compaction.ts

@@ -1,226 +0,0 @@
-import type {
-	CoreCompactionConfig,
-	CoreCompactionContext,
-	CoreCompactionResult,
-	CoreCompactionStrategy,
-	CoreSessionConfig,
-} from "../../types/config";
-import type { ProviderConfig } from "../../types/provider-settings";
-import { runAgenticCompaction } from "./agentic-compaction";
-import { runBasicCompaction } from "./basic-compaction";
-import {
-	createTokenEstimator,
-	DEFAULT_CONTEXT_WINDOW_TOKENS,
-	DEFAULT_PRESERVE_RECENT_TOKENS,
-	DEFAULT_THRESHOLD_RATIO,
-} from "./compaction-shared";
-
-export interface ContextPipelinePrepareTurnInput {
-	agentId: string;
-	conversationId: string;
-	parentAgentId: string | null;
-	iteration: number;
-	messages: CoreCompactionContext["messages"];
-	apiMessages: CoreCompactionContext["messages"];
-	abortSignal: AbortSignal;
-	systemPrompt: string;
-	tools: unknown[];
-	model: CoreCompactionContext["model"];
-	emitStatusNotice?: (
-		message: string,
-		metadata?: Record<string, unknown>,
-	) => void;
-}
-
-export interface ContextPipelinePrepareTurnResult {
-	messages: CoreCompactionContext["messages"];
-	systemPrompt?: string;
-}
-
-type EstimateMessageTokens = ReturnType<typeof createTokenEstimator>;
-
-type BuiltinCompactionStrategyOptions = {
-	context: CoreCompactionContext;
-	providerConfig: ProviderConfig;
-	compaction: CoreCompactionConfig | undefined;
-	estimateMessageTokens: EstimateMessageTokens;
-	logger: Pick<CoreSessionConfig, "logger">["logger"];
-};
-
-type BuiltinCompactionStrategyRunner = (
-	options: BuiltinCompactionStrategyOptions,
-) =>
-	| Promise<CoreCompactionResult | undefined>
-	| CoreCompactionResult
-	| undefined;
-
-const BUILTIN_COMPACTION_STRATEGIES = {
-	basic: ({ context, estimateMessageTokens, logger }) =>
-		runBasicCompaction({
-			context,
-			estimateMessageTokens,
-			logger,
-		}),
-	agentic: ({
-		context,
-		providerConfig,
-		compaction,
-		estimateMessageTokens,
-		logger,
-	}) =>
-		runAgenticCompaction({
-			context,
-			providerConfig,
-			summarizer: compaction?.summarizer,
-			preserveRecentTokens:
-				compaction?.preserveRecentTokens ?? DEFAULT_PRESERVE_RECENT_TOKENS,
-			estimateMessageTokens,
-			logger,
-		}),
-} satisfies Record<CoreCompactionStrategy, BuiltinCompactionStrategyRunner>;
-
-function resolveTriggerState(input: {
-	inputTokens: number;
-	contextWindowTokens: number;
-	config: CoreCompactionConfig;
-}): { shouldCompact: boolean; triggerTokens: number; thresholdRatio: number } {
-	if (typeof input.config.reserveTokens === "number") {
-		const reserveTokens = Math.max(0, input.config.reserveTokens);
-		const triggerTokens = Math.max(
-			0,
-			input.contextWindowTokens - reserveTokens,
-		);
-		return {
-			shouldCompact: input.inputTokens > triggerTokens,
-			triggerTokens,
-			thresholdRatio:
-				input.contextWindowTokens > 0
-					? triggerTokens / input.contextWindowTokens
-					: 0,
-		};
-	}
-
-	const thresholdRatio = input.config.thresholdRatio ?? DEFAULT_THRESHOLD_RATIO;
-	const triggerTokens = input.contextWindowTokens * thresholdRatio;
-	return {
-		shouldCompact: input.inputTokens > triggerTokens,
-		triggerTokens,
-		thresholdRatio,
-	};
-}
-
-export function createContextCompactionPrepareTurn(
-	config: Pick<
-		CoreSessionConfig,
-		"providerConfig" | "providerId" | "modelId" | "compaction" | "logger"
-	>,
-):
-	| ((
-			context: ContextPipelinePrepareTurnInput,
-	  ) => Promise<ContextPipelinePrepareTurnResult | undefined>)
-	| undefined {
-	const userCompaction = config.compaction;
-	if (userCompaction?.enabled !== true) {
-		return undefined;
-	}
-
-	const providerConfig =
-		config.providerConfig ??
-		({
-			providerId: config.providerId,
-			modelId: config.modelId,
-		} as ProviderConfig);
-	const estimateMessageTokens = createTokenEstimator();
-	const strategy = userCompaction?.strategy ?? "basic";
-	const runBuiltinStrategy = BUILTIN_COMPACTION_STRATEGIES[strategy];
-
-	return async (context) => {
-		const inputTokens = context.apiMessages.reduce(
-			(total: number, message) => total + estimateMessageTokens(message),
-			0,
-		);
-		const contextWindowTokens =
-			userCompaction?.contextWindowTokens ??
-			context.model.info?.contextWindow ??
-			DEFAULT_CONTEXT_WINDOW_TOKENS;
-		if (
-			typeof contextWindowTokens !== "number" ||
-			!Number.isFinite(contextWindowTokens) ||
-			contextWindowTokens <= 0
-		) {
-			return undefined;
-		}
-
-		const triggerState = resolveTriggerState({
-			inputTokens,
-			contextWindowTokens,
-			config: {
-				reserveTokens: userCompaction?.reserveTokens,
-				thresholdRatio: userCompaction?.thresholdRatio,
-			},
-		});
-		if (!triggerState.shouldCompact) {
-			return undefined;
-		}
-
-		const compactionContext = {
-			agentId: context.agentId,
-			conversationId: context.conversationId,
-			parentAgentId: context.parentAgentId,
-			iteration: context.iteration,
-			messages: context.messages,
-			model: context.model,
-			contextWindowTokens,
-			triggerTokens: triggerState.triggerTokens,
-			thresholdRatio: triggerState.thresholdRatio,
-			utilizationRatio:
-				contextWindowTokens > 0 ? inputTokens / contextWindowTokens : 0,
-		};
-
-		context.emitStatusNotice?.("auto-compacting", {
-			kind: "auto_compaction",
-			reason: "auto_compaction",
-			iteration: context.iteration,
-			triggerTokens: triggerState.triggerTokens,
-			contextWindowTokens,
-		});
-
-		const beforeMessageCount = context.messages.length;
-
-		const result = userCompaction?.compact
-			? await userCompaction.compact(compactionContext)
-			: await runBuiltinStrategy({
-					context: compactionContext,
-					providerConfig: {
-						...providerConfig,
-						abortSignal: context.abortSignal,
-					},
-					compaction: userCompaction,
-					estimateMessageTokens,
-					logger: config.logger,
-				});
-
-		if (result?.messages) {
-			const afterTokens = result.messages.reduce(
-				(total: number, message) => total + estimateMessageTokens(message),
-				0,
-			);
-			config.logger?.log("Context compaction completed", {
-				severity: "info",
-				strategy: strategy,
-				contextWindowTokens,
-				inputTokens,
-				afterTokens,
-				tokensSaved: inputTokens - afterTokens,
-				utilizationBefore: `${((inputTokens / contextWindowTokens) * 100).toFixed(1)}%`,
-				utilizationAfter: `${((afterTokens / contextWindowTokens) * 100).toFixed(1)}%`,
-				thresholdTrigger: `${(triggerState.thresholdRatio * 100).toFixed(1)}%`,
-				messagesBefore: beforeMessageCount,
-				messagesAfter: result.messages.length,
-				messagesRemoved: beforeMessageCount - result.messages.length,
-			} as Record<string, unknown>);
-		}
-
-		return result;
-	};
-}

package/src/extensions/index.ts

@@ -1,18 +0,0 @@
-export type { ResolveAgentPluginPathsOptions } from "./plugin/plugin-config-loader";
-export {
-	discoverPluginModulePaths,
-	resolveAgentPluginPaths,
-	resolveAndLoadAgentPlugins,
-	resolvePluginConfigSearchPaths,
-} from "./plugin/plugin-config-loader";
-export type {
-	PluginInitializationFailure,
-	PluginInitializationWarning,
-	PluginLoadDiagnostics,
-} from "./plugin/plugin-load-report";
-export type { LoadAgentPluginFromPathOptions } from "./plugin/plugin-loader";
-export {
-	loadAgentPluginFromPath,
-	loadAgentPluginsFromPaths,
-	loadAgentPluginsFromPathsWithDiagnostics,
-} from "./plugin/plugin-loader";