openrecall 0.2.2 → 0.4.0

package/src/dcp/strategies/deduplication.ts

@@ -0,0 +1,123 @@
+import type { PluginConfig } from "../config"
+import type { Logger } from "../logger"
+import type { SessionState, WithParts } from "../state"
+import { getFilePathsFromParameters, isProtected } from "../protected-file-patterns"
+import { getTotalToolTokens } from "./utils"
+
+/**
+ * Deduplication strategy - prunes older tool calls that have identical
+ * tool name and parameters, keeping only the most recent occurrence.
+ * Modifies the session state in place to add pruned tool call IDs.
+ */
+export const deduplicate = (
+    state: SessionState,
+    logger: Logger,
+    config: PluginConfig,
+    messages: WithParts[],
+): void => {
+    if (state.manualMode && !config.manualMode.automaticStrategies) {
+        return
+    }
+
+    if (!config.strategies.deduplication.enabled) {
+        return
+    }
+
+    const allToolIds = state.toolIdList
+    if (allToolIds.length === 0) {
+        return
+    }
+
+    // Filter out IDs already pruned
+    const unprunedIds = allToolIds.filter((id) => !state.prune.tools.has(id))
+
+    if (unprunedIds.length === 0) {
+        return
+    }
+
+    const protectedTools = config.strategies.deduplication.protectedTools
+
+    // Group by signature (tool name + normalized parameters)
+    const signatureMap = new Map<string, string[]>()
+
+    for (const id of unprunedIds) {
+        const metadata = state.toolParameters.get(id)
+        if (!metadata) {
+            // logger.warn(`Missing metadata for tool call ID: ${id}`)
+            continue
+        }
+
+        // Skip protected tools
+        if (protectedTools.includes(metadata.tool)) {
+            continue
+        }
+
+        const filePaths = getFilePathsFromParameters(metadata.tool, metadata.parameters)
+        if (isProtected(filePaths, config.protectedFilePatterns)) {
+            continue
+        }
+
+        const signature = createToolSignature(metadata.tool, metadata.parameters)
+        if (!signatureMap.has(signature)) {
+            signatureMap.set(signature, [])
+        }
+        const ids = signatureMap.get(signature)
+        if (ids) {
+            ids.push(id)
+        }
+    }
+
+    // Find duplicates - keep only the most recent (last) in each group
+    const newPruneIds: string[] = []
+
+    for (const [, ids] of signatureMap.entries()) {
+        if (ids.length > 1) {
+            // All except last (most recent) should be pruned
+            const idsToRemove = ids.slice(0, -1)
+            newPruneIds.push(...idsToRemove)
+        }
+    }
+
+    state.stats.totalPruneTokens += getTotalToolTokens(state, newPruneIds)
+
+    if (newPruneIds.length > 0) {
+        for (const id of newPruneIds) {
+            const entry = state.toolParameters.get(id)
+            state.prune.tools.set(id, entry?.tokenCount ?? 0)
+        }
+        logger.debug(`Marked ${newPruneIds.length} duplicate tool calls for pruning`)
+    }
+}
+
+function createToolSignature(tool: string, parameters?: any): string {
+    if (!parameters) {
+        return tool
+    }
+    const normalized = normalizeParameters(parameters)
+    const sorted = sortObjectKeys(normalized)
+    return `${tool}::${JSON.stringify(sorted)}`
+}
+
+function normalizeParameters(params: any): any {
+    if (typeof params !== "object" || params === null) return params
+    if (Array.isArray(params)) return params
+
+    const normalized: any = {}
+    for (const [key, value] of Object.entries(params)) {
+        if (value !== undefined && value !== null) {
+            normalized[key] = value
+        }
+    }
+    return normalized
+}
+
+function sortObjectKeys(obj: any): any {
+    if (typeof obj !== "object" || obj === null) return obj
+    if (Array.isArray(obj)) return obj.map(sortObjectKeys)
+
+    const sorted: any = {}
+    for (const key of Object.keys(obj).sort()) {
+        sorted[key] = sortObjectKeys(obj[key])
+    }
+    return sorted
+}

package/src/dcp/strategies/index.ts

@@ -0,0 +1,4 @@
+export { deduplicate } from "./deduplication"
+export { createPruneTool, createDistillTool, createCompressTool } from "../tools"
+export { supersedeWrites } from "./supersede-writes"
+export { purgeErrors } from "./purge-errors"

package/src/dcp/strategies/purge-errors.ts

@@ -0,0 +1,84 @@
+import type { PluginConfig } from "../config"
+import type { Logger } from "../logger"
+import type { SessionState, WithParts } from "../state"
+import { getFilePathsFromParameters, isProtected } from "../protected-file-patterns"
+import { getTotalToolTokens } from "./utils"
+
+/**
+ * Purge Errors strategy - prunes tool inputs for tools that errored
+ * after they are older than a configurable number of turns.
+ * The error message is preserved, but the (potentially large) inputs
+ * are removed to save context.
+ *
+ * Modifies the session state in place to add pruned tool call IDs.
+ */
+export const purgeErrors = (
+    state: SessionState,
+    logger: Logger,
+    config: PluginConfig,
+    messages: WithParts[],
+): void => {
+    if (state.manualMode && !config.manualMode.automaticStrategies) {
+        return
+    }
+
+    if (!config.strategies.purgeErrors.enabled) {
+        return
+    }
+
+    const allToolIds = state.toolIdList
+    if (allToolIds.length === 0) {
+        return
+    }
+
+    // Filter out IDs already pruned
+    const unprunedIds = allToolIds.filter((id) => !state.prune.tools.has(id))
+
+    if (unprunedIds.length === 0) {
+        return
+    }
+
+    const protectedTools = config.strategies.purgeErrors.protectedTools
+    const turnThreshold = config.strategies.purgeErrors.turns
+
+    const newPruneIds: string[] = []
+
+    for (const id of unprunedIds) {
+        const metadata = state.toolParameters.get(id)
+        if (!metadata) {
+            continue
+        }
+
+        // Skip protected tools
+        if (protectedTools.includes(metadata.tool)) {
+            continue
+        }
+
+        const filePaths = getFilePathsFromParameters(metadata.tool, metadata.parameters)
+        if (isProtected(filePaths, config.protectedFilePatterns)) {
+            continue
+        }
+
+        // Only process error tools
+        if (metadata.status !== "error") {
+            continue
+        }
+
+        // Check if the tool is old enough to prune
+        const turnAge = state.currentTurn - metadata.turn
+        if (turnAge >= turnThreshold) {
+            newPruneIds.push(id)
+        }
+    }
+
+    if (newPruneIds.length > 0) {
+        state.stats.totalPruneTokens += getTotalToolTokens(state, newPruneIds)
+        for (const id of newPruneIds) {
+            const entry = state.toolParameters.get(id)
+            state.prune.tools.set(id, entry?.tokenCount ?? 0)
+        }
+        logger.debug(
+            `Marked ${newPruneIds.length} error tool calls for pruning (older than ${turnThreshold} turns)`,
+        )
+    }
+}

package/src/dcp/strategies/supersede-writes.ts

@@ -0,0 +1,115 @@
+import type { PluginConfig } from "../config"
+import type { Logger } from "../logger"
+import type { SessionState, WithParts } from "../state"
+import { getFilePathsFromParameters, isProtected } from "../protected-file-patterns"
+import { getTotalToolTokens } from "./utils"
+
+/**
+ * Supersede Writes strategy - prunes write tool inputs for files that have
+ * subsequently been read. When a file is written and later read, the original
+ * write content becomes redundant since the current file state is captured
+ * in the read result.
+ *
+ * Modifies the session state in place to add pruned tool call IDs.
+ */
+export const supersedeWrites = (
+    state: SessionState,
+    logger: Logger,
+    config: PluginConfig,
+    messages: WithParts[],
+): void => {
+    if (state.manualMode && !config.manualMode.automaticStrategies) {
+        return
+    }
+
+    if (!config.strategies.supersedeWrites.enabled) {
+        return
+    }
+
+    const allToolIds = state.toolIdList
+    if (allToolIds.length === 0) {
+        return
+    }
+
+    // Filter out IDs already pruned
+    const unprunedIds = allToolIds.filter((id) => !state.prune.tools.has(id))
+    if (unprunedIds.length === 0) {
+        return
+    }
+
+    // Track write tools by file path: filePath -> [{ id, index }]
+    // We track index to determine chronological order
+    const writesByFile = new Map<string, { id: string; index: number }[]>()
+
+    // Track read file paths with their index
+    const readsByFile = new Map<string, number[]>()
+
+    for (let i = 0; i < allToolIds.length; i++) {
+        const id = allToolIds[i]!
+        const metadata = state.toolParameters.get(id)
+        if (!metadata) {
+            continue
+        }
+
+        const filePaths = getFilePathsFromParameters(metadata.tool, metadata.parameters)
+        if (filePaths.length === 0) {
+            continue
+        }
+        const filePath = filePaths[0]!
+
+        if (isProtected(filePaths, config.protectedFilePatterns)) {
+            continue
+        }
+
+        if (metadata.tool === "write") {
+            if (!writesByFile.has(filePath)) {
+                writesByFile.set(filePath, [])
+            }
+            const writes = writesByFile.get(filePath)
+            if (writes) {
+                writes.push({ id, index: i })
+            }
+        } else if (metadata.tool === "read") {
+            if (!readsByFile.has(filePath)) {
+                readsByFile.set(filePath, [])
+            }
+            const reads = readsByFile.get(filePath)
+            if (reads) {
+                reads.push(i)
+            }
+        }
+    }
+
+    // Find writes that are superseded by subsequent reads
+    const newPruneIds: string[] = []
+
+    for (const [filePath, writes] of writesByFile.entries()) {
+        const reads = readsByFile.get(filePath)
+        if (!reads || reads.length === 0) {
+            continue
+        }
+
+        // For each write, check if there's a read that comes after it
+        for (const write of writes) {
+            // Skip if already pruned
+            if (state.prune.tools.has(write.id)) {
+                continue
+            }
+
+            // Check if any read comes after this write
+            const hasSubsequentRead = reads.some((readIndex) => readIndex > write.index)
+            if (hasSubsequentRead) {
+                newPruneIds.push(write.id)
+            }
+        }
+    }
+
+    if (newPruneIds.length > 0) {
+        state.stats.totalPruneTokens += getTotalToolTokens(state, newPruneIds)
+        for (const id of newPruneIds) {
+            const entry = state.toolParameters.get(id)
+            state.prune.tools.set(id, entry?.tokenCount ?? 0)
+        }
+        logger.debug(`Marked ${newPruneIds.length} superseded write tool calls for pruning`)
+    }
+}

package/src/dcp/strategies/utils.ts

@@ -0,0 +1,135 @@
+import type { SessionState, WithParts } from "../state"
+import type { AssistantMessage, UserMessage } from "@opencode-ai/sdk/v2"
+import type { Logger } from "../logger"
+import { countTokens as anthropicCountTokens } from "@anthropic-ai/tokenizer"
+import { getLastUserMessage } from "../shared-utils"
+
+/**
+ * Get current token usage from the last assistant message.
+ * Returns total tokens (input + output + reasoning + cache).
+ */
+export function getCurrentTokenUsage(messages: WithParts[]): number {
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const msg = messages[i]!
+        if (msg.info.role === "assistant") {
+            const assistantInfo = msg.info as AssistantMessage
+            if (assistantInfo.tokens?.output > 0) {
+                const input = assistantInfo.tokens?.input || 0
+                const output = assistantInfo.tokens?.output || 0
+                const reasoning = assistantInfo.tokens?.reasoning || 0
+                const cacheRead = assistantInfo.tokens?.cache?.read || 0
+                const cacheWrite = assistantInfo.tokens?.cache?.write || 0
+                return input + output + reasoning + cacheRead + cacheWrite
+            }
+        }
+    }
+    return 0
+}
+
+export function getCurrentParams(
+    state: SessionState,
+    messages: WithParts[],
+    logger: Logger,
+): {
+    providerId: string | undefined
+    modelId: string | undefined
+    agent: string | undefined
+    variant: string | undefined
+} {
+    const userMsg = getLastUserMessage(messages)
+    if (!userMsg) {
+        logger.debug("No user message found when determining current params")
+        return {
+            providerId: undefined,
+            modelId: undefined,
+            agent: undefined,
+            variant: state.variant,
+        }
+    }
+    const userInfo = userMsg.info as UserMessage
+    const agent: string = userInfo.agent
+    const providerId: string | undefined = userInfo.model.providerID
+    const modelId: string | undefined = userInfo.model.modelID
+    const variant: string | undefined = state.variant ?? userInfo.variant
+
+    return { providerId, modelId, agent, variant }
+}
+
+export function countTokens(text: string): number {
+    if (!text) return 0
+    try {
+        return anthropicCountTokens(text)
+    } catch {
+        return Math.round(text.length / 4)
+    }
+}
+
+export function estimateTokensBatch(texts: string[]): number {
+    if (texts.length === 0) return 0
+    return countTokens(texts.join(" "))
+}
+
+export function extractToolContent(part: any): string[] {
+    const contents: string[] = []
+
+    if (part.tool === "question") {
+        const questions = part.state?.input?.questions
+        if (questions !== undefined) {
+            const content = typeof questions === "string" ? questions : JSON.stringify(questions)
+            contents.push(content)
+        }
+        return contents
+    }
+
+    if (part.tool === "edit" || part.tool === "write") {
+        if (part.state?.input) {
+            const inputContent =
+                typeof part.state.input === "string"
+                    ? part.state.input
+                    : JSON.stringify(part.state.input)
+            contents.push(inputContent)
+        }
+    }
+
+    if (part.state?.status === "completed" && part.state?.output) {
+        const content =
+            typeof part.state.output === "string"
+                ? part.state.output
+                : JSON.stringify(part.state.output)
+        contents.push(content)
+    } else if (part.state?.status === "error" && part.state?.error) {
+        const content =
+            typeof part.state.error === "string"
+                ? part.state.error
+                : JSON.stringify(part.state.error)
+        contents.push(content)
+    }
+
+    return contents
+}
+
+export function countToolTokens(part: any): number {
+    const contents = extractToolContent(part)
+    return estimateTokensBatch(contents)
+}
+
+export function getTotalToolTokens(state: SessionState, toolIds: string[]): number {
+    let total = 0
+    for (const id of toolIds) {
+        const entry = state.toolParameters.get(id)
+        total += entry?.tokenCount ?? 0
+    }
+    return total
+}
+
+export function countMessageTextTokens(msg: WithParts): number {
+    const texts: string[] = []
+    const parts = Array.isArray(msg.parts) ? msg.parts : []
+    for (const part of parts) {
+        if (part.type === "text") {
+            texts.push(part.text)
+        }
+    }
+    if (texts.length === 0) return 0
+    return estimateTokensBatch(texts)
+}

package/src/dcp/tools/compress.ts

@@ -0,0 +1,218 @@
+import { tool } from "@opencode-ai/plugin"
+import type { WithParts, CompressSummary } from "../state"
+import type { PruneToolContext } from "./types"
+import { ensureSessionInitialized } from "../state"
+import { saveSessionState } from "../state/persistence"
+import { loadPrompt } from "../prompts"
+import { getCurrentParams, getTotalToolTokens, countMessageTextTokens } from "../strategies/utils"
+import { findStringInMessages, collectToolIdsInRange, collectMessageIdsInRange } from "./utils"
+import { sendCompressNotification } from "../ui/notification"
+import { prune as applyPruneTransforms } from "../messages/prune"
+
+const COMPRESS_TOOL_DESCRIPTION = loadPrompt("compress-tool-spec")
+const COMPRESS_SUMMARY_PREFIX = "[Compressed conversation block]\n\n"
+
+export function createCompressTool(ctx: PruneToolContext): ReturnType<typeof tool> {
+    return tool({
+        description: COMPRESS_TOOL_DESCRIPTION,
+        args: {
+            topic: tool.schema
+                .string()
+                .describe("Short label (3-5 words) for display - e.g., 'Auth System Exploration'"),
+            content: tool.schema
+                .object({
+                    startString: tool.schema
+                        .string()
+                        .describe("Unique text from conversation marking the beginning of range"),
+                    endString: tool.schema
+                        .string()
+                        .describe("Unique text marking the end of range"),
+                    summary: tool.schema
+                        .string()
+                        .describe("Complete technical summary replacing all content in range"),
+                })
+                .describe("The compression details: boundaries and replacement summary"),
+        },
+        async execute(args, toolCtx) {
+            const { client, state, logger } = ctx
+            const sessionId = toolCtx.sessionID
+
+            await toolCtx.ask({
+                permission: "compress",
+                patterns: ["*"],
+                always: ["*"],
+                metadata: {},
+            })
+
+            const { topic, content } = args
+            const { startString, endString, summary } = content || {}
+
+            if (!topic || typeof topic !== "string") {
+                throw new Error("topic is required and must be a non-empty string")
+            }
+            if (!startString || typeof startString !== "string") {
+                throw new Error("content.startString is required and must be a non-empty string")
+            }
+            if (!endString || typeof endString !== "string") {
+                throw new Error("content.endString is required and must be a non-empty string")
+            }
+            if (!summary || typeof summary !== "string") {
+                throw new Error("content.summary is required and must be a non-empty string")
+            }
+
+            // logger.info("Compress tool invoked")
+            // logger.info(
+            //     JSON.stringify({
+            //         startString: startString?.substring(0, 50) + "...",
+            //         endString: endString?.substring(0, 50) + "...",
+            //         topic: topic,
+            //         summaryLength: summary?.length,
+            //     }),
+            // )
+
+            const messagesResponse = await client.session.messages({
+                path: { id: sessionId },
+            })
+            const messages: WithParts[] = messagesResponse.data || messagesResponse
+
+            await ensureSessionInitialized(
+                client,
+                state,
+                sessionId,
+                logger,
+                messages,
+                ctx.config.manualMode.enabled,
+            )
+
+            const transformedMessages = structuredClone(messages) as WithParts[]
+            applyPruneTransforms(state, logger, ctx.config, transformedMessages)
+
+            const startResult = findStringInMessages(
+                transformedMessages,
+                startString,
+                logger,
+                "startString",
+            )
+            const endResult = findStringInMessages(
+                transformedMessages,
+                endString,
+                logger,
+                "endString",
+            )
+
+            let rawStartIndex = messages.findIndex((m) => m.info.id === startResult.messageId)
+            let rawEndIndex = messages.findIndex((m) => m.info.id === endResult.messageId)
+
+            // If a boundary matched inside a synthetic compress summary message,
+            // resolve it back to the summary's anchor message in the raw messages
+            if (rawStartIndex === -1) {
+                const summary = state.compressSummaries.find((s) => s.summary.includes(startString))
+                if (summary) {
+                    rawStartIndex = messages.findIndex((m) => m.info.id === summary.anchorMessageId)
+                }
+            }
+            if (rawEndIndex === -1) {
+                const summary = state.compressSummaries.find((s) => s.summary.includes(endString))
+                if (summary) {
+                    rawEndIndex = messages.findIndex((m) => m.info.id === summary.anchorMessageId)
+                }
+            }
+
+            if (rawStartIndex === -1 || rawEndIndex === -1) {
+                throw new Error(`Failed to map boundary matches back to raw messages`)
+            }
+
+            if (rawStartIndex > rawEndIndex) {
+                throw new Error(
+                    `startString appears after endString in the conversation. Start must come before end.`,
+                )
+            }
+
+            const containedToolIds = collectToolIdsInRange(messages, rawStartIndex, rawEndIndex)
+
+            const containedMessageIds = collectMessageIdsInRange(
+                messages,
+                rawStartIndex,
+                rawEndIndex,
+            )
+
+            // Remove any existing summaries whose anchors are now inside this range
+            // This prevents duplicate injections when a larger compress subsumes a smaller one
+            const removedSummaries = state.compressSummaries.filter((s) =>
+                containedMessageIds.includes(s.anchorMessageId),
+            )
+            if (removedSummaries.length > 0) {
+                state.compressSummaries = state.compressSummaries.filter(
+                    (s) => !containedMessageIds.includes(s.anchorMessageId),
+                )
+            }
+
+            const compressSummary: CompressSummary = {
+                anchorMessageId: startResult.messageId,
+                summary: COMPRESS_SUMMARY_PREFIX + summary,
+            }
+            state.compressSummaries.push(compressSummary)
+
+            const compressedMessageIds = containedMessageIds.filter(
+                (id) => !state.prune.messages.has(id),
+            )
+            const compressedToolIds = containedToolIds.filter((id) => !state.prune.tools.has(id))
+
+            let textTokens = 0
+            for (const msgId of compressedMessageIds) {
+                const msg = messages.find((m) => m.info.id === msgId)
+                if (msg) {
+                    const tokens = countMessageTextTokens(msg)
+                    textTokens += tokens
+                    state.prune.messages.set(msgId, tokens)
+                }
+            }
+            const toolTokens = getTotalToolTokens(state, compressedToolIds)
+            for (const id of compressedToolIds) {
+                const entry = state.toolParameters.get(id)
+                state.prune.tools.set(id, entry?.tokenCount ?? 0)
+            }
+            const estimatedCompressedTokens = textTokens + toolTokens
+
+            state.stats.pruneTokenCounter += estimatedCompressedTokens
+
+            const rawStartResult = { messageId: startResult.messageId, messageIndex: rawStartIndex }
+            const rawEndResult = { messageId: endResult.messageId, messageIndex: rawEndIndex }
+
+            const currentParams = getCurrentParams(state, messages, logger)
+            await sendCompressNotification(
+                client,
+                logger,
+                ctx.config,
+                state,
+                sessionId,
+                compressedToolIds,
+                compressedMessageIds,
+                topic,
+                summary,
+                rawStartResult,
+                rawEndResult,
+                messages.length,
+                currentParams,
+            )
+
+            state.stats.totalPruneTokens += state.stats.pruneTokenCounter
+            state.stats.pruneTokenCounter = 0
+            state.nudgeCounter = 0
+
+            // logger.info("Compress range created", {
+            //     startMessageId: startResult.messageId,
+            //     endMessageId: endResult.messageId,
+            //     toolIdsRemoved: containedToolIds.length,
+            //     messagesInRange: containedMessageIds.length,
+            //     estimatedTokens: estimatedCompressedTokens,
+            // })
+
+            saveSessionState(state, logger).catch((err) =>
+                logger.error("Failed to persist state", { error: err.message }),
+            )
+
+            return `Compressed ${compressedMessageIds.length} messages (${compressedToolIds.length} tool calls) into summary. The content will be replaced with your summary.`
+        },
+    })
+}