compressed-context 0.0.0-stage → 0.1.1

package/src/compression/decompress.ts

@@ -0,0 +1,292 @@
+/**
+ * 解压：把已压缩块的内容还给模型。
+ *
+ * - `full:false`（默认）展开一层：合并块返回直接子块的摘要；T1 块返回磁盘原文。
+ * - `full:true` 递归到 T1，原文从插件自己的 transcript storage 读取。
+ * - 媒体 bytes/base64 写临时文件；url/ref/缺失内容输出占位符。
+ * - 返回分级：≤ INLINE_MAX 字符内联进 tool result；超出边读边写 `os.tmpdir()` 文件并回路径，
+ *   模型用 read 工具打开。临时文件有 reaper（上限 TEMP_CAP，删最老），防无限堆积。
+ * - copy-paste 语义：不修改任何块状态；块保持折叠，内容只是 tool result 文本。
+ */
+
+import { appendFileSync, unlinkSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { countText } from "../shared/tokens.ts"
+import type { Block, StorageLike } from "./state.ts"
+import { loadState } from "./state.ts"
+import { renderBlock } from "./render.ts"
+import {
+  loadTranscriptManifest,
+  loadTranscriptMessage,
+  type StoredMessage,
+  type StoredPart,
+} from "./transcript-store.ts"
+
+/** 内联返回的默认字符上限；运行时由设置覆盖。 */
+export const INLINE_MAX = 10_000
+/** 临时文件数量默认上限；运行时由设置覆盖。 */
+export const TEMP_CAP = 50
+
+/** decompress 的运行时限制。 */
+export interface DecompressLimits {
+  readonly inlineMax: number
+  readonly tempCap: number
+  readonly showTokens: boolean
+}
+
+const DEFAULT_LIMITS: DecompressLimits = {
+  inlineMax: INLINE_MAX,
+  tempCap: TEMP_CAP,
+  showTokens: true,
+}
+
+/** 解压工具结果；ok 时 inline 或 path 至少一项存在。 */
+export type DecompressResult =
+  | { readonly status: "ok"; readonly inline: string; readonly path?: string; readonly tokens?: number }
+  | { readonly status: "error"; readonly message: string }
+
+const tempFiles: { path: string; mtimeMs: number; sessionID: string }[] = []
+
+/** 输入：上限；输出：清理过期临时文件。预期：超上限删最老，绝不抛错。 */
+function reapTempFiles(cap: number): void {
+  while (tempFiles.length > cap) {
+    tempFiles.sort((a, b) => a.mtimeMs - b.mtimeMs)
+    const oldest = tempFiles.shift()
+    if (oldest === undefined) break
+    try {
+      unlinkSync(oldest.path)
+    } catch {
+      // 已被外部清理或权限问题：跳过，不阻塞解压
+    }
+  }
+}
+
+/** 输入：会话 + 块 id + 正文 + 上限；输出：临时文件路径。 */
+function writeTempFile(sessionID: string, blockId: string, body: string, cap: number): string {
+  const safeId = blockId.replace(/[^a-zA-Z0-9_-]/g, "-")
+  const path = join(tmpdir(), `compressed-context-decompress-${safeId}-${String(Date.now())}.txt`)
+  writeFileSync(path, body, { encoding: "utf8", mode: 0o600 })
+  tempFiles.push({ path, mtimeMs: Date.now(), sessionID })
+  reapTempFiles(cap)
+  return path
+}
+
+/** 输入：会话；输出：无；预期：删除该会话全部解压临时文件。 */
+export function deleteSessionTempFiles(sessionID: string): void {
+  for (let index = tempFiles.length - 1; index >= 0; index -= 1) {
+    const entry = tempFiles[index]
+    if (entry?.sessionID !== sessionID) continue
+    try {
+      unlinkSync(entry.path)
+    } catch {
+      // best effort
+    }
+    tempFiles.splice(index, 1)
+  }
+}
+
+/** 输入：会话 + 块 + 序号 + base64 + mime + 文件名；输出：媒体临时文件路径。 */
+function writeMediaTempFile(
+  sessionID: string,
+  blockID: string,
+  index: number,
+  data: string,
+  filename: string | undefined,
+  cap: number,
+): string {
+  const safeId = `${blockID}-media-${index.toString()}`.replace(/[^a-zA-Z0-9_-]/g, "-")
+  const safeName = (filename ?? "").replace(/[^a-zA-Z0-9._-]/g, "-")
+  const suffix = safeName.length > 0 ? `-${safeName}` : ""
+  const path = join(tmpdir(), `compressed-context-${safeId}${suffix}-${String(Date.now())}.bin`)
+  writeFileSync(path, Buffer.from(data, "base64"), { mode: 0o600 })
+  tempFiles.push({ path, mtimeMs: Date.now(), sessionID })
+  reapTempFiles(cap)
+  return path
+}
+
+/** 输入：任意值；输出：可读 JSON 文本；不可序列化时返回空串。 */
+function stringifyStored(value: unknown): string {
+  if (typeof value === "string") return value
+  if (value === undefined) return ""
+  try {
+    const encoded: unknown = JSON.stringify(value)
+    return typeof encoded === "string" ? encoded : ""
+  } catch {
+    return ""
+  }
+}
+
+/** 输入：落盘 part + 上下文；输出：可读文本；不可还原的媒体输出占位符。 */
+function renderStoredPart(
+  part: StoredPart,
+  sessionID: string,
+  blockID: string,
+  index: number,
+  cap: number,
+): string {
+  if (part.type === "text") return part.text ?? ""
+  if (part.type === "tool-call") {
+    return `[Tool call ${part.name ?? "?"}] ${stringifyStored(part.input)}`
+  }
+  if (part.type === "tool-result") {
+    return `[Tool result ${part.name ?? "?"}] ${stringifyStored(part.result)}`
+  }
+  const source = part.media?.source
+  const filename = part.filename === undefined ? "" : ` "${part.filename}"`
+  const mime = source?.mediaType ?? "unknown"
+  if (source?.type === "base64" && source.data !== undefined && source.data.length > 0) {
+    const path = writeMediaTempFile(sessionID, blockID, index, source.data, part.filename, cap)
+    return `[media ${mime}${filename} saved to: ${path}]`
+  }
+  const kind = source?.type ?? "unknown"
+  return `[unavailable media: ${mime}${filename} (${kind} not stored)]`
+}
+
+/** 输入：落盘消息 + 上下文；输出：带角色前缀的文本。 */
+function renderStoredMessage(
+  message: StoredMessage,
+  sessionID: string,
+  blockID: string,
+  messageIndex: number,
+  cap: number,
+): string {
+  const lines: string[] = []
+  for (const [partIndex, part] of message.content.entries()) {
+    const text = renderStoredPart(
+      part,
+      sessionID,
+      blockID,
+      messageIndex * 1000 + partIndex,
+      cap,
+    )
+    if (text.length === 0) continue
+    lines.push(message.role === "user" ? `[User]\n${text}` : text)
+  }
+  return lines.join("\n")
+}
+
+/**
+ * 输入：storage + 会话 + 块 + 写入口。
+ * 输出：无。
+ * 预期：逐条读取并立即追加，缺 manifest/消息时写占位符，不把整段原文拼成字符串。
+ */
+async function appendStoredTranscript(
+  storage: StorageLike,
+  sessionID: string,
+  block: Block,
+  cap: number,
+  append: (text: string) => void,
+): Promise<void> {
+  const manifest = await loadTranscriptManifest(storage, sessionID, block.id)
+  if (manifest === undefined) {
+    append(`[original transcript for ${block.id} is unavailable]`)
+    return
+  }
+  for (let index = 0; index < manifest.count; index += 1) {
+    const message = await loadTranscriptMessage(storage, sessionID, block.id, index)
+    if (message === undefined) {
+      append(`[missing transcript message ${index.toString()} for ${block.id}]`)
+      continue
+    }
+    const text = renderStoredMessage(message, sessionID, block.id, index, cap)
+    if (text.length > 0) append(text)
+  }
+}
+
+/**
+ * 输入：工具参数（block 必填，full 可选）+ 会话上下文。
+ * 输出：分级结果（内联文本或临时文件路径）。
+ * 预期：从磁盘 transcript 按需读取；缺原文/媒体时输出占位符，不报错。
+ *       超过内联上限时边读边写临时文件，避免把全文常驻内存。
+ */
+export async function resolveDecompress(
+  args: Record<string, unknown>,
+  ctx: { readonly storage: StorageLike },
+  sessionID: string,
+  limits: DecompressLimits = DEFAULT_LIMITS,
+): Promise<DecompressResult> {
+  const raw = args["block"]
+  if (typeof raw !== "string" || raw.trim().length === 0) {
+    return { status: "error", message: "block is required (e.g. b3)" }
+  }
+  const blockId = raw.trim()
+  const full = args["full"] === true
+
+  const state = await loadState(ctx.storage, sessionID)
+  if (state.blocks.length === 0) {
+    return { status: "error", message: `block ${blockId} not found (session has no compressed blocks)` }
+  }
+
+  const byId = new Map(state.blocks.map((block) => [block.id, block]))
+  const target = byId.get(blockId)
+  if (target === undefined) {
+    return { status: "error", message: `block ${blockId} not found` }
+  }
+
+  const chunks: string[] = []
+  let filePath: string | undefined
+  let chars = 0
+  let tokens = 0
+  let first = true
+
+  /** 追加一段文本；超过内联上限后转为临时文件，避免全文常驻内存。 */
+  const append = (text: string): void => {
+    if (text.length === 0) return
+    const separator = first ? "" : "\n\n"
+    first = false
+    const payload = separator + text
+    chars += payload.length
+    tokens += countText(payload)
+    if (filePath !== undefined) {
+      appendFileSync(filePath, payload)
+      return
+    }
+    chunks.push(payload)
+    if (chars > limits.inlineMax) {
+      filePath = writeTempFile(sessionID, blockId, chunks.join(""), limits.tempCap)
+      chunks.length = 0
+    }
+  }
+
+  const visited = new Set<string>()
+  const walk = async (block: Block, depth: number): Promise<void> => {
+    if (visited.has(block.id)) return
+    visited.add(block.id)
+    const sources = block.sourceBlockIds ?? []
+    if (sources.length === 0) {
+      await appendStoredTranscript(ctx.storage, sessionID, block, limits.tempCap, append)
+      return
+    }
+    if (!full && depth > 0) {
+      append(renderBlock(block))
+      return
+    }
+    for (const sourceId of sources) {
+      const source = byId.get(sourceId)
+      if (source === undefined) continue
+      if (!full) {
+        append(renderBlock(source))
+        continue
+      }
+      await walk(source, depth + 1)
+    }
+  }
+
+  await walk(target, 0)
+  if (chars === 0) append("(no restorable content for this block)")
+
+  const tokenLabel = limits.showTokens ? ` · ≈${tokens.toString()} tok` : ""
+  const header = `[Block ${blockId} content${full ? ", full" : ""}${tokenLabel}]`
+
+  if (filePath !== undefined) {
+    return {
+      status: "ok",
+      inline: `${header}\nContent (${chars.toString()} chars) written to: ${filePath}\nUse the read tool to access it.`,
+      path: filePath,
+      tokens,
+    }
+  }
+  return { status: "ok", inline: `${header}\n${chunks.join("")}`, tokens }
+}

package/src/compression/deleted-sessions.ts

@@ -0,0 +1,24 @@
+/**
+ * 已删除会话的内存墓碑。
+ *
+ * 输入：会话删除事件。
+ * 输出：无。
+ * 预期：删除后到达的 saveState / saveTranscript 直接跳过，避免把状态复活；
+ *       进程重启后墓碑消失，由启动 GC 清理孤儿数据。
+ */
+const deleted = new Set<string>()
+
+/** 输入：sessionID；输出：无；预期：标记会话已删除。 */
+export function markSessionDeleted(sessionID: string): void {
+  deleted.add(sessionID)
+}
+
+/** 输入：sessionID；输出：是否已删除。 */
+export function isSessionDeleted(sessionID: string): boolean {
+  return deleted.has(sessionID)
+}
+
+/** 输入：无；输出：无；预期：测试或多租户重置时清空墓碑。 */
+export function clearDeletedSessions(): void {
+  deleted.clear()
+}

package/src/compression/distill.ts

@@ -0,0 +1,220 @@
+import type { Plugin } from "@opencode/plugin"
+
+import type { SummaryFormat, SummaryLanguage } from "../shared/settings.ts"
+import { countText } from "../shared/tokens.ts"
+import { activeBlocks, renderAll, renderBlock } from "./render.ts"
+import { nextBlockId, type Block, type CompactionPhase, type State, type Tier } from "./state.ts"
+import { summaryChars, summaryGoal, summaryTitle } from "./summary.ts"
+import {
+  buildMergeInstruction,
+  sessionGenerator,
+  summarizeChoiceInstruction,
+  type SummaryChoiceCandidate,
+} from "./summarize.ts"
+import { minSummaryChars, validateSummary } from "./validate.ts"
+
+/** 一个合法的相邻合并候选。 */
+export interface MergeCandidate {
+  readonly left: Block
+  readonly right: Block
+  readonly tier: Tier
+}
+
+export interface DistillOnceInput {
+  readonly ctx: Plugin.Context
+  readonly sessionID: string
+  readonly state: State
+  /** 上一层还没提交的块；T2/T3 钩子里可能来自 pending。 */
+  readonly pending: Block | undefined
+  readonly budgetTokens: number
+  readonly summaryBlockChars?: number
+  readonly summaryTotalChars?: number
+  readonly summaryLanguage?: SummaryLanguage
+  readonly summaryFormat?: SummaryFormat
+  readonly phase: Exclude<CompactionPhase, undefined>
+  readonly showTimeline?: boolean
+  readonly showBlockTokens?: boolean
+}
+
+export interface DistillOnceResult {
+  readonly blocks: readonly Block[]
+  readonly pending: Block | undefined
+  readonly summary: string
+  readonly merged: boolean
+  readonly nextPhase?: CompactionPhase
+  readonly error?: string
+}
+
+/** 输入：active 链；输出：所有保持 tier 单调不增的相邻合并候选。 */
+export function mergeCandidates(blocks: readonly Block[]): readonly MergeCandidate[] {
+  const candidates: MergeCandidate[] = []
+  for (let index = 0; index < blocks.length - 1; index += 1) {
+    const left = blocks[index]
+    const right = blocks[index + 1]
+    if (left === undefined || right === undefined) continue
+    const tier = Math.min(3, Math.max(left.tier, right.tier) + 1) as Tier
+    const previous = blocks[index - 1]
+    const next = blocks[index + 2]
+    if (previous !== undefined && previous.tier < tier) continue
+    if (next !== undefined && tier < next.tier) continue
+    candidates.push({ left, right, tier })
+  }
+  return candidates
+}
+
+/** 输入：T1 后的摘要链；输出：是否要自动排 T2。 */
+export function planT1Phase(
+  summary: string,
+  active: readonly Block[],
+  budgetTokens: number,
+  totalChars: number = Number.POSITIVE_INFINITY,
+): CompactionPhase | undefined {
+  return (countText(summary) > budgetTokens || summary.length > totalChars) &&
+    mergeCandidates(active).length > 0
+    ? "t2"
+    : undefined
+}
+
+/**
+ * 输入：一次蒸馏结果 + active 链 + 预算。
+ * 输出：是否已经无法继续压缩（仍超预算，且没有任何合法合并候选）。
+ * 预期：与层级无关——全 T3 只要还有两块可合并，或链本身没超预算，都不算终止。
+ */
+export function isTerminal(
+  result: DistillOnceResult,
+  active: readonly Block[],
+  budgetTokens: number,
+  totalChars: number,
+): boolean {
+  if (result.merged || result.error !== undefined) return false
+  if (countText(result.summary) <= budgetTokens && result.summary.length <= totalChars) return false
+  return mergeCandidates(active).length === 0
+}
+
+/** 输入：块；输出：候选列表里的简短描述。 */
+function describe(block: Block): string {
+  const goal = summaryGoal(block.summary).replace(/\s+/g, " ").slice(0, 160)
+  return `${block.id} T${block.tier.toString()} "${summaryTitle(block.summary)}" — ${goal}`
+}
+
+/** 输入：候选；输出：这两块的完整 transcript（只用于本地校验，不发给模型）。 */
+function pairTranscript(candidate: MergeCandidate): string {
+  return `${renderBlock(candidate.left)}\n\n${renderBlock(candidate.right)}`
+}
+
+/**
+ * 输入：当前状态 + 上一层 pending + 预算 + 本次是 T2 还是 T3。
+ * 输出：只合并一对后的 blocks / pending / summary。
+ * 预期：每层只合一次；模型在会话前缀后只看到追加的候选指令。
+ */
+export async function distillOnce(input: DistillOnceInput): Promise<DistillOnceResult> {
+  const renderOptions = {
+    showTimeline: input.showTimeline === true,
+    showBlockTokens: input.showBlockTokens === true,
+  }
+  const active = activeBlocks(input.state.blocks, input.pending)
+  const currentSummary = renderAll(input.state.blocks, input.pending, renderOptions)
+  const totalChars = input.summaryTotalChars ?? Number.POSITIVE_INFINITY
+  if (
+    active.length < 2 ||
+    (countText(currentSummary) <= input.budgetTokens && currentSummary.length <= totalChars)
+  ) {
+    return { blocks: input.state.blocks, pending: input.pending, summary: currentSummary, merged: false }
+  }
+
+  const candidates = mergeCandidates(active)
+  if (candidates.length === 0) {
+    return { blocks: input.state.blocks, pending: input.pending, summary: currentSummary, merged: false }
+  }
+
+  const choices: SummaryChoiceCandidate[] = candidates.map((candidate, index) => ({
+    index: index + 1,
+    description: `${describe(candidate.left)} + ${describe(candidate.right)} → T${candidate.tier.toString()}`,
+    sourceChars: pairTranscript(candidate).length,
+    tier: candidate.tier,
+  }))
+
+  const generator = sessionGenerator(input.ctx.session, input.sessionID)
+  let failure = "distill summary failed"
+  const promptOptions = {
+    ...(input.summaryBlockChars === undefined ? {} : { maxChars: input.summaryBlockChars }),
+    ...(input.summaryTotalChars === undefined ? {} : { totalChars: input.summaryTotalChars }),
+    ...(input.summaryLanguage === undefined ? {} : { language: input.summaryLanguage }),
+    format: input.summaryFormat ?? "json",
+  }
+  const picked = await summarizeChoiceInstruction({
+    prompt: buildMergeInstruction(choices, promptOptions),
+    candidates: choices,
+    generate: generator,
+    ...(input.summaryBlockChars === undefined ? {} : { maxChars: input.summaryBlockChars }),
+    format: input.summaryFormat ?? "json",
+    onFailure: (reason) => {
+      failure = `distill summary failed: ${reason}`
+    },
+  })
+  if (picked === undefined) {
+    return {
+      blocks: input.state.blocks,
+      pending: input.pending,
+      summary: currentSummary,
+      merged: false,
+      error: failure,
+    }
+  }
+
+  const chosen = candidates[picked.candidate.index - 1]
+  if (chosen === undefined) {
+    return {
+      blocks: input.state.blocks,
+      pending: input.pending,
+      summary: currentSummary,
+      merged: false,
+      error: "model picked an unknown candidate",
+    }
+  }
+
+  const validation = validateSummary(
+    picked.summary,
+    pairTranscript(chosen).length,
+    minSummaryChars(chosen.tier),
+    input.summaryBlockChars,
+  )
+  if (!validation.ok) {
+    return {
+      blocks: input.state.blocks,
+      pending: input.pending,
+      summary: currentSummary,
+      merged: false,
+      error: validation.message,
+    }
+  }
+
+  // 上一层 pending 先转成已落盘块，新合并块保持 pending，交给 compaction 事件转正。
+  const working = input.pending === undefined ? [...input.state.blocks] : [...input.state.blocks, input.pending]
+  const merged: Block = {
+    id: nextBlockId(working),
+    tier: chosen.tier,
+    summary: picked.summary,
+    fromId: chosen.left.fromId,
+    toId: chosen.right.toId,
+    tokensBefore: chosen.left.tokensAfter + chosen.right.tokensAfter,
+    tokensAfter: Math.ceil(summaryChars(picked.summary) / 4),
+    at: Math.max(chosen.left.at, chosen.right.at),
+    order: chosen.left.order,
+    sourceBlockIds: [chosen.left.id, chosen.right.id],
+  }
+  const summary = renderAll(working, merged, renderOptions)
+  const stillOver =
+    countText(summary) > input.budgetTokens || summary.length > totalChars
+  // 只要还超预算且还有合法候选就继续合并；T3+T3 产出仍是 T3，直到没有候选或压进预算。
+  const nextPhase: CompactionPhase | undefined =
+    stillOver && mergeCandidates(activeBlocks(working, merged)).length > 0 ? "t3" : undefined
+
+  return {
+    blocks: working,
+    pending: merged,
+    summary,
+    merged: true,
+    ...(nextPhase === undefined ? {} : { nextPhase }),
+  }
+}