@chroxify/eunoe 0.0.0-stage → 0.1.0

package/src/context/strategies.ts

@@ -0,0 +1,89 @@
+import { CLEARED_INPUT_KEPT, CLEARED_INPUT_THRESHOLD } from "./constants"
+import { blocks, withoutCacheControl } from "./content"
+import { compactionNotice, truncationNotice } from "./prompts"
+import { hasContent, isHarness, reduceTurn, splitTurns } from "./turns"
+import type { Block, Body, CompactOptions, Message } from "./types"
+
+export function trimMessages(messages: Message[]): Message[] {
+  const { preamble, turns } = splitTurns(messages)
+  if (turns.length < 2) return messages
+  return [...preamble, ...turns.slice(0, -1).flatMap(reduceTurn), ...turns.at(-1)!]
+}
+
+export function compactMessages(
+  messages: Message[],
+  cutIndex: number,
+  options: CompactOptions,
+): Message[] {
+  const { preamble, turns } = splitTurns(messages)
+  if (cutIndex <= 0 || cutIndex >= turns.length) return messages
+  const keptFrom = Math.max(1, cutIndex - options.keepTurns)
+  const notice = compactionNotice({ omitted: keptFrom - 1 + (options.omittedBefore ?? 0), kept: cutIndex - keptFrom, guide: options.guide })
+  const removed = [...turns[0].slice(1), ...turns.slice(1, keptFrom).flat()]
+  const carried = removed
+    .filter((m) => isHarness(m) && hasContent(m))
+    .flatMap((m) => blocks(m).filter((b) => b.type === "text"))
+    .map((b) => ({ type: "text", text: `<system-reminder>\n${b.text}\n</system-reminder>` }))
+  return [
+    ...preamble,
+    turns[0][0],
+    { role: "user", content: [{ type: "text", text: notice }, ...carried] },
+    ...turns.slice(keptFrom, cutIndex).flatMap(reduceTurn),
+    ...turns.slice(cutIndex).flat(),
+  ]
+}
+
+export function tailMessages(messages: Message[], from: number, guide: string | null): Message[] {
+  const { preamble, turns } = splitTurns(messages)
+  if (turns.length === 0 || from <= 1) return messages
+  const notice = truncationNotice({ omitted: from - 1, kept: turns.length - from, guide })
+  return [
+    ...preamble,
+    turns[0][0],
+    { role: "user", content: [{ type: "text", text: notice }] },
+    ...turns.slice(from).flat(),
+  ]
+}
+
+export function countToolResults(messages: Message[], from: number): number {
+  return messages.slice(from).reduce((total, m) => total + (m.role === "user" ? blocks(m).filter((b) => b.type === "tool_result").length : 0), 0)
+}
+
+export function clearToolResults(messages: Message[], from: number, count: number, placeholder: string): Message[] {
+  let left = count
+  const cleared = new Set<string>()
+  const results = messages.map((message, index) => {
+    if (left <= 0 || index < from || message.role !== "user" || typeof message.content === "string") return message
+    if (!message.content.some((b) => b.type === "tool_result")) return message
+    return {
+      ...message,
+      content: message.content.map((b) => {
+        if (b.type !== "tool_result" || left <= 0) return b
+        left -= 1
+        cleared.add(b.tool_use_id)
+        return { type: "tool_result", tool_use_id: b.tool_use_id, ...(b.is_error ? { is_error: true } : {}), content: placeholder }
+      }),
+    }
+  })
+  return results.map((message) => {
+    if (message.role !== "assistant" || typeof message.content === "string") return message
+    if (!message.content.some((b) => b.type === "tool_use" && cleared.has(b.id))) return message
+    return { ...message, content: message.content.map((b) => (b.type === "tool_use" && cleared.has(b.id) ? { ...b, input: shortenInput(b.input) } : b)) }
+  })
+}
+
+function shortenInput(input: unknown): unknown {
+  if (typeof input === "string") return input.length > CLEARED_INPUT_THRESHOLD ? `${input.slice(0, CLEARED_INPUT_KEPT)}… [${input.length - CLEARED_INPUT_KEPT} chars cleared; in the transcript]` : input
+  if (Array.isArray(input)) return input.map(shortenInput)
+  if (input && typeof input === "object") return Object.fromEntries(Object.entries(input).map(([k, v]) => [k, shortenInput(v)]))
+  return input
+}
+
+export function appendSystem(system: Body["system"], text: string): Block[] {
+  const list: Block[] = typeof system === "string" ? [{ type: "text", text: system }] : [...(system ?? [])]
+  const lastCached = list.findLastIndex((b) => b.cache_control)
+  const cacheControl = lastCached >= 0 ? list[lastCached].cache_control : undefined
+  if (lastCached >= 0) list[lastCached] = withoutCacheControl(list[lastCached])
+  list.push({ type: "text", text, ...(cacheControl ? { cache_control: cacheControl } : {}) })
+  return list
+}

package/src/context/turns.ts

@@ -0,0 +1,82 @@
+import { SUMMARY_ARGUMENT_CHARS, SUMMARY_CALLS_SHOWN, SUMMARY_COMMAND_CHARS, SUMMARY_LAST_WORDS_CHARS } from "./constants"
+import { blocks, clean, clip, withoutCacheControl } from "./content"
+import type { Block, Message } from "./types"
+
+export function isPrompt(message: Message) {
+  if (message.role !== "user") return false
+  const content = blocks(message)
+  return !content.some((block) => block.type === "tool_result")
+    && content.some((block) => block.type === "text" || block.type === "image" || block.type === "document")
+}
+
+export function splitTurns(messages: Message[]): { preamble: Message[]; turns: Message[][] } {
+  const preamble: Message[] = []
+  const turns: Message[][] = []
+  for (const message of messages) {
+    if (isPrompt(message) && !(turns.length > 0 && turns.at(-1)!.every((m) => m.role === "user"))) {
+      turns.push([message])
+    } else if (turns.length > 0) {
+      turns.at(-1)!.push(message)
+    } else {
+      preamble.push(message)
+    }
+  }
+  return { preamble, turns }
+}
+
+export function turnFingerprint(turn: Message[]): string {
+  const hasher = new Bun.CryptoHasher("sha256")
+  hasher.update(JSON.stringify(blocks(turn[0]).map(withoutCacheControl)))
+  return hasher.digest("hex").slice(0, 24)
+}
+
+export function isHarness(message: Message) {
+  return message.role === "system"
+}
+
+export function hasContent(message: Message) {
+  return blocks(message).some((block) => block.type !== "text" || String(block.text ?? "").trim())
+}
+
+function mergedHarness(turn: Message[]): Message[] {
+  const content = turn.filter((m) => isHarness(m) && hasContent(m)).flatMap((m) => blocks(m).map(withoutCacheControl))
+  return content.length > 0 ? [{ role: "system", content }] : []
+}
+
+function describeToolCall(block: Block): string {
+  const input = block.input ?? {}
+  if (typeof input.command === "string") return `${block.name} \`${clip(input.command, SUMMARY_COMMAND_CHARS)}\``
+  if (typeof input.file_path === "string") return `${block.name} ${input.file_path}`
+  if (typeof input.pattern === "string") return `${block.name} "${clip(input.pattern, SUMMARY_ARGUMENT_CHARS)}"`
+  return `${block.name} ${clip(JSON.stringify(input), SUMMARY_ARGUMENT_CHARS)}`
+}
+
+function interruptedSummary(turn: Message[]): string {
+  const assistant = turn.filter((m) => m.role === "assistant")
+  const lastWords = [...assistant].reverse()
+    .map((m) => blocks(m).filter((b) => b.type === "text").map((b) => String(b.text ?? "")).join("\n").trim())
+    .find(Boolean)
+  const calls = assistant.flatMap((m) => blocks(m).filter((b) => b.type === "tool_use")).map(describeToolCall)
+  const shown = calls.slice(-SUMMARY_CALLS_SHOWN)
+  return [
+    "[This turn was interrupted before a final reply.]",
+    lastWords ? `Last thing you said: ${lastWords.length > SUMMARY_LAST_WORDS_CHARS ? `${lastWords.slice(0, SUMMARY_LAST_WORDS_CHARS)}…` : lastWords}` : "",
+    calls.length > 0 ? `Tool calls (${calls.length}${calls.length > shown.length ? `, last ${shown.length} shown` : ""}): ${shown.join("; ")}` : "",
+    "Their output is in the full transcript.",
+  ].filter(Boolean).join("\n")
+}
+
+export function reduceTurn(turn: Message[]): Message[] {
+  const opening: Message[] = []
+  for (let i = 0; i < turn.length && turn[i].role !== "assistant"; i += 1) {
+    if (turn[i].role === "user") opening.push(clean(turn[i]))
+  }
+  opening.push(...mergedHarness(turn))
+  const last = [...turn].reverse().find((m) => m.role !== "system")!
+  const content = last.role === "assistant" ? blocks(last) : []
+  const text = content.filter((b) => b.type === "text" && String(b.text ?? "").trim())
+  const final = text.length > 0 && !content.some((b) => b.type === "tool_use")
+    ? text.map((b) => ({ type: "text", text: b.text }))
+    : [{ type: "text", text: interruptedSummary(turn) }]
+  return [...opening, { role: "assistant", content: final }]
+}

package/src/context/types.ts

@@ -0,0 +1,24 @@
+export type Block = { type: string; [key: string]: any }
+
+export type Message = { role: string; content: string | Block[] }
+
+export type Body = {
+  model?: string
+  system?: string | Block[]
+  messages: Message[]
+  tools?: unknown[]
+  metadata?: { user_id?: string }
+  [key: string]: any
+}
+
+export interface CompactOptions {
+  keepTurns: number
+  guide: string | null
+  omittedBefore?: number
+}
+
+export interface NoticeCounts {
+  omitted: number
+  kept: number
+  guide: string | null
+}

package/src/proxy/compaction.ts

@@ -0,0 +1,93 @@
+import { CLEARED_OUTPUT } from "../context/prompts"
+import { clearToolResults, compactMessages, countToolResults } from "../context/strategies"
+import { splitTurns, turnFingerprint } from "../context/turns"
+import { DEFAULT_RATIO, PROTECTED_OUTPUTS } from "./constants"
+import { estimator, targetSize } from "./estimate"
+import { resetCut } from "./state"
+import type { CompactInput, CompactResult } from "./types"
+
+export function compact({ body, config, state, window, guide, mayCompact, omittedBefore = 0 }: CompactInput): CompactResult {
+  const { turns } = splitTurns(body.messages)
+  let stateChanged = false
+
+  let cutIndex = state.cut ? turns.findIndex((turn) => turnFingerprint(turn) === state.cut) : -1
+  if (state.cut && cutIndex === -1) {
+    resetCut(state)
+    stateChanged = true
+  }
+  if (cutIndex <= 0 && config.eval?.forceCut && turns.length >= 3) {
+    let forced = turns.length - 2
+    while (forced > 1 && JSON.stringify(turns[forced][0].content).includes("[Request interrupted")) forced -= 1
+    cutIndex = forced
+    state.cut = turnFingerprint(turns[forced])
+    stateChanged = true
+  }
+
+  const configuredKeep = config.keepTurns === "all" ? Number.MAX_SAFE_INTEGER : config.keepTurns
+  const liveLength = () => turns.slice(Math.max(cutIndex, 0)).flat().length
+  const compacted = () => cutIndex > 0
+    ? compactMessages(body.messages, cutIndex, { keepTurns: state.keep ?? configuredKeep, guide, omittedBefore })
+    : body.messages
+  const build = () => {
+    const messages = compacted()
+    return state.cleared ? clearToolResults(messages, messages.length - liveLength(), state.cleared, CLEARED_OUTPUT) : messages
+  }
+
+  const size = estimator(body, state.ratio ?? DEFAULT_RATIO)
+  const limit = window ? config.compactAt * window : Number.POSITIVE_INFINITY
+  const target = window ? targetSize(size, window) : Number.POSITIVE_INFINITY
+
+  let messages = build()
+  let estimate = size.of(messages)
+  const actions: string[] = []
+  const current = turns.length - 1
+
+  if (mayCompact && estimate > limit) {
+    if (current > 0 && current > cutIndex) {
+      cutIndex = current
+      resetCut(state)
+      state.cut = turnFingerprint(turns[current])
+      actions.push("cut")
+      messages = build()
+      estimate = size.of(messages)
+    }
+    while (estimate > target) {
+      const keep = Math.min(state.keep ?? configuredKeep, cutIndex - 1)
+      if (cutIndex > 0 && keep > 0) {
+        state.keep = keep - 1
+        actions.push(`keep=${state.keep}`)
+      } else {
+        const unclearedLive = compacted()
+        const total = countToolResults(unclearedLive, unclearedLive.length - liveLength())
+        const remaining = total - (state.cleared ?? 0) - PROTECTED_OUTPUTS
+        if (remaining <= 0) {
+          actions.push("stuck")
+          break
+        }
+        state.cleared = (state.cleared ?? 0) + Math.ceil(remaining / 2)
+        actions.push(`cleared=${state.cleared}`)
+      }
+      messages = build()
+      estimate = size.of(messages)
+    }
+    stateChanged = true
+  }
+
+  return {
+    messages,
+    stateChanged,
+    note: {
+      action: cutIndex > 0 ? "compact" : "full",
+      compacted: actions.includes("cut"),
+      actions: actions.length ? actions : undefined,
+      cutTurn: cutIndex > 0 ? cutIndex : null,
+      keep: state.keep,
+      cleared: state.cleared,
+      turns: turns.length,
+      messagesIn: body.messages.length,
+      messagesOut: messages.length,
+      estimate,
+      window,
+    },
+  }
+}

package/src/proxy/constants.ts

@@ -0,0 +1,19 @@
+export const HEALTH_PATH = "/_eunoe/health"
+export const SESSION_HEADER = "x-claude-code-session-id"
+export const TRANSCRIPT_HEADER = "x-eunoe-transcript"
+export const OMITTED_BEFORE_HEADER = "x-eunoe-omitted-before"
+
+export const TARGET_SHARE = 0.3
+export const PROTECTED_OUTPUTS = 5
+
+export const DEFAULT_RATIO = 1 / 3.6
+export const MEDIA_CHARS = 5_800
+export const MEDIA_MIN_LENGTH = 2_000
+
+export const MODEL_HEADERS = ["authorization", "x-api-key", "anthropic-version", "anthropic-beta"]
+export const ANTHROPIC_VERSION = "2023-06-01"
+export const MODEL_LOOKUP_TIMEOUT_MS = 5_000
+
+export const MAX_EVENT_BUFFER = 4_000_000
+export const LOGGED_ERROR_CHARS = 500
+export const IDLE_TIMEOUT_SECONDS = 255

package/src/proxy/estimate.ts

@@ -0,0 +1,26 @@
+import type { Body, Message } from "../context/types"
+import { MEDIA_CHARS, MEDIA_MIN_LENGTH, TARGET_SHARE } from "./constants"
+import type { Estimator } from "./types"
+
+export function sizeOf(body: Pick<Body, "system" | "tools" | "messages">): number {
+  let media = 0
+  const json = JSON.stringify({ system: body.system, tools: body.tools, messages: body.messages }, (key, value) => {
+    if (key === "data" && typeof value === "string" && value.length > MEDIA_MIN_LENGTH) {
+      media += 1
+      return ""
+    }
+    return value
+  })
+  return json.length + media * MEDIA_CHARS
+}
+
+export function estimator(body: Body, ratio: number): Estimator {
+  return {
+    of: (messages: Message[]) => Math.round(sizeOf({ ...body, messages }) * ratio),
+    fixed: Math.round(sizeOf({ ...body, messages: [] }) * ratio),
+  }
+}
+
+export function targetSize(size: Estimator, window: number): number {
+  return size.fixed + TARGET_SHARE * Math.max(0, window - size.fixed)
+}

package/src/proxy/log.ts

@@ -0,0 +1,9 @@
+import { appendFileSync, mkdirSync } from "node:fs"
+import { EUNOE_DIR, paths } from "../config/constants"
+
+export function logRequest(record: Record<string, unknown>) {
+  try {
+    mkdirSync(EUNOE_DIR, { recursive: true })
+    appendFileSync(paths.requests, JSON.stringify({ ts: new Date().toISOString(), ...record }) + "\n")
+  } catch {}
+}

package/src/proxy/rewrite.ts

@@ -0,0 +1,72 @@
+import { searchOnlyInstructions, trimInstructions, TRANSCRIPT_FALLBACK } from "../context/prompts"
+import { appendSystem, tailMessages, trimMessages } from "../context/strategies"
+import { splitTurns, turnFingerprint } from "../context/turns"
+import type { Body } from "../context/types"
+import { searchGuide } from "../transcript/search"
+import { compact } from "./compaction"
+import { DEFAULT_RATIO } from "./constants"
+import { estimator, targetSize } from "./estimate"
+import { loadState, saveState } from "./state"
+import type { Rewrite, RewriteInput } from "./types"
+
+const pass = (body: Body): Rewrite => ({ body, threadKey: null, note: { action: "pass" } })
+
+const isAgentThread = (body: Body) => Array.isArray(body.tools) && body.tools.length > 0
+
+export function rewrite(input: RewriteInput): Rewrite {
+  const { body, sessionId, config, transcript } = input
+  if (config.mode === "off" || !sessionId || !Array.isArray(body.messages) || body.messages.length === 0) return pass(body)
+  const { turns } = splitTurns(body.messages)
+  if (turns.length === 0) return pass(body)
+  const threadKey = `${sessionId}-${turnFingerprint(turns[0])}`
+  const guide = transcript ? searchGuide(transcript) : null
+
+  switch (config.eval?.strategy) {
+    case "guide": {
+      if (!isAgentThread(body)) return pass(body)
+      return {
+        body: guide ? { ...body, system: appendSystem(body.system, searchOnlyInstructions(guide)) } : body,
+        threadKey,
+        note: { action: "guide", turns: turns.length, messagesIn: body.messages.length },
+      }
+    }
+    case "tail": {
+      if (!isAgentThread(body)) return pass(body)
+      const size = estimator(body, loadState(threadKey).ratio ?? DEFAULT_RATIO)
+      let from = 1
+      if (config.eval.tailTurns) {
+        from = Math.max(1, turns.length - config.eval.tailTurns)
+      } else if (input.window) {
+        const target = targetSize(size, input.window)
+        from = turns.length - 1
+        while (from > 1 && size.of(tailMessages(body.messages, from - 1, guide)) <= target) from -= 1
+      }
+      const messages = tailMessages(body.messages, from, guide)
+      return {
+        body: { ...body, messages },
+        threadKey,
+        note: { action: "tail", keptTurns: turns.length - from, turns: turns.length, messagesIn: body.messages.length, messagesOut: messages.length, estimate: size.of(messages) },
+      }
+    }
+  }
+
+  if (config.mode === "trim") {
+    if (!isAgentThread(body)) return pass(body)
+    const messages = trimMessages(body.messages)
+    return {
+      body: { ...body, system: appendSystem(body.system, trimInstructions(guide ?? TRANSCRIPT_FALLBACK)), messages },
+      threadKey,
+      note: { action: "trim", turns: turns.length, messagesIn: body.messages.length, messagesOut: messages.length },
+    }
+  }
+
+  if (body.messages.length < 2) return { body, threadKey, note: { action: "full", turns: turns.length } }
+  const state = loadState(threadKey)
+  const result = compact({ body, config, state, window: input.window, guide, mayCompact: input.mayCompact, omittedBefore: input.omittedBefore })
+  if (result.stateChanged) saveState(threadKey, state)
+  return {
+    body: result.messages === body.messages ? body : { ...body, messages: result.messages },
+    threadKey,
+    note: result.note,
+  }
+}

package/src/proxy/server.ts

@@ -0,0 +1,120 @@
+import path from "node:path"
+import { claudeConfigDirs, loadConfig } from "../config"
+import type { Config } from "../config/types"
+import type { Body } from "../context/types"
+import { findSessionFile } from "../transcript/locate"
+import { prepareTranscript } from "../transcript/search"
+import { VERSION } from "../version"
+import { HEALTH_PATH, IDLE_TIMEOUT_SECONDS, LOGGED_ERROR_CHARS, OMITTED_BEFORE_HEADER, SESSION_HEADER, TRANSCRIPT_HEADER } from "./constants"
+import { sizeOf } from "./estimate"
+import { logRequest } from "./log"
+import { rewrite } from "./rewrite"
+import { loadState, saveState } from "./state"
+import type { Health, Note } from "./types"
+import { inputTokens, watchUsage } from "./usage"
+import { resolveWindow } from "./window"
+
+export function sessionIdOf(headers: Headers, body: Body): string | null {
+  const header = headers.get(SESSION_HEADER)
+  if (header) return header
+  try {
+    return JSON.parse(body.metadata?.user_id ?? "{}").session_id ?? null
+  } catch {
+    return null
+  }
+}
+
+function forwardableHeaders(headers: Headers) {
+  const out = new Headers(headers)
+  out.delete("content-encoding")
+  out.delete("content-length")
+  return out
+}
+
+async function prepare(request: Request, body: Body, config: Config, isCountTokens: boolean) {
+  const sessionId = sessionIdOf(request.headers, body)
+  const trusted = config.eval?.trustHeaders === true
+  const transcriptOverride = trusted ? request.headers.get(TRANSCRIPT_HEADER) : null
+  const sessionFile = transcriptOverride ?? (sessionId ? findSessionFile(sessionId, claudeConfigDirs(config)) : null)
+  const transcript = sessionFile && sessionId
+    ? prepareTranscript(config.search, sessionFile, transcriptOverride ? path.basename(transcriptOverride, ".jsonl") : sessionId)
+    : null
+  const needsWindow = config.mode === "compact" || config.eval?.strategy === "tail"
+  const window = config.window ?? (needsWindow ? await resolveWindow(body.model, request.headers, config.upstream) : null)
+  return rewrite({
+    body,
+    sessionId,
+    config,
+    transcript,
+    window,
+    mayCompact: !isCountTokens,
+    omittedBefore: trusted ? Number(request.headers.get(OMITTED_BEFORE_HEADER) ?? 0) : 0,
+  })
+}
+
+export async function handle(request: Request): Promise<Response> {
+  const config = loadConfig()
+  const url = new URL(request.url)
+  if (url.pathname === HEALTH_PATH) return Response.json({ ok: true, version: VERSION, mode: config.mode, pid: process.pid } satisfies Health)
+  const headers = new Headers(request.headers)
+  headers.delete("host")
+  headers.delete("content-length")
+  headers.delete("accept-encoding")
+
+  let bodyText = request.method === "GET" || request.method === "HEAD" ? undefined : await request.text()
+  const isMessages = request.method === "POST" && url.pathname.startsWith("/v1/messages")
+  const isCountTokens = url.pathname.includes("count_tokens")
+  let note: Note = { action: "pass" }
+  let threadKey: string | null = null
+  let sentSize = 0
+
+  if (bodyText && isMessages) {
+    try {
+      const body = JSON.parse(bodyText) as Body
+      const result = await prepare(request, body, config, isCountTokens)
+      note = result.note
+      threadKey = result.threadKey
+      if (result.body !== body) bodyText = JSON.stringify(result.body)
+      sentSize = sizeOf(result.body)
+    } catch (error) {
+      note = { action: "error-passthrough", error: String(error) }
+    }
+  }
+
+  const upstream = await fetch(config.upstream + url.pathname + url.search, { method: request.method, headers, body: bodyText })
+
+  if (upstream.status >= 400 && note.action !== "pass") {
+    const errorText = await upstream.text()
+    logRequest({ path: url.pathname, status: upstream.status, thread: threadKey, ...note, error: errorText.slice(0, LOGGED_ERROR_CHARS) })
+    return new Response(errorText, { status: upstream.status, statusText: upstream.statusText, headers: forwardableHeaders(upstream.headers) })
+  }
+
+  const response = new Response(upstream.body, { status: upstream.status, statusText: upstream.statusText, headers: forwardableHeaders(upstream.headers) })
+  if (note.action === "pass" || !isMessages || isCountTokens) {
+    if (note.action !== "pass") logRequest({ path: url.pathname, status: upstream.status, ...note })
+    return response
+  }
+
+  return watchUsage(response, (usage) => {
+    const input = inputTokens(usage)
+    logRequest({
+      path: url.pathname,
+      status: upstream.status,
+      thread: threadKey,
+      ...note,
+      input,
+      cacheRead: usage.cache_read_input_tokens ?? 0,
+      cacheWrite: usage.cache_creation_input_tokens ?? 0,
+      uncached: usage.input_tokens ?? 0,
+    })
+    if (threadKey && input > 0 && sentSize > 0) {
+      const state = loadState(threadKey)
+      state.ratio = input / sentSize
+      saveState(threadKey, state)
+    }
+  })
+}
+
+export function serve(port: number) {
+  return Bun.serve({ port, hostname: "127.0.0.1", idleTimeout: IDLE_TIMEOUT_SECONDS, fetch: handle })
+}

package/src/proxy/state.ts

@@ -0,0 +1,27 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+import { paths } from "../config/constants"
+import type { ThreadState } from "./types"
+
+function stateFile(key: string) {
+  return path.join(paths.threads, `${key.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`)
+}
+
+export function loadState(key: string): ThreadState {
+  try {
+    return JSON.parse(readFileSync(stateFile(key), "utf8"))
+  } catch {
+    return {}
+  }
+}
+
+export function saveState(key: string, state: ThreadState) {
+  mkdirSync(paths.threads, { recursive: true })
+  writeFileSync(stateFile(key), JSON.stringify(state))
+}
+
+export function resetCut(state: ThreadState) {
+  delete state.cut
+  delete state.keep
+  delete state.cleared
+}

package/src/proxy/types.ts

@@ -0,0 +1,58 @@
+import type { Config } from "../config/types"
+import type { Body, Message } from "../context/types"
+import type { TranscriptRef } from "../transcript/types"
+
+export interface ThreadState {
+  cut?: string
+  keep?: number
+  cleared?: number
+  ratio?: number
+}
+
+export type Note = Record<string, unknown>
+
+export interface RewriteInput {
+  body: Body
+  sessionId: string | null
+  config: Config
+  transcript: TranscriptRef | null
+  window: number | null
+  mayCompact: boolean
+  omittedBefore?: number
+}
+
+export interface Rewrite {
+  body: Body
+  threadKey: string | null
+  note: Note
+}
+
+export interface CompactInput {
+  body: Body
+  config: Config
+  state: ThreadState
+  window: number | null
+  guide: string | null
+  mayCompact: boolean
+  omittedBefore?: number
+}
+
+export interface CompactResult {
+  messages: Message[]
+  stateChanged: boolean
+  note: Note
+}
+
+export interface Estimator {
+  of: (messages: Message[]) => number
+  fixed: number
+}
+
+export type Usage = Record<string, number>
+
+export interface Health {
+  ok: boolean
+  version: string
+  mode: Config["mode"]
+  pid: number
+}

package/src/proxy/usage.ts

@@ -0,0 +1,44 @@
+import { MAX_EVENT_BUFFER } from "./constants"
+import type { Usage } from "./types"
+
+export function watchUsage(response: Response, onUsage: (usage: Usage) => void): Response {
+  if (!response.body) return response
+  const [toClient, toUs] = response.body.tee()
+  void (async () => {
+    const reader = toUs.pipeThrough(new TextDecoderStream()).getReader()
+    let buffer = ""
+    let found = false
+    while (true) {
+      const { value, done } = await reader.read()
+      if (done) break
+      if (found) continue
+      buffer += value
+      let newline: number
+      while (!found && (newline = buffer.indexOf("\n")) !== -1) {
+        const line = buffer.slice(0, newline).trim()
+        buffer = buffer.slice(newline + 1)
+        if (!line.startsWith("data:") || !line.includes("message_start")) continue
+        try {
+          const event = JSON.parse(line.slice(5))
+          if (event.message?.usage) {
+            found = true
+            onUsage(event.message.usage)
+          }
+        } catch {}
+      }
+      if (!found && buffer.length > MAX_EVENT_BUFFER) buffer = ""
+    }
+    if (!found && buffer.trim().startsWith("{")) {
+      try {
+        const json = JSON.parse(buffer)
+        if (json.usage) onUsage(json.usage)
+        else if (typeof json.input_tokens === "number") onUsage({ input_tokens: json.input_tokens })
+      } catch {}
+    }
+  })().catch(() => undefined)
+  return new Response(toClient, { status: response.status, statusText: response.statusText, headers: response.headers })
+}
+
+export function inputTokens(usage: Usage) {
+  return (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0)
+}