openhermes 1.12.1 → 2.5.0

package/lib/ohc/compress/search.mjs

@@ -0,0 +1,152 @@
+import { parseBoundaryId, formatBlockRef } from "../message-ids.mjs"
+import { totalTokens } from "../reaper.mjs"
+
+export function buildSearchContext(state, rawMessages) {
+  const rawMessagesById = new Map()
+  const rawIndexById = new Map()
+
+  for (const msg of rawMessages) {
+    if (msg.info?.id) rawMessagesById.set(msg.info.id, msg)
+  }
+  for (let i = 0; i < rawMessages.length; i++) {
+    const msg = rawMessages[i]
+    if (msg?.info?.id) rawIndexById.set(msg.info.id, i)
+  }
+
+  const summaryByBlockId = new Map()
+  const ms = state.prune?.messages
+  if (ms?.blocksById) {
+    for (const [blockId, block] of ms.blocksById) {
+      if (block.active) summaryByBlockId.set(blockId, block)
+    }
+  }
+
+  return { rawMessages, rawMessagesById, rawIndexById, summaryByBlockId }
+}
+
+export function resolveBoundaryIds(context, state, startId, endId) {
+  const lookup = buildBoundaryLookup(context, state)
+
+  const parsedStart = parseBoundaryId(startId)
+  const parsedEnd = parseBoundaryId(endId)
+
+  if (!parsedStart) throw new Error(`startId "${startId}" is invalid. Use ohcNNNN (message) or bkNN (block).`)
+  if (!parsedEnd) throw new Error(`endId "${endId}" is invalid. Use ohcNNNN (message) or bkNN (block).`)
+
+  const startRef = lookup.get(parsedStart.ref)
+  const endRef = lookup.get(parsedEnd.ref)
+
+  if (!startRef) throw new Error(`startId ${parsedStart.ref} not found in context.`)
+  if (!endRef) throw new Error(`endId ${parsedEnd.ref} not found in context.`)
+
+  if (startRef.rawIndex > endRef.rawIndex) {
+    throw new Error(`startId ${parsedStart.ref} appears after endId ${parsedEnd.ref}. Start must come before end.`)
+  }
+
+  return { startReference: startRef, endReference: endRef }
+}
+
+function buildBoundaryLookup(context, state) {
+  const lookup = new Map()
+
+  for (const [msgRef, rawId] of state.messageIds.byRef) {
+    const rawMsg = context.rawMessagesById.get(rawId)
+    if (!rawMsg) continue
+
+    const rawIndex = context.rawIndexById.get(rawId)
+    if (rawIndex === undefined) continue
+
+    lookup.set(msgRef, { kind: "message", rawIndex, messageId: rawId })
+  }
+
+  const summaries = Array.from(context.summaryByBlockId.values()).sort((a, b) => a.blockId - b.blockId)
+  for (const summary of summaries) {
+    const anchorMsg = context.rawMessagesById.get(summary.anchorMessageId)
+    if (!anchorMsg) continue
+
+    const rawIndex = context.rawIndexById.get(summary.anchorMessageId)
+    if (rawIndex === undefined) continue
+
+    const bkRef = formatBlockRef(summary.blockId)
+    if (!lookup.has(bkRef)) {
+      lookup.set(bkRef, {
+        kind: "compressed-block",
+        rawIndex,
+        blockId: summary.blockId,
+        anchorMessageId: summary.anchorMessageId,
+      })
+    }
+  }
+
+  return lookup
+}
+
+export function resolveSelection(context, startReference, endReference) {
+  const messageIds = []
+  const messageSeen = new Set()
+  const toolIds = []
+  const toolSeen = new Set()
+  const requiredBlockIds = []
+  const requiredBlockSeen = new Set()
+
+  for (let i = startReference.rawIndex; i <= endReference.rawIndex; i++) {
+    const msg = context.rawMessages[i]
+    if (!msg) continue
+
+    const mid = msg.info?.id
+    if (mid && !messageSeen.has(mid)) {
+      messageSeen.add(mid)
+      messageIds.push(mid)
+    }
+
+    const parts = Array.isArray(msg.parts) ? msg.parts : []
+    for (const part of parts) {
+      if (part.type !== "tool" || !part.callID || toolSeen.has(part.callID)) continue
+      toolSeen.add(part.callID)
+      toolIds.push(part.callID)
+    }
+  }
+
+  const selectedIds = new Set(messageIds)
+  const summariesInRange = []
+  for (const block of context.summaryByBlockId.values()) {
+    if (!selectedIds.has(block.anchorMessageId)) continue
+    const idx = context.rawIndexById.get(block.anchorMessageId)
+    if (idx === undefined) continue
+    summariesInRange.push({ blockId: block.blockId, rawIndex: idx })
+  }
+
+  summariesInRange.sort((a, b) => a.rawIndex - b.rawIndex || a.blockId - b.blockId)
+  for (const s of summariesInRange) {
+    if (!requiredBlockSeen.has(s.blockId)) {
+      requiredBlockSeen.add(s.blockId)
+      requiredBlockIds.push(s.blockId)
+    }
+  }
+
+  if (messageIds.length === 0) {
+    throw new Error("No messages found in the specified range.")
+  }
+
+  return { startReference, endReference, messageIds, toolIds, requiredBlockIds }
+}
+
+export function resolveAnchorMessageId(startReference) {
+  if (startReference.kind === "compressed-block") {
+    if (!startReference.anchorMessageId) throw new Error("Compressed block has no anchor message ID")
+    return startReference.anchorMessageId
+  }
+  if (!startReference.messageId) throw new Error("No message ID in start reference")
+  return startReference.messageId
+}
+
+export function validateNonOverlapping(ranges) {
+  const sorted = [...ranges].sort((a, b) => a.selection.startReference.rawIndex - b.selection.startReference.rawIndex)
+  for (let i = 1; i < sorted.length; i++) {
+    const prev = sorted[i - 1]
+    const curr = sorted[i]
+    if (curr.selection.startReference.rawIndex <= prev.selection.endReference.rawIndex) {
+      throw new Error(`content[${prev.index}] (${prev.entry.startId}..${prev.entry.endId}) overlaps content[${curr.index}] (${curr.entry.startId}..${curr.entry.endId}).`)
+    }
+  }
+}

package/lib/ohc/compress/state.mjs

@@ -0,0 +1,76 @@
+export function allocateBlockId(state) {
+  const ms = state.prune.messages
+  const id = ms.nextBlockId
+  ms.nextBlockId++
+  return id
+}
+
+export function allocateRunId(state) {
+  const ms = state.prune.messages
+  const id = ms.nextRunId
+  ms.nextRunId++
+  return id
+}
+
+export function wrapBlockSummary(blockId, summary) {
+  return `[OHC: Compressed bk${blockId}]\n\n${summary}\n<ohc-ref>bk${blockId}</ohc-ref>`
+}
+
+export function applyCompressionState(state, input, selection, anchorMessageId, blockId, storedSummary, consumedBlockIds) {
+  const ms = state.prune.messages
+
+  const block = {
+    blockId,
+    active: true,
+    topic: input.topic,
+    batchTopic: input.batchTopic || input.topic,
+    mode: input.mode || "range",
+    runId: input.runId,
+    compressMessageId: input.compressMessageId,
+    compressCallId: input.compressCallId || null,
+    anchorMessageId,
+    startId: input.startId,
+    endId: input.endId,
+    summary: storedSummary,
+    summaryTokens: input.summaryTokens || 0,
+    compressedTokens: 0,
+    consumedBlockIds: Array.isArray(consumedBlockIds) ? consumedBlockIds : [],
+    deactivatedByBlockId: undefined,
+    deactivatedByUser: false,
+    deactivatedAt: undefined,
+    createdAt: Date.now(),
+  }
+
+  ms.blocksById.set(blockId, block)
+  ms.activeBlockIds.add(blockId)
+
+  if (anchorMessageId) {
+    ms.activeByAnchorMessageId.set(anchorMessageId, blockId)
+  }
+
+  for (const rawMessageId of selection.messageIds) {
+    let entry = ms.byMessageId.get(rawMessageId)
+    if (!entry) {
+      entry = { tokenCount: 0, allBlockIds: [], activeBlockIds: [] }
+      ms.byMessageId.set(rawMessageId, entry)
+    }
+    if (!entry.allBlockIds.includes(blockId)) {
+      entry.allBlockIds.push(blockId)
+    }
+    if (!entry.activeBlockIds.includes(blockId)) {
+      entry.activeBlockIds.push(blockId)
+    }
+  }
+
+  for (const cid of consumedBlockIds) {
+    const cb = ms.blocksById.get(cid)
+    if (cb) {
+      cb.active = false
+      cb.deactivatedAt = Date.now()
+      cb.deactivatedByBlockId = blockId
+      ms.activeBlockIds.delete(cid)
+    }
+  }
+
+  return block
+}

package/lib/ohc/config.mjs

@@ -2,29 +2,185 @@ import fs from "node:fs"
 import path from "node:path"
 import os from "node:os"
 
-const CONFIG_PATH = path.join(os.homedir(), ".config", "opencode", "ohc.json")
+const GLOBAL_DIR = path.join(os.homedir(), ".config", "opencode")
+const GLOBAL_PATH = path.join(GLOBAL_DIR, "ohc.json")
+const GLOBAL_PATH_JSONC = path.join(GLOBAL_DIR, "ohc.jsonc")
 
-const DEFAULTS = { enabled: true, max: 200000, min: 50000 }
+const DEFAULTS = {
+  enabled: true,
+  notification: "chat",
+  notificationMode: "detailed",
+  max: 150000,
+  min: 50000,
+  manualMode: { enabled: false, automaticStrategies: true },
+  turnProtection: { enabled: false, turns: 4 },
+  protectedFilePatterns: [],
+  compress: {
+    maxContextLimit: 150000,
+    minContextLimit: 50000,
+    nudgeFrequency: 5,
+    iterationNudgeThreshold: 15,
+    nudgeForce: "soft",
+    protectedTools: ["task", "skill", "todowrite", "todoread"],
+    protectUserMessages: false,
+    summaryBuffer: true,
+  },
+  strategies: {
+    deduplication: { enabled: true, protectedTools: [] },
+    purgeErrors: { enabled: true, turns: 4, protectedTools: [] },
+  },
+}
 
-function writeDefaults() {
-  try {
-    fs.mkdirSync(path.dirname(CONFIG_PATH), { recursive: true })
-    fs.writeFileSync(CONFIG_PATH, JSON.stringify(DEFAULTS, null, 2) + "\n", "utf8")
-  } catch {}
+function findOpencodeDir(startDir) {
+  let current = startDir
+  while (current && current.length > 3) {
+    const candidate = path.join(current, ".opencode")
+    if (fs.existsSync(candidate) && fs.statSync(candidate).isDirectory()) {
+      return candidate
+    }
+    const parent = path.dirname(current)
+    if (parent === current) break
+    current = parent
+  }
+  return null
 }
 
-export function loadConfig() {
-  let raw = {}
+function loadFile(filePath) {
   try {
-    raw = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"))
+    const content = fs.readFileSync(filePath, "utf8").trim()
+    if (!content) return null
+    if (filePath.endsWith(".jsonc")) {
+      const stripped = content
+        .replace(/"(?:[^"\\]|\\.)*"|\/\/.*/gm, m => m.startsWith('"') ? m : "")
+        .replace(/\/\*[\s\S]*?\*\//g, "")
+      return JSON.parse(stripped)
+    }
+    return JSON.parse(content)
   } catch {
-    writeDefaults()
-    return { ...DEFAULTS }
+    return null
+  }
+}
+
+function mergeDeep(base, override) {
+  if (!override || typeof override !== "object") return base
+  const result = { ...base }
+  for (const key of Object.keys(override)) {
+    if (override[key] === undefined) continue
+    if (typeof override[key] === "object" && !Array.isArray(override[key]) && override[key] !== null) {
+      result[key] = mergeDeep(base[key] || {}, override[key])
+    } else {
+      result[key] = override[key]
+    }
+  }
+  return result
+}
+
+function mergeArrays(base, override) {
+  if (!override || !Array.isArray(override)) return base
+  return [...new Set([...base, ...override])]
+}
+
+function mergeManualMode(base, override) {
+  if (!override || typeof override !== "object") return base
+  return {
+    enabled: override.enabled ?? base.enabled,
+    automaticStrategies: override.automaticStrategies ?? base.automaticStrategies,
+  }
+}
+
+function mergeTurnProtection(base, override) {
+  if (!override || typeof override !== "object") return base
+  return {
+    enabled: override.enabled ?? base.enabled,
+    turns: override.turns ?? base.turns,
+  }
+}
+
+function mergeCompress(base, override) {
+  if (!override || typeof override !== "object") return base
+  return {
+    maxContextLimit: override.maxContextLimit ?? base.maxContextLimit,
+    minContextLimit: override.minContextLimit ?? base.minContextLimit,
+    nudgeFrequency: override.nudgeFrequency ?? base.nudgeFrequency,
+    iterationNudgeThreshold: override.iterationNudgeThreshold ?? base.iterationNudgeThreshold,
+    nudgeForce: override.nudgeForce ?? base.nudgeForce,
+    protectedTools: mergeArrays(base.protectedTools, override.protectedTools),
+    protectUserMessages: override.protectUserMessages ?? base.protectUserMessages,
+    summaryBuffer: override.summaryBuffer ?? base.summaryBuffer,
   }
+}
+
+function mergeStrategies(base, override) {
+  if (!override || typeof override !== "object") return base
+  return {
+    deduplication: {
+      enabled: override.deduplication?.enabled ?? base.deduplication.enabled,
+      protectedTools: mergeArrays(base.deduplication.protectedTools, override.deduplication?.protectedTools),
+    },
+    purgeErrors: {
+      enabled: override.purgeErrors?.enabled ?? base.purgeErrors.enabled,
+      turns: override.purgeErrors?.turns ?? base.purgeErrors.turns,
+      protectedTools: mergeArrays(base.purgeErrors.protectedTools, override.purgeErrors?.protectedTools),
+    },
+  }
+}
 
+function mergeLayer(base, data) {
+  if (!data) return base
   return {
-    enabled: raw.enabled !== false,
-    max: typeof raw.max === "number" && raw.max > 0 ? raw.max : DEFAULTS.max,
-    min: typeof raw.min === "number" ? Math.max(10000, raw.min) : DEFAULTS.min,
+    enabled: data.enabled ?? base.enabled,
+    notification: data.notification ?? base.notification,
+    notificationMode: data.notificationMode ?? base.notificationMode,
+    max: data.max ?? base.max,
+    min: data.min ?? base.min,
+    manualMode: mergeManualMode(base.manualMode, data.manualMode),
+    turnProtection: mergeTurnProtection(base.turnProtection, data.turnProtection),
+    protectedFilePatterns: mergeArrays(base.protectedFilePatterns, data.protectedFilePatterns),
+    compress: mergeCompress(base.compress, data.compress),
+    strategies: mergeStrategies(base.strategies, data.strategies),
   }
-}
+}
+
+export function loadConfig(ctx) {
+  let config = { ...DEFAULTS }
+
+  const layers = [
+    { path: fs.existsSync(GLOBAL_PATH_JSONC) ? GLOBAL_PATH_JSONC : (fs.existsSync(GLOBAL_PATH) ? GLOBAL_PATH : null), name: "global" },
+  ]
+
+  const opencodeConfigDir = process.env.OPENCODE_CONFIG_DIR
+  if (opencodeConfigDir) {
+    const cdJsonc = path.join(opencodeConfigDir, "ohc.jsonc")
+    const cdJson = path.join(opencodeConfigDir, "ohc.json")
+    const cdPath = fs.existsSync(cdJsonc) ? cdJsonc : (fs.existsSync(cdJson) ? cdJson : null)
+    if (cdPath) layers.push({ path: cdPath, name: "configDir" })
+  }
+
+  if (ctx?.directory) {
+    const opencodeDir = findOpencodeDir(ctx.directory)
+    if (opencodeDir) {
+      const pjJsonc = path.join(opencodeDir, "ohc.jsonc")
+      const pjJson = path.join(opencodeDir, "ohc.json")
+      const pjPath = fs.existsSync(pjJsonc) ? pjJsonc : (fs.existsSync(pjJson) ? pjJson : null)
+      if (pjPath) layers.push({ path: pjPath, name: "project" })
+    }
+  }
+
+  for (const layer of layers) {
+    if (!layer.path) continue
+    const data = loadFile(layer.path)
+    if (data) config = mergeLayer(config, data)
+  }
+
+  config.max = typeof config.max === "number" && config.max > 0 ? config.max : DEFAULTS.max
+  config.min = typeof config.min === "number" ? Math.max(10000, Math.min(config.min, config.max - 10000)) : DEFAULTS.min
+
+  return config
+}
+
+function writeDefaults() {
+  fs.mkdirSync(GLOBAL_DIR, { recursive: true })
+  if (!fs.existsSync(GLOBAL_PATH) && !fs.existsSync(GLOBAL_PATH_JSONC)) {
+    fs.writeFileSync(GLOBAL_PATH, JSON.stringify(DEFAULTS, null, 2) + "\n", "utf8")
+  }
+}

package/lib/ohc/message-ids.mjs

@@ -0,0 +1,168 @@
+const MESSAGE_REF_REGEX = /^ohc(\d{4})$/
+const BLOCK_REF_REGEX = /^bk([1-9]\d*)$/
+const OHCTAG = "ohc-ref"
+
+export function formatMessageRef(index) {
+  if (!Number.isInteger(index) || index < 1 || index > 9999) {
+    throw new Error(`OHC ref index out of bounds: ${index}`)
+  }
+  return `ohc${index.toString().padStart(4, "0")}`
+}
+
+export function formatBlockRef(blockId) {
+  if (!Number.isInteger(blockId) || blockId < 1) {
+    throw new Error(`Invalid block ID: ${blockId}`)
+  }
+  return `bk${blockId}`
+}
+
+export function parseMessageRef(ref) {
+  const m = (ref || "").trim().toLowerCase().match(MESSAGE_REF_REGEX)
+  if (!m) return null
+  const idx = parseInt(m[1], 10)
+  return Number.isInteger(idx) && idx >= 1 && idx <= 9999 ? idx : null
+}
+
+export function parseBlockRef(ref) {
+  const m = (ref || "").trim().toLowerCase().match(BLOCK_REF_REGEX)
+  if (!m) return null
+  const id = parseInt(m[1], 10)
+  return Number.isInteger(id) ? id : null
+}
+
+export function parseBoundaryId(id) {
+  const norm = (id || "").trim().toLowerCase()
+  const msgIdx = parseMessageRef(norm)
+  if (msgIdx !== null) {
+    return { kind: "message", ref: formatMessageRef(msgIdx), index: msgIdx }
+  }
+  const bkId = parseBlockRef(norm)
+  if (bkId !== null) {
+    return { kind: "compressed-block", ref: formatBlockRef(bkId), blockId: bkId }
+  }
+  return null
+}
+
+function escapeXml(s) {
+  return String(s).replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
+}
+
+export function formatOhcTag(ref, attrs) {
+  const serialized = Object.entries(attrs || {})
+    .filter(([, v]) => typeof v === "string" && v.length > 0)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([k, v]) => ` ${k}="${escapeXml(v)}"`)
+    .join("")
+  return `\n<${OHCTAG}${serialized}>${ref}</${OHCTAG}>`
+}
+
+export function isIgnoredUserMessage(message) {
+  if (!message?.info || message.info.role !== "user") return false
+  const parts = Array.isArray(message.parts) ? message.parts : []
+  if (parts.length === 0) return true
+  return parts.every(p => p.ignored)
+}
+
+export function assignMessageRefs(state, messages) {
+  let assigned = 0
+  let skippedFirstUser = false
+
+  for (const msg of messages) {
+    if (isIgnoredUserMessage(msg)) continue
+
+    if (state.isSubAgent && !skippedFirstUser && msg.info.role === "user") {
+      skippedFirstUser = true
+      continue
+    }
+
+    const rawId = msg.info?.id
+    if (typeof rawId !== "string" || !rawId) continue
+
+    const existing = state.messageIds.byRawId.get(rawId)
+    if (existing) {
+      if (state.messageIds.byRef.get(existing) !== rawId) {
+        state.messageIds.byRef.set(existing, rawId)
+      }
+      continue
+    }
+
+    const ref = allocateNextRef(state)
+    state.messageIds.byRawId.set(rawId, ref)
+    state.messageIds.byRef.set(ref, rawId)
+    assigned++
+  }
+
+  return assigned
+}
+
+function allocateNextRef(state) {
+  let candidate = Number.isInteger(state.messageIds.nextRef)
+    ? Math.max(1, state.messageIds.nextRef)
+    : 1
+
+  for (let attempt = 0; attempt < 9999; attempt++) {
+    if (candidate > 9999) candidate = 1
+    const ref = formatMessageRef(candidate)
+    if (!state.messageIds.byRef.has(ref)) {
+      state.messageIds.nextRef = candidate + 1
+      return ref
+    }
+    candidate++
+  }
+
+  state.messageIds.nextRef = 1
+  return formatMessageRef(1)
+}
+
+export function injectMessageIds(state, messages) {
+  for (const msg of messages) {
+    if (isIgnoredUserMessage(msg)) continue
+    const msgRef = state.messageIds.byRawId.get(msg.info?.id)
+    if (!msgRef) continue
+    const tag = formatOhcTag(msgRef)
+
+    if (msg.info.role === "user") {
+      let injected = false
+      for (const part of msg.parts) {
+        if (part.type === "text") {
+          part.text += tag
+          injected = true
+          break
+        }
+      }
+      if (!injected) {
+        msg.parts.push({ type: "text", text: tag })
+      }
+      continue
+    }
+
+    if (msg.info.role !== "assistant") continue
+
+    let hasContent = Array.isArray(msg.parts) && msg.parts.some(p => p.type === "text" || p.type === "tool")
+    if (!hasContent) continue
+
+    let injected = false
+    for (const part of msg.parts) {
+      if (part.type !== "tool" || typeof part.state?.output !== "string") continue
+      part.state.output = (part.state.output || "") + tag
+      injected = true
+      break
+    }
+
+    if (injected) continue
+
+    const lastText = [...msg.parts].reverse().find(p => p.type === "text")
+    if (lastText) {
+      lastText.text += tag
+      continue
+    }
+
+    const firstToolIdx = msg.parts.findIndex(p => p.type === "tool")
+    const synth = { type: "text", text: tag }
+    if (firstToolIdx === -1) {
+      msg.parts.push(synth)
+    } else {
+      msg.parts.splice(firstToolIdx, 0, synth)
+    }
+  }
+}