openhermes 1.13.1 → 2.5.0

package/lib/memory-tools-plugin.mjs

@@ -7,6 +7,9 @@ import os from "node:os"
 import { atomicWriteJson, fingerprintEnvironment, readJson, readJsonl, sanitizeRecord, truncateText } from "./hardening.mjs"
 import { findUnsupportedSchemaKeywords, validateSchema } from "./schema-validator.mjs"
 import { getMemoryRoot, getRuntimeRoot } from "./paths.mjs"
+import { loadConfig } from "./ohc/config.mjs"
+import { sendMemoryNotification } from "./ohc/notify.mjs"
+import { scoreRelevance } from "./search.mjs"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const SCHEMAS_DIR = path.resolve(__dirname, "..", "schemas")
@@ -85,10 +88,9 @@ function queryList(cls, limit = 10) {
     return sortRecent(filterActive(readJsonl(path.join(classDir(cls), "mistakes.jsonl")))).slice(0, limit)
   }
   const dir = classDir(cls)
-  let files = []
-  try { files = fs.readdirSync(dir).filter(f => f.endsWith(".json") && f !== "index.json").map(f => path.join(dir, f)) } catch { return [] }
-  const entries = files.map(f => readJson(f, null)).filter(Boolean).map(r => buildEntry(cls, r))
-  return sortRecent(filterActive(entries)).slice(0, limit)
+  const index = readJson(path.join(dir, "index.json"), [])
+  if (!Array.isArray(index)) return []
+  return sortRecent(filterActive(index)).slice(0, limit)
 }
 
 function queryGet(cls, id) {
@@ -96,27 +98,6 @@ function queryGet(cls, id) {
   return readJson(path.join(classDir(cls), `${id}.json`), null)
 }
 
-function scoreRelevance(r, query, project) {
-  const q = query.toLowerCase()
-  let score = 0
-  const fields = [r.summary, r.id, r.description, r.mission, r.current_state, r.failure, r.root_cause, r.fix, r.prevention, r.command, r.project, r.scope, ...(Array.isArray(r.tags) ? r.tags : []), ...(Array.isArray(r.next_actions) ? r.next_actions : []), ...(Array.isArray(r.refs) ? r.refs : [])].filter(Boolean)
-  for (const f of fields) {
-    const str = String(f).toLowerCase()
-    let idx = 0; let count = 0
-    while ((idx = str.indexOf(q, idx)) !== -1) { count++; idx += q.length }
-    score += count * 10
-    if (str.startsWith(q)) score += 5
-    if (str.includes(q)) score += 2
-  }
-  if (r.project && r.project.toLowerCase() === (project || "").toLowerCase()) score += 20
-  if (r.project && project && r.project.toLowerCase().includes(project.toLowerCase())) score += 10
-  const age = Date.now() - Date.parse(r.updated_at || r.created_at || 0)
-  if (!Number.isNaN(age)) score += Math.max(0, 10 - age / 604800000)
-  if (r.status === "active") score += 3
-  if (r.status === "closed") score -= 2
-  return score
-}
-
 function enforceAuditEvidence(record) {
   const prov = isPlainObject(record.provenance) ? record.provenance : {}
   return ["db_refs", "file_refs", "log_refs"].some(k => Array.isArray(prov[k]) && prov[k].some(i => typeof i === "string" && i.trim()))
@@ -127,22 +108,71 @@ function setToolTitle(context, title) {
   context.metadata({ title })
 }
 
+function fillSchemaDefaults(record, schema) {
+  const required = Array.isArray(schema.required) ? schema.required : []
+  for (const field of required) {
+    const prop = schema.properties?.[field]
+    if (!prop) { if (!(field in record)) record[field] = null; continue }
+
+    const types = Array.isArray(prop.type) ? prop.type : [prop.type]
+    const t = types.find(t => t !== "null") || types[0]
+
+    if (t === "object" && prop.properties) {
+      if (record[field] == null) record[field] = {}
+      if (isPlainObject(record[field])) fillSchemaDefaults(record[field], prop)
+      continue
+    }
+
+    if (field in record) continue
+    if (prop.default !== undefined) { record[field] = structuredClone(prop.default); continue }
+    if (prop.const !== undefined) { record[field] = prop.const; continue }
+    if (prop.enum) { record[field] = prop.enum[0]; continue }
+    if (t === "integer" || t === "number") {
+      record[field] = prop.minimum !== undefined ? prop.minimum : 0
+      continue
+    }
+    switch (t) {
+      case "string":
+        record[field] = prop.minLength > 1 ? "x".repeat(prop.minLength) : field
+        break
+      case "boolean": record[field] = false; break
+      case "array": record[field] = []; break
+      default: record[field] = field; break
+    }
+  }
+}
+
+function sanitizeId(raw) {
+  return raw.replace(/[<>:"/\\|?*]/g, "_").replace(/\.\./g, "_").trim()
+}
+
 function handleAdd(cls, id, dataStr) {
   let parsed
-  try { parsed = JSON.parse(dataStr) } catch (e) { return `invalid JSON: ${e.message}` }
-  if (!isPlainObject(parsed)) return "data must be a JSON object"
+  try { parsed = JSON.parse(dataStr) } catch (e) { parsed = null }
+  if (!isPlainObject(parsed)) {
+    const txt = String(dataStr ?? parsed ?? "").slice(0, 200)
+    parsed = { summary: txt }
+  }
   if (!id?.trim()) return "non-blank id is required"
+  id = sanitizeId(id.trim())
 
   const now = new Date().toISOString()
-  const existing = queryGet(cls, id.trim())
+  const existing = queryGet(cls, id)
+  const existingIsActive = existing && !hasExpired(existing)
   const record = { ...parsed, id, class: cls, source: parsed.source ?? "agent", status: parsed.status ?? "active", created_at: parsed.created_at ?? (existing?.created_at ?? now), updated_at: now }
 
   const schema = readJson(path.join(SCHEMAS_DIR, `${cls}.schema.json`), null)
   if (schema) {
+    fillSchemaDefaults(record, schema)
+
     const required = Array.isArray(schema.required) ? schema.required : []
     if (required.includes("provenance")) {
-      if (!record.provenance) record.provenance = { session_id: `auto-${now}` }
-      else if (!record.provenance.session_id) record.provenance.session_id = `auto-${now}`
+      if (!record.provenance?.session_id) record.provenance = { ...record.provenance, session_id: `auto-${now}` }
+      if (cls === "audit" && !enforceAuditEvidence(record)) {
+        const prov = record.provenance
+        if (!Array.isArray(prov.file_refs)) prov.file_refs = []
+        if (!prov.file_refs.some(r => typeof r === "string" && r.trim())) prov.file_refs.push("auto-filled")
+      }
     }
     if (required.includes("trigger") && !record.trigger) record.trigger = "manual"
     if (required.includes("success_count") && record.success_count == null) record.success_count = 0
@@ -159,13 +189,14 @@ function handleAdd(cls, id, dataStr) {
   if (cls === "mistake") upsertMistake(record)
   else writeObject(cls, record)
 
-  const action = existing ? "updated" : "saved"
+  const action = existingIsActive ? "updated" : "saved"
   return `${action}: ${id.trim()}`
 }
 
 function handleFetch(cls, id) {
   if (!id?.trim()) return "non-blank id is required"
-  const record = queryGet(cls, id.trim())
+  id = sanitizeId(id.trim())
+  const record = queryGet(cls, id)
   if (!record) return `not found: ${id}`
   return stableStringify(record)
 }
@@ -190,35 +221,46 @@ function handleSearch(query, scope, classes, project, limit) {
   if (!q) return "non-blank query is required"
   const clsList = Array.isArray(classes) && classes.length ? classes : CLASSES
   const lim = Math.min(limit ?? 10, 50)
-  let records = []
+  let candidates = []
   for (const cls of clsList) {
     if (cls === "mistake") {
       for (const m of readJsonl(path.join(classDir(cls), "mistakes.jsonl"))) {
-        if (!hasExpired(m)) records.push(m)
+        if (!hasExpired(m)) candidates.push({ ...m, _cls: cls })
       }
     } else {
       const dir = classDir(cls)
-      let files = []
-      try { files = fs.readdirSync(dir).filter(f => f.endsWith(".json") && f !== "index.json") } catch { continue }
-      for (const f of files) {
-        const r = readJson(path.join(dir, f), null)
-        if (r && !hasExpired(r)) records.push(r)
+      const index = readJson(path.join(dir, "index.json"), [])
+      if (!Array.isArray(index)) continue
+      for (const entry of index) {
+        if (!hasExpired(entry)) candidates.push({ ...entry, _cls: cls })
       }
     }
   }
-  if (scope === "global") records = records.filter(r => r.scope === "global" || !r.scope)
-  else if (scope === "local") records = records.filter(r => r.scope === "project" || r.scope === "session")
-  const scored = records.map(r => ({ ...buildEntry(r.class || "verification_receipt", r), score: scoreRelevance(r, q, project || "") }))
+  if (scope === "global") candidates = candidates.filter(r => r.scope === "global" || !r.scope)
+  else if (scope === "local") candidates = candidates.filter(r => r.scope === "project" || r.scope === "session")
+  
+  // Score using index entries (summary-based)
+  const scored = candidates
+    .map(e => ({ ...e, score: scoreRelevance(e, q, project || "") }))
     .filter(e => e.score > 0)
     .sort((a, b) => b.score - a.score)
     .slice(0, lim)
-  const lines = scored.map(e => `  ${e.id} (${e.score}pt): ${truncateText(e.summary || "", 80)}`)
-  return `${scored.length} result${scored.length === 1 ? "" : "s"} for '${q}'\n` + lines.join("\n")
+  
+  // Fetch full records for top results to get richer fields
+  const results = scored.map(e => {
+    if (e._cls === "mistake") return e
+    const full = queryGet(e._cls, e.id)
+    return full || e
+  })
+  
+  const lines = results.map(e => `  ${e.id} (${e.score}pt): ${truncateText(e.summary || "", 80)}`)
+  return `${results.length} result${results.length === 1 ? "" : "s"} for '${q}'\n` + lines.join("\n")
 }
 
 function handleArchive(cls, id) {
   if (!id?.trim()) return "non-blank id is required"
-  const record = queryGet(cls, id.trim())
+  id = sanitizeId(id.trim())
+  const record = queryGet(cls, id)
   if (!record) return `not found: ${id}`
   record.status = "expired"
   record.updated_at = new Date().toISOString()
@@ -227,13 +269,15 @@ function handleArchive(cls, id) {
   return `archived: ${id.trim()}`
 }
 
-export const MemoryToolsPlugin = async () => {
+export const MemoryToolsPlugin = async (ctx) => {
   fs.mkdirSync(MEMORY_DIR, { recursive: true })
   fs.mkdirSync(getRuntimeRoot(), { recursive: true })
 
+  const config = loadConfig()
+
   return {
     tool: {
-      add_memory: tool({
+      ohc_save: tool({
         description: "Save a new memory record or update an existing one by class and id",
         args: {
           class: tool.schema.enum(CLASSES),
@@ -242,11 +286,16 @@ export const MemoryToolsPlugin = async () => {
         },
         async execute(args, context) {
           setToolTitle(context, `save ${args.class}: ${args.id}`)
-          return handleAdd(args.class, args.id, args.data)
+          const result = handleAdd(args.class, args.id, args.data)
+          if (result.startsWith("saved:") || result.startsWith("updated:")) {
+            const action = result.startsWith("saved:") ? "Saved" : "Updated"
+            sendMemoryNotification(ctx?.client, context.sessionID, config, action, args.class, args.id, null).catch(() => {})
+          }
+          return result
         },
       }),
 
-      fetch_memory: tool({
+      ohc_get: tool({
         description: "Get a specific memory record by class and id",
         args: {
           class: tool.schema.enum(CLASSES).describe("Memory class"),
@@ -258,7 +307,7 @@ export const MemoryToolsPlugin = async () => {
         },
       }),
 
-      list_memory: tool({
+      ohc_list: tool({
         description: "List recent memory records by class, sorted by recency",
         args: {
           class: tool.schema.enum(CLASSES).describe("Memory class"),
@@ -270,7 +319,7 @@ export const MemoryToolsPlugin = async () => {
         },
       }),
 
-      latest_memory: tool({
+      ohc_latest: tool({
         description: "Get the latest active memory record by class",
         args: {
           class: tool.schema.enum(CLASSES).describe("Memory class"),
@@ -281,7 +330,7 @@ export const MemoryToolsPlugin = async () => {
         },
       }),
 
-      search_memory: tool({
+      ohc_search: tool({
         description: "Search memory records with keyword matching and relevance ranking across all classes",
         args: {
           query: tool.schema.string().describe("Search query string"),
@@ -296,7 +345,7 @@ export const MemoryToolsPlugin = async () => {
         },
       }),
 
-      archive_memory: tool({
+      ohc_archive: tool({
         description: "Soft-delete a memory record by setting its status to expired",
         args: {
           class: tool.schema.enum(CLASSES),
@@ -304,7 +353,11 @@ export const MemoryToolsPlugin = async () => {
         },
         async execute(args, context) {
           setToolTitle(context, `archive ${args.class}: ${args.id}`)
-          return handleArchive(args.class, args.id)
+          const result = handleArchive(args.class, args.id)
+          if (result.startsWith("archived:")) {
+            sendMemoryNotification(ctx?.client, context.sessionID, config, "Archived", args.class, args.id, null).catch(() => {})
+          }
+          return result
         },
       }),
     },

package/lib/ohc/block-sync.mjs

@@ -0,0 +1,69 @@
+export function syncCompressionBlocks(state, messages) {
+  const ms = state.prune.messages
+  if (!ms?.blocksById?.size) return
+
+  const messageIds = new Set()
+  for (const msg of messages) {
+    if (msg.info?.id) messageIds.add(msg.info.id)
+  }
+
+  const prevActive = new Set(ms.activeBlockIds)
+
+  ms.activeBlockIds.clear()
+  ms.activeByAnchorMessageId?.clear()
+
+  const now = Date.now()
+  const ordered = [...ms.blocksById.values()].sort(
+    (a, b) => (a.createdAt || 0) - (b.createdAt || 0) || a.blockId - b.blockId,
+  )
+
+  for (const block of ordered) {
+    const hasOrigin = typeof block.compressMessageId === "string" &&
+      block.compressMessageId.length > 0 &&
+      messageIds.has(block.compressMessageId)
+
+    if (!hasOrigin) {
+      block.active = false
+      block.deactivatedAt = now
+      continue
+    }
+
+    if (block.deactivatedByUser) {
+      block.active = false
+      if (block.deactivatedAt === undefined) block.deactivatedAt = now
+      continue
+    }
+
+    const consumed = Array.isArray(block.consumedBlockIds) ? block.consumedBlockIds : []
+    for (const cid of consumed) {
+      if (!ms.activeBlockIds.has(cid)) continue
+      const cb = ms.blocksById.get(cid)
+      if (cb) {
+        cb.active = false
+        cb.deactivatedAt = now
+        cb.deactivatedByBlockId = block.blockId
+      }
+      ms.activeBlockIds.delete(cid)
+    }
+
+    block.active = true
+    block.deactivatedAt = undefined
+    block.deactivatedByBlockId = undefined
+    ms.activeBlockIds.add(block.blockId)
+  }
+
+  for (const entry of ms.byMessageId.values()) {
+    const all = Array.isArray(entry.allBlockIds)
+      ? [...new Set(entry.allBlockIds.filter(id => Number.isInteger(id) && id > 0))]
+      : []
+    entry.allBlockIds = all
+    entry.activeBlockIds = all.filter(id => ms.activeBlockIds.has(id))
+  }
+
+  let deactivated = 0
+  let reactivated = 0
+  for (const id of prevActive) { if (!ms.activeBlockIds.has(id)) deactivated++ }
+  for (const id of ms.activeBlockIds) { if (!prevActive.has(id)) reactivated++ }
+
+  return { deactivated, reactivated }
+}

package/lib/ohc/compress/search.mjs

@@ -0,0 +1,152 @@
+import { parseBoundaryId, formatBlockRef } from "../message-ids.mjs"
+import { totalTokens } from "../reaper.mjs"
+
+export function buildSearchContext(state, rawMessages) {
+  const rawMessagesById = new Map()
+  const rawIndexById = new Map()
+
+  for (const msg of rawMessages) {
+    if (msg.info?.id) rawMessagesById.set(msg.info.id, msg)
+  }
+  for (let i = 0; i < rawMessages.length; i++) {
+    const msg = rawMessages[i]
+    if (msg?.info?.id) rawIndexById.set(msg.info.id, i)
+  }
+
+  const summaryByBlockId = new Map()
+  const ms = state.prune?.messages
+  if (ms?.blocksById) {
+    for (const [blockId, block] of ms.blocksById) {
+      if (block.active) summaryByBlockId.set(blockId, block)
+    }
+  }
+
+  return { rawMessages, rawMessagesById, rawIndexById, summaryByBlockId }
+}
+
+export function resolveBoundaryIds(context, state, startId, endId) {
+  const lookup = buildBoundaryLookup(context, state)
+
+  const parsedStart = parseBoundaryId(startId)
+  const parsedEnd = parseBoundaryId(endId)
+
+  if (!parsedStart) throw new Error(`startId "${startId}" is invalid. Use ohcNNNN (message) or bkNN (block).`)
+  if (!parsedEnd) throw new Error(`endId "${endId}" is invalid. Use ohcNNNN (message) or bkNN (block).`)
+
+  const startRef = lookup.get(parsedStart.ref)
+  const endRef = lookup.get(parsedEnd.ref)
+
+  if (!startRef) throw new Error(`startId ${parsedStart.ref} not found in context.`)
+  if (!endRef) throw new Error(`endId ${parsedEnd.ref} not found in context.`)
+
+  if (startRef.rawIndex > endRef.rawIndex) {
+    throw new Error(`startId ${parsedStart.ref} appears after endId ${parsedEnd.ref}. Start must come before end.`)
+  }
+
+  return { startReference: startRef, endReference: endRef }
+}
+
+function buildBoundaryLookup(context, state) {
+  const lookup = new Map()
+
+  for (const [msgRef, rawId] of state.messageIds.byRef) {
+    const rawMsg = context.rawMessagesById.get(rawId)
+    if (!rawMsg) continue
+
+    const rawIndex = context.rawIndexById.get(rawId)
+    if (rawIndex === undefined) continue
+
+    lookup.set(msgRef, { kind: "message", rawIndex, messageId: rawId })
+  }
+
+  const summaries = Array.from(context.summaryByBlockId.values()).sort((a, b) => a.blockId - b.blockId)
+  for (const summary of summaries) {
+    const anchorMsg = context.rawMessagesById.get(summary.anchorMessageId)
+    if (!anchorMsg) continue
+
+    const rawIndex = context.rawIndexById.get(summary.anchorMessageId)
+    if (rawIndex === undefined) continue
+
+    const bkRef = formatBlockRef(summary.blockId)
+    if (!lookup.has(bkRef)) {
+      lookup.set(bkRef, {
+        kind: "compressed-block",
+        rawIndex,
+        blockId: summary.blockId,
+        anchorMessageId: summary.anchorMessageId,
+      })
+    }
+  }
+
+  return lookup
+}
+
+export function resolveSelection(context, startReference, endReference) {
+  const messageIds = []
+  const messageSeen = new Set()
+  const toolIds = []
+  const toolSeen = new Set()
+  const requiredBlockIds = []
+  const requiredBlockSeen = new Set()
+
+  for (let i = startReference.rawIndex; i <= endReference.rawIndex; i++) {
+    const msg = context.rawMessages[i]
+    if (!msg) continue
+
+    const mid = msg.info?.id
+    if (mid && !messageSeen.has(mid)) {
+      messageSeen.add(mid)
+      messageIds.push(mid)
+    }
+
+    const parts = Array.isArray(msg.parts) ? msg.parts : []
+    for (const part of parts) {
+      if (part.type !== "tool" || !part.callID || toolSeen.has(part.callID)) continue
+      toolSeen.add(part.callID)
+      toolIds.push(part.callID)
+    }
+  }
+
+  const selectedIds = new Set(messageIds)
+  const summariesInRange = []
+  for (const block of context.summaryByBlockId.values()) {
+    if (!selectedIds.has(block.anchorMessageId)) continue
+    const idx = context.rawIndexById.get(block.anchorMessageId)
+    if (idx === undefined) continue
+    summariesInRange.push({ blockId: block.blockId, rawIndex: idx })
+  }
+
+  summariesInRange.sort((a, b) => a.rawIndex - b.rawIndex || a.blockId - b.blockId)
+  for (const s of summariesInRange) {
+    if (!requiredBlockSeen.has(s.blockId)) {
+      requiredBlockSeen.add(s.blockId)
+      requiredBlockIds.push(s.blockId)
+    }
+  }
+
+  if (messageIds.length === 0) {
+    throw new Error("No messages found in the specified range.")
+  }
+
+  return { startReference, endReference, messageIds, toolIds, requiredBlockIds }
+}
+
+export function resolveAnchorMessageId(startReference) {
+  if (startReference.kind === "compressed-block") {
+    if (!startReference.anchorMessageId) throw new Error("Compressed block has no anchor message ID")
+    return startReference.anchorMessageId
+  }
+  if (!startReference.messageId) throw new Error("No message ID in start reference")
+  return startReference.messageId
+}
+
+export function validateNonOverlapping(ranges) {
+  const sorted = [...ranges].sort((a, b) => a.selection.startReference.rawIndex - b.selection.startReference.rawIndex)
+  for (let i = 1; i < sorted.length; i++) {
+    const prev = sorted[i - 1]
+    const curr = sorted[i]
+    if (curr.selection.startReference.rawIndex <= prev.selection.endReference.rawIndex) {
+      throw new Error(`content[${prev.index}] (${prev.entry.startId}..${prev.entry.endId}) overlaps content[${curr.index}] (${curr.entry.startId}..${curr.entry.endId}).`)
+    }
+  }
+}

package/lib/ohc/compress/state.mjs

@@ -0,0 +1,76 @@
+export function allocateBlockId(state) {
+  const ms = state.prune.messages
+  const id = ms.nextBlockId
+  ms.nextBlockId++
+  return id
+}
+
+export function allocateRunId(state) {
+  const ms = state.prune.messages
+  const id = ms.nextRunId
+  ms.nextRunId++
+  return id
+}
+
+export function wrapBlockSummary(blockId, summary) {
+  return `[OHC: Compressed bk${blockId}]\n\n${summary}\n<ohc-ref>bk${blockId}</ohc-ref>`
+}
+
+export function applyCompressionState(state, input, selection, anchorMessageId, blockId, storedSummary, consumedBlockIds) {
+  const ms = state.prune.messages
+
+  const block = {
+    blockId,
+    active: true,
+    topic: input.topic,
+    batchTopic: input.batchTopic || input.topic,
+    mode: input.mode || "range",
+    runId: input.runId,
+    compressMessageId: input.compressMessageId,
+    compressCallId: input.compressCallId || null,
+    anchorMessageId,
+    startId: input.startId,
+    endId: input.endId,
+    summary: storedSummary,
+    summaryTokens: input.summaryTokens || 0,
+    compressedTokens: 0,
+    consumedBlockIds: Array.isArray(consumedBlockIds) ? consumedBlockIds : [],
+    deactivatedByBlockId: undefined,
+    deactivatedByUser: false,
+    deactivatedAt: undefined,
+    createdAt: Date.now(),
+  }
+
+  ms.blocksById.set(blockId, block)
+  ms.activeBlockIds.add(blockId)
+
+  if (anchorMessageId) {
+    ms.activeByAnchorMessageId.set(anchorMessageId, blockId)
+  }
+
+  for (const rawMessageId of selection.messageIds) {
+    let entry = ms.byMessageId.get(rawMessageId)
+    if (!entry) {
+      entry = { tokenCount: 0, allBlockIds: [], activeBlockIds: [] }
+      ms.byMessageId.set(rawMessageId, entry)
+    }
+    if (!entry.allBlockIds.includes(blockId)) {
+      entry.allBlockIds.push(blockId)
+    }
+    if (!entry.activeBlockIds.includes(blockId)) {
+      entry.activeBlockIds.push(blockId)
+    }
+  }
+
+  for (const cid of consumedBlockIds) {
+    const cb = ms.blocksById.get(cid)
+    if (cb) {
+      cb.active = false
+      cb.deactivatedAt = Date.now()
+      cb.deactivatedByBlockId = blockId
+      ms.activeBlockIds.delete(cid)
+    }
+  }
+
+  return block
+}