dsh-vision-router 2.2.8 → 3.0.0

package/lib/session-vision-index.js

@@ -1,28 +1,18 @@
 import { currentSessionSurfacePolicy } from './session-surface-policy.js'
 import { sessionSurfaceReplacementIntent } from './session-surface-compat.js'
 import { createSessionVisionStateStore } from './session-vision-state.js'
+import { legacySessionEvents } from './session-event-history-compat.js'
+import { createSessionVisionAttachmentIndex } from './session-vision-attachment-index.js'
 
 const MAX_PENDING_REPAIR_EVENTS = 256
+const MAX_REPAIR_OVERFLOW_RANGES = 32
+const MAX_REPAIR_OVERFLOW_INSPECTIONS_PER_PASS = 512
+const MAX_REPAIR_OVERFLOW_READS_PER_PASS = 32
 
 function isObject(value) {
   return value !== null && typeof value === 'object'
 }
 
-function legacySessionEvents(session) {
-  try {
-    // dsh 0.1.2-alpha.4 removed the bare `session.events` array in favor of
-    // `session.snapshotEvents()`; both are handled so one build runs on both
-    // harness generations.
-    if (session && typeof session.snapshotEvents === 'function') {
-      const events = session.snapshotEvents()
-      return Array.isArray(events) ? events : undefined
-    }
-    return Array.isArray(session?.events) ? session.events : undefined
-  } catch {
-    return undefined
-  }
-}
-
 function surfaceNodes(session) {
   try {
     return Array.isArray(session?.surface?.nodes) ? session.surface.nodes : undefined
@@ -49,9 +39,10 @@ function isReplacementEvent(event) {
 /**
  * Session visual data-plane index.
  *
- * SessionVisionStateStore remains the bounded storage owner. This object owns
- * target-only durable attachment recovery, event-feed-backed tool-result image
- * surface repair and expired structured guard-stop surface repair.
+ * SessionVisionStateStore remains the bounded storage owner. Attachment cache
+ * access and target-only durable recovery are delegated to one typed attachment
+ * index; this object owns event-feed-backed tool-result image surface repair and
+ * expired structured guard-stop surface repair.
  *
  * Supported DSH Hosts expose the post-commit `session/event` feed. Production
  * surface repair consumes only the exact frozen events observed on that feed;
@@ -80,105 +71,37 @@ export function createSessionVisionIndex({
   if (!store || typeof store !== 'object') {
     throw new TypeError('session vision index requires a state store')
   }
-  const primitiveLookup = typeof store.lookupAttachment === 'function'
-    ? store.lookupAttachment.bind(store)
-    : undefined
+  const attachmentIndex = createSessionVisionAttachmentIndex({
+    stateStore: store,
+    collectEventAttachmentRefs:
+      typeof core.collectEventAttachmentRefs === 'function'
+        ? (events) => core.collectEventAttachmentRefs(events)
+        : undefined,
+    readSessionLog,
+    logger,
+  })
+  const {
+    recordAttachments,
+    lookupAttachment,
+    resolveAttachment,
+    resolveAttachments,
+  } = attachmentIndex
+
+  // Surface repair/feed state remains owned by SessionVisionIndex. Attachment
+  // cache/recovery state lives exclusively in SessionVisionAttachmentIndex.
   const toolSurfaceScans = new WeakMap()
   const guardSurfaceScans = new WeakMap()
   const pendingToolRepairEvents = new WeakMap()
   const pendingGuardRepairEvents = new WeakMap()
+  const toolRepairOverflowRanges = new WeakMap()
+  const guardRepairOverflowRanges = new WeakMap()
   const unsupportedSurfaceContracts = new WeakSet()
   const repairReadWarnings = new WeakMap()
   const repairFeedOverflowWarnings = new WeakSet()
   const repairFeedObserverWarnings = new WeakMap()
   const surfaceFeedBackfills = new WeakMap()
-  const attachmentRecoveryWarnings = new WeakMap()
   let surfaceEventFeedActive = false
 
-  const recordAttachments = (session, refs) => {
-    if (!session || !Array.isArray(refs) || refs.length === 0) return
-    store.recordAttachments(session, refs)
-  }
-
-  // Synchronous lookup is cache-only. Projected short handles first warm this
-  // cache from current derived messages and must not trigger historical I/O.
-  const lookupAttachment = (session, id) => {
-    if (!primitiveLookup) return undefined
-    return primitiveLookup(session, id)
-  }
-
-  const recoverAttachmentsFromEvents = (session, ids, events) => {
-    const found = new Map()
-    if (!Array.isArray(events) || events.length === 0 || typeof core.collectEventAttachmentRefs !== 'function') {
-      return found
-    }
-    const wanted = new Set(ids.map((id) => String(id)))
-    for (const ref of core.collectEventAttachmentRefs(events)) {
-      if (!ref) continue
-      const id = String(ref.attachmentId ?? ref.id)
-      if (wanted.has(id) && !found.has(id)) found.set(id, ref)
-    }
-    if (found.size > 0) store.recordAttachments(session, [...found.values()])
-    return found
-  }
-
-  const warnAttachmentRecoveryFailure = (session, id, error) => {
-    const message = boundedMessage(error)
-    const key = `${String(id)}:${message}`
-    if (attachmentRecoveryWarnings.get(session) === key) return
-    attachmentRecoveryWarnings.set(session, key)
-    logger?.warn?.(
-      'vision-router: session attachment recovery failed id=%s error=%s',
-      String(id),
-      message,
-    )
-  }
-
-  const resolveAttachments = async (session, ids) => {
-    const requested = [...new Set((Array.isArray(ids) ? ids : []).map((id) => String(id)))]
-    const resolved = new Map()
-    const missing = []
-    for (const id of requested) {
-      const cached = lookupAttachment(session, id)
-      if (cached !== undefined) resolved.set(id, cached)
-      else missing.push(id)
-    }
-    if (missing.length === 0 || session === undefined) return resolved
-
-    let events
-    if (typeof readSessionLog === 'function') {
-      let result
-      try {
-        result = await readSessionLog(session)
-      } catch (error) {
-        warnAttachmentRecoveryFailure(session, missing[0], error)
-        return resolved
-      }
-      if (result?.supported === true) {
-        attachmentRecoveryWarnings.delete(session)
-        events = result.events
-      } else if (result?.supported !== false) {
-        warnAttachmentRecoveryFailure(
-          session,
-          missing[0],
-          new Error('Session log reader returned an invalid capability result'),
-        )
-        return resolved
-      }
-    }
-
-    // Historical/partial Hosts without SessionQuery keep the released fallback.
-    // Public rc8+ Hosts use readSessionLog above and never touch sync history.
-    if (events === undefined) events = legacySessionEvents(session)
-    const recovered = recoverAttachmentsFromEvents(session, missing, events)
-    for (const [id, ref] of recovered) resolved.set(id, ref)
-    return resolved
-  }
-
-  const resolveAttachment = async (session, id) => {
-    return (await resolveAttachments(session, [id])).get(String(id))
-  }
-
   const pendingSurfaceScan = (scans, session) => {
     if (!session) return undefined
     const nodes = surfaceNodes(session)
@@ -295,7 +218,132 @@ export function createSessionVisionIndex({
     return undefined
   }
 
-  const cachePendingRepairEvent = (pendingBySession, session, event) => {
+  // Feed overflow stores only dropped sequence ranges, never frozen event
+  // objects. Adjacent ranges coalesce; pathological sparse overflow is
+  // conservatively collapsed once the range count reaches a fixed bound. The
+  // recovery loop then advances that range in bounded numeric/read batches and
+  // consults only Host-owned readSessionEvent(), so memory remains bounded
+  // without returning to whole-Session history materialization.
+  const addRepairOverflowSeq = (overflowBySession, session, seq) => {
+    if (!Number.isSafeInteger(seq) || seq < 0 || Object.is(seq, -0)) return
+    let ranges = overflowBySession.get(session)
+    if (!ranges) {
+      ranges = []
+      overflowBySession.set(session, ranges)
+    }
+
+    let start = seq
+    let end = seq
+    let insertAt = 0
+    while (insertAt < ranges.length && ranges[insertAt].end + 1 < start) insertAt += 1
+    while (insertAt < ranges.length && ranges[insertAt].start - 1 <= end) {
+      start = Math.min(start, ranges[insertAt].start)
+      end = Math.max(end, ranges[insertAt].end)
+      ranges.splice(insertAt, 1)
+    }
+    ranges.splice(insertAt, 0, { start, end })
+
+    if (ranges.length > MAX_REPAIR_OVERFLOW_RANGES) {
+      overflowBySession.set(session, [{
+        start: ranges[0].start,
+        end: ranges[ranges.length - 1].end,
+      }])
+    }
+  }
+
+  const advanceRepairOverflowRange = (ranges) => {
+    const range = ranges?.[0]
+    if (!range) return
+    if (range.start >= range.end) ranges.shift()
+    else range.start += 1
+  }
+
+  const recoverRepairOverflowEvents = async (overflowBySession, pendingBySession, session) => {
+    const ranges = overflowBySession.get(session)
+    if (!ranges || ranges.length === 0) return []
+
+    const nodes = surfaceNodes(session)
+    if (!nodes || nodes.length === 0) {
+      overflowBySession.delete(session)
+      repairFeedOverflowWarnings.delete(session)
+      return []
+    }
+    const current = new Set(nodes)
+    const pending = pendingBySession.get(session)
+    const recovered = []
+    let inspections = 0
+    let reads = 0
+
+    while (
+      ranges.length > 0
+      && inspections < MAX_REPAIR_OVERFLOW_INSPECTIONS_PER_PASS
+      && reads < MAX_REPAIR_OVERFLOW_READS_PER_PASS
+    ) {
+      const seq = ranges[0].start
+      inspections += 1
+
+      // A dropped repair that is no longer on the live surface is settled. A
+      // seq still present in the fast-path map will be consumed normally.
+      if (!current.has(seq) || pending?.has(seq)) {
+        advanceRepairOverflowRange(ranges)
+        continue
+      }
+
+      if (typeof readSessionEvent !== 'function') {
+        warnRepairReadFailure(
+          session,
+          seq,
+          new Error('Session event reader is unavailable during feed overflow recovery'),
+        )
+        break
+      }
+
+      let result
+      try {
+        result = await readSessionEvent(session, seq)
+      } catch (error) {
+        warnRepairReadFailure(session, seq, error)
+        break
+      }
+      if (result?.supported !== true) {
+        warnRepairReadFailure(
+          session,
+          seq,
+          result?.supported === false
+            ? new Error('Session event reader is unsupported during feed overflow recovery')
+            : new Error('Session event reader returned an invalid capability result during feed overflow recovery'),
+        )
+        break
+      }
+      const event = result.event
+      if (!event || typeof event !== 'object' || (event.seq !== undefined && event.seq !== seq)) {
+        warnRepairReadFailure(
+          session,
+          seq,
+          new Error(`Session event reader returned no matching event for overflow seq ${seq}`),
+        )
+        break
+      }
+
+      reads += 1
+      advanceRepairOverflowRange(ranges)
+      repairReadWarnings.delete(session)
+      // The live feed may deliver the same committed event while the exact read
+      // is in flight. In that case the fast-path map owns it; do not return a
+      // second copy from recovery.
+      if (!pendingBySession.get(session)?.has(seq) && !isReplacementEvent(event)) {
+        recovered.push({ seq, event })
+      }
+    }
+
+    if (ranges.length === 0) {
+      overflowBySession.delete(session)
+      repairFeedOverflowWarnings.delete(session)
+    }
+    return recovered
+  }
+
+  const cachePendingRepairEvent = (pendingBySession, overflowBySession, session, event) => {
     if (!session || !Number.isSafeInteger(event?.seq) || event.seq < 0 || Object.is(event.seq, -0)) return
     let pending = pendingBySession.get(session)
     if (!pending) {
@@ -304,11 +352,14 @@ export function createSessionVisionIndex({
     }
     if (!pending.has(event.seq) && pending.size >= MAX_PENDING_REPAIR_EVENTS) {
       const oldest = pending.keys().next().value
-      if (oldest !== undefined) pending.delete(oldest)
+      if (oldest !== undefined) {
+        pending.delete(oldest)
+        addRepairOverflowSeq(overflowBySession, session, oldest)
+      }
       if (!repairFeedOverflowWarnings.has(session)) {
         repairFeedOverflowWarnings.add(session)
         logger?.warn?.(
-          'vision-router: session surface repair event feed exceeded %s pending events; dropping oldest entry',
+          'vision-router: session surface repair event feed exceeded %s pending events; scheduling bounded exact-event recovery',
           MAX_PENDING_REPAIR_EVENTS,
         )
       }
@@ -340,10 +391,10 @@ export function createSessionVisionIndex({
           event,
           (events, seqs) => core.planToolResultImageShadows(events, seqs, () => true),
         )
-        if (plans.length > 0) cachePendingRepairEvent(pendingToolRepairEvents, session, event)
+        if (plans.length > 0) cachePendingRepairEvent(pendingToolRepairEvents, toolRepairOverflowRanges, session, event)
       } else if (event.type === 'user/message' && typeof core.planGuardStopShadows === 'function') {
         const plans = planAgainstOneEvent(event, (events, seqs) => core.planGuardStopShadows(events, seqs))
-        if (plans.length > 0) cachePendingRepairEvent(pendingGuardRepairEvents, session, event)
+        if (plans.length > 0) cachePendingRepairEvent(pendingGuardRepairEvents, guardRepairOverflowRanges, session, event)
       }
       repairFeedObserverWarnings.delete(session)
     } catch (error) {
@@ -439,11 +490,17 @@ export function createSessionVisionIndex({
 
   const repairToolResultSurfaceFromFeed = async (session) => {
     if (typeof session?.append !== 'function' || typeof core.planToolResultImageShadows !== 'function') return 0
+    const recovered = await recoverRepairOverflowEvents(
+      toolRepairOverflowRanges,
+      pendingToolRepairEvents,
+      session,
+    )
     const pending = consumePendingRepairEvents(pendingToolRepairEvents, session)
-    if (pending.length === 0) return 0
+    const entries = [...recovered, ...pending]
+    if (entries.length === 0) return 0
     const preserve = keepToolResultImages()
     let repaired = 0
-    for (const { seq, event } of pending) {
+    for (const { seq, event } of entries) {
       const events = { [seq]: event }
       const plans = core.planToolResultImageShadows(events, [seq], () => !preserve)
       for (const plan of plans) {
@@ -470,10 +527,16 @@ export function createSessionVisionIndex({
 
   const repairGuardStopSurfaceFromFeed = async (session) => {
     if (typeof session?.append !== 'function' || typeof core.planGuardStopShadows !== 'function') return 0
+    const recovered = await recoverRepairOverflowEvents(
+      guardRepairOverflowRanges,
+      pendingGuardRepairEvents,
+      session,
+    )
     const pending = consumePendingRepairEvents(pendingGuardRepairEvents, session)
-    if (pending.length === 0) return 0
+    const entries = [...recovered, ...pending]
+    if (entries.length === 0) return 0
     let repaired = 0
-    for (const { seq, event } of pending) {
+    for (const { seq, event } of entries) {
       const events = { [seq]: event }
       const plans = core.planGuardStopShadows(events, [seq])
       for (const plan of plans) {
@@ -587,7 +650,7 @@ export function createSessionVisionIndex({
     if (typeof core.rewriteImageBlocks === 'function') {
       const found = core.rewriteImageBlocks(messages)
       if (Array.isArray(found?.attachments) && found.attachments.length > 0) {
-        store.recordAttachments(session, found.attachments)
+        recordAttachments(session, found.attachments)
       }
     }
     await ensureSurfaceEventFeedBackfill(session)